Repository: vmware-tanzu/velero Branch: main Commit: bb9a94bebe90 Files: 3082 Total size: 19.9 MB Directory structure: gitextract_b3viilp2/ ├── .dockerignore ├── .github/ │ ├── ISSUE_TEMPLATE/ │ │ ├── bug_report.md │ │ ├── config.yml │ │ └── feature-enhancement-request.md │ ├── auto-assignees.yml │ ├── dependabot.yml │ ├── labeler.yml │ ├── labels.yaml │ ├── pull_request_template.md │ └── workflows/ │ ├── auto_assign_prs.yml │ ├── auto_label_prs.yml │ ├── auto_request_review.yml │ ├── e2e-test-kind.yaml │ ├── get-go-version.yaml │ ├── nightly-trivy-scan.yml │ ├── pr-changelog-check.yml │ ├── pr-ci-check.yml │ ├── pr-codespell.yml │ ├── pr-containers.yml │ ├── pr-goreleaser.yml │ ├── pr-linter-check.yml │ ├── prow-action.yml │ ├── push-builder.yml │ ├── push.yml │ ├── rebase.yml │ └── stale-issues.yml ├── .gitignore ├── .golangci.yaml ├── .goreleaser.yml ├── ADOPTERS.md ├── CHANGELOG.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Dockerfile ├── Dockerfile-Windows ├── GOVERNANCE.md ├── LICENSE ├── MAINTAINERS.md ├── Makefile ├── OWNERS ├── README.md ├── ROADMAP.md ├── SECURITY.md ├── SUPPORT.md ├── Tiltfile ├── assets/ │ ├── README.md │ ├── one-line/ │ │ ├── 199150-vmw-os-lgo-velero-final_gry.eps │ │ └── 199150-vmw-os-lgo-velero-final_wht.eps │ └── stacked/ │ ├── 199150-vmw-os-lgo-velero-final_stacked-gry.eps │ └── 199150-vmw-os-lgo-velero-final_stacked-wht.eps ├── changelogs/ │ ├── CHANGELOG-0.10.md │ ├── CHANGELOG-0.11.md │ ├── CHANGELOG-0.3.md │ ├── CHANGELOG-0.4.md │ ├── CHANGELOG-0.5.md │ ├── CHANGELOG-0.6.md │ ├── CHANGELOG-0.7.md │ ├── CHANGELOG-0.8.md │ ├── CHANGELOG-0.9.md │ ├── CHANGELOG-1.0.md │ ├── CHANGELOG-1.1.md │ ├── CHANGELOG-1.10.md │ ├── CHANGELOG-1.11.md │ ├── CHANGELOG-1.12.md │ ├── CHANGELOG-1.13.md │ ├── CHANGELOG-1.14.md │ ├── CHANGELOG-1.15.md │ ├── CHANGELOG-1.16.md │ ├── CHANGELOG-1.17.md │ ├── CHANGELOG-1.18.md │ ├── CHANGELOG-1.2.md │ ├── CHANGELOG-1.3.md │ ├── CHANGELOG-1.4.md │ ├── CHANGELOG-1.5.md │ ├── CHANGELOG-1.6.md │ ├── CHANGELOG-1.7.md │ ├── CHANGELOG-1.8.md │ ├── CHANGELOG-1.9.md │ └── unreleased/ │ ├── 9502-Joeavaikath │ ├── 9508-kaovilai │ ├── 9532-Lyndon-Li │ ├── 9533-Lyndon-Li‎‎ │ ├── 9547-blackpiglet │ ├── 9554-testsabirweb │ ├── 9560-Lyndon-Li‎‎ │ ├── 9561-Lyndon-Li‎‎ │ ├── 9570-H-M-Quang-Ngo │ ├── 9574-blackpiglet │ └── 9581-shubham-pampattiwar ├── cmd/ │ ├── velero/ │ │ └── velero.go │ ├── velero-helper/ │ │ └── velero-helper.go │ └── velero-restore-helper/ │ └── velero-restore-helper.go ├── config/ │ ├── crd/ │ │ ├── v1/ │ │ │ ├── bases/ │ │ │ │ ├── velero.io_backuprepositories.yaml │ │ │ │ ├── velero.io_backups.yaml │ │ │ │ ├── velero.io_backupstoragelocations.yaml │ │ │ │ ├── velero.io_deletebackuprequests.yaml │ │ │ │ ├── velero.io_downloadrequests.yaml │ │ │ │ ├── velero.io_podvolumebackups.yaml │ │ │ │ ├── velero.io_podvolumerestores.yaml │ │ │ │ ├── velero.io_restores.yaml │ │ │ │ ├── velero.io_schedules.yaml │ │ │ │ ├── velero.io_serverstatusrequests.yaml │ │ │ │ └── velero.io_volumesnapshotlocations.yaml │ │ │ └── crds/ │ │ │ ├── crds.go │ │ │ └── doc.go │ │ └── v2alpha1/ │ │ ├── bases/ │ │ │ ├── velero.io_datadownloads.yaml │ │ │ └── velero.io_datauploads.yaml │ │ └── crds/ │ │ ├── crds.go │ │ └── doc.go │ └── rbac/ │ └── role.yaml ├── design/ │ ├── 2082-bsl-delete-associated-resources_design.md │ ├── CLI/ │ │ └── PoC/ │ │ ├── base/ │ │ │ ├── CRDs.yaml │ │ │ ├── backupstoragelocations.yaml │ │ │ ├── deployment.yaml │ │ │ ├── kustomization.yaml │ │ │ ├── minio.yaml │ │ │ ├── podvolumes.yaml │ │ │ ├── resticrepository.yaml │ │ │ └── volumesnapshotlocations.yaml │ │ └── overlays/ │ │ └── plugins/ │ │ ├── aws-plugin.yaml │ │ ├── azure-plugin.yaml │ │ ├── cloud │ │ ├── kustomization.yaml │ │ └── node-agent.yaml │ ├── Implemented/ │ │ ├── AsyncActionFSM.graffle │ │ ├── Extend-VolumePolicies-to-support-more-actions.md │ │ ├── apply-flag.md │ │ ├── backup-performance-improvements.md │ │ ├── backup-pvc-config.md │ │ ├── backup-repo-cache-volume.md │ │ ├── backup-repo-config.md │ │ ├── backup-resource-list.md │ │ ├── backup-resources-order.md │ │ ├── biav2-design.md │ │ ├── bsl-certificate-support_design.md │ │ ├── clean_artifacts_in_csi_flow.md │ │ ├── cluster-scope-resource-filter.md │ │ ├── concurrent-backup-processing.md │ │ ├── csi-snapshots.md │ │ ├── custom-ca-support.md │ │ ├── delete-item-action.md │ │ ├── deletion-plugins.md │ │ ├── existing-resource-policy_design.md │ │ ├── feature-flags.md │ │ ├── general-progress-monitoring.md │ │ ├── generating-velero-crds-with-structural-schema.md │ │ ├── handle-backup-of-volumes-by-resources-filters.md │ │ ├── include-exclude-in-resource-policy.md │ │ ├── json-substitution-action-design.md │ │ ├── merge-patch-and-strategic-in-resource-modifier.md │ │ ├── move-gh-org.md │ │ ├── move-plugin-repos.md │ │ ├── multiple-arch-build-with-windows.md │ │ ├── multiple-csi-volumesnapshotclass-support.md │ │ ├── multiple-label-selectors_design.md │ │ ├── node-agent-affinity.md │ │ ├── node-agent-concurrency.md │ │ ├── node-agent-load-soothing.md │ │ ├── plugin-backup-and-restore-progress-design.md │ │ ├── plugin-versioning.md │ │ ├── priority-class-name-support_design.md │ │ ├── pv-cloning.md │ │ ├── pv_backup_info.md │ │ ├── pv_restore_info.md │ │ ├── repo_maintenance_job_config.md │ │ ├── repository-maintenance.md │ │ ├── resource-status-restore.md │ │ ├── restic-backup-and-restore-progress.md │ │ ├── restore-finalizing-phase_design.md │ │ ├── restore-hooks.md │ │ ├── restore-with-EnableAPIGroupVersions-feature.md │ │ ├── retry-patching-configuration_design.md │ │ ├── riav2-design.md │ │ ├── schedule-skip-immediately-config_design.md │ │ ├── secrets.md │ │ ├── supporting-volumeattributes-resource-policy.md │ │ ├── unified-repo-and-kopia-integration/ │ │ │ └── unified-repo-and-kopia-integration.md │ │ ├── velero-debug.md │ │ ├── velero-uploader-configuration.md │ │ ├── vgdp-affinity-enhancement.md │ │ ├── vgdp-micro-service/ │ │ │ └── vgdp-micro-service.md │ │ ├── vgdp-micro-service-for-fs-backup/ │ │ │ └── vgdp-micro-service-for-fs-backup.md │ │ ├── volume-group-snapshot.md │ │ ├── volume-policy-label-selector-criteria.md │ │ ├── volume-snapshot-data-movement/ │ │ │ └── volume-snapshot-data-movement.md │ │ ├── wait-for-additional-items.md │ │ └── wildcard-namespace-support-design.md │ ├── UploadFSM.graffle │ ├── _template.md │ ├── cli-install-changes.md │ ├── graph-manifest.md │ ├── new-prepost-backuprestore-plugin-hooks.md │ ├── restore-progress.md │ ├── upload-progress.md │ └── vsv2-design.md ├── examples/ │ ├── README.md │ ├── minio/ │ │ └── 00-minio-deployment.yaml │ └── nginx-app/ │ ├── README.md │ ├── base.yaml │ └── with-pv.yaml ├── go.mod ├── go.sum ├── hack/ │ ├── boilerplate.go.txt │ ├── build-image/ │ │ └── Dockerfile │ ├── build-restic.sh │ ├── build.sh │ ├── changelog-check.sh │ ├── ci-check.sh │ ├── crd-gen/ │ │ └── v1/ │ │ └── main.go │ ├── docker-push.sh │ ├── fix_restic_cve.txt │ ├── issue-template-gen/ │ │ └── main.go │ ├── lint.sh │ ├── release-tools/ │ │ ├── brew-update.sh │ │ ├── changelog.sh │ │ ├── chk_version.go │ │ ├── chk_version_test.go │ │ ├── gen-docs.sh │ │ ├── goreleaser.sh │ │ └── tag-release.sh │ ├── test.sh │ ├── update-1fmt.sh │ ├── update-2proto.sh │ ├── update-3generated-crd-code.sh │ ├── update-4generated-issue-template.sh │ ├── update-all.sh │ ├── verify-all.sh │ ├── verify-fmt.sh │ ├── verify-generated-crd-code.sh │ └── verify-generated-issue-template.sh ├── internal/ │ ├── credentials/ │ │ ├── file_store.go │ │ ├── file_store_test.go │ │ ├── getter.go │ │ ├── local.go │ │ ├── mocks/ │ │ │ ├── FileStore.go │ │ │ └── SecretStore.go │ │ └── secret_store.go │ ├── delete/ │ │ ├── actions/ │ │ │ └── csi/ │ │ │ ├── volumesnapshotcontent_action.go │ │ │ └── volumesnapshotcontent_action_test.go │ │ ├── delete_item_action_handler.go │ │ └── delete_item_action_handler_test.go │ ├── hook/ │ │ ├── hook_tracker.go │ │ ├── hook_tracker_test.go │ │ ├── item_hook_handler.go │ │ ├── item_hook_handler_test.go │ │ ├── wait_exec_hook_handler.go │ │ └── wait_exec_hook_handler_test.go │ ├── resourcemodifiers/ │ │ ├── json_merge_patch.go │ │ ├── json_merge_patch_test.go │ │ ├── json_patch.go │ │ ├── resource_modifiers.go │ │ ├── resource_modifiers_test.go │ │ ├── resource_modifiers_validator.go │ │ ├── resource_modifiers_validator_test.go │ │ ├── strategic_merge_patch.go │ │ └── strategic_merge_patch_test.go │ ├── resourcepolicies/ │ │ ├── resource_policies.go │ │ ├── resource_policies_test.go │ │ ├── volume_filter_data.go │ │ ├── volume_filter_data_test.go │ │ ├── volume_resources.go │ │ ├── volume_resources_test.go │ │ ├── volume_resources_validator.go │ │ ├── volume_resources_validator_test.go │ │ ├── volume_types_conditions.go │ │ └── volume_types_conditions_test.go │ ├── restartabletest/ │ │ └── restartable_delegate.go │ ├── storage/ │ │ ├── storagelocation.go │ │ └── storagelocation_test.go │ ├── velero/ │ │ ├── images.go │ │ ├── images_test.go │ │ └── serverstatusrequest.go │ ├── volume/ │ │ ├── native_snapshot.go │ │ ├── snapshotlocation.go │ │ ├── utils.go │ │ ├── utils_test.go │ │ ├── volumes_information.go │ │ └── volumes_information_test.go │ └── volumehelper/ │ ├── volume_policy_helper.go │ └── volume_policy_helper_test.go ├── netlify.toml ├── pkg/ │ ├── apis/ │ │ └── velero/ │ │ ├── shared/ │ │ │ └── data_move_operation_progress.go │ │ ├── v1/ │ │ │ ├── backup_repository_types.go │ │ │ ├── backup_types.go │ │ │ ├── backupstoragelocation_types.go │ │ │ ├── backupstoragelocation_types_test.go │ │ │ ├── constants.go │ │ │ ├── delete_backup_request_types.go │ │ │ ├── doc.go │ │ │ ├── download_request_types.go │ │ │ ├── groupversion_info.go │ │ │ ├── labels_annotations.go │ │ │ ├── pod_volume_backup_types.go │ │ │ ├── pod_volume_restore_type.go │ │ │ ├── register.go │ │ │ ├── restore_types.go │ │ │ ├── schedule_types.go │ │ │ ├── server_status_request_types.go │ │ │ ├── volume_snapshot_location_type.go │ │ │ └── zz_generated.deepcopy.go │ │ └── v2alpha1/ │ │ ├── data_download_types.go │ │ ├── data_upload_types.go │ │ ├── doc.go │ │ ├── groupversion_info.go │ │ ├── register.go │ │ └── zz_generated.deepcopy.go │ ├── archive/ │ │ ├── extractor.go │ │ ├── extractor_test.go │ │ ├── filesystem.go │ │ ├── filesystem_test.go │ │ ├── parser.go │ │ └── parser_test.go │ ├── backup/ │ │ ├── actions/ │ │ │ ├── backup_pv_action.go │ │ │ ├── backup_pv_action_test.go │ │ │ ├── csi/ │ │ │ │ ├── pvc_action.go │ │ │ │ ├── pvc_action_test.go │ │ │ │ ├── volumesnapshot_action.go │ │ │ │ ├── volumesnapshot_action_test.go │ │ │ │ ├── volumesnapshotclass_action.go │ │ │ │ ├── volumesnapshotclass_action_test.go │ │ │ │ ├── volumesnapshotcontent_action.go │ │ │ │ └── volumesnapshotcontent_action_test.go │ │ │ ├── pod_action.go │ │ │ ├── pod_action_test.go │ │ │ ├── remap_crd_version_action.go │ │ │ ├── remap_crd_version_action_test.go │ │ │ ├── service_account_action.go │ │ │ ├── service_account_action_test.go │ │ │ └── testdata/ │ │ │ ├── v1/ │ │ │ │ ├── alertmanagers.monitoring.coreos.com.json │ │ │ │ ├── elasticsearches.elasticsearch.k8s.elastic.co.json │ │ │ │ ├── gcpsamples.gcp.stacks.crossplane.io.json │ │ │ │ ├── kibanas.kibana.k8s.elastic.co.json │ │ │ │ ├── pprometheuses.monitoring.coreos.com.json │ │ │ │ └── prometheuses.monitoring.coreos.com.json │ │ │ └── v1beta1/ │ │ │ ├── alertmanagers.monitoring.coreos.com.json │ │ │ ├── elasticsearches.elasticsearch.k8s.elastic.co.json │ │ │ ├── gcpsamples.gcp.stacks.crossplane.io.json │ │ │ ├── kibanas.kibana.k8s.elastic.co.json │ │ │ └── prometheuses.monitoring.coreos.com.json │ │ ├── backed_up_items_map.go │ │ ├── backup.go │ │ ├── backup_test.go │ │ ├── delete_helpers.go │ │ ├── item_backupper.go │ │ ├── item_backupper_test.go │ │ ├── item_block_worker_pool.go │ │ ├── item_collector.go │ │ ├── item_collector_test.go │ │ ├── itemblock.go │ │ ├── pv_skip_tracker.go │ │ ├── pv_skip_tracker_test.go │ │ ├── request.go │ │ ├── request_test.go │ │ ├── snapshots.go │ │ └── volume_snapshotter_cache.go │ ├── builder/ │ │ ├── backup_builder.go │ │ ├── backup_storage_location_builder.go │ │ ├── config_map_builder.go │ │ ├── container_builder.go │ │ ├── container_builder_test.go │ │ ├── customresourcedefinition_v1beta1_builder.go │ │ ├── data_download_builder.go │ │ ├── data_upload_builder.go │ │ ├── delete_backup_request_builder.go │ │ ├── deployment_builder.go │ │ ├── download_request_builder.go │ │ ├── item_operation_builder.go │ │ ├── job_builder.go │ │ ├── json_schema_props_builder.go │ │ ├── namespace_builder.go │ │ ├── node_builder.go │ │ ├── node_selector_builder.go │ │ ├── object_meta.go │ │ ├── persistent_volume_builder.go │ │ ├── persistent_volume_claim_builder.go │ │ ├── pod_builder.go │ │ ├── pod_volume_backup_builder.go │ │ ├── pod_volume_restore_builder.go │ │ ├── priority_class_builder.go │ │ ├── restore_builder.go │ │ ├── role_builder.go │ │ ├── schedule_builder.go │ │ ├── secret_builder.go │ │ ├── secret_key_selector_builder.go │ │ ├── server_status_request_builder.go │ │ ├── service_account_builder.go │ │ ├── service_builder.go │ │ ├── statefulset_builder.go │ │ ├── storage_class_builder.go │ │ ├── testcr_builder.go │ │ ├── v1_customresourcedefinition_builder.go │ │ ├── volume_builder.go │ │ ├── volume_mount_builder.go │ │ ├── volume_snapshot_builder.go │ │ ├── volume_snapshot_class_builder.go │ │ ├── volume_snapshot_content_builder.go │ │ └── volume_snapshot_location_builder.go │ ├── buildinfo/ │ │ ├── buildinfo.go │ │ └── buildinfo_test.go │ ├── client/ │ │ ├── auth_providers.go │ │ ├── client.go │ │ ├── client_test.go │ │ ├── config.go │ │ ├── config_test.go │ │ ├── dynamic.go │ │ ├── factory.go │ │ ├── factory_test.go │ │ ├── kubeconfig │ │ ├── mocks/ │ │ │ └── Factory.go │ │ └── retry.go │ ├── cmd/ │ │ ├── cli/ │ │ │ ├── backup/ │ │ │ │ ├── backup.go │ │ │ │ ├── backup_test.go │ │ │ │ ├── create.go │ │ │ │ ├── create_test.go │ │ │ │ ├── delete.go │ │ │ │ ├── delete_test.go │ │ │ │ ├── describe.go │ │ │ │ ├── describe_test.go │ │ │ │ ├── download.go │ │ │ │ ├── download_test.go │ │ │ │ ├── get.go │ │ │ │ ├── get_test.go │ │ │ │ ├── logs.go │ │ │ │ └── logs_test.go │ │ │ ├── backuplocation/ │ │ │ │ ├── backup_location.go │ │ │ │ ├── backup_location_test.go │ │ │ │ ├── create.go │ │ │ │ ├── create_test.go │ │ │ │ ├── delete.go │ │ │ │ ├── delete_test.go │ │ │ │ ├── get.go │ │ │ │ ├── get_test.go │ │ │ │ ├── set.go │ │ │ │ └── set_test.go │ │ │ ├── bug/ │ │ │ │ └── bug.go │ │ │ ├── client/ │ │ │ │ ├── client.go │ │ │ │ └── config/ │ │ │ │ ├── config.go │ │ │ │ ├── get.go │ │ │ │ └── set.go │ │ │ ├── completion/ │ │ │ │ └── completion.go │ │ │ ├── create/ │ │ │ │ └── create.go │ │ │ ├── datamover/ │ │ │ │ ├── backup.go │ │ │ │ ├── backup_test.go │ │ │ │ ├── data_mover.go │ │ │ │ ├── mocks/ │ │ │ │ │ └── Cache.go │ │ │ │ ├── restore.go │ │ │ │ └── restore_test.go │ │ │ ├── debug/ │ │ │ │ ├── cshd-scripts/ │ │ │ │ │ └── velero.cshd │ │ │ │ └── debug.go │ │ │ ├── delete/ │ │ │ │ └── delete.go │ │ │ ├── delete_options.go │ │ │ ├── describe/ │ │ │ │ └── describe.go │ │ │ ├── get/ │ │ │ │ └── get.go │ │ │ ├── install/ │ │ │ │ ├── install.go │ │ │ │ └── install_test.go │ │ │ ├── nodeagent/ │ │ │ │ ├── node_agent.go │ │ │ │ ├── server.go │ │ │ │ └── server_test.go │ │ │ ├── plugin/ │ │ │ │ ├── add.go │ │ │ │ ├── get.go │ │ │ │ ├── helpers.go │ │ │ │ ├── plugin.go │ │ │ │ └── remove.go │ │ │ ├── podvolume/ │ │ │ │ ├── backup.go │ │ │ │ ├── backup_test.go │ │ │ │ ├── podvolume.go │ │ │ │ ├── restore.go │ │ │ │ └── restore_test.go │ │ │ ├── repo/ │ │ │ │ ├── get.go │ │ │ │ └── repo.go │ │ │ ├── repomantenance/ │ │ │ │ └── maintenance.go │ │ │ ├── restore/ │ │ │ │ ├── create.go │ │ │ │ ├── create_test.go │ │ │ │ ├── delete.go │ │ │ │ ├── delete_test.go │ │ │ │ ├── describe.go │ │ │ │ ├── describe_test.go │ │ │ │ ├── get.go │ │ │ │ ├── get_test.go │ │ │ │ ├── logs.go │ │ │ │ ├── logs_test.go │ │ │ │ ├── restore.go │ │ │ │ └── restore_test.go │ │ │ ├── schedule/ │ │ │ │ ├── create.go │ │ │ │ ├── delete.go │ │ │ │ ├── describe.go │ │ │ │ ├── get.go │ │ │ │ ├── pause.go │ │ │ │ ├── schedule.go │ │ │ │ ├── skip_options.go │ │ │ │ └── unpause.go │ │ │ ├── select_option.go │ │ │ ├── select_option_test.go │ │ │ ├── serverstatus/ │ │ │ │ └── server_status.go │ │ │ ├── snapshotlocation/ │ │ │ │ ├── create.go │ │ │ │ ├── get.go │ │ │ │ ├── set.go │ │ │ │ └── snapshot_location.go │ │ │ ├── uninstall/ │ │ │ │ └── uninstall.go │ │ │ └── version/ │ │ │ ├── version.go │ │ │ └── version_test.go │ │ ├── const.go │ │ ├── errors.go │ │ ├── server/ │ │ │ ├── config/ │ │ │ │ ├── config.go │ │ │ │ └── config_test.go │ │ │ ├── plugin/ │ │ │ │ └── plugin.go │ │ │ ├── server.go │ │ │ └── server_test.go │ │ ├── test/ │ │ │ └── const.go │ │ ├── util/ │ │ │ ├── cacert/ │ │ │ │ ├── bsl_cacert.go │ │ │ │ └── bsl_cacert_test.go │ │ │ ├── confirm/ │ │ │ │ └── confirm.go │ │ │ ├── downloadrequest/ │ │ │ │ ├── downloadrequest.go │ │ │ │ └── downloadrequest_test.go │ │ │ ├── flag/ │ │ │ │ ├── accessors.go │ │ │ │ ├── accessors_test.go │ │ │ │ ├── array.go │ │ │ │ ├── array_test.go │ │ │ │ ├── enum.go │ │ │ │ ├── enum_test.go │ │ │ │ ├── label_selector_test.go │ │ │ │ ├── labelselector.go │ │ │ │ ├── map.go │ │ │ │ ├── map_test.go │ │ │ │ ├── optional_bool.go │ │ │ │ ├── optional_bool_test.go │ │ │ │ ├── orlabelselector.go │ │ │ │ └── orlabelselector_test.go │ │ │ ├── output/ │ │ │ │ ├── backup_describer.go │ │ │ │ ├── backup_describer_test.go │ │ │ │ ├── backup_printer.go │ │ │ │ ├── backup_printer_test.go │ │ │ │ ├── backup_repo_printer.go │ │ │ │ ├── backup_storage_location_printer.go │ │ │ │ ├── backup_structured_describer.go │ │ │ │ ├── backup_structured_describer_test.go │ │ │ │ ├── describe.go │ │ │ │ ├── describe_test.go │ │ │ │ ├── output.go │ │ │ │ ├── output_test.go │ │ │ │ ├── plugin_printer.go │ │ │ │ ├── restore_describer.go │ │ │ │ ├── restore_describer_test.go │ │ │ │ ├── restore_printer.go │ │ │ │ ├── schedule_describe_test.go │ │ │ │ ├── schedule_describer.go │ │ │ │ ├── schedule_printer.go │ │ │ │ └── volume_snapshot_location_printer.go │ │ │ └── signals/ │ │ │ └── signals.go │ │ └── velero/ │ │ └── velero.go │ ├── constant/ │ │ └── constant.go │ ├── controller/ │ │ ├── backup_controller.go │ │ ├── backup_controller_test.go │ │ ├── backup_deletion_controller.go │ │ ├── backup_deletion_controller_test.go │ │ ├── backup_finalizer_controller.go │ │ ├── backup_finalizer_controller_test.go │ │ ├── backup_operations_controller.go │ │ ├── backup_operations_controller_test.go │ │ ├── backup_queue_controller.go │ │ ├── backup_queue_controller_test.go │ │ ├── backup_repository_controller.go │ │ ├── backup_repository_controller_test.go │ │ ├── backup_storage_location_controller.go │ │ ├── backup_storage_location_controller_test.go │ │ ├── backup_sync_controller.go │ │ ├── backup_sync_controller_test.go │ │ ├── backup_tracker.go │ │ ├── backup_tracker_test.go │ │ ├── data_download_controller.go │ │ ├── data_download_controller_test.go │ │ ├── data_upload_controller.go │ │ ├── data_upload_controller_test.go │ │ ├── download_request_controller.go │ │ ├── download_request_controller_test.go │ │ ├── gc_controller.go │ │ ├── gc_controller_test.go │ │ ├── interface.go │ │ ├── pod_volume_backup_controller.go │ │ ├── pod_volume_backup_controller_test.go │ │ ├── pod_volume_restore_controller.go │ │ ├── pod_volume_restore_controller_legacy.go │ │ ├── pod_volume_restore_controller_legacy_test.go │ │ ├── pod_volume_restore_controller_test.go │ │ ├── restore_controller.go │ │ ├── restore_controller_test.go │ │ ├── restore_finalizer_controller.go │ │ ├── restore_finalizer_controller_test.go │ │ ├── restore_operations_controller.go │ │ ├── restore_operations_controller_test.go │ │ ├── schedule_controller.go │ │ ├── schedule_controller_test.go │ │ ├── server_status_request_controller.go │ │ ├── server_status_request_controller_test.go │ │ └── suite_test.go │ ├── datamover/ │ │ ├── backup_micro_service.go │ │ ├── backup_micro_service_test.go │ │ ├── dataupload_delete_action.go │ │ ├── restore_micro_service.go │ │ ├── restore_micro_service_test.go │ │ ├── util.go │ │ └── util_test.go │ ├── datapath/ │ │ ├── error.go │ │ ├── error_test.go │ │ ├── file_system.go │ │ ├── file_system_test.go │ │ ├── manager.go │ │ ├── manager_test.go │ │ ├── micro_service_watcher.go │ │ ├── micro_service_watcher_test.go │ │ ├── mocks/ │ │ │ └── asyncBR.go │ │ └── types.go │ ├── discovery/ │ │ ├── helper.go │ │ ├── helper_test.go │ │ └── mocks/ │ │ └── Helper.go │ ├── exposer/ │ │ ├── cache_volume.go │ │ ├── cache_volume_test.go │ │ ├── csi_snapshot.go │ │ ├── csi_snapshot_priority_test.go │ │ ├── csi_snapshot_test.go │ │ ├── generic_restore.go │ │ ├── generic_restore_priority_test.go │ │ ├── generic_restore_test.go │ │ ├── host_path.go │ │ ├── host_path_test.go │ │ ├── image.go │ │ ├── image_test.go │ │ ├── mocks/ │ │ │ ├── GenericRestoreExposer.go │ │ │ └── PodVolumeExposer.go │ │ ├── pod_volume.go │ │ ├── pod_volume_test.go │ │ ├── snapshot.go │ │ ├── types.go │ │ ├── vgdp_counter.go │ │ └── vgdp_counter_test.go │ ├── features/ │ │ ├── feature_flags.go │ │ └── feature_flags_test.go │ ├── install/ │ │ ├── daemonset.go │ │ ├── daemonset_test.go │ │ ├── deployment.go │ │ ├── deployment_test.go │ │ ├── doc.go │ │ ├── import_test.go │ │ ├── install.go │ │ ├── install_test.go │ │ ├── resources.go │ │ └── resources_test.go │ ├── itemblock/ │ │ ├── actions/ │ │ │ ├── pod_action.go │ │ │ ├── pod_action_test.go │ │ │ ├── pvc_action.go │ │ │ ├── pvc_action_test.go │ │ │ ├── service_account_action.go │ │ │ └── service_account_action_test.go │ │ └── itemblock.go │ ├── itemoperation/ │ │ ├── backup_operation.go │ │ ├── restore_operation.go │ │ └── shared.go │ ├── itemoperationmap/ │ │ ├── backup_operation_map.go │ │ └── restore_operation_map.go │ ├── kopia/ │ │ ├── kopia_log.go │ │ └── kopia_log_test.go │ ├── kuberesource/ │ │ └── kuberesource.go │ ├── label/ │ │ ├── label.go │ │ └── label_test.go │ ├── metrics/ │ │ ├── metrics.go │ │ └── metrics_test.go │ ├── nodeagent/ │ │ ├── node_agent.go │ │ └── node_agent_test.go │ ├── persistence/ │ │ ├── in_memory_object_store.go │ │ ├── mocks/ │ │ │ ├── backup_store.go │ │ │ └── object_store.go │ │ ├── object_store.go │ │ ├── object_store_layout.go │ │ └── object_store_test.go │ ├── plugin/ │ │ ├── clientmgmt/ │ │ │ ├── backupitemaction/ │ │ │ │ ├── v1/ │ │ │ │ │ ├── restartable_backup_item_action.go │ │ │ │ │ └── restartable_backup_item_action_test.go │ │ │ │ └── v2/ │ │ │ │ ├── restartable_backup_item_action.go │ │ │ │ └── restartable_backup_item_action_test.go │ │ │ ├── itemblockaction/ │ │ │ │ └── v1/ │ │ │ │ ├── restartable_item_block_action.go │ │ │ │ └── restartable_item_block_action_test.go │ │ │ ├── manager.go │ │ │ ├── manager_test.go │ │ │ ├── process/ │ │ │ │ ├── client_builder.go │ │ │ │ ├── client_builder_test.go │ │ │ │ ├── logrus_adapter.go │ │ │ │ ├── logrus_adapter_test.go │ │ │ │ ├── process.go │ │ │ │ ├── process_test.go │ │ │ │ ├── registry.go │ │ │ │ ├── registry_test.go │ │ │ │ └── restartable_process.go │ │ │ ├── restartable_delete_item_action.go │ │ │ ├── restartable_delete_item_action_test.go │ │ │ ├── restartable_object_store.go │ │ │ ├── restartable_object_store_test.go │ │ │ ├── restoreitemaction/ │ │ │ │ ├── v1/ │ │ │ │ │ ├── restartable_restore_item_action.go │ │ │ │ │ └── restartable_restore_item_action_test.go │ │ │ │ └── v2/ │ │ │ │ ├── restartable_restore_item_action.go │ │ │ │ └── restartable_restore_item_action_test.go │ │ │ └── volumesnapshotter/ │ │ │ └── v1/ │ │ │ ├── restartable_volume_snapshotter.go │ │ │ └── restartable_volume_snapshotter_test.go │ │ ├── framework/ │ │ │ ├── action_resolver.go │ │ │ ├── action_resolver_test.go │ │ │ ├── backup_item_action.go │ │ │ ├── backup_item_action_client.go │ │ │ ├── backup_item_action_server.go │ │ │ ├── backup_item_action_test.go │ │ │ ├── backupitemaction/ │ │ │ │ └── v2/ │ │ │ │ ├── backup_item_action.go │ │ │ │ ├── backup_item_action_client.go │ │ │ │ ├── backup_item_action_server.go │ │ │ │ └── backup_item_action_test.go │ │ │ ├── common/ │ │ │ │ ├── client_dispenser.go │ │ │ │ ├── client_dispenser_test.go │ │ │ │ ├── client_errors.go │ │ │ │ ├── handle_panic.go │ │ │ │ ├── plugin_base.go │ │ │ │ ├── plugin_base_test.go │ │ │ │ ├── plugin_config.go │ │ │ │ ├── plugin_config_test.go │ │ │ │ ├── plugin_kinds.go │ │ │ │ ├── server_errors.go │ │ │ │ ├── server_mux.go │ │ │ │ └── server_mux_test.go │ │ │ ├── delete_item_action.go │ │ │ ├── delete_item_action_client.go │ │ │ ├── delete_item_action_server.go │ │ │ ├── doc.go │ │ │ ├── examples_test.go │ │ │ ├── handshake.go │ │ │ ├── import_test.go │ │ │ ├── interface.go │ │ │ ├── itemblockaction/ │ │ │ │ └── v1/ │ │ │ │ ├── item_block_action.go │ │ │ │ ├── item_block_action_client.go │ │ │ │ ├── item_block_action_server.go │ │ │ │ └── item_block_action_test.go │ │ │ ├── logger.go │ │ │ ├── logger_test.go │ │ │ ├── object_store.go │ │ │ ├── object_store_client.go │ │ │ ├── object_store_server.go │ │ │ ├── plugin_lister.go │ │ │ ├── plugin_types_test.go │ │ │ ├── restore_item_action.go │ │ │ ├── restore_item_action_client.go │ │ │ ├── restore_item_action_server.go │ │ │ ├── restoreitemaction/ │ │ │ │ └── v2/ │ │ │ │ ├── restore_item_action.go │ │ │ │ ├── restore_item_action_client.go │ │ │ │ └── restore_item_action_server.go │ │ │ ├── server.go │ │ │ ├── stream_reader.go │ │ │ ├── stream_reader_test.go │ │ │ ├── validation.go │ │ │ ├── validation_test.go │ │ │ ├── volume_snapshotter.go │ │ │ ├── volume_snapshotter_client.go │ │ │ └── volume_snapshotter_server.go │ │ ├── generated/ │ │ │ ├── BackupItemAction.pb.go │ │ │ ├── BackupItemAction_grpc.pb.go │ │ │ ├── DeleteItemAction.pb.go │ │ │ ├── DeleteItemAction_grpc.pb.go │ │ │ ├── ObjectStore.pb.go │ │ │ ├── ObjectStore_grpc.pb.go │ │ │ ├── PluginLister.pb.go │ │ │ ├── PluginLister_grpc.pb.go │ │ │ ├── RestoreItemAction.pb.go │ │ │ ├── RestoreItemAction_grpc.pb.go │ │ │ ├── Shared.pb.go │ │ │ ├── VolumeSnapshotter.pb.go │ │ │ ├── VolumeSnapshotter_grpc.pb.go │ │ │ ├── backupitemaction/ │ │ │ │ └── v2/ │ │ │ │ ├── BackupItemAction.pb.go │ │ │ │ └── BackupItemAction_grpc.pb.go │ │ │ ├── itemblockaction/ │ │ │ │ └── v1/ │ │ │ │ ├── ItemBlockAction.pb.go │ │ │ │ └── ItemBlockAction_grpc.pb.go │ │ │ └── restoreitemaction/ │ │ │ └── v2/ │ │ │ ├── RestoreItemAction.pb.go │ │ │ └── RestoreItemAction_grpc.pb.go │ │ ├── mocks/ │ │ │ ├── manager.go │ │ │ └── process_factory.go │ │ ├── proto/ │ │ │ ├── BackupItemAction.proto │ │ │ ├── DeleteItemAction.proto │ │ │ ├── ObjectStore.proto │ │ │ ├── PluginLister.proto │ │ │ ├── RestoreItemAction.proto │ │ │ ├── Shared.proto │ │ │ ├── VolumeSnapshotter.proto │ │ │ ├── backupitemaction/ │ │ │ │ └── v2/ │ │ │ │ └── BackupItemAction.proto │ │ │ ├── itemblockaction/ │ │ │ │ └── v1/ │ │ │ │ └── ItemBlockAction.proto │ │ │ └── restoreitemaction/ │ │ │ └── v2/ │ │ │ └── RestoreItemAction.proto │ │ ├── utils/ │ │ │ └── volumehelper/ │ │ │ ├── volume_policy_helper.go │ │ │ └── volume_policy_helper_test.go │ │ └── velero/ │ │ ├── backupitemaction/ │ │ │ ├── v1/ │ │ │ │ └── backup_item_action.go │ │ │ └── v2/ │ │ │ └── backup_item_action.go │ │ ├── delete_item_action.go │ │ ├── itemblockaction/ │ │ │ └── v1/ │ │ │ └── item_block_action.go │ │ ├── mocks/ │ │ │ ├── DeleteItemAction.go │ │ │ ├── backupitemaction/ │ │ │ │ ├── v1/ │ │ │ │ │ └── BackupItemAction.go │ │ │ │ └── v2/ │ │ │ │ └── BackupItemAction.go │ │ │ ├── itemblockaction/ │ │ │ │ └── v1/ │ │ │ │ └── ItemBlockAction.go │ │ │ ├── object_store.go │ │ │ ├── restoreitemaction/ │ │ │ │ ├── v1/ │ │ │ │ │ └── RestoreItemAction.go │ │ │ │ └── v2/ │ │ │ │ └── RestoreItemAction.go │ │ │ └── volumesnapshotter/ │ │ │ └── v1/ │ │ │ └── VolumeSnapshotter.go │ │ ├── object_store.go │ │ ├── restore_item_action_shared.go │ │ ├── restoreitemaction/ │ │ │ ├── v1/ │ │ │ │ └── restore_item_action.go │ │ │ └── v2/ │ │ │ └── restore_item_action.go │ │ ├── shared.go │ │ └── volumesnapshotter/ │ │ └── v1/ │ │ └── volume_snapshotter.go │ ├── podexec/ │ │ ├── pod_command_executor.go │ │ └── pod_command_executor_test.go │ ├── podvolume/ │ │ ├── backup_micro_service.go │ │ ├── backup_micro_service_test.go │ │ ├── backupper.go │ │ ├── backupper_factory.go │ │ ├── backupper_test.go │ │ ├── configs/ │ │ │ └── configs.go │ │ ├── mocks/ │ │ │ └── restorer.go │ │ ├── restore_micro_service.go │ │ ├── restore_micro_service_test.go │ │ ├── restorer.go │ │ ├── restorer_factory.go │ │ ├── restorer_test.go │ │ ├── snaphost_tracker_test.go │ │ ├── snapshot_tracker.go │ │ ├── util.go │ │ └── util_test.go │ ├── repository/ │ │ ├── backup_repo_op.go │ │ ├── backup_repo_op_test.go │ │ ├── config/ │ │ │ ├── aws.go │ │ │ ├── aws_test.go │ │ │ ├── azure.go │ │ │ ├── azure_test.go │ │ │ ├── config.go │ │ │ ├── config_test.go │ │ │ ├── gcp.go │ │ │ └── gcp_test.go │ │ ├── ensurer.go │ │ ├── ensurer_test.go │ │ ├── keys/ │ │ │ ├── keys.go │ │ │ └── keys_test.go │ │ ├── locker.go │ │ ├── maintenance/ │ │ │ ├── maintenance.go │ │ │ └── maintenance_test.go │ │ ├── manager/ │ │ │ ├── manager.go │ │ │ └── manager_test.go │ │ ├── mocks/ │ │ │ ├── ConfigManager.go │ │ │ ├── Manager.go │ │ │ └── RepositoryWriter.go │ │ ├── provider/ │ │ │ ├── provider.go │ │ │ ├── restic.go │ │ │ ├── unified_repo.go │ │ │ └── unified_repo_test.go │ │ ├── restic/ │ │ │ └── repository.go │ │ ├── types/ │ │ │ └── snapshotidentifier.go │ │ └── udmrepo/ │ │ ├── kopialib/ │ │ │ ├── backend/ │ │ │ │ ├── azure/ │ │ │ │ │ └── azure_storage_wrapper.go │ │ │ │ ├── azure.go │ │ │ │ ├── azure_test.go │ │ │ │ ├── backend.go │ │ │ │ ├── common.go │ │ │ │ ├── common_test.go │ │ │ │ ├── file_system.go │ │ │ │ ├── file_system_test.go │ │ │ │ ├── gcs.go │ │ │ │ ├── gcs_test.go │ │ │ │ ├── logging/ │ │ │ │ │ └── context.go │ │ │ │ ├── mocks/ │ │ │ │ │ ├── Logger.go │ │ │ │ │ ├── Reader.go │ │ │ │ │ ├── Storage.go │ │ │ │ │ ├── Store.go │ │ │ │ │ ├── Writer.go │ │ │ │ │ ├── repository.go │ │ │ │ │ └── repository_writer.go │ │ │ │ ├── s3.go │ │ │ │ ├── s3_test.go │ │ │ │ ├── utils.go │ │ │ │ └── utils_test.go │ │ │ ├── lib_repo.go │ │ │ ├── lib_repo_test.go │ │ │ ├── repo_init.go │ │ │ └── repo_init_test.go │ │ ├── mocks/ │ │ │ ├── BackupRepo.go │ │ │ ├── BackupRepoService.go │ │ │ ├── ObjectReader.go │ │ │ └── ObjectWriter.go │ │ ├── repo.go │ │ ├── repo_options.go │ │ └── service/ │ │ └── service.go │ ├── restic/ │ │ ├── command.go │ │ ├── command_factory.go │ │ ├── command_factory_test.go │ │ ├── command_test.go │ │ ├── common.go │ │ ├── common_test.go │ │ ├── exec_commands.go │ │ └── exec_commands_test.go │ ├── restore/ │ │ ├── actions/ │ │ │ ├── add_pvc_from_pod_action.go │ │ │ ├── add_pvc_from_pod_action_test.go │ │ │ ├── admissionwebhook_config_action.go │ │ │ ├── admissionwebhook_config_action_test.go │ │ │ ├── apiservice_action.go │ │ │ ├── apiservice_action_test.go │ │ │ ├── change_image_name_action.go │ │ │ ├── change_image_name_action_test.go │ │ │ ├── change_storageclass_action.go │ │ │ ├── change_storageclass_action_test.go │ │ │ ├── clusterrolebinding_action.go │ │ │ ├── clusterrolebinding_action_test.go │ │ │ ├── crd_v1_preserve_unknown_fields_action.go │ │ │ ├── crd_v1_preserve_unknown_fields_action_test.go │ │ │ ├── csi/ │ │ │ │ ├── pvc_action.go │ │ │ │ ├── pvc_action_test.go │ │ │ │ ├── volumesnapshot_action.go │ │ │ │ ├── volumesnapshot_action_test.go │ │ │ │ ├── volumesnapshotclass_action.go │ │ │ │ ├── volumesnapshotclass_action_test.go │ │ │ │ ├── volumesnapshotcontent_action.go │ │ │ │ └── volumesnapshotcontent_action_test.go │ │ │ ├── dataupload_retrieve_action.go │ │ │ ├── dataupload_retrieve_action_test.go │ │ │ ├── init_restorehook_pod_action.go │ │ │ ├── init_restorehook_pod_action_test.go │ │ │ ├── job_action.go │ │ │ ├── job_action_test.go │ │ │ ├── pod_action.go │ │ │ ├── pod_action_test.go │ │ │ ├── pod_volume_restore_action.go │ │ │ ├── pod_volume_restore_action_test.go │ │ │ ├── pvc_action.go │ │ │ ├── pvc_action_test.go │ │ │ ├── rolebinding_action.go │ │ │ ├── rolebinding_action_test.go │ │ │ ├── secret_action.go │ │ │ ├── secret_action_test.go │ │ │ ├── service_account_action.go │ │ │ ├── service_account_action_test.go │ │ │ ├── service_action.go │ │ │ └── service_action_test.go │ │ ├── merge_service_account.go │ │ ├── merge_service_account_test.go │ │ ├── prioritize_group_version.go │ │ ├── prioritize_group_version_test.go │ │ ├── pv_restorer.go │ │ ├── pv_restorer_test.go │ │ ├── request.go │ │ ├── request_test.go │ │ ├── restore.go │ │ ├── restore_test.go │ │ └── restore_wildcard_test.go │ ├── restorehelper/ │ │ └── util.go │ ├── test/ │ │ ├── api_server.go │ │ ├── comparisons.go │ │ ├── discovery_client.go │ │ ├── fake_controller_runtime_client.go │ │ ├── fake_credential_file_store.go │ │ ├── fake_discovery_helper.go │ │ ├── fake_dynamic.go │ │ ├── fake_file_system.go │ │ ├── fake_mapper.go │ │ ├── fake_namespace.go │ │ ├── fake_volume_snapshotter.go │ │ ├── helpers.go │ │ ├── mock_pod_command_executor.go │ │ ├── mocks/ │ │ │ └── VolumeSnapshotLister.go │ │ ├── mocks.go │ │ ├── resources.go │ │ ├── tar_writer.go │ │ └── test_logger.go │ ├── types/ │ │ ├── node_agent.go │ │ ├── priority.go │ │ ├── priority_test.go │ │ └── repo_maintenance.go │ ├── uploader/ │ │ ├── kopia/ │ │ │ ├── block_backup.go │ │ │ ├── block_backup_windows.go │ │ │ ├── block_restore.go │ │ │ ├── block_restore_windows.go │ │ │ ├── flush_volume_linux.go │ │ │ ├── flush_volume_other.go │ │ │ ├── progress.go │ │ │ ├── progress_test.go │ │ │ ├── restore_output.go │ │ │ ├── shim.go │ │ │ ├── shim_test.go │ │ │ ├── snapshot.go │ │ │ └── snapshot_test.go │ │ ├── mocks/ │ │ │ ├── policy.go │ │ │ ├── shim.go │ │ │ ├── snapshot.go │ │ │ └── uploader.go │ │ ├── provider/ │ │ │ ├── kopia.go │ │ │ ├── kopia_test.go │ │ │ ├── mocks/ │ │ │ │ └── Provider.go │ │ │ ├── provider.go │ │ │ ├── provider_test.go │ │ │ ├── restic.go │ │ │ └── restic_test.go │ │ ├── types.go │ │ ├── types_test.go │ │ └── util/ │ │ ├── uploader_config.go │ │ └── uploader_config_test.go │ └── util/ │ ├── actionhelpers/ │ │ ├── pod_helper.go │ │ ├── pvc_helper.go │ │ ├── rbac.go │ │ └── service_account_helper.go │ ├── azure/ │ │ ├── credential.go │ │ ├── credential_test.go │ │ ├── storage.go │ │ ├── storage_test.go │ │ ├── testdata/ │ │ │ └── certificate.pem │ │ ├── util.go │ │ └── util_test.go │ ├── boolptr/ │ │ └── boolptr.go │ ├── collections/ │ │ ├── includes_excludes.go │ │ └── includes_excludes_test.go │ ├── csi/ │ │ ├── util.go │ │ ├── util_test.go │ │ ├── volume_snapshot.go │ │ └── volume_snapshot_test.go │ ├── encode/ │ │ └── encode.go │ ├── exec/ │ │ └── exec.go │ ├── filesystem/ │ │ └── file_system.go │ ├── kube/ │ │ ├── client.go │ │ ├── event.go │ │ ├── event_handler.go │ │ ├── event_test.go │ │ ├── list_watch.go │ │ ├── list_watch_test.go │ │ ├── mocks/ │ │ │ └── Client.go │ │ ├── mocks.go │ │ ├── node.go │ │ ├── node_test.go │ │ ├── periodical_enqueue_source.go │ │ ├── periodical_enqueue_source_test.go │ │ ├── pod.go │ │ ├── pod_test.go │ │ ├── predicate.go │ │ ├── predicate_test.go │ │ ├── priority_class.go │ │ ├── priority_class_test.go │ │ ├── pvc_pv.go │ │ ├── pvc_pv_test.go │ │ ├── resource_deletionstatus_tracker.go │ │ ├── resource_requirements.go │ │ ├── resource_requirements_test.go │ │ ├── secrets.go │ │ ├── secrets_test.go │ │ ├── security_context.go │ │ ├── security_context_test.go │ │ ├── utils.go │ │ └── utils_test.go │ ├── logging/ │ │ ├── default_logger.go │ │ ├── default_logger_test.go │ │ ├── dual_mode_logger.go │ │ ├── dual_mode_logger_test.go │ │ ├── error_location_hook.go │ │ ├── error_location_hook_test.go │ │ ├── format_flag.go │ │ ├── hclog_level_hook.go │ │ ├── log_counter_hook.go │ │ ├── log_counter_hook_test.go │ │ ├── log_level_flag.go │ │ ├── log_location_hook.go │ │ ├── log_location_hook_test.go │ │ ├── log_merge_hook.go │ │ └── log_merge_hook_test.go │ ├── podvolume/ │ │ ├── pod_volume.go │ │ └── pod_volume_test.go │ ├── results/ │ │ ├── result.go │ │ └── result_test.go │ ├── scheme.go │ ├── stringptr/ │ │ └── stringptr.go │ ├── stringslice/ │ │ ├── stringslice.go │ │ └── stringslice_test.go │ ├── third_party.go │ ├── util.go │ ├── util_test.go │ ├── velero/ │ │ ├── restore/ │ │ │ ├── util.go │ │ │ └── util_test.go │ │ ├── velero.go │ │ └── velero_test.go │ └── wildcard/ │ ├── expand.go │ └── expand_test.go ├── restore-hooks_product-requirements.md ├── site/ │ ├── Dockerfile │ ├── README-HUGO.md │ ├── assets/ │ │ ├── _scss/ │ │ │ ├── _styles.scss │ │ │ ├── bootstrap-4.1.3/ │ │ │ │ ├── _alert.scss │ │ │ │ ├── _badge.scss │ │ │ │ ├── _breadcrumb.scss │ │ │ │ ├── _button-group.scss │ │ │ │ ├── _buttons.scss │ │ │ │ ├── _card.scss │ │ │ │ ├── _carousel.scss │ │ │ │ ├── _close.scss │ │ │ │ ├── _code.scss │ │ │ │ ├── _custom-forms.scss │ │ │ │ ├── _dropdown.scss │ │ │ │ ├── _forms.scss │ │ │ │ ├── _functions.scss │ │ │ │ ├── _grid.scss │ │ │ │ ├── _images.scss │ │ │ │ ├── _input-group.scss │ │ │ │ ├── _jumbotron.scss │ │ │ │ ├── _list-group.scss │ │ │ │ ├── _media.scss │ │ │ │ ├── _mixins.scss │ │ │ │ ├── _modal.scss │ │ │ │ ├── _nav.scss │ │ │ │ ├── _navbar.scss │ │ │ │ ├── _pagination.scss │ │ │ │ ├── _popover.scss │ │ │ │ ├── _print.scss │ │ │ │ ├── _progress.scss │ │ │ │ ├── _reboot.scss │ │ │ │ ├── _root.scss │ │ │ │ ├── _tables.scss │ │ │ │ ├── _tooltip.scss │ │ │ │ ├── _transitions.scss │ │ │ │ ├── _type.scss │ │ │ │ ├── _utilities.scss │ │ │ │ ├── _variables.scss │ │ │ │ ├── bootstrap-grid.scss │ │ │ │ ├── bootstrap-reboot.scss │ │ │ │ ├── bootstrap.scss │ │ │ │ ├── mixins/ │ │ │ │ │ ├── _alert.scss │ │ │ │ │ ├── _background-variant.scss │ │ │ │ │ ├── _badge.scss │ │ │ │ │ ├── _border-radius.scss │ │ │ │ │ ├── _box-shadow.scss │ │ │ │ │ ├── _breakpoints.scss │ │ │ │ │ ├── _buttons.scss │ │ │ │ │ ├── _caret.scss │ │ │ │ │ ├── _clearfix.scss │ │ │ │ │ ├── _float.scss │ │ │ │ │ ├── _forms.scss │ │ │ │ │ ├── _gradients.scss │ │ │ │ │ ├── _grid-framework.scss │ │ │ │ │ ├── _grid.scss │ │ │ │ │ ├── _hover.scss │ │ │ │ │ ├── _image.scss │ │ │ │ │ ├── _list-group.scss │ │ │ │ │ ├── _lists.scss │ │ │ │ │ ├── _nav-divider.scss │ │ │ │ │ ├── _pagination.scss │ │ │ │ │ ├── _reset-text.scss │ │ │ │ │ ├── _resize.scss │ │ │ │ │ ├── _screen-reader.scss │ │ │ │ │ ├── _size.scss │ │ │ │ │ ├── _table-row.scss │ │ │ │ │ ├── _text-emphasis.scss │ │ │ │ │ ├── _text-hide.scss │ │ │ │ │ ├── _text-truncate.scss │ │ │ │ │ ├── _transition.scss │ │ │ │ │ └── _visibility.scss │ │ │ │ └── utilities/ │ │ │ │ ├── _align.scss │ │ │ │ ├── _background.scss │ │ │ │ ├── _borders.scss │ │ │ │ ├── _clearfix.scss │ │ │ │ ├── _display.scss │ │ │ │ ├── _embed.scss │ │ │ │ ├── _flex.scss │ │ │ │ ├── _float.scss │ │ │ │ ├── _position.scss │ │ │ │ ├── _screenreaders.scss │ │ │ │ ├── _shadows.scss │ │ │ │ ├── _sizing.scss │ │ │ │ ├── _spacing.scss │ │ │ │ ├── _text.scss │ │ │ │ └── _visibility.scss │ │ │ └── site/ │ │ │ ├── common/ │ │ │ │ ├── _core.scss │ │ │ │ ├── _fonts.scss │ │ │ │ └── _type.scss │ │ │ ├── layouts/ │ │ │ │ ├── _container.scss │ │ │ │ ├── _docsearch.scss │ │ │ │ └── _documentation.scss │ │ │ ├── objects/ │ │ │ │ ├── _alternating-cards.scss │ │ │ │ ├── _button.scss │ │ │ │ ├── _card.scss │ │ │ │ ├── _footer.scss │ │ │ │ ├── _header.scss │ │ │ │ ├── _home-hero.scss │ │ │ │ ├── _post.scss │ │ │ │ ├── _section.scss │ │ │ │ └── _thumbnail-grid.scss │ │ │ ├── settings/ │ │ │ │ └── _variables.scss │ │ │ └── utilities/ │ │ │ ├── _image.scss │ │ │ └── _type.scss │ │ └── styles.scss │ ├── config.yaml │ ├── content/ │ │ ├── _index.md │ │ ├── casestudies/ │ │ │ ├── index.md │ │ │ ├── sample1.md │ │ │ ├── sample2.md │ │ │ └── sample3.md │ │ ├── community/ │ │ │ └── _index.md │ │ ├── contributors/ │ │ │ ├── 01-daniel-jiang.md │ │ │ ├── 02-scott-seago.md │ │ │ ├── 02-shubham-pampattiwar.md │ │ │ ├── 02-wenkai-yin.md │ │ │ ├── 02-xun-jiang.md │ │ │ ├── 04-pradeep-chaturvedi.md │ │ │ ├── 06-anshul-ahuja.md │ │ │ ├── 07-tiger-kaovilai.md │ │ │ └── index.md │ │ ├── docs/ │ │ │ ├── main/ │ │ │ │ ├── _index.md │ │ │ │ ├── api-types/ │ │ │ │ │ ├── README.md │ │ │ │ │ ├── _index.md │ │ │ │ │ ├── backup.md │ │ │ │ │ ├── backupstoragelocation.md │ │ │ │ │ ├── restore.md │ │ │ │ │ ├── schedule.md │ │ │ │ │ └── volumesnapshotlocation.md │ │ │ │ ├── backup-hooks.md │ │ │ │ ├── backup-reference.md │ │ │ │ ├── backup-repository-configuration.md │ │ │ │ ├── backup-restore-windows.md │ │ │ │ ├── basic-install.md │ │ │ │ ├── build-from-source.md │ │ │ │ ├── code-standards.md │ │ │ │ ├── contributions/ │ │ │ │ │ ├── ibm-config.md │ │ │ │ │ ├── minio.md │ │ │ │ │ ├── oracle-config.md │ │ │ │ │ └── tencent-config.md │ │ │ │ ├── csi-snapshot-data-movement.md │ │ │ │ ├── csi.md │ │ │ │ ├── custom-plugins.md │ │ │ │ ├── customize-installation.md │ │ │ │ ├── data-movement-backup-pvc-configuration.md │ │ │ │ ├── data-movement-cache-volume.md │ │ │ │ ├── data-movement-node-selection.md │ │ │ │ ├── data-movement-pod-resource-configuration.md │ │ │ │ ├── data-movement-restore-pvc-configuration.md │ │ │ │ ├── debugging-install.md │ │ │ │ ├── debugging-restores.md │ │ │ │ ├── development.md │ │ │ │ ├── disaster-case.md │ │ │ │ ├── enable-api-group-versions-feature.md │ │ │ │ ├── examples.md │ │ │ │ ├── file-system-backup.md │ │ │ │ ├── how-velero-works.md │ │ │ │ ├── image-tagging.md │ │ │ │ ├── img/ │ │ │ │ │ └── README.md │ │ │ │ ├── locations.md │ │ │ │ ├── maintainers.md │ │ │ │ ├── manual-testing.md │ │ │ │ ├── migration-case.md │ │ │ │ ├── namespace-glob-patterns.md │ │ │ │ ├── namespace.md │ │ │ │ ├── node-agent-concurrency.md │ │ │ │ ├── node-agent-prepare-queue-length.md │ │ │ │ ├── on-premises.md │ │ │ │ ├── output-file-format.md │ │ │ │ ├── overview-plugins.md │ │ │ │ ├── performance-guidance.md │ │ │ │ ├── plugin-release-instructions.md │ │ │ │ ├── proxy.md │ │ │ │ ├── rbac.md │ │ │ │ ├── release-instructions.md │ │ │ │ ├── release-schedule.md │ │ │ │ ├── repository-maintenance.md │ │ │ │ ├── resource-filtering.md │ │ │ │ ├── restore-hooks.md │ │ │ │ ├── restore-reference.md │ │ │ │ ├── restore-resource-modifiers.md │ │ │ │ ├── run-locally.md │ │ │ │ ├── self-signed-certificates.md │ │ │ │ ├── start-contributing.md │ │ │ │ ├── style-guide.md │ │ │ │ ├── support-process.md │ │ │ │ ├── supported-configmaps/ │ │ │ │ │ ├── _index.md │ │ │ │ │ └── node-agent-configmap.md │ │ │ │ ├── supported-providers.md │ │ │ │ ├── tilt.md │ │ │ │ ├── troubleshooting.md │ │ │ │ ├── uninstalling.md │ │ │ │ ├── upgrade-to-1.18.md │ │ │ │ ├── velero-install.md │ │ │ │ ├── volume-group-snapshots.md │ │ │ │ └── website-guidelines.md │ │ │ ├── v0.10.0/ │ │ │ │ ├── _index.md │ │ │ │ ├── about.md │ │ │ │ ├── api-types/ │ │ │ │ │ ├── README.md │ │ │ │ │ ├── backup.md │ │ │ │ │ ├── backupstoragelocation.md │ │ │ │ │ └── volumesnapshotlocation.md │ │ │ │ ├── aws-config.md │ │ │ │ ├── azure-config.md │ │ │ │ ├── build-from-scratch.md │ │ │ │ ├── debugging-install.md │ │ │ │ ├── debugging-restores.md │ │ │ │ ├── disaster-case.md │ │ │ │ ├── extend.md │ │ │ │ ├── faq.md │ │ │ │ ├── gcp-config.md │ │ │ │ ├── get-started.md │ │ │ │ ├── hooks.md │ │ │ │ ├── ibm-config.md │ │ │ │ ├── image-tagging.md │ │ │ │ ├── img/ │ │ │ │ │ └── README.md │ │ │ │ ├── install-overview.md │ │ │ │ ├── locations.md │ │ │ │ ├── migration-case.md │ │ │ │ ├── namespace.md │ │ │ │ ├── output-file-format.md │ │ │ │ ├── plugins.md │ │ │ │ ├── rbac.md │ │ │ │ ├── restic.md │ │ │ │ ├── storage-layout-reorg-v0.10.md │ │ │ │ ├── support-matrix.md │ │ │ │ ├── troubleshooting.md │ │ │ │ ├── upgrading-to-v0.10.md │ │ │ │ ├── vendoring-dependencies.md │ │ │ │ ├── versions.md │ │ │ │ └── zenhub.md │ │ │ ├── v0.11.0/ │ │ │ │ ├── _index.md │ │ │ │ ├── about.md │ │ │ │ ├── api-types/ │ │ │ │ │ ├── README.md │ │ │ │ │ ├── backup.md │ │ │ │ │ ├── backupstoragelocation.md │ │ │ │ │ └── volumesnapshotlocation.md │ │ │ │ ├── aws-config.md │ │ │ │ ├── azure-config.md │ │ │ │ ├── build-from-scratch.md │ │ │ │ ├── debugging-install.md │ │ │ │ ├── debugging-restores.md │ │ │ │ ├── disaster-case.md │ │ │ │ ├── expose-minio.md │ │ │ │ ├── extend.md │ │ │ │ ├── faq.md │ │ │ │ ├── gcp-config.md │ │ │ │ ├── get-started.md │ │ │ │ ├── hooks.md │ │ │ │ ├── ibm-config.md │ │ │ │ ├── image-tagging.md │ │ │ │ ├── img/ │ │ │ │ │ └── README.md │ │ │ │ ├── install-overview.md │ │ │ │ ├── locations.md │ │ │ │ ├── migrating-to-velero.md │ │ │ │ ├── migration-case.md │ │ │ │ ├── namespace.md │ │ │ │ ├── output-file-format.md │ │ │ │ ├── plugins.md │ │ │ │ ├── rbac.md │ │ │ │ ├── restic.md │ │ │ │ ├── support-matrix.md │ │ │ │ ├── troubleshooting.md │ │ │ │ ├── vendoring-dependencies.md │ │ │ │ ├── versions.md │ │ │ │ └── zenhub.md │ │ │ ├── v0.3.0/ │ │ │ │ ├── _index.md │ │ │ │ ├── build-from-scratch.md │ │ │ │ ├── cli-reference/ │ │ │ │ │ ├── README.md │ │ │ │ │ ├── ark.md │ │ │ │ │ ├── ark_backup.md │ │ │ │ │ ├── ark_backup_create.md │ │ │ │ │ ├── ark_backup_get.md │ │ │ │ │ ├── ark_restore.md │ │ │ │ │ ├── ark_restore_create.md │ │ │ │ │ ├── ark_restore_delete.md │ │ │ │ │ ├── ark_restore_get.md │ │ │ │ │ ├── ark_schedule.md │ │ │ │ │ ├── ark_schedule_create.md │ │ │ │ │ ├── ark_schedule_delete.md │ │ │ │ │ ├── ark_schedule_get.md │ │ │ │ │ ├── ark_server.md │ │ │ │ │ └── ark_version.md │ │ │ │ ├── cloud-provider-specifics.md │ │ │ │ ├── concepts.md │ │ │ │ ├── config-definition.md │ │ │ │ ├── debugging-restores.md │ │ │ │ ├── img/ │ │ │ │ │ └── README.md │ │ │ │ ├── output-file-format.md │ │ │ │ └── use-cases.md │ │ │ ├── v0.4.0/ │ │ │ │ ├── _index.md │ │ │ │ ├── build-from-scratch.md │ │ │ │ ├── cli-reference/ │ │ │ │ │ ├── README.md │ │ │ │ │ ├── ark.md │ │ │ │ │ ├── ark_backup.md │ │ │ │ │ ├── ark_backup_create.md │ │ │ │ │ ├── ark_backup_download.md │ │ │ │ │ ├── ark_backup_get.md │ │ │ │ │ ├── ark_backup_logs.md │ │ │ │ │ ├── ark_restore.md │ │ │ │ │ ├── ark_restore_create.md │ │ │ │ │ ├── ark_restore_delete.md │ │ │ │ │ ├── ark_restore_get.md │ │ │ │ │ ├── ark_restore_logs.md │ │ │ │ │ ├── ark_schedule.md │ │ │ │ │ ├── ark_schedule_create.md │ │ │ │ │ ├── ark_schedule_delete.md │ │ │ │ │ ├── ark_schedule_get.md │ │ │ │ │ ├── ark_server.md │ │ │ │ │ └── ark_version.md │ │ │ │ ├── cloud-provider-specifics.md │ │ │ │ ├── concepts.md │ │ │ │ ├── config-definition.md │ │ │ │ ├── debugging-restores.md │ │ │ │ ├── img/ │ │ │ │ │ └── README.md │ │ │ │ ├── output-file-format.md │ │ │ │ ├── use-cases.md │ │ │ │ └── vendoring-dependencies.md │ │ │ ├── v0.5.0/ │ │ │ │ ├── _index.md │ │ │ │ ├── api-types/ │ │ │ │ │ ├── README.md │ │ │ │ │ └── backup.md │ │ │ │ ├── build-from-scratch.md │ │ │ │ ├── cli-reference/ │ │ │ │ │ ├── README.md │ │ │ │ │ ├── ark.md │ │ │ │ │ ├── ark_backup.md │ │ │ │ │ ├── ark_backup_create.md │ │ │ │ │ ├── ark_backup_download.md │ │ │ │ │ ├── ark_backup_get.md │ │ │ │ │ ├── ark_backup_logs.md │ │ │ │ │ ├── ark_create.md │ │ │ │ │ ├── ark_create_backup.md │ │ │ │ │ ├── ark_create_restore.md │ │ │ │ │ ├── ark_create_schedule.md │ │ │ │ │ ├── ark_get.md │ │ │ │ │ ├── ark_get_backups.md │ │ │ │ │ ├── ark_get_restores.md │ │ │ │ │ ├── ark_get_schedules.md │ │ │ │ │ ├── ark_restore.md │ │ │ │ │ ├── ark_restore_create.md │ │ │ │ │ ├── ark_restore_delete.md │ │ │ │ │ ├── ark_restore_get.md │ │ │ │ │ ├── ark_restore_logs.md │ │ │ │ │ ├── ark_schedule.md │ │ │ │ │ ├── ark_schedule_create.md │ │ │ │ │ ├── ark_schedule_delete.md │ │ │ │ │ ├── ark_schedule_get.md │ │ │ │ │ ├── ark_server.md │ │ │ │ │ └── ark_version.md │ │ │ │ ├── cloud-provider-specifics.md │ │ │ │ ├── concepts.md │ │ │ │ ├── config-definition.md │ │ │ │ ├── debugging-restores.md │ │ │ │ ├── faq.md │ │ │ │ ├── hooks.md │ │ │ │ ├── img/ │ │ │ │ │ └── README.md │ │ │ │ ├── output-file-format.md │ │ │ │ ├── use-cases.md │ │ │ │ └── vendoring-dependencies.md │ │ │ ├── v0.6.0/ │ │ │ │ ├── _index.md │ │ │ │ ├── api-types/ │ │ │ │ │ ├── README.md │ │ │ │ │ └── backup.md │ │ │ │ ├── build-from-scratch.md │ │ │ │ ├── cli-reference/ │ │ │ │ │ ├── README.md │ │ │ │ │ ├── ark.md │ │ │ │ │ ├── ark_backup.md │ │ │ │ │ ├── ark_backup_create.md │ │ │ │ │ ├── ark_backup_describe.md │ │ │ │ │ ├── ark_backup_download.md │ │ │ │ │ ├── ark_backup_get.md │ │ │ │ │ ├── ark_backup_logs.md │ │ │ │ │ ├── ark_create.md │ │ │ │ │ ├── ark_create_backup.md │ │ │ │ │ ├── ark_create_restore.md │ │ │ │ │ ├── ark_create_schedule.md │ │ │ │ │ ├── ark_describe.md │ │ │ │ │ ├── ark_describe_backups.md │ │ │ │ │ ├── ark_describe_restores.md │ │ │ │ │ ├── ark_describe_schedules.md │ │ │ │ │ ├── ark_get.md │ │ │ │ │ ├── ark_get_backups.md │ │ │ │ │ ├── ark_get_restores.md │ │ │ │ │ ├── ark_get_schedules.md │ │ │ │ │ ├── ark_plugin.md │ │ │ │ │ ├── ark_plugin_add.md │ │ │ │ │ ├── ark_plugin_remove.md │ │ │ │ │ ├── ark_restore.md │ │ │ │ │ ├── ark_restore_create.md │ │ │ │ │ ├── ark_restore_delete.md │ │ │ │ │ ├── ark_restore_describe.md │ │ │ │ │ ├── ark_restore_get.md │ │ │ │ │ ├── ark_restore_logs.md │ │ │ │ │ ├── ark_schedule.md │ │ │ │ │ ├── ark_schedule_create.md │ │ │ │ │ ├── ark_schedule_delete.md │ │ │ │ │ ├── ark_schedule_describe.md │ │ │ │ │ ├── ark_schedule_get.md │ │ │ │ │ ├── ark_server.md │ │ │ │ │ └── ark_version.md │ │ │ │ ├── cloud-provider-specifics.md │ │ │ │ ├── concepts.md │ │ │ │ ├── config-definition.md │ │ │ │ ├── debugging-restores.md │ │ │ │ ├── faq.md │ │ │ │ ├── hooks.md │ │ │ │ ├── img/ │ │ │ │ │ └── README.md │ │ │ │ ├── output-file-format.md │ │ │ │ ├── plugins.md │ │ │ │ ├── use-cases.md │ │ │ │ └── vendoring-dependencies.md │ │ │ ├── v0.7.0/ │ │ │ │ ├── _index.md │ │ │ │ ├── about.md │ │ │ │ ├── api-types/ │ │ │ │ │ ├── README.md │ │ │ │ │ └── backup.md │ │ │ │ ├── aws-config.md │ │ │ │ ├── azure-config.md │ │ │ │ ├── build-from-scratch.md │ │ │ │ ├── cli-reference/ │ │ │ │ │ ├── README.md │ │ │ │ │ ├── ark.md │ │ │ │ │ ├── ark_backup.md │ │ │ │ │ ├── ark_backup_create.md │ │ │ │ │ ├── ark_backup_delete.md │ │ │ │ │ ├── ark_backup_describe.md │ │ │ │ │ ├── ark_backup_download.md │ │ │ │ │ ├── ark_backup_get.md │ │ │ │ │ ├── ark_backup_logs.md │ │ │ │ │ ├── ark_client.md │ │ │ │ │ ├── ark_client_config.md │ │ │ │ │ ├── ark_client_config_get.md │ │ │ │ │ ├── ark_client_config_set.md │ │ │ │ │ ├── ark_create.md │ │ │ │ │ ├── ark_create_backup.md │ │ │ │ │ ├── ark_create_restore.md │ │ │ │ │ ├── ark_create_schedule.md │ │ │ │ │ ├── ark_delete.md │ │ │ │ │ ├── ark_delete_backup.md │ │ │ │ │ ├── ark_delete_restore.md │ │ │ │ │ ├── ark_delete_schedule.md │ │ │ │ │ ├── ark_describe.md │ │ │ │ │ ├── ark_describe_backups.md │ │ │ │ │ ├── ark_describe_restores.md │ │ │ │ │ ├── ark_describe_schedules.md │ │ │ │ │ ├── ark_get.md │ │ │ │ │ ├── ark_get_backups.md │ │ │ │ │ ├── ark_get_restores.md │ │ │ │ │ ├── ark_get_schedules.md │ │ │ │ │ ├── ark_plugin.md │ │ │ │ │ ├── ark_plugin_add.md │ │ │ │ │ ├── ark_plugin_remove.md │ │ │ │ │ ├── ark_restore.md │ │ │ │ │ ├── ark_restore_create.md │ │ │ │ │ ├── ark_restore_delete.md │ │ │ │ │ ├── ark_restore_describe.md │ │ │ │ │ ├── ark_restore_get.md │ │ │ │ │ ├── ark_restore_logs.md │ │ │ │ │ ├── ark_schedule.md │ │ │ │ │ ├── ark_schedule_create.md │ │ │ │ │ ├── ark_schedule_delete.md │ │ │ │ │ ├── ark_schedule_describe.md │ │ │ │ │ ├── ark_schedule_get.md │ │ │ │ │ ├── ark_server.md │ │ │ │ │ └── ark_version.md │ │ │ │ ├── cloud-common.md │ │ │ │ ├── config-definition.md │ │ │ │ ├── debugging-restores.md │ │ │ │ ├── extend.md │ │ │ │ ├── faq.md │ │ │ │ ├── gcp-config.md │ │ │ │ ├── hooks.md │ │ │ │ ├── img/ │ │ │ │ │ └── README.md │ │ │ │ ├── namespace.md │ │ │ │ ├── output-file-format.md │ │ │ │ ├── plugins.md │ │ │ │ ├── use-cases.md │ │ │ │ └── vendoring-dependencies.md │ │ │ ├── v0.7.1/ │ │ │ │ ├── _index.md │ │ │ │ ├── about.md │ │ │ │ ├── api-types/ │ │ │ │ │ ├── README.md │ │ │ │ │ └── backup.md │ │ │ │ ├── aws-config.md │ │ │ │ ├── azure-config.md │ │ │ │ ├── build-from-scratch.md │ │ │ │ ├── cli-reference/ │ │ │ │ │ ├── README.md │ │ │ │ │ ├── ark.md │ │ │ │ │ ├── ark_backup.md │ │ │ │ │ ├── ark_backup_create.md │ │ │ │ │ ├── ark_backup_delete.md │ │ │ │ │ ├── ark_backup_describe.md │ │ │ │ │ ├── ark_backup_download.md │ │ │ │ │ ├── ark_backup_get.md │ │ │ │ │ ├── ark_backup_logs.md │ │ │ │ │ ├── ark_client.md │ │ │ │ │ ├── ark_client_config.md │ │ │ │ │ ├── ark_client_config_get.md │ │ │ │ │ ├── ark_client_config_set.md │ │ │ │ │ ├── ark_create.md │ │ │ │ │ ├── ark_create_backup.md │ │ │ │ │ ├── ark_create_restore.md │ │ │ │ │ ├── ark_create_schedule.md │ │ │ │ │ ├── ark_delete.md │ │ │ │ │ ├── ark_delete_backup.md │ │ │ │ │ ├── ark_delete_restore.md │ │ │ │ │ ├── ark_delete_schedule.md │ │ │ │ │ ├── ark_describe.md │ │ │ │ │ ├── ark_describe_backups.md │ │ │ │ │ ├── ark_describe_restores.md │ │ │ │ │ ├── ark_describe_schedules.md │ │ │ │ │ ├── ark_get.md │ │ │ │ │ ├── ark_get_backups.md │ │ │ │ │ ├── ark_get_restores.md │ │ │ │ │ ├── ark_get_schedules.md │ │ │ │ │ ├── ark_plugin.md │ │ │ │ │ ├── ark_plugin_add.md │ │ │ │ │ ├── ark_plugin_remove.md │ │ │ │ │ ├── ark_restore.md │ │ │ │ │ ├── ark_restore_create.md │ │ │ │ │ ├── ark_restore_delete.md │ │ │ │ │ ├── ark_restore_describe.md │ │ │ │ │ ├── ark_restore_get.md │ │ │ │ │ ├── ark_restore_logs.md │ │ │ │ │ ├── ark_schedule.md │ │ │ │ │ ├── ark_schedule_create.md │ │ │ │ │ ├── ark_schedule_delete.md │ │ │ │ │ ├── ark_schedule_describe.md │ │ │ │ │ ├── ark_schedule_get.md │ │ │ │ │ ├── ark_server.md │ │ │ │ │ └── ark_version.md │ │ │ │ ├── cloud-common.md │ │ │ │ ├── config-definition.md │ │ │ │ ├── debugging-deletes.md │ │ │ │ ├── debugging-restores.md │ │ │ │ ├── extend.md │ │ │ │ ├── faq.md │ │ │ │ ├── gcp-config.md │ │ │ │ ├── hooks.md │ │ │ │ ├── img/ │ │ │ │ │ └── README.md │ │ │ │ ├── namespace.md │ │ │ │ ├── output-file-format.md │ │ │ │ ├── plugins.md │ │ │ │ ├── troubleshooting.md │ │ │ │ ├── use-cases.md │ │ │ │ └── vendoring-dependencies.md │ │ │ ├── v0.8.0/ │ │ │ │ ├── _index.md │ │ │ │ ├── about.md │ │ │ │ ├── api-types/ │ │ │ │ │ ├── README.md │ │ │ │ │ └── backup.md │ │ │ │ ├── aws-config.md │ │ │ │ ├── azure-config.md │ │ │ │ ├── build-from-scratch.md │ │ │ │ ├── cli-reference/ │ │ │ │ │ ├── README.md │ │ │ │ │ ├── ark.md │ │ │ │ │ ├── ark_backup.md │ │ │ │ │ ├── ark_backup_create.md │ │ │ │ │ ├── ark_backup_delete.md │ │ │ │ │ ├── ark_backup_describe.md │ │ │ │ │ ├── ark_backup_download.md │ │ │ │ │ ├── ark_backup_get.md │ │ │ │ │ ├── ark_backup_logs.md │ │ │ │ │ ├── ark_client.md │ │ │ │ │ ├── ark_client_config.md │ │ │ │ │ ├── ark_client_config_get.md │ │ │ │ │ ├── ark_client_config_set.md │ │ │ │ │ ├── ark_completion.md │ │ │ │ │ ├── ark_create.md │ │ │ │ │ ├── ark_create_backup.md │ │ │ │ │ ├── ark_create_restore.md │ │ │ │ │ ├── ark_create_schedule.md │ │ │ │ │ ├── ark_delete.md │ │ │ │ │ ├── ark_delete_backup.md │ │ │ │ │ ├── ark_delete_restore.md │ │ │ │ │ ├── ark_delete_schedule.md │ │ │ │ │ ├── ark_describe.md │ │ │ │ │ ├── ark_describe_backups.md │ │ │ │ │ ├── ark_describe_restores.md │ │ │ │ │ ├── ark_describe_schedules.md │ │ │ │ │ ├── ark_get.md │ │ │ │ │ ├── ark_get_backups.md │ │ │ │ │ ├── ark_get_restores.md │ │ │ │ │ ├── ark_get_schedules.md │ │ │ │ │ ├── ark_plugin.md │ │ │ │ │ ├── ark_plugin_add.md │ │ │ │ │ ├── ark_plugin_remove.md │ │ │ │ │ ├── ark_restore.md │ │ │ │ │ ├── ark_restore_create.md │ │ │ │ │ ├── ark_restore_delete.md │ │ │ │ │ ├── ark_restore_describe.md │ │ │ │ │ ├── ark_restore_get.md │ │ │ │ │ ├── ark_restore_logs.md │ │ │ │ │ ├── ark_schedule.md │ │ │ │ │ ├── ark_schedule_create.md │ │ │ │ │ ├── ark_schedule_delete.md │ │ │ │ │ ├── ark_schedule_describe.md │ │ │ │ │ ├── ark_schedule_get.md │ │ │ │ │ ├── ark_server.md │ │ │ │ │ └── ark_version.md │ │ │ │ ├── cloud-common.md │ │ │ │ ├── config-definition.md │ │ │ │ ├── debugging-deletes.md │ │ │ │ ├── debugging-restores.md │ │ │ │ ├── extend.md │ │ │ │ ├── faq.md │ │ │ │ ├── gcp-config.md │ │ │ │ ├── hooks.md │ │ │ │ ├── ibm-config.md │ │ │ │ ├── img/ │ │ │ │ │ └── README.md │ │ │ │ ├── namespace.md │ │ │ │ ├── output-file-format.md │ │ │ │ ├── plugins.md │ │ │ │ ├── troubleshooting.md │ │ │ │ ├── use-cases.md │ │ │ │ └── vendoring-dependencies.md │ │ │ ├── v0.8.1/ │ │ │ │ ├── _index.md │ │ │ │ ├── about.md │ │ │ │ ├── api-types/ │ │ │ │ │ ├── README.md │ │ │ │ │ └── backup.md │ │ │ │ ├── aws-config.md │ │ │ │ ├── azure-config.md │ │ │ │ ├── build-from-scratch.md │ │ │ │ ├── cli-reference/ │ │ │ │ │ ├── README.md │ │ │ │ │ ├── ark.md │ │ │ │ │ ├── ark_backup.md │ │ │ │ │ ├── ark_backup_create.md │ │ │ │ │ ├── ark_backup_delete.md │ │ │ │ │ ├── ark_backup_describe.md │ │ │ │ │ ├── ark_backup_download.md │ │ │ │ │ ├── ark_backup_get.md │ │ │ │ │ ├── ark_backup_logs.md │ │ │ │ │ ├── ark_client.md │ │ │ │ │ ├── ark_client_config.md │ │ │ │ │ ├── ark_client_config_get.md │ │ │ │ │ ├── ark_client_config_set.md │ │ │ │ │ ├── ark_completion.md │ │ │ │ │ ├── ark_create.md │ │ │ │ │ ├── ark_create_backup.md │ │ │ │ │ ├── ark_create_restore.md │ │ │ │ │ ├── ark_create_schedule.md │ │ │ │ │ ├── ark_delete.md │ │ │ │ │ ├── ark_delete_backup.md │ │ │ │ │ ├── ark_delete_restore.md │ │ │ │ │ ├── ark_delete_schedule.md │ │ │ │ │ ├── ark_describe.md │ │ │ │ │ ├── ark_describe_backups.md │ │ │ │ │ ├── ark_describe_restores.md │ │ │ │ │ ├── ark_describe_schedules.md │ │ │ │ │ ├── ark_get.md │ │ │ │ │ ├── ark_get_backups.md │ │ │ │ │ ├── ark_get_restores.md │ │ │ │ │ ├── ark_get_schedules.md │ │ │ │ │ ├── ark_plugin.md │ │ │ │ │ ├── ark_plugin_add.md │ │ │ │ │ ├── ark_plugin_remove.md │ │ │ │ │ ├── ark_restore.md │ │ │ │ │ ├── ark_restore_create.md │ │ │ │ │ ├── ark_restore_delete.md │ │ │ │ │ ├── ark_restore_describe.md │ │ │ │ │ ├── ark_restore_get.md │ │ │ │ │ ├── ark_restore_logs.md │ │ │ │ │ ├── ark_schedule.md │ │ │ │ │ ├── ark_schedule_create.md │ │ │ │ │ ├── ark_schedule_delete.md │ │ │ │ │ ├── ark_schedule_describe.md │ │ │ │ │ ├── ark_schedule_get.md │ │ │ │ │ ├── ark_server.md │ │ │ │ │ └── ark_version.md │ │ │ │ ├── cloud-common.md │ │ │ │ ├── config-definition.md │ │ │ │ ├── debugging-deletes.md │ │ │ │ ├── debugging-install.md │ │ │ │ ├── debugging-restores.md │ │ │ │ ├── extend.md │ │ │ │ ├── faq.md │ │ │ │ ├── gcp-config.md │ │ │ │ ├── hooks.md │ │ │ │ ├── ibm-config.md │ │ │ │ ├── img/ │ │ │ │ │ └── README.md │ │ │ │ ├── namespace.md │ │ │ │ ├── output-file-format.md │ │ │ │ ├── plugins.md │ │ │ │ ├── troubleshooting.md │ │ │ │ ├── use-cases.md │ │ │ │ └── vendoring-dependencies.md │ │ │ ├── v0.9.0/ │ │ │ │ ├── _index.md │ │ │ │ ├── about.md │ │ │ │ ├── api-types/ │ │ │ │ │ ├── README.md │ │ │ │ │ └── backup.md │ │ │ │ ├── aws-config.md │ │ │ │ ├── azure-config.md │ │ │ │ ├── build-from-scratch.md │ │ │ │ ├── cli-reference/ │ │ │ │ │ ├── README.md │ │ │ │ │ ├── ark.md │ │ │ │ │ ├── ark_backup.md │ │ │ │ │ ├── ark_backup_create.md │ │ │ │ │ ├── ark_backup_delete.md │ │ │ │ │ ├── ark_backup_describe.md │ │ │ │ │ ├── ark_backup_download.md │ │ │ │ │ ├── ark_backup_get.md │ │ │ │ │ ├── ark_backup_logs.md │ │ │ │ │ ├── ark_client.md │ │ │ │ │ ├── ark_client_config.md │ │ │ │ │ ├── ark_client_config_get.md │ │ │ │ │ ├── ark_client_config_set.md │ │ │ │ │ ├── ark_completion.md │ │ │ │ │ ├── ark_create.md │ │ │ │ │ ├── ark_create_backup.md │ │ │ │ │ ├── ark_create_restore.md │ │ │ │ │ ├── ark_create_schedule.md │ │ │ │ │ ├── ark_delete.md │ │ │ │ │ ├── ark_delete_backup.md │ │ │ │ │ ├── ark_delete_restore.md │ │ │ │ │ ├── ark_delete_schedule.md │ │ │ │ │ ├── ark_describe.md │ │ │ │ │ ├── ark_describe_backups.md │ │ │ │ │ ├── ark_describe_restores.md │ │ │ │ │ ├── ark_describe_schedules.md │ │ │ │ │ ├── ark_get.md │ │ │ │ │ ├── ark_get_backups.md │ │ │ │ │ ├── ark_get_restores.md │ │ │ │ │ ├── ark_get_schedules.md │ │ │ │ │ ├── ark_plugin.md │ │ │ │ │ ├── ark_plugin_add.md │ │ │ │ │ ├── ark_plugin_remove.md │ │ │ │ │ ├── ark_restic.md │ │ │ │ │ ├── ark_restic_repo.md │ │ │ │ │ ├── ark_restic_repo_get.md │ │ │ │ │ ├── ark_restic_server.md │ │ │ │ │ ├── ark_restore.md │ │ │ │ │ ├── ark_restore_create.md │ │ │ │ │ ├── ark_restore_delete.md │ │ │ │ │ ├── ark_restore_describe.md │ │ │ │ │ ├── ark_restore_get.md │ │ │ │ │ ├── ark_restore_logs.md │ │ │ │ │ ├── ark_schedule.md │ │ │ │ │ ├── ark_schedule_create.md │ │ │ │ │ ├── ark_schedule_delete.md │ │ │ │ │ ├── ark_schedule_describe.md │ │ │ │ │ ├── ark_schedule_get.md │ │ │ │ │ ├── ark_server.md │ │ │ │ │ └── ark_version.md │ │ │ │ ├── config-definition.md │ │ │ │ ├── debugging-install.md │ │ │ │ ├── debugging-restores.md │ │ │ │ ├── extend.md │ │ │ │ ├── faq.md │ │ │ │ ├── gcp-config.md │ │ │ │ ├── hooks.md │ │ │ │ ├── ibm-config.md │ │ │ │ ├── image-tagging.md │ │ │ │ ├── img/ │ │ │ │ │ └── README.md │ │ │ │ ├── install-overview.md │ │ │ │ ├── namespace.md │ │ │ │ ├── output-file-format.md │ │ │ │ ├── plugins.md │ │ │ │ ├── quickstart.md │ │ │ │ ├── restic.md │ │ │ │ ├── support-matrix.md │ │ │ │ ├── troubleshooting.md │ │ │ │ ├── use-cases.md │ │ │ │ └── vendoring-dependencies.md │ │ │ ├── v1.0.0/ │ │ │ │ ├── _index.md │ │ │ │ ├── about.md │ │ │ │ ├── api-types/ │ │ │ │ │ ├── README.md │ │ │ │ │ ├── backup.md │ │ │ │ │ ├── backupstoragelocation.md │ │ │ │ │ └── volumesnapshotlocation.md │ │ │ │ ├── aws-config.md │ │ │ │ ├── azure-config.md │ │ │ │ ├── build-from-source.md │ │ │ │ ├── debugging-install.md │ │ │ │ ├── debugging-restores.md │ │ │ │ ├── disaster-case.md │ │ │ │ ├── extend.md │ │ │ │ ├── faq.md │ │ │ │ ├── gcp-config.md │ │ │ │ ├── get-started.md │ │ │ │ ├── hooks.md │ │ │ │ ├── ibm-config.md │ │ │ │ ├── image-tagging.md │ │ │ │ ├── img/ │ │ │ │ │ └── README.md │ │ │ │ ├── install-overview.md │ │ │ │ ├── locations.md │ │ │ │ ├── migrating-to-velero.md │ │ │ │ ├── migration-case.md │ │ │ │ ├── namespace.md │ │ │ │ ├── oracle-config.md │ │ │ │ ├── output-file-format.md │ │ │ │ ├── plugins.md │ │ │ │ ├── rbac.md │ │ │ │ ├── restic.md │ │ │ │ ├── support-matrix.md │ │ │ │ ├── troubleshooting.md │ │ │ │ ├── upgrade-to-1.0.md │ │ │ │ ├── vendoring-dependencies.md │ │ │ │ └── zenhub.md │ │ │ ├── v1.1.0/ │ │ │ │ ├── _index.md │ │ │ │ ├── about.md │ │ │ │ ├── api-types/ │ │ │ │ │ ├── README.md │ │ │ │ │ ├── backup.md │ │ │ │ │ ├── backupstoragelocation.md │ │ │ │ │ └── volumesnapshotlocation.md │ │ │ │ ├── aws-config.md │ │ │ │ ├── azure-config.md │ │ │ │ ├── backup-reference.md │ │ │ │ ├── build-from-source.md │ │ │ │ ├── debugging-install.md │ │ │ │ ├── debugging-restores.md │ │ │ │ ├── development.md │ │ │ │ ├── disaster-case.md │ │ │ │ ├── faq.md │ │ │ │ ├── gcp-config.md │ │ │ │ ├── get-started.md │ │ │ │ ├── hooks.md │ │ │ │ ├── ibm-config.md │ │ │ │ ├── image-tagging.md │ │ │ │ ├── img/ │ │ │ │ │ └── README.md │ │ │ │ ├── install-overview.md │ │ │ │ ├── locations.md │ │ │ │ ├── migrating-to-velero.md │ │ │ │ ├── migration-case.md │ │ │ │ ├── namespace.md │ │ │ │ ├── oracle-config.md │ │ │ │ ├── output-file-format.md │ │ │ │ ├── plugins.md │ │ │ │ ├── rbac.md │ │ │ │ ├── restic.md │ │ │ │ ├── restore-reference.md │ │ │ │ ├── run-locally.md │ │ │ │ ├── start-contributing.md │ │ │ │ ├── support-matrix.md │ │ │ │ ├── troubleshooting.md │ │ │ │ ├── upgrade-to-1.1.md │ │ │ │ ├── vendoring-dependencies.md │ │ │ │ └── zenhub.md │ │ │ ├── v1.10/ │ │ │ │ ├── _index.md │ │ │ │ ├── api-types/ │ │ │ │ │ ├── README.md │ │ │ │ │ ├── _index.md │ │ │ │ │ ├── backup.md │ │ │ │ │ ├── backupstoragelocation.md │ │ │ │ │ ├── restore.md │ │ │ │ │ ├── schedule.md │ │ │ │ │ └── volumesnapshotlocation.md │ │ │ │ ├── backup-hooks.md │ │ │ │ ├── backup-reference.md │ │ │ │ ├── basic-install.md │ │ │ │ ├── build-from-source.md │ │ │ │ ├── code-standards.md │ │ │ │ ├── contributions/ │ │ │ │ │ ├── ibm-config.md │ │ │ │ │ ├── minio.md │ │ │ │ │ ├── oracle-config.md │ │ │ │ │ └── tencent-config.md │ │ │ │ ├── csi.md │ │ │ │ ├── custom-plugins.md │ │ │ │ ├── customize-installation.md │ │ │ │ ├── debugging-install.md │ │ │ │ ├── debugging-restores.md │ │ │ │ ├── development.md │ │ │ │ ├── disaster-case.md │ │ │ │ ├── enable-api-group-versions-feature.md │ │ │ │ ├── examples.md │ │ │ │ ├── file-system-backup.md │ │ │ │ ├── how-velero-works.md │ │ │ │ ├── image-tagging.md │ │ │ │ ├── img/ │ │ │ │ │ └── README.md │ │ │ │ ├── locations.md │ │ │ │ ├── maintainers.md │ │ │ │ ├── manual-testing.md │ │ │ │ ├── migration-case.md │ │ │ │ ├── namespace.md │ │ │ │ ├── on-premises.md │ │ │ │ ├── output-file-format.md │ │ │ │ ├── overview-plugins.md │ │ │ │ ├── performance-guidance.md │ │ │ │ ├── plugin-release-instructions.md │ │ │ │ ├── proxy.md │ │ │ │ ├── rbac.md │ │ │ │ ├── release-instructions.md │ │ │ │ ├── release-schedule.md │ │ │ │ ├── resource-filtering.md │ │ │ │ ├── restore-hooks.md │ │ │ │ ├── restore-reference.md │ │ │ │ ├── run-locally.md │ │ │ │ ├── self-signed-certificates.md │ │ │ │ ├── start-contributing.md │ │ │ │ ├── style-guide.md │ │ │ │ ├── support-process.md │ │ │ │ ├── supported-providers.md │ │ │ │ ├── tilt.md │ │ │ │ ├── troubleshooting.md │ │ │ │ ├── uninstalling.md │ │ │ │ ├── upgrade-to-1.10.md │ │ │ │ ├── velero-install.md │ │ │ │ └── website-guidelines.md │ │ │ ├── v1.11/ │ │ │ │ ├── _index.md │ │ │ │ ├── api-types/ │ │ │ │ │ ├── README.md │ │ │ │ │ ├── _index.md │ │ │ │ │ ├── backup.md │ │ │ │ │ ├── backupstoragelocation.md │ │ │ │ │ ├── restore.md │ │ │ │ │ ├── schedule.md │ │ │ │ │ └── volumesnapshotlocation.md │ │ │ │ ├── backup-hooks.md │ │ │ │ ├── backup-reference.md │ │ │ │ ├── basic-install.md │ │ │ │ ├── build-from-source.md │ │ │ │ ├── code-standards.md │ │ │ │ ├── contributions/ │ │ │ │ │ ├── ibm-config.md │ │ │ │ │ ├── minio.md │ │ │ │ │ ├── oracle-config.md │ │ │ │ │ └── tencent-config.md │ │ │ │ ├── csi.md │ │ │ │ ├── custom-plugins.md │ │ │ │ ├── customize-installation.md │ │ │ │ ├── debugging-install.md │ │ │ │ ├── debugging-restores.md │ │ │ │ ├── development.md │ │ │ │ ├── disaster-case.md │ │ │ │ ├── enable-api-group-versions-feature.md │ │ │ │ ├── examples.md │ │ │ │ ├── file-system-backup.md │ │ │ │ ├── how-velero-works.md │ │ │ │ ├── image-tagging.md │ │ │ │ ├── img/ │ │ │ │ │ └── README.md │ │ │ │ ├── locations.md │ │ │ │ ├── maintainers.md │ │ │ │ ├── manual-testing.md │ │ │ │ ├── migration-case.md │ │ │ │ ├── namespace.md │ │ │ │ ├── on-premises.md │ │ │ │ ├── output-file-format.md │ │ │ │ ├── overview-plugins.md │ │ │ │ ├── performance-guidance.md │ │ │ │ ├── plugin-release-instructions.md │ │ │ │ ├── proxy.md │ │ │ │ ├── rbac.md │ │ │ │ ├── release-instructions.md │ │ │ │ ├── release-schedule.md │ │ │ │ ├── resource-filtering.md │ │ │ │ ├── restore-hooks.md │ │ │ │ ├── restore-reference.md │ │ │ │ ├── run-locally.md │ │ │ │ ├── self-signed-certificates.md │ │ │ │ ├── start-contributing.md │ │ │ │ ├── style-guide.md │ │ │ │ ├── support-process.md │ │ │ │ ├── supported-providers.md │ │ │ │ ├── tilt.md │ │ │ │ ├── troubleshooting.md │ │ │ │ ├── uninstalling.md │ │ │ │ ├── upgrade-to-1.11.md │ │ │ │ ├── velero-install.md │ │ │ │ └── website-guidelines.md │ │ │ ├── v1.12/ │ │ │ │ ├── _index.md │ │ │ │ ├── api-types/ │ │ │ │ │ ├── README.md │ │ │ │ │ ├── _index.md │ │ │ │ │ ├── backup.md │ │ │ │ │ ├── backupstoragelocation.md │ │ │ │ │ ├── restore.md │ │ │ │ │ ├── schedule.md │ │ │ │ │ └── volumesnapshotlocation.md │ │ │ │ ├── backup-hooks.md │ │ │ │ ├── backup-reference.md │ │ │ │ ├── basic-install.md │ │ │ │ ├── build-from-source.md │ │ │ │ ├── code-standards.md │ │ │ │ ├── contributions/ │ │ │ │ │ ├── ibm-config.md │ │ │ │ │ ├── minio.md │ │ │ │ │ ├── oracle-config.md │ │ │ │ │ └── tencent-config.md │ │ │ │ ├── csi-snapshot-data-movement.md │ │ │ │ ├── csi.md │ │ │ │ ├── custom-plugins.md │ │ │ │ ├── customize-installation.md │ │ │ │ ├── debugging-install.md │ │ │ │ ├── debugging-restores.md │ │ │ │ ├── development.md │ │ │ │ ├── disaster-case.md │ │ │ │ ├── enable-api-group-versions-feature.md │ │ │ │ ├── examples.md │ │ │ │ ├── file-system-backup.md │ │ │ │ ├── how-velero-works.md │ │ │ │ ├── image-tagging.md │ │ │ │ ├── img/ │ │ │ │ │ └── README.md │ │ │ │ ├── locations.md │ │ │ │ ├── maintainers.md │ │ │ │ ├── manual-testing.md │ │ │ │ ├── migration-case.md │ │ │ │ ├── namespace.md │ │ │ │ ├── on-premises.md │ │ │ │ ├── output-file-format.md │ │ │ │ ├── overview-plugins.md │ │ │ │ ├── performance-guidance.md │ │ │ │ ├── plugin-release-instructions.md │ │ │ │ ├── proxy.md │ │ │ │ ├── rbac.md │ │ │ │ ├── release-instructions.md │ │ │ │ ├── release-schedule.md │ │ │ │ ├── resource-filtering.md │ │ │ │ ├── restore-hooks.md │ │ │ │ ├── restore-reference.md │ │ │ │ ├── restore-resource-modifiers.md │ │ │ │ ├── run-locally.md │ │ │ │ ├── self-signed-certificates.md │ │ │ │ ├── start-contributing.md │ │ │ │ ├── style-guide.md │ │ │ │ ├── support-process.md │ │ │ │ ├── supported-providers.md │ │ │ │ ├── tilt.md │ │ │ │ ├── troubleshooting.md │ │ │ │ ├── uninstalling.md │ │ │ │ ├── upgrade-to-1.12.md │ │ │ │ ├── velero-install.md │ │ │ │ └── website-guidelines.md │ │ │ ├── v1.13/ │ │ │ │ ├── _index.md │ │ │ │ ├── api-types/ │ │ │ │ │ ├── README.md │ │ │ │ │ ├── _index.md │ │ │ │ │ ├── backup.md │ │ │ │ │ ├── backupstoragelocation.md │ │ │ │ │ ├── restore.md │ │ │ │ │ ├── schedule.md │ │ │ │ │ └── volumesnapshotlocation.md │ │ │ │ ├── backup-hooks.md │ │ │ │ ├── backup-reference.md │ │ │ │ ├── basic-install.md │ │ │ │ ├── build-from-source.md │ │ │ │ ├── code-standards.md │ │ │ │ ├── contributions/ │ │ │ │ │ ├── ibm-config.md │ │ │ │ │ ├── minio.md │ │ │ │ │ ├── oracle-config.md │ │ │ │ │ └── tencent-config.md │ │ │ │ ├── csi-snapshot-data-movement.md │ │ │ │ ├── csi.md │ │ │ │ ├── custom-plugins.md │ │ │ │ ├── customize-installation.md │ │ │ │ ├── debugging-install.md │ │ │ │ ├── debugging-restores.md │ │ │ │ ├── development.md │ │ │ │ ├── disaster-case.md │ │ │ │ ├── enable-api-group-versions-feature.md │ │ │ │ ├── examples.md │ │ │ │ ├── file-system-backup.md │ │ │ │ ├── how-velero-works.md │ │ │ │ ├── image-tagging.md │ │ │ │ ├── img/ │ │ │ │ │ └── README.md │ │ │ │ ├── locations.md │ │ │ │ ├── maintainers.md │ │ │ │ ├── manual-testing.md │ │ │ │ ├── migration-case.md │ │ │ │ ├── namespace.md │ │ │ │ ├── node-agent-concurrency.md │ │ │ │ ├── on-premises.md │ │ │ │ ├── output-file-format.md │ │ │ │ ├── overview-plugins.md │ │ │ │ ├── performance-guidance.md │ │ │ │ ├── plugin-release-instructions.md │ │ │ │ ├── proxy.md │ │ │ │ ├── rbac.md │ │ │ │ ├── release-instructions.md │ │ │ │ ├── release-schedule.md │ │ │ │ ├── resource-filtering.md │ │ │ │ ├── restore-hooks.md │ │ │ │ ├── restore-reference.md │ │ │ │ ├── restore-resource-modifiers.md │ │ │ │ ├── run-locally.md │ │ │ │ ├── self-signed-certificates.md │ │ │ │ ├── start-contributing.md │ │ │ │ ├── style-guide.md │ │ │ │ ├── support-process.md │ │ │ │ ├── supported-providers.md │ │ │ │ ├── tilt.md │ │ │ │ ├── troubleshooting.md │ │ │ │ ├── uninstalling.md │ │ │ │ ├── upgrade-to-1.13.md │ │ │ │ ├── velero-install.md │ │ │ │ └── website-guidelines.md │ │ │ ├── v1.14/ │ │ │ │ ├── _index.md │ │ │ │ ├── api-types/ │ │ │ │ │ ├── README.md │ │ │ │ │ ├── _index.md │ │ │ │ │ ├── backup.md │ │ │ │ │ ├── backupstoragelocation.md │ │ │ │ │ ├── restore.md │ │ │ │ │ ├── schedule.md │ │ │ │ │ └── volumesnapshotlocation.md │ │ │ │ ├── backup-hooks.md │ │ │ │ ├── backup-reference.md │ │ │ │ ├── basic-install.md │ │ │ │ ├── build-from-source.md │ │ │ │ ├── code-standards.md │ │ │ │ ├── contributions/ │ │ │ │ │ ├── ibm-config.md │ │ │ │ │ ├── minio.md │ │ │ │ │ ├── oracle-config.md │ │ │ │ │ └── tencent-config.md │ │ │ │ ├── csi-snapshot-data-movement.md │ │ │ │ ├── csi.md │ │ │ │ ├── custom-plugins.md │ │ │ │ ├── customize-installation.md │ │ │ │ ├── data-movement-backup-node-selection.md │ │ │ │ ├── debugging-install.md │ │ │ │ ├── debugging-restores.md │ │ │ │ ├── development.md │ │ │ │ ├── disaster-case.md │ │ │ │ ├── enable-api-group-versions-feature.md │ │ │ │ ├── examples.md │ │ │ │ ├── file-system-backup.md │ │ │ │ ├── how-velero-works.md │ │ │ │ ├── image-tagging.md │ │ │ │ ├── img/ │ │ │ │ │ └── README.md │ │ │ │ ├── locations.md │ │ │ │ ├── maintainers.md │ │ │ │ ├── manual-testing.md │ │ │ │ ├── migration-case.md │ │ │ │ ├── namespace.md │ │ │ │ ├── node-agent-concurrency.md │ │ │ │ ├── on-premises.md │ │ │ │ ├── output-file-format.md │ │ │ │ ├── overview-plugins.md │ │ │ │ ├── performance-guidance.md │ │ │ │ ├── plugin-release-instructions.md │ │ │ │ ├── proxy.md │ │ │ │ ├── rbac.md │ │ │ │ ├── release-instructions.md │ │ │ │ ├── release-schedule.md │ │ │ │ ├── repository-maintenance.md │ │ │ │ ├── resource-filtering.md │ │ │ │ ├── restore-hooks.md │ │ │ │ ├── restore-reference.md │ │ │ │ ├── restore-resource-modifiers.md │ │ │ │ ├── run-locally.md │ │ │ │ ├── self-signed-certificates.md │ │ │ │ ├── start-contributing.md │ │ │ │ ├── style-guide.md │ │ │ │ ├── support-process.md │ │ │ │ ├── supported-providers.md │ │ │ │ ├── tilt.md │ │ │ │ ├── troubleshooting.md │ │ │ │ ├── uninstalling.md │ │ │ │ ├── upgrade-to-1.14.md │ │ │ │ ├── velero-install.md │ │ │ │ └── website-guidelines.md │ │ │ ├── v1.15/ │ │ │ │ ├── _index.md │ │ │ │ ├── api-types/ │ │ │ │ │ ├── README.md │ │ │ │ │ ├── _index.md │ │ │ │ │ ├── backup.md │ │ │ │ │ ├── backupstoragelocation.md │ │ │ │ │ ├── restore.md │ │ │ │ │ ├── schedule.md │ │ │ │ │ └── volumesnapshotlocation.md │ │ │ │ ├── backup-hooks.md │ │ │ │ ├── backup-reference.md │ │ │ │ ├── backup-repository-configuration.md │ │ │ │ ├── basic-install.md │ │ │ │ ├── build-from-source.md │ │ │ │ ├── code-standards.md │ │ │ │ ├── contributions/ │ │ │ │ │ ├── ibm-config.md │ │ │ │ │ ├── minio.md │ │ │ │ │ ├── oracle-config.md │ │ │ │ │ └── tencent-config.md │ │ │ │ ├── csi-snapshot-data-movement.md │ │ │ │ ├── csi.md │ │ │ │ ├── custom-plugins.md │ │ │ │ ├── customize-installation.md │ │ │ │ ├── data-movement-backup-node-selection.md │ │ │ │ ├── data-movement-backup-pvc-configuration.md │ │ │ │ ├── data-movement-pod-resource-configuration.md │ │ │ │ ├── debugging-install.md │ │ │ │ ├── debugging-restores.md │ │ │ │ ├── development.md │ │ │ │ ├── disaster-case.md │ │ │ │ ├── enable-api-group-versions-feature.md │ │ │ │ ├── examples.md │ │ │ │ ├── file-system-backup.md │ │ │ │ ├── how-velero-works.md │ │ │ │ ├── image-tagging.md │ │ │ │ ├── img/ │ │ │ │ │ └── README.md │ │ │ │ ├── locations.md │ │ │ │ ├── maintainers.md │ │ │ │ ├── manual-testing.md │ │ │ │ ├── migration-case.md │ │ │ │ ├── namespace.md │ │ │ │ ├── node-agent-concurrency.md │ │ │ │ ├── on-premises.md │ │ │ │ ├── output-file-format.md │ │ │ │ ├── overview-plugins.md │ │ │ │ ├── performance-guidance.md │ │ │ │ ├── plugin-release-instructions.md │ │ │ │ ├── proxy.md │ │ │ │ ├── rbac.md │ │ │ │ ├── release-instructions.md │ │ │ │ ├── release-schedule.md │ │ │ │ ├── repository-maintenance.md │ │ │ │ ├── resource-filtering.md │ │ │ │ ├── restore-hooks.md │ │ │ │ ├── restore-reference.md │ │ │ │ ├── restore-resource-modifiers.md │ │ │ │ ├── run-locally.md │ │ │ │ ├── self-signed-certificates.md │ │ │ │ ├── start-contributing.md │ │ │ │ ├── style-guide.md │ │ │ │ ├── support-process.md │ │ │ │ ├── supported-providers.md │ │ │ │ ├── tilt.md │ │ │ │ ├── troubleshooting.md │ │ │ │ ├── uninstalling.md │ │ │ │ ├── upgrade-to-1.15.md │ │ │ │ ├── velero-install.md │ │ │ │ └── website-guidelines.md │ │ │ ├── v1.16/ │ │ │ │ ├── _index.md │ │ │ │ ├── api-types/ │ │ │ │ │ ├── README.md │ │ │ │ │ ├── _index.md │ │ │ │ │ ├── backup.md │ │ │ │ │ ├── backupstoragelocation.md │ │ │ │ │ ├── restore.md │ │ │ │ │ ├── schedule.md │ │ │ │ │ └── volumesnapshotlocation.md │ │ │ │ ├── backup-hooks.md │ │ │ │ ├── backup-reference.md │ │ │ │ ├── backup-repository-configuration.md │ │ │ │ ├── backup-restore-windows.md │ │ │ │ ├── basic-install.md │ │ │ │ ├── build-from-source.md │ │ │ │ ├── code-standards.md │ │ │ │ ├── contributions/ │ │ │ │ │ ├── ibm-config.md │ │ │ │ │ ├── minio.md │ │ │ │ │ ├── oracle-config.md │ │ │ │ │ └── tencent-config.md │ │ │ │ ├── csi-snapshot-data-movement.md │ │ │ │ ├── csi.md │ │ │ │ ├── custom-plugins.md │ │ │ │ ├── customize-installation.md │ │ │ │ ├── data-movement-backup-node-selection.md │ │ │ │ ├── data-movement-backup-pvc-configuration.md │ │ │ │ ├── data-movement-pod-resource-configuration.md │ │ │ │ ├── data-movement-restore-pvc-configuration.md │ │ │ │ ├── debugging-install.md │ │ │ │ ├── debugging-restores.md │ │ │ │ ├── development.md │ │ │ │ ├── disaster-case.md │ │ │ │ ├── enable-api-group-versions-feature.md │ │ │ │ ├── examples.md │ │ │ │ ├── file-system-backup.md │ │ │ │ ├── how-velero-works.md │ │ │ │ ├── image-tagging.md │ │ │ │ ├── img/ │ │ │ │ │ └── README.md │ │ │ │ ├── locations.md │ │ │ │ ├── maintainers.md │ │ │ │ ├── manual-testing.md │ │ │ │ ├── migration-case.md │ │ │ │ ├── namespace.md │ │ │ │ ├── node-agent-concurrency.md │ │ │ │ ├── on-premises.md │ │ │ │ ├── output-file-format.md │ │ │ │ ├── overview-plugins.md │ │ │ │ ├── performance-guidance.md │ │ │ │ ├── plugin-release-instructions.md │ │ │ │ ├── proxy.md │ │ │ │ ├── rbac.md │ │ │ │ ├── release-instructions.md │ │ │ │ ├── release-schedule.md │ │ │ │ ├── repository-maintenance.md │ │ │ │ ├── resource-filtering.md │ │ │ │ ├── restore-hooks.md │ │ │ │ ├── restore-reference.md │ │ │ │ ├── restore-resource-modifiers.md │ │ │ │ ├── run-locally.md │ │ │ │ ├── self-signed-certificates.md │ │ │ │ ├── start-contributing.md │ │ │ │ ├── style-guide.md │ │ │ │ ├── support-process.md │ │ │ │ ├── supported-providers.md │ │ │ │ ├── tilt.md │ │ │ │ ├── troubleshooting.md │ │ │ │ ├── uninstalling.md │ │ │ │ ├── upgrade-to-1.16.md │ │ │ │ ├── velero-install.md │ │ │ │ └── website-guidelines.md │ │ │ ├── v1.17/ │ │ │ │ ├── _index.md │ │ │ │ ├── api-types/ │ │ │ │ │ ├── README.md │ │ │ │ │ ├── _index.md │ │ │ │ │ ├── backup.md │ │ │ │ │ ├── backupstoragelocation.md │ │ │ │ │ ├── restore.md │ │ │ │ │ ├── schedule.md │ │ │ │ │ └── volumesnapshotlocation.md │ │ │ │ ├── backup-hooks.md │ │ │ │ ├── backup-reference.md │ │ │ │ ├── backup-repository-configuration.md │ │ │ │ ├── backup-restore-windows.md │ │ │ │ ├── basic-install.md │ │ │ │ ├── build-from-source.md │ │ │ │ ├── code-standards.md │ │ │ │ ├── contributions/ │ │ │ │ │ ├── ibm-config.md │ │ │ │ │ ├── minio.md │ │ │ │ │ ├── oracle-config.md │ │ │ │ │ └── tencent-config.md │ │ │ │ ├── csi-snapshot-data-movement.md │ │ │ │ ├── csi.md │ │ │ │ ├── custom-plugins.md │ │ │ │ ├── customize-installation.md │ │ │ │ ├── data-movement-backup-pvc-configuration.md │ │ │ │ ├── data-movement-node-selection.md │ │ │ │ ├── data-movement-pod-resource-configuration.md │ │ │ │ ├── data-movement-restore-pvc-configuration.md │ │ │ │ ├── debugging-install.md │ │ │ │ ├── debugging-restores.md │ │ │ │ ├── development.md │ │ │ │ ├── disaster-case.md │ │ │ │ ├── enable-api-group-versions-feature.md │ │ │ │ ├── examples.md │ │ │ │ ├── file-system-backup.md │ │ │ │ ├── how-velero-works.md │ │ │ │ ├── image-tagging.md │ │ │ │ ├── img/ │ │ │ │ │ └── README.md │ │ │ │ ├── locations.md │ │ │ │ ├── maintainers.md │ │ │ │ ├── manual-testing.md │ │ │ │ ├── migration-case.md │ │ │ │ ├── namespace.md │ │ │ │ ├── node-agent-concurrency.md │ │ │ │ ├── node-agent-prepare-queue-length.md │ │ │ │ ├── on-premises.md │ │ │ │ ├── output-file-format.md │ │ │ │ ├── overview-plugins.md │ │ │ │ ├── performance-guidance.md │ │ │ │ ├── plugin-release-instructions.md │ │ │ │ ├── proxy.md │ │ │ │ ├── rbac.md │ │ │ │ ├── release-instructions.md │ │ │ │ ├── release-schedule.md │ │ │ │ ├── repository-maintenance.md │ │ │ │ ├── resource-filtering.md │ │ │ │ ├── restore-hooks.md │ │ │ │ ├── restore-reference.md │ │ │ │ ├── restore-resource-modifiers.md │ │ │ │ ├── run-locally.md │ │ │ │ ├── self-signed-certificates.md │ │ │ │ ├── start-contributing.md │ │ │ │ ├── style-guide.md │ │ │ │ ├── support-process.md │ │ │ │ ├── supported-providers.md │ │ │ │ ├── tilt.md │ │ │ │ ├── troubleshooting.md │ │ │ │ ├── uninstalling.md │ │ │ │ ├── upgrade-to-1.17.md │ │ │ │ ├── velero-install.md │ │ │ │ ├── volume-group-snapshots.md │ │ │ │ └── website-guidelines.md │ │ │ ├── v1.18/ │ │ │ │ ├── _index.md │ │ │ │ ├── api-types/ │ │ │ │ │ ├── README.md │ │ │ │ │ ├── _index.md │ │ │ │ │ ├── backup.md │ │ │ │ │ ├── backupstoragelocation.md │ │ │ │ │ ├── restore.md │ │ │ │ │ ├── schedule.md │ │ │ │ │ └── volumesnapshotlocation.md │ │ │ │ ├── backup-hooks.md │ │ │ │ ├── backup-reference.md │ │ │ │ ├── backup-repository-configuration.md │ │ │ │ ├── backup-restore-windows.md │ │ │ │ ├── basic-install.md │ │ │ │ ├── build-from-source.md │ │ │ │ ├── code-standards.md │ │ │ │ ├── contributions/ │ │ │ │ │ ├── ibm-config.md │ │ │ │ │ ├── minio.md │ │ │ │ │ ├── oracle-config.md │ │ │ │ │ └── tencent-config.md │ │ │ │ ├── csi-snapshot-data-movement.md │ │ │ │ ├── csi.md │ │ │ │ ├── custom-plugins.md │ │ │ │ ├── customize-installation.md │ │ │ │ ├── data-movement-backup-pvc-configuration.md │ │ │ │ ├── data-movement-cache-volume.md │ │ │ │ ├── data-movement-node-selection.md │ │ │ │ ├── data-movement-pod-resource-configuration.md │ │ │ │ ├── data-movement-restore-pvc-configuration.md │ │ │ │ ├── debugging-install.md │ │ │ │ ├── debugging-restores.md │ │ │ │ ├── development.md │ │ │ │ ├── disaster-case.md │ │ │ │ ├── enable-api-group-versions-feature.md │ │ │ │ ├── examples.md │ │ │ │ ├── file-system-backup.md │ │ │ │ ├── how-velero-works.md │ │ │ │ ├── image-tagging.md │ │ │ │ ├── img/ │ │ │ │ │ └── README.md │ │ │ │ ├── locations.md │ │ │ │ ├── maintainers.md │ │ │ │ ├── manual-testing.md │ │ │ │ ├── migration-case.md │ │ │ │ ├── namespace-glob-patterns.md │ │ │ │ ├── namespace.md │ │ │ │ ├── node-agent-concurrency.md │ │ │ │ ├── node-agent-prepare-queue-length.md │ │ │ │ ├── on-premises.md │ │ │ │ ├── output-file-format.md │ │ │ │ ├── overview-plugins.md │ │ │ │ ├── performance-guidance.md │ │ │ │ ├── plugin-release-instructions.md │ │ │ │ ├── proxy.md │ │ │ │ ├── rbac.md │ │ │ │ ├── release-instructions.md │ │ │ │ ├── release-schedule.md │ │ │ │ ├── repository-maintenance.md │ │ │ │ ├── resource-filtering.md │ │ │ │ ├── restore-hooks.md │ │ │ │ ├── restore-reference.md │ │ │ │ ├── restore-resource-modifiers.md │ │ │ │ ├── run-locally.md │ │ │ │ ├── self-signed-certificates.md │ │ │ │ ├── start-contributing.md │ │ │ │ ├── style-guide.md │ │ │ │ ├── support-process.md │ │ │ │ ├── supported-configmaps/ │ │ │ │ │ ├── _index.md │ │ │ │ │ └── node-agent-configmap.md │ │ │ │ ├── supported-providers.md │ │ │ │ ├── tilt.md │ │ │ │ ├── troubleshooting.md │ │ │ │ ├── uninstalling.md │ │ │ │ ├── upgrade-to-1.18.md │ │ │ │ ├── velero-install.md │ │ │ │ ├── volume-group-snapshots.md │ │ │ │ └── website-guidelines.md │ │ │ ├── v1.2.0/ │ │ │ │ ├── _index.md │ │ │ │ ├── api-types/ │ │ │ │ │ ├── README.md │ │ │ │ │ ├── backup.md │ │ │ │ │ ├── backupstoragelocation.md │ │ │ │ │ ├── schedule.md │ │ │ │ │ └── volumesnapshotlocation.md │ │ │ │ ├── backup-reference.md │ │ │ │ ├── basic-install.md │ │ │ │ ├── build-from-source.md │ │ │ │ ├── code-standards.md │ │ │ │ ├── contributions/ │ │ │ │ │ ├── ibm-config.md │ │ │ │ │ ├── minio.md │ │ │ │ │ └── oracle-config.md │ │ │ │ ├── custom-plugins.md │ │ │ │ ├── customize-installation.md │ │ │ │ ├── debugging-install.md │ │ │ │ ├── debugging-restores.md │ │ │ │ ├── development.md │ │ │ │ ├── disaster-case.md │ │ │ │ ├── examples.md │ │ │ │ ├── faq.md │ │ │ │ ├── hooks.md │ │ │ │ ├── how-velero-works.md │ │ │ │ ├── image-tagging.md │ │ │ │ ├── img/ │ │ │ │ │ └── README.md │ │ │ │ ├── locations.md │ │ │ │ ├── migration-case.md │ │ │ │ ├── namespace.md │ │ │ │ ├── on-premises.md │ │ │ │ ├── output-file-format.md │ │ │ │ ├── overview-plugins.md │ │ │ │ ├── rbac.md │ │ │ │ ├── restic.md │ │ │ │ ├── restore-reference.md │ │ │ │ ├── run-locally.md │ │ │ │ ├── start-contributing.md │ │ │ │ ├── support-process.md │ │ │ │ ├── supported-providers.md │ │ │ │ ├── troubleshooting.md │ │ │ │ ├── uninstalling.md │ │ │ │ ├── upgrade-to-1.2.md │ │ │ │ ├── velero-install.md │ │ │ │ ├── vendoring-dependencies.md │ │ │ │ ├── website-guidelines.md │ │ │ │ └── zenhub.md │ │ │ ├── v1.3.0/ │ │ │ │ ├── _index.md │ │ │ │ ├── api-types/ │ │ │ │ │ ├── README.md │ │ │ │ │ ├── backup.md │ │ │ │ │ ├── backupstoragelocation.md │ │ │ │ │ ├── restore.md │ │ │ │ │ ├── schedule.md │ │ │ │ │ └── volumesnapshotlocation.md │ │ │ │ ├── backup-reference.md │ │ │ │ ├── basic-install.md │ │ │ │ ├── build-from-source.md │ │ │ │ ├── code-standards.md │ │ │ │ ├── contributions/ │ │ │ │ │ ├── ibm-config.md │ │ │ │ │ ├── minio.md │ │ │ │ │ └── oracle-config.md │ │ │ │ ├── csi.md │ │ │ │ ├── custom-plugins.md │ │ │ │ ├── customize-installation.md │ │ │ │ ├── debugging-install.md │ │ │ │ ├── debugging-restores.md │ │ │ │ ├── development.md │ │ │ │ ├── disaster-case.md │ │ │ │ ├── examples.md │ │ │ │ ├── faq.md │ │ │ │ ├── hooks.md │ │ │ │ ├── how-velero-works.md │ │ │ │ ├── image-tagging.md │ │ │ │ ├── img/ │ │ │ │ │ └── README.md │ │ │ │ ├── locations.md │ │ │ │ ├── migration-case.md │ │ │ │ ├── namespace.md │ │ │ │ ├── on-premises.md │ │ │ │ ├── output-file-format.md │ │ │ │ ├── overview-plugins.md │ │ │ │ ├── rbac.md │ │ │ │ ├── release-instructions.md │ │ │ │ ├── restic.md │ │ │ │ ├── restore-reference.md │ │ │ │ ├── run-locally.md │ │ │ │ ├── start-contributing.md │ │ │ │ ├── support-process.md │ │ │ │ ├── supported-providers.md │ │ │ │ ├── troubleshooting.md │ │ │ │ ├── uninstalling.md │ │ │ │ ├── upgrade-to-1.3.md │ │ │ │ ├── velero-install.md │ │ │ │ ├── vendoring-dependencies.md │ │ │ │ ├── website-guidelines.md │ │ │ │ └── zenhub.md │ │ │ ├── v1.3.1/ │ │ │ │ ├── _index.md │ │ │ │ ├── api-types/ │ │ │ │ │ ├── README.md │ │ │ │ │ ├── backup.md │ │ │ │ │ ├── backupstoragelocation.md │ │ │ │ │ ├── restore.md │ │ │ │ │ ├── schedule.md │ │ │ │ │ └── volumesnapshotlocation.md │ │ │ │ ├── backup-reference.md │ │ │ │ ├── basic-install.md │ │ │ │ ├── build-from-source.md │ │ │ │ ├── code-standards.md │ │ │ │ ├── contributions/ │ │ │ │ │ ├── ibm-config.md │ │ │ │ │ ├── minio.md │ │ │ │ │ └── oracle-config.md │ │ │ │ ├── csi.md │ │ │ │ ├── custom-plugins.md │ │ │ │ ├── customize-installation.md │ │ │ │ ├── debugging-install.md │ │ │ │ ├── debugging-restores.md │ │ │ │ ├── development.md │ │ │ │ ├── disaster-case.md │ │ │ │ ├── examples.md │ │ │ │ ├── faq.md │ │ │ │ ├── hooks.md │ │ │ │ ├── how-velero-works.md │ │ │ │ ├── image-tagging.md │ │ │ │ ├── img/ │ │ │ │ │ └── README.md │ │ │ │ ├── locations.md │ │ │ │ ├── migration-case.md │ │ │ │ ├── namespace.md │ │ │ │ ├── on-premises.md │ │ │ │ ├── output-file-format.md │ │ │ │ ├── overview-plugins.md │ │ │ │ ├── rbac.md │ │ │ │ ├── release-instructions.md │ │ │ │ ├── restic.md │ │ │ │ ├── restore-reference.md │ │ │ │ ├── run-locally.md │ │ │ │ ├── start-contributing.md │ │ │ │ ├── support-process.md │ │ │ │ ├── supported-providers.md │ │ │ │ ├── troubleshooting.md │ │ │ │ ├── uninstalling.md │ │ │ │ ├── upgrade-to-1.3.md │ │ │ │ ├── velero-install.md │ │ │ │ ├── vendoring-dependencies.md │ │ │ │ ├── website-guidelines.md │ │ │ │ └── zenhub.md │ │ │ ├── v1.3.2/ │ │ │ │ ├── _index.md │ │ │ │ ├── api-types/ │ │ │ │ │ ├── README.md │ │ │ │ │ ├── _index.md │ │ │ │ │ ├── backup.md │ │ │ │ │ ├── backupstoragelocation.md │ │ │ │ │ ├── restore.md │ │ │ │ │ ├── schedule.md │ │ │ │ │ └── volumesnapshotlocation.md │ │ │ │ ├── backup-reference.md │ │ │ │ ├── basic-install.md │ │ │ │ ├── build-from-source.md │ │ │ │ ├── code-standards.md │ │ │ │ ├── contributions/ │ │ │ │ │ ├── ibm-config.md │ │ │ │ │ ├── minio.md │ │ │ │ │ └── oracle-config.md │ │ │ │ ├── csi.md │ │ │ │ ├── custom-plugins.md │ │ │ │ ├── customize-installation.md │ │ │ │ ├── debugging-install.md │ │ │ │ ├── debugging-restores.md │ │ │ │ ├── development.md │ │ │ │ ├── disaster-case.md │ │ │ │ ├── examples.md │ │ │ │ ├── faq.md │ │ │ │ ├── hooks.md │ │ │ │ ├── how-velero-works.md │ │ │ │ ├── image-tagging.md │ │ │ │ ├── img/ │ │ │ │ │ └── README.md │ │ │ │ ├── locations.md │ │ │ │ ├── migration-case.md │ │ │ │ ├── namespace.md │ │ │ │ ├── on-premises.md │ │ │ │ ├── output-file-format.md │ │ │ │ ├── overview-plugins.md │ │ │ │ ├── rbac.md │ │ │ │ ├── release-instructions.md │ │ │ │ ├── restic.md │ │ │ │ ├── restore-reference.md │ │ │ │ ├── run-locally.md │ │ │ │ ├── start-contributing.md │ │ │ │ ├── support-process.md │ │ │ │ ├── supported-providers.md │ │ │ │ ├── troubleshooting.md │ │ │ │ ├── uninstalling.md │ │ │ │ ├── upgrade-to-1.3.md │ │ │ │ ├── velero-install.md │ │ │ │ ├── vendoring-dependencies.md │ │ │ │ ├── website-guidelines.md │ │ │ │ └── zenhub.md │ │ │ ├── v1.4/ │ │ │ │ ├── _index.md │ │ │ │ ├── api-types/ │ │ │ │ │ ├── README.md │ │ │ │ │ ├── _index.md │ │ │ │ │ ├── backup.md │ │ │ │ │ ├── backupstoragelocation.md │ │ │ │ │ ├── restore.md │ │ │ │ │ ├── schedule.md │ │ │ │ │ └── volumesnapshotlocation.md │ │ │ │ ├── backup-reference.md │ │ │ │ ├── basic-install.md │ │ │ │ ├── build-from-source.md │ │ │ │ ├── code-standards.md │ │ │ │ ├── contributions/ │ │ │ │ │ ├── ibm-config.md │ │ │ │ │ ├── minio.md │ │ │ │ │ └── oracle-config.md │ │ │ │ ├── csi.md │ │ │ │ ├── custom-plugins.md │ │ │ │ ├── customize-installation.md │ │ │ │ ├── debugging-install.md │ │ │ │ ├── debugging-restores.md │ │ │ │ ├── development.md │ │ │ │ ├── disaster-case.md │ │ │ │ ├── examples.md │ │ │ │ ├── hooks.md │ │ │ │ ├── how-velero-works.md │ │ │ │ ├── image-tagging.md │ │ │ │ ├── img/ │ │ │ │ │ └── README.md │ │ │ │ ├── locations.md │ │ │ │ ├── migration-case.md │ │ │ │ ├── namespace.md │ │ │ │ ├── on-premises.md │ │ │ │ ├── output-file-format.md │ │ │ │ ├── overview-plugins.md │ │ │ │ ├── rbac.md │ │ │ │ ├── release-instructions.md │ │ │ │ ├── resource-filtering.md │ │ │ │ ├── restic.md │ │ │ │ ├── restore-reference.md │ │ │ │ ├── run-locally.md │ │ │ │ ├── self-signed-certificates.md │ │ │ │ ├── start-contributing.md │ │ │ │ ├── support-process.md │ │ │ │ ├── supported-providers.md │ │ │ │ ├── troubleshooting.md │ │ │ │ ├── uninstalling.md │ │ │ │ ├── upgrade-to-1.4.md │ │ │ │ ├── velero-install.md │ │ │ │ ├── website-guidelines.md │ │ │ │ └── zenhub.md │ │ │ ├── v1.7/ │ │ │ │ ├── _index.md │ │ │ │ ├── api-types/ │ │ │ │ │ ├── README.md │ │ │ │ │ ├── _index.md │ │ │ │ │ ├── backup.md │ │ │ │ │ ├── backupstoragelocation.md │ │ │ │ │ ├── restore.md │ │ │ │ │ ├── schedule.md │ │ │ │ │ └── volumesnapshotlocation.md │ │ │ │ ├── backup-hooks.md │ │ │ │ ├── backup-reference.md │ │ │ │ ├── basic-install.md │ │ │ │ ├── build-from-source.md │ │ │ │ ├── code-standards.md │ │ │ │ ├── contributions/ │ │ │ │ │ ├── ibm-config.md │ │ │ │ │ ├── minio.md │ │ │ │ │ ├── oracle-config.md │ │ │ │ │ └── tencent-config.md │ │ │ │ ├── csi.md │ │ │ │ ├── custom-plugins.md │ │ │ │ ├── customize-installation.md │ │ │ │ ├── debugging-install.md │ │ │ │ ├── debugging-restores.md │ │ │ │ ├── development.md │ │ │ │ ├── disaster-case.md │ │ │ │ ├── enable-api-group-versions-feature.md │ │ │ │ ├── examples.md │ │ │ │ ├── how-velero-works.md │ │ │ │ ├── image-tagging.md │ │ │ │ ├── img/ │ │ │ │ │ └── README.md │ │ │ │ ├── locations.md │ │ │ │ ├── maintainers.md │ │ │ │ ├── manual-testing.md │ │ │ │ ├── migration-case.md │ │ │ │ ├── namespace.md │ │ │ │ ├── on-premises.md │ │ │ │ ├── output-file-format.md │ │ │ │ ├── overview-plugins.md │ │ │ │ ├── plugin-release-instructions.md │ │ │ │ ├── rbac.md │ │ │ │ ├── release-instructions.md │ │ │ │ ├── release-schedule.md │ │ │ │ ├── resource-filtering.md │ │ │ │ ├── restic.md │ │ │ │ ├── restore-hooks.md │ │ │ │ ├── restore-reference.md │ │ │ │ ├── run-locally.md │ │ │ │ ├── self-signed-certificates.md │ │ │ │ ├── start-contributing.md │ │ │ │ ├── style-guide.md │ │ │ │ ├── support-process.md │ │ │ │ ├── supported-providers.md │ │ │ │ ├── tilt.md │ │ │ │ ├── troubleshooting.md │ │ │ │ ├── uninstalling.md │ │ │ │ ├── upgrade-to-1.7.md │ │ │ │ ├── velero-install.md │ │ │ │ └── website-guidelines.md │ │ │ └── v1.9/ │ │ │ ├── _index.md │ │ │ ├── api-types/ │ │ │ │ ├── README.md │ │ │ │ ├── _index.md │ │ │ │ ├── backup.md │ │ │ │ ├── backupstoragelocation.md │ │ │ │ ├── restore.md │ │ │ │ ├── schedule.md │ │ │ │ └── volumesnapshotlocation.md │ │ │ ├── backup-hooks.md │ │ │ ├── backup-reference.md │ │ │ ├── basic-install.md │ │ │ ├── build-from-source.md │ │ │ ├── code-standards.md │ │ │ ├── contributions/ │ │ │ │ ├── ibm-config.md │ │ │ │ ├── minio.md │ │ │ │ ├── oracle-config.md │ │ │ │ └── tencent-config.md │ │ │ ├── csi.md │ │ │ ├── custom-plugins.md │ │ │ ├── customize-installation.md │ │ │ ├── debugging-install.md │ │ │ ├── debugging-restores.md │ │ │ ├── development.md │ │ │ ├── disaster-case.md │ │ │ ├── enable-api-group-versions-feature.md │ │ │ ├── examples.md │ │ │ ├── how-velero-works.md │ │ │ ├── image-tagging.md │ │ │ ├── img/ │ │ │ │ └── README.md │ │ │ ├── locations.md │ │ │ ├── maintainers.md │ │ │ ├── manual-testing.md │ │ │ ├── migration-case.md │ │ │ ├── namespace.md │ │ │ ├── on-premises.md │ │ │ ├── output-file-format.md │ │ │ ├── overview-plugins.md │ │ │ ├── plugin-release-instructions.md │ │ │ ├── rbac.md │ │ │ ├── release-instructions.md │ │ │ ├── release-schedule.md │ │ │ ├── resource-filtering.md │ │ │ ├── restic.md │ │ │ ├── restore-hooks.md │ │ │ ├── restore-reference.md │ │ │ ├── run-locally.md │ │ │ ├── self-signed-certificates.md │ │ │ ├── start-contributing.md │ │ │ ├── style-guide.md │ │ │ ├── support-process.md │ │ │ ├── supported-providers.md │ │ │ ├── tilt.md │ │ │ ├── troubleshooting.md │ │ │ ├── uninstalling.md │ │ │ ├── upgrade-to-1.9.md │ │ │ ├── velero-install.md │ │ │ └── website-guidelines.md │ │ ├── plugins/ │ │ │ ├── _index.md │ │ │ └── list/ │ │ │ ├── 01-amazon-web-services.md │ │ │ ├── 01-google-cloud-platform.md │ │ │ ├── 01-microsoft-azure.md │ │ │ ├── 01-vsphere.md │ │ │ ├── 05-alibaba-cloud.md │ │ │ ├── 05-digitalocean.md │ │ │ ├── 05-hpe-storage.md │ │ │ ├── 05-openebs.md │ │ │ ├── 05-openshift.md │ │ │ ├── 05-portworx.md │ │ │ ├── 05-storj.md │ │ │ ├── 10-container-storage-interface.md │ │ │ ├── 10-openstack.md │ │ │ └── index.md │ │ ├── posts/ │ │ │ ├── 2019-04-09-Velero-is-an-Open-Source-Tool-to-Back-up-and-Migrate-Kubernetes-Clusters.md │ │ │ ├── 2019-05-20-velero-1.0-has-arrived.md │ │ │ ├── 2019-08-22-announcing-velero-1.1.md │ │ │ ├── 2019-10-01-announcing-gh-move.md │ │ │ ├── 2019-10-08-Velero-v1-1-on-vSphere.md │ │ │ ├── 2019-10-10-Velero-v1-1-Stateful-Backup-vSphere.md │ │ │ ├── 2019-11-07-Velero-1.2-Sets-Sail.md │ │ │ ├── 2020-03-02-Velero-1.3-Voyage-Continues.md │ │ │ ├── 2020-05-26-Velero-1.4-Community-Wave.md │ │ │ ├── 2020-05-27-CSI-integration.md │ │ │ ├── 2020-09-16-Velero-1.5-For-And-By-Community.md │ │ │ ├── 2021-04-13-Velero-1.6-Bring-All-Your-Credentials.md │ │ │ ├── 2023-04-26-Velero-1.11.md │ │ │ └── _index.md │ │ └── resources/ │ │ └── _index.md │ ├── data/ │ │ └── docs/ │ │ ├── default.yml │ │ ├── main-toc.yml │ │ ├── toc-mapping.yml │ │ ├── v010-toc.yml │ │ ├── v011-toc.yml │ │ ├── v1-0-0-toc.yml │ │ ├── v1-1-0-toc.yml │ │ ├── v1-10-toc.yml │ │ ├── v1-11-toc.yml │ │ ├── v1-12-toc.yml │ │ ├── v1-13-toc.yml │ │ ├── v1-14-toc.yml │ │ ├── v1-15-toc.yml │ │ ├── v1-16-toc.yml │ │ ├── v1-17-toc.yml │ │ ├── v1-18-toc.yml │ │ ├── v1-2-0-toc.yml │ │ ├── v1-3-0-toc.yml │ │ ├── v1-3-1-toc.yml │ │ ├── v1-3-2-toc.yml │ │ ├── v1-4-toc.yml │ │ ├── v1-5-toc.yml │ │ ├── v1-6-toc.yml │ │ ├── v1-7-toc.yml │ │ ├── v1-8-toc.yml │ │ ├── v1-9-toc.yml │ │ ├── v7-1-toc.yml │ │ ├── v7-toc.yml │ │ ├── v8-1-toc.yml │ │ ├── v8-toc.yml │ │ └── v9-toc.yml │ ├── layouts/ │ │ ├── 404.html │ │ ├── _default/ │ │ │ ├── _markup/ │ │ │ │ ├── render-image.html │ │ │ │ └── render-link.html │ │ │ ├── baseof.html │ │ │ ├── footer.html │ │ │ ├── section.html │ │ │ ├── site-header.html │ │ │ └── tag.html │ │ ├── docs/ │ │ │ ├── docs.html │ │ │ ├── nav.html │ │ │ ├── version-warning.html │ │ │ └── versions.html │ │ ├── index.html │ │ ├── index.redirects │ │ ├── partials/ │ │ │ ├── blog-post-card.html │ │ │ ├── blog-posts.html │ │ │ ├── case-studies-alternating.html │ │ │ ├── contributors.html │ │ │ ├── head-docs.html │ │ │ ├── plugins.html │ │ │ └── related-posts.html │ │ ├── plugins/ │ │ │ └── list.html │ │ ├── posts/ │ │ │ ├── section.html │ │ │ └── single.html │ │ ├── robots.txt │ │ └── shortcodes/ │ │ ├── table.html │ │ └── youtube.html │ └── static/ │ ├── fonts/ │ │ └── Metropolis/ │ │ ├── Open Font License.md │ │ └── README.md │ └── js/ │ ├── jquery.matchHeight.js │ └── scripts.js ├── test/ │ ├── Makefile │ ├── e2e/ │ │ ├── README.md │ │ ├── backup/ │ │ │ └── backup.go │ │ ├── backups/ │ │ │ ├── deletion.go │ │ │ ├── sync_backups.go │ │ │ └── ttl.go │ │ ├── basic/ │ │ │ ├── api-group/ │ │ │ │ ├── enable_api_group_extentions.go │ │ │ │ └── enable_api_group_versions.go │ │ │ ├── backup-volume-info/ │ │ │ │ ├── base.go │ │ │ │ ├── csi_data_mover.go │ │ │ │ ├── csi_snapshot.go │ │ │ │ ├── filesystem_upload.go │ │ │ │ ├── native_snapshot.go │ │ │ │ └── skipped_volumes.go │ │ │ ├── namespace-mapping.go │ │ │ ├── nodeport.go │ │ │ ├── resources-check/ │ │ │ │ ├── namespaces.go │ │ │ │ ├── namespaces_annotation.go │ │ │ │ ├── rbac.go │ │ │ │ └── resources_check.go │ │ │ ├── restore_exec_hooks.go │ │ │ └── storage-class-changing.go │ │ ├── bsl-mgmt/ │ │ │ └── deletion.go │ │ ├── e2e_suite_test.go │ │ ├── migration/ │ │ │ └── migration.go │ │ ├── nodeagentconfig/ │ │ │ └── node-agent-config.go │ │ ├── parallelfilesdownload/ │ │ │ └── parallel_files_download.go │ │ ├── parallelfilesupload/ │ │ │ └── parallel_files_upload.go │ │ ├── privilegesmgmt/ │ │ │ └── ssr.go │ │ ├── pv-backup/ │ │ │ └── pv-backup-filter.go │ │ ├── repomaintenance/ │ │ │ └── repo_maintenance_config.go │ │ ├── resource-filtering/ │ │ │ ├── base.go │ │ │ ├── exclude_label.go │ │ │ ├── exclude_namespaces.go │ │ │ ├── exclude_resources.go │ │ │ ├── include_namespaces.go │ │ │ ├── include_resources.go │ │ │ ├── label_selector.go │ │ │ └── wildcard_namespaces.go │ │ ├── resourcemodifiers/ │ │ │ └── resource_modifiers.go │ │ ├── resourcepolicies/ │ │ │ └── resource_policies.go │ │ ├── scale/ │ │ │ └── multiple_namespaces.go │ │ ├── schedule/ │ │ │ ├── in_progress.go │ │ │ ├── ordered_resources.go │ │ │ └── periodical.go │ │ ├── test/ │ │ │ └── test.go │ │ └── upgrade/ │ │ └── upgrade.go │ ├── perf/ │ │ ├── README.md │ │ ├── backup/ │ │ │ └── backup.go │ │ ├── basic/ │ │ │ └── basic.go │ │ ├── e2e_suite_test.go │ │ ├── metrics/ │ │ │ ├── minio.go │ │ │ ├── monitor.go │ │ │ ├── nfs.go │ │ │ ├── pod.go │ │ │ └── time.go │ │ ├── restore/ │ │ │ └── restore.go │ │ └── test/ │ │ └── test.go │ ├── pkg/ │ │ └── client/ │ │ ├── auth_providers.go │ │ ├── client.go │ │ ├── client_test.go │ │ ├── config.go │ │ ├── config_test.go │ │ ├── dynamic.go │ │ ├── factory.go │ │ └── factory_test.go │ ├── testdata/ │ │ ├── enable_api_group_versions/ │ │ │ ├── README.md │ │ │ ├── case-a-source-v1beta1.yaml │ │ │ ├── case-a-source.yaml │ │ │ ├── case-a-target.yaml │ │ │ ├── case-b-source-manually-added-mutations.yaml │ │ │ ├── case-b-target-manually-added-mutations.yaml │ │ │ ├── case-c-target-manually-added-mutations.yaml │ │ │ ├── case-d-target-manually-added-mutations.yaml │ │ │ ├── music_v1_rockband.yaml │ │ │ ├── music_v1alpha1_rockband.yaml │ │ │ ├── music_v2_rockband.yaml │ │ │ ├── music_v2beta1_rockband.yaml │ │ │ └── music_v2beta2_rockband.yaml │ │ ├── storage-class/ │ │ │ ├── README.md │ │ │ ├── aws-legecy.yaml │ │ │ ├── aws.yaml │ │ │ ├── azure-legacy.yaml │ │ │ ├── azure.yaml │ │ │ ├── gcp-legacy.yaml │ │ │ ├── gcp.yaml │ │ │ ├── kind.yaml │ │ │ ├── vanilla-zfs.yaml │ │ │ ├── vsphere-legacy.yaml │ │ │ └── vsphere.yaml │ │ └── volume-snapshot-class/ │ │ ├── aws.yaml │ │ ├── azure.yaml │ │ ├── gcp.yaml │ │ ├── vanilla-zfs.yaml │ │ └── vsphere.yaml │ ├── types.go │ └── util/ │ ├── common/ │ │ └── common.go │ ├── csi/ │ │ └── common.go │ ├── eks/ │ │ └── eks.go │ ├── k8s/ │ │ ├── client.go │ │ ├── clusterrolebinding.go │ │ ├── common.go │ │ ├── configmap.go │ │ ├── crd.go │ │ ├── deployment.go │ │ ├── namespace.go │ │ ├── node.go │ │ ├── persistentvolumes.go │ │ ├── pod.go │ │ ├── pvc.go │ │ ├── rbac.go │ │ ├── sc.go │ │ ├── secret.go │ │ ├── service.go │ │ ├── serviceaccount.go │ │ └── statefulset.go │ ├── kibishii/ │ │ ├── kibishii_utils.go │ │ └── kibishii_utils_test.go │ ├── metrics/ │ │ ├── minio.go │ │ ├── nfs.go │ │ └── pod.go │ ├── providers/ │ │ ├── aws_utils.go │ │ ├── azure_utils.go │ │ ├── common.go │ │ └── gcloud_utils.go │ ├── report/ │ │ └── report.go │ └── velero/ │ ├── install.go │ ├── install_test.go │ ├── velero_utils.go │ └── velero_utils_test.go ├── third_party/ │ └── kubernetes/ │ └── pkg/ │ └── kubectl/ │ └── cmd/ │ ├── completion.go │ └── util/ │ └── shortcut_expander.go └── tilt-resources/ ├── examples/ │ ├── cloud │ ├── deployment.yaml │ ├── node-agent.yaml │ ├── tilt-settings.json │ └── velero_v1_backupstoragelocation.yaml └── kustomization.yaml ================================================ FILE CONTENTS ================================================ ================================================ FILE: .dockerignore ================================================ .go/ .go.std/ site/ ================================================ FILE: .github/ISSUE_TEMPLATE/bug_report.md ================================================ --- name: Bug report about: Tell us about a problem you are experiencing --- **What steps did you take and what happened:** **What did you expect to happen:** **The following information will help us better understand what's going on**: _If you are using velero v1.7.0+:_ Please use `velero debug --backup --restore ` to generate the support bundle, and attach to this issue, more options please refer to `velero debug --help` _If you are using earlier versions:_ Please provide the output of the following commands (Pasting long output into a [GitHub gist](https://gist.github.com) or other pastebin is fine.) - `kubectl logs deployment/velero -n velero` - `velero backup describe ` or `kubectl get backup/ -n velero -o yaml` - `velero backup logs ` - `velero restore describe ` or `kubectl get restore/ -n velero -o yaml` - `velero restore logs ` **Anything else you would like to add:** **Environment:** - Velero version (use `velero version`): - Velero features (use `velero client config get features`): - Kubernetes version (use `kubectl version`): - Kubernetes installer & version: - Cloud provider or hardware configuration: - OS (e.g. from `/etc/os-release`): **Vote on this issue!** This is an invitation to the Velero community to vote on issues, you can see the project's [top voted issues listed here](https://github.com/vmware-tanzu/velero/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc). Use the "reaction smiley face" up to the right of this comment to vote. - :+1: for "I would like to see this bug fixed as soon as possible" - :-1: for "There are more important bugs to focus on right now" ================================================ FILE: .github/ISSUE_TEMPLATE/config.yml ================================================ blank_issues_enabled: false contact_links: - name: Velero Q&A url: https://github.com/vmware-tanzu/velero/discussions/categories/community-support-q-a about: Have questions about Velero? Please ask them here. ================================================ FILE: .github/ISSUE_TEMPLATE/feature-enhancement-request.md ================================================ --- name: Feature enhancement request about: Suggest an idea for this project --- **Describe the problem/challenge you have** **Describe the solution you'd like** **Anything else you would like to add:** **Environment:** - Velero version (use `velero version`): - Kubernetes version (use `kubectl version`): - Kubernetes installer & version: - Cloud provider or hardware configuration: - OS (e.g. from `/etc/os-release`): **Vote on this issue!** This is an invitation to the Velero community to vote on issues, you can see the project's [top voted issues listed here](https://github.com/vmware-tanzu/velero/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc). Use the "reaction smiley face" up to the right of this comment to vote. - :+1: for "The project would be better with this feature added" - :-1: for "This feature will not enhance the project in a meaningful way" ================================================ FILE: .github/auto-assignees.yml ================================================ --- # This assigns a PR to its author addAssignees: author reviewers: # The default reviewers defaults: - maintainers groups: maintainers: - sseago - reasonerjt - ywk253100 - blackpiglet - shubham-pampattiwar - Lyndon-Li - anshulahuja98 - kaovilai tech-writer: - sseago - reasonerjt - ywk253100 - Lyndon-Li files: 'site/**': - tech-writer '**/*.md': - tech-writer # Technical design requests are ".md" files but should # be reviewed by maintainers '/design/**': - maintainers options: ignore_draft: true ignored_keywords: - WIP - wip - DO NOT MERGE enable_group_assignment: true number_of_reviewers: 2 ================================================ FILE: .github/dependabot.yml ================================================ version: 2 updates: # Dependencies listed in .github/workflows - package-ecosystem: "github-actions" directory: "/" schedule: interval: "weekly" labels: - "Dependencies" - "github_actions" - "kind/changelog-not-required" # Dependencies listed in go.mod - package-ecosystem: "gomod" directory: "/" # Location of package manifests schedule: interval: "weekly" labels: - "kind/changelog-not-required" ignore: - dependency-name: "*" update-types: ["version-update:semver-major", "version-update:semver-minor", "version-update:semver-patch"] ================================================ FILE: .github/labeler.yml ================================================ # This file is used by Auto Label PRs action. # Works with https://github.com/actions/labeler/ # Below this line, the keys are labels to be applied, and the values are the file globs to match against. # Anything in the `design` directory gets the `Design` label. Area/Design: - changed-files: - any-glob-to-any-file: design/* # Anything that has plugin infra will be labeled. # Individual plugins don't necessarily live here, though Area/Plugins: - changed-files: - any-glob-to-any-file: pkg/plugins/**/* Dependencies: - changed-files: - any-glob-to-any-file: go.mod Documentation: - changed-files: - any-glob-to-any-file: site/content/docs/**/* # Anything in the site directory gets the website label *EXCEPT* docs Website: - all: - changed-files: - any-glob-to-any-file: site/**/* - all-globs-to-all-files: '!site/content/docs/**/*' has-changelog: - changed-files: - any-glob-to-any-file: changelogs/** has-e2e-2tests: - changed-files: - any-glob-to-any-file: test/e2e/**/* has-unit-tests: - changed-files: - any-glob-to-any-file: pkg/**/*_test.go ================================================ FILE: .github/labels.yaml ================================================ # This file is used by [prow github action](https://github.com/jpmcb/prow-github-actions/) in .github/workflows/prow-action.yml. # This file only has values for kind and area commands. area: - CLI - CSI - Cloud/AWS - Cloud/Azure - Cloud/DigitalOcean - Cloud/GCP - Cloud/vSphere - Design - Documentation - Filters - Plugins - Process - Storage/Minio - Storage/Cinder - WindowsSupport - datamover - fs-backup - fs-backup/deletion - fs-backup/file-selectable - fs-uploader - kopia-integration - migration - multi-tenancy - progress-monitoring - resilience - schedule - storage/IBM-ObjectStorage - upgrade - volume-snapshot-dm kind: - changelog-not-required - question - refactor - requirement - release-note - release-blocker - spike - tech-debt - usage-error - voting ================================================ FILE: .github/pull_request_template.md ================================================ Thank you for contributing to Velero! # Please add a summary of your change # Does your change fix a particular issue? Fixes #(issue) # Please indicate you've done the following: - [ ] [Accepted the DCO](https://velero.io/docs/v1.5/code-standards/#dco-sign-off). Commits without the DCO will delay acceptance. - [ ] [Created a changelog file (`make new-changelog`)](https://velero.io/docs/main/code-standards/#adding-a-changelog) or comment `/kind changelog-not-required` on this PR. - [ ] Updated the corresponding documentation in `site/content/docs/main`. ================================================ FILE: .github/workflows/auto_assign_prs.yml ================================================ --- name: "Auto Assign Author" # pull_request_target means that this will run on pull requests, but in # the context of the base repo. This should mean PRs from forks are supported. on: pull_request_target: types: [opened, reopened, ready_for_review] jobs: # Automatically assigns reviewers and owner add-reviews: runs-on: ubuntu-latest steps: - name: Set the author of a PR as the assignee uses: kentaro-m/auto-assign-action@v2.0.0 with: configuration-path: ".github/auto-assignees.yml" repo-token: "${{ secrets.GITHUB_TOKEN }}" ================================================ FILE: .github/workflows/auto_label_prs.yml ================================================ name: "Auto Label PRs" # pull_request_target means that this will run on pull requests, but in the context of the base repo. # This should mean PRs from forks are supported. # Because it includes the `synchronize` parameter, any push of a new commit to the HEAD ref of a pull request # will trigger this process. on: pull_request_target: types: [opened, reopened, synchronize, ready_for_review] jobs: # Automatically labels PRs based on file globs in the change. triage: runs-on: ubuntu-latest steps: - uses: actions/labeler@v5 with: repo-token: "${{ secrets.GITHUB_TOKEN }}" configuration-path: .github/labeler.yml ================================================ FILE: .github/workflows/auto_request_review.yml ================================================ --- name: "Auto Request Review" on: pull_request_target: types: [opened, ready_for_review, reopened] jobs: auto-request-review: name: Auto Request Review runs-on: ubuntu-latest steps: - name: Request a PR review based on files types/paths, and/or groups the author belongs to uses: necojackarc/auto-request-review@v0.13.0 with: token: ${{ secrets.GITHUB_TOKEN }} config: .github/auto-assignees.yml ================================================ FILE: .github/workflows/e2e-test-kind.yaml ================================================ name: "Run the E2E test on kind" on: push: pull_request: # Do not run when the change only includes these directories. paths-ignore: - "site/**" - "design/**" - "**/*.md" jobs: get-go-version: uses: ./.github/workflows/get-go-version.yaml with: ref: ${{ github.event.pull_request.base.ref }} # Build the Velero CLI and image once for all Kubernetes versions, and cache it so the fan-out workers can get it. build: runs-on: ubuntu-latest needs: get-go-version outputs: minio-dockerfile-sha: ${{ steps.minio-version.outputs.dockerfile_sha }} steps: - name: Check out the code uses: actions/checkout@v6 - name: Set up Go version uses: actions/setup-go@v6 with: go-version: ${{ needs.get-go-version.outputs.version }} # Look for a CLI that's made for this PR - name: Fetch built CLI id: cli-cache uses: actions/cache@v4 with: path: ./_output/bin/linux/amd64/velero # The cache key a combination of the current PR number and the commit SHA key: velero-cli-${{ github.event.pull_request.number }}-${{ github.sha }} - name: Fetch built image id: image-cache uses: actions/cache@v4 with: path: ./velero.tar # The cache key a combination of the current PR number and the commit SHA key: velero-image-${{ github.event.pull_request.number }}-${{ github.sha }} # If no binaries were built for this PR, build it now. - name: Build Velero CLI if: steps.cli-cache.outputs.cache-hit != 'true' run: | make local # If no image were built for this PR, build it now. - name: Build Velero Image if: steps.image-cache.outputs.cache-hit != 'true' run: | IMAGE=velero VERSION=pr-test BUILD_OUTPUT_TYPE=docker make container docker save velero:pr-test-linux-amd64 -o ./velero.tar # Check and build MinIO image once for all e2e tests - name: Check Bitnami MinIO Dockerfile version id: minio-version run: | DOCKERFILE_SHA=$(curl -s https://api.github.com/repos/bitnami/containers/commits?path=bitnami/minio/2025/debian-12/Dockerfile\&per_page=1 | jq -r '.[0].sha') echo "dockerfile_sha=${DOCKERFILE_SHA}" >> $GITHUB_OUTPUT - name: Cache MinIO Image uses: actions/cache@v4 id: minio-cache with: path: ./minio-image.tar key: minio-bitnami-${{ steps.minio-version.outputs.dockerfile_sha }} - name: Build MinIO Image from Bitnami Dockerfile if: steps.minio-cache.outputs.cache-hit != 'true' run: | echo "Building MinIO image from Bitnami Dockerfile..." git clone --depth 1 https://github.com/bitnami/containers.git /tmp/bitnami-containers cd /tmp/bitnami-containers/bitnami/minio/2025/debian-12 docker build -t bitnami/minio:local . docker save bitnami/minio:local > ${{ github.workspace }}/minio-image.tar # Create json of k8s versions to test # from guide: https://stackoverflow.com/a/65094398/4590470 setup-test-matrix: runs-on: ubuntu-latest env: GH_TOKEN: ${{ github.token }} outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - name: Set k8s versions id: set-matrix # everything excluding older tags. limits needs to be high enough to cover all latest versions # and test labels # grep -E "v[1-9]\.(2[5-9]|[3-9][0-9])" filters for v1.25 to v9.99 # and removes older patches of the same minor version # awk -F. '{if(!a[$1"."$2]++)print $1"."$2"."$NF}' run: | echo "matrix={\ \"k8s\":$(wget -q -O - "https://hub.docker.com/v2/namespaces/kindest/repositories/node/tags?page_size=50" | grep -o '"name": *"[^"]*' | grep -o '[^"]*$' | grep -v -E "alpha|beta" | grep -E "v[1-9]\.(2[5-9]|[3-9][0-9])" | awk -F. '{if(!a[$1"."$2]++)print $1"."$2"."$NF}' | sort -r | sed s/v//g | jq -R -c -s 'split("\n")[:-1]'),\ \"labels\":[\ \"Basic && (ClusterResource || NodePort || StorageClass)\", \ \"ResourceFiltering && !Restic\", \ \"ResourceModifier || (Backups && BackupsSync) || PrivilegesMgmt || OrderedResources\", \ \"(NamespaceMapping && Single && Restic) || (NamespaceMapping && Multiple && Restic)\"\ ]}" >> $GITHUB_OUTPUT # Run E2E test against all Kubernetes versions on kind run-e2e-test: needs: - build - setup-test-matrix - get-go-version runs-on: ubuntu-latest strategy: matrix: ${{fromJson(needs.setup-test-matrix.outputs.matrix)}} fail-fast: false steps: - name: Check out the code uses: actions/checkout@v6 - name: Set up Go version uses: actions/setup-go@v6 with: go-version: ${{ needs.get-go-version.outputs.version }} # Fetch the pre-built MinIO image from the build job - name: Fetch built MinIO Image uses: actions/cache@v4 id: minio-cache with: path: ./minio-image.tar key: minio-bitnami-${{ needs.build.outputs.minio-dockerfile-sha }} - name: Load MinIO Image run: | echo "Loading MinIO image..." docker load < ./minio-image.tar - name: Install MinIO run: | docker run -d --rm -p 9000:9000 -e "MINIO_ROOT_USER=minio" -e "MINIO_ROOT_PASSWORD=minio123" -e "MINIO_DEFAULT_BUCKETS=bucket,additional-bucket" bitnami/minio:local - uses: engineerd/setup-kind@v0.6.2 with: skipClusterLogsExport: true version: "v0.27.0" image: "kindest/node:v${{ matrix.k8s }}" - name: Fetch built CLI id: cli-cache uses: actions/cache@v4 with: path: ./_output/bin/linux/amd64/velero key: velero-cli-${{ github.event.pull_request.number }}-${{ github.sha }} - name: Fetch built Image id: image-cache uses: actions/cache@v4 with: path: ./velero.tar key: velero-image-${{ github.event.pull_request.number }}-${{ github.sha }} - name: Load Velero Image run: kind load image-archive velero.tar - name: Run E2E test run: | cat << EOF > /tmp/credential [default] aws_access_key_id=minio aws_secret_access_key=minio123 EOF # Match kubectl version to k8s server version curl -LO https://dl.k8s.io/release/v${{ matrix.k8s }}/bin/linux/amd64/kubectl sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl git clone https://github.com/vmware-tanzu-experiments/distributed-data-generator.git -b main /tmp/kibishii GOPATH=~/go \ CLOUD_PROVIDER=kind \ OBJECT_STORE_PROVIDER=aws \ BSL_CONFIG=region=minio,s3ForcePathStyle="true",s3Url=http://$(hostname -i):9000 \ CREDS_FILE=/tmp/credential \ BSL_BUCKET=bucket \ ADDITIONAL_OBJECT_STORE_PROVIDER=aws \ ADDITIONAL_BSL_CONFIG=region=minio,s3ForcePathStyle="true",s3Url=http://$(hostname -i):9000 \ ADDITIONAL_CREDS_FILE=/tmp/credential \ ADDITIONAL_BSL_BUCKET=additional-bucket \ VELERO_IMAGE=velero:pr-test-linux-amd64 \ PLUGINS=velero/velero-plugin-for-aws:latest \ GINKGO_LABELS="${{ matrix.labels }}" \ KIBISHII_DIRECTORY=/tmp/kibishii/kubernetes/yaml/ \ make -C test/ run-e2e timeout-minutes: 30 - name: Upload debug bundle if: ${{ failure() }} uses: actions/upload-artifact@v5 with: name: DebugBundle-k8s-${{ matrix.k8s }}-job-${{ strategy.job-index }} path: /home/runner/work/velero/velero/test/e2e/debug-bundle* ================================================ FILE: .github/workflows/get-go-version.yaml ================================================ on: workflow_call: inputs: ref: description: "The target branch's ref" required: true type: string outputs: version: description: "The expected Go version" value: ${{ jobs.extract.outputs.version }} jobs: extract: runs-on: ubuntu-latest outputs: version: ${{ steps.pick-version.outputs.version }} steps: - name: Check out the code uses: actions/checkout@v6 - id: pick-version run: | if [ "${{ inputs.ref }}" == "main" ]; then version=$(grep '^go ' go.mod | awk '{print $2}' | cut -d. -f1-2) else goDirectiveVersion=$(grep '^go ' go.mod | awk '{print $2}') toolChainVersion=$(grep '^toolchain ' go.mod | awk '{print $2}') version=$(printf "%s\n%s\n" "$goDirectiveVersion" "$toolChainVersion" | sort -V | tail -n1) fi echo "version=$version" echo "version=$version" >> $GITHUB_OUTPUT ================================================ FILE: .github/workflows/nightly-trivy-scan.yml ================================================ name: Trivy Nightly Scan on: schedule: - cron: '0 2 * * *' # run at 2 AM UTC jobs: nightly-scan: name: Trivy nightly scan runs-on: ubuntu-latest strategy: fail-fast: false matrix: # maintain the versions of Velero those need security scan versions: [main] # list of images that need scan images: [velero, velero-plugin-for-aws, velero-plugin-for-gcp, velero-plugin-for-microsoft-azure] permissions: security-events: write # for github/codeql-action/upload-sarif to upload SARIF results steps: - name: Checkout code uses: actions/checkout@v6 - name: Run Trivy vulnerability scanner uses: aquasecurity/trivy-action@master with: image-ref: 'docker.io/velero/${{ matrix.images }}:${{ matrix.versions }}' severity: 'CRITICAL,HIGH,MEDIUM' format: 'template' template: '@/contrib/sarif.tpl' output: 'trivy-results.sarif' - name: Upload Trivy scan results to GitHub Security tab uses: github/codeql-action/upload-sarif@v3 with: sarif_file: 'trivy-results.sarif' ================================================ FILE: .github/workflows/pr-changelog-check.yml ================================================ name: Pull Request Changelog Check # by setting `on: [pull_request]`, that means action will be trigger when PR is opened, synchronize, reopened. # Add labeled and unlabeled events too. on: pull_request: types: [opened, synchronize, reopened, labeled, unlabeled] jobs: build: name: Run Changelog Check runs-on: ubuntu-latest steps: - name: Check out the code uses: actions/checkout@v6 - name: Changelog check if: ${{ !(contains(github.event.pull_request.labels.*.name, 'kind/changelog-not-required') || contains(github.event.pull_request.labels.*.name, 'Design') || contains(github.event.pull_request.labels.*.name, 'Website') || contains(github.event.pull_request.labels.*.name, 'Documentation'))}} run: ./hack/changelog-check.sh ================================================ FILE: .github/workflows/pr-ci-check.yml ================================================ name: Pull Request CI Check on: [pull_request] jobs: get-go-version: uses: ./.github/workflows/get-go-version.yaml with: ref: ${{ github.event.pull_request.base.ref }} build: name: Run CI needs: get-go-version runs-on: ubuntu-latest strategy: fail-fast: false steps: - name: Check out the code uses: actions/checkout@v6 - name: Set up Go version uses: actions/setup-go@v6 with: go-version: ${{ needs.get-go-version.outputs.version }} - name: Make ci run: make ci - name: Upload test coverage uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} files: coverage.out verbose: true fail_ci_if_error: true ================================================ FILE: .github/workflows/pr-codespell.yml ================================================ name: Pull Request Codespell Check on: [pull_request] jobs: codespell: name: Run Codespell runs-on: ubuntu-latest steps: - name: Check out the code uses: actions/checkout@v6 - name: Codespell uses: codespell-project/actions-codespell@master with: # ignore the config/.../crd.go file as it's generated binary data that is edited elsewhere. skip: .git,*.png,*.jpg,*.woff,*.ttf,*.gif,*.ico,./config/crd/v1beta1/crds/crds.go,./config/crd/v1/crds/crds.go,./config/crd/v2alpha1/crds/crds.go,./go.sum,./LICENSE ignore_words_list: iam,aks,ist,bridget,ue,shouldnot,atleast,notin,sme,optin,sie check_filenames: true check_hidden: true - name: Velero.io word list check shell: bash {0} run: | IGNORE_COMMENT="Velero.io word list : ignore" FILES_TO_CHECK=$(find . -type f \ ! -path "./.git/*" \ ! -path "./site/content/docs/v*" \ ! -path "./changelogs/CHANGELOG-*" \ ! -path "./.github/workflows/pr-codespell.yml" \ ! -path "./site/static/fonts/Metropolis/Open Font License.md" \ ! -regex '.*\.\(png\|jpg\|woff\|ttf\|gif\|ico\|svg\)' ) function check_word_in_files() { local word=$1 xargs grep -Iinr "$word" <<< "$FILES_TO_CHECK" | \ grep -v "$IGNORE_COMMENT" | \ grep -i --color=always "$word" && \ EXIT_STATUS=1 } function check_word_case_sensitive_in_files() { local word=$1 xargs grep -Inr "$word" <<< "$FILES_TO_CHECK" | \ grep -v "$IGNORE_COMMENT" | \ grep --color=always "$word" && \ EXIT_STATUS=1 } EXIT_STATUS=0 check_word_case_sensitive_in_files ' kubernetes ' check_word_in_files 'on-premise\b' check_word_in_files 'back-up' check_word_in_files 'plug-in' check_word_in_files 'whitelist' check_word_in_files 'blacklist' exit $EXIT_STATUS ================================================ FILE: .github/workflows/pr-containers.yml ================================================ name: build Velero containers on Dockerfile change on: pull_request: branches: - 'main' - 'release-**' paths: - 'Dockerfile' jobs: build: name: Build runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 name: Checkout - name: Set up QEMU id: qemu uses: docker/setup-qemu-action@v3 with: platforms: all - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v3 with: version: latest # Although this action also calls docker-push.sh, it is not triggered # by push, so BRANCH and TAG are empty by default. docker-push.sh will # only build Velero image without pushing. - name: Make Velero container without pushing to registry. if: github.repository == 'vmware-tanzu/velero' run: | ./hack/docker-push.sh ================================================ FILE: .github/workflows/pr-goreleaser.yml ================================================ name: Verify goreleaser change on: pull_request: branches: - 'main' - 'release-**' paths: - '.goreleaser.yml' - 'hack/release-tools/goreleaser.sh' jobs: build: name: Build runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 name: Checkout - name: Verify .goreleaser.yml and try a dryrun release. if: github.repository == 'vmware-tanzu/velero' run: | CHANGELOG=$(ls changelogs | sort -V -r | head -n 1) GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} \ REGISTRY=velero \ RELEASE_NOTES_FILE=changelogs/$CHANGELOG \ PUBLISH=false \ make release ================================================ FILE: .github/workflows/pr-linter-check.yml ================================================ name: Pull Request Linter Check on: pull_request: # Do not run when the change only includes these directories. paths-ignore: - "site/**" - "design/**" - "**/*.md" jobs: get-go-version: uses: ./.github/workflows/get-go-version.yaml with: ref: ${{ github.event.pull_request.base.ref }} build: name: Run Linter Check runs-on: ubuntu-latest needs: get-go-version steps: - name: Check out the code uses: actions/checkout@v6 - name: Set up Go version uses: actions/setup-go@v6 with: go-version: ${{ needs.get-go-version.outputs.version }} - name: Linter check uses: golangci/golangci-lint-action@v9 with: version: v2.5.0 args: --verbose ================================================ FILE: .github/workflows/prow-action.yml ================================================ # Adds support for prow-like commands # Uses .github/labels.yaml to define areas and kinds name: "Prow github actions" on: issue_comment: types: [created] jobs: execute: runs-on: ubuntu-latest steps: - uses: jpmcb/prow-github-actions@v1.1.3 with: # TODO: before allowing the /lgtm command, see if we can block merging if changelog labels are missing. prow-commands: | /approve /area /assign /cc /close /hold /kind /milestone /retitle /remove /reopen /uncc /unassign github-token: "${{ secrets.GITHUB_TOKEN }}" ================================================ FILE: .github/workflows/push-builder.yml ================================================ name: build-image on: push: branches: [ main ] paths: - 'hack/build-image/Dockerfile' jobs: build: name: Build runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 with: # The default value is "1" which fetches only a single commit. If we merge PR without squash or rebase, # there are at least two commits: the first one is the merge commit and the second one is the real commit # contains the changes. # As we use the Dockerfile's commit ID as the tag of the build-image, fetching only 1 commit causes the merge # commit ID to be the tag. # While when running make commands locally, as the local git repository usually contains all commits, the Dockerfile's # commit ID is the second one. This is mismatch with the images in Dockerhub fetch-depth: 2 - name: Build run: make build-image # Only try to publish the container image from the root repo; forks don't have permission to do so and will always get failures. - name: Publish container image if: github.repository == 'vmware-tanzu/velero' run: | docker login -u ${{ secrets.DOCKER_USER }} -p ${{ secrets.DOCKER_PASSWORD }} make push-build-image ================================================ FILE: .github/workflows/push.yml ================================================ name: Main CI on: push: branches: - 'main' - 'release-**' tags: - '*' jobs: get-go-version: uses: ./.github/workflows/get-go-version.yaml with: ref: ${{ github.ref_name }} build: name: Build runs-on: ubuntu-latest needs: get-go-version steps: - name: Check out the code uses: actions/checkout@v6 - name: Set up Go version uses: actions/setup-go@v6 with: go-version: ${{ needs.get-go-version.outputs.version }} - name: Set up QEMU id: qemu uses: docker/setup-qemu-action@v3 with: platforms: all - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v3 with: version: latest - name: Build run: | make local # Clean go cache to ease the build environment storage pressure. go clean -modcache -cache - name: Test run: make test - name: Upload test coverage uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} files: coverage.out verbose: true # Only try to publish the container image from the root repo; forks don't have permission to do so and will always get failures. - name: Publish container image if: github.repository == 'vmware-tanzu/velero' run: | sudo swapoff -a sudo rm -f /mnt/swapfile docker system prune -a --force # Build and push Velero image to docker registry docker login -u ${{ secrets.DOCKER_USER }} -p ${{ secrets.DOCKER_PASSWORD }} ./hack/docker-push.sh ================================================ FILE: .github/workflows/rebase.yml ================================================ on: issue_comment: types: [created] name: Automatic Rebase jobs: rebase: name: Rebase if: github.event.issue.pull_request != '' && contains(github.event.comment.body, '/rebase') runs-on: ubuntu-latest steps: - name: Checkout the latest code uses: actions/checkout@v6 with: fetch-depth: 0 - name: Automatic Rebase uses: cirrus-actions/rebase@1.8 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} ================================================ FILE: .github/workflows/stale-issues.yml ================================================ name: "Close stale issues and PRs" on: schedule: - cron: "30 1 * * *" # Every day at 1:30 UTC jobs: stale: runs-on: ubuntu-latest steps: - uses: actions/stale@v10.1.1 with: repo-token: ${{ secrets.GITHUB_TOKEN }} stale-issue-message: "This issue is stale because it has been open 60 days with no activity. Remove stale label or comment or this will be closed in 14 days. If a Velero team member has requested log or more information, please provide the output of the shared commands." close-issue-message: "This issue was closed because it has been stalled for 14 days with no activity." days-before-issue-stale: 60 days-before-issue-close: 14 stale-issue-label: staled # Disable stale PRs for now; they can remain open. days-before-pr-stale: -1 days-before-pr-close: -1 # Only issues made after Feb 09 2021. start-date: "2021-09-02T00:00:00" exempt-issue-labels: "Epic,Area/CLI,Area/Cloud/AWS,Area/Cloud/Azure,Area/Cloud/GCP,Area/Cloud/vSphere,Area/CSI,Area/Design,Area/Documentation,Area/Plugins,Bug,Enhancement/User,kind/requirement,kind/refactor,kind/tech-debt,limitation,Needs investigation,Needs triage,Needs Product,P0 - Hair on fire,P1 - Important,P2 - Long-term important,P3 - Wouldn't it be nice if...,Product Requirements,Restic - GA,Restic,release-blocker,Security,backlog" ================================================ FILE: .gitignore ================================================ # Compiled Object files, Static and Dynamic libs (Shared Objects) *.o *.a *.so # Folders _obj _test _output # Architecture specific extensions/prefixes *.[568vq] [568vq].out *.cgo1.go *.cgo2.c _cgo_defun.c _cgo_gotypes.go _cgo_export.* _testmain.go *.exe *.test *.prof /velero .idea/ .container-* .vimrc .go .DS_Store .push-* .vscode *.diff # Hugo compiled data site/public site/resources site/.hugo_build.lock .vs # these are files for local Tilt development: _tiltbuild tilt-resources/tilt-settings.json tilt-resources/velero_v1_backupstoragelocation.yaml tilt-resources/deployment.yaml tilt-resources/node-agent.yaml tilt-resources/cloud # test generated files test/e2e/report.xml coverage.out __debug_bin* debug.test* # make lint cache .cache/ # Go telemetry directory created when container sets HOME to working directory # This happens because Makefile uses 'docker run -w /github.com/vmware-tanzu/velero' # and Go's os.UserConfigDir() falls back to $HOME/.config when XDG_CONFIG_HOME is unset .config/ ================================================ FILE: .golangci.yaml ================================================ # This file contains all available configuration options # with their default values. # options for analysis running run: # default concurrency is a available CPU number concurrency: 4 # timeout for analysis, e.g. 30s, 5m, default is 0 timeout: 20m # exit code when at least one issue was found, default is 1 issues-exit-code: 1 # by default isn't set. If set we pass it to "go list -mod={option}". From "go help modules": # If invoked with -mod=readonly, the go command is disallowed from the implicit # automatic updating of go.mod described above. Instead, it fails when any changes # to go.mod are needed. This setting is most useful to check that go.mod does # not need updates, such as in a continuous integration and testing system. # If invoked with -mod=vendor, the go command assumes that the vendor # directory holds the correct copies of dependencies and ignores # the dependency descriptions in go.mod. # modules-download-mode: readonly|release|vendor modules-download-mode: readonly # Allow multiple parallel golangci-lint instances running. # If false (default) - golangci-lint acquires file lock on start. allow-parallel-runners: false # output configuration options output: formats: text: path: stdout # print lines of code with issue, default is true print-issued-lines: true # print linter name in the end of issue text, default is true print-linter-name: true # Show statistics per linter. show-stats: false linters: # all available settings of specific linters settings: depguard: rules: main: deny: # specify an error message to output when a denylisted package is used - pkg: github.com/sirupsen/logrus desc: "logging is allowed only by logutils.Log" dogsled: # checks assignments with too many blank identifiers; default is 2 max-blank-identifiers: 2 dupl: # tokens count to trigger issue, 150 by default threshold: 100 errcheck: # report about not checking of errors in type assertions: `a := b.(MyStruct)`; # default is false: such cases aren't reported by default. check-type-assertions: false # report about assignment of errors to blank identifier: `num, _ := strconv.Atoi(numStr)`; # default is false: such cases aren't reported by default. check-blank: false exhaustive: # indicates that switch statements are to be considered exhaustive if a # 'default' case is present, even if all enum members aren't listed in the # switch default-signifies-exhaustive: false funlen: lines: 60 statements: 40 gocognit: # minimal code complexity to report, 30 by default (but we recommend 10-20) min-complexity: 10 nestif: # minimal complexity of if statements to report, 5 by default min-complexity: 4 goconst: # minimal length of string constant, 3 by default min-len: 3 # minimal occurrences count to trigger, 3 by default min-occurrences: 5 gocritic: # Which checks should be enabled; can't be combined with 'disabled-checks'; # See https://go-critic.github.io/overview#checks-overview # To check which checks are enabled run `GL_DEBUG=gocritic golangci-lint run` # By default list of stable checks is used. settings: # settings passed to gocritic captLocal: # must be valid enabled check name paramsOnly: true gocyclo: # minimal code complexity to report, 30 by default (but we recommend 10-20) min-complexity: 10 godot: # check all top-level comments, not only declarations check-all: false godox: # report any comments starting with keywords, this is useful for TODO or FIXME comments that # might be left in the code accidentally and should be resolved before merging keywords: # default keywords are TODO, BUG, and FIXME, these can be overwritten by this setting - NOTE - OPTIMIZE # marks code that should be optimized before merging - HACK # marks hack-arounds that should be removed before merging gosec: excludes: - G115 govet: # enable or disable analyzers by name enable: - atomicalign enable-all: false disable: - shadow disable-all: false importas: alias: - alias: appsv1api pkg: k8s.io/api/apps/v1 - alias: corev1api pkg: k8s.io/api/core/v1 - alias: rbacv1 pkg: k8s.io/api/rbac/v1 - alias: apierrors pkg: k8s.io/apimachinery/pkg/api/errors - alias: apiextv1 pkg: k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1 - alias: metav1 pkg: k8s.io/apimachinery/pkg/apis/meta/v1 - alias: storagev1api pkg: k8s.io/api/storage/v1 - alias: batchv1api pkg: k8s.io/api/batch/v1 lll: # max line length, lines longer will be reported. Default is 120. # '\t' is counted as 1 character by default, and can be changed with the tab-width option line-length: 120 # tab width in spaces. Default to 1. tab-width: 1 misspell: # Correct spellings using locale preferences for US or UK. # Default is to use a neutral variety of English. # Setting locale to US will correct the British spelling of 'colour' to 'color'. locale: US ignore-rules: - someword nakedret: # make an issue if func has more lines of code than this setting and it has naked returns; default is 30 max-func-lines: 30 prealloc: # XXX: we don't recommend using this linter before doing performance profiling. # For most programs usage of prealloc will be a premature optimization. # Report preallocation suggestions only on simple loops that have no returns/breaks/continues/gotos in them. # True by default. simple: true range-loops: true # Report preallocation suggestions on range loops, true by default for-loops: false # Report preallocation suggestions on for loops, false by default nolintlint: # Enable to ensure that nolint directives are all used. Default is true. allow-unused: false # Exclude following linters from requiring an explanation. Default is []. allow-no-explanation: [] # Enable to require an explanation of nonzero length after each nolint directive. Default is false. require-explanation: true # Enable to require nolint directives to mention the specific linter being suppressed. Default is false. require-specific: true perfsprint: strconcat: false sprintf1: false errorf: false int-conversion: true revive: rules: - name: blank-imports disabled: true - name: context-as-argument disabled: true - name: context-keys-type - name: dot-imports disabled: true - name: early-return disabled: true arguments: - "preserveScope" - name: empty-block disabled: true - name: error-naming disabled: true - name: error-return disabled: true - name: error-strings disabled: true - name: errorf disabled: true - name: increment-decrement - name: indent-error-flow disabled: true - name: range - name: receiver-naming disabled: true - name: redefines-builtin-id disabled: true - name: superfluous-else disabled: true arguments: - "preserveScope" - name: time-naming - name: unexported-return disabled: true - name: unnecessary-stmt - name: unreachable-code - name: unused-parameter disabled: true - name: use-any - name: var-declaration - name: var-naming disabled: true rowserrcheck: packages: - github.com/jmoiron/sqlx staticcheck: checks: - all - -QF1001 # FIXME - -QF1003 # FIXME - -QF1004 # FIXME - -QF1007 # FIXME - -QF1008 # FIXME - -QF1009 # FIXME - -QF1012 # FIXME testifylint: # TODO: enable them all disable: - float-compare - go-require enable-all: true testpackage: # regexp pattern to skip files skip-regexp: (export|internal)_test\.go unparam: # Inspect exported functions, default is false. Set to true if no external program/library imports your code. # XXX: if you enable this setting, unparam will report a lot of false-positives in text editors: # if it's called for subdir of a project it can't find external interfaces. All text editor integrations # with golangci-lint call it on a directory with the changed file. check-exported: false usetesting: os-setenv: false whitespace: multi-if: false # Enforces newlines (or comments) after every multi-line if statement multi-func: false # Enforces newlines (or comments) after every multi-line function signature wsl: # If true append is only allowed to be cuddled if appending value is # matching variables, fields or types on line above. Default is true. strict-append: true # Allow calls and assignments to be cuddled as long as the lines have any # matching variables, fields or types. Default is true. allow-assign-and-call: true # Allow multiline assignments to be cuddled. Default is true. allow-multiline-assign: true # Allow declarations (var) to be cuddled. allow-cuddle-declarations: false # Allow trailing comments in ending of blocks allow-trailing-comment: false # Force newlines in end of case at this limit (0 = never). force-case-trailing-whitespace: 0 # Force cuddling of err checks with err var assignment force-err-cuddling: false # Allow leading comments to be separated with empty lines allow-separated-leading-comment: false default: none enable: - asasalint - asciicheck - bidichk - bodyclose - copyloopvar - dogsled - dupword - durationcheck - errcheck - errchkjson - exptostd - ginkgolinter - goconst - goheader - goprintffuncname - gosec - govet - importas - ineffassign - misspell - nakedret - nilerr - noctx - nolintlint - nosprintfhostport - perfsprint - revive - staticcheck - testifylint - thelper - unconvert - unparam - unused - usestdlibvars - usetesting - whitespace exclusions: # which dirs to skip: issues from them won't be reported; # can use regexp here: generated.*, regexp is applied on full path; # default value is empty list, but default dirs are skipped independently # from this option's value (see skip-dirs-use-default). # "/" will be replaced by current OS file path separator to properly work # on Windows. paths: - pkg/plugin/generated/* - third_party rules: - linters: - staticcheck text: "DefaultVolumesToRestic" # No need to report deprecate for DefaultVolumesToRestic. - path: ".*_test.go$" linters: - errcheck - goconst - gosec - govet - staticcheck - unparam - unused - path: test/ linters: - errcheck - goconst - gosec - nilerr - staticcheck - unparam - unused - path: ".*data_upload_controller_test.go$" linters: - dupword text: "type" - path: ".*config_test.go$" linters: - dupword text: "bucket" generated: lax presets: - comments - common-false-positives - legacy - std-error-handling issues: # Maximum issues count per one linter. Set to 0 to disable. Default is 50. max-issues-per-linter: 0 # Maximum count of issues with the same text. Set to 0 to disable. Default is 3. max-same-issues: 0 # make issues output unique by line, default is true uniq-by-line: true # This file contains all available configuration options # with their default values. formatters: enable: - gofmt - goimports exclusions: generated: lax paths: - pkg/plugin/generated/* - third_party settings: gofmt: # simplify code: gofmt with `-s` option, true by default simplify: true goimports: local-prefixes: - github.com/vmware-tanzu/velero severity: default: error # Default value is empty list. # When a list of severity rules are provided, severity information will be added to lint # issues. Severity rules have the same filtering capability as exclude rules except you # are allowed to specify one matcher per severity rule. # Only affects out formats that support setting severity information. rules: - linters: - dupl severity: info version: "2" ================================================ FILE: .goreleaser.yml ================================================ # Copyright 2018 the Velero contributors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. dist: _output builds: - main: ./cmd/velero/velero.go env: - CGO_ENABLED=0 goos: - linux - darwin - windows goarch: - amd64 - arm - arm64 - ppc64le - s390x ignore: # don't build arm for darwin and arm/arm64 for windows - goos: darwin goarch: arm - goos: darwin goarch: ppc64le - goos: darwin goarch: s390x - goos: windows goarch: arm - goos: windows goarch: arm64 - goos: windows goarch: ppc64le - goos: windows goarch: s390x ldflags: - -X "github.com/vmware-tanzu/velero/pkg/buildinfo.Version={{ .Tag }}" -X "github.com/vmware-tanzu/velero/pkg/buildinfo.GitSHA={{ .FullCommit }}" -X "github.com/vmware-tanzu/velero/pkg/buildinfo.GitTreeState={{ .Env.GIT_TREE_STATE }}" -X "github.com/vmware-tanzu/velero/pkg/buildinfo.ImageRegistry={{ .Env.REGISTRY }}" archives: - name_template: "{{ .ProjectName }}-{{ .Tag }}-{{ .Os }}-{{ .Arch }}" wrap_in_directory: true files: - LICENSE - examples/**/* checksum: name_template: 'CHECKSUM' release: github: owner: vmware-tanzu name: velero draft: true prerelease: auto git: # What should be used to sort tags when gathering the current and previous # tags if there are more than one tag in the same commit. # # Default: `-version:refname` tag_sort: -version:creatordate ================================================ FILE: ADOPTERS.md ================================================ # Velero Adopters If you're using Velero and want to add your organization to this list, [follow these directions][1]! pitsdatarecovery.net bitgo.com      nirmata.com      kyma-project.io      redhat.com      dellemc.com      bugsnag.com      okteto.com      banzaicloud.com      sighup.io      mayadata.io      replicated.com cloudcasa.io azure.com broadcom.com ## Success Stories Below is a list of adopters of Velero in **production environments** that have publicly shared the details of how they use it. **[BitGo][20]** BitGo uses Velero backup and restore capabilities to seamlessly provision and scale fullnode statefulsets on the fly as well as having it serve an integral piece for our Kubernetes disaster-recovery story. **[Bugsnag][30]** We use Velero for managing backups of an internal instance of our on-premise clustered solution. We also recommend our users of [on-premise Bugsnag installations](https://www.bugsnag.com/on-premise) use Velero for [managing their own backups](https://docs.bugsnag.com/on-premise/clustered/backup-restore/). **[Banzai Cloud][60]** [Banzai Cloud Pipeline][61] is a Kubernetes-based microservices platform that integrates services needed for Day-1 and Day-2 operations along with first-class support both for on-prem and hybrid multi-cloud deployments. We use Velero to periodically [backup and restore these clusters in case of disasters][62]. ## Solutions built with Velero Below is a list of solutions where Velero is being used as a component. **[Nirmata][10]** We have integrated our [solution with Velero][11] to provide our customers with out of box backup/DR. **[Kyma][40]** Kyma [integrates with Velero][41] to effortlessly back up and restore Kyma clusters with all its resources. Velero capabilities allow Kyma users to define and run manual and scheduled backups in order to successfully handle a disaster-recovery scenario. **[Red Hat][50]** Red Hat has developed 2 operators for the OpenShift platform: - [Migration Toolkit for Containers][51] (Crane): This operator uses [Velero and Restic][52] to drive the migration of applications between OpenShift clusters. - [OADP (OpenShift API for Data Protection) Operator][53]: This operator sets up and installs Velero on the OpenShift platform, allowing users to backup and restore applications. **[Dell EMC][70]** For Kubernetes environments, [PowerProtect Data Manager][71] leverages the Container Storage Interface (CSI) framework to take snapshots to back up the persistent data or the data that the application creates e.g. databases. [Dell EMC leverages Velero][72] to backup the namespace configuration files (also known as Namespace meta data) for enterprise grade data protection. **[SIGHUP][80]** SIGHUP integrates Velero in its [Fury Kubernetes Distribution][81] providing predefined schedules and configurations to ensure an optimized disaster recovery experience. [Fury Kubernetes Disaster Recovery Module][82] is ready to be deployed into any Kubernetes cluster running anywhere. **[MayaData][90]** MayaData is a large user of Velero as well as a contributor. MayaData offers a Data Agility platform called [OpenEBS Director][91], that helps customers confidently and easily manage stateful workloads in Kubernetes. Velero is one of the core software building block of the OpenEBS Director's [DMaaS or data migration as a service offering][92] used to enable data protection strategies. **[Okteto][93]** Okteto integrates Velero in [Okteto Cloud][94] and [Okteto Enterprise][95] to periodically backup and restore our clusters for disaster recovery. Velero is also a core software building block to provide namespace cloning capabilities, a feature that allows our users cloning staging environments into their personal development namespace for providing production-like development environments. **[Replicated][100]**
Replicated uses the Velero open source project to enable snapshots in [KOTS][101] to backup Kubernetes manifests & persistent volumes. In addition to the default functionality that Velero provides, [KOTS][101] provides a detailed interface in the [Admin Console][102] that can be used to manage the storage destination and schedule, and to perform and monitor the backup and restore process.
**[CloudCasa][103]**
[Catalogic Software][104] integrates Velero with [CloudCasa][103] - A Smart Home in the Cloud for Backups. CloudCasa is a full-featured, scalable, cloud-native solution providing Kubernetes data protection, disaster recovery, and migration as a service. An option to manage existing Velero instances and an enterprise self-hosted option are also available.
**[Microsoft Azure][105]**
[Azure Backup for AKS][106] is an Azure native, Kubernetes aware, Enterprise ready backup for containerized applications deployed on Azure Kubernetes Service (AKS). AKS Backup utilizes Velero to perform backup and restore operations to protect stateful applications in AKS clusters.
**[Broadcom][107]**
[VMware Cloud Foundation][108] (VCF) offers built-in [vSphere Kubernetes Service][109] (VKS), a Kubernetes runtime that includes a CNCF certified Kubernetes distribution, to deploy and manage containerized workloads. VCF empowers platform engineers with native [Kubernetes multi-cluster management][110] capability for managing Kubernetes (K8s) infrastructure at scale. VCF utilizes Velero for Kubernetes data protection enabling platform engineers to back up and restore containerized workloads manifests & persistent volumes, helping to increase the resiliency of stateful applications in VKS cluster. ## Adding your organization to the list of Velero Adopters If you are using Velero and would like to be included in the list of `Velero Adopters`, add an SVG version of your logo to the `site/static/img/adopters` directory in this repo and submit a [pull request][3] with your change. Name the image file something that reflects your company (e.g., if your company is called Acme, name the image acme.png). See this for an example [PR][4]. ### Adding a logo to velero.io If you would like to add your logo to a future `Adopters of Velero` section on [velero.io][2], follow the steps above to add your organization to the list of Velero Adopters. Our community will follow up and publish it to the [velero.io][2] website. [1]: #adding-a-logo-to-veleroio [2]: https://velero.io [3]: https://github.com/vmware-tanzu/velero/pulls [4]: https://github.com/vmware-tanzu/velero/pull/2242 [10]: https://www.nirmata.com/2019/08/14/kubernetes-disaster-recovery-using-velero-and-nirmata/ [11]: https://nirmata.com [20]: https://bitgo.com [30]: https://bugsnag.com [40]: https://kyma-project.io [41]: https://kyma-project.io/docs/components/backup/#overview-overview [50]: https://redhat.com [51]: https://github.com/fusor/mig-operator [52]: https://github.com/fusor/mig-operator/blob/master/docs/usage/2.md [53]: https://github.com/openshift/oadp-operator [60]: https://banzaicloud.com [61]: https://banzaicloud.com/products/pipeline/ [62]: https://banzaicloud.com/blog/vault-backup-velero/ [70]: https://dellemc.com [71]: https://dellemc.com/dataprotection [72]: https://www.dellemc.com/resources/en-us/asset/briefs-handouts/solutions/h18141-dellemc-dpd-kubernetes.pdf [80]: https://sighup.io [81]: https://github.com/sighupio/fury-distribution [82]: https://github.com/sighupio/fury-kubernetes-dr [90]: https://mayadata.io [91]: https://director.mayadata.io/ [92]: https://help.mayadata.io/hc/en-us/articles/360033401591-DMaaS [93]: https://okteto.com [94]: https://cloud.okteto.com [95]: https://okteto.com/enterprise/ [100]: https://www.replicated.com [101]: https://kots.io [102]: https://kots.io/kotsadm/snapshots/overview/ [103]: https://cloudcasa.io/ [104]: https://www.catalogicsoftware.com/ [105]: https://azure.microsoft.com/ [106]: https://learn.microsoft.com/azure/backup/backup-overview [107]: https://www.broadcom.com/ [108]: https://www.vmware.com/products/cloud-infrastructure/vmware-cloud-foundation [109]: https://www.vmware.com/products/cloud-infrastructure/vsphere-kubernetes-service [110]: https://blogs.vmware.com/cloud-foundation/2025/09/29/empowering-platform-engineers-with-native-kubernetes-multi-cluster-management-in-vmware-cloud-foundation/ ================================================ FILE: CHANGELOG.md ================================================ ## Current release: * [CHANGELOG-1.15.md][25] ## Older releases: * [CHANGELOG-1.14.md][24] * [CHANGELOG-1.13.md][23] * [CHANGELOG-1.12.md][22] * [CHANGELOG-1.11.md][21] * [CHANGELOG-1.10.md][20] * [CHANGELOG-1.9.md][19] * [CHANGELOG-1.8.md][18] * [CHANGELOG-1.7.md][17] * [CHANGELOG-1.6.md][16] * [CHANGELOG-1.5.md][15] * [CHANGELOG-1.4.md][14] * [CHANGELOG-1.3.md][13] * [CHANGELOG-1.2.md][12] * [CHANGELOG-1.1.md][11] * [CHANGELOG-1.0.md][10] * [CHANGELOG-0.11.md][9] * [CHANGELOG-0.10.md][8] * [CHANGELOG-0.9.md][7] * [CHANGELOG-0.8.md][6] * [CHANGELOG-0.7.md][5] * [CHANGELOG-0.6.md][4] * [CHANGELOG-0.5.md][3] * [CHANGELOG-0.4.md][2] * [CHANGELOG-0.3.md][1] [25]: https://github.com/vmware-tanzu/velero/blob/main/changelogs/CHANGELOG-1.15.md [24]: https://github.com/vmware-tanzu/velero/blob/main/changelogs/CHANGELOG-1.14.md [23]: https://github.com/vmware-tanzu/velero/blob/main/changelogs/CHANGELOG-1.13.md [22]: https://github.com/vmware-tanzu/velero/blob/main/changelogs/CHANGELOG-1.12.md [21]: https://github.com/vmware-tanzu/velero/blob/main/changelogs/CHANGELOG-1.11.md [20]: https://github.com/vmware-tanzu/velero/blob/main/changelogs/CHANGELOG-1.10.md [19]: https://github.com/vmware-tanzu/velero/blob/main/changelogs/CHANGELOG-1.9.md [18]: https://github.com/vmware-tanzu/velero/blob/main/changelogs/CHANGELOG-1.8.md [17]: https://github.com/vmware-tanzu/velero/blob/main/changelogs/CHANGELOG-1.7.md [16]: https://github.com/vmware-tanzu/velero/blob/main/changelogs/CHANGELOG-1.6.md [15]: https://github.com/vmware-tanzu/velero/blob/main/changelogs/CHANGELOG-1.5.md [14]: https://github.com/vmware-tanzu/velero/blob/main/changelogs/CHANGELOG-1.4.md [13]: https://github.com/vmware-tanzu/velero/blob/main/changelogs/CHANGELOG-1.3.md [12]: https://github.com/vmware-tanzu/velero/blob/main/changelogs/CHANGELOG-1.2.md [11]: https://github.com/vmware-tanzu/velero/blob/main/changelogs/CHANGELOG-1.1.md [10]: https://github.com/vmware-tanzu/velero/blob/main/changelogs/CHANGELOG-1.0.md [9]: https://github.com/vmware-tanzu/velero/blob/main/changelogs/CHANGELOG-0.11.md [8]: https://github.com/vmware-tanzu/velero/blob/main/changelogs/CHANGELOG-0.10.md [7]: https://github.com/vmware-tanzu/velero/blob/main/changelogs/CHANGELOG-0.9.md [6]: https://github.com/vmware-tanzu/velero/blob/main/changelogs/CHANGELOG-0.8.md [5]: https://github.com/vmware-tanzu/velero/blob/main/changelogs/CHANGELOG-0.7.md [4]: https://github.com/vmware-tanzu/velero/blob/main/changelogs/CHANGELOG-0.6.md [3]: https://github.com/vmware-tanzu/velero/blob/main/changelogs/CHANGELOG-0.5.md [2]: https://github.com/vmware-tanzu/velero/blob/main/changelogs/CHANGELOG-0.4.md [1]: https://github.com/vmware-tanzu/velero/blob/main/changelogs/CHANGELOG-0.3.md [0]: https://github.com/vmware-tanzu/velero/blob/main/changelogs/unreleased ================================================ FILE: CODE_OF_CONDUCT.md ================================================ # Contributor Covenant Code of Conduct ## Our Pledge We as members, contributors, and leaders pledge to make participation in the Velero project and our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socioeconomic status, nationality, personal appearance, race, religion, or sexual identity and orientation. We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community. ## Our Standards Examples of behavior that contributes to a positive environment for our community include: * Demonstrating empathy and kindness toward other people * Being respectful of differing opinions, viewpoints, and experiences * Giving and gracefully accepting constructive feedback * Accepting responsibility and apologizing to those affected by our mistakes, and learning from the experience * Focusing on what is best not just for us as individuals, but for the overall community Examples of unacceptable behavior include: * The use of sexualized language or imagery, and sexual attention or advances of any kind * Trolling, insulting or derogatory comments, and personal or political attacks * Public or private harassment * Publishing others' private information, such as a physical or email address, without their explicit permission * Other conduct which could reasonably be considered inappropriate in a professional setting ## Enforcement Responsibilities Community leaders are responsible for clarifying and enforcing our standards of acceptable behavior and will take appropriate and fair corrective action in response to any behavior that they deem inappropriate, threatening, offensive, or harmful. Community leaders have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, and will communicate reasons for moderation decisions when appropriate. ## Scope This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing the community in public spaces. Examples of representing our community include using an official e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. ## Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement at oss-coc@vmware.com. All complaints will be reviewed and investigated promptly and fairly. All community leaders are obligated to respect the privacy and security of the reporter of any incident. ## Enforcement Guidelines Community leaders will follow these Community Impact Guidelines in determining the consequences for any action they deem in violation of this Code of Conduct: ### 1. Correction **Community Impact**: Use of inappropriate language or other behavior deemed unprofessional or unwelcome in the community. **Consequence**: A private, written warning from community leaders, providing clarity around the nature of the violation and an explanation of why the behavior was inappropriate. A public apology may be requested. ### 2. Warning **Community Impact**: A violation through a single incident or series of actions. **Consequence**: A warning with consequences for continued behavior. No interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, for a specified period of time. This includes avoiding interactions in community spaces as well as external channels like social media. Violating these terms may lead to a temporary or permanent ban. ### 3. Temporary Ban **Community Impact**: A serious violation of community standards, including sustained inappropriate behavior. **Consequence**: A temporary ban from any sort of interaction or public communication with the community for a specified period of time. No public or private interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, is allowed during this period. Violating these terms may lead to a permanent ban. ### 4. Permanent Ban **Community Impact**: Demonstrating a pattern of violation of community standards, including sustained inappropriate behavior, harassment of an individual, or aggression toward or disparagement of classes of individuals. **Consequence**: A permanent ban from any sort of public interaction within the community. ## Attribution This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 2.0, available at https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. Community Impact Guidelines were inspired by [Mozilla's code of conduct enforcement ladder](https://github.com/mozilla/diversity). [homepage]: https://www.contributor-covenant.org For answers to common questions about this code of conduct, see the FAQ at https://www.contributor-covenant.org/faq. Translations are available at https://www.contributor-covenant.org/translations. ================================================ FILE: CONTRIBUTING.md ================================================ # Contributing Authors are expected to follow some guidelines when submitting PRs. Please see [our documentation](https://velero.io/docs/main/code-standards/) for details. ================================================ FILE: Dockerfile ================================================ # Copyright 2020 the Velero contributors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Velero binary build section FROM --platform=$BUILDPLATFORM golang:1.25-bookworm AS velero-builder ARG GOPROXY ARG BIN ARG PKG ARG VERSION ARG REGISTRY ARG GIT_SHA ARG GIT_TREE_STATE ARG TARGETOS ARG TARGETARCH ARG TARGETVARIANT ENV CGO_ENABLED=0 \ GO111MODULE=on \ GOPROXY=${GOPROXY} \ GOOS=${TARGETOS} \ GOARCH=${TARGETARCH} \ GOARM=${TARGETVARIANT} \ LDFLAGS="-X ${PKG}/pkg/buildinfo.Version=${VERSION} -X ${PKG}/pkg/buildinfo.GitSHA=${GIT_SHA} -X ${PKG}/pkg/buildinfo.GitTreeState=${GIT_TREE_STATE} -X ${PKG}/pkg/buildinfo.ImageRegistry=${REGISTRY}" WORKDIR /go/src/github.com/vmware-tanzu/velero COPY . /go/src/github.com/vmware-tanzu/velero RUN mkdir -p /output/usr/bin && \ export GOARM=$( echo "${GOARM}" | cut -c2-) && \ go build -o /output/${BIN} \ -ldflags "${LDFLAGS}" ${PKG}/cmd/${BIN} && \ go build -o /output/velero-restore-helper \ -ldflags "${LDFLAGS}" ${PKG}/cmd/velero-restore-helper && \ go build -o /output/velero-helper \ -ldflags "${LDFLAGS}" ${PKG}/cmd/velero-helper && \ go clean -modcache -cache # Restic binary build section FROM --platform=$BUILDPLATFORM golang:1.25-bookworm AS restic-builder ARG GOPROXY ARG BIN ARG TARGETOS ARG TARGETARCH ARG TARGETVARIANT ARG RESTIC_VERSION ENV CGO_ENABLED=0 \ GO111MODULE=on \ GOPROXY=${GOPROXY} \ GOOS=${TARGETOS} \ GOARCH=${TARGETARCH} \ GOARM=${TARGETVARIANT} COPY . /go/src/github.com/vmware-tanzu/velero RUN mkdir -p /output/usr/bin && \ export GOARM=$(echo "${GOARM}" | cut -c2-) && \ /go/src/github.com/vmware-tanzu/velero/hack/build-restic.sh && \ go clean -modcache -cache # Velero image packing section FROM paketobuildpacks/run-jammy-tiny:latest LABEL maintainer="Xun Jiang " COPY --from=velero-builder /output / COPY --from=restic-builder /output / USER cnb:cnb ================================================ FILE: Dockerfile-Windows ================================================ # Copyright the Velero contributors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ARG OS_VERSION=1809 # Velero binary build section FROM --platform=$BUILDPLATFORM golang:1.25-bookworm AS velero-builder ARG GOPROXY ARG BIN ARG PKG ARG VERSION ARG REGISTRY ARG GIT_SHA ARG GIT_TREE_STATE ARG TARGETOS ARG TARGETARCH ARG TARGETVARIANT ENV CGO_ENABLED=0 \ GO111MODULE=on \ GOPROXY=${GOPROXY} \ GOOS=${TARGETOS} \ GOARCH=${TARGETARCH} \ GOARM=${TARGETVARIANT} \ LDFLAGS="-X ${PKG}/pkg/buildinfo.Version=${VERSION} -X ${PKG}/pkg/buildinfo.GitSHA=${GIT_SHA} -X ${PKG}/pkg/buildinfo.GitTreeState=${GIT_TREE_STATE} -X ${PKG}/pkg/buildinfo.ImageRegistry=${REGISTRY}" WORKDIR /go/src/github.com/vmware-tanzu/velero COPY . /go/src/github.com/vmware-tanzu/velero RUN mkdir -p /output/usr/bin && \ export GOARM=$( echo "${GOARM}" | cut -c2-) && \ go build -o /output/${BIN}.exe \ -ldflags "${LDFLAGS}" ${PKG}/cmd/${BIN} && \ go build -o /output/velero-restore-helper.exe \ -ldflags "${LDFLAGS}" ${PKG}/cmd/velero-restore-helper && \ go build -o /output/velero-helper.exe \ -ldflags "${LDFLAGS}" ${PKG}/cmd/velero-helper && \ go clean -modcache -cache # Velero image packing section FROM mcr.microsoft.com/windows/nanoserver:${OS_VERSION} COPY --from=velero-builder /output / USER ContainerUser ================================================ FILE: GOVERNANCE.md ================================================ # Velero Governance This document defines the project governance for Velero. ## Overview **Velero**, an open source project, is committed to building an open, inclusive, productive and self-governing open source community focused on building a high quality tool that enables users to safely backup and restore, perform disaster recovery, and migrate Kubernetes cluster resources and persistent volumes. The community is governed by this document with the goal of defining how community should work together to achieve this goal. ## Code Repositories The following code repositories are governed by Velero community and maintained under the `vmware-tanzu\Velero` organization. * **[Velero](https://github.com/vmware-tanzu/velero):** Main Velero codebase * **[Helm Chart](https://github.com/vmware-tanzu/helm-charts/tree/main/charts/velero):** The Helm chart for the Velero server component * **[Velero CSI Plugin](https://github.com/vmware-tanzu/velero-plugin-for-csi):** This repository contains Velero plugins for snapshotting CSI backed PVCs using the CSI beta snapshot APIs * **[Velero Plugin for vSphere](https://github.com/vmware-tanzu/velero-plugin-for-vsphere):** This repository contains the Velero Plugin for vSphere. This plugin is a volume snapshotter plugin that provides crash-consistent snapshots of vSphere block volumes and backup of volume data into S3 compatible storage. * **[Velero Plugin for AWS](https://github.com/vmware-tanzu/velero-plugin-for-aws):** This repository contains the plugins to support running Velero on AWS, including the object store plugin and the volume snapshotter plugin * **[Velero Plugin for GCP](https://github.com/vmware-tanzu/velero-plugin-for-gcp):** This repository contains the plugins to support running Velero on GCP, including the object store plugin and the volume snapshotter plugin * **[Velero Plugin for Azure](https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure):** This repository contains the plugins to support running Velero on Azure, including the object store plugin and the volume snapshotter plugin * **[Velero Plugin Example](https://github.com/vmware-tanzu/velero-plugin-example):** This repository contains example plugins for Velero ## Community Roles * **Users:** Members that engage with the Velero community via any medium (Slack, GitHub, mailing lists, etc.). * **Contributors:** Regular contributions to projects (documentation, code reviews, responding to issues, participation in proposal discussions, contributing code, etc.). * **Maintainers**: The Velero project leaders. They are responsible for the overall health and direction of the project; final reviewers of PRs and responsible for releases. Some Maintainers are responsible for one or more components within a project, acting as technical leads for that component. Maintainers are expected to contribute code and documentation, review PRs including ensuring quality of code, triage issues, proactively fix bugs, and perform maintenance tasks for these components. ### Maintainers New maintainers must be nominated by an existing maintainer and must be elected by a supermajority of existing maintainers. Likewise, maintainers can be removed by a supermajority of the existing maintainers or can resign by notifying one of the maintainers. ### Supermajority A supermajority is defined as two-thirds of members in the group. A supermajority of [Maintainers](#maintainers) is required for certain decisions as outlined above. A supermajority vote is equivalent to the number of votes in favor being at least twice the number of votes against. For example, if you have 5 maintainers, a supermajority vote is 4 votes. Voting on decisions can happen on the mailing list, GitHub, Slack, email, or via a voting service, when appropriate. Maintainers can either vote "agree, yes, +1", "disagree, no, -1", or "abstain". A vote passes when supermajority is met. An abstain vote equals not voting at all. ### Decision Making Ideally, all project decisions are resolved by consensus. If impossible, any maintainer may call a vote. Unless otherwise specified in this document, any vote will be decided by a supermajority of maintainers. Votes by maintainers belonging to the same company will count as one vote; e.g., 4 maintainers employed by fictional company **Valerium** will only have **one** combined vote. If voting members from a given company do not agree, the company's vote is determined by a supermajority of voters from that company. If no supermajority is achieved, the company is considered to have abstained. ## Proposal Process One of the most important aspects in any open source community is the concept of proposals. Large changes to the codebase and / or new features should be preceded by a proposal in our community repo. This process allows for all members of the community to weigh in on the concept (including the technical details), share their comments and ideas, and offer to help. It also ensures that members are not duplicating work or inadvertently stepping on toes by making large conflicting changes. The project roadmap is defined by accepted proposals. Proposals should cover the high-level objectives, use cases, and technical recommendations on how to implement. In general, the community member(s) interested in implementing the proposal should be either deeply engaged in the proposal process or be an author of the proposal. The proposal should be documented as a separated markdown file pushed to the root of the `design` folder in the [Velero](https://github.com/vmware-tanzu/velero/tree/main/design) repository via PR. The name of the file should follow the name pattern `_design.md`, e.g: `restore-hooks-design.md`. Use the [Proposal Template](https://github.com/vmware-tanzu/velero/blob/main/design/_template.md) as a starting point. ### Proposal Lifecycle The proposal PR can follow the GitHub lifecycle of the PR to indicate its status: * **Open**: Proposal is created and under review and discussion. * **Merged**: Proposal has been reviewed and is accepted (either by consensus or through a vote). * **Closed**: Proposal has been reviewed and was rejected (either by consensus or through a vote). ## Lazy Consensus To maintain velocity in a project as busy as Velero, the concept of [Lazy Consensus](http://en.osswiki.info/concepts/lazy_consensus) is practiced. Ideas and / or proposals should be shared by maintainers via GitHub with the appropriate maintainer groups (e.g., `@vmware-tanzu/velero-maintainers`) tagged. Out of respect for other contributors, major changes should also be accompanied by a ping on Slack or a note on the Velero mailing list as appropriate. Author(s) of proposal, Pull Requests, issues, etc. will give a time period of no less than five (5) working days for comment and remain cognizant of popular observed world holidays. Other maintainers may chime in and request additional time for review, but should remain cognizant of blocking progress and abstain from delaying progress unless absolutely needed. The expectation is that blocking progress is accompanied by a guarantee to review and respond to the relevant action(s) (proposals, PRs, issues, etc.) in short order. Lazy Consensus is practiced for all projects in the `Velero` org, including the main project repository and the additional repositories. Lazy consensus does _not_ apply to the process of: * Removal of maintainers from Velero ## Deprecation Policy ### Deprecation Process Any contributor may introduce a request to deprecate a feature or an option of a feature by opening a feature request issue in the vmware-tanzu/velero GitHub project. The issue should describe why the feature is no longer needed or has become detrimental to Velero, as well as whether and how it has been superseded. The submitter should give as much detail as possible. Once the issue is filed, a one-month discussion period begins. Discussions take place within the issue itself as well as in the community meetings. The person who opens the issue, or a maintainer, should add the date and time marking the end of the discussion period in a comment on the issue as soon as possible after it is opened. A decision on the issue needs to be made within this one-month period. The feature will be deprecated by a supermajority vote of 50% plus one of the project maintainers at the time of the vote tallying, which is 72 hours after the end of the community meeting that is the end of the comment period. (Maintainers are permitted to vote in advance of the deadline, but should hold their votes until as close as possible to hear all possible discussion.) Votes will be tallied in comments on the issue. Non-maintainers may add non-binding votes in comments to the issue as well; these are opinions to be taken into consideration by maintainers, but they do not count as votes. If the vote passes, the deprecation window takes effect in the subsequent release, and the removal follows the schedule. ### Schedule If depreciation proposal passes by supermajority votes, the feature is deprecated in the next minor release and the feature can be removed completely after two minor version or equivalent major version e.g., if feature gets deprecated in Nth minor version, then feature can be removed after N+2 minor version or its equivalent if the major version number changes. ### Deprecation Window The deprecation window is the period from the release in which the deprecation takes effect through the release in which the feature is removed. During this period, only critical security vulnerabilities and catastrophic bugs should be fixed. **Note:** If a backup relies on a deprecated feature, then backups made with the last Velero release before this feature is removed must still be restorable in version `n+2`. For instance, something like restic feature support, that might mean that restic is removed from the list of supported uploader types in version `n` but the underlying implementation required to restore from a restic backup won't be removed until release `n+2`. ## Updating Governance All substantive changes in Governance require a supermajority agreement by all maintainers. ================================================ FILE: LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "{}" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright {yyyy} {name of copyright owner} Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: MAINTAINERS.md ================================================ # Velero Maintainers [GOVERNANCE.md](https://github.com/vmware-tanzu/velero/blob/main/GOVERNANCE.md) describes governance guidelines and maintainer responsibilities. ## Maintainers | Maintainer | GitHub ID | Affiliation | |---------------------|---------------------------------------------------------------|--------------------------------------------------| | Scott Seago | [sseago](https://github.com/sseago) | [OpenShift](https://github.com/openshift) | | Daniel Jiang | [reasonerjt](https://github.com/reasonerjt) | Broadcom | | Wenkai Yin | [ywk253100](https://github.com/ywk253100) | Broadcom | | Xun Jiang | [blackpiglet](https://github.com/blackpiglet) | Broadcom | | Shubham Pampattiwar | [shubham-pampattiwar](https://github.com/shubham-pampattiwar) | [OpenShift](https://github.com/openshift) | | Yonghui Li | [Lyndon-Li](https://github.com/Lyndon-Li) | Broadcom | | Anshul Ahuja | [anshulahuja98](https://github.com/anshulahuja98) | [Microsoft Azure](https://www.github.com/azure/) | | Tiger Kaovilai | [kaovilai](https://github.com/kaovilai) | [OpenShift](https://github.com/openshift) | ## Emeritus Maintainers * Adnan Abdulhussein ([prydonius](https://github.com/prydonius)) * Andy Goldstein ([ncdc](https://github.com/ncdc)) * Steve Kriss ([skriss](https://github.com/skriss)) * Carlos Panato ([cpanato](https://github.com/cpanato)) * Nolan Brubaker ([nrb](https://github.com/nrb)) * Ashish Amarnath ([ashish-amarnath](https://github.com/ashish-amarnath)) * Carlisia Thompson ([carlisia](https://github.com/carlisia)) * Bridget McErlean ([zubron](https://github.com/zubron)) * JenTing Hsiao ([jenting](https://github.com/jenting)) * Dave Smith-Uchida ([dsu-igeek](https://github.com/dsu-igeek)) * Ming Qiu ([qiuming-best](https://github.com/qiuming-best)) ================================================ FILE: Makefile ================================================ # Copyright 2016 The Kubernetes Authors. # # Modifications Copyright 2020 the Velero contributors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # The binary to build (just the basename). BIN ?= velero # This repo's root import path (under GOPATH). PKG := github.com/vmware-tanzu/velero # Where to push the docker image. REGISTRY ?= velero # In order to push images to an insecure registry, follow the two steps: # 1. Set "INSECURE_REGISTRY=true" # 2. Provide your own buildx builder instance by setting "BUILDX_INSTANCE=your-own-builder-instance" # The builder can be created with the following command: # cat << EOF > buildkitd.toml # [registry."insecure-registry-ip:port"] # http = true # insecure = true # EOF # docker buildx create --name=velero-builder --driver=docker-container --bootstrap --use --config ./buildkitd.toml # Refer to https://github.com/docker/buildx/issues/1370#issuecomment-1288516840 for more details INSECURE_REGISTRY ?= false # Image name IMAGE ?= $(REGISTRY)/$(BIN) # We allow the Dockerfile to be configurable to enable the use of custom Dockerfiles # that pull base images from different registries. VELERO_DOCKERFILE ?= Dockerfile VELERO_DOCKERFILE_WINDOWS ?= Dockerfile-Windows BUILDER_IMAGE_DOCKERFILE ?= hack/build-image/Dockerfile # Calculate the realpath of the build-image Dockerfile as we `cd` into the hack/build # directory before this Dockerfile is used and any relative path will not be valid. BUILDER_IMAGE_DOCKERFILE_REALPATH := $(shell realpath $(BUILDER_IMAGE_DOCKERFILE)) # Build image handling. We push a build image for every changed version of # /hack/build-image/Dockerfile. We tag the dockerfile with the short commit hash # of the commit that changed it. When determining if there is a build image in # the registry to use we look for one that matches the current "commit" for the # Dockerfile else we make one. # In the case where the Dockerfile for the build image has been overridden using # the BUILDER_IMAGE_DOCKERFILE variable, we always force a build. ifneq "$(origin BUILDER_IMAGE_DOCKERFILE)" "file" BUILDER_IMAGE_TAG := "custom" else BUILDER_IMAGE_TAG := $(shell git log -1 --pretty=%h $(BUILDER_IMAGE_DOCKERFILE)) endif BUILDER_IMAGE := $(REGISTRY)/build-image:$(BUILDER_IMAGE_TAG) BUILDER_IMAGE_CACHED := $(shell docker images -q ${BUILDER_IMAGE} 2>/dev/null ) HUGO_IMAGE := ghcr.io/gohugoio/hugo # Which architecture to build - see $(ALL_ARCH) for options. # if the 'local' rule is being run, detect the ARCH from 'go env' # if it wasn't specified by the caller. local : ARCH ?= $(shell go env GOOS)-$(shell go env GOARCH) ARCH ?= linux-amd64 VERSION ?= main TAG_LATEST ?= false ifeq ($(TAG_LATEST), true) IMAGE_TAGS ?= $(IMAGE):$(VERSION) $(IMAGE):latest else IMAGE_TAGS ?= $(IMAGE):$(VERSION) endif # check buildx is enabled only if docker is in path # macOS/Windows docker cli without Docker Desktop license: https://github.com/abiosoft/colima # To add buildx to docker cli: https://github.com/abiosoft/colima/discussions/273#discussioncomment-2684502 ifeq ($(shell which docker 2>/dev/null 1>&2 && docker buildx inspect 2>/dev/null | awk '/Status/ { print $$2 }'), running) BUILDX_ENABLED ?= true # if emulated docker cli from podman, assume enabled # emulated docker cli from podman: https://podman-desktop.io/docs/migrating-from-docker/emulating-docker-cli-with-podman # podman known issues: # - on remote podman, such as on macOS, # --output issue: https://github.com/containers/podman/issues/15922 else ifeq ($(shell which docker 2>/dev/null 1>&2 && cat $(shell which docker) | grep -c "exec podman"), 1) BUILDX_ENABLED ?= true else BUILDX_ENABLED ?= false endif define BUILDX_ERROR buildx not enabled, refusing to run this recipe see: https://velero.io/docs/main/build-from-source/#making-images-and-updating-velero for more info endef # comma cannot be escaped and can only be used in Make function arguments by putting into variable comma=, # The version of restic binary to be downloaded RESTIC_VERSION ?= 0.15.0 CLI_PLATFORMS ?= linux-amd64 linux-arm linux-arm64 darwin-amd64 darwin-arm64 windows-amd64 linux-ppc64le linux-s390x BUILD_OUTPUT_TYPE ?= docker BUILD_OS ?= linux BUILD_ARCH ?= amd64 BUILD_WINDOWS_VERSION ?= ltsc2022 ifeq ($(BUILD_OUTPUT_TYPE), docker) ALL_OS = linux ALL_ARCH.linux = $(word 2, $(subst -, ,$(shell go env GOOS)-$(shell go env GOARCH))) else ALL_OS = $(subst $(comma), ,$(BUILD_OS)) ALL_ARCH.linux = $(subst $(comma), ,$(BUILD_ARCH)) endif ALL_ARCH.windows = $(if $(filter windows,$(ALL_OS)),amd64,) ALL_OSVERSIONS.windows = $(if $(filter windows,$(ALL_OS)),$(BUILD_WINDOWS_VERSION),) ALL_OS_ARCH.linux = $(foreach os, $(filter linux,$(ALL_OS)), $(foreach arch, ${ALL_ARCH.linux}, ${os}-$(arch))) ALL_OS_ARCH.windows = $(foreach os, $(filter windows,$(ALL_OS)), $(foreach arch, $(ALL_ARCH.windows), $(foreach osversion, ${ALL_OSVERSIONS.windows}, ${os}-${osversion}-${arch}))) ALL_OS_ARCH = $(ALL_OS_ARCH.linux)$(ALL_OS_ARCH.windows) ALL_IMAGE_TAGS = $(IMAGE_TAGS) # set git sha and tree state GIT_SHA = $(shell git rev-parse HEAD) ifneq ($(shell git status --porcelain 2> /dev/null),) GIT_TREE_STATE ?= dirty else GIT_TREE_STATE ?= clean endif ### ### These variables should not need tweaking. ### platform_temp = $(subst -, ,$(ARCH)) GOOS = $(word 1, $(platform_temp)) GOARCH = $(word 2, $(platform_temp)) GOPROXY ?= https://proxy.golang.org GOBIN=$$(pwd)/.go/bin # If you want to build all binaries, see the 'all-build' rule. # If you want to build all containers, see the 'all-containers' rule. all: @$(MAKE) build build-%: @$(MAKE) --no-print-directory ARCH=$* build all-build: $(addprefix build-, $(CLI_PLATFORMS)) all-containers: @$(MAKE) --no-print-directory container local: build-dirs # Add DEBUG=1 to enable debug locally GOOS=$(GOOS) \ GOARCH=$(GOARCH) \ GOBIN=$(GOBIN) \ VERSION=$(VERSION) \ REGISTRY=$(REGISTRY) \ PKG=$(PKG) \ BIN=$(BIN) \ GIT_SHA=$(GIT_SHA) \ GIT_TREE_STATE=$(GIT_TREE_STATE) \ OUTPUT_DIR=$$(pwd)/_output/bin/$(GOOS)/$(GOARCH) \ ./hack/build.sh build: _output/bin/$(GOOS)/$(GOARCH)/$(BIN) _output/bin/$(GOOS)/$(GOARCH)/$(BIN): build-dirs @echo "building: $@" $(MAKE) shell CMD="-c '\ GOOS=$(GOOS) \ GOARCH=$(GOARCH) \ GOBIN=$(GOBIN) \ VERSION=$(VERSION) \ REGISTRY=$(REGISTRY) \ PKG=$(PKG) \ BIN=$(BIN) \ GIT_SHA=$(GIT_SHA) \ GIT_TREE_STATE=$(GIT_TREE_STATE) \ OUTPUT_DIR=/output/$(GOOS)/$(GOARCH) \ ./hack/build.sh'" TTY := $(shell tty -s && echo "-t") # Example: make shell CMD="date > datefile" shell: build-dirs build-env @# bind-mount the Velero root dir in at /github.com/vmware-tanzu/velero @# because the Kubernetes code-generator tools require the project to @# exist in a directory hierarchy ending like this (but *NOT* necessarily @# under $GOPATH). @docker run \ -e GOFLAGS \ -e GOPROXY \ -i $(TTY) \ --rm \ -u $$(id -u):$$(id -g) \ -v "$$(pwd):/github.com/vmware-tanzu/velero:delegated" \ -v "$$(pwd)/_output/bin:/output:delegated" \ -v "$$(pwd)/.go/pkg:/go/pkg:delegated" \ -v "$$(pwd)/.go/std:/go/std:delegated" \ -v "$$(pwd)/.go/std/$(GOOS)/$(GOARCH):/usr/local/go/pkg/$(GOOS)_$(GOARCH)_static:delegated" \ -v "$$(pwd)/.go/go-build:/.cache/go-build:delegated" \ -v "$$(pwd)/.go/golangci-lint:/.cache/golangci-lint:delegated" \ -w /github.com/vmware-tanzu/velero \ $(BUILDER_IMAGE) \ /bin/sh $(CMD) container: ifneq ($(BUILDX_ENABLED), true) $(error $(BUILDX_ERROR)) endif ifeq ($(BUILDX_INSTANCE),) @echo creating a buildx instance -docker buildx rm velero-builder || true @docker buildx create --use --name=velero-builder else @echo using a specified buildx instance $(BUILDX_INSTANCE) @docker buildx use $(BUILDX_INSTANCE) endif @mkdir -p _output @for osarch in $(ALL_OS_ARCH); do \ $(MAKE) container-$${osarch}; \ done ifeq ($(BUILD_OUTPUT_TYPE), registry) @for tag in $(ALL_IMAGE_TAGS); do \ IMAGE_TAG=$${tag} $(MAKE) push-manifest; \ done endif container-linux-%: @BUILDX_ARCH=$* $(MAKE) container-linux container-linux: @echo "building container: $(IMAGE):$(VERSION)-linux-$(BUILDX_ARCH)" @docker buildx build --pull \ --output="type=$(BUILD_OUTPUT_TYPE)$(if $(findstring tar, $(BUILD_OUTPUT_TYPE)),$(comma)dest=_output/$(BIN)-$(VERSION)-linux-$(BUILDX_ARCH).tar,)" \ --platform="linux/$(BUILDX_ARCH)" \ $(addprefix -t , $(addsuffix "-linux-$(BUILDX_ARCH)",$(ALL_IMAGE_TAGS))) \ --build-arg=GOPROXY=$(GOPROXY) \ --build-arg=PKG=$(PKG) \ --build-arg=BIN=$(BIN) \ --build-arg=VERSION=$(VERSION) \ --build-arg=GIT_SHA=$(GIT_SHA) \ --build-arg=GIT_TREE_STATE=$(GIT_TREE_STATE) \ --build-arg=REGISTRY=$(REGISTRY) \ --build-arg=RESTIC_VERSION=$(RESTIC_VERSION) \ --provenance=false \ --sbom=false \ -f $(VELERO_DOCKERFILE) . @echo "built container: $(IMAGE):$(VERSION)-linux-$(BUILDX_ARCH)" container-windows-%: @BUILDX_OSVERSION=$(firstword $(subst -, ,$*)) BUILDX_ARCH=$(lastword $(subst -, ,$*)) $(MAKE) container-windows container-windows: @echo "building container: $(IMAGE):$(VERSION)-windows-$(BUILDX_OSVERSION)-$(BUILDX_ARCH)" @docker buildx build --pull \ --output="type=$(BUILD_OUTPUT_TYPE)$(if $(findstring tar, $(BUILD_OUTPUT_TYPE)),$(comma)dest=_output/$(BIN)-$(VERSION)-windows-$(BUILDX_OSVERSION)-$(BUILDX_ARCH).tar,)" \ --platform="windows/$(BUILDX_ARCH)" \ $(addprefix -t , $(addsuffix "-windows-$(BUILDX_OSVERSION)-$(BUILDX_ARCH)",$(ALL_IMAGE_TAGS))) \ --build-arg=GOPROXY=$(GOPROXY) \ --build-arg=PKG=$(PKG) \ --build-arg=BIN=$(BIN) \ --build-arg=VERSION=$(VERSION) \ --build-arg=OS_VERSION=$(BUILDX_OSVERSION) \ --build-arg=GIT_SHA=$(GIT_SHA) \ --build-arg=GIT_TREE_STATE=$(GIT_TREE_STATE) \ --build-arg=REGISTRY=$(REGISTRY) \ --provenance=false \ --sbom=false \ -f $(VELERO_DOCKERFILE_WINDOWS) . @echo "built container: $(IMAGE):$(VERSION)-windows-$(BUILDX_OSVERSION)-$(BUILDX_ARCH)" push-manifest: @echo "building manifest: $(IMAGE_TAG) for $(foreach osarch, $(ALL_OS_ARCH), $(IMAGE_TAG)-${osarch})" @docker manifest create --amend --insecure=$(INSECURE_REGISTRY) $(IMAGE_TAG) $(foreach osarch, $(ALL_OS_ARCH), $(IMAGE_TAG)-${osarch}) @set -x; \ for arch in $(ALL_ARCH.windows); do \ for osversion in $(ALL_OSVERSIONS.windows); do \ BASEIMAGE=mcr.microsoft.com/windows/nanoserver:$${osversion}; \ full_version=`docker manifest inspect --insecure=$(INSECURE_REGISTRY) $${BASEIMAGE} | jq -r '.manifests[0].platform["os.version"]'`; \ docker manifest annotate --os windows --arch $${arch} --os-version $${full_version} $(IMAGE_TAG) $(IMAGE_TAG)-windows-$${osversion}-$${arch}; \ done; \ done @echo "pushing manifest $(IMAGE_TAG)" @docker manifest push --purge --insecure=$(INSECURE_REGISTRY) $(IMAGE_TAG) @echo "pushed manifest $(IMAGE_TAG):" @docker manifest inspect --insecure=$(INSECURE_REGISTRY) $(IMAGE_TAG) SKIP_TESTS ?= test: build-dirs ifneq ($(SKIP_TESTS), 1) @$(MAKE) shell CMD="-c 'hack/test.sh $(WHAT)'" endif test-local: build-dirs ifneq ($(SKIP_TESTS), 1) hack/test.sh $(WHAT) endif verify: ifneq ($(SKIP_TESTS), 1) @$(MAKE) shell CMD="-c 'hack/verify-all.sh'" endif lint: ifneq ($(SKIP_TESTS), 1) @$(MAKE) shell CMD="-c 'hack/lint.sh'" endif local-lint: ifneq ($(SKIP_TESTS), 1) @hack/lint.sh endif update: @$(MAKE) shell CMD="-c 'hack/update-all.sh'" # update-crd is for development purpose only, it is faster than update, so is a shortcut when you want to generate CRD changes only update-crd: @$(MAKE) shell CMD="-c 'hack/update-3generated-crd-code.sh'" build-dirs: @mkdir -p _output/bin/$(GOOS)/$(GOARCH) @mkdir -p .go/src/$(PKG) .go/pkg .go/bin .go/std/$(GOOS)/$(GOARCH) .go/go-build .go/golangci-lint build-env: @# if we have overridden the value for the build-image Dockerfile, @# force a build using that Dockerfile @# if we detect changes in dockerfile force a new build-image @# else if we dont have a cached image make one @# finally use the cached image ifneq "$(origin BUILDER_IMAGE_DOCKERFILE)" "file" @echo "Dockerfile for builder image has been overridden to $(BUILDER_IMAGE_DOCKERFILE)" @echo "Preparing a new builder-image" $(MAKE) build-image else ifneq ($(shell git diff --quiet HEAD -- $(BUILDER_IMAGE_DOCKERFILE); echo $$?), 0) @echo "Local changes detected in $(BUILDER_IMAGE_DOCKERFILE)" @echo "Preparing a new builder-image" $(MAKE) build-image else ifneq ($(BUILDER_IMAGE_CACHED),) @echo "Using Cached Image: $(BUILDER_IMAGE)" else @echo "Trying to pull build-image: $(BUILDER_IMAGE)" docker pull -q $(BUILDER_IMAGE) || $(MAKE) build-image endif build-image: @# When we build a new image we just untag the old one. @# This makes sure we don't leave the orphaned image behind. $(eval old_id=$(shell docker image inspect --format '{{ .ID }}' ${BUILDER_IMAGE} 2>/dev/null)) ifeq ($(BUILDX_ENABLED), true) @cd hack/build-image && docker buildx build --build-arg=GOPROXY=$(GOPROXY) --output=type=docker --pull -t $(BUILDER_IMAGE) -f $(BUILDER_IMAGE_DOCKERFILE_REALPATH) . else @cd hack/build-image && docker build --build-arg=GOPROXY=$(GOPROXY) --pull -t $(BUILDER_IMAGE) -f $(BUILDER_IMAGE_DOCKERFILE_REALPATH) . endif $(eval new_id=$(shell docker image inspect --format '{{ .ID }}' ${BUILDER_IMAGE} 2>/dev/null)) @if [ "$(old_id)" != "" ] && [ "$(old_id)" != "$(new_id)" ]; then \ docker rmi -f $$id || true; \ fi push-build-image: @# this target will push the build-image it assumes you already have docker @# credentials needed to accomplish this. @# Pushing will be skipped if a custom Dockerfile was used to build the image. ifneq "$(origin BUILDER_IMAGE_DOCKERFILE)" "file" @echo "Dockerfile for builder image has been overridden" @echo "Skipping push of custom image" else docker push $(BUILDER_IMAGE) endif build-image-hugo: cd site && docker build --pull -t $(HUGO_IMAGE) . clean: # if we have a cached image then use it to run go clean --modcache # this test checks if we there is an image id in the BUILDER_IMAGE_CACHED variable. ifneq ($(strip $(BUILDER_IMAGE_CACHED)),) $(MAKE) shell CMD="-c 'go clean --modcache'" docker rmi -f $(BUILDER_IMAGE) || true endif rm -rf .go _output docker rmi $(HUGO_IMAGE) .PHONY: modules modules: go mod tidy .PHONY: verify-modules verify-modules: modules @if !(git diff --quiet HEAD -- go.sum go.mod); then \ echo "go module files are out of date, please commit the changes to go.mod and go.sum"; exit 1; \ fi ci: verify-modules verify all test changelog: hack/release-tools/changelog.sh # release builds a GitHub release using goreleaser within the build container. # # To dry-run the release, which will build the binaries/artifacts locally but # will *not* create a GitHub release: # GITHUB_TOKEN=an-invalid-token-so-you-dont-accidentally-push-release \ # RELEASE_NOTES_FILE=changelogs/CHANGELOG-1.2.md \ # PUBLISH=false \ # make release # # To run the release, which will publish a *DRAFT* GitHub release in github.com/vmware-tanzu/velero # (you still need to review/publish the GitHub release manually): # GITHUB_TOKEN=your-github-token \ # RELEASE_NOTES_FILE=changelogs/CHANGELOG-1.2.md \ # PUBLISH=true \ # make release release: $(MAKE) shell CMD="-c '\ GITHUB_TOKEN=$(GITHUB_TOKEN) \ RELEASE_NOTES_FILE=$(RELEASE_NOTES_FILE) \ PUBLISH=$(PUBLISH) \ REGISTRY=$(REGISTRY) \ ./hack/release-tools/goreleaser.sh'" serve-docs: build-image-hugo docker run \ --rm \ -v "$$(pwd)/site:/project" \ -it -p 1313:1313 \ $(HUGO_IMAGE) \ server --bind=0.0.0.0 --enableGitInfo=false # gen-docs generates a new versioned docs directory under site/content/docs. # Please read the documentation in the script for instructions on how to use it. gen-docs: @hack/release-tools/gen-docs.sh .PHONY: test-e2e test-e2e: local $(MAKE) -e VERSION=$(VERSION) -C test/ run-e2e .PHONY: test-perf test-perf: local $(MAKE) -e VERSION=$(VERSION) -C test/ run-perf go-generate: go generate ./pkg/... # requires an authenticated gh cli # gh: https://cli.github.com/ # First create a PR # gh pr create --title 'Title name' --body 'PR body' # by default uses PR title as changelog body but can be overwritten like so # make new-changelog CHANGELOG_BODY="Changes you have made" new-changelog: GH_LOGIN ?= $(shell gh pr view --json author --jq .author.login 2> /dev/null) new-changelog: GH_PR_NUMBER ?= $(shell gh pr view --json number --jq .number 2> /dev/null) new-changelog: CHANGELOG_BODY ?= '$(shell gh pr view --json title --jq .title)' new-changelog: @if [ "$(GH_LOGIN)" = "" ]; then \ echo "branch does not have PR or cli not logged in, try 'gh auth login' or 'gh pr create'"; \ exit 1; \ fi @mkdir -p ./changelogs/unreleased/ && \ echo $(CHANGELOG_BODY) > ./changelogs/unreleased/$(GH_PR_NUMBER)-$(GH_LOGIN) && \ echo \"$(CHANGELOG_BODY)\" added to "./changelogs/unreleased/$(GH_PR_NUMBER)-$(GH_LOGIN)" ================================================ FILE: OWNERS ================================================ # This file is used by the [PROW action](https://github.com/jpmcb/prow-github-actions) to approve and merge PRs. # The file's format follows the [OWNERS SPEC](https://www.kubernetes.dev/docs/guide/owners/#owners-spec). # List of usernames who may use /lgtm reviewers: - @Lyndon-Li - @anshulahuja98 - @blackpiglet - @qiuming-best - @reasonerjt - @shubham-pampattiwar - @sseago - @ywk253100 # List of usernames who may use /approve approvers: - @Lyndon-Li - @anshulahuja98 - @blackpiglet - @qiuming-best - @reasonerjt - @shubham-pampattiwar - @sseago - @ywk253100 ================================================ FILE: README.md ================================================ ![100] [![Build Status][1]][2] [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/3811/badge)](https://bestpractices.coreinfrastructure.org/projects/3811) ![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/vmware-tanzu/velero) ## Overview Velero (formerly Heptio Ark) gives you tools to back up and restore your Kubernetes cluster resources and persistent volumes. You can run Velero with a public cloud platform or on-premises. Velero lets you: * Take backups of your cluster and restore in case of loss. * Migrate cluster resources to other clusters. * Replicate your production cluster to development and testing clusters. Velero consists of: * A server that runs on your cluster * A command-line client that runs locally ## Documentation [The documentation][29] provides a getting started guide and information about building from source, architecture, extending Velero and more. Please use the version selector at the top of the site to ensure you are using the appropriate documentation for your version of Velero. ## Troubleshooting If you encounter issues, review the [troubleshooting docs][30], [file an issue][4], or talk to us on the [#velero channel][25] on the Kubernetes Slack server. ## Contributing If you are ready to jump in and test, add code, or help with documentation, follow the instructions on our [Start contributing][31] documentation for guidance on how to setup Velero for development. ## Changelog See [the list of releases][6] to find out about feature changes. ### Velero compatibility matrix The following is a list of the supported Kubernetes versions for each Velero version. | Velero version | Expected Kubernetes version compatibility | Tested on Kubernetes version | |----------------|-------------------------------------------|-------------------------------------| | 1.18 | 1.18-latest | 1.33.7, 1.34.1, and 1.35.0 | | 1.17 | 1.18-latest | 1.31.7, 1.32.3, 1.33.1, and 1.34.0 | | 1.16 | 1.18-latest | 1.31.4, 1.32.3, and 1.33.0 | | 1.15 | 1.18-latest | 1.28.8, 1.29.8, 1.30.4 and 1.31.1 | | 1.14 | 1.18-latest | 1.27.9, 1.28.9, and 1.29.4 | Velero supports IPv4, IPv6, and dual stack environments. Support for this was tested against Velero v1.8. The Velero maintainers are continuously working to expand testing coverage, but are not able to test every combination of Velero and supported Kubernetes versions for each Velero release. The table above is meant to track the current testing coverage and the expected supported Kubernetes versions for each Velero version. If you are interested in using a different version of Kubernetes with a given Velero version, we'd recommend that you perform testing before installing or upgrading your environment. For full information around capabilities within a release, also see the Velero [release notes](https://github.com/vmware-tanzu/velero/releases) or Kubernetes [release notes](https://github.com/kubernetes/kubernetes/tree/master/CHANGELOG). See the Velero [support page](https://velero.io/docs/latest/support-process/) for information about supported versions of Velero. For each release, Velero maintainers run the test to ensure the upgrade path from n-2 minor release. For example, before the release of v1.10.x, the test will verify that the backup created by v1.9.x and v1.8.x can be restored using the build to be tagged as v1.10.x. [1]: https://github.com/vmware-tanzu/velero/workflows/Main%20CI/badge.svg [2]: https://github.com/vmware-tanzu/velero/actions?query=workflow%3A"Main+CI" [4]: https://github.com/vmware-tanzu/velero/issues [6]: https://github.com/vmware-tanzu/velero/releases [9]: https://kubernetes.io/docs/setup/ [10]: https://kubernetes.io/docs/tasks/tools/install-kubectl/#install-with-homebrew-on-macos [11]: https://kubernetes.io/docs/tasks/tools/install-kubectl/#tabset-1 [12]: https://github.com/kubernetes/kubernetes/blob/master/cluster/addons/dns/README.md [14]: https://github.com/kubernetes/kubernetes [24]: https://groups.google.com/forum/#!forum/projectvelero [25]: https://kubernetes.slack.com/messages/velero [29]: https://velero.io/docs/ [30]: https://velero.io/docs/troubleshooting [31]: https://velero.io/docs/start-contributing [100]: https://velero.io/docs/main/img/velero.png ================================================ FILE: ROADMAP.md ================================================ # Please go to the [Velero Wiki](https://github.com/vmware-tanzu/velero/wiki/) to see our latest roadmap, archived roadmaps and roadmap guidance. ================================================ FILE: SECURITY.md ================================================ # Security Release Process Velero is an open source tool with a growing community devoted to safe backup and restore, disaster recovery, and data migration of Kubernetes resources and persistent volumes. The community has adopted this security disclosure and response policy to ensure we responsibly handle critical issues. ## Supported Versions The Velero project maintains the following [governance document](https://github.com/vmware-tanzu/velero/blob/main/GOVERNANCE.md), [release document](https://github.com/vmware-tanzu/velero/blob/f42c63af1b9af445e38f78a7256b1c48ef79c10e/site/docs/main/release-instructions.md), and [support document](https://velero.io/docs/main/support-process/). Please refer to these for release and related details. Only the most recent version of Velero is supported. Each [release](https://github.com/vmware-tanzu/velero/releases) includes information about upgrading to the latest version. ## Reporting a Vulnerability - Private Disclosure Process Security is of the highest importance and all security vulnerabilities or suspected security vulnerabilities should be reported to Velero privately, to minimize attacks against current users of Velero before they are fixed. Vulnerabilities will be investigated and patched on the next patch (or minor) release as soon as possible. This information could be kept entirely internal to the project. If you know of a publicly disclosed security vulnerability for Velero, please **IMMEDIATELY** contact the Security Team (velero-security.pdl@broadcom.com). **IMPORTANT: Do not file public issues on GitHub for security vulnerabilities** To report a vulnerability or a security-related issue, please contact the email address with the details of the vulnerability. The email will be fielded by the Security Team and then shared with the Velero maintainers who have committer and release permissions. Emails will be addressed within 3 business days, including a detailed plan to investigate the issue and any potential workarounds to perform in the meantime. Do not report non-security-impacting bugs through this channel. Use [GitHub issues](https://github.com/vmware-tanzu/velero/issues/new/choose) instead. ## Proposed Email Content Provide a descriptive subject line and in the body of the email include the following information: * Basic identity information, such as your name and your affiliation or company. * Detailed steps to reproduce the vulnerability (POC scripts, screenshots, and logs are all helpful to us). * Description of the effects of the vulnerability on Velero and the related hardware and software configurations, so that the Security Team can reproduce it. * How the vulnerability affects Velero usage and an estimation of the attack surface, if there is one. * List other projects or dependencies that were used in conjunction with Velero to produce the vulnerability. ## When to report a vulnerability * When you think Velero has a potential security vulnerability. * When you suspect a potential vulnerability but you are unsure that it impacts Velero. * When you know of or suspect a potential vulnerability on another project that is used by Velero. ## Patch, Release, and Disclosure The Security Team will respond to vulnerability reports as follows: 1. The Security Team will investigate the vulnerability and determine its effects and criticality. 2. If the issue is not deemed to be a vulnerability, the Security Team will follow up with a detailed reason for rejection. 3. The Security Team will initiate a conversation with the reporter within 3 business days. 4. If a vulnerability is acknowledged and the timeline for a fix is determined, the Security Team will work on a plan to communicate with the appropriate community, including identifying mitigating steps that affected users can take to protect themselves until the fix is rolled out. 5. The Security Team will also create a [CVSS](https://www.first.org/cvss/specification-document) using the [CVSS Calculator](https://www.first.org/cvss/calculator/3.0). The Security Team makes the final call on the calculated CVSS; it is better to move quickly than making the CVSS perfect. Issues may also be reported to [Mitre](https://cve.mitre.org/) using this [scoring calculator](https://nvd.nist.gov/vuln-metrics/cvss/v3-calculator). The CVE will initially be set to private. 6. The Security Team will work on fixing the vulnerability and perform internal testing before preparing to roll out the fix. 7. The Security Team will provide early disclosure of the vulnerability by emailing the [Velero Distributors](https://groups.google.com/u/1/g/projectvelero-distributors) mailing list. Distributors can initially plan for the vulnerability patch ahead of the fix, and later can test the fix and provide feedback to the Velero team. See the section **Early Disclosure to Velero Distributors List** for details about how to join this mailing list. 8. A public disclosure date is negotiated by the SecurityTeam, the bug submitter, and the distributors list. We prefer to fully disclose the bug as soon as possible once a user mitigation or patch is available. It is reasonable to delay disclosure when the bug or the fix is not yet fully understood, the solution is not well-tested, or for distributor coordination. The timeframe for disclosure is from immediate (especially if it’s already publicly known) to a few weeks. For a critical vulnerability with a straightforward mitigation, we expect the report date for the public disclosure date to be on the order of 14 business days. The Security Team holds the final say when setting a public disclosure date. 9. Once the fix is confirmed, the Security Team will patch the vulnerability in the next patch or minor release, and backport a patch release into all earlier supported releases. Upon release of the patched version of Velero, we will follow the **Public Disclosure Process**. ## Public Disclosure Process The Security Team publishes a [public advisory](https://github.com/vmware-tanzu/velero/security/advisories) to the Velero community via GitHub. In most cases, additional communication via Slack, Twitter, mailing lists, blog and other channels will assist in educating Velero users and rolling out the patched release to affected users. The Security Team will also publish any mitigating steps users can take until the fix can be applied to their Velero instances. Velero distributors will handle creating and publishing their own security advisories. ## Mailing lists * Use velero-security.pdl@broadcom.com to report security concerns to the Security Team, who uses the list to privately discuss security issues and fixes prior to disclosure. * Join the [Velero Distributors](https://groups.google.com/u/1/g/projectvelero-distributors) mailing list for early private information and vulnerability disclosure. Early disclosure may include mitigating steps and additional information on security patch releases. See below for information on how Velero distributors or vendors can apply to join this list. ## Early Disclosure to Velero Distributors List The private list is intended to be used primarily to provide actionable information to multiple distributor projects at once. This list is not intended to inform individuals about security issues. ## Membership Criteria To be eligible to join the [Velero Distributors](https://groups.google.com/u/1/g/projectvelero-distributors) mailing list, you should: 1. Be an active distributor of Velero. 2. Have a user base that is not limited to your own organization. 3. Have a publicly verifiable track record up to the present day of fixing security issues. 4. Not be a downstream or rebuild of another distributor. 5. Be a participant and active contributor in the Velero community. 6. Accept the Embargo Policy that is outlined below. 7. Have someone who is already on the list vouch for the person requesting membership on behalf of your distribution. **The terms and conditions of the Embargo Policy apply to all members of this mailing list. A request for membership represents your acceptance to the terms and conditions of the Embargo Policy.** ## Embargo Policy The information that members receive on the Velero Distributors mailing list must not be made public, shared, or even hinted at anywhere beyond those who need to know within your specific team, unless you receive explicit approval to do so from the Security Team. This remains true until the public disclosure date/time agreed upon by the list. Members of the list and others cannot use the information for any reason other than to get the issue fixed for your respective distribution's users. Before you share any information from the list with members of your team who are required to fix the issue, these team members must agree to the same terms, and only be provided with information on a need-to-know basis. In the unfortunate event that you share information beyond what is permitted by this policy, you must urgently inform the Security Team (velero-security.pdl@broadcom.com) of exactly what information was leaked and to whom. If you continue to leak information and break the policy outlined here, you will be permanently removed from the list. ## Requesting to Join Send new membership requests to projectvelero-distributors@googlegroups.com. In the body of your request please specify how you qualify for membership and fulfill each criterion listed in the Membership Criteria section above. ## Confidentiality, integrity and availability We consider vulnerabilities leading to the compromise of data confidentiality, elevation of privilege, or integrity to be our highest priority concerns. Availability, in particular in areas relating to DoS and resource exhaustion, is also a serious security concern. The Security Team takes all vulnerabilities, potential vulnerabilities, and suspected vulnerabilities seriously and will investigate them in an urgent and expeditious manner. Note that we do not currently consider the default settings for Velero to be secure-by-default. It is necessary for operators to explicitly configure settings, role based access control, and other resource related features in Velero to provide a hardened Velero environment. We will not act on any security disclosure that relates to a lack of safe defaults. Over time, we will work towards improved safe-by-default configuration, taking into account backwards compatibility. ================================================ FILE: SUPPORT.md ================================================ # Velero Support Thanks for trying out Velero! We welcome all feedback, find all the ways to connect with us on our Community page: - [Velero Community](https://velero.io/community/) You can find details on the Velero maintainers' support process [here](https://velero.io/docs/main/support-process/). ================================================ FILE: Tiltfile ================================================ # -*- mode: Python -*- k8s_yaml([ 'config/crd/v1/bases/velero.io_backups.yaml', 'config/crd/v1/bases/velero.io_backupstoragelocations.yaml', 'config/crd/v1/bases/velero.io_deletebackuprequests.yaml', 'config/crd/v1/bases/velero.io_downloadrequests.yaml', 'config/crd/v1/bases/velero.io_podvolumebackups.yaml', 'config/crd/v1/bases/velero.io_podvolumerestores.yaml', 'config/crd/v1/bases/velero.io_backuprepositories.yaml', 'config/crd/v1/bases/velero.io_restores.yaml', 'config/crd/v1/bases/velero.io_schedules.yaml', 'config/crd/v1/bases/velero.io_serverstatusrequests.yaml', 'config/crd/v1/bases/velero.io_volumesnapshotlocations.yaml', 'config/crd/v2alpha1/bases/velero.io_datauploads.yaml', 'config/crd/v2alpha1/bases/velero.io_datadownloads.yaml', ]) # default values settings = { "default_registry": "docker.io/velero", "use_node_agent": False, "enable_debug": False, "debug_continue_on_start": True, # Continue the velero process by default when in debug mode "create_backup_locations": False, "setup-minio": False, } # global settings settings.update(read_json( "tilt-resources/tilt-settings.json", default = {}, )) k8s_yaml(kustomize('tilt-resources')) k8s_yaml('tilt-resources/deployment.yaml') if settings.get("enable_debug"): k8s_resource('velero', port_forwards = '2345') # TODO: Need to figure out how to apply port forwards for all node-agent pods if settings.get("use_node_agent"): k8s_yaml('tilt-resources/node-agent.yaml') if settings.get("create_backup_locations"): k8s_yaml('tilt-resources/velero_v1_backupstoragelocation.yaml') if settings.get("setup-minio"): k8s_yaml('examples/minio/00-minio-deployment.yaml', allow_duplicates = True) # By default, Tilt automatically allows Minikube, Docker for Desktop, Microk8s, Red Hat CodeReady Containers, Kind, K3D, and Krucible. allow_k8s_contexts(settings.get("allowed_contexts")) default_registry(settings.get("default_registry")) local_goos = str(local("go env GOOS", quiet = True, echo_off = True)).strip() git_sha = str(local("git rev-parse HEAD", quiet = True, echo_off = True)).strip() tilt_helper_dockerfile_header = """ # Tilt image FROM golang:1.25 as tilt-helper # Support live reloading with Tilt RUN wget --output-document /restart.sh --quiet https://raw.githubusercontent.com/windmilleng/rerun-process-wrapper/master/restart.sh && \ wget --output-document /start.sh --quiet https://raw.githubusercontent.com/windmilleng/rerun-process-wrapper/master/start.sh && \ chmod +x /start.sh && chmod +x /restart.sh """ additional_docker_helper_commands = """ # Install delve to allow debugging RUN go install github.com/go-delve/delve/cmd/dlv@latest RUN wget -qO- https://dl.k8s.io/v1.25.2/kubernetes-client-linux-amd64.tar.gz | tar xvz RUN wget -qO- https://get.docker.com | sh """ additional_docker_build_commands = """ COPY --from=tilt-helper /go/bin/dlv /usr/bin/dlv COPY --from=tilt-helper /usr/bin/docker /usr/bin/docker COPY --from=tilt-helper /go/kubernetes/client/bin/kubectl /usr/bin/kubectl """ ############################## # Setup Velero ############################## def get_debug_flag(): """ Returns the flag to enable debug building of Velero if debug mode is enabled. """ if settings.get('enable_debug'): return "DEBUG=1" return "" # Set up a local_resource build of the Velero binary. The binary is written to _tiltbuild/velero. local_resource( "velero_server_binary", cmd = 'cd ' + '.' + ';mkdir -p _tiltbuild;PKG=. BIN=velero GOOS=linux GOARCH=amd64 GIT_SHA=' + git_sha + ' VERSION=main GIT_TREE_STATE=dirty OUTPUT_DIR=_tiltbuild ' + get_debug_flag() + ' REGISTRY=' + settings.get("default_registry") + ' ./hack/build.sh', deps = ["cmd", "internal", "pkg"], ignore = ["pkg/cmd"], ) local_resource( "velero_local_binary", cmd = 'cd ' + '.' + ';mkdir -p _tiltbuild/local;PKG=. BIN=velero GOOS=' + local_goos + ' GOARCH=amd64 GIT_SHA=' + git_sha + ' VERSION=main GIT_TREE_STATE=dirty OUTPUT_DIR=_tiltbuild/local ' + get_debug_flag() + ' REGISTRY=' + settings.get("default_registry") + ' ./hack/build.sh', deps = ["internal", "pkg/cmd"], ) local_resource( "restic_binary", cmd = 'cd ' + '.' + ';mkdir -p _tiltbuild/restic; BIN=velero GOOS=linux GOARCH=amd64 GOARM="" RESTIC_VERSION=0.13.1 OUTPUT_DIR=_tiltbuild/restic ./hack/build-restic.sh', ) # Note: we need a distro with a bash shell to exec into the Velero container tilt_dockerfile_header = """ FROM ubuntu:22.04 as tilt RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -qq -y ca-certificates tzdata && rm -rf /var/lib/apt/lists/* WORKDIR / COPY --from=tilt-helper /start.sh . COPY --from=tilt-helper /restart.sh . COPY velero . COPY restic/restic /usr/bin/restic """ dockerfile_contents = "\n".join([ tilt_helper_dockerfile_header, additional_docker_helper_commands, tilt_dockerfile_header, additional_docker_build_commands, ]) def get_velero_entrypoint(): """ Returns the entrypoint for the Velero container image. """ entrypoint = ["sh", "/start.sh"] if settings.get("enable_debug"): # If debug mode is enabled, start the velero process using Delve entrypoint.extend( ["dlv", "--listen=:2345", "--headless=true", "--api-version=2", "--accept-multiclient", "exec"]) # Set whether or not to continue the debugged process on start # See https://github.com/go-delve/delve/blob/master/Documentation/usage/dlv_exec.md if settings.get("debug_continue_on_start"): entrypoint.append("--continue") entrypoint.append("--") entrypoint.append("/velero") return entrypoint # Set up an image build for Velero. The live update configuration syncs the output from the local_resource # build into the container. docker_build( ref = "velero/velero", context = "_tiltbuild", dockerfile_contents = dockerfile_contents, target = "tilt", entrypoint = get_velero_entrypoint(), live_update = [ sync("./_tiltbuild/velero", "/velero"), run("sh /restart.sh"), ]) ############################## # Setup plugins ############################## def load_provider_tiltfiles(): all_providers = settings.get("providers", {}) enable_providers = settings.get("enable_providers", []) providers = [] ## Load settings only for providers to enable for name in enable_providers: repo = all_providers.get(name) if not repo: print("Enabled provider '{}' does not exist in list of supported providers".format(name)) continue file = repo + "/tilt-provider.json" if not os.path.exists(file): print("Provider settings not found for \"{}\". Please ensure this plugin repository has a tilt-provider.json file included.".format(name)) continue provider_details = read_json(file, default = {}) if type(provider_details) == "dict": provider_details["name"] = name if "context" in provider_details: provider_details["context"] = os.path.join(repo, "/", provider_details["context"]) else: provider_details["context"] = repo if "go_main" not in provider_details: provider_details["go_main"] = "main.go" providers.append(provider_details) return providers # Enable each provider def enable_providers(providers): if not providers: print("No providers to enable.") return for p in providers: enable_provider(p) # Configures a provider by doing the following: # # 1. Enables a local_resource go build of the provider's local binary # 2. Configures a docker build for the provider, with live updating of the local binary def enable_provider(provider): name = provider.get("name") plugin_name = provider.get("plugin_name") # Note: we need a distro with a shell to do a copy of the plugin binary tilt_dockerfile_header = """ FROM ubuntu:22.04 as tilt WORKDIR / COPY --from=tilt-helper /start.sh . COPY --from=tilt-helper /restart.sh . COPY """ + plugin_name + """ . """ dockerfile_contents = "\n".join([ tilt_helper_dockerfile_header, additional_docker_helper_commands, tilt_dockerfile_header, additional_docker_build_commands, ]) context = provider.get("context") go_main = provider.get("go_main", "main.go") live_reload_deps = [] for d in provider.get("live_reload_deps", []): live_reload_deps.append(os.path.join(context, "/", d)) # Set up a local_resource build of the plugin binary. The main.go path must be provided via go_main option. The binary is written to _tiltbuild/. local_resource( name + "_plugin", cmd = 'cd ' + context + ';mkdir -p _tiltbuild;PKG=' + context + ' BIN=' + go_main + ' GOOS=linux GOARCH=amd64 OUTPUT_DIR=_tiltbuild ./hack/build.sh', deps = live_reload_deps, ) # Set up an image build for the plugin. The live update configuration syncs the output from the local_resource # build into the init container, and that restarts the Velero container. docker_build( ref = provider.get("image"), context = os.path.join(context, "/_tiltbuild/"), dockerfile_contents = dockerfile_contents, target = "tilt", entrypoint = ["/bin/bash", "-c", "cp /" + plugin_name + " /target/."], live_update = [ sync(os.path.join(context, "/_tiltbuild/", plugin_name), os.path.join("/", plugin_name)) ] ) ############################## # Start ############################# enable_providers(load_provider_tiltfiles()) ================================================ FILE: assets/README.md ================================================ # Velero Assets This folder contains logo images for Velero in gray (for light backgrounds) and white (for dark backgrounds like black t-shirts or dark mode!) – horizontal and stacked… in .eps and .svg. ## Some general guidelines for usage • Don’t alter the logos/graphics: resize, reformat, recolor. Keep them intact. • Don’t separate the word mark (Velero) from the icon) – we are still building a strong name and identity – and the logo by itself doesn’t have any strong recognition or association with as yet: so best practice keep the two together. Nike kept its name with the swoosh for quite some time before the swoosh became iconic. • Don’t append the name to another brand – let it stand alone! ================================================ FILE: changelogs/CHANGELOG-0.10.md ================================================ - [v0.10.2](#v0102) - [v0.10.1](#v0101) - [v0.10.0](#v0100) ## v0.10.2 #### 2019-02-28 ### Download - https://github.com/heptio/ark/releases/tag/v0.10.2 ### Changes * upgrade restic to v0.9.4 & replace --hostname flag with --host (#1156, @skriss) * use 'restic stats' instead of 'restic check' to determine if repo exists (#1171, @skriss) * Fix concurrency bug in code ensuring restic repository exists (#1235, @skriss) ## v0.10.1 #### 2019-01-10 ### Download - https://github.com/heptio/ark/releases/tag/v0.10.1 ### Changes * Fix minio setup job command (#1118, @acbramley) * Add debugging-install link in doc get-started.md (#1131, @hex108) * `ark version`: show full git SHA & combine git tree state indicator with git SHA line (#1124, @skriss) * Delete spec.priority in pod restore action (#879, @mwieczorek) * Allow to use AWS Signature v1 for creating signed AWS urls (#811, @bashofmann) * add multizone/regional support to gcp (#765, @wwitzel3) * Fixed the newline output when deleting a schedule. (#1120, @jwhitcraft) * Remove obsolete make targets and rename 'make goreleaser' to 'make release' (#1114, @skriss) * Update to go 1.11 (#1069, @gliptak) * Update CHANGELOGs (#1063, @wwitzel3) * Initialize empty schedule metrics on server init (#1054, @cbeneke) * Added brew reference (#1051, @omerlh) * Remove default token from all service accounts (#1048, @ncdc) * Add pprof support to the Ark server (#234, @ncdc) ## v0.10.0 #### 2018-11-15 ### Download - https://github.com/heptio/ark/releases/tag/v0.10.0 ### Highlights - We've introduced two new custom resource definitions, `BackupStorageLocation` and `VolumeSnapshotLocation`, that replace the `Config` CRD from previous versions. As part of this, you may now configure more than one possible location for where backups and snapshots are stored, and when you create a `Backup` you can select the location where you'd like that particular backup to be stored. See the [Locations documentation][2] for an overview of this feature. - Ark's plugin system has been significantly refactored to improve robustness and ease of development. Plugin processes are now automatically restarted if they unexpectedly terminate. Additionally, plugin binaries can now contain more than one plugin implementation (e.g. and object store *and* a block store, or many backup item actions). - The sync process, which ensures that Backup custom resources exist for each backup in object storage, has been revamped to run much more frequently (once per minute rather than once per hour), to use significantly fewer cloud provider API calls, and to not generate spurious Kubernetes API errors. - Ark can now be configured to store all data under a prefix within an object storage bucket. This means that you no longer need a separate bucket per Ark instance; you can now have all of your clusters' Ark backups go into a single bucket, with each cluster having its own prefix/subdirectory within that bucket. - Restic backup data is now automatically stored within the same bucket/prefix as the rest of the Ark data. A separate bucket is no longer required (or allowed). - Ark resources (backups, restores, schedules) can now be bulk-deleted through the `ark` CLI, using the `--all` or `--selector` flags, or by specifying multiple resource names as arguments to the `delete` commands. - The `ark` CLI now supports waiting for backups and restores to complete with the `--wait` flag for `ark backup create` and `ark restore create` - Restores can be created directly from the most recent backup for a schedule, using `ark restore create --from-schedule SCHEDULE_NAME` ### Breaking Changes Heptio Ark v0.10 contains a number of breaking changes. Upgrading will require some additional steps beyond just updating your client binary and your container image tag. We've provided a [detailed set of instructions][1] to help you with the upgrade process. **Please read and follow these instructions carefully to ensure a successful upgrade!** - The `Config` CRD has been replaced by `BackupStorageLocation` and `VolumeSnapshotLocation` CRDs. - The interface for external plugins (object/block stores, backup/restore item actions) has changed. If you have authored any custom plugins, they'll need to be updated for v0.10. - The [`ObjectStore.ListCommonPrefixes`](https://github.com/vmware-tanzu/velero/blob/main/pkg/cloudprovider/object_store.go#L50) signature has changed to add a `prefix` parameter. - Registering plugins has changed. Create a new plugin server with the `NewServer` function, and register plugins with the appropriate functions. See the [`Server`](https://github.com/vmware-tanzu/velero/blob/main/pkg/plugin/server.go#L37) interface for details. - The organization of Ark data in object storage has changed. Existing data will need to be moved around to conform to the new layout. ### All Changes - [b9de44ff](https://github.com/heptio/ark/commit/b9de44ff) update docs to reference config/ dir within release tarballs - [eace0255](https://github.com/heptio/ark/commit/eace0255) goreleaser: update example image tags to match version being released - [cff02159](https://github.com/heptio/ark/commit/cff02159) add rbac content, rework get-started for NodePort and publicUrl, add versioning information - [fa14255e](https://github.com/heptio/ark/commit/fa14255e) add content for docs issue 819 - [22959071](https://github.com/heptio/ark/commit/22959071) add doc explaining locations - [e5556fe6](https://github.com/heptio/ark/commit/e5556fe6) Added qps and burst to server's client - [9ae861c9](https://github.com/heptio/ark/commit/9ae861c9) Support a separate URL base for pre-signed URLs - [698420b6](https://github.com/heptio/ark/commit/698420b6) Update storage-layout-reorg-v0.10.md - [6c9e1f18](https://github.com/heptio/ark/commit/6c9e1f18) lower some noisy logs to debug level - [318fd8a8](https://github.com/heptio/ark/commit/318fd8a8) add troubleshooting for loadbalancer restores - [defb8aa8](https://github.com/heptio/ark/commit/defb8aa8) remove code that checks directly for a backup from restore controller - [7abe1156](https://github.com/heptio/ark/commit/7abe1156) Move clearing up of metadata before plugin's actions - [ec013e6f](https://github.com/heptio/ark/commit/ec013e6f) Document upgrading plugins in the deployment - [d6162e94](https://github.com/heptio/ark/commit/d6162e94) fix goreleaser bugs - [a15df276](https://github.com/heptio/ark/commit/a15df276) Add correct link and change role - [46bed015](https://github.com/heptio/ark/commit/46bed015) add 0.10 breaking changes warning to readme in main - [e3a7d6a2](https://github.com/heptio/ark/commit/e3a7d6a2) add content for issue 994 - [400911e9](https://github.com/heptio/ark/commit/400911e9) address docs issue #978 - [b818cc27](https://github.com/heptio/ark/commit/b818cc27) don't require a default provider VSL if there's only 1 - [90638086](https://github.com/heptio/ark/commit/90638086) v0.10 changelog - [6e2166c4](https://github.com/heptio/ark/commit/6e2166c4) add docs page on versions and upgrading - [18b434cb](https://github.com/heptio/ark/commit/18b434cb) goreleaser scripts for building/creating a release on a workstation - [bb65d67a](https://github.com/heptio/ark/commit/bb65d67a) update restic prerequisite with min k8s version - [b5a2ccd5](https://github.com/heptio/ark/commit/b5a2ccd5) Silence git detached HEAD advice in build container - [67749141](https://github.com/heptio/ark/commit/67749141) instructions for upgrading to v0.10 - [516422c2](https://github.com/heptio/ark/commit/516422c2) sync controller: fill in missing .spec.storageLocation - [195e6aaf](https://github.com/heptio/ark/commit/195e6aaf) fix bug preventing PV snapshots from v0.10 backups from restoring - [bca58516](https://github.com/heptio/ark/commit/bca58516) Run 'make update' to update formatting - [573ce7d0](https://github.com/heptio/ark/commit/573ce7d0) Update formatting script - [90d9be59](https://github.com/heptio/ark/commit/90d9be59) support restoring/deleting legacy backups with .status.volumeBackups - [ef194972](https://github.com/heptio/ark/commit/ef194972) rename variables #967 - [6d4e702c](https://github.com/heptio/ark/commit/6d4e702c) fix broken link - [596eea1b](https://github.com/heptio/ark/commit/596eea1b) restore storageclasses before pvs and pvcs - [f014cab1](https://github.com/heptio/ark/commit/f014cab1) backup describer: show snapshot summary by default, details optionally - [8acc66d0](https://github.com/heptio/ark/commit/8acc66d0) remove pvProviderExists param from NewRestoreController - [57ce590f](https://github.com/heptio/ark/commit/57ce590f) create a struct for multiple return of same type in restore_contoroller #967 - [028fafb6](https://github.com/heptio/ark/commit/028fafb6) Corrected grammatical error - [db856aff](https://github.com/heptio/ark/commit/db856aff) Specify return arguments - [9952dfb0](https://github.com/heptio/ark/commit/9952dfb0) Address #424: Add CRDs to list of prioritized resources - [cf2c2714](https://github.com/heptio/ark/commit/cf2c2714) fix bugs in GetBackupVolumeSnapshots and add test - [ec124673](https://github.com/heptio/ark/commit/ec124673) remove all references to Config from docs/examples - [c36131a0](https://github.com/heptio/ark/commit/c36131a0) remove Config-related code - [406b50a7](https://github.com/heptio/ark/commit/406b50a7) update restore process using snapshot locations - [268080ad](https://github.com/heptio/ark/commit/268080ad) avoid panics if can't get block store during deletion - [4a03370f](https://github.com/heptio/ark/commit/4a03370f) update backup deletion controller for snapshot locations - [38c72b8c](https://github.com/heptio/ark/commit/38c72b8c) include snapshot locations in created schedule's backup spec - [0ec2de55](https://github.com/heptio/ark/commit/0ec2de55) azure: update blockstore to allow storing snaps in different resource group - [35bb533c](https://github.com/heptio/ark/commit/35bb533c) close gzip writer before uploading volumesnapshots file - [da9ed38c](https://github.com/heptio/ark/commit/da9ed38c) store volume snapshot info as JSON in backup storage - [e24248e0](https://github.com/heptio/ark/commit/e24248e0) add --volume-snapshot-locations flag to ark backup create - [df07b7dc](https://github.com/heptio/ark/commit/df07b7dc) update backup code to work with volume snapshot locations - [4af89fa8](https://github.com/heptio/ark/commit/4af89fa8) add unit test for getDefaultVolumeSnapshotLocations - [02f50b9c](https://github.com/heptio/ark/commit/02f50b9c) add default-volume-snapshot-locations to server cmd - [1aa712d2](https://github.com/heptio/ark/commit/1aa712d2) Default and validate VolumeSnapshotLocations - [bbf76985](https://github.com/heptio/ark/commit/bbf76985) add create CLI command for snapshot locations - [aeb221ea](https://github.com/heptio/ark/commit/aeb221ea) Add printer for snapshot locations - [ffc612ac](https://github.com/heptio/ark/commit/ffc612ac) Add volume snapshot CLI get command - [f20342aa](https://github.com/heptio/ark/commit/f20342aa) Add VolumeLocation and Snapshot. - [7172db8a](https://github.com/heptio/ark/commit/7172db8a) upgrade to restic v0.9.3 - [99adc4fa](https://github.com/heptio/ark/commit/99adc4fa) Remove broken references to docs that are not existing - [474efde6](https://github.com/heptio/ark/commit/474efde6) Fixed relative link for image - [41735154](https://github.com/heptio/ark/commit/41735154) don't require a default backup storage location to exist - [0612c5de](https://github.com/heptio/ark/commit/0612c5de) templatize error message in DeleteOptions - [66bcbc05](https://github.com/heptio/ark/commit/66bcbc05) add support for bulk deletion to ark schedule delete - [3af43b49](https://github.com/heptio/ark/commit/3af43b49) add azure-specific code to support multi-location restic - [d009163b](https://github.com/heptio/ark/commit/d009163b) update restic to support multiple backup storage locations - [f4c99c77](https://github.com/heptio/ark/commit/f4c99c77) Change link for the support matrix - [91e45d56](https://github.com/heptio/ark/commit/91e45d56) Fix broken storage providers link - [ed0eb865](https://github.com/heptio/ark/commit/ed0eb865) fix backup storage location example YAMLs - [eb709b8f](https://github.com/heptio/ark/commit/eb709b8f) only sync a backup location if it's changed since last sync - [af3af1b5](https://github.com/heptio/ark/commit/af3af1b5) clarify Azure resource group usage in docs - [9fdf8513](https://github.com/heptio/ark/commit/9fdf8513) Minor code cleanup - [2073e15a](https://github.com/heptio/ark/commit/2073e15a) Fix formatting for live site - [0fc3e8d8](https://github.com/heptio/ark/commit/0fc3e8d8) add documentation on running Ark on-premises - [e46e89cb](https://github.com/heptio/ark/commit/e46e89cb) have restic share main Ark bucket - [42b54586](https://github.com/heptio/ark/commit/42b54586) refactor to make valid dirs part of an object store layout - [8bc7e4f6](https://github.com/heptio/ark/commit/8bc7e4f6) store backups & restores in backups/, restores/ subdirs in obj storage - [e3232b7e](https://github.com/heptio/ark/commit/e3232b7e) add support for bulk deletion to ark restore delete - [17be71e1](https://github.com/heptio/ark/commit/17be71e1) remove deps used for docs gen - [20635106](https://github.com/heptio/ark/commit/20635106) remove script for generating docs - [6fd9ea9d](https://github.com/heptio/ark/commit/6fd9ea9d) remove cli reference docs and related scripts - [4833607a](https://github.com/heptio/ark/commit/4833607a) Fix infinite sleep in fsfreeze container - [7668bfd4](https://github.com/heptio/ark/commit/7668bfd4) Add links for Portworx plugin support - [468006e6](https://github.com/heptio/ark/commit/468006e6) Fix Portworx name in doc - [e6b44539](https://github.com/heptio/ark/commit/e6b44539) Make fsfreeze image building consistent - [fcd27a13](https://github.com/heptio/ark/commit/fcd27a13) get a new metadata accessor after calling backup item actions - [ffef86e3](https://github.com/heptio/ark/commit/ffef86e3) Adding support for the AWS_CLUSTER_NAME env variable allowing to claim volumes ownership - [cda3dff8](https://github.com/heptio/ark/commit/cda3dff8) Document single binary plugins - [f049e078](https://github.com/heptio/ark/commit/f049e078) Remove ROADMAP.md, update ZenHub link to Ark board - [94617b30](https://github.com/heptio/ark/commit/94617b30) convert all controllers to use genericController, logContext -> log - [779cb428](https://github.com/heptio/ark/commit/779cb428) Document SignatureDoesNotMatch error and triaging - [7d8813a9](https://github.com/heptio/ark/commit/7d8813a9) move ObjectStore mock into pkg/cloudprovider/mocks - [f0edf733](https://github.com/heptio/ark/commit/f0edf733) add a BackupStore to pkg/persistence that supports prefixes - [af64069d](https://github.com/heptio/ark/commit/af64069d) create pkg/persistence and move relevant code from pkg/cloudprovider into it - [29d75d72](https://github.com/heptio/ark/commit/29d75d72) move object and block store interfaces to their own files - [211aa7b7](https://github.com/heptio/ark/commit/211aa7b7) Set schedule labels to subsequent backups - [d34994cb](https://github.com/heptio/ark/commit/d34994cb) set azure restic env vars based on default backup location's config - [a50367f1](https://github.com/heptio/ark/commit/a50367f1) Regenerate CLI docs - [7bc27bbb](https://github.com/heptio/ark/commit/7bc27bbb) Pin cobra version - [e94277ac](https://github.com/heptio/ark/commit/e94277ac) Update pflag version - [df69b274](https://github.com/heptio/ark/commit/df69b274) azure: update documentation and examples - [cb321db2](https://github.com/heptio/ark/commit/cb321db2) azure: refactor to not use helpers/ pkg, validate all env/config inputs - [9d7ea748](https://github.com/heptio/ark/commit/9d7ea748) azure: support different RGs/storage accounts per backup location - [cd4e9f53](https://github.com/heptio/ark/commit/cd4e9f53) azure: fix for breaking change in blob.GetSASURI - [a440029c](https://github.com/heptio/ark/commit/a440029c) bump Azure SDK version and include storage mgmt package - [b31e25bf](https://github.com/heptio/ark/commit/b31e25bf) server: remove unused code, replace deprecated func - [729d7339](https://github.com/heptio/ark/commit/729d7339) controllers: take a newPluginManager func in constructors - [6445dbf1](https://github.com/heptio/ark/commit/6445dbf1) Update examples and docs for backup locations - [133dc185](https://github.com/heptio/ark/commit/133dc185) backup sync: process the default location first - [7a1e6d16](https://github.com/heptio/ark/commit/7a1e6d16) generic controller: allow controllers with only a resync func - [6f7bfe54](https://github.com/heptio/ark/commit/6f7bfe54) remove Config CRD's BackupStorageProvider & other obsolete code - [bd4d97b9](https://github.com/heptio/ark/commit/bd4d97b9) move server's defaultBackupLocation into config struct - [0e94fa37](https://github.com/heptio/ark/commit/0e94fa37) update sync controller for backup locations - [2750aa71](https://github.com/heptio/ark/commit/2750aa71) Use backup storage location during restore - [20f89fbc](https://github.com/heptio/ark/commit/20f89fbc) use the default backup storage location for restic - [833a6307](https://github.com/heptio/ark/commit/833a6307) Add storage location to backup get/describe - [cf7c8587](https://github.com/heptio/ark/commit/cf7c8587) download request: fix setting of log level for plugin manager - [3234124a](https://github.com/heptio/ark/commit/3234124a) backup deletion: fix setting of log level in plugin manager - [74043ab4](https://github.com/heptio/ark/commit/74043ab4) download request controller: fix bug in determining expiration - [7007f198](https://github.com/heptio/ark/commit/7007f198) refactor download request controller test and add test cases - [8f534615](https://github.com/heptio/ark/commit/8f534615) download request controller: use backup location for object store - [bab08ed1](https://github.com/heptio/ark/commit/bab08ed1) backup deletion controller: use backup location for object store - [c6f488f7](https://github.com/heptio/ark/commit/c6f488f7) Use backup location in the backup controller - [06b5af44](https://github.com/heptio/ark/commit/06b5af44) add create and get CLI commands for backup locations - [adbcd370](https://github.com/heptio/ark/commit/adbcd370) add --default-backup-storage-location flag to server cmd - [2a34772e](https://github.com/heptio/ark/commit/2a34772e) Add --storage-location argument to create commands - [56f16170](https://github.com/heptio/ark/commit/56f16170) Correct metadata for BackupStorageLocationList - [345c3c39](https://github.com/heptio/ark/commit/345c3c39) Generate clients for BackupStorageLocation - [a25eb032](https://github.com/heptio/ark/commit/a25eb032) Add BackupStorageLocation API type - [575c4ddc](https://github.com/heptio/ark/commit/575c4ddc) apply annotations on single line, no restore mode - [030ea6c0](https://github.com/heptio/ark/commit/030ea6c0) minor word updates and command wrapping - [d32f8dbb](https://github.com/heptio/ark/commit/d32f8dbb) Update hooks/fsfreeze example - [342a1c64](https://github.com/heptio/ark/commit/342a1c64) add an ark bug command - [9c11ba90](https://github.com/heptio/ark/commit/9c11ba90) Add DigitalOcean to S3-compatible backup providers - [ea50ebf2](https://github.com/heptio/ark/commit/ea50ebf2) Fix map merging logic - [9508e4a2](https://github.com/heptio/ark/commit/9508e4a2) Switch Config CRD elements to server flags - [0c3ac67b](https://github.com/heptio/ark/commit/0c3ac67b) start using a namespaced label on restored objects, deprecate old label - [6e53aa03](https://github.com/heptio/ark/commit/6e53aa03) Bring back 'make local' - [5acccaa7](https://github.com/heptio/ark/commit/5acccaa7) add bulk deletion support to ark backup delete - [3aa241a7](https://github.com/heptio/ark/commit/3aa241a7) Preserve node ports during restore when annotations hold specification. - [c5f5862c](https://github.com/heptio/ark/commit/c5f5862c) Add --wait support to ark backup create - [eb6f742b](https://github.com/heptio/ark/commit/eb6f742b) Document CRD not found errors - [fb4d507c](https://github.com/heptio/ark/commit/fb4d507c) Extend doc about synchronization - [e7bb5926](https://github.com/heptio/ark/commit/e7bb5926) Add --wait support to `ark restore create` - [8ce513ac](https://github.com/heptio/ark/commit/8ce513ac) Only delete unused backup if they are complete - [1c26fbde](https://github.com/heptio/ark/commit/1c26fbde) remove SnapshotService, replace with direct BlockStore usage - [13051218](https://github.com/heptio/ark/commit/13051218) Refactor plugin management - [74dbf387](https://github.com/heptio/ark/commit/74dbf387) Add restore failed phase and metrics - [8789ae5c](https://github.com/heptio/ark/commit/8789ae5c) update testify to latest released version - [fe9d61a9](https://github.com/heptio/ark/commit/fe9d61a9) Add schedule command info to quickstart - [ca5656c2](https://github.com/heptio/ark/commit/ca5656c2) fix bug preventing backup item action item updates from saving - [d2e629f5](https://github.com/heptio/ark/commit/d2e629f5) Delete backups from etcd if they're not in storage - [625ba481](https://github.com/heptio/ark/commit/625ba481) Fix ZenHub link on Readme.md - [dcae6eb0](https://github.com/heptio/ark/commit/dcae6eb0) Update gcp-config.md - [06d6665a](https://github.com/heptio/ark/commit/06d6665a) check s3URL scheme upon AWS ObjectStore Init() - [cc359f6e](https://github.com/heptio/ark/commit/cc359f6e) Add contributor docs for our ZenHub usage - [f6204562](https://github.com/heptio/ark/commit/f6204562) cleanup service account action log statement - [450fa72f](https://github.com/heptio/ark/commit/450fa72f) Initialize schedule Prometheus metrics to have them created beforehand (see https://prometheus.io/docs/practices/instrumentation/#avoid-missing-metrics) - [39c4267a](https://github.com/heptio/ark/commit/39c4267a) Clarify that object storage should per-cluster - [78cbdf95](https://github.com/heptio/ark/commit/78cbdf95) delete old deletion requests for backup when processing a new one - [85a61b8e](https://github.com/heptio/ark/commit/85a61b8e) return nil error if 404 encountered when deleting snapshots - [a2a7dbda](https://github.com/heptio/ark/commit/a2a7dbda) fix tagging latest by using make's ifeq - [b4a52e45](https://github.com/heptio/ark/commit/b4a52e45) Add commands for context to the bug template - [3efe6770](https://github.com/heptio/ark/commit/3efe6770) Update Ark library code to work with Kubernetes 1.11 - [7e8c8c69](https://github.com/heptio/ark/commit/7e8c8c69) Add some basic troubleshooting commands - [d1955120](https://github.com/heptio/ark/commit/d1955120) require namespace for backups/etc. to exist at server startup - [683f7afc](https://github.com/heptio/ark/commit/683f7afc) switch to using .status.startTimestamp for sorting backups - [b71a37db](https://github.com/heptio/ark/commit/b71a37db) Record backup completion time before uploading - [217084cd](https://github.com/heptio/ark/commit/217084cd) Add example ark version command to issue templates - [040788bb](https://github.com/heptio/ark/commit/040788bb) Add minor improvements and aws exampledelimitMateCR - [5b89f7b6](https://github.com/heptio/ark/commit/5b89f7b6) Skip backup sync if it already exists in k8s - [c6050845](https://github.com/heptio/ark/commit/c6050845) restore controller: switch to 'c' for receiver name - [706ae07d](https://github.com/heptio/ark/commit/706ae07d) enable a schedule to be provided as the source for a restore - [aea68414](https://github.com/heptio/ark/commit/aea68414) fix up Slack link in troubleshooting on main branch - [bb8e2e91](https://github.com/heptio/ark/commit/bb8e2e91) Document how to run the Ark server locally - [dc84e591](https://github.com/heptio/ark/commit/dc84e591) Remove outdated namespace deletion content - [23abbc9a](https://github.com/heptio/ark/commit/23abbc9a) fix paths - [f0426538](https://github.com/heptio/ark/commit/f0426538) use posix-compliant conditional for checking TAG_LATEST - [cf336d80](https://github.com/heptio/ark/commit/cf336d80) Added new templates - [795dc262](https://github.com/heptio/ark/commit/795dc262) replace pkg/restore's osFileSystem with pkg/util/filesystem's - [eabef085](https://github.com/heptio/ark/commit/eabef085) Update generated Ark code based on the 1.11 k8s.io/code-generator script - [f5eac0b4](https://github.com/heptio/ark/commit/f5eac0b4) Update vendored library code for Kubernetes 1.11 [1]: https://heptio.github.io/velero/v0.10.0/upgrading-to-v0.10 [2]: locations.md ================================================ FILE: changelogs/CHANGELOG-0.11.md ================================================ ## v0.11.1 #### 2019-05-17 ### Download - https://github.com/heptio/velero/releases/tag/v0.11.1 ### Highlights * Added the `velero migrate-backups` command to migrate legacy Ark backup metadata to the current Velero format in object storage. This command needs to be run in preparation for upgrading to v1.0, **if** you have backups that were originally created prior to v0.11 (i.e. when the project was named Ark). ## v0.11.0 #### 2019-02-28 ### Download - https://github.com/heptio/velero/releases/tag/v0.11.0 ### Highlights * Heptio Ark is now Velero! This release is the first one to use the new name. For details on the changes and how to migrate to v0.11, see the [migration instructions][1]. **Please follow the instructions to ensure a successful upgrade to v0.11.** * Restic has been upgraded to v0.9.4, which brings significantly faster restores thanks to a new multi-threaded restorer. * Velero now waits for terminating namespaces and persistent volumes to delete before attempting to restore them, rather than trying and failing to restore them while they're being deleted. ### All Changes * Fix concurrency bug in code ensuring restic repository exists (#1235, @skriss) * Wait for PVs and namespaces to delete before attempting to restore them. (#826, @nrb) * Set the zones for GCP regional disks on restore. This requires the `compute.zones.get` permission on the GCP serviceaccount in order to work correctly. (#1200, @nrb) * Renamed Heptio Ark to Velero. Changed internal imports, environment variables, and binary name. (#1184, @nrb) * use 'restic stats' instead of 'restic check' to determine if repo exists (#1171, @skriss) * upgrade restic to v0.9.4 & replace --hostname flag with --host (#1156, @skriss) * Clarify restore log when object unchanged (#1153, @daved) * Add backup-version file in backup tarball. (#1117, @wwitzel3) * add ServerStatusRequest CRD and show server version in `ark version` output (#1116, @skriss) [1]: https://heptio.github.io/velero/v0.11.0/migrating-to-velero ================================================ FILE: changelogs/CHANGELOG-0.3.md ================================================ - [v0.3.3](#v033) - [v0.3.2](#v032) - [v0.3.1](#v031) - [v0.3.0](#v030) ## v0.3.3 #### 2017-08-10 ### Download - https://github.com/heptio/ark/tree/v0.3.3 ### Bug Fixes * Treat the first field in a schedule's cron expression as minutes, not seconds ## v0.3.2 #### 2017-08-07 ### Download - https://github.com/heptio/ark/tree/v0.3.2 ### New Features * Add client-go auth provider plugins for Azure, GCP, OIDC ## v0.3.1 #### 2017-08-03 ### Download - https://github.com/heptio/ark/tree/v0.3.1 ### Bug Fixes * Fix Makefile VERSION ## v0.3.0 #### 2017-08-03 ### Download - https://github.com/heptio/ark/tree/v0.3.0 ### All New Features * Initial Release ================================================ FILE: changelogs/CHANGELOG-0.4.md ================================================ - [v0.4.0](#v040) ## v0.4.0 #### 2017-09-14 ### Download - https://github.com/heptio/ark/tree/v0.4.0 ### Breaking changes * Snapshotting and restoring volumes is now enabled by default * The --namespaces flag for 'ark restore create' has been replaced by --include-namespaces and --exclude-namespaces ### New features * Support for S3 SSE with KMS * Cloud provider configurations are validated at startup * The persistentVolumeProvider is now optional * Restore objects are garbage collected * Each backup now has an associated log file, viewable via 'ark backup logs' * Each restore now has an associated log file, viewable via 'ark restore logs' * Add --include-resources/--exclude-resources for restores ### Bug fixes * Only save/use iops for io1 volumes on AWS * When restoring, try to retrieve the Backup directly from object storage if it's not found * When syncing Backups from object storage to Kubernetes, don't return at the first error encountered * More closely match how kubectl performs kubeconfig resolution * Increase default Azure API request timeout to 2 minutes * Update Azure diskURI to match diskName ================================================ FILE: changelogs/CHANGELOG-0.5.md ================================================ - [v0.5.1](#v051) - [v0.5.0](#v050) ## v0.5.1 #### 2017-11-06 ### Download - https://github.com/heptio/ark/tree/v0.5.1 ### Bug fixes * If a Service is headless, retain ClusterIP = None when backing up and restoring. * Use the specified --label-selector when listing backups, schedules, and restores. * Restore namespace mapping functionality that was accidentally broken in 0.5.0. * Always include namespaces in the backup, regardless of the --include-cluster-resources setting. ## v0.5.0 #### 2017-10-26 ### Download - https://github.com/heptio/ark/tree/v0.5.0 ### Breaking changes * The backup tar file format has changed. Backups created using previous versions of Ark cannot be restored using v0.5.0. * When backing up one or more specific namespaces, cluster-scoped resources are no longer backed up by default, with the exception of PVs that are used within the target namespace(s). Cluster-scoped resources can still be included by explicitly specifying `--include-cluster-resources`. ### New features * Add customized user-agent string for Ark CLI * Switch from glog to logrus * Exclude nodes from restoration * Add a FAQ * Record PV availability zone and use it when restoring volumes from snapshots * Back up the PV associated with a PVC * Add `--include-cluster-resources` flag to `ark backup create` * Add `--include-cluster-resources` flag to `ark restore create` * Properly support resource restore priorities across cluster-scoped and namespace-scoped resources * Support `ark create ...` and `ark get ...` * Make ark run as cluster-admin * Add pod exec backup hooks * Support cross-compilation & upgrade to go 1.9 ### Bug fixes * Make config change detection more robust ================================================ FILE: changelogs/CHANGELOG-0.6.md ================================================ - [v0.6.0](#v060) ## v0.6.0 #### 2017-11-30 ### Download - https://github.com/heptio/ark/tree/v0.6.0 ### Highlights * **Plugins** - We now support user-defined plugins that can extend Ark functionality to meet your custom backup/restore needs without needing to be compiled into the core binary. We support pluggable block and object stores as well as per-item backup and restore actions that can execute arbitrary logic, including modifying the items being backed up or restored. For more information see the [documentation](docs/plugins.md), which includes a reference to a fully-functional sample plugin repository. (#174 #188 #206 #213 #215 #217 #223 #226) * **Describers** - The Ark CLI now includes `describe` commands for `backups`, `restores`, and `schedules` that provide human-friendly representations of the relevant API objects. ### Breaking Changes * The config object format has changed. In order to upgrade to v0.6.0, the config object will have to be updated to match the new format. See the [examples](examples) and [documentation](docs/config-definition.md) for more information. * The restore object format has changed. The `warnings` and `errors` fields are now ints containing the counts, while full warnings and errors are now stored in the object store instead of etcd. Restore objects created prior to v.0.6.0 should be deleted, or a new bucket used, and the old restore objects deleted from Kubernetes (`kubectl -n heptio-ark delete restore --all`). ### All New Features * Add `ark plugin add` and `ark plugin remove` commands #217, @skriss * Add plugin support for block/object stores, backup/restore item actions #174 #188 #206 #213 #215 #223 #226, @skriss @ncdc * Improve Azure deployment instructions #216, @ncdc * Change default TTL for backups to 30 days #204, @nrb * Improve logging for backups and restores #199, @ncdc * Add `ark backup describe`, `ark schedule describe` #196, @ncdc * Add `ark restore describe` and move restore warnings/errors to object storage #173 #201 #202, @ncdc * Upgrade to client-go v5.0.1, kubernetes v1.8.2 #157, @ncdc * Add Travis CI support #165 #166, @ncdc ### Bug Fixes * Fix log location hook prefix stripping #222, @ncdc * When running `ark backup download`, remove file if there's an error #154, @ncdc * Update documentation for AWS KMS Key alias support #163, @lli-hiya * Remove clock from `volume_snapshot_action` #137, @athampy ================================================ FILE: changelogs/CHANGELOG-0.7.md ================================================ - [v0.7.1](#v071) - [v0.7.0](#v070) ## v0.7.1 #### 2018-02-22 ### Download - https://github.com/heptio/ark/releases/tag/v0.7.1 ### Bug Fixes: * Run the Ark server in its own namespace, separate from backups/schedules/restores/config (#322, @ncdc) ## v0.7.0 #### 2018-02-15 ### Download - https://github.com/heptio/ark/releases/tag/v0.7.0 ### New Features: * Run the Ark server in any namespace (#272, @ncdc) * Add ability to delete backups and their associated data (#252, @skriss) * Support both pre and post backup hooks (#243, @ncdc) ### Bug Fixes / Other Changes: * Switch from Update() to Patch() when updating Ark resources (#241, @skriss) * Don't fail the backup if a PVC is not bound to a PV (#256, @skriss) * Restore serviceaccounts prior to workload controllers (#258, @ncdc) * Stop removing annotations from PVs when restoring them (#263, @skriss) * Update GCP client libraries (#249, @skriss) * Clarify backup and restore creation messages (#270, @nrb) * Update S3 bucket creation docs for us-east-1 (#285, @lypht) ================================================ FILE: changelogs/CHANGELOG-0.8.md ================================================ - [v0.8.3](#v083) - [v0.8.2](#v082) - [v0.8.1](#v081) - [v0.8.0](#v080) ## v0.8.3 #### 2018-06-29 ### Download - https://github.com/heptio/ark/releases/tag/v0.8.3 ### Bug Fixes: * Don't restore backup and restore resources to avoid possible data corruption (#622, @ncdc) ## v0.8.2 #### 2018-06-01 ### Download - https://github.com/heptio/ark/releases/tag/v0.8.2 ### Bug Fixes: * Don't crash when a persistent volume claim is missing spec.volumeName (#520, @ncdc) ## v0.8.1 #### 2018-04-23 ### Download - https://github.com/heptio/ark/releases/tag/v0.8.1 ### Bug Fixes: * Azure: allow pre-v0.8.0 backups with disk snapshots to be restored and deleted (#446 #449, @skriss) ## v0.8.0 #### 2018-04-19 ### Download - https://github.com/heptio/ark/releases/tag/v0.8.0 ### Highlights: * Backup deletion has been completely revamped to make it simpler and less error-prone. As a user, you still use the `ark backup delete` command to request deletion of a backup and its associated cloud resources; behind the scenes, we've switched to using a new `DeleteBackupRequest` Custom Resource and associated controller for processing deletion requests. * We've reduced the number of required fields in the Ark config. For Azure, `location` is no longer required, and for GCP, `project` is not needed. * Ark now copies tags from volumes to snapshots during backup, and from snapshots to new volumes during restore. ### Breaking Changes: * Ark has moved back to a single namespace (`heptio-ark` by default) as part of #383. ### All New Features: * Add global `--kubecontext` flag to Ark CLI (#296, @blakebarnett) * Azure: support cross-resource group restores of volumes (#356 #378, @skriss) * AWS/Azure/GCP: copy tags from volumes to snapshots, and from snapshots to volumes (#341, @skriss) * Replace finalizer for backup deletion with `DeleteBackupRequest` custom resource & controller (#383 #431, @ncdc @nrb) * Don't log warnings during restore if an identical object already exists in the cluster (#405, @nrb) * Add bash & zsh completion support (#384, @containscafeine) ### Bug Fixes / Other Changes: * Error from the Ark CLI if attempting to restore a non-existent backup (#302, @ncdc) * Enable running the Ark server locally for development purposes (#334, @ncdc) * Add examples to `ark schedule create` documentation (#331, @lypht) * GCP: Remove `project` requirement from Ark config (#345, @skriss) * Add `--from-backup` flag to `ark restore create` and allow custom restore names (#342 #409, @skriss) * Azure: remove `location` requirement from Ark config (#344, @skriss) * Add documentation/examples for storing backups in IBM Cloud Object Storage (#321, @roytman) * Reduce verbosity of hooks logging (#362, @skriss) * AWS: Add minimal IAM policy to documentation (#363 #419, @hopkinsth) * Don't restore events (#374, @sanketjpatel) * Azure: reduce API polling interval from 60s to 5s (#359, @skriss) * Switch from hostPath to emptyDir volume type for minio example (#386, @containscafeine) * Add limit ranges as a prioritized resource for restores (#392, @containscafeine) * AWS: Add documentation on using Ark with kube2iam (#402, @domderen) * Azure: add node selector so Ark pod is scheduled on a linux node (#415, @ffd2subroutine) * Error from the Ark CLI if attempting to get logs for a non-existent restore (#391, @containscafeine) * GCP: Add minimal IAM policy to documentation (#429, @skriss @jody-frankowski) ### Upgrading from v0.7.1: Ark v0.7.1 moved the Ark server deployment into a separate namespace, `heptio-ark-server`. As of v0.8.0 we've returned to a single namespace, `heptio-ark`, for all Ark-related resources. If you're currently running v0.7.1, here are the steps you can take to upgrade: 1. Execute the steps from the **Credentials and configuration** section for your cloud: * [AWS](https://heptio.github.io/velero/v0.8.0/aws-config#credentials-and-configuration) * [Azure](https://heptio.github.io/velero/v0.8.0/azure-config#credentials-and-configuration) * [GCP](https://heptio.github.io/velero/v0.8.0/gcp-config#credentials-and-configuration) When you get to the secret creation step, if you don't have your `credentials-ark` file handy, you can copy the existing secret from your `heptio-ark-server` namespace into the `heptio-ark` namespace: ```bash kubectl get secret/cloud-credentials -n heptio-ark-server --export -o json | \ jq '.metadata.namespace="heptio-ark"' | \ kubectl apply -f - ``` 2. You can now safely delete the `heptio-ark-server` namespace: ```bash kubectl delete namespace heptio-ark-server ``` 3. Execute the commands from the **Start the server** section for your cloud: * [AWS](https://heptio.github.io/velero/v0.8.0/aws-config#start-the-server) * [Azure](https://heptio.github.io/velero/v0.8.0/azure-config#start-the-server) * [GCP](https://heptio.github.io/velero/v0.8.0/gcp-config#start-the-server) ================================================ FILE: changelogs/CHANGELOG-0.9.md ================================================ - [v0.9.11](#v0911) - [v0.9.10](#v0910) - [v0.9.9](#v099) - [v0.9.8](#v098) - [v0.9.7](#v097) - [v0.9.6](#v096) - [v0.9.5](#v095) - [v0.9.4](#v094) - [v0.9.3](#v093) - [v0.9.2](#v092) - [v0.9.1](#v091) - [v0.9.0](#v090) ## v0.9.11 #### 2018-11-08 ### Download - https://github.com/heptio/ark/releases/tag/v0.9.11 ### Bug Fixes * Fix bug preventing PV snapshots from being restored (#1040, @ncdc) ## v0.9.10 #### 2018-11-01 ### Download - https://github.com/heptio/ark/releases/tag/v0.9.10 ### Bug Fixes * restore storageclasses before pvs and pvcs (#594, @shubheksha) * AWS: Ensure that the order returned by ListObjects is consistent (#999, @bashofmann) * Add CRDs to list of prioritized resources (#424, @domenicrosati) * Verify PV doesn't exist before creating new volume (#609, @nrb) * Update README.md - Grammar mistake corrected (#1018, @midhunbiju) ## v0.9.9 #### 2018-10-24 ### Download - https://github.com/heptio/ark/releases/tag/v0.9.9 ### Bug Fixes * Check if initContainers key exists before attempting to remove volume mounts. (#927, @skriss) ## v0.9.8 #### 2018-10-18 ### Download - https://github.com/heptio/ark/releases/tag/v0.9.8 ### Bug Fixes * Discard service account token volume mounts from init containers on restore (#910, @james-powis) * Support --include-cluster-resources flag when creating schedule (#942, @captjt) * Remove logic to get a GCP project (#926, @shubheksha) * Only try to back up PVCs linked PV if the PVC's phase is Bound (#920, @skriss) * Claim ownership of new AWS volumes on Kubernetes cluster being restored into (#801, @ljakimczuk) * Remove timeout check when taking snapshots (#928, @carlisia) ## v0.9.7 #### 2018-10-04 ### Download - https://github.com/heptio/ark/releases/tag/v0.9.7 ### Bug Fixes * Preserve explicitly-specified node ports during restore (#712, @timoreimann) * Enable restoring resources with ownerReference set (#837, @mwieczorek) * Fix error when restoring ExternalName services (#869, @shubheksha) * remove restore log helper for accurate line numbers (#891, @skriss) * Display backup StartTimestamp in `ark backup get` output (#894, @marctc) * Fix restic restores when using namespace mappings (#900, @skriss) ## v0.9.6 #### 2018-09-21 ### Download - https://github.com/heptio/ark/releases/tag/v0.9.6 ### Bug Fixes * Discard service account tokens from non-default service accounts on restore (#843, @james-powis) * Update Docker images to use `alpine:3.8` (#852, @nrb) ## v0.9.5 #### 2018-09-17 ### Download - https://github.com/heptio/ark/releases/tag/v0.9.5 ### Bug Fixes * Fix issue causing restic restores not to work (#834, @skriss) ## v0.9.4 #### 20180-09-05 ### Download - https://github.com/heptio/ark/releases/tag/v0.9.4 ### Bug Fixes * Terminate plugin clients to resolve memory leaks (#797, @skriss) * Fix nil map errors when merging annotations (#812, @nrb) ## v0.9.3 #### 2018-08-10 ### Download - https://github.com/heptio/ark/releases/tag/v0.9.3 ### Bug Fixes * Initialize Prometheus metrics when creating a new schedule (#689, @lemaral) ## v0.9.2 #### 2018-07-26 ### Download - https://github.com/heptio/ark/releases/tag/v0.9.2) - 2018-07-26 ### Bug Fixes: * Fix issue where modifications made by backup item actions were not being saved to backup tarball (#704, @skriss) ## v0.9.1 #### 2018-07-23 ### Download - https://github.com/heptio/ark/releases/tag/v0.9.1 ### Bug Fixes: * Require namespace for Ark's CRDs to already exist at server startup (#676, @skriss) * Require all Ark CRDs to exist at server startup (#683, @skriss) * Fix `latest` tagging in Makefile (#690, @skriss) * Make Ark compatible with clusters that don't have the `rbac.authorization.k8s.io/v1` API group (#682, @nrb) * Don't consider missing snapshots an error during backup deletion, limit backup deletion requests per backup to 1 (#687, @skriss) ## v0.9.0 #### 2018-07-06 ### Download - https://github.com/heptio/ark/releases/tag/v0.9.0 ### Highlights: * Ark now has support for backing up and restoring Kubernetes volumes using a free open-source backup tool called [restic](https://github.com/restic/restic). This provides users an out-of-the-box solution for backing up and restoring almost any type of Kubernetes volume, whether or not it has snapshot support integrated with Ark. For more information, see the [documentation](https://github.com/vmware-tanzu/velero/blob/main/docs/restic.md). * Support for Prometheus metrics has been added! View total number of backup attempts (including success or failure), total backup size in bytes, and backup durations. More metrics coming in future releases! ### All New Features: * Add restic support (#508 #532 #533 #534 #535 #537 #540 #541 #545 #546 #547 #548 #555 #557 #561 #563 #569 #570 #571 #606 #608 #610 #621 #631 #636, @skriss) * Add prometheus metrics (#531 #551 #564, @ashish-amarnath @nrb) * When backing up a service account, include cluster roles/cluster role bindings that reference it (#470, @skriss) * When restoring service accounts, copy secrets/image pull secrets into the target cluster even if the service account already exists (#403, @nrb) ### Bug Fixes / Other Changes: * Upgrade to Kubernetes 1.10 dependencies (#417, @skriss) * Upgrade to go 1.10 and alpine 3.7 (#456, @skriss) * Display no excluded resources/namespaces as `` rather than `*` (#453, @nrb) * Skip completed jobs and pods when restoring (#463, @nrb) * Set namespace correctly when syncing backups from object storage (#472, @skriss) * When building on macOS, bind-mount volumes with delegated config (#478, @skriss) * Add replica sets and daemonsets to cohabiting resources so they're not backed up twice (#482 #485, @skriss) * Shut down the Ark server gracefully on SIGINT/SIGTERM (#483, @skriss) * Only back up resources that support GET and DELETE in addition to LIST and CREATE (#486, @nrb) * Show a better error message when trying to get an incomplete restore's logs (#496, @nrb) * Stop processing when setting a backup deletion request's phase to `Deleting` fails (#500, @nrb) * Add library code to install Ark's server components (#437 #506, @marpaia) * Properly handle errors when backing up additional items (#512, @carlpett) * Run post hooks even if backup actions fail (#514, @carlpett) * GCP: fail backup if upload to object storage fails (#510, @nrb) * AWS: don't require `region` as part of backup storage provider config (#455, @skriss) * Ignore terminating resources while doing a backup (#526, @yastij) * Log to stdout instead of stderr (#553, @ncdc) * Move sample minio deployment's config to an emptyDir (#566, @runyontr) * Add `omitempty` tag to optional API fields (@580, @nikhita) * Don't restore PVs with a reclaim policy of `Delete` and no snapshot (#613, @ncdc) * Don't restore mirror pods (#619, @ncdc) ### Docs Contributors: * @gianrubio * @castrojo * @dhananjaysathe * @c-knowles * @mattkelly * @ae-v * @hamidzr ================================================ FILE: changelogs/CHANGELOG-1.0.md ================================================ ## v1.0.0 #### 2019-05-20 ### Highlights - We've added a new command, `velero install`, to make it easier to get up and running with Velero. This CLI command replaces the static YAML installation files that were previously part of release tarballs. See the updated [install instructions][3] for more information. - We've made a number of improvements to the plugin framework: - we've reorganized the relevant packages to minimize the import surface for plugin authors - all plugins are now wrapped in panic handlers that will report information on panics back to Velero - Velero's `--log-level` flag is now passed to plugin implementations - Errors logged within plugins are now annotated with the file/line of where the error occurred - Restore item actions can now optionally return a list of additional related items that should be restored - Restore item actions can now indicate that an item *should not* be restored - For Azure installation, the `cloud-credentials` secret can now be created from a file containing a list of environment variables. Note that `velero install` always uses this method of providing credentials for Azure. For more details, see [Run on Azure][0]. - We've added a new phase, `PartiallyFailed`, for both backups and restores. This new phase is used for backups/restores that successfully process some but not all of their items. - We removed all legacy Ark references, including API types, prometheus metrics, restic & hook annotations, etc. - The restic integration remains a **beta feature**. Please continue to try it out and provide feedback, and we'll be working over the next couple of releases to bring it to GA. ### Breaking Changes #### API * All legacy Ark data types and pre-1.0 compatibility code has been removed. Users should migrate any backups created pre-v0.11.0 with the `velero migrate-backups` command, available in [v0.11.1][2]. #### Image * The base container image has been switched to `ubuntu:bionic` #### Labels/Annotations/Metrics * The "ark" annotations for specifying hooks are no longer supported, and have been replaced with "velero"-based equivalents. * The "ark" annotation for specifying restic backups is no longer supported, and has been replaced with a "velero"-based equivalent. * The "ark" prometheus metrics no longer exist, and have been replaced with "velero"-based equivalents. #### Plugin Development * `BlockStore` plugins are now named `VolumeSnapshotter` plugins * Plugin APIs have moved to reduce the import surface: * Plugin gRPC servers live in `github.com/heptio/velero/pkg/plugin/framework` * Plugin interface types live in `github.com/heptio/velero/pkg/plugin/velero` * RestoreItemAction interface now takes the original item from the backup as a parameter * RestoreItemAction plugins can now return additional items to restore * RestoreItemAction plugins can now skip restoring an item * Plugins may now send stack traces with errors to the Velero server, so that the errors may be put into the server log * Plugins must now be "namespaced," using `example.domain.com/plugin-name` format * For external ObjectStore and VolumeSnapshotter plugins. this name will also be the provider name in BackupStorageLoction and VolumeSnapshotLocation objects * `--log-level` flag is now passed to all plugins #### Validation * Configs for Azure, AWS, and GCP are now checked for invalid or extra keys, and the server is halted if any are found ### Download - https://github.com/heptio/velero/releases/tag/v1.0.0 ### Container Image `gcr.io/heptio-images/velero:v1.0.0` ### Documentation https://velero.io/docs/v1.0.0/ ### Upgrading To upgrade from a previous version of Velero, see our [upgrade instructions][1]. ### All Changes * Change base images to ubuntu:bionic (#1488, @skriss) * Expose the timestamp of the last successful backup in a gauge (#1448, @fabito) * check backup existence before download (#1447, @fabito) * Use `latest` image tag if no version information is provided at build time (#1439, @nrb) * switch from `restic stats` to `restic snapshots` for checking restic repository existence (#1416, @skriss) * GCP: add optional 'project' config to volume snapshot location for if snapshots are in a different project than the IAM account (#1405, @skriss) * Disallow bucket names starting with '-' (#1407, @nrb) * Shorten label values when they're longer than 63 characters (#1392, @anshulc) * Fail backup if it already exists in object storage. (#1390, @ncdc,carlisia) * Add PartiallyFailed phase for backups, log + continue on errors during backup process (#1386, @skriss) * Remove deprecated "hooks" for backups (they've been replaced by "pre hooks") (#1384, @skriss) * Restic repo ensurer: return error if new repository does not become ready within a minute, and fix channel closing/deletion (#1367, @skriss) * Support non-namespaced names for built-in plugins (#1366, @nrb) * Change container base images to debian:stretch-slim and upgrade to go 1.12 (#1365, @skriss) * Azure: allow credentials to be provided in a .env file (path specified by $AZURE_CREDENTIALS_FILE), formatted like (#1364, @skriss): ``` AZURE_TENANT_ID=${AZURE_TENANT_ID} AZURE_SUBSCRIPTION_ID=${AZURE_SUBSCRIPTION_ID} AZURE_CLIENT_ID=${AZURE_CLIENT_ID} AZURE_CLIENT_SECRET=${AZURE_CLIENT_SECRET} AZURE_RESOURCE_GROUP=${AZURE_RESOURCE_GROUP} ``` * Instantiate the plugin manager with the per-restore logger so plugin logs are captured in the per-restore log (#1358, @skriss) * Add gauge metrics for number of existing backups and restores (#1353, @fabito) * Set default TTL for backups (#1352, @vorar) * Validate that there can't be any duplicate plugin name, and that the name format is `example.io/name`. (#1339, @carlisia) * AWS/Azure/GCP: fail fast if unsupported keys are provided in BackupStorageLocation/VolumeSnapshotLocation config (#1338, @skriss) * `velero backup logs` & `velero restore logs`: show helpful error message if backup/restore does not exist or is not finished processing (#1337, @skriss) * Add support for allowing a RestoreItemAction to skip item restore. (#1336, @sseago) * Improve error message around invalid S3 URLs, and gracefully handle trailing backslashes. (#1331, @skriss) * Set backup's start timestamp before patching it to InProgress so start times display in `velero backup get` while in progress (#1330, @skriss) * Added ability to dynamically disable controllers (#1326, @amanw) * Remove deprecated code in preparation for v1.0 release (#1323, @skriss): - remove ark.heptio.com API group - remove support for reading ark-backup.json files from object storage - remove Ark field from RestoreResult type - remove support for "hook.backup.ark.heptio.com/..." annotations for specifying hooks - remove support for $HOME/.config/ark/ client config directory - remove support for restoring Azure snapshots using short snapshot ID formats in backup metadata - stop applying "velero-restore" label to restored resources and remove it from the API pkg - remove code that strips the "gc.ark.heptio.com" finalizer from backups - remove support for "backup.ark.heptio.com/..." annotations for requesting restic backups - remove "ark"-prefixed prometheus metrics - remove VolumeBackups field and related code from Backup's status * Rename BlockStore plugin to VolumeSnapshotter (#1321, @skriss) * Bump plugin ProtocolVersion to version 2 (#1319, @carlisia) * Remove Warning field from restore item action output (#1318, @skriss) * Fix for #1312, use describe to determine if AWS EBS snapshot is encrypted and explicitly pass that value in EC2 CreateVolume call. (#1316, @mstump) * Allow restic restore helper image name to be optionally specified via ConfigMap (#1311, @skriss) * Compile only once to lower the initialization cost for regexp.MustCompile. (#1306, @pei0804) * Enable restore item actions to return additional related items to be restored; have pods return PVCs and PVCs return PVs (#1304, @skriss) * Log error locations from plugin logger, and don't overwrite them in the client logger if they exist already (#1301, @skriss) * Send stack traces from plugin errors to Velero via gRPC so error location info can be logged (#1300, @skriss) * Azure: restore volumes in the original region's zone (#1298, @sylr) * Check for and exclude hostPath-based persistent volumes from restic backup (#1297, @skriss) * Make resticrepositories non-restorable resources (#1296, @skriss) * Gracefully handle failed API groups from the discovery API (#1293, @fabito) * Add `velero install` command for basic use cases. (#1287, @nrb) * Collect 3 new metrics: backup_deletion_{attempt|failure|success}_total (#1280, @fabito) * Pass --log-level flag to internal/external plugins, matching Velero server's log level (#1278, @skriss) * AWS EBS Volume IDs now contain AZ (#1274, @tsturzl) * Add panic handlers to all server-side plugin methods (#1270, @skriss) * Move all the interfaces and associated types necessary to implement all of the Velero plugins to under the new package `velero`. (#1264, @carlisia) * Update `velero restore` to not open every single file open during extraction of the data (#1261, @asaf) * Remove restore code that waits for a PV to become Available (#1254, @skriss) * Improve `describe` output * Move Phase to right under Metadata(name/namespace/label/annotations) * Move Validation errors: section right after Phase: section and only show it if the item has a phase of FailedValidation * For restores move Warnings and Errors under Validation errors. Leave their display as is. (#1248, @DheerajSShetty) * Don't remove storage class from a persistent volume when restoring it (#1246, @skriss) * Need to defer closing the the ReadCloser in ObjectStoreGRPCServer.GetObject (#1236, @DheerajSShetty) * Update Kubernetes dependencies to match v1.12, and update Azure SDK to v19.0.0 (GA) (#1231, @skriss) * Remove pkg/util/collections/map_utils.go, replace with structured API types and apimachinery's unstructured helpers (#1146, @skriss) * Add original resource (from backup) to restore item action interface (#1123, @mwieczorek) [0]: https://velero.io/docs/v1.0.0/azure-config [1]: https://velero.io/docs/v1.0.0/upgrade-to-1.0 [2]: https://github.com/heptio/velero/releases/tag/v0.11.1 [3]: https://velero.io/docs/v1.0.0/install-overview ================================================ FILE: changelogs/CHANGELOG-1.1.md ================================================ ## v1.1.0 #### 2019-08-22 ### Download - https://github.com/heptio/velero/releases/tag/v1.1.0 ### Container Image `gcr.io/heptio-images/velero:v1.1.0` ### Documentation https://velero.io/docs/v1.1.0/ ### Upgrading **If you are running Velero in a non-default namespace**, i.e. any namespace other than `velero`, manual intervention is required when upgrading to v1.1. See [upgrading to v1.1](https://velero.io/docs/v1.1.0/upgrade-to-1.1/) for details. ### Highlights #### Improved Restic Support A big focus of our work this cycle was continuing to improve support for restic. To that end, we’ve fixed the following bugs: - Prior to version 1.1, restic backups could be delayed or failed due to long-lived locks on the repository. Now, Velero removes stale locks from restic repositories every 5 minutes, ensuring they do not interrupt normal operations. - Previously, the PodVolumeBackup custom resources that represented a restic backup within a cluster were not synchronized between clusters, making it unclear what restic volumes were available to restore into a new cluster. In version 1.1, these resources are synced into clusters, so they are more visible to you when you are trying to restore volumes. - Originally, Velero would not validate the host path in which volumes were mounted on a given node. If a node did not expose the filesystem correctly, you wouldn’t know about it until a backup failed. Now, Velero’s restic server will validate that the directory structure is correct on startup, providing earlier feedback when it’s not. - Velero’s restic support is intended to work on a broad range of volume types. With the general release of the [Container Storage Interface API](https://kubernetes.io/blog/2019/01/15/container-storage-interface-ga/), Velero can now use restic to back up CSI volumes. Along with our bug fixes, we’ve provided an easier way to move restic backups between storage providers. Different providers often have different StorageClasses, requiring user intervention to make restores successfully complete. To make cross-provider moves simpler, we’ve introduced a StorageClass remapping plug-in. It allows you to automatically translate one StorageClass on PersistentVolumeClaims and PersistentVolumes to another. You can read more about it in our [documentation](https://velero.io/docs/v1.1.0/restore-reference/#changing-pv-pvc-storage-classes). #### Quality-of-Life Improvements We’ve also made several other enhancements to Velero that should benefit all users. Users sometimes ask about recommendations for Velero’s resource allocation within their cluster. To help with this concern, we’ve added default resource requirements to the Velero Deployment and restic init containers, along with configurable requests and limits for the restic DaemonSet. All these values can be adjusted if your environment requires it. We’ve also taken some time to improve Velero for the future by updating the Deployment and DaemonSet to use the apps/v1 API group, which will be the [default in Kubernetes 1.16](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.16.md#action-required-3). This change means that `velero install` and the `velero plugin` commands will require Kubernetes 1.9 or later to work. Existing Velero installs will continue to work without needing changes, however. In order to help you better understand what resources have been backed up, we’ve added a list of resources in the `velero backup describe --details` command. This change makes it easier to inspect a backup without having to download and extract it. In the same vein, we’ve added the ability to put custom tags on cloud-provider snapshots. This approach should provide a better way to keep track of the resources being created in your cloud account. To add a label to a snapshot at backup time, use the `--labels` argument in the `velero backup create` command. Our final change for increasing visibility into your Velero installation is the `velero plugin get` command. This command will report all the plug-ins within the Velero deployment.. Velero has previously used a restore-only flag on the server to control whether a cluster could write backups to object storage. With Velero 1.1, we’ve now moved the restore-only behavior into read-only BackupStorageLocations. This move means that the Velero server can use a BackupStorageLocation as a source to restore from, but not for backups, while still retaining the ability to back up to other configured locations. In the future, the `--restore-only` flag will be removed in favor of configuring read-only BackupStorageLocations. #### Community Contributions We appreciate all community contributions, whether they be pull requests, bug reports, feature requests, or just questions. With this release, we wanted to draw attention to a few contributions in particular: For users of node-based IAM authentication systems such as kube2iam, `velero install` now supports the `--pod-annotations` argument for applying necessary annotations at install time. This support should make `velero install` more flexible for scenarios that do not use Secrets for access to their cloud buckets and volumes. You can read more about how to use this new argument in our [AWS documentation](https://velero.io/docs/v1.1.0/aws-config/#alternative-setup-permissions-using-kube2iam). Huge thanks to [Traci Kamp](https://github.com/tlkamp) for this contribution. Structured logging is important for any application, and Velero is no different. Starting with version 1.1, the Velero server can now output its logs in a JSON format, allowing easier parsing and ingestion. Thank you to [Donovan Carthew](https://github.com/carthewd) for this feature. AWS supports multiple profiles for accessing object storage, but in the past Velero only used the default. With v.1.1, you can set the `profile` key on yourBackupStorageLocation to specify an alternate profile. If no profile is set, the default one is used, making this change backward compatible. Thanks [Pranav Gaikwad](https://github.com/pranavgaikwad) for this change. Finally, thanks to testing by [Dylan Murray](https://github.com/dymurray) and [Scott Seago](https://github.com/sseago), an issue with running Velero in non-default namespaces was found in our beta version for this release. If you’re running Velero in a namespace other than `velero`, please follow the [upgrade instructions](https://velero.io/docs/v1.1.0/upgrade-to-1.1/). ### All Changes * Add the prefix to BSL config map so that object stores can use it when initializing (#1767, @betta1) * Use `VELERO_NAMESPACE` to determine what namespace Velero server is running in. For any v1.0 installations using a different namespace, the `VELERO_NAMESPACE` environment variable will need to be set to the correct namespace. (#1748, @nrb) * support setting CPU/memory requests with unbounded limits using velero install (#1745, @prydonius) * sort output of resource list in `velero backup describe --details` (#1741, @prydonius) * adds the ability to define custom tags to be added to snapshots by specifying custom labels on the Backup CR with the velero backup create --labels flag (#1729, @prydonius) * Restore restic volumes from PodVolumeBackups CRs (#1723, @carlisia) * properly restore PVs backed up with restic and a reclaim policy of "Retain" (#1713, @skriss) * Make `--secret-file` flag on `velero install` optional, add `--no-secret` flag for explicit confirmation (#1699, @nrb) * Add low cpu/memory limits to the restic init container. This allows for restoration into namespaces with quotas defined. (#1677, @nrb) * Adds configurable CPU/memory requests and limits to the restic DaemonSet generated by velero install. (#1710, @prydonius) * remove any stale locks from restic repositories every 5m (#1708, @skriss) * error if backup storage location's Bucket field also contains a prefix, and gracefully handle leading/trailing slashes on Bucket and Prefix fields. (#1694, @skriss) * enhancement: allow option to choose JSON log output (#1654, @carthewd) * Adds configurable CPU/memory requests and limits to the Velero Deployment generated by velero install. (#1678, @prydonius) * Store restic PodVolumeBackups in obj storage & use that as source of truth like regular backups. (#1577, @carlisia) * Update Velero Deployment to use apps/v1 API group. `velero install` and `velero plugin add/remove` commands will now require Kubernetes 1.9+ (#1673, @nrb) * Respect the --kubecontext and --kubeconfig arguments for `velero install`. (#1656, @nrb) * add plugin for updating PV & PVC storage classes on restore based on a config map (#1621, @skriss) * Add restic support for CSI volumes (#1615, @nrb) * bug fix: Fixed namespace usage with cli command 'version' (#1630, @jwmatthews) * enhancement: allow users to specify additional Velero/Restic pod annotations on the command line with the pod-annotations flag. (#1626, @tlkamp) * adds validation for pod volumes hostPath mount on restic server startup (#1616, @prydonius) * enable support for ppc64le architecture (#1605, @prajyot) * bug fix: only restore additional items returned from restore item actions if they match the restore's namespace/resource selectors (#1612, @skriss) * add startTimestamp and completionTimestamp to PodVolumeBackup and PodVolumeRestore status fields (#1609, @prydonius) * bug fix: respect namespace selector when determining which restore item actions to run (#1607, @skriss) * ensure correct backup item actions run with namespace selector (#1601, @prydonius) * allows excluding resources from backups with the `velero.io/exclude-from-backup=true` label (#1588, @prydonius) * ensures backup item action modifications to an item's namespace/name are saved in the file path in the tarball (#1587, @prydonius) * Hides `velero server` and `velero restic server` commands from the list of available commands as these are not intended for use by the velero CLI user. (#1561, @prydonius) * remove dependency on glog, update to klog (#1559, @skriss) * move issue-template-gen from docs/ to hack/ (#1558, @skriss) * fix panic when processing DeleteBackupRequest objects without labels (#1556, @prydonius) * support for multiple AWS profiles (#1548, @pranavgaikwad) * Add CLI command to list (get) all Velero plugins (#1535, @carlisia) * Added author as a tag on blog post. Should fix 404 error when trying to follow link as specified in issue #1522. (#1522, @coonsd) * Allow individual backup storage locations to be read-only (#1517, @skriss) * Stop returning an error when a restic volume is empty since it is a valid scenario. (#1480, @carlisia) * add ability to use wildcard in includes/excludes (#1428, @guilhem) ================================================ FILE: changelogs/CHANGELOG-1.10.md ================================================ ## v1.10.0 ### 2022-11-23 ### Download https://github.com/vmware-tanzu/velero/releases/tag/v1.10.0 ### Container Image `velero/velero:v1.10.0` ### Documentation https://velero.io/docs/v1.10/ ### Upgrading https://velero.io/docs/v1.10/upgrade-to-1.10/ ### Highlights #### Unified Repository and Kopia integration In this release, we introduced the Unified Repository architecture to build a data path where data movers and the backup repository are decoupled and a unified backup repository could serve various data movement activities. In this release, we also deeply integrate Velero with Kopia, specifically, Kopia's uploader modules are isolated as a generic file system uploader; Kopia's repository modules are encapsulated as the unified backup repository. For more information, refer to the [design document](https://github.com/vmware-tanzu/velero/blob/v1.10.0/design/unified-repo-and-kopia-integration/unified-repo-and-kopia-integration.md). #### File system backup refactor Velero's file system backup (a.k.s. pod volume backup or formerly restic backup) is refactored as the first user of the Unified Repository architecture. Specifically, we added a new path, the Kopia path, besides the existing Restic path. While Restic path is still available and set as default, you can opt in Kopia path by specifying the `uploader-type` parameter at installation time. Meanwhile, you are free to restore from existing backups under either path, Velero dynamically switches to the correct path to process the restore. Because of the new path, we renamed some modules and parameters, refer to the Break Changes section for more details. For more information, visit the [file system backup document](https://velero.io/docs/v1.10/file-system-backup/) and [v1.10 upgrade guide document](https://velero.io/docs/v1.10/upgrade-to-1.10/). Meanwhile, we've created a performance guide for both Restic path and Kopia path, which helps you to choose between the two paths and provides you the best practice to configure them under different scenarios. Please note that the results in the guide are based on our testing environments, you may get different results when testing in your own ones. For more information, visit the [performance guide document](https://velero.io/docs/v1.10/performance-guidance/). #### Plugin versioning V1 refactor In this release, Velero moves plugins BackupItemAction, RestoreItemAction and VolumeSnapshotterAction to version v1, this allows future plugin changes that do not support backward compatibility, so is a preparation for various complex tasks, for example, data movement tasks. For more information, refer to the [plugin versioning design document](https://github.com/vmware-tanzu/velero/blob/v1.10.0/design/plugin-versioning.md). #### Refactor the controllers using Kubebuilder v3 In this release we continued our code modernization work, rewriting some controllers using Kubebuilder v3. This work is ongoing and we will continue to make progress in future releases. #### Add credentials to volume snapshot locations In this release, we enabled dedicate credentials options to volume snapshot locations so that you can specify credentials per volume snapshot location as same as backup storage location. For more information, please visit the [locations document](https://velero.io/docs/v1.10/locations/). #### CSI snapshot enhancements In this release we added several changes to enhance the robustness of CSI snapshot procedures, for example, some protection code for error handling, and a mechanism to skip exclusion checks so that CSI snapshot works with various backup resource filters. #### Backup schedule pause/unpause In this release, Velero supports to pause/unpause a backup schedule during or after its creation. Specifically: At creation time, you can specify `–paused` flag to `velero schedule create` command, if so, you will create a paused schedule that will not run until it is unpaused After creation, you can run `velero schedule pause` or `velero schedule unpause` command to pause/unpause a schedule #### Runtime and dependencies In order to fix CVEs, we changed Velero's runtime and dependencies as follows: Bump go runtime to v1.18.8 Bump some core dependent libraries to newer versions Compile Restic (v0.13.1) with go 1.18.8 instead of packaging the official binary #### Breaking changes Due to file system backup refactor, below modules and parameters name have been changed in this release: `restic` daemonset is renamed to `node-agent` `resticRepository` CR is renamed to `backupRepository` `velero restic repo` command is renamed to `velero repo` `velero-restic-credentials` secret is renamed to `velero-repo-credentials` `default-volumes-to-restic` parameter is renamed to `default-volumes-to-fs-backup` `restic-timeout` parameter is renamed to `fs-backup-timeout` `default-restic-prune-frequency` parameter is renamed to `default-repo-maintain-frequency` #### Upgrade Due to the major changes of file system backup, the old upgrade steps are not suitable any more. For the new upgrade steps, visit [v1.10 upgrade guide document](https://velero.io/docs/v1.10/upgrade-to-1.10/). #### Limitations/Known issues In this release, Kopia backup repository (so the Kopia path of file system backup) doesn't support self signed certificate for S3 compatible storage. To track this problem, refer to this [Velero issue](https://github.com/vmware-tanzu/velero/issues/5123) or [Kopia issue](https://github.com/kopia/kopia/issues/1443). Due to the code change in Velero, there will be some code change required in vSphere plugin, without which the functionality may be impacted. Therefore, if you are using vSphere plugin in your workflow, please hold the upgrade until the issue [#485](https://github.com/vmware-tanzu/velero-plugin-for-vsphere/issues/485) is fixed in vSphere plugin. ### All changes * Restore ClusterBootstrap before Cluster otherwise a new default ClusterBootstrap object is create for the cluster (#5616, @ywk253100) * Add compile restic binary for CVE fix (#5574, @qiuming-best) * Fix controller problematic log output (#5572, @qiuming-best) * Enhance the restore priorities list to support specifying the low prioritized resources that need to be restored in the last (#5535, @ywk253100) * fix restic backup progress error (#5534, @qiuming-best) * fix restic backup failure with self-signed certification backend storage (#5526, @qiuming-best) * Add credential store in backup deletion controller to support VSL credential. (#5521, @blackpiglet) * Fix issue 5505: the pod volume backups/restores except the first one fail under the kopia path if "AZURE_CLOUD_NAME" is specified (#5512, @Lyndon-Li) * After Pod Volume Backup/Restore refactor, remove all the unreasonable appearance of "restic" word from documents (#5499, @Lyndon-Li) * Refactor Pod Volume Backup/Restore doc to match the new behavior (#5484, @Lyndon-Li) * Remove redundancy code block left by #5388. (#5483, @blackpiglet) * Issue fix 5477: create the common way to support S3 compatible object storages that work for both Restic and Kopia; Keep the resticRepoPrefix parameter for compatibility (#5478, @Lyndon-Li) * Update the k8s.io dependencies to 0.24.0. This also required an update to github.com/bombsimon/logrusr/v3. Removed the `WithClusterName` method as it is a "legacy field that was always cleared by the system and never used" as per upstream k8s https://github.com/kubernetes/apimachinery/blob/release-1.24/pkg/apis/meta/v1/types.go#L257-L259 (#5471, @kcboyle) * Add v1.10 velero upgrade doc (#5468, @qiuming-best) * Upgrade velero docker image to use go 1.18 and upgrade golangci-lint to 1.45.0 (#5459, @Lyndon-Li) * Add VolumeSnapshot client back. (#5449, @blackpiglet) * Change subcommand `velero restic repo` to `velero repo` (#5446, @allenxu404) * Remove irrational "Restic" names in Velero code after the PVBR refactor (#5444, @Lyndon-Li) * moved RIA execute input/output structs back to velero package (#5441, @sseago) * Rename Velero pod volume restore init helper from "velero-restic-restore-helper" to "velero-restore-helper" (#5432, @Lyndon-Li) * Skip the exclusion check for additional resources returned by BIA (#5429, @reasonerjt) * Change B/R describe CLI to support Kopia (#5412, @allenxu404) * Add nil check before execution of csi snapshot delete (#5401, @shubham-pampattiwar) * update velero using klog to version v2.9.0 (#5396, @blackpiglet) * Fix Test_prepareBackupRequest_BackupStorageLocation UT failure. (#5394, @blackpiglet) * Rename Velero daemonset from "restic" to "node-agent" (#5390, @Lyndon-Li) * Add some corner cases checking for CSI snapshot in backup controller. (#5388, @blackpiglet) * Fix issue 5386: Velero providers a full URL as the S3Url while the underlying minio client only accept the host part of the URL as the endpoint and the schema should be specified separately. (#5387, @Lyndon-Li) * Fix restore error with flag namespace-mappings (#5377, @qiuming-best) * Pod Volume Backup/Restore Refactor: Rename parameters in CRDs and commands to remove "Restic" word (#5370, @Lyndon-Li) * Added backupController's UT to test the prepareBackupRequest() method BackupStorageLocation processing logic (#5362, @niulechuan) * Fix a repoEnsurer problem introduced by the refactor - The repoEnsurer didn't check "" state of BackupRepository, as a result, the function GetBackupRepository always returns without an error even though the ensreReady is specified. (#5359, @Lyndon-Li) * Add E2E test for schedule backup (#5355, @danfengliu) * Add useOwnerReferencesInBackup field doc for schedule. (#5353, @cleverhu) * Clarify the help message for the default value of parameter --snapshot-volumes, when it's not set. (#5350, @blackpiglet) * Fix restore cmd extraflag overwrite bug (#5347, @qiuming-best) * Resolve gopkg.in/yaml.v3 vulnerabilities by upgrading gopkg.in/yaml.v3 to v3.0.1 (#5344, @kaovilai) * Increase ensure restic repository timeout to 5m (#5335, @shubham-pampattiwar) * Add opt-in and opt-out PersistentVolume backup to E2E tests (#5331, @danfengliu) * Cancel downloadRequest when timeout without downloadURL (#5329, @kaovilai) * Fix PVB finds wrong parent snapshot (#5322, @qiuming-best) * Fix issue 4874 and 4752: check the daemonset pod is running in the node where the workload pod resides before running the PVB for the pod (#5319, @Lyndon-Li) * plugin versioning v1 refactor for VolumeSnapshotter (#5318, @sseago) * Change the status of restore to completed from partially failed when restore empty backup (#5314, @allenxu404) * RestoreItemAction v1 refactoring for plugin api versioning (#5312, @sseago) * Refactor the repoEnsurer code to use controller runtime client and wrap some common BackupRepository operations to share with other modules (#5308, @Lyndon-Li) * Remove snapshot related lister, informer and client from backup controller. (#5299, @jxun) * Remove github.com/apex/log logger. (#5297, @blackpiglet) * change CSISnapshotTimeout from pointer to normal variables. (#5294, @cleverhu) * Optimize code for restore exists resources. (#5293, @cleverhu) * Add more detailed comments for labels columns. (#5291, @cleverhu) * Add backup status checking in schedule controller. (#5283, @blackpiglet) * Add changes for problems/enhancements found during smoking test for Kopia pod volume backup/restore (#5282, @Lyndon-Li) * Support pause/unpause schedules (#5279, @ywk253100) * plugin/clientmgmt refactoring for BackupItemAction v1 (#5271, @sseago) * Don't move velero v1 plugins to new proto dir (#5263, @sseago) * Fill gaps for Kopia path of PVBR: integrate Repo Manager with Unified Repo; pass UploaderType to PVBR backupper and restorer; pass RepositoryType to BackupRepository controller and Repo Ensurer (#5259, @Lyndon-Li) * Add csiSnapshotTimeout for describe backup (#5252, @cleverhu) * equip gc controller with configurable frequency (#5248, @allenxu404) * Fix nil pointer panic when restoring StatefulSets (#5247, @divolgin) * Controller refactor code modifications. (#5241, @jxun) * Fix edge cases for already exists resources (#5239, @shubham-pampattiwar) * Check for empty ns list before checking nslist[0] (#5236, @sseago) * Remove reference to non-existent doc (#5234, @reasonerjt) * Add changes for Kopia Integration: Kopia Lib - method implementation. Add changes to write Kopia Repository logs to Velero log (#5233, @Lyndon-Li) * Add changes for Kopia Integration: Kopia Lib - initialize Kopia repo (#5231, @Lyndon-Li) * Uploader Implementation: Kopia backup and restore (#5221, @qiuming-best) * Migrate backup sync controller from code-generator to kubebuilder. (#5218, @jxun) * check vsc null pointer (#5217, @lilongfeng0902) * Refactor GCController with kubebuilder (#5215, @allenxu404) * Uploader Implementation: Restic backup and restore (#5214, @qiuming-best) * Add parameter "uploader-type" to velero server (#5212, @reasonerjt) * Add annotation "pv.kubernetes.io/migrated-to" for CSI checking. (#5181, @jxun) * Add changes for Kopia Integration: Unified Repository Provider - method implementation (#5179, @Lyndon-Li) * Treat namespaces with exclude label as excludedNamespaces Related issue: #2413 (#5178, @allenxu404) * Reduce CRD size. (#5174, @jxun) * Fix restic backups to multiple backup storage locations bug (#5172, @qiuming-best) * Add changes for Kopia Integration: Unified Repository Provider - Repo Password (#5167, @Lyndon-Li) * Skip registering "crd-remap-version" plugin when feature flag "EnableAPIGroupVersions" is set (#5165, @reasonerjt) * Kopia uploader integration on shim progress uploader module (#5163, @qiuming-best) * Add labeled and unlabeled events for PR changelog check action. (#5157, @jxun) * VolumeSnapshotLocation refactor with kubebuilder. (#5148, @jxun) * Delay CA file deletion in PVB controller. (#5145, @jxun) * This commit splits the pkg/restic package into several packages to support Kopia integration works (#5143, @ywk253100) * Kopia Integration: Add the Unified Repository Interface definition. Kopia Integration: Add the changes for Unified Repository storage config. Related Issues; #5076, #5080 (#5142, @Lyndon-Li) * Update the CRD for kopia integration (#5135, @reasonerjt) * Let "make shell xxx" respect GOPROXY (#5128, @reasonerjt) * Modify BackupStoreGetter to avoid BSL spec changes (#5122, @sseago) * Dump stack trace when the plugin server handles panic (#5110, @reasonerjt) * Make CSI snapshot creation timeout configurable. (#5104, @jxun) * Fix bsl validation bug: the BSL is validated continually and doesn't respect the validation period configured (#5101, @ywk253100) * Exclude "csinodes.storage.k8s.io" and "volumeattachments.storage.k8s.io" from restore by default. (#5064, @jxun) * Move 'velero.io/exclude-from-backup' label string to const (#5053, @niulechuan) * Modify Github actions. (#5052, @jxun) * Fix typo in doc, in https://velero.io/docs/main/restore-reference/ "Restore order" section, "Mamespace" should be "Namespace". (#5051, @niulechuan) * Delete opened issues triage action. (#5041, @jxun) * When spec.RestoreStatus is empty, don't restore status (#5008, @sseago) * Added DownloadTargetKindCSIBackupVolumeSnapshots for retrieving the signed URL to download only the ``-csi-volumesnapshots.json.gz and DownloadTargetKindCSIBackupVolumeSnapshotContents to download only ``-csi-volumesnapshotcontents.json.gz in the DownloadRequest CR structure. These files are already present in the backup layout. (#4980, @anshulahuja98) * Refactor BackupItemAction proto and related code to backupitemaction/v1 package. This is part of implementation of the plugin version design https://github.com/vmware-tanzu/velero/blob/main/design/plugin-versioning.md (#4943, @phuongatemc) * Unified Repository Design (#4926, @Lyndon-Li) * Add credentials to volume snapshot locations (#4864, @sseago) ================================================ FILE: changelogs/CHANGELOG-1.11.md ================================================ ## v1.11 ### 2023-04-07 ### Download https://github.com/vmware-tanzu/velero/releases/tag/v1.11.0 ### Container Image `velero/velero:v1.11.0` ### Documentation https://velero.io/docs/v1.11/ ### Upgrading https://velero.io/docs/v1.11/upgrade-to-1.11/ ### Highlights #### BackupItemAction v2 This feature implements the BackupItemAction v2. BIA v2 has two new methods: Progress() and Cancel() and modifies the Execute() return value. The API change is needed to facilitate long-running BackupItemAction plugin actions that may not be complete when the Execute() method returns. This will allow long-running BackupItemAction plugin actions to continue in the background while the Velero moves to the following plugin or the next item. #### RestoreItemAction v2 This feature implemented the RestoreItemAction v2. RIA v2 has three new methods: Progress(), Cancel(), and AreAdditionalItemsReady(), and it modifies RestoreItemActionExecuteOutput() structure in the RIA return value. The Progress() and Cancel() methods are needed to facilitate long-running RestoreItemAction plugin actions that may not be complete when the Execute() method returns. This will allow long-running RestoreItemAction plugin actions to continue in the background while the Velero moves to the following plugin or the next item. The AreAdditionalItemsReady() method is needed to allow plugins to tell Velero to wait until the returned additional items have been restored and are ready for use in the cluster before restoring the current item. #### Plugin Progress Monitoring This is intended as a replacement for the previously-approved Upload Progress Monitoring design ([Upload Progress Monitoring](https://github.com/vmware-tanzu/velero/blob/main/design/upload-progress.md)) to expand the supported use cases beyond snapshot upload to include what was previously called Async Backup/Restore Item Actions. #### Flexible resource policy that can filter volumes to skip in the backup This feature provides a flexible policy to filter volumes in the backup without requiring patching any labels or annotations to the pods or volumes. This policy is configured as k8s ConfigMap and maintained by the users themselves, and it can be extended to more scenarios in the future. By now, the policy rules out volumes from backup depending on the CSI driver, NFS setting, volume size, and StorageClass setting. Please refer to [policy API design](https://github.com/vmware-tanzu/velero/blob/main/design/Implemented/handle-backup-of-volumes-by-resources-filters.md#api-design) for the policy's ConifgMap format. It is not guaranteed to work on unofficial third-party plugins as it may not follow the existing backup workflow code logic of Velero. #### Resource Filters that can distinguish cluster scope and namespace scope resources This feature adds four new resource filters for backup. The new filters are separated into cluster scope and namespace scope. Before this feature, Velero could not filter cluster scope resources precisely. This feature provides the ability and refactors existing resource filter parameters. #### Add a parameter for setting the Velero server connection with the k8s API server's timeout In Velero, some code pieces need to communicate with the k8s API server. Before v1.11, these code pieces used hard-code timeout settings. This feature adds a resource-timeout parameter in the velero server binary to make it configurable. #### Add resource list in the output of the restore describe command Before this feature, Velero restore didn't have a restored resources list as the Velero backup. It's not convenient for users to learn what is restored. This feature adds the resources list and the handling result of the resources (including created, updated, failed, and skipped). #### Refactor controllers with controller-runtime In v1.11, Backup Controller and Restore controller are refactored with controller-runtime. Till v1.11, all Velero controllers use the controller-runtime framework. #### Runtime and dependencies To fix CVEs and keep pace with Golang, Velero made changes as follows: * Bump Golang runtime to v1.19.8. * Bump several dependent libraries to new versions. * Compile Restic (v0.15.0) with Golang v1.19.8 instead of packaging the official binary. ### Breaking changes * The Velero CSI plugin now determines whether to restore Volume's data from snapshots on the restore's restorePVs setting. Before v1.11, the CSI plugin doesn't check the restorePVs parameter setting. ### Limitations/Known issues * The Flexible resource policy that can filter volumes to skip in the backup is not guaranteed to work on unofficial third-party plugins because the plugins may not follow the existing backup workflow code logic of Velero. The ConfigMap used as the policy is supposed to be maintained by users. ### All Changes * Modify new scope resource filters name. (#6089, @blackpiglet) * Make Velero not exits when EnableCSI is on and CSI snapshot not installed (#6062, @blackpiglet) * Restore Services before Clusters (#6057, @ywk253100) * Fixed backup deletion bug related to async operations (#6041, @sseago) * Update Golang version to v1.19 for branch main. (#6039, @blackpiglet) * Fix issue #5972, don't assume errorField as error type when dealing with logger.WithError (#6028, @Lyndon-Li) * distinguish between New and InProgress operations (#6012, @sseago) * Modify golangci.yaml file. Resolve found lint issues. (#6008, @blackpiglet) * Remove Reference of itemsnapshotter (#5997, @reasonerjt) * minor fixes for backup_operations_controller (#5996, @sseago) * RIAv2 async operations controller work (#5993, @sseago) * Follow-on fixes for BIAv2 controller work (#5971, @sseago) * Refactor backup controller based on the controller-runtime framework. (#5969, @qiuming-best) * Fix client wait problem after async operation change, velero backup/restore --wait should check a full list of the terminal status (#5964, @Lyndon-Li) * Fix issue #5935, refactor the logics for backup/restore persistent log, so as to remove the contest to gzip writer (#5956, @Lyndon-Li) * Switch the base image to distroless/base-nossl-debian11 to reduce the CVE triage efforts (#5939, @ywk253100) * Wait for additional items to be ready before restoring current item (#5933, @sseago) * Add configurable server setting for default timeouts (#5926, @eemcmullan) * Add warning/error result to cmd `velero backup describe` (#5916, @allenxu404) * Fix Dependabot alerts. Use 1.18 and 1.19 golang instead of patch image in dockerfile. Add release-1.10 and release-1.9 in Trivy daily scan. (#5911, @blackpiglet) * Update client-go to v0.25.6 (#5907, @kaovilai) * Limit the concurrent number for backup's VolumeSnapshot operation. (#5900, @blackpiglet) * Fix goreleaser issue for resolving tags and updated it's version. (#5899, @anshulahuja98) * This is to fix issue 5881, enhance the PVB tracker in two modes, Track and Taken (#5894, @Lyndon-Li) * Add labels for velero installed namespace to support PSA. (#5873, @blackpiglet) * Add restored resource list in the restore describe command (#5867, @ywk253100) * Add a json output to cmd velero backup describe (#5865, @allenxu404) * Make restore controller adopting the controller-runtime framework. (#5864, @blackpiglet) * Replace k8s.io/apimachinery/pkg/util/clock with k8s.io/utils/clock (#5859, @hezhizhen) * Restore finalizer and managedFields of metadata during the restoration (#5853, @ywk253100) * BIAv2 async operations controller work (#5849, @sseago) * Add secret restore item action to handle service account token secret (#5843, @ywk253100) * Add new resource filters can separate cluster and namespace scope resources. (#5838, @blackpiglet) * Correct PVB/PVR Failed Phase patching during startup (#5828, @kaovilai) * bump up golang net to fix CVE-2022-41721 (#5812, @Lyndon-Li) * Update CRD descriptions for SnapshotVolumes and restorePVs (#5807, @shubham-pampattiwar) * Add mapped selected-node existence check (#5806, @blackpiglet) * Add option "--service-account-name" to install cmd (#5802, @reasonerjt) * Enable staticcheck linter. (#5788, @blackpiglet) * Set Kopia IgnoreUnknownTypes in ErrorHandlingPolicy to True for ignoring backup unknown file type (#5786, @qiuming-best) * Bump up Restic version to 0.15.0 (#5784, @qiuming-best) * Add File system backup related metrics to Grafana dashboard - Add metrics backup_warning_total for record of total warnings - Add metrics backup_last_status for record of last status of the backup (#5779, @allenxu404) * Design for Handling backup of volumes by resources filters (#5773, @qiuming-best) * Add PR container build action, which will not push image. Add GOARM parameter. (#5771, @blackpiglet) * Fix issue 5458, track pod volume backup until the CR is submitted in case it is skipped half way (#5769, @Lyndon-Li) * Fix issue 5226, invalidate the related backup repositories whenever the backup storage info change in BSL (#5768, @Lyndon-Li) * Add Restic builder in Dockerfile, and keep the used built Golang image version in accordance with upstream Restic. (#5764, @blackpiglet) * Fix issue 5043, after the restore pod is scheduled, check if the node-agent pod is running in the same node. (#5760, @Lyndon-Li) * Remove restore controller's redundant client. (#5759, @blackpiglet) * Define itemoperations.json format and update DownloadRequest API (#5752, @sseago) * Add Trivy nightly scan. (#5740, @jxun) * Fix issue 5696, check if the repo is still openable before running the prune and forget operation, if not, try to reconnect the repo (#5715, @Lyndon-Li) * Fix error with Restic backup empty volumes (#5713, @qiuming-best) * new backup and restore phases to support async plugin operations: - WaitingForPluginOperations - WaitingForPluginOperationsPartiallyFailed (#5710, @sseago) * Prevent nil panic on exec restore hooks (#5675, @dymurray) * Fix CVEs scanned by trivy (#5653, @qiuming-best) * Publish backupresults json to enhance error info during backups. (#5576, @anshulahuja98) * RestoreItemAction v2 API implementation (#5569, @sseago) * add new RestoreItemAction of "velero.io/change-image-name" to handle the issue mentioned at #5519 (#5540, @wenterjoy) * BackupItemAction v2 API implementation (#5442, @sseago) * Proposal to separate resource filter into cluster scope and namespace scope (#5333, @blackpiglet) ================================================ FILE: changelogs/CHANGELOG-1.12.md ================================================ ## v1.12 ### 2023-08-18 ### Download https://github.com/vmware-tanzu/velero/releases/tag/v1.12.0 ### Container Image `velero/velero:v1.12.0` ### Documentation https://velero.io/docs/v1.12/ ### Upgrading https://velero.io/docs/v1.12/upgrade-to-1.12/ ### Highlights #### CSI Snapshot Data Movement CSI Snapshot Data Movement refers to back up CSI snapshot data from the volatile and limited production environment into durable, heterogeneous, and scalable backup storage in a consistent manner; and restore the data to volumes in the original or alternative environment. CSI Snapshot Data Movement is useful in below scenarios: * For on-premises users, the storage usually doesn't support durable snapshots, so it is impossible/less efficient/cost ineffective to keep volume snapshots by the storage This feature helps to move the snapshot data to a storage with lower cost and larger scale for long time preservation. * For public cloud users, this feature helps users to fulfill the multiple cloud strategy. It allows users to back up volume snapshots from one cloud provider and preserve or restore the data to another cloud provider. Then users will be free to flow their business data across cloud providers based on Velero backup and restore CSI Snapshot Data Movement is built according to the Volume Snapshot Data Movement design ([Volume Snapshot Data Movement](https://github.com/vmware-tanzu/velero/blob/main/design/Implemented/unified-repo-and-kopia-integration/unified-repo-and-kopia-integration.md)). More details can be found in the design. #### Resource Modifiers In many use cases, customers often need to substitute specific values in Kubernetes resources during the restoration process like changing the namespace, changing the storage class, etc. To address this need, Resource Modifiers (also known as JSON Substitutions) offer a generic solution in the restore workflow. It allows the user to define filters for specific resources and then specify a JSON patch (operator, path, value) to apply to the resource. This feature simplifies the process of making substitutions without requiring the implementation of a new RestoreItemAction plugin. More details can be found in Volume Snapshot Resource Modifiers design ([Resource Modifiers](https://github.com/vmware-tanzu/velero/blob/main/design/Implemented/json-substitution-action-design.md)). #### Multiple VolumeSnapshotClasses Prior to version 1.12, the Velero CSI plugin would choose the VolumeSnapshotClass in the cluster based on matching driver names and the presence of the "velero.io/csi-volumesnapshot-class" label. However, this approach proved inadequate for many user scenarios. With the introduction of version 1.12, Velero now offers support for multiple VolumeSnapshotClasses in the CSI Plugin, enabling users to select a specific class for a particular backup. More details can be found in Multiple VolumeSnapshotClasses design ([Multiple VolumeSnapshotClasses](https://github.com/vmware-tanzu/velero/blob/main/design/Implemented/multiple-csi-volumesnapshotclass-support.md)). #### Restore Finalizer Before v1.12, the restore controller would only delete restore resources but wouldn’t delete restore data from the backup storage location when the command `velero restore delete` was executed. The only chance Velero deletes restores data from the backup storage location is when the associated backup is deleted. In this version, Velero introduces a finalizer that ensures the cleanup of all associated data for restores when running the command `velero restore delete`. #### Runtime and dependencies To fix CVEs and keep pace with Golang, Velero made changes as follows: * Bump Golang runtime to v1.20.7. * Bump several dependent libraries to new versions. * Bump Kopia to v0.13. ### Breaking changes * Prior to v1.12, the parameter `uploader-type` for Velero installation had a default value of "restic". However, starting from this version, the default value has been changed to "kopia". This means that Velero will now use Kopia as the default path for file system backup. * The ways of setting CSI snapshot time have changed in v1.12. First, the sync waiting time for creating a snapshot handle in the CSI plugin is changed from the fixed 10 minutes into backup.Spec.CSISnapshotTimeout. The second, the async waiting time for VolumeSnapshot and VolumeSnapshotContent's status turning into `ReadyToUse` in operation uses the operation's timeout. The default value is 4 hours. * As from [Velero helm chart v4.0.0](https://github.com/vmware-tanzu/helm-charts/releases/tag/velero-4.0.0), it supports multiple BSL and VSL, and the BSL and VSL have changed from the map into a slice, and[ this breaking change](https://github.com/vmware-tanzu/helm-charts/pull/413) is not backward compatible. So it would be best to change the BSL and VSL configuration into slices before the Upgrade. ### Limitations/Known issues * The Azure plugin supports Azure AD Workload identity way, but it only works for Velero native snapshots. It cannot support filesystem backup and snapshot data mover scenarios. ### All Changes * Fixes #6498. Get resource client again after restore actions in case resource's gv is changed. This is an improvement of pr #6499, to support group changes. A group change usually happens in a restore plugin which is used for resource conversion: convert a resource from a not supported gv to a supported gv (#6634, @27149chen) * Add API support for volMode block, only error for now. (#6608, @shawn-hurley) * Fix how the AWS credentials are obtained from configuration (#6598, @aws_creds) * Add performance E2E test (#6569, @qiuming-best) * Non default s3 credential profiles work on Unified Repository Provider (kopia) (#6558, @kaovilai) * Fix issue #6571, fix the problem for restore item operation to set the errors correctly so that they can be recorded by Velero restore and then reflect the correct status for Velero restore. (#6594, @Lyndon-Li) * Fix issue 6575, flush the repo after delete the snapshot, otherwise, the changes(deleting repo snapshot) cannot be committed to the repo. (#6587, @Lyndon-Li) * Delete moved snapshots when the backup is deleted (#6547, @reasonerjt) * check if restore crd exist before operating restores (#6544, @allenxu404) * Remove PVC's selector in backup's PVC action. (#6481, @blackpiglet) * Delete the expired deletebackuprequests that are stuck in "InProgress" (#6476, @reasonerjt) * Fix issue #6534, reset PVB CR's StorageLocation to the latest one during backup sync as same as the backup CR. Also fix similar problem with DataUploadResult for data mover restore. (#6533, @Lyndon-Li) * Fix issue #6519. Restrict the client manager of node-agent server to include only Velero resources from the server's namespace, otherwise, the controllers will try to reconcile CRs from all the installed Velero namespaces. (#6523, @Lyndon-Li) * Track the skipped PVC and print the summary in backup log (#6496, @reasonerjt) * Add restore finalizer to clean up external resources (#6479, @allenxu404) * fix: Typos and add more spell checking rules to CI (#6415, @mateusoliveira43) * Add missing CompletionTimestamp and metrics when restore moved into terminal phase in restoreOperationsReconciler (#6397, @Nutrymaco) * Add support for resource Modifications in the restore flow. Also known as JSON Substitutions. (#6452, @anshulahuja98) * Remove dependency of the legacy client code from pkg/cmd directory part 2 (#6497, @blackpiglet) * Add data upload and download metrics (#6493, @allenxu404) * Fix issue 6490, If a backup/restore has multiple async operations and one operation fails while others are still in-progress, when all the operations finish, the backup/restore will be set as Completed falsely (#6491, @Lyndon-Li) * Velero Plugins no longer need kopia indirect dependency in their go.mod (#6484, @kaovilai) * Remove dependency of the legacy client code from pkg/cmd directory (#6469, @blackpiglet) * Add support for OpenStack CSI drivers topology keys (#6464, @openstack-csi-topology-keys) * Add exit code log and possible memory shortage warning log for Restic command failure. (#6459, @blackpiglet) * Modify DownloadRequest controller logic (#6433, @blackpiglet) * Add data download controller for data mover (#6436, @qiuming-best) * Fix hook filter display issue for backup describer (#6434, @allenxu404) * Retrieve DataUpload into backup result ConfigMap during volume snapshot restore. (#6410, @blackpiglet) * Design to add support for Multiple VolumeSnapshotClasses in CSI Plugin. (#5774, @anshulahuja98) * Clarify the deletion frequency for gc controller (#6414, @allenxu404) * Add unit tests for pkg/archive (#6396, @allenxu404) * Add UT for pkg/discovery (#6394, @qiuming-best) * Add UT for pkg/util (#6368, @Lyndon-Li) * Add the code for data mover restore expose (#6357, @Lyndon-Li) * Restore Endpoints before Services (#6315, @ywk253100) * Add warning message for volume snapshotter in data mover case. (#6377, @blackpiglet) * Add unit test for pkg/uploader (#6374, @qiuming-best) * Change kopia as the default path of PVB (#6370, @Lyndon-Li) * Do not persist VolumeSnapshot and VolumeSnapshotContent for snapshot DataMover case. (#6366, @blackpiglet) * Add data mover related options in CLI (#6365, @ywk253100) * Add dataupload controller (#6337, @qiuming-best) * Add UT cases for pkg/podvolume (#6336, @Lyndon-Li) * Remove Wait VolumeSnapshot to ReadyToUse logic. (#6327, @blackpiglet) * Enhance the code because of #6297, the return value of GetBucketRegion is not recorded, as a result, when it fails, we have no way to get the cause (#6326, @Lyndon-Li) * Skip updating status when CRDs are restored (#6325, @reasonerjt) * Include namespaces needed by namespaced-scope resources in backup. (#6320, @blackpiglet) * Update metrics when backup failed with validation error (#6318, @ywk253100) * Add the code for data mover backup expose (#6308, @Lyndon-Li) * Fix a PVR issue for generic data path -- the namespace remap was not honored, and enhance the code for better error handling (#6303, @Lyndon-Li) * Add default values for defaultItemOperationTimeout and itemOperationSyncFrequency in velero CLI (#6298, @shubham-pampattiwar) * Add UT cases for pkg/repository (#6296, @Lyndon-Li) * Fix issue #5875. Since Kopia has supported IAM, Velero should not require static credentials all the time (#6283, @Lyndon-Li) * Fixed a bug where status.progress is not getting updated for backups. (#6276, @kkothule) * Add code change for async generic data path that is used by both PVB/PVR and data mover (#6226, @Lyndon-Li) * Add data mover CRD under v2alpha1, include DataUpload CRD and DataDownload CRD (#6176, @Lyndon-Li) * Remove any dataSource or dataSourceRef fields from PVCs in PVC BIA for cases of prior PVC restores with CSI (#6111, @eemcmullan) * Add the design for Volume Snapshot Data Movement (#5968, @Lyndon-Li) * Fix issue #5123, Kopia repository supports self-cert CA for S3 compatible storage. (#6268, @Lyndon-Li) * Bump up Kopia to v0.13 (#6248, @Lyndon-Li) * log volumes to backup to help debug why `IsPodRunning` is called. (#6232, @kaovilai) * Enable errcheck linter and resolve found issues (#6208, @blackpiglet) * Enable more linters, and remove mal-functioned milestoned issue action. (#6194, @blackpiglet) * Enable stylecheck linter and resolve found issues. (#6185, @blackpiglet) * Fix issue #6182. If pod is not running, don't treat it as an error, let it go and leave a warning. (#6184, @Lyndon-Li) * Enable staticcheck and resolve found issues (#6183, @blackpiglet) * Enable linter revive and resolve found errors: part 2 (#6177, @blackpiglet) * Enable linter revive and resolve found errors: part 1 (#6173, @blackpiglet) * Fix usestdlibvars and whitespace linters issues. (#6162, @blackpiglet) * Update Golang to v1.20 for main. (#6158, @blackpiglet) * Make GetPluginConfig accessible from other packages. (#6151, @tkaovila) * Ignore not found error during patching managedFields (#6136, @ywk253100) * Fix the goreleaser issues and add a new goreleaser action (#6109, @blackpiglet) ================================================ FILE: changelogs/CHANGELOG-1.13.md ================================================ ## v1.13 ### 2024-01-10 ### Download https://github.com/vmware-tanzu/velero/releases/tag/v1.13.0 ### Container Image `velero/velero:v1.13.0` ### Documentation https://velero.io/docs/v1.13/ ### Upgrading https://velero.io/docs/v1.13/upgrade-to-1.13/ ### Highlights #### Resource Modifier Enhancement Velero introduced the Resource Modifiers in v1.12.0. This feature allows users to specify a ConfigMap with a set of rules to modify the resources during restoration. However, only the JSON Patch is supported when creating the rules, and JSON Patch has some limitations, which cannot cover all use cases. In v1.13.0, Velero adds new support for JSON Merge Patch and Strategic Merge Patch, which provide more power and flexibility and allow users to use the same ConfigMap to apply patches on the resources. More design details can be found in [Support JSON Merge Patch and Strategic Merge Patch in Resource Modifiers](https://github.com/vmware-tanzu/velero/blob/main/design/Implemented/merge-patch-and-strategic-in-resource-modifier.md) design. For instructions on how to use the feature, please refer to the [Resource Modifiers](https://velero.io/docs/v1.13/restore-resource-modifiers/) doc. #### Node-Agent Concurrency Velero data movement activities from fs-backups and CSI snapshot data movements run in Velero node-agent, so may be hosted by every node in the cluster and consume resources (i.e. CPU, memory, network bandwidth) from there. With v1.13, users are allowed to configure how many data movement activities (a.k.a, loads) run in each node globally or by node, so that users can better leverage the performance of Velero data movement activities and the resource consumption in the cluster. For more information, check the [Node-Agent Concurrency](https://velero.io/docs/v1.13/node-agent-concurrency/) document. #### Parallel Files Upload Options Velero now supports configurable options for parallel files upload when using Kopia uploader to do fs-backups or CSI snapshot data movements which makes speed up backup possible. For more information, please check [Here](https://velero.io/docs/v1.13/backup-reference/#parallel-files-upload). #### Write Sparse Files Options If using fs-restore or CSI snapshot data movements, it’s supported to write sparse files during restore. For more information, please check [Here](https://velero.io/docs/v1.13/restore-reference/#write-sparse-files). #### Backup Describe In v1.13, the Backup Volume section is added to the velero backup describe command output. The backup Volumes section describes information for all the volumes included in the backup of various backup types, i.e. native snapshot, fs-backup, CSI snapshot, and CSI snapshot data movement. Particularly, the velero backup description now supports showing the information of CSI snapshot data movements, which is not supported in v1.12. Additionally, backup describe command will not check EnableCSI feature gate from client side, so if a backup has volumes with CSI snapshot or CSI snapshot data movement, backup describe command always shows the corresponding information in its output. #### Backup's new VolumeInfo metadata Create a new metadata file in the backup repository's backup name sub-directory to store the backup-including PVC and PV information. The information includes the backing-up method of the PVC and PV data, snapshot information, and status. The VolumeInfo metadata file determines how the PV resource should be restored. The Velero downstream software can also use this metadata file to get a summary of the backup's volume data information. #### Enhancement for CSI Snapshot Data Movements when Velero Pod Restart When performing backup and restore operations, enhancements have been implemented for Velero server pods or node agents to ensure that the current backup or restore process is not stuck or interrupted after restart due to certain exceptional circumstances. #### New status fields added to show hook execution details Hook execution status is now included in the backup/restore CR status and displayed in the backup/restore describe command output. Specifically, it will show the number of hooks which attempted to execute under the HooksAttempted field and the number of hooks which failed to execute under the HooksFailed field. #### AWS SDK Bump Up Bump up AWS SDK for Go to version 2, which offers significant performance improvements in CPU and memory utilization over version 1. #### Azure AD/Workload Identity Support Azure AD/Workload Identity is the recommended approach to do the authentication with Azure services/AKS, Velero has introduced support for Azure AD/Workload Identity on the Velero Azure plugin side in previous releases, and in v1.13.0 Velero adds new support for Kopia operations(file system backup/data mover/etc.) with Azure AD/Workload Identity. #### Runtime and dependencies To fix CVEs and keep pace with Golang, Velero made changes as follows: * Bump Golang runtime to v1.21.6. * Bump several dependent libraries to new versions. * Bump Kopia to v0.15.0. ### Breaking changes * Backup describe command: due to the backup describe output enhancement, some existing information (i.e. the output for native snapshot, CSI snapshot, and fs-backup) has been moved to the Backup Volumes section with some format changes. * API type changes: changes the field [DataMoverConfig](https://github.com/vmware-tanzu/velero/blob/v1.13.0/pkg/apis/velero/v2alpha1/data_upload_types.go#L54) in DataUploadSpec from `*map[string][string]`` to `map[string]string` * Velero install command: due to the issue [#7264](https://github.com/vmware-tanzu/velero/issues/7264), v1.13.0 introduces a break change that make the informer cache enabled by default to keep the actual behavior consistent with the helper message(the informer cache is disabled by default before the change). ### Limitations/Known issues * The backup's VolumeInfo metadata doesn't have the information updated in the async operations. This function could be supported in v1.14 release. ### Note * Velero introduces the informer cache which is enabled by default. The informer cache improves the restore performance but may cause higher memory consumption. Increase the memory limit of the Velero pod or disable the informer cache by specifying the `--disable-informer-cache` option when installing Velero if you get the OOM error. ### Deprecation announcement * The generated k8s clients, informers, and listers are deprecated in the Velero v1.13 release. They are put in the Velero repository's pkg/generated directory. According to the n+2 supporting policy, the deprecated are kept for two more releases. The pkg/generated directory should be deleted in the v1.15 release. * After the backup VolumeInfo metadata file is added to the backup, Velero decides how to restore the PV resource according to the VolumeInfo content. To support the backup generated by the older version of Velero, the old logic is also kept. The support for the backup without the VolumeInfo metadata file will be kept for two releases. The support logic will be deleted in the v1.15 release. ### All Changes * Make "disable-informer-cache" option false(enabled) by default to keep it consistent with the help message (#7294, @ywk253100) * Fix issue #6928, remove snapshot deletion timeout for PVB (#7282, @Lyndon-Li) * Do not set "targetNamespace" to namespace items (#7274, @reasonerjt) * Fix issue #7244. By the end of the upload, check the outstanding incomplete snapshots and delete them by calling ApplyRetentionPolicy (#7245, @Lyndon-Li) * Adjust the newline output of resource list in restore describer (#7238, @allenxu404) * Remove the redundant newline in backup describe output (#7229, @allenxu404) * Fix issue #7189, data mover generic restore - don't assume the first volume as the restore volume (#7201, @Lyndon-Li) * Update CSIVolumeSnapshotsCompleted in backup's status and the metric during backup finalize stage according to async operations content. (#7184, @blackpiglet) * Refactor DownloadRequest Stream function (#7175, @blackpiglet) * Add `--skip-immediately` flag to schedule commands; `--schedule-skip-immediately` server and install (#7169, @kaovilai) * Add node-agent concurrency doc and change the config name from dataPathConcurrency to loadCocurrency (#7161, @Lyndon-Li) * Enhance hooks tracker by adding a returned error to record function (#7153, @allenxu404) * Track the skipped PV when SnapshotVolumes set as false (#7152, @reasonerjt) * Add more linters part 2. (#7151, @blackpiglet) * Fix issue #7135, check pod status before checking node-agent pod status (#7150, @Lyndon-Li) * Treat namespace as a regular restorable item (#7143, @reasonerjt) * Allow sparse option for Kopia & Restic restore (#7141, @qiuming-best) * Use VolumeInfo to help restore the PV. (#7138, @blackpiglet) * Node agent restart enhancement (#7130, @qiuming-best) * Fix issue #6695, add describe for data mover backups (#7125, @Lyndon-Li) * Add hooks status to backup/restore CR (#7117, @allenxu404) * Include plugin name in the error message by operations (#7115, @reasonerjt) * Fix issue #7068, due to a behavior of CSI external snapshotter, manipulations of VS and VSC may not be handled in the same order inside external snapshotter as the API is called. So add a protection finalizer to ensure the order (#7102, @Lyndon-Li) * Generate VolumeInfo for backup. (#7100, @blackpiglet) * Fix issue #7094, fallback to full backup if previous snapshot is not found (#7096, @Lyndon-Li) * Fix issue #7068, due to an behavior of CSI external snapshotter, manipulations of VS and VSC may not be handled in the same order inside external snapshotter as the API is called. So add a protection finalizer to ensure the order (#7095, @Lyndon-Li) * Skip syncing the backup which doesn't contain backup metadata (#7081, @ywk253100) * Fix issue #6693, partially fail restore if CSI snapshot is involved but CSI feature is not ready, i.e., CSI feature gate is not enabled or CSI plugin is not installed. (#7077, @Lyndon-Li) * Truncate the credential file to avoid the change of secret content messing it up (#7072, @ywk253100) * Add VolumeInfo metadata structures. (#7070, @blackpiglet) * improve discoveryHelper.Refresh() in restore (#7069, @27149chen) * Add DataUpload Result and CSI VolumeSnapshot check for restore PV. (#7061, @blackpiglet) * Add the implementation for design #6950, configurable data path concurrency (#7059, @Lyndon-Li) * Make data mover fail early (#7052, @qiuming-best) * Remove dependency of generated client part 3. (#7051, @blackpiglet) * Update Backup.Status.CSIVolumeSnapshotsCompleted during finalize (#7046, @kaovilai) * Remove the Velero generated client. (#7041, @blackpiglet) * Fix issue #7027, data mover backup exposer should not assume the first volume as the backup volume in backup pod (#7038, @Lyndon-Li) * Read information from the credential specified by BSL (#7034, @ywk253100) * Fix #6857. Added check for matching Owner References when synchronizing backups, removing references that are not found/have mismatched uid. (#7032, @deefdragon) * Add description markers for dataupload and datadownload CRDs (#7028, @shubham-pampattiwar) * Add HealthCheckNodePort deletion logic for Service restore. (#7026, @blackpiglet) * Fix inconsistent behavior of Backup and Restore hook execution (#7022, @allenxu404) * Fix #6964. Don't use csiSnapshotTimeout (10 min) for waiting snapshot to readyToUse for data mover, so as to make the behavior complied with CSI snapshot backup (#7011, @Lyndon-Li) * restore: Use warning when Create IsAlreadyExist and Get error (#7004, @kaovilai) * Bump kopia to 0.15.0 (#7001, @Lyndon-Li) * Make Kopia file parallelism configurable (#7000, @qiuming-best) * Fix unified repository (kopia) s3 credentials profile selection (#6995, @kaovilai) * Fix #6988, always get region from BSL if it is not empty (#6990, @Lyndon-Li) * Limit PVC block mode logic to non-Windows platform. (#6989, @blackpiglet) * It is a valid case that the Status.RestoreSize field in VolumeSnapshot is not set, if so, get the volume size from the source PVC to create the backup PVC (#6976, @Lyndon-Li) * Check whether the action is a CSI action and whether CSI feature is enabled, before executing the action. (#6968, @blackpiglet) * Add the PV backup information design document. (#6962, @blackpiglet) * Change controller-runtime List option from MatchingFields to ListOptions (#6958, @blackpiglet) * Add the design for node-agent concurrency (#6950, @Lyndon-Li) * Import auth provider plugins (#6947, @0x113) * Fix #6668, add a limitation for file system restore parallelism with other types of restores (CSI snapshot restore, CSI snapshot movement restore) (#6946, @Lyndon-Li) * Add MSI Support for Azure plugin. (#6938, @yanggangtony) * Partially fix #6734, guide Kubernetes' scheduler to spread backup pods evenly across nodes as much as possible, so that data mover backup could achieve better parallelism (#6926, @Lyndon-Li) * Bump up aws sdk to aws-sdk-go-v2 (#6923, @reasonerjt) * Optional check if targeted container is ready before executing a hook (#6918, @Ripolin) * Support JSON Merge Patch and Strategic Merge Patch in Resource Modifiers (#6917, @27149chen) * Fix issue 6913: Velero Built-in Datamover: Backup stucks in phase WaitingForPluginOperations when Node Agent pod gets restarted (#6914, @shubham-pampattiwar) * Set ParallelUploadAboveSize as MaxInt64 and flush repo after setting up policy so that policy is retrieved correctly by TreeForSource (#6885, @Lyndon-Li) * Replace the base image with paketobuildpacks image (#6883, @ywk253100) * Fix issue #6859, move plugin depending podvolume functions to util pkg, so as to remove the dependencies to unnecessary repository packages like kopia, azure, etc. (#6875, @Lyndon-Li) * Fix #6861. Only Restic path requires repoIdentifier, so for non-restic path, set the repoIdentifier fields as empty in PVB and PVR and also remove the RepoIdentifier column in the get output of PVBs and PVRs (#6872, @Lyndon-Li) * Add volume types filter in resource policies (#6863, @qiuming-best) * change the metrics backup_attempt_total default value to 1. (#6838, @yanggangtony) * Bump kopia to v0.14 (#6833, @Lyndon-Li) * Retry failed create when using generateName (#6830, @sseago) * Fix issue #6786, always delete VSC regardless of the deletion policy (#6827, @Lyndon-Li) * Proposal to support JSON Merge Patch and Strategic Merge Patch in Resource Modifiers (#6797, @27149chen) * Fix the node-agent missing metrics-address defines. (#6784, @yanggangtony) * Fix default BSL setting not work (#6771, @qiuming-best) * Update restore controller logic for restore deletion (#6770, @ywk253100) * Fix #6752: add namespace exclude check. (#6760, @blackpiglet) * Fix issue #6753, remove the check for read-only BSL in restore async operation controller since Velero cannot fully support read-only mode BSL in restore at present (#6757, @Lyndon-Li) * Fix issue #6647, add the --default-snapshot-move-data parameter to Velero install, so that users don't need to specify --snapshot-move-data per backup when they want to move snapshot data for all backups (#6751, @Lyndon-Li) * Use old(origin) namespace in resource modifier conditions in case namespace may change during restore (#6724, @27149chen) * Perf improvements for existing resource restore (#6723, @sseago) * Remove schedule-related metrics on schedule delete (#6715, @nilesh-akhade) * Kubernetes 1.27 new job label batch.kubernetes.io/controller-uid are deleted during restore per https://github.com/kubernetes/kubernetes/pull/114930 (#6712, @kaovilai) * This pr made some improvements in Resource Modifiers: 1. add label selector 2. change the field name from groupKind to groupResource (#6704, @27149chen) * Make Kopia support Azure AD (#6686, @ywk253100) * Add support for block volumes with Kopia (#6680, @dzaninovic) * Delete PartiallyFailed orphaned backups as well as Completed ones (#6649, @sseago) * Add CSI snapshot data movement doc (#6637, @Lyndon-Li) * Fixes #6636, skip subresource in resource discovery (#6635, @27149chen) * Add `orLabelSelectors` for backup, restore commands (#6475, @nilesh-akhade) * fix run preHook and postHook on completed pods (#5211, @cleverhu) ================================================ FILE: changelogs/CHANGELOG-1.14.md ================================================ ## v1.14 ### Download https://github.com/vmware-tanzu/velero/releases/tag/v1.14.0 ### Container Image `velero/velero:v1.14.0` ### Documentation https://velero.io/docs/v1.14/ ### Upgrading https://velero.io/docs/v1.14/upgrade-to-1.14/ ### Highlights #### The maintenance work for kopia/restic backup repositories is run in jobs Since velero started using kopia as the approach for filesystem-level backup/restore, we've noticed an issue when velero connects to the kopia backup repositories and performs maintenance, it sometimes consumes excessive memory that can cause the velero pod to get OOM Killed. To mitigate this issue, the maintenance work will be moved out of velero pod to a separate kubernetes job, and the user will be able to specify the resource request in "velero install". #### Volume Policies are extended to support more actions to handle volumes In an earlier release, a flexible volume policy was introduced to skip certain volumes from a backup. In v1.14 we've made enhancement to this policy to allow the user to set how the volumes should be backed up. The user will be able to set "fs-backup" or "snapshot" as value of “action" in the policy and velero will backup the volumes accordingly. This enhancement allows the user to achieve a fine-grained control like "opt-in/out" without having to update the target workload. For more details please refer to https://velero.io/docs/v1.14/resource-filtering/#supported-volumepolicy-actions #### Node Selection for Data Movement Backup In velero the data movement flow relies on datamover pods, and these pods may take substantial resources and keep running for a long time. In v1.14, the user will be able to create a configmap to define the eligible nodes on which the datamover pods are launched. For more details refer to https://velero.io/docs/v1.14/data-movement-backup-node-selection/ #### VolumeInfo metadata for restored volumes In v1.13, we introduced volumeinfo metadata for backup to help velero CLI and downstream adopter understand how velero handles each volume during backup. In v1.14, similar metadata will be persisted for each restore. velero CLI is also updated to bring more info in the output of "velero restore describe". #### "Finalizing" phase is introduced to restores The "Finalizing" phase is added to the state transition flow to restore, which helps us fix several issues: The labels added to PVs will be restored after the data in the PV is restored via volumesnapshotter. The post restore hook will be executed after datamovement is finished. #### Certificate-based authentication support for Azure Besides the service principal with secret(password)-based authentication, Velero introduces the new support for service principal with certificate-based authentication in v1.14.0. This approach enables you to adopt a phishing resistant authentication by using conditional access policies, which better protects Azure resources and is the recommended way by Azure. ### Runtime and dependencies * Golang runtime: v1.22.2 * kopia: v0.17.0 ### Limitations/Known issues * For the external BackupItemAction plugins that take snapshots for PVs, such as vsphere plugin. If the plugin checks the value of the field "snapshotVolumes" in the backup spec as a criteria for snapshot, the settings in the volume policy will not take effect. For example, if the "snapshotVolumes" is set to False in the backup spec, but a volume meets the condition in the volume policy for "snapshot" action, because the plugin will not check the settings in the volume policy, the plugin will not take snapshot for the volume. For more details please refer to #7818 ### Breaking changes * CSI plugin has been merged into velero repo in v1.14 release. It will be installed by default as an internal plugin, and should not be installed via "–plugins " parameter in "velero install" command. * The default resource requests and limitations for node agent are removed in v1.14, to make the node agent pods have the QoS class of "BestEffort", more details please refer to #7391 * There's a change in namespace filtering behavior during backup: In v1.14, when the includedNamespaces/excludedNamespaces fields are not set and the labelSelector/OrLabelSelectors are set in the backup spec, the backup will only include the namespaces which contain the resources that match the label selectors, while in previous releases all namespaces will be included in the backup with such settings. More details refer to #7105 * Patching the PV in the "Finalizing" state may cause the restore to be in "PartiallyFailed" state when the PV is blocked in "Pending" state, while in the previous release the restore may end up being in "Complete" state. For more details refer to #7866 ### All Changes * Fix backup log to show error string, not index (#7805, @piny940) * Modify the volume helper logic. (#7794, @blackpiglet) * Add documentation for extension of volume policy feature (#7779, @shubham-pampattiwar) * Surface errors when waiting for backupRepository and timeout occurs (#7762, @kaovilai) * Add existingResourcePolicy restore CR validation to controller (#7757, @kaovilai) * Fix condition matching in resource modifier when there are multiple rules (#7715, @27149chen) * Bump up the version of KinD and k8s in github actions (#7702, @reasonerjt) * Implementation for Extending VolumePolicies to support more actions (#7664, @shubham-pampattiwar) * Migrate from `github.com/Azure/azure-storage-blob-go` to `github.com/Azure/azure-sdk-for-go/sdk/storage/azblob` (#7598, @mmorel-35) * When Included/ExcludedNamespaces are omitted, and LabelSelector or OrLabelSelector is used, namespaces without selected items are excluded from backup. (#7697, @blackpiglet) * Display CSI snapshot restores in restore describe (#7687, @reasonerjt) * Use specific credential rather than the credential chain for Azure (#7680, @ywk253100) * Modify hook docs for clarity on displaying hook execution results (#7679, @allenxu404) * Wait for results of restore exec hook executions in Finalizing phase instead of InProgress phase (#7619, @allenxu404) * migrating to `sdk/resourcemanager/**/arm**` from `services/**/mgmt/**` (#7596, @mmorel-35) * Bump up to go1.22 (#7666, @reasonerjt) * Fix issue #7648. Adjust the exposing logic to avoid exposing failure and snapshot leak when expose fails (#7662, @Lyndon-Li) * Track and persist restore volume info (#7630, @reasonerjt) * Check the existence of the namespaces provided in the "--include-namespaces" option (#7569, @ywk253100) * Add the finalization phase to the restore workflow (#7377, @allenxu404) * Upgrade the version of go plugin related libs/tools (#7373, @ywk253100) * Check resource Group Version and Kind is available in cluster before attempting restore to prevent being stuck. (#7322, @kaovilai) * Merge CSI plugin code into Velero. (#7609, @blackpiglet) * Fix issue #7391, remove the default constraint for node-agent pods (#7488, @Lyndon-Li) * Fix DataDownload fails during restore for empty PVC workload (#7521, @qiuming-best) * Add repository maintenance job (#7451, @qiuming-best) * Check whether the VolumeSnapshot's source PVC is nil before using it. Skip populate VolumeInfo for data-moved PV when CSI is not enabled. (#7515, @blackpiglet) * Fix issue #7308, change the data path requeue time to 5 second for data mover backup/restore, PVB and PVR. (#7458, @Lyndon-Li) * Patch newly dynamically provisioned PV with volume info to restore custom setting of PV (#7504, @allenxu404) * Adjust the logic for the backup_last_status metrics to stop incorrectly incrementing over time (#7445, @allenxu404) * dependabot: support github-actions updates (#7594, @mmorel-35) * Include the design for adding the finalization phase to the restore workflow (#7317, @allenxu404) * Fix issue #7211. Enable advanced feature capability and add support to concatenate objects for unified repo. (#7452, @Lyndon-Li) * Add design to introduce restore volume info (#7610, @reasonerjt) * Increase the k8s client QPS/burst to avoid throttling request errors (#7311, @ywk253100) * Support update the backup VolumeInfos by the Async ops result. (#7554, @blackpiglet) * FS backup create PodVolumeBackup when the backup excluded PVC, so I added logic to skip PVC volume type when PVC is not included in the backup resources to be backed up. (#7472, @sbahar619) * Respect and use `credentialsFile` specified in BSL.spec.config when IRSA is configured over Velero Pod Environment credentials (#7374, @reasonerjt) * Move the native snapshot definition code into internal directory (#7544, @blackpiglet) * Fix issue #7036. Add the implementation of node selection for data mover backups (#7437, @Lyndon-Li) * Fix issue #7535, add the MustHave resource check during item collection and item filter for restore (#7585, @Lyndon-Li) * build(deps): bump json-patch to v5.8.0 (#7584, @mmorel-35) * Add confirm flag to velero plugin add (#7566, @kaovilai) * do not skip unknown gvr at the beginning and get new gr when kind is changed (#7523, @27149chen) * Fix snapshot leak for backup (#7558, @qiuming-best) * For issue #7036, add the document for data mover node selection (#7640, @Lyndon-Li) * Add design for Extending VolumePolicies to support more actions (#6956, @shubham-pampattiwar) * BackupRepositories associated with a BSL are invalidated when BSL is (re-)created. (#7380, @kaovilai) * Improve the concurrency for PVBs in different pods (#7571, @ywk253100) * Bump up Kopia to v0.16.0 and open kopia repo with no index change (#7559, @Lyndon-Li) * Bump up the versions of several Kubernetes-related libs (#7489, @ywk253100) * Make parallel restore configurable (#7512, @qiuming-best) * Support certificate-based authentication for Azure (#7549, @ywk253100) * Fix issue #7281, batch delete snapshots in the same repo (#7438, @Lyndon-Li) * Add CRD name to error message when it is not ready to use (#7295, @josemarevalo) * Add the design for node selection for data mover backup (#7383, @Lyndon-Li) * Bump up aws-sdk to latest version to leverage Pod Identity credentials. (#7307, @guikcd) * Fix issue #7246. Document the behavior for repo snapshot deletion (#7622, @Lyndon-Li) * Fix issue #7583, set backupName optional for Restore CRD (#7617, @Lyndon-Li) ================================================ FILE: changelogs/CHANGELOG-1.15.md ================================================ ## v1.15 ### Download https://github.com/vmware-tanzu/velero/releases/tag/v1.15.0 ### Container Image `velero/velero:v1.15.0` ### Documentation https://velero.io/docs/v1.15/ ### Upgrading https://velero.io/docs/v1.15/upgrade-to-1.15/ ### Highlights #### Data mover micro service Data transfer activities for CSI Snapshot Data Movement are moved from node-agent pods to dedicate backupPods or restorePods. This brings many benefits such as: - This avoids to access volume data through host path, while host path access is privileged and may involve security escalations, which are concerned by users. - This enables users to to control resource (i.e., cpu, memory) allocations in a granular manner, e.g., control them per backup/restore of a volume. - This enhances the resilience, crash of one data movement activity won't affect others. - This prevents unnecessary full backup because of host path changes after workload pods restart. - For more information, check the design https://github.com/vmware-tanzu/velero/blob/main/design/Implemented/vgdp-micro-service/vgdp-micro-service.md. #### Item Block concepts and ItemBlockAction (IBA) plugin Item Block concepts are introduced for resource backups to help to achieve multiple thread backups. Specifically, correlated resources are categorized in the same item block and item blocks could be processed concurrently in multiple threads. ItemBlockAction plugin is introduced to help Velero to categorize resources into item blocks. At present, Velero provides built-in IBAs for pods and PVCs and Velero also supports customized IBAs for any resources. In v1.15, Velero doesn't support multiple thread process of item blocks though item block concepts and IBA plugins are fully supported. The multiple thread support will be delivered in future releases. For more information, check the design https://github.com/vmware-tanzu/velero/blob/main/design/backup-performance-improvements.md. #### Node selection for repository maintenance job Repository maintenance are resource consuming tasks, Velero now allows you to configure the nodes to run repository maintenance jobs, so that you can run repository maintenance jobs in idle nodes or avoid them to run in nodes hosting critical workloads. To support the configuration, a new repository maintenance configuration configMap is introduced. For more information, check the document https://velero.io/docs/v1.15/repository-maintenance/. #### Backup PVC read-only configuration In 1.15, Velero allows you to configure the data mover backupPods to read-only mount the backupPVCs. In this way, the data mover expose process could be significantly accelerated for some storages (i.e., ceph). To support the configuration, a new backup PVC configuration configMap is introduced. For more information, check the document https://velero.io/docs/v1.15/data-movement-backup-pvc-configuration/. #### Backup PVC storage class configuration In 1.15, Velero allows you to configure the storageclass used by the data mover backupPods. In this way, the provision of backupPVCs don't need to adhere to the same pattern as workload PVCs, e.g., for a backupPVC, it only needs one replica, whereas, the a workload PVC may have multiple replicas. To support the configuration, the same backup PVC configuration configMap is used. For more information, check the document https://velero.io/docs/v1.15/data-movement-backup-pvc-configuration/. #### Backup repository data cache configuration The backup repository may need to cache data on the client side during various repository operations, i.e., read, write, maintenance, etc. The cache consumes the root file system space of the pod where the repository access happens. In 1.15, Velero allows you to configure the total size of the cache per repository. In this way, if your pod doesn't have enough space in its root file system, the pod won't be evicted due to running out of ephemeral storage. To support the configuration, a new backup repository configuration configMap is introduced. For more information, check the document https://velero.io/docs/v1.15/backup-repository-configuration/. #### Performance improvements In 1.15, several performance related issues/enhancements are included, which makes significant performance improvements in specific scenarios: - There was a memory leak of Velero server after plugin calls, now it is fixed, see issue https://github.com/vmware-tanzu/velero/issues/7925 - The `client-burst/client-qps` parameters are automatically inherited to plugins, so that you can use the same velero server parameters to accelerate the plugin executions when large number of API server calls happen, see issue https://github.com/vmware-tanzu/velero/issues/7806 - Maintenance of Kopia repository takes huge memory in scenarios that huge number of files have been backed up, Velero 1.15 has included the Kopia upstream enhancement to fix the problem, see issue https://github.com/vmware-tanzu/velero/issues/7510 ### Runtime and dependencies Golang runtime: v1.22.8 kopia: v0.17.0 ### Limitations/Known issues #### Read-only backup PVC may not work on SELinux environments Due to an issue of Kubernetes upstream, if a volume is mounted as read-only in SELinux environments, the read privilege is not granted to any user, as a result, the data mover backup will fail. On the other hand, the backupPVC must be mounted as read-only in order to accelerate the data mover expose process. Therefore, a user option is added in the same backup PVC configuration configMap, once the option is enabled, the backupPod container will run as a super privileged container and disable SELinux access control. If you have concern in this super privileged container or you have configured [pod security admissions](https://kubernetes.io/docs/concepts/security/pod-security-admission/) and don't allow super privileged containers, you will not be able to use this read-only backupPVC feature and lose the benefit to accelerate the data mover expose process. ### Breaking changes #### Deprecation of Restic Restic path for fs-backup is in deprecation process starting from 1.15. According to [Velero deprecation policy](https://github.com/vmware-tanzu/velero/blob/v1.15/GOVERNANCE.md#deprecation-policy), for 1.15, if Restic path is used the backup/restore of fs-backup still creates and succeeds, but you will see warnings in below scenarios: - When `--uploader-type=restic` is used in Velero installation - When Restic path is used to create backup/restore of fs-backup #### node-agent configuration name is configurable Previously, a fixed name is searched for node-agent configuration configMap. Now in 1.15, Velero allows you to customize the name of the configMap, on the other hand, the name must be specified by node-agent server parameter `node-agent-configmap`. #### Repository maintenance job configurations in Velero server parameter are moved to repository maintenance job configuration configMap In 1.15, below Velero server parameters for repository maintenance jobs are moved to the repository maintenance job configuration configMap. While for back compatibility reason, the same Velero sever parameters are preserved as is. But the configMap is recommended and the same values in the configMap take preference if they exist in both places: ``` --keep-latest-maintenance-jobs --maintenance-job-cpu-request --maintenance-job-mem-request --maintenance-job-cpu-limit --maintenance-job-mem-limit ``` #### Changing PVC selected-node feature is deprecated In 1.15, the [Changing PVC selected-node feature](https://velero.io/docs/v1.15/restore-reference/#changing-pvc-selected-node) enters deprecation process and will be removed in future releases according to [Velero deprecation policy](https://github.com/vmware-tanzu/velero/blob/v1.15/GOVERNANCE.md#deprecation-policy). Usage of this feature for any purpose is not recommended. ### All Changes * add no-relabeling option to backupPVC configmap (#8288, @sseago) * only set spec.volumes readonly if PVC is readonly for datamover (#8284, @sseago) * Add labels to maintenance job pods (#8256, @shubham-pampattiwar) * Add the Carvel package related resources to the restore priority list (#8228, @ywk253100) * Reduces indirect imports for plugin/framework importers (#8208, @kaovilai) * Add controller name to periodical_enqueue_source. The logger parameter now includes an additional field with the value of reflect.TypeOf(objList).String() and another field with the value of controllerName. (#8198, @kaovilai) * Update Openshift SCC docs link (#8170, @shubham-pampattiwar) * Partially fix issue #8138, add doc for node-agent memory preserve (#8167, @Lyndon-Li) * Pass Velero server command args to the plugins (#8166, @ywk253100) * Fix issue #8155, Merge Kopia upstream commits for critical issue fixes and performance improvements (#8158, @Lyndon-Li) * Implement the Repo maintenance Job configuration. (#8145, @blackpiglet) * Add document for data mover micro service (#8144, @Lyndon-Li) * Fix issue #8134, allow to config resource request/limit for data mover micro service pods (#8143, @Lyndon-Li) * Apply backupPVCConfig to backupPod volume spec (#8141, @shubham-pampattiwar) * Add resource modifier for velero restore describe CLI (#8139, @blackpiglet) * Fix issue #7620, add doc for backup repo config (#8131, @Lyndon-Li) * Modify E2E and perf test report generated directory (#8129, @blackpiglet) * Add docs for backup pvc config support (#8119, @shubham-pampattiwar) * Delete generated k8s client and informer. (#8114, @blackpiglet) * Add support for backup PVC configuration (#8109, @shubham-pampattiwar) * ItemBlock model and phase 1 (single-thread) workflow changes (#8102, @sseago) * Fix issue #8032, make node-agent configMap name configurable (#8097, @Lyndon-Li) * Fix issue #8072, add the warning messages for restic deprecation (#8096, @Lyndon-Li) * Fix issue #7620, add backup repository configuration implementation and support cacheLimit configuration for Kopia repo (#8093, @Lyndon-Li) * Patch dbr's status when error happens (#8086, @reasonerjt) * According to design #7576, after node-agent restarts, if a DU/DD is in InProgress status, re-capture the data mover ms pod and continue the execution (#8085, @Lyndon-Li) * Updates to IBM COS documentation to match current version (#8082, @gjanders) * Data mover micro service DUCR/DDCR controller refactor according to design #7576 (#8074, @Lyndon-Li) * add retries with timeout to existing patch calls that moves a backup/restore from InProgress/Finalizing to a final status phase. (#8068, @kaovilai) * Data mover micro service restore according to design #7576 (#8061, @Lyndon-Li) * Internal ItemBlockAction plugins (#8054, @sseago) * Data mover micro service backup according to design #7576 (#8046, @Lyndon-Li) * Avoid wrapping failed PVB status with empty message. (#8028, @mrnold) * Created new ItemBlockAction (IBA) plugin type (#8026, @sseago) * Make PVPatchMaximumDuration timeout configurable (#8021, @shubham-pampattiwar) * Reuse existing plugin manager for get/put volume info (#8012, @sseago) * Data mover ms watcher according to design #7576 (#7999, @Lyndon-Li) * New data path for data mover ms according to design #7576 (#7988, @Lyndon-Li) * For issue #7700 and #7747, add the design for backup PVC configurations (#7982, @Lyndon-Li) * Only get VolumeSnapshotClass when DataUpload exists. (#7974, @blackpiglet) * Fix issue #7972, sync the backupPVC deletion in expose clean up (#7973, @Lyndon-Li) * Expose the VolumeHelper to third-party plugins. (#7969, @blackpiglet) * Check whether the volume's source is PVC before fetching its PV. (#7967, @blackpiglet) * Check whether the namespaces specified in namespace filter exist. (#7965, @blackpiglet) * Add design for backup repository configurations for issue #7620, #7301 (#7963, @Lyndon-Li) * New data path for data mover ms according to design #7576 (#7955, @Lyndon-Li) * Skip PV patch step in Restoe workflow for WaitForFirstConsumer VolumeBindingMode Pending state PVCs (#7953, @shubham-pampattiwar) * Fix issue #7904, add the deprecation and limitation clarification for change PVC selected-node feature (#7948, @Lyndon-Li) * Expose the VolumeHelper to third-party plugins. (#7944, @blackpiglet) * Don't consider unschedulable pods unrecoverable (#7899, @sseago) * Upgrade to robfig/cron/v3 to support time zone specification. (#7793, @kaovilai) * Add the result in the backup's VolumeInfo. (#7775, @blackpiglet) * Migrate from github.com/golang/protobuf to google.golang.org/protobuf (#7593, @mmorel-35) * Add the design for data mover micro service (#7576, @Lyndon-Li) * Descriptive restore error when restoring into a terminating namespace. (#7424, @kaovilai) * Ignore missing path error in conditional match (#7410, @seanblong) * Propose a deprecation process for velero (#5532, @shubham-pampattiwar) ================================================ FILE: changelogs/CHANGELOG-1.16.md ================================================ ## v1.16 ### Download https://github.com/vmware-tanzu/velero/releases/tag/v1.16.0 ### Container Image `velero/velero:v1.16.0` ### Documentation https://velero.io/docs/v1.16/ ### Upgrading https://velero.io/docs/v1.16/upgrade-to-1.16/ ### Highlights #### Windows cluster support In v1.16, Velero supports to run in Windows clusters and backup/restore Windows workloads, either stateful or stateless: * Hybrid build and all-in-one image: the build process is enhanced to build an all-in-one image for hybrid CPU architecture and hybrid platform. For more information, check the design https://github.com/vmware-tanzu/velero/blob/main/design/multiple-arch-build-with-windows.md * Deployment in Windows clusters: Velero node-agent, data mover pods and maintenance jobs now support to run in both linux and Windows nodes * Data mover backup/restore Windows workloads: Velero built-in data mover supports Windows workloads throughout its full cycle, i.e., discovery, backup, restore, pre/post hook, etc. It automatically identifies Windows workloads and schedules data mover pods to the right group of nodes Check the epic issue https://github.com/vmware-tanzu/velero/issues/8289 for more information. #### Parallel Item Block backup v1.16 now supports to back up item blocks in parallel. Specifically, during backup, correlated resources are grouped in item blocks and Velero backup engine creates a thread pool to back up the item blocks in parallel. This significantly improves the backup throughput, especially when there are large scale of resources. Pre/post hooks also belongs to item blocks, so will also run in parallel along with the item blocks. Users are allowed to configure the parallelism through the `--item-block-worker-count` Velero server parameter. If not configured, the default parallelism is 1. For more information, check issue https://github.com/vmware-tanzu/velero/issues/8334. #### Data mover restore enhancement in scalability In previous releases, for each volume of WaitForFirstConsumer mode, data mover restore is only allowed to happen in the node that the volume is attached. This severely degrades the parallelism and the balance of node resource(CPU, memory, network bandwidth) consumption for data mover restore (https://github.com/vmware-tanzu/velero/issues/8044). In v1.16, users are allowed to configure data mover restores running and spreading evenly across all nodes in the cluster. The configuration is done through a new flag `ignoreDelayBinding` in node-agent configuration (https://github.com/vmware-tanzu/velero/issues/8242). #### Data mover enhancements in observability In 1.16, some observability enhancements are added: * Output various statuses of intermediate objects for failures of data mover backup/restore (https://github.com/vmware-tanzu/velero/issues/8267) * Output the errors when Velero fails to delete intermediate objects during clean up (https://github.com/vmware-tanzu/velero/issues/8125) The outputs are in the same node-agent log and enabled automatically. #### CSI snapshot backup/restore enhancement in usability In previous releases, a unnecessary VolumeSnapshotContent object is retained for each backup and synced to other clusters sharing the same backup storage location. And during restore, the retained VolumeSnapshotContent is also restored unnecessarily. In 1.16, the retained VolumeSnapshotContent is removed from the backup, so no unnecessary CSI objects are synced or restored. For more information, check issue https://github.com/vmware-tanzu/velero/issues/8725. #### Backup Repository Maintenance enhancement in resiliency and observability In v1.16, some enhancements of backup repository maintenance are added to improve the observability and resiliency: * A new backup repository maintenance history section, called `RecentMaintenance`, is added to the BackupRepository CR. Specifically, for each BackupRepository, including start/completion time, completion status and error message. (https://github.com/vmware-tanzu/velero/issues/7810) * Running maintenance jobs are now recaptured after Velero server restarts. (https://github.com/vmware-tanzu/velero/issues/7753) * The maintenance job will not be launched for readOnly BackupStorageLocation. (https://github.com/vmware-tanzu/velero/issues/8238) * The backup repository will not try to initialize a new repository for readOnly BackupStorageLocation. (https://github.com/vmware-tanzu/velero/issues/8091) * Users now are allowed to configure the intervals of an effective maintenance in the way of `normalGC`, `fastGC` and `eagerGC`, through the `fullMaintenanceInterval` parameter in backupRepository configuration. (https://github.com/vmware-tanzu/velero/issues/8364) #### Volume Policy enhancement of filtering volumes by PVC labels In v1.16, Volume Policy is extended to support filtering volumes by PVC labels. (https://github.com/vmware-tanzu/velero/issues/8256). #### Resource Status restore per object In v1.16, users are allowed to define whether to restore resource status per object through an annotation `velero.io/restore-status` set on the object. (https://github.com/vmware-tanzu/velero/issues/8204). #### Velero Restore Helper binary is merged into Velero image In v1.16, Velero banaries, i.e., velero, velero-helper and velero-restore-helper, are all included into the single Velero image. (https://github.com/vmware-tanzu/velero/issues/8484). ### Runtime and dependencies Golang runtime: 1.23.7 kopia: 0.19.0 ### Limitations/Known issues #### Limitations of Windows support * fs-backup is not supported for Windows workloads and so fs-backup runs only in linux nodes for linux workloads * Backup/restore of NTFS extended attributes/advanced features are not supported, i.e., Security Descriptors, System/Hidden/ReadOnly attributes, Creation Time, NTFS Streams, etc. ### All Changes * Add third party annotation support for maintenance job, so that the declared third party annotations could be added to the maintenance job pods (#8812, @Lyndon-Li) * Fix issue #8803, use deterministic name to create backupRepository (#8808, @Lyndon-Li) * Refactor restoreItem and related functions to differentiate the backup resource name and the restore target resource name. (#8797, @blackpiglet) * ensure that PV is removed before VS is deleted (#8777, @ix-rzi) * host_pods should not be mandatory to node-agent (#8774, @mpryc) * Log doesn't show pv name, but displays %!s(MISSING) instead (#8771, @hu-keyu) * Fix issue #8754, add third party annotation support for data mover (#8770, @Lyndon-Li) * Add docs for volume policy with labels as a criteria (#8759, @shubham-pampattiwar) * Move pvc annotation removal from CSI RIA to regular PVC RIA (#8755, @sseago) * Add doc for maintenance history (#8747, @Lyndon-Li) * Fix issue #8733, add doc for restorePVC (#8737, @Lyndon-Li) * Fix issue #8426, add doc for Windows support (#8736, @Lyndon-Li) * Fix issue #8475, refactor build-from-source doc for hybrid image build (#8729, @Lyndon-Li) * Return directly if no pod volme backup are tracked (#8728, @ywk253100) * Fix issue #8706, for immediate volumes, there is no selected-node annotation on PVC, so deduce the attached node from VolumeAttachment CRs (#8715, @Lyndon-Li) * Add labels as a criteria for volume policy (#8713, @shubham-pampattiwar) * Copy SecurityContext from Containers[0] if present for PVR (#8712, @sseago) * Support pushing images to an insecure registry (#8703, @ywk253100) * Modify golangci configuration to make it work. (#8695, @blackpiglet) * Run backup post hooks inside ItemBlock synchronously (#8694, @ywk253100) * Add docs for object level status restore (#8693, @shubham-pampattiwar) * Clean artifacts generated during CSI B/R. (#8684, @blackpiglet) * Don't run maintenance on the ReadOnly BackupRepositories. (#8681, @blackpiglet) * Fix #8657: WaitGroup panic issue (#8679, @ywk253100) * Fixes issue #8214, validate `--from-schedule` flag in create backup command to prevent empty or whitespace-only values. (#8665, @aj-2000) * Implement parallel ItemBlock processing via backup_controller goroutines (#8659, @sseago) * Clean up leaked CSI snapshot for incomplete backup (#8637, @raesonerjt) * Handle update conflict when restoring the status (#8630, @ywk253100) * Fix issue #8419, support repo maintenance job to run on Windows nodes (#8626, @Lyndon-Li) * Always create DataUpload configmap in restore namespace (#8621, @sseago) * Fix issue #8091, avoid to create new repo when BSL is readonly (#8615, @Lyndon-Li) * Fix issue #8242, distribute dd evenly across nodes (#8611, @Lyndon-Li) * Fix issue #8497, update du/dd progress on completion (#8608, @Lyndon-Li) * Fix issue #8418, add Windows toleration to data mover pods (#8606, @Lyndon-Li) * Check the PVB status via podvolume Backupper rather than calling API server to avoid API server issue (#8603, @ywk253100) * Fix issue #8067, add tmp folder (/tmp for linux, C:\Windows\Temp for Windows) as an alternative of udmrepo's config file location (#8602, @Lyndon-Li) * Data mover restore for Windows (#8594, @Lyndon-Li) * Skip patching the PV in finalization for failed operation (#8591, @reasonerjt) * Fix issue #8579, set event burst to block event broadcaster from filtering events (#8590, @Lyndon-Li) * Configurable Kopia Maintenance Interval. backup-repository-configmap adds an option for configurable`fullMaintenanceInterval` where fastGC (12 hours), and eagerGC (6 hours) allowing for faster removal of deleted velero backups from kopia repo. (#8581, @kaovilai) * Fix issue #7753, recall repo maintenance history on Velero server restart (#8580, @Lyndon-Li) * Clear validation errors when schedule is valid (#8575, @ywk253100) * Merge restore helper image into Velero server image (#8574, @ywk253100) * Don't include excluded items in ItemBlocks (#8572, @sseago) * fs uploader and block uploader support Windows nodes (#8569, @Lyndon-Li) * Fix issue #8418, support data mover backup for Windows nodes (#8555, @Lyndon-Li) * Fix issue #8044, allow users to ignore delay binding the restorePVC of data mover when it is in WaitForFirstConsumer mode (#8550, @Lyndon-Li) * Fix issue #8539, validate uploader types when o.CRDsOnly is set to false only since CRD installation doesn't rely on uploader types (#8538, @Lyndon-Li) * Fix issue #7810, add maintenance history for backupRepository CRs (#8532, @Lyndon-Li) * Make fs-backup work on linux nodes with the new Velero deployment and disable fs-backup if the source/target pod is running in non-linux node (#8424) (#8518, @Lyndon-Li) * Fix issue: backup schedule pause/unpause doesn't work (#8512, @ywk253100) * Fix backup post hook issue #8159 (caused by #7571): always execute backup post hooks after PVBs are handled (#8509, @ywk253100) * Fix issue #8267, enhance the error message when expose fails (#8508, @Lyndon-Li) * Fix issue #8416, #8417, deploy Velero server and node-agent in linux/Windows hybrid env (#8504, @Lyndon-Li) * Design to add label selector as a criteria for volume policy (#8503, @shubham-pampattiwar) * Related to issue #8485, move the acceptedByNode and acceptedTimestamp to Status of DU/DD CRD (#8498, @Lyndon-Li) * Add SecurityContext to restore-helper (#8491, @reasonerjt) * Fix issue #8433, add third party labels to data mover pods when the same labels exist in node-agent pods (#8487, @Lyndon-Li) * Fix issue #8485, add an accepted time so as to count the prepare timeout (#8486, @Lyndon-Li) * Fix issue #8125, log diagnostic info for data mover exposers when expose timeout (#8482, @Lyndon-Li) * Fix issue #8415, implement multi-arch build and Windows build (#8476, @Lyndon-Li) * Pin kopia to 0.18.2 (#8472, @Lyndon-Li) * Add nil check for updating DataUpload VolumeInfo in finalizing phase (#8471, @blackpiglet) * Allowing Object-Level Resource Status Restore (#8464, @shubham-pampattiwar) * For issue #8429. Add the design for multi-arch build and windows build (#8459, @Lyndon-Li) * Upgrade go.mod k8s.io/ go.mod to v0.31.3 and implemented proper logger configuration for both client-go and controller-runtime libraries. This change ensures that logging format and level settings are properly applied throughout the codebase. The update improves logging consistency and control across the Velero system. (#8450, @kaovilai) * Add Design for Allowing Object-Level Resource Status Restore (#8403, @shubham-pampattiwar) * Fix issue #8391, check ErrCancelled from suffix of data mover pod's termination message (#8396, @Lyndon-Li) * Fix issue #8394, don't call closeDataPath in VGDP callbacks, otherwise, the VGDP cleanup will hang (#8395, @Lyndon-Li) * Adding support in velero Resource Policies for filtering PVs based on additional VolumeAttributes properties under CSI PVs (#8383, @mayankagg9722) * Add --item-block-worker-count flag to velero install and server (#8380, @sseago) * Make BackedUpItems thread safe (#8366, @sseago) * Include --annotations flag in backup and restore create commands (#8354, @alromeros) * Use aggregated discovery API to discovery API groups and resources (#8353, @ywk253100) * Copy "envFrom" from Velero server when creating maintenance jobs (#8343, @evhan) * Set hinting region to use for GetBucketRegion() in pkg/repository/config/aws.go (#8297, @kaovilai) * Bump up version of client-go and controller-runtime (#8275, @ywk253100) * fix(pkg/repository/maintenance): don't panic when there's no container statuses (#8271, @mcluseau) * Add Backup warning for inclusion of NS managed by ArgoCD (#8257, @shubham-pampattiwar) * Added tracking for deleted namespace status check in restore flow. (#8233, @sangitaray2021) ================================================ FILE: changelogs/CHANGELOG-1.17.md ================================================ ## v1.17 ### Download https://github.com/vmware-tanzu/velero/releases/tag/v1.17.0 ### Container Image `velero/velero:v1.17.0` ### Documentation https://velero.io/docs/v1.17/ ### Upgrading https://velero.io/docs/v1.17/upgrade-to-1.17/ ### Highlights #### Modernized fs-backup In v1.17, Velero fs-backup is modernized to the micro-service architecture, which brings below benefits: - Many features that were absent to fs-backup are now available, i.e., load concurrency control, cancel, resume on restart, etc. - fs-backup is more robust, the running backup/restore could survive from node-agent restart; and the resource allocation is in a more granular manner, the failure of one backup/restore won't impact others. - The resource usage of node-agent is steady, especially, the node-agent pods won't request huge memory and hold it for a long time. Check design https://github.com/vmware-tanzu/velero/blob/main/design/vgdp-micro-service-for-fs-backup/vgdp-micro-service-for-fs-backup.md for more details. #### fs-backup support Windows cluster In v1.17, Velero fs-backup supports to backup/restore Windows workloads. By leveraging the new micro-service architecture for fs-backup, data mover pods could run in Windows nodes and backup/restore Windows volumes. Together with CSI snapshot data movement for Windows which is delivered in 1.16, Velero now supports Windows workload backup/restore in full scenarios. Check design https://github.com/vmware-tanzu/velero/blob/main/design/vgdp-micro-service-for-fs-backup/vgdp-micro-service-for-fs-backup.md for more details. #### Volume group snapshot support In v1.17, Velero supports [volume group snapshots](https://kubernetes.io/blog/2024/12/18/kubernetes-1-32-volume-group-snapshot-beta/) which is a beta feature in Kubernetes upstream, for both CSI snapshot backup and CSI snapshot data movement. This allows a snapshot to be taken from multiple volumes at the same point-in-time to achieve write order consistency, which is helpful to achieve better data consistency when multiple volumes being backed up are correlated. Check the document https://velero.io/docs/main/volume-group-snapshots/ for more details. #### Priority class support In v1.17, [Kubernetes priority class](https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#priorityclass) is supported for all modules across Velero. Specifically, users are allowed to configure priority class to Velero server, node-agent, data mover pods, backup repository maintenance jobs separately. Check design https://github.com/vmware-tanzu/velero/blob/main/design/Implemented/priority-class-name-support_design.md for more details. #### Scalability and Resiliency improvements of data movers ##### Reduce excessive number of data mover pods in Pending state In v1.17, Velero allows users to set a `PrepareQueueLength` in the node-agent configuration, data mover pods and volumes out of this number won't be created until data path quota is available, so that excessive number cluster resources won't be taken unnecessarily, which is particularly helpful for large scale environments. This improvement applies to all kinds of data movements, including fs-backup and CSI snapshot data movement. Check design https://github.com/vmware-tanzu/velero/blob/main/design/node-agent-load-soothing.md for more details. ##### Enhancement on node-agent restart handling for data movements In v1.17, data movements in all phases could survive from node-agent restart and resume themselves; when a data movement gets orphaned in special cases, e.g., cluster node absent, it could also be canceled appropriately after the restart. This improvement applies to all kinds of data movements, including fs-backup and CSI snapshot data movement. Check issue https://github.com/vmware-tanzu/velero/issues/8534 for more details. ##### CSI snapshot data movement restore node-selection and node-selection by storage class In v1.17, CSI snapshot data movement restore acquires the same node-selection capability as backup, that is, users could specify which nodes can/cannot run data mover pods for both backup and restore now. And users are also allowed to configure the node-selection per storage class, which is particularly helpful to the environments where a storage class are not usable by all cluster nodes. Check issue https://github.com/vmware-tanzu/velero/issues/8186 and https://github.com/vmware-tanzu/velero/issues/8223 for more details. #### Include/exclude policy support for resource policy In v1.17, Velero resource policy supports `includeExcludePolicy` besides the existing `volumePolicy`. This allows users to set include/exclude filters for resources in a resource policy configmap, so that these filters are reusable among multiple backups. Check the document https://velero.io/docs/main/resource-filtering/#creating-resource-policies:~:text=resources%3D%22*%22-,Resource%20policies,-Velero%20provides%20resource for more details. ### Runtime and dependencies Golang runtime: 1.24.6 kopia: 0.21.1 ### Limitations/Known issues ### Breaking changes #### Deprecation of Restic According to [Velero deprecation policy](https://github.com/vmware-tanzu/velero/blob/main/GOVERNANCE.md#deprecation-policy), backup of fs-backup under Restic path is removed in v1.17, so `--uploader-type=restic` is not a valid installation configuration anymore. This means you cannot create a backup under Restic path, but you can still restore from the previous backups under Restic path until v1.19. #### Repository maintenance job configurations are removed from Velero server parameter Since the repository maintenance job configurations are moved to repository maintenance job configMap, in v1.17 below Velero sever parameters are removed: - --keep-latest-maintenance-jobs - --maintenance-job-cpu-request - --maintenance-job-mem-request - --maintenance-job-cpu-limit - --maintenance-job-mem-limit ### All Changes * Add ConfigMap parameters validation for install CLI and server start. (#9200, @blackpiglet) * Add priorityclasses to high priority restore list (#9175, @kaovilai) * Introduced context-based logger for backend implementations (Azure, GCS, S3, and Filesystem) (#9168, @priyansh17) * Fix issue #9140, add os=windows:NoSchedule toleration for Windows pods (#9165, @Lyndon-Li) * Remove the repository maintenance job parameters from velero server. (#9147, @blackpiglet) * Add include/exclude policy to resources policy (#9145, @reasonerjt) * Add ConfigMap support for keepLatestMaintenanceJobs with CLI parameter fallback (#9135, @shubham-pampattiwar) * Fix the dd and du's node affinity issue. (#9130, @blackpiglet) * Remove the WaitUntilVSCHandleIsReady from vs BIA. (#9124, @blackpiglet) * Add comprehensive Volume Group Snapshots documentation with workflow diagrams and examples (#9123, @shubham-pampattiwar) * Fix issue #9065, add doc for node-agent prepare queue length (#9118, @Lyndon-Li) * Fix issue #9095, update restore doc for PVC selected-node (#9117, @Lyndon-Li) * Update CSI Snapshot Data Movement doc for issue #8534, #8185 (#9113, @Lyndon-Li) * Fix issue #8986, refactor fs-backup doc after VGDP Micro Service for fs-backup (#9112, @Lyndon-Li) * Return error if timeout when checking server version (#9111, @ywk253100) * Update "Default Volumes to Fs Backup" to "File System Backup (Default)" (#9105, @shubham-pampattiwar) * Fix issue #9077, don't block backup deletion on list VS error (#9100, @Lyndon-Li) * Bump up Kopia to v0.21.1 (#9098, @Lyndon-Li) * Add imagePullSecrets inheritance for VGDP pod and maintenance job. (#9096, @blackpiglet) * Avoid checking the VS and VSC status in the backup finalizing phase. (#9092, @blackpiglet) * Fix issue #9053, Always remove selected-node annotation during PVC restore when no node mapping exists. Breaking change: Previously, the annotation was preserved if the node existed. (#9076, @Lyndon-Li) * Enable parameterized kubelet mount path during node-agent installation (#9074, @longxiucai) * Fix issue #8857, support third party tolerations for data mover pods (#9072, @Lyndon-Li) * Fix issue #8813, remove restic from the valid uploader type (#9069, @Lyndon-Li) * Fix issue #8185, allow users to disable pod volume host path mount for node-agent (#9068, @Lyndon-Li) * Fix #8344, add the design for a mechanism to soothe creation of data mover pods for DataUpload, DataDownload, PodVolumeBackup and PodVolumeRestore (#9067, @Lyndon-Li) * Fix #8344, add a mechanism to soothe creation of data mover pods for DataUpload, DataDownload, PodVolumeBackup and PodVolumeRestore (#9064, @Lyndon-Li) * Add Gauge metric for BSL availability (#9059, @reasonerjt) * Fix missing defaultVolumesToFsBackup flag output in Velero describe backup cmd (#9056, @shubham-pampattiwar) * Allow for proper tracking of multiple hooks per container (#9048, @sseago) * Make the backup repository controller doesn't invalidate the BSL on restart (#9046, @blackpiglet) * Removed username/password credential handling from newConfigCredential as azidentity.UsernamePasswordCredentialOptions is reported as deprecated. (#9041, @priyansh17) * Remove dependency with VolumeSnapshotClass in DataUpload. (#9040, @blackpiglet) * Fix issue #8961, cancel PVB/PVR on Velero server restart (#9031, @Lyndon-Li) * Fix issue #8962, resume PVB/PVR during node-agent restarts (#9030, @Lyndon-Li) * Bump kopia v0.20.1 (#9027, @Lyndon-Li) * Fix issue #8965, support PVB/PVR's cancel state in the backup/restore (#9026, @Lyndon-Li) * Fix Issue 8816 When specifying LabelSelector on restore, related items such as PVC and VolumeSnapshot are not included (#9024, @amastbau) * Fix issue #8963, add legacy PVR controller for Restic path (#9022, @Lyndon-Li) * Fix issue #8964, add Windows support for VGDP MS for fs-backup (#9021, @Lyndon-Li) * Accommodate VGS workflows in PVC CSI plugin (#9019, @shubham-pampattiwar) * Fix issue #8958, add VGDP MS PVB controller (#9015, @Lyndon-Li) * Fix issue #8959, add VGDP MS PVR controller (#9014, @Lyndon-Li) * Fix issue #8988, add data path for VGDP ms PVR (#9005, @Lyndon-Li) * Fix issue #8988, add data path for VGDP ms pvb (#8998, @Lyndon-Li) * Skip VS and VSC not created by backup. (#8990, @blackpiglet) * Make ResticIdentifier optional for kopia BackupRepositories (#8987, @kaovilai) * Fix issue #8960, implement PodVolume exposer for PVB/PVR (#8985, @Lyndon-Li) * fix: update mc command in minio-deployment example (#8982, @vishal-chdhry) * Fix issue #8957, add design for VGDP MS for fs-backup (#8979, @Lyndon-Li) * Add BSL status check for backup/restore operations. (#8976, @blackpiglet) * Mark BackupRepository not ready when BSL changed (#8975, @ywk253100) * Add support for [distributed snapshotting](https://github.com/kubernetes-csi/external-snapshotter/tree/4cedb3f45790ac593ebfa3324c490abedf739477?tab=readme-ov-file#distributed-snapshotting) (#8969, @flx5) * Fix issue #8534, refactor dm controllers to tolerate cancel request in more cases, e.g., node restart, node drain (#8952, @Lyndon-Li) * The backup and restore VGDP affinity enhancement implementation. (#8949, @blackpiglet) * Remove CSI VS and VSC metadata from backup. (#8946, @blackpiglet) * Extend PVCAction itemblock plugin to support grouping PVCs under VGS label key (#8944, @shubham-pampattiwar) * Copy security context from origin pod (#8943, @farodin91) * Add support for configuring VGS label key (#8938, @shubham-pampattiwar) * Add VolumeSnapshotContent into the RIA and the mustHave resource list. (#8924, @blackpiglet) * Mounted cloud credentials should not be world-readable (#8919, @sseago) * Warn for not found error in patching managed fields (#8902, @sseago) * Fix issue 8878, relief node os deduction error checks (#8891, @Lyndon-Li) * Skip namespace in terminating state in backup resource collection. (#8890, @blackpiglet) * Implement PriorityClass Support (#8883, @kaovilai) * Fix Velero adding restore-wait init container when not needed. (#8880, @kaovilai) * Pass the logger in kopia related operations. (#8875, @hu-keyu) * Inherit the dnsPolicy and dnsConfig from the node agent pod. This is done so that the kopia task uses the same configuration. (#8845, @flx5) * Add design for VolumeGroupSnapshot support (#8778, @shubham-pampattiwar) * Inherit k8s default volumeSnapshotClass. (#8719, @hu-keyu) * CLI automatically discovers and uses cacert from BSL for download requests (#8557, @kaovilai) * This PR aims to add s390x support to Velero binary. (#7505, @pandurangkhandeparker) ================================================ FILE: changelogs/CHANGELOG-1.18.md ================================================ ## v1.18 ### Download https://github.com/vmware-tanzu/velero/releases/tag/v1.18.0 ### Container Image `velero/velero:v1.18.0` ### Documentation https://velero.io/docs/v1.18/ ### Upgrading https://velero.io/docs/v1.18/upgrade-to-1.18/ ### Highlights #### Concurrent backup In v1.18, Velero is capable to process multiple backups concurrently. This is a significant usability improvement, especially for multiple tenants or multiple users case, backups submitted from different users could run their backups simultaneously without interfering with each other. Check design https://github.com/vmware-tanzu/velero/blob/main/design/Implemented/concurrent-backup-processing.md for more details. #### Cache volume for data movers In v1.18, Velero allows users to configure cache volumes for data mover pods during restore for CSI snapshot data movement and fs-backup. This brings below benefits: - Solve the problem that data mover pods fail to when pod's ephemeral disk is limited - Solve the problem that multiple data mover pods fail to run concurrently in one node when the node's ephemeral disk is limited - Working together with backup repository's cache limit configuration, cache volume with appropriate size helps to improve the restore throughput Check design https://github.com/vmware-tanzu/velero/blob/main/design/Implemented/backup-repo-cache-volume.md for more details. #### Incremental size for data movers In v1.18, Velero allows users to observe the incremental size of data movers backups for CSI snapshot data movement and fs-backup, so that users could visually see the data reduction due to incremental backup. #### Wildcard support for namespaces In v1.18, Velero allows to use Glob regular expressions for namespace filters during backup and restore, so that users could filter namespaces in a batch manner. #### VolumePolicy for PVC phase In v1.18, Velero VolumePolicy supports actions by PVC phase, which help users to do special operations for PVCs with a specific phase, e.g., skip PVCs in Pending/Lost status from the backup. #### Scalability and Resiliency improvements ##### Prevent Velero server OOM Kill for large backup repositories In v1.18, some backup repository operations are delay executed out of Velero server, so Velero server won't be OOM Killed. #### Performance improvement for VolumePolicy In v1.18, VolumePolicy is enhanced for large number of pods/PVCs so that the performance is significantly improved. #### Events for data mover pod diagnostic In v1.18, events are recorded into data mover pod diagnostic, which allows user to see more information for troubleshooting when the data mover pod fails. ### Runtime and dependencies Golang runtime: 1.25.7 kopia: 0.22.3 ### Limitations/Known issues ### Breaking changes #### Deprecation of PVC selected node feature According to [Velero deprecation policy](https://github.com/vmware-tanzu/velero/blob/main/GOVERNANCE.md#deprecation-policy), PVC selected node feature is deprecated in v1.18. Velero could appropriately handle PVC's selected-node annotation, so users don't need to do anything particularly. ### All Changes * Remove backup from running list when backup fails validation (#9498, @sseago) * Maintenance Job only uses the first element of the LoadAffinity array (#9494, @blackpiglet) * Fix issue #9478, add diagnose info on expose peek fails (#9481, @Lyndon-Li) * Add Role, RoleBinding, ClusterRole, and ClusterRoleBinding in restore sequence. (#9474, @blackpiglet) * Add maintenance job and data mover pod's labels and annotations setting. (#9452, @blackpiglet) * Fix plugin init container names exceeding DNS-1123 limit (#9445, @mpryc) * Add PVC-to-Pod cache to improve volume policy performance (#9441, @shubham-pampattiwar) * Remove VolumeSnapshotClass from CSI B/R process. (#9431, @blackpiglet) * Use hookIndex for recording multiple restore exec hooks. (#9366, @blackpiglet) * Sanitize Azure HTTP responses in BSL status messages (#9321, @shubham-pampattiwar) * Remove labels associated with previous backups (#9206, @Joeavaikath) * Add VolumePolicy support for PVC Phase conditions to allow skipping Pending PVCs (#9166, @claude) * feat: Enhance BackupStorageLocation with Secret-based CA certificate support (#9141, @kaovilai) * Add `--apply` flag to `install` command, allowing usage of Kubernetes apply to make changes to existing installs (#9132, @mjnagel) * Fix issue #9194, add doc for GOMAXPROCS behavior change (#9420, @Lyndon-Li) * Apply volume policies to VolumeGroupSnapshot PVC filtering (#9419, @shubham-pampattiwar) * Fix issue #9276, add doc for cache volume support (#9418, @Lyndon-Li) * Add Prometheus metrics for maintenance jobs (#9414, @shubham-pampattiwar) * Fix issue #9400, connect repo first time after creation so that init params could be written (#9407, @Lyndon-Li) * Cache volume for PVR (#9397, @Lyndon-Li) * Cache volume support for DataDownload (#9391, @Lyndon-Li) * don't copy securitycontext from first container if configmap found (#9389, @sseago) * Refactor repo provider interface for static configuration (#9379, @Lyndon-Li) * Fix issue #9365, prevent fake completion notification due to multiple update of single PVR (#9375, @Lyndon-Li) * Add cache volume configuration (#9370, @Lyndon-Li) * Track actual resource names for GenerateName in restore status (#9368, @shubham-pampattiwar) * Fix managed fields patch for resources using GenerateName (#9367, @shubham-pampattiwar) * Support cache volume for generic restore exposer and pod volume exposer (#9362, @Lyndon-Li) * Add incrementalSize to DU/PVB for reporting new/changed size (#9357, @sseago) * Add snapshotSize for DataDownload, PodVolumeRestore (#9354, @Lyndon-Li) * Add cache dir configuration for udmrepo (#9353, @Lyndon-Li) * Fix the Job build error when BackupReposiotry name longer than 63. (#9350, @blackpiglet) * Add cache configuration to VGDP (#9342, @Lyndon-Li) * Fix issue #9332, add bytesDone for cache files (#9333, @Lyndon-Li) * Fix typos in documentation (#9329, @T4iFooN-IX) * Concurrent backup processing (#9307, @sseago) * VerifyJSONConfigs verify every elements in Data. (#9302, @blackpiglet) * Fix issue #9267, add events to data mover prepare diagnostic (#9296, @Lyndon-Li) * Add option for privileged fs-backup pod (#9295, @sseago) * Fix issue #9193, don't connect repo in repo controller (#9291, @Lyndon-Li) * Implement concurrency control for cache of native VolumeSnapshotter plugin. (#9281, @0xLeo258) * Fix issue #7904, remove the code and doc for PVC node selection (#9269, @Lyndon-Li) * Fix schedule controller to prevent backup queue accumulation during extended blocking scenarios by properly handling empty backup phases (#9264, @shubham-pampattiwar) * Fix repository maintenance jobs to inherit allowlisted tolerations from Velero deployment (#9256, @shubham-pampattiwar) * Implement wildcard namespace pattern expansion for backup namespace includes/excludes. This change adds support for wildcard patterns (*, ?, [abc], {a,b,c}) in namespace includes and excludes during backup operations (#9255, @Joeavaikath) * Protect VolumeSnapshot field from race condition during multi-thread backup (#9248, @0xLeo258) * Update AzureAD Microsoft Authentication Library to v1.5.0 (#9244, @priyansh17) * Get pod list once per namespace in pvc IBA (#9226, @sseago) * Fix issue #7725, add design for backup repo cache configuration (#9148, @Lyndon-Li) * Fix issue #9229, don't attach backupPVC to the source node (#9233, @Lyndon-Li) * feat: Permit specifying annotations for the BackupPVC (#9173, @clementnuss) ================================================ FILE: changelogs/CHANGELOG-1.2.md ================================================ ## v1.2.0 #### 2019-11-07 ### Download https://github.com/vmware-tanzu/velero/releases/tag/v1.2.0 ### Container Image `velero/velero:v1.2.0` Please note that as of this release we are no longer publishing new container images to `gcr.io/heptio-images`. The existing ones will remain there for the foreseeable future. ### Documentation https://velero.io/docs/v1.2.0/ ### Upgrading https://velero.io/docs/v1.2.0/upgrade-to-1.2/ ### Highlights ## Moving Cloud Provider Plugins Out of Tree Velero has had built-in support for AWS, Microsoft Azure, and Google Cloud Platform (GCP) since day 1. When Velero moved to a plugin architecture for object store providers and volume snapshotters in version 0.6, the code for these three providers was converted to use the plugin interface provided by this new architecture, but the cloud provider code still remained inside the Velero codebase. This put the AWS, Azure, and GCP plugins in a different position compared with other providers’ plugins, since they automatically shipped with the Velero binary and could include documentation in-tree. With version 1.2, we’ve extracted the AWS, Azure, and GCP plugins into their own repositories, one per provider. We now also publish one plugin image per provider. This change brings these providers to parity with other providers’ plugin implementations, reduces the size of the core Velero binary by not requiring each provider’s SDK to be included, and opens the door for the plugins to be maintained and released independently of core Velero. ## Restic Integration Improvements We’ve continued to work on improving Velero’s restic integration. With this release, we’ve made the following enhancements: - Restic backup and restore progress is now captured during execution and visible to the user through the `velero backup/restore describe --details` command. The details are updated every 10 seconds. This provides a new level of visibility into restic operations for users. - Restic backups of persistent volume claims (PVCs) now remain incremental across the rescheduling of a pod. Previously, if the pod using a PVC was rescheduled, the next restic backup would require a full rescan of the volume’s contents. This improvement potentially makes such backups significantly faster. - Read-write-many volumes are no longer backed up once for every pod using the volume, but instead just once per Velero backup. This improvement speeds up backups and prevents potential restore issues due to multiple copies of the backup being processed simultaneously. ## Clone PVs When Cloning a Namespace Before version 1.2, you could clone a Kubernetes namespace by backing it up and then restoring it to a different namespace in the same cluster by using the `--namespace-mappings` flag with the `velero restore create` command. However, in this scenario, Velero was unable to clone persistent volumes used by the namespace, leading to errors for users. In version 1.2, Velero automatically detects when you are trying to clone an existing namespace, and clones the persistent volumes used by the namespace as well. This doesn’t require the user to specify any additional flags for the `velero restore create` command. This change lets you fully achieve your goal of cloning namespaces using persistent storage within a cluster. ## Improved Server-Side Encryption Support To help you secure your important backup data, we’ve added support for more forms of server-side encryption of backup data on both AWS and GCP. Specifically: - On AWS, Velero now supports Amazon S3-managed encryption keys (SSE-S3), which uses AES256 encryption, by specifying `serverSideEncryption: AES256` in a backup storage location’s config. - On GCP, Velero now supports using a specific Cloud KMS key for server-side encryption by specifying `kmsKeyName: ` in a backup storage location’s config. ## CRD Structural Schema In Kubernetes 1.16, custom resource definitions (CRDs) reached general availability. Structural schemas are required for CRDs created in the `apiextensions.k8s.io/v1` API group. Velero now defines a structural schema for each of its CRDs and automatically applies it the user runs the `velero install` command. The structural schemas enable the user to get quicker feedback when their backup, restore, or schedule request is invalid, so they can immediately remediate their request. ### All Changes * Ensure object store plugin processes are cleaned up after restore and after BSL validation during server start up (#2041, @betta1) * bug fix: don't try to restore pod volume backups that don't have a snapshot ID (#2031, @skriss) * Restore Documentation: Updated Restore Documentation with Clarification implications of removing restore object. (#1957, @nainav) * add `--allow-partially-failed` flag to `velero restore create` for use with `--from-schedule` to allow partially-failed backups to be restored (#1994, @skriss) * Allow backup storage locations to specify backup sync period or toggle off sync (#1936, @betta1) * Remove cloud provider code (#1985, @carlisia) * Restore action for cluster/namespace role bindings (#1974, @alexander-demichev) * Add `--no-default-backup-location` flag to `velero install` (#1931, @Frank51) * If includeClusterResources is nil/auto, pull in necessary CRDs in backupResource (#1831, @sseago) * Azure: add support for Azure China/German clouds (#1938, @andyzhangx) * Add a new required `--plugins` flag for `velero install` command. `--plugins` takes a list of container images to add as initcontainers. (#1930, @nrb) * restic: only backup read-write-many PVCs at most once, even if they're annotated for backup from multiple pods. (#1896, @skriss) * Azure: add support for cross-subscription backups (#1895, @boxcee) * adds `insecureSkipTLSVerify` server config for AWS storage and `--insecure-skip-tls-verify` flag on client for self-signed certs (#1793, @s12chung) * Add check to update resource field during backupItem (#1904, @spiffcs) * Add `LD_LIBRARY_PATH` (=/plugins) to the env variables of velero deployment. (#1893, @lintongj) * backup sync controller: stop using `metadata/revision` file, do a full diff of bucket contents vs. cluster contents each sync interval (#1892, @skriss) * bug fix: during restore, check item's original namespace, not the remapped one, for inclusion/exclusion (#1909, @skriss) * adds structural schema to Velero CRDs created on Velero install, enabling validation of Velero API fields (#1898, @prydonius) * GCP: add support for specifying a Cloud KMS key name to use for encrypting backups in a storage location. (#1879, @skriss) * AWS: add support for SSE-S3 AES256 encryption via `serverSideEncryption` config field in BackupStorageLocation (#1869, @skriss) * change default `restic prune` interval to 7 days, add `velero server/install` flags for specifying an alternate default value. (#1864, @skriss) * velero install: if `--use-restic` and `--wait` are specified, wait up to a minute for restic daemonset to be ready (#1859, @skriss) * report restore progress in PodVolumeRestores and expose progress in the velero restore describe --details command (#1854, @prydonius) * Jekyll Site updates - modifies documentation to use a wider layout; adds better markdown table formatting (#1848, @ccbayer) * fix excluding additional items with the velero.io/exclude-from-backup=true label (#1843, @prydonius) * report backup progress in PodVolumeBackups and expose progress in the velero backup describe --details command. Also upgrades restic to v0.9.5 (#1821, @prydonius) * Add `--features` argument to all velero commands to provide feature flags that can control enablement of pre-release features. (#1798, @nrb) * when backing up PVCs with restic, specify `--parent` flag to prevent full volume rescans after pod reschedules (#1807, @skriss) * remove 'restic check' calls from before/after 'restic prune' since they're redundant (#1794, @skriss) * fix error formatting due interpreting % as printf formatted strings (#1781, @s12chung) * when using `velero restore create --namespace-mappings ...` to create a second copy of a namespace in a cluster, create copies of the PVs used (#1779, @skriss) * adds --from-schedule flag to the `velero create backup` command to create a Backup from an existing Schedule (#1734, @prydonius) ================================================ FILE: changelogs/CHANGELOG-1.3.md ================================================ ## v1.3.2 ### 2020-04-03 ### Download https://github.com/vmware-tanzu/velero/releases/tag/v1.3.2 ### Container Image `velero/velero:v1.3.2` ### Documentation https://velero.io/docs/v1.3.2/ ### Upgrading https://velero.io/docs/v1.3.2/upgrade-to-1.3/ ### All Changes * Allow `plugins/` as a valid top-level directory within backup storage locations. This directory is a place for plugin authors to store arbitrary data as needed. It is recommended to create an additional subdirectory under `plugins/` specifically for your plugin, e.g. `plugins/my-plugin-data/`. (#2350, @skriss) * bug fix: don't panic in `velero restic repo get` when last maintenance time is `nil` (#2315, @skriss) ## v1.3.1 ### 2020-03-10 ### Download https://github.com/vmware-tanzu/velero/releases/tag/v1.3.1 ### Container Image `velero/velero:v1.3.1` ### Documentation https://velero.io/docs/v1.3.1/ ### Upgrading https://velero.io/docs/v1.3.1/upgrade-to-1.3/ ### Highlights Fixed a bug that caused failures when backing up CustomResourceDefinitions with whole numbers in numeric fields. ### All Changes * Fix CRD backup failures when fields contained a whole number. (#2322, @nrb) ## v1.3.0 #### 2020-03-02 ### Download https://github.com/vmware-tanzu/velero/releases/tag/v1.3.0 ### Container Image `velero/velero:v1.3.0` ### Documentation https://velero.io/docs/v1.3.0/ ### Upgrading https://velero.io/docs/v1.3.0/upgrade-to-1.3/ ### Highlights #### Custom Resource Definition Backup and Restore Improvements This release includes a number of related bug fixes and improvements to how Velero backs up and restores custom resource definitions (CRDs) and instances of those CRDs. We found and fixed three issues around restoring CRDs that were originally created via the `v1beta1` CRD API. The first issue affected CRDs that had the `PreserveUnknownFields` field set to `true`. These CRDs could not be restored into 1.16+ Kubernetes clusters, because the `v1` CRD API does not allow this field to be set to `true`. We added code to the restore process to check for this scenario, to set the `PreserveUnknownFields` field to `false`, and to instead set `x-kubernetes-preserve-unknown-fields` to `true` in the OpenAPIv3 structural schema, per Kubernetes guidance. For more information on this, see the [Kubernetes documentation](https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/#pruning-versus-preserving-unknown-fields). The second issue affected CRDs without structural schemas. These CRDs need to be backed up/restored through the `v1beta1` API, since all CRDs created through the `v1` API must have structural schemas. We added code to detect these CRDs and always back them up/restore them through the `v1beta1` API. Finally, related to the previous issue, we found that our restore code was unable to handle backups with multiple API versions for a given resource type, and we’ve remediated this as well. We also improved the CRD restore process to enable users to properly restore CRDs and instances of those CRDs in a single restore operation. Previously, users found that they needed to run two separate restores: one to restore the CRD(s), and another to restore instances of the CRD(s). This was due to two deficiencies in the Velero code. First, Velero did not wait for a CRD to be fully accepted by the Kubernetes API server and ready for serving before moving on; and second, Velero did not refresh its cached list of available APIs in the target cluster after restoring CRDs, so it was not aware that it could restore instances of those CRDs. We fixed both of these issues by (1) adding code to wait for CRDs to be “ready” after restore before moving on, and (2) refreshing the cached list of APIs after restoring CRDs, so any instances of newly-restored CRDs could subsequently be restored. With all of these fixes and improvements in place, we hope that the CRD backup and restore experience is now seamless across all supported versions of Kubernetes. #### Multi-Arch Docker Images Thanks to community members [@Prajyot-Parab](https://github.com/Prajyot-Parab) and [@shaneutt](https://github.com/shaneutt), Velero now provides multi-arch container images by using Docker manifest lists. We are currently publishing images for `linux/amd64`, `linux/arm64`, `linux/arm`, and `linux/ppc64le` in [our Docker repository](https://hub.docker.com/r/velero/velero/tags?page=1&name=v1.3&ordering=last_updated). Users don’t need to change anything other than updating their version tag - the v1.3 image is `velero/velero:v1.3.0`, and Docker will automatically pull the proper architecture for the host. For more information on manifest lists, see [Docker’s documentation](https://docs.docker.com/registry/spec/manifest-v2-2/). #### Bug Fixes, Usability Enhancements, and More We fixed a large number of bugs and made some smaller usability improvements in this release. Here are a few highlights: - Support private registries with custom ports for the restic restore helper image ([PR #1999](https://github.com/vmware-tanzu/velero/pull/1999), [@cognoz](https://github.com/cognoz)) - Use AWS profile from BackupStorageLocation when invoking restic ([PR #2096](https://github.com/vmware-tanzu/velero/pull/2096), [@dinesh](https://github.com/dinesh)) - Allow restores from schedules in other clusters ([PR #2218](https://github.com/vmware-tanzu/velero/pull/2218), [@cpanato](https://github.com/cpanato)) - Fix memory leak & race condition in restore code ([PR #2201](https://github.com/vmware-tanzu/velero/pull/2201), [@skriss](https://github.com/skriss)) ### All Changes * Corrected the selfLink for Backup CR in site/docs/main/output-file-format.md (#2292, @RushinthJohn) * Back up schema-less CustomResourceDefinitions as v1beta1, even if they are retrieved via the v1 endpoint. (#2264, @nrb) * Bug fix: restic backup volume snapshot to the second location failed (#2244, @jenting) * Added support of using PV name from volumesnapshotter('SetVolumeID') in case of PV renaming during the restore (#2216, @mynktl) * Replaced deprecated helm repo url at all it appearance at docs. (#2209, @markrity) * added support for arm and arm64 images (#2227, @shaneutt) * when restoring from a schedule, validate by checking for backup(s) labeled with the schedule name rather than existence of the schedule itself, to allow for restoring from deleted schedules and schedules in other clusters (#2218, @cpanato) * bug fix: back up server-preferred version of CRDs rather than always the `v1beta1` version (#2230, @skriss) * Wait for CustomResourceDefinitions to be ready before restoring CustomResources. Also refresh the resource list from the Kubernetes API server after restoring CRDs in order to properly restore CRs. (#1937, @nrb) * When restoring a v1 CRD with PreserveUnknownFields = True, make sure that the preservation behavior is maintained by copying the flag into the Open API V3 schema, but update the flag so as to allow the Kubernetes API server to accept the CRD without error. (#2197, @nrb) * Enable pruning unknown CRD fields (#2187, @jenting) * bump restic to 0.9.6 to fix some issues with non AWS standard regions (#2210, @Sh4d1) * bug fix: fix race condition resulting in restores sometimes succeeding despite restic restore failures (#2201, @skriss) * Bug fix: Check for nil LastMaintenanceTime in ResticRepository dueForMaintenance (#2200, @sseago) * repopulate backup_last_successful_timestamp metrics for each schedule after server restart (#2196, @skriss) * added support for ppc64le images and manifest lists (#1768, @prajyot) * bug fix: only prioritize restoring `replicasets.apps`, not `replicasets.extensions` (#2157, @skriss) * bug fix: restore both `replicasets.apps` *and* `replicasets.extensions` before `deployments` (#2120, @skriss) * bug fix: don't restore cluster-scoped resources when restoring specific namespaces and IncludeClusterResources is nil (#2118, @skriss) * Enabling Velero to switch credentials (`AWS_PROFILE`) if multiple s3-compatible backupLocations are present (#2096, @dinesh) * bug fix: deep-copy backup's labels when constructing snapshot tags, so the PV name isn't added as a label to the backup (#2075, @skriss) * remove the `fsfreeze-pause` image being published from this repo; replace it with `ubuntu:bionic` in the nginx example app (#2068, @skriss) * add support for a private registry with a custom port in a restic-helper image (#1999, @cognoz) * return better error message to user when cluster config can't be found via `--kubeconfig`, `$KUBECONFIG`, or in-cluster config (#2057, @skriss) ================================================ FILE: changelogs/CHANGELOG-1.4.md ================================================ ## v1.4.2 ### 2020-07-13 ### Download https://github.com/vmware-tanzu/velero/releases/tag/v1.4.2 ### Container Image `velero/velero:v1.4.2` ### Documentation https://velero.io/docs/v1.4/ ### Upgrading https://velero.io/docs/v1.4/upgrade-to-1.4/ ### All Changes * log a warning instead of erroring if an additional item returned from a plugin can't be found in the Kubernetes API (#2595, @skriss) * Adjust restic default time out to 4 hours and base pod resource requests to 500m CPU/512Mi memory. (#2696, @nrb) * capture version of the CRD prior before invoking the remap_crd_version backup item action (#2683, @ashish-amarnath) ## v1.4.1 This tag was created in code, but has no associated docker image due to misconfigured building infrastructure. v1.4.2 fixes this. ## v1.4.0 ### 2020-05-26 ### Download https://github.com/vmware-tanzu/velero/releases/tag/v1.4.0 ### Container Image `velero/velero:v1.4.0` ### Documentation https://velero.io/docs/v1.4/ ### Upgrading https://velero.io/docs/v1.4/upgrade-to-1.4/ ### Highlights * Added beta-level CSI support! * Added custom CA certificate support * Backup progress reporting * Changed backup tarball format to support all versions of a given resource ### All Changes * increment restic volumesnapshot count after successful pvb create (#2542, @ashish-amarnath) * Add details of CSI volumesnapshotcontents associated with a backup to `velero backup describe` when the `EnableCSI` feature flag is given on the velero client. (#2448, @nrb) * Allow users the option to retrieve all versions of a given resource (instead of just the preferred version) from the API server with the `EnableAPIGroupVersions` feature flag. (#2373, @brito-rafa) * Changed backup tarball format to store all versions of a given resource, updated backup tarball format to 1.1.0. (#2373, @brito-rafa) * allow feature flags to be passed from install CLI (#2503, @ashish-amarnath) * sync backups' CSI API objects into the cluster as part of the backup sync controller (#2496, @ashish-amarnath) * bug fix: in error location logging hook, if the item logged under the `error` key doesn't implement the `error` interface, don't return an error since this is a valid scenario (#2487, @skriss) * bug fix: in CRD restore plugin, don't use runtime.DefaultUnstructuredConverter.FromUnstructured(...) to avoid conversion issues when float64 fields contain int values (#2484, @skriss) * during backup deletion also delete CSI volumesnapshotcontents that were created as a part of the backup but the associated volumesnapshot object does not exist (#2480, @ashish-amarnath) * If plugins don't support the `--features` flag, don't pass it to them. Also, update the standard plugin server to ignore unknown flags. (#2479, @skriss) * At backup time, if a CustomResourceDefinition appears to have been created via the v1beta1 endpoint, retrieve it from the v1beta1 endpoint instead of simply changing the APIVersion. (#2478, @nrb) * update container base images from ubuntu:bionic to ubuntu:focal (#2471, @skriss) * bug fix: when a resource includes/excludes list contains unresolvable items, don't remove them from the list, so that the list doesn't inadvertently end up matching *all* resources. (#2462, @skriss) * Azure: add support for getting storage account key for restic directly from an environment variable (#2455, @jaygridley) * Support to skip VSL validation for the backup having SnapshotVolumes set to false or created with `--snapshot-volumes=false` (#2450, @mynktl) * report backup progress (number of items backed up so far out of an estimated total number of items) during backup in the logs and as status fields on the Backup custom resource (#2440, @skriss) * bug fix: populate namespace in logs for backup errors (#2438, @skriss) * during backup deletion also delete CSI volumesnapshots that were created as a part of the backup (#2411, @ashish-amarnath) * bump Kubernetes module dependencies to v0.17.4 to get fix for https://github.com/kubernetes/kubernetes/issues/86149 (#2407, @skriss) * bug fix: save PodVolumeBackup manifests to object storage even if the volume was empty, so that on restore, the PV is dynamically reprovisioned if applicable (#2390, @skriss) * Adding new restoreItemAction for PVC to update the selected-node annotation (#2377, @mynktl) * Added a --cacert flag to the install command to provide the CA bundle to use when verifying TLS connections to object storage (#2368, @mansam) * Added a `--cacert` flag to the velero client describe, download, and logs commands to allow passing a path to a certificate to use when verifying TLS connections to object storage. Also added a corresponding client config option called `cacert` which takes a path to a certificate bundle to use as a default when `--cacert` is not specified. (#2364, @mansam) * support setting a custom CA certificate on a BSL to use when verifying TLS connections (#2353, @mansam) * adding annotations on backup CRD for k8s major, minor and git versions (#2346, @brito-rafa) * When the EnableCSI feature flag is provided, upload CSI VolumeSnapshots and VolumeSnapshotContents to object storage as gzipped JSON. (#2323, @nrb) * add CSI snapshot API types into default restore priorities (#2318, @ashish-amarnath) * refactoring: wait for all informer caches to sync before running controllers (#2299, @skriss) * refactor restore code to lazily resolve resources via discovery and eliminate second restore loop for instances of restored CRDs (#2248, @skriss) * upgrade to go 1.14 and migrate from `dep` to go modules (#2214, @skriss) * clarify the wording for restore describe for namespaces included ================================================ FILE: changelogs/CHANGELOG-1.5.md ================================================ ## v1.5.1 ### 2020-09-16 ### Download https://github.com/vmware-tanzu/velero/releases/tag/v1.5.1 ### Container Image `velero/velero:v1.5.1` ### Documentation https://velero.io/docs/v1.5/ ### Upgrading https://velero.io/docs/v1.5/upgrade-to-1.5/ ### Highlights * Auto Volume Backup Using Restic with `--default-volumes-to-restic` flag * DeleteItemAction plugins * Code modernization * Restore Hooks: InitContianer Restore Hooks and Exec Restore Hooks ### All Changes * 🏃‍♂️ add shortnames for CRDs (#2911, @ashish-amarnath) * Use format version instead of version on `velero backup describe` since version has been deprecated (#2901, @jenting) * fix EnableAPIGroupersions output log format (#2882, @jenting) * Convert ServerStatusRequest controller to kubebuilder (#2838, @carlisia) * rename the PV if VolumeSnapshotter has modified the PV name (#2835, @pawanpraka1) * Implement post-restore exec hooks in pod containers (#2804, @areed) * Check for errors on restic backup command (#2863, @dymurray) * 🐛 fix passing LDFLAGS across build stages (#2853, @ashish-amarnath) * Feature: Invoke DeleteItemAction plugins based on backup contents when a backup is deleted. (#2815, @nrb) * When JSON logging format is enabled, place error message at "error.message" instead of "error" for compatibility with Elasticsearch/ELK and the Elastic Common Schema (#2830, @bgagnon) * discovery Helper support get GroupVersionResource and an APIResource from GroupVersionKind (#2764, @runzexia) * Migrate site from Jekyll to Hugo (#2720, @tbatard) * Add the DeleteItemAction plugin type (#2808, @nrb) * 🐛 Manually patch the generated yaml for restore CRD as a hacky workaround (#2814, @ashish-amarnath) * Setup crd validation github action on k8s versions (#2805, @ashish-amarnath) * 🐛 Supply command to run restic-wait init container (#2802, @ashish-amarnath) * Make init and exec restore hooks as optional in restore hookSpec (#2793, @ashish-amarnath) * Implement restore hooks injecting init containers into pod spec (#2787, @ashish-amarnath) * Pass default-volumes-to-restic flag from create schedule to backup (#2776, @ashish-amarnath) * Enhance Backup to support backing up resources in specific orders and add --ordered-resources option to support this feature. (#2724, @phuong) * Fix inconsistent type for the "resource" structured logging field (#2796, @bgagnon) * Add the ability to set the allowPrivilegeEscalation flag in the securityContext for the Restic restore helper. (#2792, @doughepi) * Add cacert flag for velero backup-location create (#2778, @jenting) * Exclude volumes mounting secrets and configmaps from defaulting volume backups to restic (#2762, @ashish-amarnath) * Add types to implement restore hooks (#2761, @ashish-amarnath) * Add wait group and error channel for restore hooks to restore context. (#2755, @areed) * Refactor image builds to use buildx for multi arch image building (#2754, @robreus) * Add annotation key constants for restore hooks (#2750, @ashish-amarnath) * Adds Start and CompletionTimestamp to RestoreStatus Displays the Timestamps when issued a print or describe (#2748, @thejasbabu) * Move pkg/backup/item_hook_handlers.go to internal/hook (#2734, @nrb) * add metrics for restic back up operation (#2719, @ashish-amarnath) * StorageGrid compatibility by removing explicit gzip accept header setting (#2712, @fvsqr) * restic: add support for setting SecurityContext (runAsUser, runAsGroup) for restore (#2621, @jaygridley) * Add backupValidationFailureTotal to metrics (#2714, @kathpeony) * bump Kubernetes module dependencies to v0.18.4 to fix https://github.com/vmware-tanzu/velero/issues/2540 by adding code compatibility with kubernetes v1.18 (#2651, @laverya) * Add a BSL controller to handle validation + update BSL status phase (validation removed from the server and no longer blocks when there's any invalid BSL) (#2674, @carlisia) * updated acceptable values on cron schedule from 0-7 to 0-6 (#2676, @dthrasher) * Improve velero download doc (#2660, @carlisia) * Update basic-install and release-instructions documentation for Windows Chocolatey package (#2638, @adamrushuk) * move CSI plugin out of prototype into beta (#2636, @ashish-amarnath) * Add a new supported provider for an object storage plugin for Storj (#2635, @jessicagreben) * Update basic-install.md documentation: Add windows cli installation option via chocolatey (#2629, @adamrushuk) * Documentation: Update Jekyll to 4.1.0. Switch from redcarpet to kramdown for Markdown renderer (#2625, @tbatard) * improve builder image handling so that we don't rebuild each `make shell` (#2620, @mauilion) * first check if there are pending changed on the build-image dockerfile if so build it. * then check if there is an image in the registry if so pull it. * then build an image cause we don't have a cached image. (this handles the backward compat case.) * fix make clean to clear go mod cache before removing dirs (for containerized builds) * Add linter checks to Makefile (#2615, @tbatard) * add a CI check for a changelog file (#2613, @ashish-amarnath) * implement option to back up all volumes by default with restic (#2611, @ashish-amarnath) * When a timeout string can't be parsed, log the error as a warning instead of silently consuming the error. (#2610, @nrb) * Azure: support using `aad-pod-identity` auth when using restic (#2602, @skriss) * log a warning instead of erroring if an additional item returned from a plugin can't be found in the Kubernetes API (#2595, @skriss) * when creating new backup from schedule from cli, allow backup name to be automatically generated (#2569, @cblecker) * Convert manifests + BSL api client to kubebuilder (#2561, @carlisia) * backup/restore: reinstantiate backup store just before uploading artifacts to ensure credentials are up-to-date (#2550, @skriss) ================================================ FILE: changelogs/CHANGELOG-1.6.md ================================================ ## v1.6.0 ### 2021-04-12 ### Download https://github.com/vmware-tanzu/velero/releases/tag/v1.6.0 ### Container Image `velero/velero:v1.6.0` ### Documentation https://velero.io/docs/v1.6/ ### Upgrading https://velero.io/docs/v1.6/upgrade-to-1.6/ ### Highlights * Support for per-BSL credentials * Progress reporting for restores * Restore API Groups by priority level * Restic v0.12.0 upgrade * End-to-end testing * CLI usability improvements ### All Changes * Add support for restic to use per-BSL credentials. Velero will now serialize the secret referenced by the `Credential` field in the BSL and use this path when setting provider specific environment variables for restic commands. (#3489, @zubron) * Upgrade restic from v0.9.6 to v0.12.0. (#3528, @ashish-amarnath) * Progress reporting added for Velero Restores (#3125, @pranavgaikwad) * Add uninstall option for velero cli (#3399, @vadasambar) * Add support for per-BSL credentials. Velero will now serialize the secret referenced by the `Credential` field in the BSL and pass this path through to Object Storage plugins via the `config` map using the `credentialsFile` key. (#3442, @zubron) * Fixed a bug where restic volumes would not be restored when using a namespace mapping. (#3475, @zubron) * Restore API group version by priority. Increase timeout to 3 minutes in DeploymentIsReady(...) function in the install package (#3133, @codegold79) * Add field and cli flag to associate a credential with a BSL on BSL create|set. (#3190, @carlisia) * Add colored output to `describe schedule/backup/restore` commands (#3275, @mike1808) * Add CAPI Cluster and ClusterResourceSets to default restore priorities so that the capi-controller-manager does not panic on restores. (#3446, @nrb) * Use label to select Velero deployment in plugin cmd (#3447, @codegold79) * feat: support setting BackupStorageLocation CA certificate via `velero backup-location set --cacert` (#3167, @jenting) * Add restic initContainer length check in pod volume restore to prevent restic plugin container disappear in runtime (#3198, @shellwedance) * Bump versions of external snapshotter and others in order to make `go get` to succeed (#3202, @georgettica) * Support fish shell completion (#3231, @jenting) * Change the logging level of PV deletion timeout from Debug to Warn (#3316, @MadhavJivrajani) * Set the BSL created at install time as the "default" (#3172, @carlisia) * Capitalize all help messages (#3209, @jenting) * Increased default Velero pod memory limit to 512Mi (#3234, @dsmithuchida) * Fixed an issue where the deletion of a backup would fail if the backup tarball couldn't be downloaded from object storage. Now the tarball is only downloaded if there are associated DeleteItemAction plugins and if downloading the tarball fails, the plugins are skipped. (#2993, @zubron) * feat: add delete sub-command for BSL (#3073, @jenting) * 🐛 BSLs with validation disabled should be validated at least once (#3084, @ashish-amarnath) * feat: support configures BackupStorageLocation custom resources to indicate which one is the default (#3092, @jenting) * Added "--preserve-nodeports" flag to preserve original nodePorts when restoring. (#3095, @yusufgungor) * Owner reference in backup when created from schedule (#3127, @matheusjuvelino) * issue: add flag to the schedule cmd to configure the `useOwnerReferencesInBackup` option #3176 (#3182, @matheusjuvelino) * cli: allow creating multiple instances of Velero across two different namespaces (#2886, @alaypatel07) * Feature: It is possible to change the timezone of the container by specifying in the manifest.. env: [TZ: Zone/Country], or in the Helm Chart.. configuration: {extraEnvVars: [TZ: 'Zone/Country']} (#2944, @mickkael) * Fix issue where bare `velero` command returned an error code. (#2947, @nrb) * Restore CRD Resource name to fix CRD wait functionality. (#2949, @sseago) * Fixed 'velero.io/change-pvc-node-selector' plugin to fetch configmap using label key "velero.io/change-pvc-node-selector" (#2970, @mynktl) * Compile with Go 1.15 (#2974, @gliptak) * Fix BSL controller to avoid invoking init() on all BSLs regardless of ValidationFrequency (#2992, @betta1) * Ensure that bound PVCs and PVs remain bound on restore. (#3007, @nrb) * Allows the restic-wait container to exist in any order in the pod being restored. Prints a warning message in the case where the restic-wait container isn't the first container in the list of initialization containers. (#3011, @doughepi) * Add warning to velero version cmd if the client and server versions mismatch. (#3024, @cvhariharan) * 🐛 Use namespace and name to match PVB to Pod restore (#3051, @ashish-amarnath) * Fixed various typos across codebase (#3057, @invidian) * 🐛 ItemAction plugins for unresolvable types should not be run for all types (#3059, @ashish-amarnath) * Basic end-to-end tests, generate data/backup/remove/restore/verify. Uses distributed data generator (#3060, @dsu-igeek) * Added GitHub Workflow running Codespell for spell checking (#3064, @invidian) * Pass annotations from schedule to backup it creates the same way it is done for labels. Add WithannotationsMap function to builder to be able to pass map instead of key/val list (#3067, @funkycode) * Add instructions to clone repository for examples in docs (#3074, @MadhavJivrajani) * 🏃‍♂️ update setup-kind github actions CI (#3085, @ashish-amarnath) * Modify wrong function name to correct one. (#3106, @shellwedance) ================================================ FILE: changelogs/CHANGELOG-1.7.md ================================================ ## v1.7.0 ### 2021-09-07 ### Download https://github.com/vmware-tanzu/velero/releases/tag/v1.7.0 ### Container Image `velero/velero:v1.7.0` ### Documentation https://velero.io/docs/v1.7/ ### Upgrading https://velero.io/docs/v1.7/upgrade-to-1.7/ ### Highlights #### Distroless images The Velero container images now use [distroless base images](https://github.com/GoogleContainerTools/distroless). Using distroless images as the base ensures that only the packages and programs necessary for running Velero are included. Unrelated libraries and OS packages, that often contain security vulnerabilities, are now excluded. This change reduces the size of both the server and restic restore helper image by approximately 62MB. As the [distroless](https://github.com/GoogleContainerTools/distroless) images do not contain a shell, it will no longer be possible to exec into Velero containers using these images. #### New "debug" command This release introduces the new `velero debug` command. This command collects information about a Velero installation, such as pod logs and resources managed by Velero, in a tarball which can be provided to the Velero maintainer team to help diagnose issues. ### All changes * Distinguish between different unnamed node ports when preserving (#4026, @sseago) * Validate namespace in Velero backup create command (#4057, @codegold79) * Empty the "ClusterIPs" along with "ClusterIP" when "ClusterIP" isn't "None" (#4101, @ywk253100) * Add a RestoreItemAction plugin (`velero.io/apiservice`) which skips the restore of any `APIService` which is managed by Kubernetes. These are identified using the `kube-aggregator.kubernetes.io/automanaged` label. (#4028, @zubron) * Change the base image to distroless (#4055, @ywk253100) * Updated the version of velero/velero-plugin-for-aws version from v1.2.0 to v1.2.1 (#4064, @kahirokunn) * Skip the backup and restore of DownwardAPI volumes when using restic. (#4076, @zubron) * Bump up Go to 1.16 (#3990, @reasonerjt) * Fix restic error when volume is emptyDir and Pod not running (#3993, @mahaupt) * Select the velero deployment with both label and container name (#3996, @ywk253100) * Wait for the namespace to be deleted before removing the CRDs during uninstall. This deprecates the `--wait` flag of the `uninstall` command (#4007, @ywk253100) * Use the cluster preferred CRD API version when polling for Velero CRD readiness. (#4015, @zubron) * Implement velero debug (#4022, @reasonerjt) * Skip the restore of volumes that originally came from a projected volume when using restic. (#3877, @zubron) * Run the E2E test with kind(provision various versions of k8s cluster) and MinIO on Github Action (#3912, @ywk253100) * Fix -install-velero flag for e2e tests (#3919, @jaidevmane) * Upgrade Velero ClusterRoleBinding to use v1 API (#3926, @jenting) * enable e2e tests to choose crd apiVersion (#3941, @sseago) * Fixing multipleNamespaceTest bug - Missing expect statement in test (#3983, @jaidevmane) * Add --client-page-size flag to server to allow chunking Kubernetes API LIST calls across multiple requests on large clusters (#3823, @dharmab) * Fix CR restore regression introduced in 1.6 restore progress. (#3845, @sseago) * Use region specified in the BackupStorageLocation spec when getting restic repo identifier. Originally fixed by @jala-dx in #3617. (#3857, @zubron) * skip backuping projected volume when using restic (#3866, @alaypatel07) * Install Kubernetes preferred CRDs API version (v1beta1/v1). (#3614, @jenting) * Add Label to BackupSpec so that labels can explicitly be provided to Schedule.Spec.Template.Metadata.Labels which will be reflected on the backups created. (#3641, @arush-sal) * Add PVC UID label to PodVolumeRestore (#3792, @sseago) * Support pulling plugin images by digest (#3803, @2uasimojo) * Added BackupPhaseUploading and BackupPhaseUploadingPartialFailure backup phases as part of Upload Progress Monitoring. (#3805, @dsmithuchida) Uploading (new) The "Uploading" phase signifies that the main part of the backup, including snapshotting has completed successfully and uploading is continuing. In the event of an error during uploading, the phase will change to UploadingPartialFailure. On success, the phase changes to Completed. The backup cannot be restored from when it is in the Uploading state. UploadingPartialFailure (new) The "UploadingPartialFailure" phase signifies that the main part of the backup, including snapshotting has completed, but there were partial failures either during the main part or during the uploading. The backup cannot be restored from when it is in the UploadingPartialFailure state. * 🐛 Fix plugin name derivation from image name (#3711, @ashish-amarnath) * ✨ ⚠️ Remove CSI volumesnapshot artifact deletion This change requires https://github.com/vmware-tanzu/velero-plugin-for-csi/pull/86 for Velero to continue deleting of CSI volumesnapshots when the corresponding backups are deleted. (#3734, @ashish-amarnath) * use unstructured to marshal selective fields for service restore action (#3789, @alaypatel07) ================================================ FILE: changelogs/CHANGELOG-1.8.md ================================================ ## v1.8.0 ### 2022-01-14 ### Download https://github.com/vmware-tanzu/velero/releases/tag/v1.8.0 ### Container Image `velero/velero:v1.8.0` ### Documentation https://velero.io/docs/v1.8 ### Upgrading https://velero.io/docs/v1.8/upgrade-to-1.8/ ### Highlights #### Velero plugins now support handling volumes created by the CSI drivers of cloud providers Versions 1.4 of the Velero plugins for AWS, Azure and GCP now support snapshotting and restoring the persistent volumes provisioned by CSI driver via the APIs of the cloud providers. With this enhancement, users can backup and restore the persistent volumes on these cloud providers without using the Velero CSI plugin. The CSI plugin will remain beta and the feature flag `EnableCSI` will be disabled by default. For the version of the plugins and the CSI drivers they support respectively please see the table: | Plugin | Version | CSI Driver | | --- | ----------- | ---------- | | velero-plugin-for-aws | v1.4.0 | ebs.csi.aws.com | | velero-plugin-for-microsoft-azure | v1.4.0 | disk.csi.azure.com | | velero-plugin-for-gcp | v1.4.0 | pd.csi.storage.gke.io | #### IPv6 dual stack support We've verified the functionality of Velero on IPv6 dual stack by successfully running the E2E test on IPv6 dual stack environment. #### Refactor the controllers using Kubebuilder v3 In this release we continued our code modernization work, rewriting some controllers using Kubebuilder v3. This work is ongoing and we will continue to make progress in future releases. #### Enhancements to E2E test cases More test cases have been added to the E2E test suite to improve the release health. #### Respect the cron setting of scheduled backup The creation time is now taken into account to calculate the next run for scheduled backup. #### Deleting BSLs also cleans up related resources When a Backup Storage Location (BSL) is deleted, backup and Restic repository resources will also be deleted. #### Breaking changes Starting in v1.8, Velero will only support Kubernetes v1 CRD meaning that Velero v1.8+ will only run on Kubernetes v1.16+. Before upgrading, make sure you are running a supported Kubernetes version. For more information, see our [compatibility matrix](https://github.com/vmware-tanzu/velero#velero-compatibility-matrix). #### Upload Progress Monitoring and Item Snapshotter Item Snapshotter plugin API was merged. This will support both Upload Progress monitoring and the planned Data Mover. Upload Progress monitoring PRs are in progress for 1.9. ### All changes * E2E test on ssr object with controller namespace mix-ups (#4521, @mqiu) * Check whether the volume is provisioned by CSI driver or not by the annotation as well (#4513, @ywk253100) * Initialize the labels field of `velero backup-location create` option to avoid #4484 (#4491, @ywk253100) * Fix e2e 2500 namespaces scale test timeout problem (#4480, @mqiu) * Add backup deletion e2e test (#4401, @danfengliu) * Return the error when getting backup store in backup deletion controller (#4465, @reasonerjt) * Ignore the provided port is already allocated error when restoring the LoadBalancer service (#4462, @ywk253100) * Revert #4423 migrate backup sync controller to kubebuilder. (#4457, @jxun) * Add rbac and annotation test cases (#4455, @mqiu) * remove --crds-version in velero install command. (#4446, @jxun) * Upgrade e2e test vsphere plugin (#4440, @mqiu) * Fix e2e test failures for the inappropriate optimize of velero install (#4438, @mqiu) * Limit backup namespaces on test resource filtering cases (#4437, @mqiu) * Bump up Go to 1.17 (#4431, @reasonerjt) * Added ``-itemsnapshots.json.gz to the backup format. This file exists when item snapshots are taken and contains an array of volume.Itemsnapshots containing the information about the snapshots. This will not be used unless upload progress monitoring and item snapshots are enabled and an ItemSnapshot plugin is used to take snapshots. Also added DownloadTargetKindBackupItemSnapshots for retrieving the signed URL to download only the ``-itemsnapshots.json.gz part of a backup for use by `velero backup describe`. (#4429, @dsmithuchida) * Migrate backup sync controller from code-generator to kubebuilder. (#4423, @jxun) * Added UploadProgressFeature flag to enable Upload Progress Monitoring and Item Snapshotters. (#4416, @dsmithuchida) * Added BackupWithResolvers and RestoreWithResolvers calls. Will eventually replace Backup and Restore methods. Adds ItemSnapshotters to Backup and Restore workflows. (#4410, @dsu) * Build for darwin-arm64 (#4409, @epk) * Add resource filtering test cases (#4404, @mqiu) * Fix the issue that the backup cannot be deleted after the application uninstalled (#4398, @ywk253100) * Add restoreactionitem plugin to handle admission webhook configurations (#4397, @reasonerjt) * Keep the annotation "pv.kubernetes.io/provisioned-by" when restoring PVs (#4391, @ywk253100) * Adjust structure of e2e test codes (#4386, @mqiu) * feat: migrate velero controller from kubebuilder v2 to v3 From Velero v1.8, apiextesions.k8s.io/v1beta1 is no longer supported, which means only CRD of apiextensions.k8s.io/v1 is supported, and the supported Kubernetes version is updated to v1.16 and later. (#4382, @jxun) * Delete backups and Restic repos associated with deleted BSL(s) (#4377, @codegold79) * Add the key for GKE zone for AZ collection (#4376, @reasonerjt) * Fix statefulsets volumeClaimTemplates storageClassName when use Changing PV/PVC Storage Classes (#4375, @Box-Cube) * Fix snapshot e2e test issue of jsonpath (#4372, @danfengliu) * Modify the timestamp in the name of a backup generated from schedule to use UTC. (#4353, @jxun) * Read Availability zone from nodeAffinity requirements (#4350, @reasonerjt) * Use factory.Namespace() to replace hardcoded velero namespace (#4346, @half-life666) * Return the error if velero failed to detect S3 region for restic repo (#4343, @reasonerjt) * Add init log option for velero controller-runtime manager. (#4341, @jxun) * Ignore the `provided port is already allocated` error when restoring the `NodePort` service (#4336, @ywk253100) * Fixed an issue with the `backup-location create` command where the BSL Credential field would be set to an invalid empty SecretKeySelector when no credential details were provided. (#4322, @zubron) * fix buggy pager func (#4306, @alaypatel07) * Don't create a backup immediately after creating a schedule (#4281, @ywk253100) * Fix CVE-2020-29652 and CVE-2020-26160 (#4274, @ywk253100) * Refine tag-release.sh to align with change in release process (#4185, @reasonerjt) * Fix plugins incompatible issue in upgrade test (#4141, @danfengliu) * Verify group before treating resource as cohabiting (#4126, @sseago) * Added ItemSnapshotter plugin definition and plugin framework - addresses #3533. Part of the Upload Progress enhancement (#3533) (#4077, @dsmithuchida) * Add upgrade test in E2E test (#4058, @danfengliu) * Handle namespace mapping for PVs without snapshots on restore (#3708, @sseago) ================================================ FILE: changelogs/CHANGELOG-1.9.md ================================================ ## v1.9.0 ### 2022-06-13 ### Download https://github.com/vmware-tanzu/velero/releases/tag/v1.9.0 ### Container Image `velero/velero:v1.9.0` ### Documentation https://velero.io/docs/v1.9/ ### Upgrading https://velero.io/docs/v1.9/upgrade-to-1.9/ ### Highlights #### Improvement to the CSI plugin - Bump up to the CSI volume snapshot v1 API - No VolumeSnapshot will be left in the source namespace of the workload - Report metrics for CSI snapshots More improvements please refer to [CSI plugin improvement](https://github.com/vmware-tanzu/velero/issues?q=is%3Aissue+label%3A%22CSI+plugin+-+GA+-+phase1%22+is%3Aclosed) With these improvements we'll provide official support for CSI snapshots on AKS/EKS clusters. (with CSI plugin v0.3.0) #### Refactor the controllers using Kubebuilder v3 In this release we continued our code modernization work, rewriting some controllers using Kubebuilder v3. This work is ongoing and we will continue to make progress in future releases. #### Optionally restore status on selected resources Options are added to the CLI and Restore spec to control the group of resources whose status will be restored. #### ExistingResourcePolicy in the restore API Users can choose to overwrite or patch the existing resources during restore by setting this policy. #### Upgrade integrated Restic version and add skip TLS validation in Restic command Upgrade integrated Restic version, which will resolve some of the CVEs, and support skip TLS validation in Restic backup/restore. #### Breaking changes With bumping up the API to v1 in CSI plugin, the v0.3.0 CSI plugin will only work for Kubernetes v1.20+ ### All changes * restic: add full support for setting SecurityContext for restore init container from configMap. (#4084, @MatthieuFin) * Add metrics backup_items_total and backup_items_errors (#4296, @tobiasgiese) * Convert PodVolumebackup controller to the Kubebuilder framework (#4436, @fgold) * Skip not mounted volumes when backing up (#4497, @dkeven) * Update doc for v1.8 (#4517, @reasonerjt) * Fix bug to make the restic prune frequency configurable (#4518, @ywk253100) * Add E2E test of backups sync from BSL (#4545, @mqiu) * Fix: OrderedResources in Schedules (#4550, @dbrekau) * Skip volumes of non-running pods when backing up (#4584, @bynare) * E2E SSR test add retry mechanism and logs (#4591, @mqiu) * Add pushing image to GCR in github workflow to facilitate some environments that have rate limitation to docker hub, e.g. vSphere. (#4623, @jxun) * Add existingResourcePolicy to Restore API (#4628, @shubham-pampattiwar) * Fix E2E backup namespaces test (#4634, @qiuming-best) * Update image used by E2E test to gcr.io (#4639, @jxun) * Add multiple label selector support to Velero Backup and Restore APIs (#4650, @shubham-pampattiwar) * Convert Pod Volume Restore resource/controller to the Kubebuilder framework (#4655, @ywk253100) * Update --use-owner-references-in-backup description in velero command line. (#4660, @jxun) * Avoid overwritten hook's exec.container parameter when running pod command executor. (#4661, @jxun) * Support regional pv for GKE (#4680, @jxun) * Bypass the remap CRD version plugin when v1beta1 CRD is not supported (#4686, @reasonerjt) * Add GINKGO_SKIP to support skip specific case in e2e test. (#4692, @jxun) * Add --pod-labels flag to velero install (#4694, @j4m3s-s) * Enable coverage in test.sh and upload to codecov (#4704, @reasonerjt) * Mark the BSL as "Unavailable" when gets any error and add a new field "Message" to the status to record the error message (#4719, @ywk253100) * Support multiple skip option for E2E test (#4725, @jxun) * Add PriorityClass to the AdditionalItems of Backup's PodAction and Restore's PodAction plugin to backup and restore PriorityClass if it is used by a Pod. (#4740, @phuongatemc) * Insert all restore errors and warnings into restore log. (#4743, @sseago) * Refactor schedule controller with kubebuilder (#4748, @ywk253100) * Garbage collector now adds labels to backups that failed to delete for BSLNotFound, BSLCannotGet, BSLReadOnly reasons. (#4757, @kaovilai) * Skip podvolumerestore creation when restore excludes pv/pvc (#4769, @half-life666) * Add parameter for e2e test to support modify kibishii install path. (#4778, @jxun) * Ensure the restore hook applied to new namespace based on the mapping (#4779, @reasonerjt) * Add ability to restore status on selected resources (#4785, @RafaeLeal) * Do not take snapshot for PV to avoid duplicated snapshotting, when CSI feature is enabled. (#4797, @jxun) * Bump up to v1 API for CSI snapshot (#4800, @reasonerjt) * fix: delete empty backups (#4817, @yuvalman) * Add CSI VolumeSnapshot related metrics. (#4818, @jxun) * Fix default-backup-ttl not work (#4831, @qiuming-best) * Make the vsc created by backup sync controller deletable (#4832, @reasonerjt) * Make in-progress backup/restore as failed when doing the reconcile to avoid hanging in in-progress status (#4833, @ywk253100) * Use controller-gen to generate the deep copy methods for objects (#4838, @ywk253100) * Update integrated Restic version and add insecureSkipTLSVerify for Restic CLI. (#4839, @jxun) * Modify CSI VolumeSnapshot metric related code. (#4854, @jxun) * Refactor backup deletion controller based on kubebuilder (#4855, @reasonerjt) * Remove VolumeSnapshots created during backup when CSI feature is enabled. (#4858, @jxun) * Convert Restic Repository resource/controller to the Kubebuilder framework (#4859, @qiuming-best) * Add ClusterClasses to the restore priority list (#4866, @reasonerjt) * Cleanup the .velero folder after restic done (#4872, @big-appled) * Delete orphan CSI snapshots in backup sync controller (#4887, @reasonerjt) * Make waiting VolumeSnapshot to ready process parallel. (#4889, @jxun) * continue rather than return for non-matching restore action label (#4890, @sseago) * Make in-progress PVB/PVR as failed when restic controller restarts to avoid hanging backup/restore (#4893, @ywk253100) * Refactor BSL controller with periodical enqueue source (#4894, @jxun) * Make garbage collection for expired backups configurable (#4897, @ywk253100) * Bump up the version of distroless to base-debian11 (#4898, @ywk253100) * Add schedule ordered resources E2E test (#4913, @qiuming-best) * Make velero completion zsh command output can be used by `source` command. (#4914, @jxun) * Enhance the map flag to support parsing input value contains entry delimiters (#4920, @ywk253100) * Fix E2E test [Backups][Deletion][Restic] on GCP. (#4968, @jxun) * Disable status as sub resource in CRDs (#4972, @ywk253100) * Add more information for failing to get path or snapshot in restic backup and restore. (#4988, @jxun) ================================================ FILE: changelogs/unreleased/9502-Joeavaikath ================================================ Support all glob wildcard characters in namespace validation ================================================ FILE: changelogs/unreleased/9508-kaovilai ================================================ Fix VolumePolicy PVC phase condition filter for unbound PVCs (#9507) ================================================ FILE: changelogs/unreleased/9532-Lyndon-Li ================================================ Fix issue #9343, include PV topology to data mover pod affinities ================================================ FILE: changelogs/unreleased/9533-Lyndon-Li‎‎ ================================================ Fix issue #9496, support customized host os ================================================ FILE: changelogs/unreleased/9547-blackpiglet ================================================ If BIA return updateObj with SkipFromBackupAnnotation, treat it as skip the resource from backup. ================================================ FILE: changelogs/unreleased/9554-testsabirweb ================================================ Issue #9544: Add test coverage for S3 bucket name in MRAP ARN notation and fix bucket validation to accept ARN format ================================================ FILE: changelogs/unreleased/9560-Lyndon-Li‎‎ ================================================ Fix issue #9475, use node-selector instead of nodName for generic restore ================================================ FILE: changelogs/unreleased/9561-Lyndon-Li‎‎ ================================================ Fix issue #9460, flush buffer before data mover completes ================================================ FILE: changelogs/unreleased/9570-H-M-Quang-Ngo ================================================ Add schedule_expected_interval_seconds metric for dynamic backup alerting thresholds (#9559) ================================================ FILE: changelogs/unreleased/9574-blackpiglet ================================================ Add ephemeral storage limit and request support for data mover and maintenance job ================================================ FILE: changelogs/unreleased/9581-shubham-pampattiwar ================================================ Fix DBR stuck when CSI snapshot no longer exists in cloud provider ================================================ FILE: cmd/velero/velero.go ================================================ /* Copyright 2017, 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package main import ( "os" "path/filepath" "k8s.io/klog/v2" "github.com/vmware-tanzu/velero/pkg/cmd" "github.com/vmware-tanzu/velero/pkg/cmd/velero" ) func main() { defer klog.Flush() baseName := filepath.Base(os.Args[0]) err := velero.NewCommand(baseName).Execute() cmd.CheckError(err) } ================================================ FILE: cmd/velero-helper/velero-helper.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package main import ( "fmt" "os" "time" ) const ( // workingModePause indicates it is for general purpose to hold the pod under running state workingModePause = "pause" ) func main() { if len(os.Args) < 2 { fmt.Fprintln(os.Stderr, "ERROR: at least one argument must be provided, the working mode") os.Exit(1) } switch os.Args[1] { case workingModePause: time.Sleep(time.Duration(1<<63 - 1)) default: fmt.Fprintln(os.Stderr, "ERROR: wrong working mode provided") os.Exit(1) } } ================================================ FILE: cmd/velero-restore-helper/velero-restore-helper.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package main import ( "fmt" "os" "path/filepath" "time" ) func main() { if len(os.Args) != 2 { fmt.Fprintln(os.Stderr, "ERROR: exactly one argument must be provided, the restore's UID") os.Exit(1) } ticker := time.NewTicker(time.Second) defer ticker.Stop() for { <-ticker.C if done() { fmt.Println("All restic restores are done") err := removeFolder() if err != nil { fmt.Println(err) } else { fmt.Println("Done cleanup .velero folder") } return } } } // done returns true if for each directory under /restores, a file exists // within the .velero/ subdirectory whose name is equal to os.Args[1], or // false otherwise func done() bool { children, err := os.ReadDir("/restores") if err != nil { fmt.Fprintf(os.Stderr, "ERROR reading /restores directory: %s\n", err) return false } for _, child := range children { if !child.IsDir() { fmt.Printf("%s is not a directory, skipping.\n", child.Name()) continue } doneFile := filepath.Join("/restores", child.Name(), ".velero", os.Args[1]) if _, err := os.Stat(doneFile); os.IsNotExist(err) { fmt.Printf("The filesystem restore done file %s is not found yet. Retry later.\n", doneFile) return false } else if err != nil { fmt.Fprintf(os.Stderr, "ERROR looking filesystem restore done file %s: %s\n", doneFile, err) return false } fmt.Printf("Found the done file %s\n", doneFile) } return true } // remove .velero folder func removeFolder() error { children, err := os.ReadDir("/restores") if err != nil { return err } for _, child := range children { if !child.IsDir() { fmt.Printf("%s is not a directory, skipping.\n", child.Name()) continue } donePath := filepath.Join("/restores", child.Name(), ".velero") err = os.RemoveAll(donePath) if err != nil { return err } fmt.Printf("Deleted %s", donePath) } return nil } ================================================ FILE: config/crd/v1/bases/velero.io_backuprepositories.yaml ================================================ --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.16.5 name: backuprepositories.velero.io spec: group: velero.io names: kind: BackupRepository listKind: BackupRepositoryList plural: backuprepositories singular: backuprepository scope: Namespaced versions: - additionalPrinterColumns: - jsonPath: .metadata.creationTimestamp name: Age type: date - jsonPath: .spec.repositoryType name: Repository Type type: string name: v1 schema: openAPIV3Schema: properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: BackupRepositorySpec is the specification for a BackupRepository. properties: backupStorageLocation: description: |- BackupStorageLocation is the name of the BackupStorageLocation that should contain this repository. type: string maintenanceFrequency: description: MaintenanceFrequency is how often maintenance should be run. type: string repositoryConfig: additionalProperties: type: string description: RepositoryConfig is for repository-specific configuration fields. nullable: true type: object repositoryType: description: RepositoryType indicates the type of the backend repository enum: - kopia - restic - "" type: string resticIdentifier: description: |- ResticIdentifier is the full restic-compatible string for identifying this repository. This field is only used when RepositoryType is "restic". type: string volumeNamespace: description: |- VolumeNamespace is the namespace this backup repository contains pod volume backups for. type: string required: - backupStorageLocation - maintenanceFrequency - volumeNamespace type: object status: description: BackupRepositoryStatus is the current status of a BackupRepository. properties: lastMaintenanceTime: description: LastMaintenanceTime is the last time repo maintenance succeeded. format: date-time nullable: true type: string message: description: Message is a message about the current status of the BackupRepository. type: string phase: description: Phase is the current state of the BackupRepository. enum: - New - Ready - NotReady type: string recentMaintenance: description: RecentMaintenance is status of the recent repo maintenance. items: properties: completeTimestamp: description: CompleteTimestamp is the completion time of the repo maintenance. format: date-time nullable: true type: string message: description: Message is a message about the current status of the repo maintenance. type: string result: description: Result is the result of the repo maintenance. enum: - Succeeded - Failed type: string startTimestamp: description: StartTimestamp is the start time of the repo maintenance. format: date-time nullable: true type: string type: object type: array type: object type: object served: true storage: true subresources: {} ================================================ FILE: config/crd/v1/bases/velero.io_backups.yaml ================================================ --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.16.5 name: backups.velero.io spec: group: velero.io names: kind: Backup listKind: BackupList plural: backups singular: backup scope: Namespaced versions: - name: v1 schema: openAPIV3Schema: description: |- Backup is a Velero resource that represents the capture of Kubernetes cluster state at a point in time (API objects and associated volume state). properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: BackupSpec defines the specification for a Velero backup. properties: csiSnapshotTimeout: description: |- CSISnapshotTimeout specifies the time used to wait for CSI VolumeSnapshot status turns to ReadyToUse during creation, before returning error as timeout. The default value is 10 minute. type: string datamover: description: |- DataMover specifies the data mover to be used by the backup. If DataMover is "" or "velero", the built-in data mover will be used. type: string defaultVolumesToFsBackup: description: |- DefaultVolumesToFsBackup specifies whether pod volume file system backup should be used for all volumes by default. nullable: true type: boolean defaultVolumesToRestic: description: |- DefaultVolumesToRestic specifies whether restic should be used to take a backup of all pod volumes by default. Deprecated: this field is no longer used and will be removed entirely in future. Use DefaultVolumesToFsBackup instead. nullable: true type: boolean excludedClusterScopedResources: description: |- ExcludedClusterScopedResources is a slice of cluster-scoped resource type names to exclude from the backup. If set to "*", all cluster-scoped resource types are excluded. The default value is empty. items: type: string nullable: true type: array excludedNamespaceScopedResources: description: |- ExcludedNamespaceScopedResources is a slice of namespace-scoped resource type names to exclude from the backup. If set to "*", all namespace-scoped resource types are excluded. The default value is empty. items: type: string nullable: true type: array excludedNamespaces: description: |- ExcludedNamespaces contains a list of namespaces that are not included in the backup. items: type: string nullable: true type: array excludedResources: description: |- ExcludedResources is a slice of resource names that are not included in the backup. items: type: string nullable: true type: array hooks: description: Hooks represent custom behaviors that should be executed at different phases of the backup. properties: resources: description: Resources are hooks that should be executed when backing up individual instances of a resource. items: description: |- BackupResourceHookSpec defines one or more BackupResourceHooks that should be executed based on the rules defined for namespaces, resources, and label selector. properties: excludedNamespaces: description: ExcludedNamespaces specifies the namespaces to which this hook spec does not apply. items: type: string nullable: true type: array excludedResources: description: ExcludedResources specifies the resources to which this hook spec does not apply. items: type: string nullable: true type: array includedNamespaces: description: |- IncludedNamespaces specifies the namespaces to which this hook spec applies. If empty, it applies to all namespaces. items: type: string nullable: true type: array includedResources: description: |- IncludedResources specifies the resources to which this hook spec applies. If empty, it applies to all resources. items: type: string nullable: true type: array labelSelector: description: LabelSelector, if specified, filters the resources to which this hook spec applies. nullable: true properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: |- operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: |- values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array x-kubernetes-list-type: atomic required: - key - operator type: object type: array x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string description: |- matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic name: description: Name is the name of this hook. type: string post: description: |- PostHooks is a list of BackupResourceHooks to execute after storing the item in the backup. These are executed after all "additional items" from item actions are processed. items: description: BackupResourceHook defines a hook for a resource. properties: exec: description: Exec defines an exec hook. properties: command: description: Command is the command and arguments to execute. items: type: string minItems: 1 type: array container: description: |- Container is the container in the pod where the command should be executed. If not specified, the pod's first container is used. type: string onError: description: OnError specifies how Velero should behave if it encounters an error executing this hook. enum: - Continue - Fail type: string timeout: description: |- Timeout defines the maximum amount of time Velero should wait for the hook to complete before considering the execution a failure. type: string required: - command type: object required: - exec type: object type: array pre: description: |- PreHooks is a list of BackupResourceHooks to execute prior to storing the item in the backup. These are executed before any "additional items" from item actions are processed. items: description: BackupResourceHook defines a hook for a resource. properties: exec: description: Exec defines an exec hook. properties: command: description: Command is the command and arguments to execute. items: type: string minItems: 1 type: array container: description: |- Container is the container in the pod where the command should be executed. If not specified, the pod's first container is used. type: string onError: description: OnError specifies how Velero should behave if it encounters an error executing this hook. enum: - Continue - Fail type: string timeout: description: |- Timeout defines the maximum amount of time Velero should wait for the hook to complete before considering the execution a failure. type: string required: - command type: object required: - exec type: object type: array required: - name type: object nullable: true type: array type: object includeClusterResources: description: |- IncludeClusterResources specifies whether cluster-scoped resources should be included for consideration in the backup. nullable: true type: boolean includedClusterScopedResources: description: |- IncludedClusterScopedResources is a slice of cluster-scoped resource type names to include in the backup. If set to "*", all cluster-scoped resource types are included. The default value is empty, which means only related cluster-scoped resources are included. items: type: string nullable: true type: array includedNamespaceScopedResources: description: |- IncludedNamespaceScopedResources is a slice of namespace-scoped resource type names to include in the backup. The default value is "*". items: type: string nullable: true type: array includedNamespaces: description: |- IncludedNamespaces is a slice of namespace names to include objects from. If empty, all namespaces are included. items: type: string nullable: true type: array includedResources: description: |- IncludedResources is a slice of resource names to include in the backup. If empty, all resources are included. items: type: string nullable: true type: array itemOperationTimeout: description: |- ItemOperationTimeout specifies the time used to wait for asynchronous BackupItemAction operations The default value is 4 hour. type: string labelSelector: description: |- LabelSelector is a metav1.LabelSelector to filter with when adding individual objects to the backup. If empty or nil, all objects are included. Optional. nullable: true properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: |- operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: |- values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array x-kubernetes-list-type: atomic required: - key - operator type: object type: array x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string description: |- matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic metadata: properties: labels: additionalProperties: type: string type: object type: object orLabelSelectors: description: |- OrLabelSelectors is list of metav1.LabelSelector to filter with when adding individual objects to the backup. If multiple provided they will be joined by the OR operator. LabelSelector as well as OrLabelSelectors cannot co-exist in backup request, only one of them can be used. items: description: |- A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: |- operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: |- values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array x-kubernetes-list-type: atomic required: - key - operator type: object type: array x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string description: |- matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic nullable: true type: array orderedResources: additionalProperties: type: string description: |- OrderedResources specifies the backup order of resources of specific Kind. The map key is the resource name and value is a list of object names separated by commas. Each resource name has format "namespace/objectname". For cluster resources, simply use "objectname". nullable: true type: object resourcePolicy: description: ResourcePolicy specifies the referenced resource policies that backup should follow properties: apiGroup: description: |- APIGroup is the group for the resource being referenced. If APIGroup is not specified, the specified Kind must be in the core API group. For any other third-party types, APIGroup is required. type: string kind: description: Kind is the type of resource being referenced type: string name: description: Name is the name of resource being referenced type: string required: - kind - name type: object x-kubernetes-map-type: atomic snapshotMoveData: description: SnapshotMoveData specifies whether snapshot data should be moved nullable: true type: boolean snapshotVolumes: description: |- SnapshotVolumes specifies whether to take snapshots of any PV's referenced in the set of objects included in the Backup. nullable: true type: boolean storageLocation: description: StorageLocation is a string containing the name of a BackupStorageLocation where the backup should be stored. type: string ttl: description: |- TTL is a time.Duration-parseable string describing how long the Backup should be retained for. type: string uploaderConfig: description: UploaderConfig specifies the configuration for the uploader. nullable: true properties: parallelFilesUpload: description: ParallelFilesUpload is the number of files parallel uploads to perform when using the uploader. type: integer type: object volumeGroupSnapshotLabelKey: description: VolumeGroupSnapshotLabelKey specifies the label key to group PVCs under a VGS. type: string volumeSnapshotLocations: description: VolumeSnapshotLocations is a list containing names of VolumeSnapshotLocations associated with this backup. items: type: string type: array type: object status: description: BackupStatus captures the current status of a Velero backup. properties: backupItemOperationsAttempted: description: |- BackupItemOperationsAttempted is the total number of attempted async BackupItemAction operations for this backup. type: integer backupItemOperationsCompleted: description: |- BackupItemOperationsCompleted is the total number of successfully completed async BackupItemAction operations for this backup. type: integer backupItemOperationsFailed: description: |- BackupItemOperationsFailed is the total number of async BackupItemAction operations for this backup which ended with an error. type: integer completionTimestamp: description: |- CompletionTimestamp records the time a backup was completed. Completion time is recorded even on failed backups. Completion time is recorded before uploading the backup object. The server's time is used for CompletionTimestamps format: date-time nullable: true type: string csiVolumeSnapshotsAttempted: description: |- CSIVolumeSnapshotsAttempted is the total number of attempted CSI VolumeSnapshots for this backup. type: integer csiVolumeSnapshotsCompleted: description: |- CSIVolumeSnapshotsCompleted is the total number of successfully completed CSI VolumeSnapshots for this backup. type: integer errors: description: |- Errors is a count of all error messages that were generated during execution of the backup. The actual errors are in the backup's log file in object storage. type: integer expiration: description: Expiration is when this Backup is eligible for garbage-collection. format: date-time nullable: true type: string failureReason: description: FailureReason is an error that caused the entire backup to fail. type: string formatVersion: description: FormatVersion is the backup format version, including major, minor, and patch version. type: string hookStatus: description: HookStatus contains information about the status of the hooks. nullable: true properties: hooksAttempted: description: |- HooksAttempted is the total number of attempted hooks Specifically, HooksAttempted represents the number of hooks that failed to execute and the number of hooks that executed successfully. type: integer hooksFailed: description: HooksFailed is the total number of hooks which ended with an error type: integer type: object phase: description: Phase is the current state of the Backup. enum: - New - Queued - ReadyToStart - FailedValidation - InProgress - WaitingForPluginOperations - WaitingForPluginOperationsPartiallyFailed - Finalizing - FinalizingPartiallyFailed - Completed - PartiallyFailed - Failed - Deleting type: string progress: description: |- Progress contains information about the backup's execution progress. Note that this information is best-effort only -- if Velero fails to update it during a backup for any reason, it may be inaccurate/stale. nullable: true properties: itemsBackedUp: description: |- ItemsBackedUp is the number of items that have actually been written to the backup tarball so far. type: integer totalItems: description: |- TotalItems is the total number of items to be backed up. This number may change throughout the execution of the backup due to plugins that return additional related items to back up, the velero.io/exclude-from-backup label, and various other filters that happen as items are processed. type: integer type: object queuePosition: description: |- QueuePosition is the position of the backup in the queue. Only relevant when Phase is "Queued" type: integer startTimestamp: description: |- StartTimestamp records the time a backup was started. Separate from CreationTimestamp, since that value changes on restores. The server's time is used for StartTimestamps format: date-time nullable: true type: string validationErrors: description: |- ValidationErrors is a slice of all validation errors (if applicable). items: type: string nullable: true type: array version: description: |- Version is the backup format major version. Deprecated: Please see FormatVersion type: integer volumeSnapshotsAttempted: description: |- VolumeSnapshotsAttempted is the total number of attempted volume snapshots for this backup. type: integer volumeSnapshotsCompleted: description: |- VolumeSnapshotsCompleted is the total number of successfully completed volume snapshots for this backup. type: integer warnings: description: |- Warnings is a count of all warning messages that were generated during execution of the backup. The actual warnings are in the backup's log file in object storage. type: integer type: object type: object served: true storage: true ================================================ FILE: config/crd/v1/bases/velero.io_backupstoragelocations.yaml ================================================ --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.16.5 name: backupstoragelocations.velero.io spec: group: velero.io names: kind: BackupStorageLocation listKind: BackupStorageLocationList plural: backupstoragelocations shortNames: - bsl singular: backupstoragelocation scope: Namespaced versions: - additionalPrinterColumns: - description: Backup Storage Location status such as Available/Unavailable jsonPath: .status.phase name: Phase type: string - description: LastValidationTime is the last time the backup store location was validated jsonPath: .status.lastValidationTime name: Last Validated type: date - jsonPath: .metadata.creationTimestamp name: Age type: date - description: Default backup storage location jsonPath: .spec.default name: Default type: boolean name: v1 schema: openAPIV3Schema: description: BackupStorageLocation is a location where Velero stores backup objects properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: BackupStorageLocationSpec defines the desired state of a Velero BackupStorageLocation properties: accessMode: description: AccessMode defines the permissions for the backup storage location. enum: - ReadOnly - ReadWrite type: string backupSyncPeriod: description: BackupSyncPeriod defines how frequently to sync backup API objects from object storage. A value of 0 disables sync. nullable: true type: string config: additionalProperties: type: string description: Config is for provider-specific configuration fields. type: object credential: description: Credential contains the credential information intended to be used with this location properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: default: "" description: |- Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object x-kubernetes-map-type: atomic default: description: Default indicates this location is the default backup storage location. type: boolean objectStorage: description: ObjectStorageLocation specifies the settings necessary to connect to a provider's object storage. properties: bucket: description: Bucket is the bucket to use for object storage. type: string caCert: description: |- CACert defines a CA bundle to use when verifying TLS connections to the provider. Deprecated: Use CACertRef instead. format: byte type: string caCertRef: description: |- CACertRef is a reference to a Secret containing the CA certificate bundle to use when verifying TLS connections to the provider. The Secret must be in the same namespace as the BackupStorageLocation. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: default: "" description: |- Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object x-kubernetes-map-type: atomic prefix: description: Prefix is the path inside a bucket to use for Velero storage. Optional. type: string required: - bucket type: object provider: description: Provider is the provider of the backup storage. type: string validationFrequency: description: ValidationFrequency defines how frequently to validate the corresponding object storage. A value of 0 disables validation. nullable: true type: string required: - objectStorage - provider type: object status: description: BackupStorageLocationStatus defines the observed state of BackupStorageLocation properties: accessMode: description: |- AccessMode is an unused field. Deprecated: there is now an AccessMode field on the Spec and this field will be removed entirely as of v2.0. enum: - ReadOnly - ReadWrite type: string lastSyncedRevision: description: |- LastSyncedRevision is the value of the `metadata/revision` file in the backup storage location the last time the BSL's contents were synced into the cluster. Deprecated: this field is no longer updated or used for detecting changes to the location's contents and will be removed entirely in v2.0. type: string lastSyncedTime: description: |- LastSyncedTime is the last time the contents of the location were synced into the cluster. format: date-time nullable: true type: string lastValidationTime: description: |- LastValidationTime is the last time the backup store location was validated the cluster. format: date-time nullable: true type: string message: description: Message is a message about the backup storage location's status. type: string phase: description: Phase is the current state of the BackupStorageLocation. enum: - Available - Unavailable type: string type: object type: object served: true storage: true subresources: {} ================================================ FILE: config/crd/v1/bases/velero.io_deletebackuprequests.yaml ================================================ --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.16.5 name: deletebackuprequests.velero.io spec: group: velero.io names: kind: DeleteBackupRequest listKind: DeleteBackupRequestList plural: deletebackuprequests singular: deletebackuprequest scope: Namespaced versions: - additionalPrinterColumns: - description: The name of the backup to be deleted jsonPath: .spec.backupName name: BackupName type: string - description: The status of the deletion request jsonPath: .status.phase name: Status type: string name: v1 schema: openAPIV3Schema: description: DeleteBackupRequest is a request to delete one or more backups. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: DeleteBackupRequestSpec is the specification for which backups to delete. properties: backupName: type: string required: - backupName type: object status: description: DeleteBackupRequestStatus is the current status of a DeleteBackupRequest. properties: errors: description: Errors contains any errors that were encountered during the deletion process. items: type: string nullable: true type: array phase: description: Phase is the current state of the DeleteBackupRequest. enum: - New - InProgress - Processed type: string type: object type: object served: true storage: true subresources: {} ================================================ FILE: config/crd/v1/bases/velero.io_downloadrequests.yaml ================================================ --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.16.5 name: downloadrequests.velero.io spec: group: velero.io names: kind: DownloadRequest listKind: DownloadRequestList plural: downloadrequests singular: downloadrequest scope: Namespaced versions: - name: v1 schema: openAPIV3Schema: description: |- DownloadRequest is a request to download an artifact from backup object storage, such as a backup log file. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: DownloadRequestSpec is the specification for a download request. properties: target: description: Target is what to download (e.g. logs for a backup). properties: kind: description: Kind is the type of file to download. enum: - BackupLog - BackupContents - BackupVolumeSnapshots - BackupItemOperations - BackupResourceList - BackupResults - RestoreLog - RestoreResults - RestoreResourceList - RestoreItemOperations - CSIBackupVolumeSnapshots - CSIBackupVolumeSnapshotContents - BackupVolumeInfos - RestoreVolumeInfo type: string name: description: Name is the name of the Kubernetes resource with which the file is associated. type: string required: - kind - name type: object required: - target type: object status: description: DownloadRequestStatus is the current status of a DownloadRequest. properties: downloadURL: description: DownloadURL contains the pre-signed URL for the target file. type: string expiration: description: Expiration is when this DownloadRequest expires and can be deleted by the system. format: date-time nullable: true type: string phase: description: Phase is the current state of the DownloadRequest. enum: - New - Processed type: string type: object type: object served: true storage: true ================================================ FILE: config/crd/v1/bases/velero.io_podvolumebackups.yaml ================================================ --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.16.5 name: podvolumebackups.velero.io spec: group: velero.io names: kind: PodVolumeBackup listKind: PodVolumeBackupList plural: podvolumebackups singular: podvolumebackup scope: Namespaced versions: - additionalPrinterColumns: - description: PodVolumeBackup status such as New/InProgress jsonPath: .status.phase name: Status type: string - description: Time duration since this PodVolumeBackup was started jsonPath: .status.startTimestamp name: Started type: date - description: Completed bytes format: int64 jsonPath: .status.progress.bytesDone name: Bytes Done type: integer - description: Total bytes format: int64 jsonPath: .status.progress.totalBytes name: Total Bytes type: integer - description: Incremental bytes format: int64 jsonPath: .status.incrementalBytes name: Incremental Bytes priority: 10 type: integer - description: Name of the Backup Storage Location where this backup should be stored jsonPath: .spec.backupStorageLocation name: Storage Location type: string - description: Time duration since this PodVolumeBackup was created jsonPath: .metadata.creationTimestamp name: Age type: date - description: Name of the node where the PodVolumeBackup is processed jsonPath: .status.node name: Node type: string - description: The type of the uploader to handle data transfer jsonPath: .spec.uploaderType name: Uploader type: string name: v1 schema: openAPIV3Schema: properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: PodVolumeBackupSpec is the specification for a PodVolumeBackup. properties: backupStorageLocation: description: |- BackupStorageLocation is the name of the backup storage location where the backup repository is stored. type: string cancel: description: |- Cancel indicates request to cancel the ongoing PodVolumeBackup. It can be set when the PodVolumeBackup is in InProgress phase type: boolean node: description: Node is the name of the node that the Pod is running on. type: string pod: description: Pod is a reference to the pod containing the volume to be backed up. properties: apiVersion: description: API version of the referent. type: string fieldPath: description: |- If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: "spec.containers{name}" (where "name" refers to the name of the container that triggered the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. type: string kind: description: |- Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: description: |- Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: description: |- Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: description: |- UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object x-kubernetes-map-type: atomic repoIdentifier: description: RepoIdentifier is the backup repository identifier. type: string tags: additionalProperties: type: string description: |- Tags are a map of key-value pairs that should be applied to the volume backup as tags. type: object uploaderSettings: additionalProperties: type: string description: |- UploaderSettings are a map of key-value pairs that should be applied to the uploader configuration. nullable: true type: object uploaderType: description: UploaderType is the type of the uploader to handle the data transfer. enum: - kopia - restic - "" type: string volume: description: |- Volume is the name of the volume within the Pod to be backed up. type: string required: - backupStorageLocation - node - pod - repoIdentifier - volume type: object status: description: PodVolumeBackupStatus is the current status of a PodVolumeBackup. properties: acceptedTimestamp: description: |- AcceptedTimestamp records the time the pod volume backup is to be prepared. The server's time is used for AcceptedTimestamp format: date-time nullable: true type: string completionTimestamp: description: |- CompletionTimestamp records the time a backup was completed. Completion time is recorded even on failed backups. Completion time is recorded before uploading the backup object. The server's time is used for CompletionTimestamps format: date-time nullable: true type: string incrementalBytes: description: IncrementalBytes holds the number of bytes new or changed since the last backup format: int64 type: integer message: description: Message is a message about the pod volume backup's status. type: string path: description: Path is the full path within the controller pod being backed up. type: string phase: description: Phase is the current state of the PodVolumeBackup. enum: - New - Accepted - Prepared - InProgress - Canceling - Canceled - Completed - Failed type: string progress: description: |- Progress holds the total number of bytes of the volume and the current number of backed up bytes. This can be used to display progress information about the backup operation. properties: bytesDone: format: int64 type: integer totalBytes: format: int64 type: integer type: object snapshotID: description: SnapshotID is the identifier for the snapshot of the pod volume. type: string startTimestamp: description: |- StartTimestamp records the time a backup was started. Separate from CreationTimestamp, since that value changes on restores. The server's time is used for StartTimestamps format: date-time nullable: true type: string type: object type: object served: true storage: true subresources: {} ================================================ FILE: config/crd/v1/bases/velero.io_podvolumerestores.yaml ================================================ --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.16.5 name: podvolumerestores.velero.io spec: group: velero.io names: kind: PodVolumeRestore listKind: PodVolumeRestoreList plural: podvolumerestores singular: podvolumerestore scope: Namespaced versions: - additionalPrinterColumns: - description: PodVolumeRestore status such as New/InProgress jsonPath: .status.phase name: Status type: string - description: Time duration since this PodVolumeRestore was started jsonPath: .status.startTimestamp name: Started type: date - description: Completed bytes format: int64 jsonPath: .status.progress.bytesDone name: Bytes Done type: integer - description: Total bytes format: int64 jsonPath: .status.progress.totalBytes name: Total Bytes type: integer - description: Name of the Backup Storage Location where the backup data is stored jsonPath: .spec.backupStorageLocation name: Storage Location type: string - description: Time duration since this PodVolumeRestore was created jsonPath: .metadata.creationTimestamp name: Age type: date - description: Name of the node where the PodVolumeRestore is processed jsonPath: .status.node name: Node type: string - description: The type of the uploader to handle data transfer jsonPath: .spec.uploaderType name: Uploader Type type: string name: v1 schema: openAPIV3Schema: properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: PodVolumeRestoreSpec is the specification for a PodVolumeRestore. properties: backupStorageLocation: description: |- BackupStorageLocation is the name of the backup storage location where the backup repository is stored. type: string cancel: description: |- Cancel indicates request to cancel the ongoing PodVolumeRestore. It can be set when the PodVolumeRestore is in InProgress phase type: boolean pod: description: Pod is a reference to the pod containing the volume to be restored. properties: apiVersion: description: API version of the referent. type: string fieldPath: description: |- If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: "spec.containers{name}" (where "name" refers to the name of the container that triggered the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. type: string kind: description: |- Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: description: |- Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: description: |- Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: description: |- UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object x-kubernetes-map-type: atomic repoIdentifier: description: RepoIdentifier is the backup repository identifier. type: string snapshotID: description: SnapshotID is the ID of the volume snapshot to be restored. type: string snapshotSize: description: SnapshotSize is the logical size in Bytes of the snapshot. format: int64 type: integer sourceNamespace: description: SourceNamespace is the original namespace for namaspace mapping. type: string uploaderSettings: additionalProperties: type: string description: |- UploaderSettings are a map of key-value pairs that should be applied to the uploader configuration. nullable: true type: object uploaderType: description: UploaderType is the type of the uploader to handle the data transfer. enum: - kopia - restic - "" type: string volume: description: Volume is the name of the volume within the Pod to be restored. type: string required: - backupStorageLocation - pod - repoIdentifier - snapshotID - sourceNamespace - volume type: object status: description: PodVolumeRestoreStatus is the current status of a PodVolumeRestore. properties: acceptedTimestamp: description: |- AcceptedTimestamp records the time the pod volume restore is to be prepared. The server's time is used for AcceptedTimestamp format: date-time nullable: true type: string completionTimestamp: description: |- CompletionTimestamp records the time a restore was completed. Completion time is recorded even on failed restores. The server's time is used for CompletionTimestamps format: date-time nullable: true type: string message: description: Message is a message about the pod volume restore's status. type: string node: description: Node is name of the node where the pod volume restore is processed. type: string phase: description: Phase is the current state of the PodVolumeRestore. enum: - New - Accepted - Prepared - InProgress - Canceling - Canceled - Completed - Failed type: string progress: description: |- Progress holds the total number of bytes of the snapshot and the current number of restored bytes. This can be used to display progress information about the restore operation. properties: bytesDone: format: int64 type: integer totalBytes: format: int64 type: integer type: object startTimestamp: description: |- StartTimestamp records the time a restore was started. The server's time is used for StartTimestamps format: date-time nullable: true type: string type: object type: object served: true storage: true subresources: {} ================================================ FILE: config/crd/v1/bases/velero.io_restores.yaml ================================================ --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.16.5 name: restores.velero.io spec: group: velero.io names: kind: Restore listKind: RestoreList plural: restores singular: restore scope: Namespaced versions: - name: v1 schema: openAPIV3Schema: description: |- Restore is a Velero resource that represents the application of resources from a Velero backup to a target Kubernetes cluster. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: RestoreSpec defines the specification for a Velero restore. properties: backupName: description: |- BackupName is the unique name of the Velero backup to restore from. type: string excludedNamespaces: description: |- ExcludedNamespaces contains a list of namespaces that are not included in the restore. items: type: string nullable: true type: array excludedResources: description: |- ExcludedResources is a slice of resource names that are not included in the restore. items: type: string nullable: true type: array existingResourcePolicy: description: ExistingResourcePolicy specifies the restore behavior for the Kubernetes resource to be restored nullable: true type: string hooks: description: Hooks represent custom behaviors that should be executed during or post restore. properties: resources: items: description: |- RestoreResourceHookSpec defines one or more RestoreResrouceHooks that should be executed based on the rules defined for namespaces, resources, and label selector. properties: excludedNamespaces: description: ExcludedNamespaces specifies the namespaces to which this hook spec does not apply. items: type: string nullable: true type: array excludedResources: description: ExcludedResources specifies the resources to which this hook spec does not apply. items: type: string nullable: true type: array includedNamespaces: description: |- IncludedNamespaces specifies the namespaces to which this hook spec applies. If empty, it applies to all namespaces. items: type: string nullable: true type: array includedResources: description: |- IncludedResources specifies the resources to which this hook spec applies. If empty, it applies to all resources. items: type: string nullable: true type: array labelSelector: description: LabelSelector, if specified, filters the resources to which this hook spec applies. nullable: true properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: |- operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: |- values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array x-kubernetes-list-type: atomic required: - key - operator type: object type: array x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string description: |- matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic name: description: Name is the name of this hook. type: string postHooks: description: PostHooks is a list of RestoreResourceHooks to execute during and after restoring a resource. items: description: RestoreResourceHook defines a restore hook for a resource. properties: exec: description: Exec defines an exec restore hook. properties: command: description: Command is the command and arguments to execute from within a container after a pod has been restored. items: type: string minItems: 1 type: array container: description: |- Container is the container in the pod where the command should be executed. If not specified, the pod's first container is used. type: string execTimeout: description: |- ExecTimeout defines the maximum amount of time Velero should wait for the hook to complete before considering the execution a failure. type: string onError: description: OnError specifies how Velero should behave if it encounters an error executing this hook. enum: - Continue - Fail type: string waitForReady: description: WaitForReady ensures command will be launched when container is Ready instead of Running. nullable: true type: boolean waitTimeout: description: |- WaitTimeout defines the maximum amount of time Velero should wait for the container to be Ready before attempting to run the command. type: string required: - command type: object init: description: Init defines an init restore hook. properties: initContainers: description: InitContainers is list of init containers to be added to a pod during its restore. items: type: object x-kubernetes-preserve-unknown-fields: true type: array x-kubernetes-preserve-unknown-fields: true timeout: description: Timeout defines the maximum amount of time Velero should wait for the initContainers to complete. type: string type: object type: object type: array required: - name type: object type: array type: object includeClusterResources: description: |- IncludeClusterResources specifies whether cluster-scoped resources should be included for consideration in the restore. If null, defaults to true. nullable: true type: boolean includedNamespaces: description: |- IncludedNamespaces is a slice of namespace names to include objects from. If empty, all namespaces are included. items: type: string nullable: true type: array includedResources: description: |- IncludedResources is a slice of resource names to include in the restore. If empty, all resources in the backup are included. items: type: string nullable: true type: array itemOperationTimeout: description: |- ItemOperationTimeout specifies the time used to wait for RestoreItemAction operations The default value is 4 hour. type: string labelSelector: description: |- LabelSelector is a metav1.LabelSelector to filter with when restoring individual objects from the backup. If empty or nil, all objects are included. Optional. nullable: true properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: |- operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: |- values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array x-kubernetes-list-type: atomic required: - key - operator type: object type: array x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string description: |- matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic namespaceMapping: additionalProperties: type: string description: |- NamespaceMapping is a map of source namespace names to target namespace names to restore into. Any source namespaces not included in the map will be restored into namespaces of the same name. type: object orLabelSelectors: description: |- OrLabelSelectors is list of metav1.LabelSelector to filter with when restoring individual objects from the backup. If multiple provided they will be joined by the OR operator. LabelSelector as well as OrLabelSelectors cannot co-exist in restore request, only one of them can be used items: description: |- A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: |- operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: |- values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array x-kubernetes-list-type: atomic required: - key - operator type: object type: array x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string description: |- matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic nullable: true type: array preserveNodePorts: description: PreserveNodePorts specifies whether to restore old nodePorts from backup. nullable: true type: boolean resourceModifier: description: ResourceModifier specifies the reference to JSON resource patches that should be applied to resources before restoration. nullable: true properties: apiGroup: description: |- APIGroup is the group for the resource being referenced. If APIGroup is not specified, the specified Kind must be in the core API group. For any other third-party types, APIGroup is required. type: string kind: description: Kind is the type of resource being referenced type: string name: description: Name is the name of resource being referenced type: string required: - kind - name type: object x-kubernetes-map-type: atomic restorePVs: description: |- RestorePVs specifies whether to restore all included PVs from snapshot nullable: true type: boolean restoreStatus: description: |- RestoreStatus specifies which resources we should restore the status field. If nil, no objects are included. Optional. nullable: true properties: excludedResources: description: ExcludedResources specifies the resources to which will not restore the status. items: type: string nullable: true type: array includedResources: description: |- IncludedResources specifies the resources to which will restore the status. If empty, it applies to all resources. items: type: string nullable: true type: array type: object scheduleName: description: |- ScheduleName is the unique name of the Velero schedule to restore from. If specified, and BackupName is empty, Velero will restore from the most recent successful backup created from this schedule. type: string uploaderConfig: description: UploaderConfig specifies the configuration for the restore. nullable: true properties: parallelFilesDownload: description: ParallelFilesDownload is the concurrency number setting for restore. type: integer writeSparseFiles: description: WriteSparseFiles is a flag to indicate whether write files sparsely or not. nullable: true type: boolean type: object type: object status: description: RestoreStatus captures the current status of a Velero restore properties: completionTimestamp: description: |- CompletionTimestamp records the time the restore operation was completed. Completion time is recorded even on failed restore. The server's time is used for StartTimestamps format: date-time nullable: true type: string errors: description: |- Errors is a count of all error messages that were generated during execution of the restore. The actual errors are stored in object storage. type: integer failureReason: description: FailureReason is an error that caused the entire restore to fail. type: string hookStatus: description: HookStatus contains information about the status of the hooks. nullable: true properties: hooksAttempted: description: |- HooksAttempted is the total number of attempted hooks Specifically, HooksAttempted represents the number of hooks that failed to execute and the number of hooks that executed successfully. type: integer hooksFailed: description: HooksFailed is the total number of hooks which ended with an error type: integer type: object phase: description: Phase is the current state of the Restore enum: - New - FailedValidation - InProgress - WaitingForPluginOperations - WaitingForPluginOperationsPartiallyFailed - Completed - PartiallyFailed - Failed - Finalizing - FinalizingPartiallyFailed type: string progress: description: |- Progress contains information about the restore's execution progress. Note that this information is best-effort only -- if Velero fails to update it during a restore for any reason, it may be inaccurate/stale. nullable: true properties: itemsRestored: description: ItemsRestored is the number of items that have actually been restored so far type: integer totalItems: description: |- TotalItems is the total number of items to be restored. This number may change throughout the execution of the restore due to plugins that return additional related items to restore type: integer type: object restoreItemOperationsAttempted: description: |- RestoreItemOperationsAttempted is the total number of attempted async RestoreItemAction operations for this restore. type: integer restoreItemOperationsCompleted: description: |- RestoreItemOperationsCompleted is the total number of successfully completed async RestoreItemAction operations for this restore. type: integer restoreItemOperationsFailed: description: |- RestoreItemOperationsFailed is the total number of async RestoreItemAction operations for this restore which ended with an error. type: integer startTimestamp: description: |- StartTimestamp records the time the restore operation was started. The server's time is used for StartTimestamps format: date-time nullable: true type: string validationErrors: description: |- ValidationErrors is a slice of all validation errors (if applicable) items: type: string nullable: true type: array warnings: description: |- Warnings is a count of all warning messages that were generated during execution of the restore. The actual warnings are stored in object storage. type: integer type: object type: object served: true storage: true ================================================ FILE: config/crd/v1/bases/velero.io_schedules.yaml ================================================ --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.16.5 name: schedules.velero.io spec: group: velero.io names: kind: Schedule listKind: ScheduleList plural: schedules singular: schedule scope: Namespaced versions: - additionalPrinterColumns: - description: Status of the schedule jsonPath: .status.phase name: Status type: string - description: A Cron expression defining when to run the Backup jsonPath: .spec.schedule name: Schedule type: string - description: The last time a Backup was run for this schedule jsonPath: .status.lastBackup name: LastBackup type: date - jsonPath: .metadata.creationTimestamp name: Age type: date - jsonPath: .spec.paused name: Paused type: boolean name: v1 schema: openAPIV3Schema: description: |- Schedule is a Velero resource that represents a pre-scheduled or periodic Backup that should be run. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: ScheduleSpec defines the specification for a Velero schedule properties: paused: description: Paused specifies whether the schedule is paused or not type: boolean schedule: description: |- Schedule is a Cron expression defining when to run the Backup. type: string skipImmediately: description: |- SkipImmediately specifies whether to skip backup if schedule is due immediately from `schedule.status.lastBackup` timestamp when schedule is unpaused or if schedule is new. If true, backup will be skipped immediately when schedule is unpaused if it is due based on .Status.LastBackupTimestamp or schedule is new, and will run at next schedule time. If false, backup will not be skipped immediately when schedule is unpaused, but will run at next schedule time. If empty, will follow server configuration (default: false). type: boolean template: description: |- Template is the definition of the Backup to be run on the provided schedule properties: csiSnapshotTimeout: description: |- CSISnapshotTimeout specifies the time used to wait for CSI VolumeSnapshot status turns to ReadyToUse during creation, before returning error as timeout. The default value is 10 minute. type: string datamover: description: |- DataMover specifies the data mover to be used by the backup. If DataMover is "" or "velero", the built-in data mover will be used. type: string defaultVolumesToFsBackup: description: |- DefaultVolumesToFsBackup specifies whether pod volume file system backup should be used for all volumes by default. nullable: true type: boolean defaultVolumesToRestic: description: |- DefaultVolumesToRestic specifies whether restic should be used to take a backup of all pod volumes by default. Deprecated: this field is no longer used and will be removed entirely in future. Use DefaultVolumesToFsBackup instead. nullable: true type: boolean excludedClusterScopedResources: description: |- ExcludedClusterScopedResources is a slice of cluster-scoped resource type names to exclude from the backup. If set to "*", all cluster-scoped resource types are excluded. The default value is empty. items: type: string nullable: true type: array excludedNamespaceScopedResources: description: |- ExcludedNamespaceScopedResources is a slice of namespace-scoped resource type names to exclude from the backup. If set to "*", all namespace-scoped resource types are excluded. The default value is empty. items: type: string nullable: true type: array excludedNamespaces: description: |- ExcludedNamespaces contains a list of namespaces that are not included in the backup. items: type: string nullable: true type: array excludedResources: description: |- ExcludedResources is a slice of resource names that are not included in the backup. items: type: string nullable: true type: array hooks: description: Hooks represent custom behaviors that should be executed at different phases of the backup. properties: resources: description: Resources are hooks that should be executed when backing up individual instances of a resource. items: description: |- BackupResourceHookSpec defines one or more BackupResourceHooks that should be executed based on the rules defined for namespaces, resources, and label selector. properties: excludedNamespaces: description: ExcludedNamespaces specifies the namespaces to which this hook spec does not apply. items: type: string nullable: true type: array excludedResources: description: ExcludedResources specifies the resources to which this hook spec does not apply. items: type: string nullable: true type: array includedNamespaces: description: |- IncludedNamespaces specifies the namespaces to which this hook spec applies. If empty, it applies to all namespaces. items: type: string nullable: true type: array includedResources: description: |- IncludedResources specifies the resources to which this hook spec applies. If empty, it applies to all resources. items: type: string nullable: true type: array labelSelector: description: LabelSelector, if specified, filters the resources to which this hook spec applies. nullable: true properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: |- operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: |- values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array x-kubernetes-list-type: atomic required: - key - operator type: object type: array x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string description: |- matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic name: description: Name is the name of this hook. type: string post: description: |- PostHooks is a list of BackupResourceHooks to execute after storing the item in the backup. These are executed after all "additional items" from item actions are processed. items: description: BackupResourceHook defines a hook for a resource. properties: exec: description: Exec defines an exec hook. properties: command: description: Command is the command and arguments to execute. items: type: string minItems: 1 type: array container: description: |- Container is the container in the pod where the command should be executed. If not specified, the pod's first container is used. type: string onError: description: OnError specifies how Velero should behave if it encounters an error executing this hook. enum: - Continue - Fail type: string timeout: description: |- Timeout defines the maximum amount of time Velero should wait for the hook to complete before considering the execution a failure. type: string required: - command type: object required: - exec type: object type: array pre: description: |- PreHooks is a list of BackupResourceHooks to execute prior to storing the item in the backup. These are executed before any "additional items" from item actions are processed. items: description: BackupResourceHook defines a hook for a resource. properties: exec: description: Exec defines an exec hook. properties: command: description: Command is the command and arguments to execute. items: type: string minItems: 1 type: array container: description: |- Container is the container in the pod where the command should be executed. If not specified, the pod's first container is used. type: string onError: description: OnError specifies how Velero should behave if it encounters an error executing this hook. enum: - Continue - Fail type: string timeout: description: |- Timeout defines the maximum amount of time Velero should wait for the hook to complete before considering the execution a failure. type: string required: - command type: object required: - exec type: object type: array required: - name type: object nullable: true type: array type: object includeClusterResources: description: |- IncludeClusterResources specifies whether cluster-scoped resources should be included for consideration in the backup. nullable: true type: boolean includedClusterScopedResources: description: |- IncludedClusterScopedResources is a slice of cluster-scoped resource type names to include in the backup. If set to "*", all cluster-scoped resource types are included. The default value is empty, which means only related cluster-scoped resources are included. items: type: string nullable: true type: array includedNamespaceScopedResources: description: |- IncludedNamespaceScopedResources is a slice of namespace-scoped resource type names to include in the backup. The default value is "*". items: type: string nullable: true type: array includedNamespaces: description: |- IncludedNamespaces is a slice of namespace names to include objects from. If empty, all namespaces are included. items: type: string nullable: true type: array includedResources: description: |- IncludedResources is a slice of resource names to include in the backup. If empty, all resources are included. items: type: string nullable: true type: array itemOperationTimeout: description: |- ItemOperationTimeout specifies the time used to wait for asynchronous BackupItemAction operations The default value is 4 hour. type: string labelSelector: description: |- LabelSelector is a metav1.LabelSelector to filter with when adding individual objects to the backup. If empty or nil, all objects are included. Optional. nullable: true properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: |- operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: |- values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array x-kubernetes-list-type: atomic required: - key - operator type: object type: array x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string description: |- matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic metadata: properties: labels: additionalProperties: type: string type: object type: object orLabelSelectors: description: |- OrLabelSelectors is list of metav1.LabelSelector to filter with when adding individual objects to the backup. If multiple provided they will be joined by the OR operator. LabelSelector as well as OrLabelSelectors cannot co-exist in backup request, only one of them can be used. items: description: |- A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: |- operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: |- values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array x-kubernetes-list-type: atomic required: - key - operator type: object type: array x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string description: |- matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic nullable: true type: array orderedResources: additionalProperties: type: string description: |- OrderedResources specifies the backup order of resources of specific Kind. The map key is the resource name and value is a list of object names separated by commas. Each resource name has format "namespace/objectname". For cluster resources, simply use "objectname". nullable: true type: object resourcePolicy: description: ResourcePolicy specifies the referenced resource policies that backup should follow properties: apiGroup: description: |- APIGroup is the group for the resource being referenced. If APIGroup is not specified, the specified Kind must be in the core API group. For any other third-party types, APIGroup is required. type: string kind: description: Kind is the type of resource being referenced type: string name: description: Name is the name of resource being referenced type: string required: - kind - name type: object x-kubernetes-map-type: atomic snapshotMoveData: description: SnapshotMoveData specifies whether snapshot data should be moved nullable: true type: boolean snapshotVolumes: description: |- SnapshotVolumes specifies whether to take snapshots of any PV's referenced in the set of objects included in the Backup. nullable: true type: boolean storageLocation: description: StorageLocation is a string containing the name of a BackupStorageLocation where the backup should be stored. type: string ttl: description: |- TTL is a time.Duration-parseable string describing how long the Backup should be retained for. type: string uploaderConfig: description: UploaderConfig specifies the configuration for the uploader. nullable: true properties: parallelFilesUpload: description: ParallelFilesUpload is the number of files parallel uploads to perform when using the uploader. type: integer type: object volumeGroupSnapshotLabelKey: description: VolumeGroupSnapshotLabelKey specifies the label key to group PVCs under a VGS. type: string volumeSnapshotLocations: description: VolumeSnapshotLocations is a list containing names of VolumeSnapshotLocations associated with this backup. items: type: string type: array type: object useOwnerReferencesInBackup: description: |- UseOwnerReferencesBackup specifies whether to use OwnerReferences on backups created by this Schedule. nullable: true type: boolean required: - schedule - template type: object status: description: ScheduleStatus captures the current state of a Velero schedule properties: lastBackup: description: |- LastBackup is the last time a Backup was run for this Schedule schedule format: date-time nullable: true type: string lastSkipped: description: LastSkipped is the last time a Schedule was skipped format: date-time nullable: true type: string phase: description: Phase is the current phase of the Schedule enum: - New - Enabled - FailedValidation type: string validationErrors: description: |- ValidationErrors is a slice of all validation errors (if applicable) items: type: string type: array type: object type: object served: true storage: true subresources: {} ================================================ FILE: config/crd/v1/bases/velero.io_serverstatusrequests.yaml ================================================ --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.16.5 name: serverstatusrequests.velero.io spec: group: velero.io names: kind: ServerStatusRequest listKind: ServerStatusRequestList plural: serverstatusrequests shortNames: - ssr singular: serverstatusrequest scope: Namespaced versions: - name: v1 schema: openAPIV3Schema: description: |- ServerStatusRequest is a request to access current status information about the Velero server. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: ServerStatusRequestSpec is the specification for a ServerStatusRequest. type: object status: description: ServerStatusRequestStatus is the current status of a ServerStatusRequest. properties: phase: description: Phase is the current lifecycle phase of the ServerStatusRequest. enum: - New - Processed type: string plugins: description: Plugins list information about the plugins running on the Velero server items: description: PluginInfo contains attributes of a Velero plugin properties: kind: type: string name: type: string required: - kind - name type: object nullable: true type: array processedTimestamp: description: |- ProcessedTimestamp is when the ServerStatusRequest was processed by the ServerStatusRequestController. format: date-time nullable: true type: string serverVersion: description: ServerVersion is the Velero server version. type: string type: object type: object served: true storage: true ================================================ FILE: config/crd/v1/bases/velero.io_volumesnapshotlocations.yaml ================================================ --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.16.5 name: volumesnapshotlocations.velero.io spec: group: velero.io names: kind: VolumeSnapshotLocation listKind: VolumeSnapshotLocationList plural: volumesnapshotlocations shortNames: - vsl singular: volumesnapshotlocation scope: Namespaced versions: - name: v1 schema: openAPIV3Schema: description: VolumeSnapshotLocation is a location where Velero stores volume snapshots. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: VolumeSnapshotLocationSpec defines the specification for a Velero VolumeSnapshotLocation. properties: config: additionalProperties: type: string description: Config is for provider-specific configuration fields. type: object credential: description: Credential contains the credential information intended to be used with this location properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: default: "" description: |- Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object x-kubernetes-map-type: atomic provider: description: Provider is the provider of the volume storage. type: string required: - provider type: object status: description: VolumeSnapshotLocationStatus describes the current status of a Velero VolumeSnapshotLocation. properties: phase: description: VolumeSnapshotLocationPhase is the lifecycle phase of a Velero VolumeSnapshotLocation. enum: - Available - Unavailable type: string type: object type: object served: true storage: true ================================================ FILE: config/crd/v1/crds/crds.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ // Code generated by crds_generate.go; DO NOT EDIT. package crds import ( "bytes" "compress/gzip" "io" apiextinstall "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/client-go/kubernetes/scheme" ) var rawCRDs = [][]byte{ []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xccXK\x8f۶\x13\xbf\xfbS\f\xf6\x7f\xfd\xcbnP\xb4(|K\xdc\x06\b\x9a\x04\v{\x91;-\x8dlf)\x92%\x87N\xdd\xc7w/\x86\x94lY\xa2\xad\xf5\x1e\x8a\xf2&r\u07bfy\xd9EQ̄\x95_\xd0yi\xf4\x12\x84\x95\xf8;\xa1\xe6/?\x7f\xfe\xc9ϥY\x1c\xde̞\xa5\xae\x96\xb0\n\x9eL\xb3Fo\x82+\xf1g\xac\xa5\x96$\x8d\x9e5H\xa2\x12$\x963\x00\xa1\xb5!\xc1מ?\x01J\xa3\xc9\x19\xa5\xd0\x15;\xd4\xf3\xe7\xb0\xc5m\x90\xaaB\x17\x85w\xaa\x0f\xdf\xcd\xdf\xfc8\xffa\x06\xa0E\x83K؊\xf29X\x87\xd6xI\xc6I\xf4\xf3\x03*tf.\xcd\xcc[,Y\xfaΙ`\x97p~Hܭ\xe6d\xf5\xbb(h\xdd\t:\xc6'%=\xfd\x9a}\xfe(=E\x12\xab\x82\x13*gH|\xf6R\xef\x82\x12nD\xc0\n|i,.\xe13\xdbbE\x89\xd5\f\xa0\xf54\xdaV\x80\xa8\xaa\x18;\xa1\x1e\x9dԄneTh\xba\x98\x15\xf0\xd5\x1b\xfd(h\xbf\x84y\x17\xddy\xe90\x06\xf6I6\xe8I46\xd2v\x01{\xbb\xc3\xf6\x9b\x8e\xac\xbc\x12\x84ca\x1c\xb9\xf9\xd9֧\xa3\xc5\v)\xe7@@\xef-I\xf4\xe4\xa4\xde\xcd\xceć7)\x14\xe5\x1e\x1b\xb1li\x8dE\xfd\xf6\xf1×\xef7\x17\xd7\x00\xd6\x19\x8b\x8ed\aO:\xbd\xf4\xeb\xdd\x02T\xe8K'-\xc5\xe4\xf8\xab\xb8x\x03`\x05\x89\v*\xceC\xf4@{\xecb\x8cUk\x13\x98\x1ah/=8\xb4\x0e=ꔙ|-4\x98\xedW,i>\x10\xbdA\xc7b\xc0\xefMP\x15\xa7\xef\x01\x1d\x81\xc3\xd2\xec\xb4\xfc\xe3$\xdb\x03\x99\xa8T\tBO\x10Q\xd4B\xc1A\xa8\x80\xff\a\xa1\xab\x81\xe4F\x1c\xc1!넠{\xf2\"\x83\x1f\xda\xf1\xc98\x04\xa9k\xb3\x84=\x91\xf5\xcb\xc5b'\xa9+\xca\xd24MВ\x8e\x8bX_r\x1b\xc88\xbf\xa8\xf0\x80j\xe1\xe5\xae\x10\xae\xdcK\u0092\x82Å\xb0\xb2\x88\x8e\xe8X\x98\xf3\xa6\xfa\x9fk\xcb\xd8_\xa8\x1d\x01\x9dN\xac\xa4;\xe0\xe1\xd2\x02\xe9A\xb4\xa2\x92\x8bg\x14\xf8\x8aC\xb7\xfee\xf3\x04\x9d%\t\xa9\x04ʙt\x14\x97\x0e\x1f\x8e\xa6\xd45\xba\xc4W;\xd3D\x99\xa8+k\xa4\xa6\xf8Q*\x89\x9a\xc0\x87m#\x89\xd3ව\x9e\x18\xba\xa1\xd8Ul\\\xb0E\b\x96K\xa7\x1a\x12|а\x12\r\xaa\x95\xf0\xf8/cŨ\xf8\x82Ax\x11Z\xfdv<$N\xe1\xed=t\xad\xf4\n\xb4\xc3\xf6\xb8\xb1X2\xb2\x1c\\f\x95\xb5,SM\xd5Ɓ\x18\xd1_F*\xdf\x02\xf8\xa4&\xba!\xe3\xc4\x0e?\x9a$sH4\x95v|\xde\xe5\x04u\x16s\xdbJ=\x01\xf3\x84\x19\x81\xb4\x17\xd4k\x06$\xa4>\xf5\x94\xac\x937\x90\x89\xe8\b\xee\x14Z\xe8\x12\xdf\xc7|\xd4\xe5q\xc2\xd1O\x19\x16vio\xbe\x81\xa9\tu_hkkƓ-\x82\v\xfa.c\xcf>\xae\x8c\xae\xe5nlh\x7f\x90]\x03wB\xc9\xc0\xdb\xf5@'{\xca\xc9u\xb6\xa5\xe82\x8f\x01\xa9\xe5.\xb8k\xe0\xd5\x12U5j!\x00:(%\xb6\n\x97@.\xe0\x95\x88\x8cj\xe52\"<\x1f'\x80[_\x10\x83\xd4\x15WK;\xacXI\x97\x8c\x9c\xfe\xa8+p\x97kJ\xff\xa0\x0e\xcdX]\x01\xcf\xc6J\x91\xb9w\xe8I\x96\x99\x87\x87\x87\xfb2\x80\xc5|\xa8\xb8\x1d\xd5\x12\xddkjr=\x90ѕc\x1d\x94j\x15\x14\xa5i\xac \xb9U\xd8\xcd\f\xc6\\&\x9ec.i`T\x86\xf0\x14'\x01c\xce*\x8cVG\b\x1e+\xf8\xb6G=\x02\xc3\xc3C\xd2\xfdpWI\x1cxQ\xc3\xd3j\xf7\x9ax|\xb9\x14\xd1\xefN\xe9\":\x96ZbϿ\xae\xfd\xf8\x8cHk\xaaֲ\x96/\xd6\xcc\x1d\x8eq_\x91\x0e\ac\xbe\xc87\xe6\x01M\xae\xa5\rH\x06Q{\xd1d\"A\xc1\xdf3\x9b\"C\x17\xcd28\x17g\x7f\xba\xe5\x95\xef\xd5\xd3I\tO\xbd&\xcc\v\xf8\x04\xee\x1f\xc7\x1c\x9da,\f\x88/\x18\xda~\xf02\xb8\xfaP\x96\x88\xd5x\x1d\x01Ʒ\x11\x94\x16\xfd\x82彮\xcb\xe5\x87\x14z/vSN~JTi\xd3kY@lM\xa0+\b\xd0>\xe7\xe3mT&,\xb5{\xe1\xa7\xec|d\x9a\\^\f\x96\x81[&\\k\xbf\x9f\xf1[\xe6v\x8d\xa2\x1a\xb7\xf0\x02>\x1b\xca?\xdd\xec\xc0%\xea~2M\x0e\x9d\x01={~\x81A+r\x94\x7fc\xaf%a\x93\x1d\xe7\xd7k%\x1dn\xe7\n\tO\xbfU\xf3d\x03\xd3WC\xae\x13h\xe9\x81W\xb9X9Ws\xa9\vٔc\xe9L\x97P:\x13\x85\x94\xce\xcd\r\an\x15U&\x12\xf7\x96\xd6\xd5P$\xb8_\x16\x8eI\x0f\x1c\xfa\xa0\xe8E\x0e\xac#i\x87_b<\xa7\xdf\xcb\xec\xc9\xd7\\:\x05l\xba\xd6x\x95⽐\xea\xea\U000e4cde\x84\xa3\xfb\xf2ws\xc1r\xfa\x9dķ\xfd\xbc\xfdO\xe6獝\xb7{\x14Ή\xe3\xf4\xe8\x1e]z\xfe\xc9^\xf5\x8c\xf3i\x9d\xe8߄\xed\xe9\x1f\x89%\xfc\xf9\xf7\xec\x9f\x00\x00\x00\xff\xff\x18\xd9g\x90\x9b\x14\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xec=]s\xdb8\x92\xef\xf9\x15(\xdd\xc3\xecnI\xf6\xa6\ue8ee\xfc\x96q\x92\x1d\xd5\xcc$\xde\xd8\xe3}\x86Ȗ\x841\bp\x00P\xb6\xf6\xee\xfe\xfb\x15\x1a\x00?DP\x04eٓ\xdd\r_\x12\x8b`\x03\xfd\xddh4\x80\xc5b\xf1\x86\x96\xec\x1e\x94fR\\\x11Z2x2 \xec_\xfa\xe2\xe1\xbf\xf5\x05\x93\x97\xbb\xb7o\x1e\x98ȯ\xc8u\xa5\x8d,\xbe\x80\x96\x95\xca\xe0=\xac\x99`\x86I\xf1\xa6\x00Csj\xe8\xd5\x1bB\xa8\x10\xd2P\xfb\xb3\xb6\x7f\x12\x92Ia\x94\xe4\x1c\xd4b\x03\xe2\xe2\xa1Z\xc1\xaab<\a\x85\xc0C\u05fb?_\xbc\xfd\xaf\x8b\xff|C\x88\xa0\x05\\\x91\x15\xcd\x1e\xaaR_쀃\x92\x17L\xbe\xd1%d\x16\xe4Fɪ\xbc\"\xcd\v\xf7\x89\xef\xce\r\xf5{\xfc\x1a\x7f\xe0L\x9b\x1f[?\xfeĴ\xc1\x17%\xaf\x14\xe5uO\xf8\x9bfbSq\xaa¯o\bљ,\xe1\x8a|\xb2]\x944\x83\xfc\r!~\xd4\xd8\xe5\xc2\x0fx\xf7\xd6AȶPP7\x16Bd\t\xe2\xdd\xcd\xf2\xfe\xdfo;?\x13\x92\x83\xce\x14+\r\xe2\xfe\xbf\x8b\xfaw\xe2GI\x98&\x94\xdc#\x8eDy\x92\x13\xb3\xa5\x86((\x15h\x10F\x13\xb3\x05\x92\xd1\xd2T\n\x88\\\x93\x1f\xab\x15(\x01\x06t\v^\xc6+m@\x11m\xa8\x01B\r\xa1\xa4\x94L\x18\xc2\x041\xac\x00\xf2\x87w7K\"W\xbfBf4\xa1\"'Tk\x991j ';ɫ\x02ܷ\x7f\xbc\xa8\xa1\x96J\x96\xa0\f\vDwOK\x92Z\xbf\x1e\xc3\xd5>\x96<\xee+\x92[\x91\x02\x87\x96'1䞢\x16?\xb3e\xbaA\x1f\x85\xcc\xfeL\x85\x1f\xfe\xc5\x01\xe8[P\x16\f\xd1[Y\xf1\xdcJ\xe2\x0e\x94%`&7\x82\xfd\xbd\x86\xad\x89\x91\xd8)\xa7\x06\xb4\xa5\x8c\x01%(';\xca+\x98[\xa2\x1c@.\xe8\x9e(\xb0}\x92J\xb4\xe0\xe1\a\xfap\x1c?K\x05\x84\x89\xb5\xbc\"[cJ}uy\xb9a&\xe8W&\x8b\xa2\x12\xcc\xec/QUت2R\xe9\xcb\x1cv\xc0/5\xdb,\xa8ʶ\xcc@f\xd9|IK\xb6@D\x04\xea\xd8E\x91\xff[\x10\x0f\xdd\xe9\xd6\xec\xad\xd8j\xa3\x98ش^\xa0~L`\x8fU\x1d'\x8c\x0e\x94C\xb1\xe1\x82\xfdɒ\xeeˇۻ\xb6\xa02\xed\x99Ғ\xd7!\xfeXj2\xb1\x06\xe5\xbe[+Y L\x10\xb9\x13U\x94s\xce@\x18\xa2\xabU\xc1\x8c\x15\x83\xdf*\xd0V\a\xe4!\xd8k\xb4Ad\x05\xa4*s+Ƈ\r\x96\x82\\\xd3\x02\xf85\xd5\xf0ʼ\xb2\\\xd1\v˄$n\xb5-\xebacG\xde\u058b` \aX\xeb\f\xcbm\tYG\xd1\xecWl\xcd2\xa7Nk\xa9\x1a\xbb\xe3l`\x97BqշO\xa6٭\xa0\xa5\xdeJs\xc7\n\x90\x959l1&kȼ\xdb\xe5\x01\x940B?^\xb4Y\x95\x86\xdc*\xed#e\x06\xc7|}\xbb$\xf7h\xac\xc2\xd7h\xb4*ML\xa5\x84\x95\x92H__\x80\xe6\xfb;\xf9\x8b\x06\x92W(ܙ\x02\xa4Ü\xac`m%A\x81\xfd\u07be\x02\xa5,m4\x0e@V=cc\x9f\xbb-X\xdaҊ\x1b\xaf'L\x93\xb7\x7f&\x05\x13\x95\xe9\x89\xda בR\xd4\xd0B\xee@\x9dB\xc4\xf7\xd4П\xed\xc7\a\xb4\xb3@\tB\xb5\xc4[y:\xae\xf6\xf82\xc6m\xf7,\xd7-\x88L\x93ٌHEf\xce\x03\xcf\xe6\xee\xeb\x8aq\xb3`\xa2\xdd\xc7#\xe3<\xf42\ryGC\xc7P}'?j'\xbc'\xd1b\x00V\x8b4\x8f[0[P\xa4\x94\xb5\xc7[3\x0eD﵁\xc2\x13&x\x11\x8fO\xa4'\xd4\x1d\xce=\bm\xe9\xea\x11\xe9#/*\xce\xe9\x8a\xc3\x151\xaa\x82\x01ڬ\xa4\xe4@\xc5\bq\xbe\x806,;\ai\x1c\xa4\ba\x94\x7fѡ\x00:M\xfa\x00\x84F@{\x9aY\xef\xccy\x8b\xb0]\xaaD\xc7T*Ȭվ\xf2ހ\x01G\x0f$$\xe1Rl@\xb9\xdem\xa4\x12\x04L\x81\x15\xb8\x9cXC\xab\x80[oB֕\xb5\xc1\x17\xc4j\xf7\xa0\f0\xa1\rЈp>\x83?\xf0\x94\xf1*\x87\xfc\xda\x05^\xb76~\xccC\xd4ܳ\x9a)|\xfap\x14\xa2\xf7Μe\x18\x04\xfaxo\x81qkLL\x1b'\xbd/\xc1\x85Ζ\x95~؍\xf7=j\x0f4\x18\xfb\xd1\xecO\xb39r\xb8\xdbk\xb7\x0fM\xa8\x82\x9a,\xc9v\x13\x8a\xd2\xec\xfb\xad\x99\x81\"Bţ\xf6$\x91\x9fT)\xba\x1f\xe0f\x1d\xff\x9f\x91\x9fC0\x0f8*B\xb3W\xe6\xe9a\xbf\xff\xcc\\=\x0f\x1f5\xcev)\x13\x96\x7fv\xe2\xd9a\x9fv\xf37K6!M\x04\x1e\x13\x0e\x1eN͎p\xebw\"\xd6Yd~H\xc8k\xd9\xf2\xc2\xfb\x0fI\xa9\xad\x94\x0fc\xd4\xf9\xc1\xb6i&E$ì\nY\xc1\x96\xee\x98T\x1e\xf5\xc6\xd5\xc2\x13d\x95\x89j=5$g\xeb5(\v\xa7\xdcR\r\xdaM\x93\x87\t2\x1c\xbe\x93\x96\x19\x89\xbe<\xc0\xa3a\xa4e\x13b>4t\x1bG\x1cz\xc9\xf0\u0601\xda\xf0\x1a\x9dq\xcev,\xaf(G\xbfLE\xe6\xf0\xa1\xf5\xb8bV\xe6\b\x93{c\x8eJ\xa6{\\@\x10\x90\xb2L\xea̔\xa4\x00\x1b\xf3\x16vN\xd0o:\x8c\xf9\x8a\xdaXE\x0eaO\x90Y\xaa\xe2\xa0}W9\x86\x91\x8d͘7L\xc1D\x04\xe1t\x05\x9ch\xe0\x90\x19\xa9\xe2\x14\x19\xe3\xb3{R\x8c\xe0\x00!#\x96\xaf;\xd3h\x108\x02\x92\xe0\x14n˲\xad\v\xf5\xac\x10!\x1c\x92K\xb0\x01\x9f!\xb4,y\xc4]4\xcfQ\xe6\xfbN\x8e\xe9z\xf3\x8ch\xfd!\xbc\x98\xfe7O\x82\xcdl\x9e(i\x1b\xfd\xeaR\xb6\x16\x87\xf8\x9c\xb6y\xfe9\t\x1b,\xff\tB{D\xfb\tf\x85\x92ezPn-U\x19\xe8\v\x1bNa\xa43'̄_\xc74\xa1\x13s\xf5\x92e\x1d\"|ݼ\x99.\xf4\x89\xacIщ\x17bL\xdd\xc5? _\xd0e\xdcz\x8f\x91̓\x9f\xda_\xcd\t[\xd7D\xcf\xe7d\u0378\x01u@\xfd\x93L}\xe0\xcc9\x88\x91\xe2\xf5\b\xa6\xefM\xb6\xfd\xf0dC0ݬT%\xd2\xe5\xf0c\x17Ȇh\xbf\xeb\x9eG\xe0\x12Lc3\x05\x05\xa6\xc7q\xc6\xd4\xfe\x05C\xabw\x9f\xde\xc7\xe7W\xed'A\xf2z\x88\x8c(\x9d{\xde\x1d`\xd4\x1e\x9f\x0f\xe1\xc3\x1b\x8c\x81\xea\t\x90[\n\x99\x13J\x1e`\xefB\x17*\x88\xe5\x0f\r\x8d\x13\xbaW\x80k2(g\x0f\xb0G0\xf1E\x96\xfe\x93*\r\xeey\x80}J\xb3\x03\x1a\xda11\xed\x17\x8f,\x9d\xec\x0fH\b̭\xa7\x8a\x81{\xbc*D\x964\xe2O\xa2-\tO\xa0\xfd\th&\x89J\xbb\x8f\xf6*%J\xc0w\xda\xf1\xd2j̖\x95hV1\xe3 \xd7\xc9\fu\xcf=\xe5,\xaf;r:\xb2\x14s\xf2I\x1a\xfbχ'\xa6\xfdB\xe6{\t\xfa\x934\xf8ˋP\xd4\r\xfc%\xe9\xe9z@E\x13\xce\xca[\x82\xb5\x97\xe2\x9cO\xb3\xd2VӞi\xb2\x14v\xba\xe2H\x92\xd8\x15\xae\xba\xba\xee\\GE\xa5q\x15MH\xb1pi\x9bXO\x9e\xdeRu\xc8\xfd\xecN}\x87w\xd6Y\xb87n\xed\x97\xd3\f\xf2\xb0\\\x83\x8b\x92\xd4\xc0\x86e\x89\xfd\x15\xa06@Jk\xc2\xd3$\"Ѱzl\xa6\x89O\x9a\xf7n?O\x8b\x87z\x8d\x7fa]\xce\xc2C0\xb2H\xa0\x81\xb7\xdd\xf98>\v\xab\xb3\t\xad\x82$\x8c6\x1dX\xb3\x1cn\x9aB\x94g\x90\x03\xbd8\x868\xa3ܥy\x8eu.\x94\xdfL\xf0(\x13da\xaaih\x8dݹ\xe0\x82\xe2R\xcb\xffXO\x8b\xda\xf4\x7f\xa4\xa4L\xe9\v\xf2\x0eKZ8t\xde\xf9\xa4Y\vLB\x97X\x92b\xe5gG\xb9\xf5\xfdր\v\x02\xdcE\x02r\u074b\x8b\xe6\xe4q+\xb5s\xdb\xf5\"\xce\xec\x01\xf6n\xc5p\xb4˶\x91\x99-\xc5\xcc\xc5\x10=\x83Q\a\x1cR\xf0=\x99\xe1\xbb\xd9sB\xa9DIMl\xd6\x11т\x96i\x12\x8a%E\xa9\x81\xba\x9d\xb0\x86 \xc4~X\x97\xca\xd8 \xfb\x18\xb6I\"ZJ\x1dY\xc8\x1f\x18ʈ\xf0\xdeHm\\\xbe\xac\x133G\x13j2$\xd1\b]\xbb\xfa%\xa9B\xb1\x895\xcac\xa9\xdf\xf6s\xb7\x05\r~\xbd\xc2'\xe6\x1cP;\xb3\x9b5\xfa\xed\xac\xfḓ\x97`'4È\x05\xbf-\x95\xcc@Gײ\x9b'\xc1_D\xaa2ڸ\xd79G\xeafI\xae$\xe3x\n4<\xe9!\xaf%\xc4\xc4\xf9\u0087\xa7VB\xd4\xea\xbe\xfd{LƦ\x8e\x8b`\xc9`Q\xd0\xc32\xa5\xa4!^\xbb/\x836x@n\xf2\xa16\x15Z\x82T_^\v\xe0\xd7\x10(\x14L,\xb1\x03\xf2\xf6\x05\x02\voCc\xc5&\xb1\xe7\xb4P\xf6:t\xd2p\xa7\xfe\xc1\xa9r)q\xa9@A\x87y\xfd\xac:ơB\x9aVBbB\xb8Y\xca\xfc;M\xd6Li\xd3\x1e\x82\x1e(S\x89\x82\x998\xf1\x12\x1f\x94:i\xde\xf5\xd9}\xd9Jwm\xe5c(\xcfr\x84I\xc4\x1cח\x80\xb05a\x86\x80\xc8d%0\x81c\xf5\x18\xbbp\xc4u\x16\x96\xa5*I\x9a\xf6\xdb\aDU\xa4\x11`\x81\x92\xc2\xc4\xd1LO\xbb\xf9G\xca\xf8K\xb0\xcd\fU\xb1Ş\xd3t\"\x94\xb8\xb5\v\xf2\n\xfaĊ\xaa \xb4\xb0\x1f\xc0\xb0\x8c\x0f\xa1\xdd+\xc5\xc8E\xc5\r+9.\xa4\xeeX\x1eM6\x98-\xec\xeb\x034~\x95\xb8\xf5ԟ\x04\xf3\xf9K-\xb5\x17\a\x91>\xd5\xe4\x118'4\xa6W=\xcc3w\x12S&\x17`\xfd\x91\xd5N\x7f0\x88?\xbei\xee\xc4\x1dwעW+b)&*\x86O\x91\x19t\x1c)\xf6\xa6\x17\xc1\xba8\x1c\x7f\xfb\xad\x02\xb5'x\x8eM\x1d\xe74\x9b\xc0\xbcbj;\x11\v\xa6\u009b\xad\xa1\xfcy/\xe8oT\x99\xbc\x13\xce\xeb\x1e\x8e\a\xbf\xb16\xa2\x99\xd4X\xc3g\xe7+\xd1>\x06>\x17\xb2\xfe:\xf2\xd9X\x80\x9c\xba[\xeae\xa78\xd3'9\xa3QEz\xe4\xf7;\xed\x82:e\xf7SZ\x01\xc0\xe8n\xa7\x97\x9a\xf2\x8cMz\x92㼴\xddL\xd3\x16\v_p\xf7\xd2K\xecZJ\xa4T\xca.\xa5itz\x85]I\xaf\xba\x1b\xe9\xb5v!%\xef>J*qI^\x05N-Q9q;\xcd\xf8\x1a\xef\xf1\xddD\t\xbb\x88\x12V\x7fǑ<\x01\xbd\x84]B\xd3v\a%\xf0,U\x15_q\x17\xd0+\xee\xfey\xed]?#\x925\xf2z\xda\ue793\x97,\xa4\xcaA\x1d]\xf6I\x95£\xf2\x972\xb7\xe9\x0e\xe4`\xbd#\x9c\xfag[u\xe2et\x0f\xfe\xa0Q\x97G\xb3\xa2\xe4{;C!\xb3\xf6\a\xa7I@T\xdaBo7\x92\xb3,\x12\xbbE\xcffr\x8d{\x87e\xe0\x89QY\xbbd\xa0\xb4\r\xe3\xa1\x1b\x86y\xdd#0גs\xf98q\xeeOK\xf6\x17<\xb9\xfb\x19١w7K\x84\x11\xc4\x03\x8f\x02\xaf\x8b\xb3jlV`\xddr\x83\xe7\x90\xee/\xd7\x1d\x88\xdd:\xc7\xf6Ḑ\xbbs\x90CX\xe0Mg&\xadu\xb9Y\xbaq\f\xf5be\x86\x8a=\x91XQc\xb6L勒*\xb3w\x85\x1a\xf3\xce\x18\x82/=\x96\xdd\x19\xf4\x1e\xfd\xb3\x9d\xa3\xe4\rG:\xe3\n\xe5\xbe\xec.\xfa\x1e\xd2\xee\x94q\f\xef^\x1cݷx\xc6q\f\x87%\v\xa4T\xe4\xe7h\xe5\xd7ٲfڟL\xfc\xb3\xdc\xc1\xfbh\xf6\xacC\x9eۃ\xe6\x91\xf2\xac\x00\xd1\x1d\xba;X\xa5\xba\x02<\x90\xb7\xff\xea\x19\xf5V\xa1k\x7f\xa6\xea)\x89\xb2\xdb.\x88\b~\xe1\x84\xd9\xd0Y\xcc>\xe1\x01\xf0{rs\x8fs\xb4ڴy\x15\xf5s\xb4\x90*\v\x8b\xc1\x118\xfe\x83\xef\xcf_\x9a\xa6\x8dTt\x03?Iw\xc6\xf6\x18ۻ\xad;g\xaf\xfb\xa8'ԏ\x06\xa5\x89\x1d\xc0\xebO\xfb>\x00\xd6\xd4|\xf7\x0e5\xb6\xa3\x9cxL\xb31\xfc\x14\xbe\xdf\xdd\xfd\xe4\xb02\xac\x80\x8b\xf7\x95+w\xb06Q\x83%q\xc0\xd6AZ\xd9\xffn\xe5#\x1e\xfe\x1b\xcfc\x86;\x13\x1ad\x14`\xb19\x96 NB\xa9*\xb9\xa49\xa8k)\xd6l3\x82\xdd/\x9d\xc6\an6\xc3\x1f=r\xb5\x8f\n\xf0\xcf\\\x83`c\x1e\u0381\x7fd\x1c\xb4\x1bV\x82\x01\xbe\xe9\x7fU\xdb\xe3\xaaX\xb9\x18nm_\xd6\x1d\f\xf88\x87\x16\xa6\xa2KP6\x8arI\xebJ\aY\x1dF\xbc\xe1\b\x13\x066П\x05\x1e\xb1\xc0\xeeTit\x9f\xc1\x9c\xe0\\\xe6\xc7X~\xab\x83\xfc\xfd\xf0\x97\a\x9cl\xa5\xbcb'\xee\xb9 \xe4\xe6\xfeZ\x93J\xe4\x98.\xbe\xff\xcb\xed$\xa9\xdbuN\xae\x0f\xda:fT\xef\xe3_\xb5\x82㖽pѱ\\G\x10\x18\x82Ӻ\a\xe4\x91\x19\x7fp\xd7yOZ\x1d\x9a\xf2\f\xddp\x80G\xfa\x8f\xdfq\xe0N\xfe\xf77\xa3xu\xac\x14\x1e\x93\xeao\x05\xc0cEO\xba\xe6`U\x17l\xd5\xc5_\xfa\x9d1P\x94&\x16k\x8c\x9b\xc3\xef\x8f\x01\xac\xe34i(oi%\r\rb\x91\xb6ދ\xecXa\x99\xb7FG\xb8yL\x1fc\x04\xb8\xf6\xfb!\xceF\x80\x1a\xe0\x10\x01t\x95e\xa0\xf5\xba\xe2|_o\xc7\xf8J\xa8\xf1\x912~>R8h\x83\x82`\xd1;\ni\x14a_\xee\r\"\x0f\x9a\x1e\xb6*M#\x85炯\x86Ԇ\x16']\xd8p\xdd\a\x83W\xf6\xa8\xbcUTI\xeb\xb1Sݰ?\xe6\\\x1ap\xeeK\x9cdYh\x90\x13\u0601 \xd6;;\x12\x87;\xa7&B\xf1;\\\x9d\x87\v\xfe.\xa4B\xa2\x17\x13\x11\x9f\xed\xd0x\x01\xcew\xba\x86\x89\xb5\xa2x\x9fI\x9f\b\xfd\xe0\xd7e+\xael\xf4\x0f\v\v\u2d285j\x9b3ͺ~\xe1yF\xee\xfav9\x04\xee\x14\x13\u05ff\xee\xe5\x99j\xdcG\xf7Y&\xad\x8f\xee$\x83\x16\x81X\xcb\xf8\xf9qGU?\xedPw\xfc\xd2\x05\x1cY\xd8CG9\xf7\x1b\x1d\vКn\xc2i\xee\x8fv\xea\xb1\x01\x01.=\xe7\x16O\"@\x9b]qݳ̝\xca\xd0\xccT\xd4w\x10\n|[\xad\xbeӄ\xcb\x18T\xbcЅ\x85\x9b\xc2\u009cl\"\xa1\x9eJ\xa6R\xe6p\x1fꆖ6\x18\t#w\x9a\xbb݀\xb3\r\xb3s\x1d˹\rU+\xba\x81E&9\a\xb4\xd6\xfdq\xbd\xa4\xae\xfb\xbd\x87_\x80\xeaQ\xd4>\xb6\xdb\xfa\x15@\xc7m\xb7\xf0M]\xb9;\xde\xdee\x98\x82\xe6\"\xbdހ$v<)PvT\x88\xde2\xd7\x1fi\xbbm\xd0:o\x96}\x9e\xd7_27\xf7y\x81\xb8<\x16\xf4W\xa9\xe6\xa4`\xc2\xfeCE\xee\x16\xf0\xc2Ǔƿ\x95\xf2\xe16\x12\xc4\xf6\x06\xffCݰY\xea`\xc2\r\x1b7\x8c\xaed\xe5W\xdf\xeb\x806\xbe\xac\x82'\xf3\x9fy\xba\x890\x8f\xf8\x83\x1e:\x83\x19\xdd\x1f:\x90F]\x81\xeby\x00\xd6m\xb8Ɍ\xf3\xfd\xfc\x10\xf2\xc1\xad\x89\r\xec\xd6\xcd\x05>\fh\xce#\x18\xe8(\xacHE\x81\xd4\a_\xb4\r\xfa)\xb3^O\xe6\xa1`\xb2G\xe3\x1f\x9a\xd6Ctt\xc3l\x85{\x03\bv\x82\xc0\xf3N\xd8\xf1\x9a\x8a\x11\u1ff1m\xea\xb3\vZ\x13\xb7P%6\x98\xa5\x8b\xef}_\x90O\xd0_\xaeX\x90\xbfVPEh\xb0\b\x17\xc3\xdd\x1a\xaa\xfa)_\xb7\r\x1er\xac\xe8@m\x8c4Y\x8a\x1b%7\nt_X\x17\xe4o\x94\x19&6\x1f\xa5\xba\xe1Ն\x89\xcf\xc3[~\x8e5\xbe\xa1\xca0+\xecn<\xb1\x812A9\xfb{̮\xb5_\x8e\x03\xba\x1e\x9c`-H\xc20\x86^\xbc\a\x1b\xe3\x0e\xe6\x05\xa2&\xb4\xf4t=%^\t<\x19\xb3\xa9u,\xd1\xc4\"\xa1\xdb\v\xf2IF\r\x83/\x87b]\x986$\x03m\x16\xb0^Ke\xdcj\xf5bA\xd8:$\x1f\xac\xcd\xc1\xbc\x99\xbb\xab\x92\xb0\xd82s]hҸ/Lz+\xf4\xc2x\x94}A\xf7ne\x8afYe#\xacKm(\x8f\x048\xcf2\xfc\x98\xe5\xb1\xca\a\xf9/\xcfZ\xc9[\xb6\x01\xf5\x93\x8e؏#)\x1e\xa6\xe1\xa2>nQ\x04A\x1e\x153\xc6\xc6T\xf2H)\x81'\x95\xb1\xb1\x15\xe7D[R\x9f\x94}$Ό.\x87Kr\xd2P\xbe\xab\xa1\f\x99g\x8f5\xde̸B\xda\x10\x1b\xf7b\xf5\x91oeٜm\xa9\xd8\f\x9eP\xb0U\xb2\xdal\x83$\x0f\x04\xd3$\xaf\x00\x93\xb5hRt\xb8X\xd8TJ\xb4J\t\x8el\xfb&A\x18p\xb84{ U9\xf7\x17\xf7\xfa{\x99/\xfd\x1d(\x8b\xb5\x92\xc5\xc2\xf7\x8b\xb9Թ_\xc9WL\xda\xc8\xc5l\xa3T'.j\xf7\xd7\f\xa0$\x94%\bB\xb5\xef9ᤨ\x93\xdd\xd4o\xd65\xdcH\xcd\x12\xa2\xfd(\xc7\xff\xda\x06\x10\x18^\x86\xbf\xbb\xcc\xf03\x18\xec3\x86\xc7g\xbf\x05\x1fvT\x187\x9d\xa8]\xe4\xcc9\xb1٤\x89\x8c\xb6\x8e\xedYI\x9a\xdb\x0e\x84\x91\xfc\fv\x17gѭ/\xd7p\a\x81]\xfb\xebWk\xc0s\xa2\x99\b\x17_\xbb\xd2\x0f'\xfdѕ@\x81\x17UJ\x15\xaf\xc6<\x9ep\xe9\"\xf4\xba\xb9\x96]\x1dI|8y*~\x7f\x00\xe3`S7\xdeKZ7\t\xd3\xe7?\xb0\xd8z\x00\x96\xf1f\x16\x95?\xfe\ue6f5wIS\xbd8E\x8e\xcd\xfcpR7<\x85\xeb\xdeCz\xc3\xc1j\x9b\x06\xe8N*'\xe9\xdc\xee\x8cٴs\xa6\xd2\xc2\x15\xef\xe7\xc9%\xedΘD{\xb1\f\xdayQ~\xa4xA\xf4IZ\xfb7\xffm$\x85\xe6\xc1\x9e;\x89\xd6ʡ\x85\x81\xbfj\x16-\xeas{?\xa2\x9d\xce[\xd6\xc2\xf7\xe4\x7f\xf9\xff\x00\x00\x00\xff\xff<\x82OF\xb8\x82\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xccZK\x93۸\x11\xbe\xebWt\xed\x1e\xf6\xb2\x94줒J\xe96\x96\x93*W\xc6\xf1\xd4hvr]\bhJ\xb0@\x80\x01@\xc9\xca㿧\x1a\x0f\x89\xe2C\x0f;q\u008b-\x12h\xf4\xf3\xeb\x0f\xc0\x14E1a\xb5|E\xeb\xa4\xd1s`\xb5\xc4/\x1e5\xfdr\xd3\xed\x1f\xdcT\x9a\xd9\xee\xedd+\xb5\x98âq\xdeT\xcf\xe8Lc9\xbe\xc7Rj\xe9\xa5ѓ\n=\x13̳\xf9\x04\x80im<\xa3\u05ce~\x02p\xa3\xbd5J\xa1-֨\xa7\xdbf\x85\xabF*\x816\b\xcfK\xef\xdeL\xdf\xfe~\xfa\xbb\t\x80f\x15\xcea\xc5\xf8\xb6\xa9\x9d7\x96\xadQ\x19\x1eENw\xa8К\xa94\x13W#\xa7\x15\xd6\xd64\xf5\x1cN\x1f\xa2\x84\xb4z\xd4\xfc]\x10\xb6\x8c\xc2\x1e\x93\xb0\xf0]I\xe7\xff<>\xe6Q:\x1f\xc6ժ\xb1L\x8d\xa9\x15\x86\xb8\x8d\xb1\xfe/\xa7\xa5\vX9\x15\xbfH\xbdn\x14\xb3#\xd3'\x00\x8e\x9b\x1a\xe7\x10f\u05cc\xa3\x98\x00$\xd7\x04i\x050!\x82\xb3\x99z\xb2R{\xb4\v\xa3\x9aJ\x1f\xd7\x12踕\xb5\x0fΌ\xb6@2\x06\xb25\xe0<\xf3\x8d\x03\xd7\xf0\r0\a\x0f;&\x15[)\x9c\xfd\xa2Y\xfe\x7f\x90\a\xf0\xd9\x19\xfd\xc4\xfcf\x0e\xd38kZo\x98\xcb_c\x8c\x9eZo\xfc\x81\fp\xdeJ\xbd\x1eR\xe9\x919\xffʔ\x14A\x93\x17Y!H\a~\x83\xa0\x98\xf3\xe0\xe9\x05\xfd\x8a\x1e\x02r\x11B\xf6\x10\xec\x99K\xeb\x00좔\xe0\xa3aMUo\xad3\xb5I\x15x\xedH\x89\xfaӛ\xa4}Kl\xce\xef)\xb7x\x14\xe9<\xab\xea3\xb9\x0fk\x1c\x13v\xe6\x8a\xf7X\xb2F\xf9\xb6\xa9\x14%\xd5\xce\xcbs\xb3j\xe4S\x11g\x9d\xad\xf8\xfe\xec]\\ue\x8cB\x16\xa5\xc4Q\xbb\xb71\v\xf9\x06+6O\x83M\x8d\xfa\xe1\xe9\xc3\xebo\x97g\xafa(\x91:EA\x81c\xad\xd8l\xd0\"\xbc\x86\xfa\x8bqsɴ\xa3L\x00\xb3\xfa\x8cܟ\x82X[S\xa3\xf52\x17K|ZX\xd4z\xdb\xd1\xe9\x9f\xc5\xd97\x002#\xce\x02A\xa0\x841\xafR\xfd\xa0H\x96\x83)\xc1o\xa4\x03\x8b\xb5E\x87:\xc2\x14\xbdf:)8\xed\x88^\xa2%1Tۍ\x12\x84e;\xb4\x1e,r\xb3\xd6\xf2\xefG\xd9\x0e\xbcI\xc9\xec\xd1y\b\x15\xaa\x99\xa2dm\xf0g`Zt$W\xec\x00\x16iMhtK^\x98\xe0\xbaz|\xa4j\x90\xba4s\xd8x_\xbb\xf9l\xb6\x96>#47U\xd5h\xe9\x0f\xb3\x00\xb6r\xd5xc\xddL\xe0\x0e\xd5\xcc\xc9u\xc1,\xdfH\x8f\xdc7\x16g\xac\x96E0DGH\xadď6a\xba;[\xb6W\xd2\xf1\t\x90zGx\b^c\xcaDQ\xd1\xc4S\x14\xe8\x15\xb9\xee\xf9\x8f\xcb\x17Ț\xc4HŠ\x9c\x86\xf6\xfc\x92\xe3Cޔ\xbaD\x1b\xe7\x95\xd6TA&jQ\x1b\xa9}\xf8\xc1\x95D\xed\xc15\xabJzJ\x83\xbf5\xe8<\x85\xae+v\x11\xba\x18\xac\x10\x9a:\x80Dw\xc0\a\r\vV\xa1Z0\x87\xdf9V\x14\x15WP\x10n\x8aV\xbb7w\aG\xf7\xb6>\xe4\x9e:\x12\xdaA4X\xd6\xc8\xcf\xeaN\xa0\x93\x96*\xc33\x8f\xa1\xba:\x0eJP1ޔ\xf33\f\x12\xf40\xceѹ\x8fF`\xf7KG\xe5\x87\xe3\xc03\x1dk\xb4\x95t\xa1\xbdBil\xb7\xf3\xb0#\x92\xb7\x9f\x8cx݀\x03\xa0n\xaa\xbe\"\x05<#\x13\x9f\xb4:\x8c|\xfa\xab\x95\xbe\xbf\xd0H \xe9\x89*.\x0f\x9a?\xa1\x95F\\1\xfe]g\xf8\xd1\x05\x1b\xb3\x872\xe4\xbf\xf6\xea@\xd8\xe5\x0e\x9a\xf7Q;?\x0fO\x1f2\x82\xc7\xdaJ\x85\x99|5\x85\x87TԦ\x847 \xa4#\"\xe1\x82о\xb3t\xa3\x02ј\x83\xb7\xcd]\xe6s\xa3K\xb9\xee\x1b\xdd\xe6Fc\x19sEt\xc7s\x8b\xb0\x12\xa1\x16eGm\xcdN\n\xb4\x05Շ,%O\x9a46v\x90R\xa2\x12=l\x1a\xad\xb2`\x8aEAE\xcdԕ\x18.\x8e\x03\x03\x93fR\xc7\f>\t\bXc\xabԚ\xb5G-\xb0\xdbm\x826&\x00\x9aC\x01{\xe97\x11)\xd5P\xdd\xc1\xc5ڣg\x8b\x87\xa1\xd7\x1d\xdd_6H#c\xe3Ep\xc8-\xfa\x90m\xa8(}(\x95\xa6\x00\x1f\x1b\x17\xb0\xb6\x8b\x13\xf9\t\x84/\xcf\xde\xe2\xa1\xefh\xb8\x16\xdcD\x85FT\x0e$j\x0e?\xfcpݤ^w\xcb\x0fQ\xf7l\xa8\xc5\x12-\xea\x1e\x9b\xc8\xcfK\xe8Q\x944\x94aX\x96Ƚܡ:\x84\x9eD\xe0\xf93\xac\x1a\x0f\xa2\xc1\x105Ʒ{f\x85\x03n\xaa\x9ay\xb9\x92J\xfa\x03H7\"\x9f)e\xf6(Rı\xaa\xfda\n\x1f\xb4\xf3LstG\x1eD\x1e\x8b\xa9\xc0t\x1c\x95\xaa8\x10:f\x8700\x8a\xaf\x8c\xf3\xc0\xd1R:\xaa\x03\xec\xad\xd1\xeb1c\a\xda!\xed\x01\xadF\x8f\xa1#\n\xc3\x1d5C\x8e\xb5w3\xb3C\xbb\x93\xb8\x9f\xed\x8d\xddJ\xbd.H\xc1\"\x81\xcf,\xec\xecf?\x86\x7f\xbe&\vL\x1dq\xe2\x86\xe4]\x86Z?\x10\xbd\xf5\x1b\x8c-b\x19s\xd0X \x02A\xa9]\xa5܍\xc8:TvC\xbc\xbc\xfd\xe4\x90\x0f\xf5\x8f-\xf6[\xc7\x05P\x01\xf8R\x9c|[T\xac.\xe2h\xe6M%\xf9\xa4km\xcc\xfb\xcb\xf8\x937+R\vɉܞ\xe3F\xdeĉ\xb3=̀\x1b\xba\xbb\x9c1\xb4\x1cvS47q\x85+\x1a\x7fj\x8f=m}#t\xa7\xfe\xef\xd0\x13\xeft\xa0\x91\xf8\x01\xb3}?\a\xc0\xe4FkB*o\x80\x1d\xdb\xc0O\xae\xdb\xff\xeeD\xcfU÷8\xe0\xf8\x9e)\xef\xc2\xc0\xec\xe38\x8dti\x1c\x86\xc6tM\r\xb8^\x11\x9c-\xd0ޢ\xcb(\xf2-\x1eH\u0091[0X<\xc0\xaa\xd1BaVu\xbfAM\xdb1Y\x1e\x88\xec\xbf<.\xb3c\x03\x01K[\xa7\xec\xde1 yO\xbb\x00JA1\x87_\x1c\xa6u\x9f\xb1\x04\xa9\x9dG\xd6#\xe9\xf1\x89\xbdq\x0e\xab\xc3\x00\u05fa\xd9A\xcfX~\xbb\x8f\x82\xae\xe4\xa1\xd4 8\xc6\xc4J\xb0\x92\xfa{\xde\x0f-\x1e\x02\xc4\x12\xdf \"}\xe6ґe\xeetth\xd0i\xf1\fdR\xc7\x02a\xd5\xd8\":\x1fR\x01\x8by9H䇃q\xb9.\xe0\x12\xb3\xe89\xfb>v1*\x13\x80\xdd\xc80\xe0z\xb2\xc0E\xa6\x017\xb0\x8d\x9e\x99\xa39\x05w\xb2\x0e\xf8\x0e\xcc\x03\xfe\xfb\xec\x03\xeef \xf0\xddY\bܖ)\x97\xd9\b|\x13#\xb9\xe0\x8bK\\\x05\xae\xf2\x15\xb8\xc8Y`\x94\xb7\xc05\xee\x02w\xf2\x17\bx\x82\xa5\xfcr\x032?\x85\x81\xb9\x93\xd6\xcco\xa8kH\x81\xc0\x06\xfaj<\xa1\x18q\xd0q\xd3\xfb)\x85\xef+\xfa\xee%\xd2\x17չ\x87\xf7e@\xbfB\x8c\x9eҰ\xa3\x17\xf2\xef\x04 \xe7\a c\x04mТ\xdd\xf1\xb4\xfdO\xf1X\x81\x0f\xa0\xf8\x992\xaf\xfd\x19\x17\x8e'\xf2\x99\xff\x10K\xa3Ͱ\xb1\x16]m\xb4\xa0\xb6w\xdb\xe1\xc4I\xe5\xff\xdc\x11\xc5pX\x8bs\xfa\xda\xf9\x96\xa3p\xd3\xf9\\\xb8߸\xfb\x84.\xde\xfa\xb4Ͽ\xccʡݵ\x0e\xe9:6~\x97\xb3\xb9\xc1\xce\xd6:\xb0#\xaa\xa4\xa1\xd1\xe1\xc8\"4\xad\xe9d`F\x9b\x17\xfa\xd0<\xa4\x03m\xf64\xb9%-v=\x13\xe9M8\xb4dZ\xa4\xe3b\xfa4 y/\x95\xa2\x1ef\xb12\xe4,\xd4^Zj\x96,\xb4\xb1\xddo\xa6o\xfewg\x81\x8a9\xbf\xc6Qq\xff\x99\xa6\x00[\x99\xc6\x0f\xf4\xfeV\xc2\x0f\xd6t\xb8e\xbfG\xc7\xf0\xb7\x03\xd7\xe8\t\x8d\xc9\x11፵\xe1\xb2._\"ݱ\xd1\x1cC\xe0\x87Ο8\xb4\xbf\xf5\xff\x00\xe2\x06\xbb\x06\xbbt\xefe촭\xb8&'\xb7\xdf4\xab\xe3\x15\xec\x1c\xfe\xf1\xafɿ\x03\x00\x00\xff\xff%\xff\\)\x99#\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcVMo\x1b7\x10\xbd\xebW\f\xd0kwU\xa3hQ\xec\xadqr0\xda\x06\x82\x1d\xe4N\x91#-c.\xc9\xce\f\xe5\xba\x1f\xff\xbd \xb9+K\xab\x95\x93\\\xb27\x91Ù\xc7\xf7f\x1e\xd54\xcdJE\xfb\x11\x89m\xf0\x1d\xa8h\xf1/A\x9f\x7fq\xfb\xf8\v\xb76\xac\x0f7\xabG\xebM\a\xb7\x89%\f\xf7\xc8!\x91Ʒ\xb8\xb3ފ\r~5\xa0(\xa3Du+\x00\xe5}\x10\x95\x979\xff\x04\xd0\xc1\v\x05琚=\xfa\xf61mq\x9b\xac3H%\xf9T\xfa\xf0C{\xf3s\xfb\xd3\n\xc0\xab\x01;0\xe8Pp\xab\xf4c\x8a\x84\x7f&d\xe1\xf6\x80\x0e)\xb46\xac8\xa2\xce\xf9\xf7\x14R\xec\xe0e\xa3\x9e\x1fkW\xdcoK\xaa7%\xd5}MUv\x9de\xf9\xedZ\xc4\xefv\x8c\x8a.\x91rˀJ\x00[\xbfON\xd1b\xc8\n\x80u\x88\xd8\xc1\xfb\f+*\x8df\x050^\xbb\xc0l@\x19S\x88TnC\xd6\v\xd2mpi\x98\bl\xc0 k\xb2Q\nQ\x1fz,W\x84\xb0\x03\xe9\x11j9\x90\x00[\x1c\x11\x98r\x0e\xe0\x13\a\xbfQ\xd2w\xd0f\xbe\xda\x1a\x9a\x81\x8c\x01\x95\xea7\xf3ey\u0380Y\xc8\xfa\xfd5\b,J\x12O J]\x1b<\xd0\t\xbf\xe7\x00J|\x1b{\xc5\xe7\xd5\x1f\xcaƵ\xca5\xe6pS\x99\xd6=\x0e\xaa\x1bcCD\xff\xeb\xe6\xee\xe3\x8f\x0fg\xcbp\x8euAZ\xb0\fjB\x9a\x89\xab\xacA\xf0\b\x81`\b4\xb1\xca\xed1i\xa4\x10\x91\xc4N\xadU\xbf\x93\xe19Y\x9dA\xf8\xb79\xdb\x03Ȩ\xeb)0y\x8a\x90\v\x89cS\xa0\x19/Zɵ\f\x84\x91\x90\xd1\u05f9\xca\xcb\xcaC\xd8~B-\xed,\xf5\x03RN\x03܇\xe4L\x1e\xbe\x03\x92\x00\xa1\x0e{o\xff>\xe6\xe6|\xef\\\xd4))\x94\xe4\xb6\xf3\xca\xc1A\xb9\x84߃\xf2f\x96yP\xcf@\x98kB\xf2'\xf9\xca\x01\x9e\xe3\xf8#\x93h\xfd.tЋD\xee\xd6뽕\xc9Rt\x18\x86\xe4\xad<\xaf\x8b;\xd8m\x92@\xbc6x@\xb7f\xbbo\x14\xe9\xde\njI\x84k\x15mS.⋭\xb4\x83\xf9\x8eF\x13Ⳳ\x17\xddS\xbf\xe2\x02_!O\xf6\x84\xda#5U\xbd\xe2\x8b\ny)Sw\xff\xee\xe1\x03LH\xaaRU\x94\x97\xd0\v^&}2\x9b\xd6\xef\x90\xea\xb9\x1d\x85\xa1\xe4Dob\xb0^\xca\x0f\xed,z\x01N\xdb\xc1\nO\x1d\x9b\xa5\x9b\xa7\xbd-\xb6\x9b\x1d E\xa3\x04\xcd<\xe0\xceí\x1a\xd0\xdd*\xc6o\xacUV\x85\x9b,\xc2\x17\xa9u\xfa\x98̃+\xbd'\x1b\xd33pEڅ\xe1\x7f\x88\xa8\xb3\xb8\x99\xdf|\xda\ueb2ec\xb5\v\x04O\xbd\xd5\xfd4\xfc3\x9a\x8eFq\xce߲1\xe4\xef\xc5n\xe7;W/\x0fEdK8k\xd8\x06.\xbc\xfbu^\x8a\xa9~%3\xd5\xd1Gnt\"*\xcdw\xf4y\xb5t\xe8K\xb9@\xa2@\x17\xab3P\xefJP\xf9Ǡ\xacgP\xfey<\b\xd2+\x81'\xa4\r\x97\x95\x1ax\x8fO\v\xabw~CaO\xc8\xf3\x96ϛ\x9b\xca\x1e\xce߃WXZlʋE\xceVhNXd\t\xa4\xf6\xa7\xbcr\xda\x1e\x9d\xbe\x83\x7f\xfe[\xfd\x1f\x00\x00\xff\xff\xbeM\x1a\xea\xb1\n\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcWMo\xe36\x10\xbd\xfbW\f\xd0K\v\xac\xe4\x06E\x8b·\xd6\xd9C\xb0\xe96\x88\xb7\xb9S\xd4HbC\x91,9t6E\x7f|1\xa4\xe4\x0fYv\x9c\xcb\xea\xe6\xe1p\xf8\xe6\xcd\xcc#]\x14\xc5B8\xf5\x84>(kV \x9c¯\x84\x86\x7f\x85\xf2\xf9\xd7P*\xbb\xdc\xde,\x9e\x95\xa9W\xb0\x8e\x81l\xff\x88\xc1F/\xf1\x16\x1be\x14)k\x16=\x92\xa8\x05\x89\xd5\x02@\x18cI\xb09\xf0O\x00i\ry\xab5\xfa\xa2ES>\xc7\n\xab\xa8t\x8d>\x05\x1f\x8f\xde\xfeX\xde\xfcR\xfe\xbc\x000\xa2\xc7\x15\xd4\xf6\xc5h+j\x8f\xffD\f\x14\xca-j\xf4\xb6Tv\x11\x1cJ\x8e\xddz\x1b\xdd\n\xf6\vy\xefpn\xc6|;\x84y\xccaҊV\x81>ͭޫ\xc1\xc3\xe9\xe8\x85>\x05\x91\x16\x832m\xd4\u009f,/\x00\x82\xb4\x0eW\xf0\x99a8!\xb1^\x00\f)&XŐ\xdd\xf6&\x87\x92\x1d\xf6\"\xe3\x05\xb0\x0e\xcdo\x0fwO?m\x8e\xcc\x005\x06镣D\xd4\x7f\xc5\xce\x0e\xd3\x04@\x05\x100\xc0\x01\xb2;\x84 \f\bO\xaa\x11\x92\xa0\xf1\xb6\x87J\xc8\xe7\xe8\xc0V\x7f\xa3$\bd\xbdh\xf1\x03\x84(;\x10\x1c%;\x1c\x9c\xa5m\v\x8d\xd2X\xeel\xce[\x87\x9e\xd4Hy\xfe\x0e\x1a\xea\xc0z)\v\xfe8\xf1\xbc\vj\xee,\f@\x1d\x8e\xe4a=p\x05\xb6\x01\xeaT\x00\x8f\xcec@\x93{\x8d\xcd\xc2\fٔ\x93\xd0\x1b\xf4\x1c\x06Bg\xa3\xae\xb9!\xb7\xe8\t\x1aE\xaf\xcb41\xaa\x8ad}XָE\xbd\f\xaa-\x84\x97\x9d\"\x94\x14=.\x85SEJĤQ+\xfb\xfa;?\ff8:\x96^\xb9!\x03yeڃ\x854\x1d\xef(\x0f\xcfK\xee\xae\x1c*\xa7\xb8\xaf\x02\x9b\x98\xbaǏ\x9b/0\"ɕ\x1aZl\xe7z\xc2\xcbX\x1ffS\x99\x06}ޗڔc\xa2\xa9\x9dU\x86\xd2\x0f\xa9\x15\x1a\x82\x10\xab^Q\x18{\x9dK7\r\xbbNR\x04\x15Bt\xb5 \xac\xa7\x0ew\x06֢G\xbd\x16\x01\xbfq\xad\xb8*\xa1\xe0\"\\U\xadC\x81\x9d:gz\x0f\x16Fy&j^\x01\x128\xe1[\xa4\xa9u\x82\xe5Kr\xe2\xe3_:q,X\xdfcٖ\xac9a\x00\x92\xf5\xe8\x87i\xa1.a\x80\xd9F\x9fE2\xf67\xd3\xc0\xbc\xb2\xa0\xb0\xd8\x1db:=\x9a?4\xb1\x9f?\xa0\x80\xdf\x13\xe6{\xdb^\\_[C<\x17\x17\x9d\x9e\xac\x8e=n\x8cp\xa1\xb3o\xf8\xde\x11\xf6\x7f:\xf4\xf9\x1a\xbe\xe8:\xde滫\xef\x82c\xd4g\xcf}D\xbeA\xf0|\xa6\x83\xc3UQ\xae\xc04x^\x95\xe8zs\xf7\x1e\nϸ\xbf\xa3Hw\xa6\xb1o\xa4\xb8w\x9c\xf5;#\x03\xe3\x97\xde\x10o\xf74\xbfBƞ\xe6-\xf9\xeeD\xf8\x14+\xf4\x06\t\xc3^\xa9_\x14u\xb3\x11\x01^:%\xbb\xb41\r\x04_\x02!X\xa9\xe6$\xf5\n\xf8\xac#\xca\xe3\xccP\x16iXg\xcc\f\xfe\xc4|F\xfd\xce\x1dP\f\x8at\x95\x82\x92\xa0\x18ޡ\xa1\xc9\x7f\xa4ZF\xef\xd3\x15\x95\xad\xfc2\x99n\xb8VDG\xe5\xf9\xeb\xf1\xfe\r%\xbd\xdd{\xa6\x17\xb7P&\xa3q\x1e\x8b\xa0Z~A\xf1\x1akiҸS2\xf2w\xfc\xc2;&j\xb6\xa2\xf8թ<\x80o@\xfc\xb8ŝ\x8f&\xdf\xf3\xd37l\n\x88\x81\x9f[ \x85\x99\xc1X!Ԩ\x91\xb0\x86\xea5\xdf\\\xaf\x81\xb0?\xc5\xddX\xdf\vZ\x01\xdf\xff\x05\xa9\x9962QkQi\\\x01\xf9x\xae\xcbf\x13w\x9d\b3cx\x94\xf3\x03\xfb\xcc5\xc6n\x18/v\x06\x9c\xbd_\n\xf8\x8c/3\xd6\ao%\x86\x80\xa7ct6\x93\xd9!81\x06~\xa4\xd5\a,\r\x7f\x19\x06\xcb\xff\x01\x00\x00\xff\xffx\xae@\xbaJ\x0e\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xc4:Ks\x1b7\xd2w\xfd\x8a.吤\xca$\xe3|ߦ\xb6x\xb3\xe5͖v\x13\xafʔ}I\xe5\xd0\x1c49\x88f\x00,\x80\x11\xcd\xcd\xe6\xbfo5\x80\xe1\xbc@R\xa2\x93\x18\x17\x89x4\xfa\xfd\xc2\xccf\xb3+4\xf2\x03Y'\xb5Z\x02\x1aI\x1f=)\xfe\xe5\xe6\x0f\x7fus\xa9\x17\x8f/\xaf\x1e\xa4\x12K\xb8i\x9c\xd7\xf5;r\xba\xb1\x05\xbd\xa1\x8dT\xd2K\xad\xaej\xf2(\xd0\xe3\xf2\n\x00\x95\xd2\x1ey\xda\xf1O\x80B+ouU\x91\x9dmI\xcd\x1f\x9a5\xad\x1bY\t\xb2\x01x{\xf5\xe37\xf3\x97\xdf\xcd\xffr\x05\xa0\xb0\xa6%\x18-\x1eu\xd5Դ\xc6\xe2\xa11n\xfeH\x15Y=\x97\xfa\xca\x19*\x18\xf6\xd6\xea\xc6,\xa1[\x88gӽ\x11\xe7;->\x040\xaf\x03\x98\xb0RI\xe7\xff\x99[\xfdA:\x1fv\x98\xaa\xb1XM\x91\b\x8bN\xaamS\xa1\x9d,_\x01\xb8B\x1bZ\xc2[F\xc3`A\xe2\n \x91\x18К\x01\n\x11\x98\x86՝\x95ʓ\xbda\b-\xb3f \xc8\x15V\x1a\x1f\x982\xc2\x0f\x9cG\xdf8pMQ\x02:xK\xbbŭ\xba\xb3zk\xc9E\xe4\x00~qZݡ/\x970\x8f\xdb\xe7\xa6DGi52w\x15\x16Ҕ\xdf3\xca\xce[\xa9\xb69$\xeeeM \x1a\x1b\x84\xca\xd4\x17\x04\xbe\x94n\x82\xdd\x0e\x1dch} ;\x8fKXg\x88\xcecm\xc6H\xf5\x8eF\xac\x04z\xca\xe1t\xa3kS\x91'\x01뽧\x96\x92\x8d\xb65\xfa%H\xe5\xbf\xfb\xff\xe3\xecH\xfc\x9a\x87\xa3o\xb4\x1a\xf2\xe65\xcfBo:b²ڒ\xcd2H{\xac>\x05\x11\xcf\x00^\xf7\xceGL\"\xdc\xfe\xfcYTnUa\xa9&u\x19B\xb2;=Ŧ\x0f\xba\xbfj\xac\xd4V\xfa\xfd\x12^~\xf3T4\xd9>@o\xc0\x97\x04IyV^[\xdc\x12\xfc\xa0\x8b\xa8h\xbb\x92lR\xb4u\xd2\xfeR7\x95\x80u+\x18\x00\xe7\xb5\xcd*\x9b\xa1b\x1eO%\xb8-ؑ\xc6\r\xef\xfc#\f\xa2\xb0\x84Y\x83h\x9d\xe6<\xec\x90Z\xe5\xad\xe2Ֆ\x9ed\x11}\x96*-\xe8\xc0?\x9a\xa0%\x1d\x18\xab\vr\ue1212\x8c\x01\"o\xbb\x89\xb3\f*)\xeci\xf1iL\xa5Q\x90\x05\xaf\xa1D%*b2\x10\xbcE\xe56IE\xa6\x02l\x8f\xdd\xef\xcd\x10\x95\xf7i\xe1\x18:q\xd7\xe3\xcb讋\x92j\\\xa6\xbdڐzuw\xfb\xe1\xffV\x83iVcm\xc8zن\x8f8z\xc1\xb17\vCr\xff;\x1b\xac\x01\xf0\x05\xf1\x14\b\x8e\x92\xe4\x02\x1bR \x91p\x8a\xec\x91\x0e,\x19K\x8eM+h\x94\xde\x00*\xd0\xeb_\xa8\xf0\xf3\x11\xe8\x15Y\x06\xd3\xdaB\xa1\xd5#Y\x0f\x96\n\xbdU\xf2?\a؎y͗V\xe8\xc9\xf9`\x8cVa\x05\x8fX5\xf4\x02P\x89\x11\xe4\x1a\xf7`\x89\xef\x84F\xf5\xe0\x85\x03n\x8cǏ\xda\x12H\xb5\xd1K(\xbd7n\xb9Xl\xa5oS\x86B\xd7u\xa3\xa4\xdf/B\xf4\x97\xeb\xc6k\xeb\x16\x82\x1e\xa9Z8\xb9\x9d\xa1-J\xe9\xa9\xf0\x8d\xa5\x05\x1a9\v\x84\xa8\x906\xcck\xf1\x85MI\x86\x1b\\;\x11t\x1c!\xd2?C<\x1c\xfb\xd9\b0\x81\x8a$vR\xe0)fݻ\xbf\xad\xee\xa1\xc5$J*\n\xa5\xdb:\xe1K+\x1f\xe6\xa6T\x1b\xd6y>\xb7\xb1\xba\x0e0I\t\xa3\xa5\xf2\xe1GQIR\x1e\\\xb3\xae\xa5g5\xf8wCγ\xe8\xc6`oBZ\x05k\xb6%\xf6\x00b\xbc\xe1V\xc1\r\xd6Tݠ\xa3?YV,\x157c!\x89wg\xf9\xc3c#\xa9\x12!s8\x7fwVsy\xdcn\"\x12!\"x\r\bFRA\x83h\fR9O(\xd2$;AKi\xedE\xf4\xf4G\x91\xe4\xd1Em\x96\t G\x1e)\xe0\x1f\xab\x7f\xbd]\xfc]G:\x00\vN\xcdB\xad\x17\xf2\xed\x17\x87zO\x90\x93\x96\x04Wo4\xafQ\xc9\r9?O\xd0Ⱥ\x9f\xbe\xfd9\xcf?\x80\xef\xb5\x05\xfa\x88\\5\xbd\x00\x19y~\bf\xad\xdaH\x17\t?@\x84\x9d\xf4e@\xd4h\x91\b\xdc\x05\x12<>\xb0%G\x12\x1a\x82J>d\xec'\x8e\xeb\x90\xcduh\xfe\xca\xd6\xf3\xdb5|\x15\x9d\xd75\xff\xbc\x8eh\x1cҖ\xbe\x81u\xe8D+\xb3r\xbb\xa5.\xef\x9f(\v\x87Y\x0eP_\x83\xb6L\xab\xd2=\x10\x010\xcb)\xc6\a\x12\x13\xf4~\xfa\xf6\xe7k\xf8jȃ#WI%\xe8#|\xcb\xde'\xf0\xc6h\xf1\xf5\x1c\xee\x83\x1e\xec\x95Ǐ|SQjG\n\xb4\xaa\xf61\x01~$p\xba&\xd8QU\xcdb\x82(`\x87{Л#\xf7\xb4\"b\xd5D0h\xfd\xc9$1\xf1\xe1\xb4\xd1L\xb3\xa6v<\xcd^B\x16\xf5$\xeb\xfdl\x19\xc8\x139\x11ʅO\xe0D\xbf\xf4\xba\x80\x13\x0f͚\xac\"O\x81\x19B\x17\x8e\xf9P\x90\xf1n\xa1\x1f\xc9>J\xda-v\xda>H\xb5\x9d\xb12\u03a2\xd4\xdd\"t\xbb\x16_\x84?\x97\x12\x1e\xdaT\x9fJ}\x00\xf2\xf9X\xc0\xb7\xbb\xc5%\x1ch\xb3\xfb\xa7Ǯ\xa3|X\xa5\x84s\f\x93m~Wʢlk\xbd\x9e\xb7\xadQDw\x8cj\xff\x99l\x87\xf9\xdcX\xc6h?K\xad\xda\x19*\xc1\xff;\xe9<\xcf_\xc2\xd8F~\x92sy\x7f\xfb\xe6sZT#/\xf1$Gj\x988>\xce:\xacf5\x9aY܍^ײ\x18\xed\xe6\x1c\xfeV\xb0\x906\x92\xec\x99\xf4\xef\xdd`s\x9b\xa0f\xaa\x81Þg\xe5\x9f\x1e\xb7\x99\x84\xaf\xdf\xc5>\x95\x16\x9e\xe4\xd7yU\xb8ǭ\x03\xb4\x04\b5\x1aֈ\a\xda\xcfb\xc6aPr\xba\xc0\x19\xc1\xa11\bhL\xc51=f\x11\x19\x88)\xffM\xecA\x17\xe8;Ɛ\xac(ۮԊ\xbc\x97\xea32\xe7\xfd\b\x91ߗQ\x87\x9e]\xa1\xd5FnS\xb7s\xca)\xd5T\x15\xae+Z\x82\xb7ͱ\x9a\xeb$#\xefy\xcbi\xfa\xdf\xf7\xb6\xb6\x1a~\xa6\xc1\x98\xa7j\xd0v\x9c\x12C\xaa\xa9\xa7\xa8\xcc\xe0A\x1b\x89\x99yK\xceO\xac\x97\x17\xae\xaf\x9fccQ)/)\xb9c\x19\x9c\xabJ\x93\xa2\xa7\x04\xbe\xadL\xbd\ueabc\xacП\xe1\x1b\xb8\xba\xe7rd\x88\xf7,\xdf.\x19\xed\xe9u\x97\xdb)\xa3\xc5hf\xe8\x06G\x8b\x91\xbe'\xf5\x90BC\xfb\x19]\xa4\xf8Ȗx\x1a\x83\xa3o\x9f\xde8\xed\xbe\xb4\x8fą\x9d\xf1$\x0e\x8d\xfeK$\xfej\f$\xf4~\xadHF!k:\x94\xfeC_\x17\x8b\xbb5\x81\xb1d0\xdb\x15\x82йw\xa1\x85\xf9\xa5\x8b\xc0\xa4\x83Ƒ\b\x1d\xb4\xc9\xdd\x13\b\xed;\x93@O3>\x7f\x99\xbf\xc87\xa6\xe2\x9b_\xff\xa5\xe4\xa2.\xd5\x14̔\x85\xd8r-<ᴏ\x8d9\x8eu\xe0\x0e\xfc\x8a\xd0H\x84*\x94\x8b\xe4\rʊ\x04\xb4/\xd9τ\xb2\xa6\r\xa78\xd1ǵ}\x9c\x84\xde\xf1\xfa\xef\xb4$3L\x98&<\x7f\xa40\xc7O\x8dg$y;\xda\x0e\xa5\xae\x92\xbcTS\xafɲa\x86\aOP\xb4㺿(Qm\xb3N\xae}\xb0#\xa8\xd0yXw\x1f\x06\xe4\x88\uffd8\x8e)\xeb\xbfpv\xa3&\xe7p{Ν\xff\x18w\xc5\xce]:\x02\xb8֍\xcf\xdb\xef\x97.\xb9\xa0\xe7u\x0f\xb3M\xb1\xa1\xf7C_\xb6\xcen\xd3TU8ӏ\x1b\xdd\a\x1c\x01\xab5\xe53\xfe\x13\xad\xc3S\b\x96\xe8α\xea\x8e\xf7\xe4\xfc\xf1!؝t\xc8p\"\xb0\xbf\xa5]f\xb6\xf5s\x99\xa5\xbb\xe4<3K\x93/1\xfa\x8b\xb17\x9e\xe3\\\xbb\x96\x85y\xf8\xce!\xb3\xf6}\xf0*\xcfbv\xc2\xef\x12\xb7y\xe8\xadw\x96\x17>[\x98\xd8\xdf0\xff@%\xfab\xcb5!\xba\xf3\xad\x06EH\xa9\x91\x96\x9e\x04\x82\xeb\xf2\x1a\x84t\xa6\xc2\xfd\x81\x96P\xfa\xb1\xa9\xe6\xdfG:\x8bj=\xa6\xa1c\xa9\xec\xe9\x0e\xf7\xe1k\x91|]{\xda_\xc0\x19\x9f\x11\xd6\xf5qg\xf8{\xdcp\"\x15w\n\x8d+\xb5\xbf}sF5V\x87\x8d\xad=vee\b,\xe1\xe9-mJ\xaa\x90A\xb5\xf3n\xcfr\x16Ï\x87.\xd1\xe2\xd5\x00\u0099\xb8\x9f\xbee\xcaE\xd7\x15{\x01v@\xe1a\xf7f\xfc\x05NjC\x90A\x9f\x1a\xe41\x1e\xe5\xba\nZ\x85:B\xdb\xe9+;\x9c\r\xe4C\x82\xfe\xcc\x18\x9eU\xa7\xc9d\xc0\\\xf4`\xa77\xcd\xfeL\xb3><\xf7/\xe1\xd7߮\xfe\x17\x00\x00\xff\xfff=C\x19\x96(\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xc4Z͒\x1b\xb7\x11\xbe\xefSt\xad\x0f\xb6\xab4d\xa4$\xae\x14o\xd2*Nmbo\xb6DI\x17\x97\x0f\xe0\xa09\x03s\x06\x80\x01\f\xb9\xb4\xe3wO5\x80\x19\xce\x0fH.\xa9\x925\x17i\xf1\xd3\xf8\xf0u\xa3\xbb\xd1`\x96e7L\x8b\x8fh\xacPr\x01L\v|r(\xe9/;\xdb\xfc\xc3΄\x9ao_\xdel\x84\xe4\v\xb8k\xacS\xf5;\xb4\xaa19\xbeŵ\x90\xc2\t%ojt\x8c3\xc7\x167\x00LJ\xe5\x185[\xfa\x13 W\xd2\x19UUh\xb2\x02\xe5lӬpՈ\x8a\xa3\xf1\xc2ۥ\xb7\x7f\x99\xbd\xfcn\xf6\xf7\x1b\x00\xc9j\\\x80V|\xab\xaa\xa6F\x83\xd6)\x83v\xb6\xc5\n\x8d\x9a\tuc5\xe6$\xbc0\xaa\xd1\v8t\x84\xc9q\xe1\x00\xfaQ\xf1\x8f^λ \xc7wUº\xff$\xbb\x7f\x10\xd6\xf9!\xbaj\f\xab\x128|\xaf\x15\xb2h*f\xa6\xfd7\x006W\x1a\x17\xf0@P4ˑ\xdf\x00\xc4}zh\x190\xce=s\xacz4B:4w$\xa2e,\x03\x8e67B;\xcf\xcc\x18\"X\xc7\\c\xc16y\t\xcc\xc2\x03\xee\xe6\xf7\xf2Ѩ\u00a0\r\xf0\x00~\xb1J>2W.`\x16\x86\xcft\xc9,\xc6\xde@\xf1\xd2w\xc4&\xb7'\xcc\xd6\x19!\x8b\x14\x8a\xf7\xa2F\xe0\x8d\xf1\xaa\xa5\xfd\xe7\b\xae\x14v\no\xc7,A4\xceo<\r\xc6\xf7\x93H\xebX\xadǨzS\x03,\xce\x1c\xa6@ݩZW\xe8\x90\xc3j\xef\xb0\xdd\xcaZ\x99\x9a\xb9\x05\b\xe9\xbe\xfb\xdbq>\"a3?\xf5\xad\x92Cr\xdeP+\xf4\x9a\x03\x12\xd2V\x81&ɐr\xac\xfa\x14 \x8e\x04\xbc\xe9\xcd\x0fH\x82\xdc~\xfbY(dz\xa0\xd6\xe0J\x847,\xdf4\x1a\x96N\x19V \xfc\xa0\xf2\xa0\xc2]\x89\x06\xfd\x88U\x18A'\x18\x04\xe9N\x99\xa4\xea4\xe6\xb306\nke\x8d\xf47\\\xe8\xb3\xd8Wn\x90%\xed\xabuE3?B(\x996\xb2\xd7\x05>\xcb\xc0\xfaDJű\xc7\xda\x04\x97\xb0\xa0\x8d\xca\xd1\xda\x13\x86OB\x06H\x1e\x0e\rg)*яi\x015\xbaR\x8c\xa3\x01\xa7\xa0d\x92W\x18t\xe8\f\x93v\x1d-c\xaa\xc2v\xda\xfb\xbd\x1eB\xf9\xd0\xca\xeb\xf5L0\x85\xa1ۗ\xc1\r\xe6%\xd6l\x11\xc7*\x8d\xf2\xf5\xe3\xfdǿ.\a\xcd@\xb4h4N\xb4\x9e9|\xbd\xc0\xd3k\x85\xe1\x9e\xff\x97\r\xfa\x00h\x810\v8E \xb4\x9e\x8b\xe8_\x91GL\x81#a\xc1\xa06hQ\x86\x98D\xcdL\x82Z\xfd\x82\xb9\x9b\x8dD/ѐ\x18\xb0\xa5j*N\x81k\x8bƁ\xc1\\\x15R\xfc\xd6ɶD8-Z1\x87\xd6\xf9\x83h$\xab`˪\x06_\x00\x93|$\xb9f{0HkB#{\xf2\xfc\x04;\xc6\xf1\xa3\xb7&\xb9V\v(\x9d\xd3v1\x9f\x17µ\xe18Wu\xddH\xe1\xf6s\x1fYŪq\xca\xd89\xc7-Vs+\x8a\x8c\x99\xbc\x14\x0es\xd7\x18\x9c3-2\xbf\x11\xe9C\xf2\xac\xe6_\x99\x18\xc0\xed`ى\xa2\xc3\xe7\x83\xe8\x05ꡨJ'\x81EQa\x8b\a-P\x13Q\xf7\xee\x9f\xcb\xf7\xd0\"\t\x9a\nJ9\f\x9d\xf0\xd2\xea\x87\xd8\x14rM\x86O\xf3\xd6F\xd5^&J\xae\x95\x90\xce\xff\x91W\x02\xa5\x03۬j\xe1\xc8\f~m\xd0:R\xddX\xec\x9dOY`E\a\x8a\xfc\x00\x1f\x0f\xb8\x97p\xc7j\xac\xee\x98\xc5?YW\xa4\x15\x9b\x91\x12\x9e\xa5\xad~\"6\x1e\x1c\xe8\xedu\xb4Y\xd4\x11Վ\xfd\xdbRcN\x9a%ri\xaaX\x8b\x18I\xd6\xca\x00\x9b\x8c\x1f2\x95v\x01\xf4%#\xcax\xd09\xb3\xa3\xefMJP\x8bX\xf6\x1cy\x8cw6\x06\xaaj\x18\xa8\xfa\xdf$F\x1a\xd4\xca\n\xa7\xcc\xfe\x10)\xc7&qT;\xf4\xe5L\xe6X]\xb3\xbd;?\x13\x84\xe4\xc4;v&M\xce(H\xf5@\x95,\x14\x1d\xb2\x89:\xe0\xde\xd18\xb2s\x8b.\xbdYy4\xb2\t\t\x87\x1c\x13\xfa\xb9\xe4x\xdb+\xa5*dc6\xb5\xe2g6\xfd\xa8\xa2\xe30\xb8F\x83>\xfe\a7\xab\x95wƎ\tٺ\x8f\x90r\x83S\x89}\xac\xc8\xdd\x1cS\xcdq;\x84\x13!)\t\xf8\xf5\xe3}\x1bvZˊ\xd0'\x91\xa5\xcfO\xd2,\xe8[\v\xac\xb8\x0f\xd4\xe7\xd7NZ\b}\xf7\xeb\x00\xc2\xfb^\xa7\x80\x81\x16\x98\xe3 \ue050\xd6!㱑܍\xc1\xd8\xf7\"\xf8ԣ \xe9;\xc4GR\t0\xf2\xf1\x82ÿ\x97\xff}\x98\xffK\x85}\x00\xcb)\x13\xf2w\x15\xacQ\xba\x17\xdd}\x85\xa3\x15\x069\xdd>pV3)\xd6h\xdd,JCc\x7fz\xf5s\x9a?\x80\xef\x95\x01|b\x94\xf4\xbf\x00\x118\xef\xc2Fk5\u0086\x8dw\x12a'\\\xe9\x81j\xc5\xe3\x06w~\v\x8em\xe8Ą-4\b\x95\xd8`\x9a}\x80[\x9f<\x1d`\xfeN.\xe5\x8f[\xf8&8\x89[\xfa\xf36\xc0\xe8\x12\x84\xbe\xd79\xc0q%s\xe0\x8c(\n<$\xda\x13c\xa1\x80F\xa1\xe0[P\x86\xf6*UO\x84\x17Lz\n\x8e\x18\xf9\x04\xdeO\xaf~\xbe\x85o\x86\x1c\x1cYJH\x8eO\xf0\x8aθ\xe7F+\xfe\xed\f\xde{;\xd8KǞh\xa5\xbcT\x16%(Y\xedC\xbe\xb9E\xb0\xaaF\xd8aUe!\x15\xe3\xb0c{P\xeb#\xeb\xb4*\"\xd3d\xa0\x99q'ӱ\xc8\xc3\xe9C3\xcdO\xda\xefy\xe7\xc5\xe7+\xcf:\xbd_,\xd6?\x93\t\x9f\x98\x7f\x02\x13\xfd\xab\xce\x15Ll\x9a\x15\x1a\x89\x0e=\x19\\\xe5\x96x\xc8Q;;W[4[\x81\xbb\xf9N\x99\x8d\x90EFƘ\x05\xad۹/\xd9̿\xf2\xff\\\xbbq_g\xf9\xd4\xdd{!_\x8e\x02Z\xddίa\xa0ͣ\x9f\x1f\xbb\x8e\U000b0319\xddX&\x9d\xf9])\xf2\xb2\xbdU\xf5\xbcm\xcdxp\xc7L\xee\xbf\xd0\xd9!\x9e\x1bC\x88\xf6Y,8fLr\xfa\xbf\x15\xd6Q\xfb5\xc46ⓜˇ\xfb\xb7_\xf2D5\xe2\x1aOr\xe4\xb6\x10\xbe\xa7\xec\x80*\xab\x99\xce\xc2h\xe6T-\xf2\xd1hʕ\xef9)i-М\xc9\xfe\xde\r\x06\xb7Y{\"\xeb\xee\xc6\\\x94v[ɴ-\x95\xbb\x7f{\x06Dz\x1b\xd8b8\xe80&\x9d\xad,:\x12's\xcdg\xe0Y\x8a\xdf\x12n+\x89\x88\x86\xb6\x98*U\x88\x9cU`}\x9b\x8c\xc5\xca\b\xb3\x95=\x05\x94\xaaG\x8e\xe1\xf6\xab\x8a=\xbc\xde\x17<\x1c\xf7\xb4C\xc8\xc3\xd1-jeD!$\xab\x0e\x1e\xdb_\x1d%\xab\x99\xff+a\xab5\xd3Z\xc8\xe2\"n\xdb\xfa\xd6\x12\x9d\x13\xb2H$\xfa\xfd\xf2\xfb\xa9\xeb\xc0\xc9sr\xde\x05|\x18\x01\x01f\x10\x18\xed\x89T\xb5\xc1}\x16\xb2N\xcd\x04\xa5\x8c\x94\x15\xc6\xd4z\x85\xc0\xb4\xae(\xaf\v\x99d\xca7\xb5պ\\ɵ(b\xe5tʔl\xaa\x8a\xad*\\\x803ͱK[\xf2\xb8\xf7\v\x85g4\xfe\xa17\xb4U\xf7\x99RezW\x83\x02\xe6t3(\x9bz\n%\x83\x8d҂%\xda\xe9pN\x1c\x13u\xdc\xde^bR\xe1\xe4\x9f\xe1 ܙS\x05\x87\xe88\xe25$^\xb1\x83\xfbHG\xf3K\x1d\x8a\xc1_\x1b\xbaS\r\x11f\xe9\xda\xcah\x8cV\xfcfLZ\xdf\x17\x8f:\x0f\x9et\xdc1<\xf4\xa3\xde@\xc1\xb3\xcaR\xbeP~Ia*<\x87E\xdeC\x1a\xe0\xdaG2\xba`\\]\x9a\xa2;\xacvȻ7\x84k\xea6\xaf\xc7B|A\xd9\xf0xHD\x8d]\x91#ډ9\x94]B\x88\xd1\x065KZ\x04\xf8G\x01\xeb\v\xa3_\xdb MXh,r\xef['\x8b\x1f\x8d\t\x9c9\xcch\xfeu\x0e$]\xec\n\xcfs\xfdW\x98\xab*_S1S\x0eYG\x9b\x7f\x1fj\x1f\x06S\x94\x1d\xe4u\x84\x05q\xc8\xfd\x95\x1b\x94\x845\x13\x15r\xe8\x1e\x9f/f>\x01z\x9a\x8c}N\xf2k\xb4\x96\x15\xe7\x9c֏aT\xa8\xbc\xc5)\xc0V\xaaqG\xac\xf2k\x1b\x8f\xd6E1Y*~\x0eɃ\xe2\x1e\x86<\xfe\xe46E\x93PK\xff\x19\xee\"\x8c\xbe\xa8y\xaeHIcR\xae\xa6\x83|\xda\xd7\xc0\x89\x18\xf6\x80\xbbDk{\x82\x13]\x8f\xd1-$\xba&\xbf\a\xe8w\x86Jr*\xa7i\xfb\x922\xbb\xc7\xf6D\xdf\xf7\xfe\xb8\\\xc4v\xc4w\x8dC\xe8\xeaХ\xaaZ\x1f\xe0\x1f\xc9eS\xafА*V\xa9\x8c\x18\x98\xe4}ͥ\x8a\t\x9d\x846\f\aQ\xb1\x1e\x16\v\xe8\xfe\x94;\x05\\X]\xb1}\xb7\x19\x7f\x83\xa3#\x9d~N8\x9c\xab\xd6WQ\xe49\x92\xb7\x9d\xaeTw?ZH\xdfOOg\xfap&\xdb\xf7\xfdݏ\x11>\xcf\n'\xf2\xce\xe1\x8fC\xae1\x90\xe5@¹`\x11\x7f\xacr\xb9\x8f\x1f.\xf3g\xba\xf7${\x93F\x8f\x9c\xf7d\xc7'\xaf~K\xb3\xeaރ\x17\xf0\xfb\x1f7\xff\x0f\x00\x00\xff\xff;\xa8N\xc3\x13&\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xdc=[s\xdb8w\xef\xf9\x15\x98\xf4a\xdb\x19\xcbi\xa6\x97\xe9\xf8\xcd\xf5:\x8d\xfb}\xebx\xec4\xfb\f\x91G\">\x83\x00\x17\x00\xa5h\xdb\xfe\xf7\x0e\x0e.$%\x90\x84d˛-^2\xa6\x80\x03\xe0\xdc\xcf\xc1\x01\xb2X,\xdeц}\x03\xa5\x99\x14W\x846\f\xbe\x1b\x10\xf6/}\xf9\xfco\xfa\x92\xc9\x0f\x9b\x8f\uf799(\xaf\xc8M\xab\x8d\xac\x1fA\xcbV\x15\xf03\xac\x98`\x86I\xf1\xae\x06CKj\xe8\xd5;B\xa8\x10\xd2P\xfbY\xdb?\t)\xa40Jr\x0ej\xb1\x06q\xf9\xdc.a\xd92^\x82B\xe0a\xea\xcd?^~\xfc\xd7\xcb\x7fyG\x88\xa05\\\x11\x05\xdaH\x05\xfar\x03\x1c\x94\xbcd\xf2\x9dn\xa0\xb00\xd7J\xb6\xcd\x15\xe9~pc\xfc|n\xad\x8fn8~\xe1L\x9b\xbf\xf4\xbf\xfe\x95i\x83\xbf4\xbcU\x94w\x93\xe1G\xcdĺ\xe5T\xc5\xcf\xef\bхl\xe0\x8a\xdc\xdbi\x1aZ@\xf9\x8e\x10\xbft\x9cv\xe1W\xbd\xf9\xe8@\x14\x15\xd4ԭ\x87\x10ـ\xb8~\xb8\xfb\xf6OO\x83τ\x94\xa0\v\xc5\x1a\x83\b\xf8\x9fE\xfcN\xc2B\tӄ\x92o\xb8Q\xbb\x1aD<1\x155DA\xa3@\x830\x9a\x98\n\bm\x1a\xce\n\xc4;\x91\xab\x1e\xa40J\x93\x95\x92u\amI\x8b\xe7\xb6!F\x12J\fUk0\xe4/\xed\x12\x94\x00\x03\x9a\x14\xbc\xd5\x06\xd4e\x04\xd4(ـ2,`ٵ\x1e\xef\xf4\xbeNm\xcc6\x8b\v7\x8a\x94\x96\x89\xc0m\xc1\xe3\x13J\x8f>\"W\xc4TLw[\r\xdb#T\x10\xb9\xfc\x1b\x14\xe6r\x0f\xf4\x13(\v\x86\xe8J\xb6\xbc\xb4\xbc\xb7\x01e\x91Uȵ`\xbfG\xd8\xdan\xdcNʩ\x01m\b\x13\x06\x94\xa0\x9cl(o\xe1\x82PQ\xeeA\xae\xe9\x8e(\xb0s\x92V\xf4\xe0\xe1\x00\xbd\xbf\x8e_\x90xb%\xafHeL\xa3\xaf>|X3\x13$\xaa\x90u\xdd\nfv\x1fP8ز5R\xe9\x0f%l\x80\x7f\xd0l\xbd\xa0\xaa\xa8\x98\x81´\n>І-p#\x02\xa5\xea\xb2.\xff.\x12u0\xad\xd9Y\x1e\xd5F1\xb1\xee\xfd\x80\x02q\x04y\xac\xa88\xc6s\xa0\xdc\x16;*\xd8O\x16u\x8f\xb7O_\xfbLɴ'J\x8f7\xc7\xe8c\xb1\xc9\xc4\n\x94\x1b\x87\xacia\x82(\x1bɄ\xc1?\n\xce@\x18\xa2\xdbe͌e\x83\xdfZЖ\xdf\xe5>\xd8\x1b\xd4:d\t\xa4mJj\xa0\xdc\xefp'\xc8\r\xad\x81\xdfP\roL+K\x15\xbd\xb0DȢV_\x97\xeewv\xe8\xed\xfd\x104\xe2\bi\xbd\x16yj\xa0\x18H\x9a\x1d\xc6VA]\xac\xa4\x1a(\x19;d\x88\xa3\xb4\xf0\xdb洈U\x8b\xfb\xbf\xccq\x99m\xff\x1eG[~\xb3+k\x05\xfb\xad\x05T\xa6N\xfc\xe1P_\xa9\x9ej\x1f6\xcbF\xfb\xd4\x1dE\xb4m\xf0\xbd\xe0m\te\xd4\xeb\a\x1b\xcc\xd9\xc6\xed\x01\x144z\x94\t+D\xd6\xfaؽ\x88\xeeWT\xe0T\x01\x11\xd2$\xe01\xe1\xe0\x11&\x10\x03I\x9a`G\x03ubœ[&D\xb4\x9c\xd3%\x87+bT{\x88F7\x96*Ew#\xd8\n\x1e\xc0\x8b\x90\x15\x81xU\xc3Y\x81$\x8f\n\x05\xf1\xf5\xe7E\x15\xd3VQ\x86]>HΊ\xdd\f\xben\x93\x83\x82\xb4z\xd9\xf5;$K\xa8\xe8\x86I\x95\x12\x03\xa9\xb0kϞwjZZ-\xe9\x81\xec۸\xcc\r'\x91UI\xf9<\xc7\x10\x9fm\x9f\xce:\x90\x02\x1dʸ\x15Omo\xbb\x97@\xe0;\x14\xadI,\x93\x90\xb2E\xd3$\x15i\xa46\xe3t\x1fW]\xa4\xef\x1c\xa5~\x9c`\x9a\x83\x9d%Y\xdd5\xaf\x84\x03Q-\x0e\x06\nY\n\xb0ۨ-Q\xbb\xbeJ\xb6\xae\xef(RȒj(\x89\x14\xa33#\xbb\xb4\x1c\xb4\x9f\xabD\xce\xe8\xf4\xd0E\xb7\x7f\xf4x\b\xa7K\xe0D\x03\x87\xc2Hu\x88\xcc\x1c\x94\xba\x96\xa3XGP\x99ЦC\t\xe860\x01\x92XN\xdfV\xac\xa8\x9c\x87a\xd9\x13\xe1\x90R\x82\xb6\xda\x04]\xe6\xdd\xd8&\xc9\x1c\xf9\xfd$Sڣk3b\xb5\x0f/\xa5Q\xba\x96\xa1\x86\xbb\x96Dm\xa7{\x0ft\x8b\xffn\xe4\xe4\xb6\xff\x7f\"6\x18\x93\x13\x98vB\xfe\t\xba\x9f\xd9<=ʷ\x18ၾ$w+\x02ucv\x17\x84\x99\xf0uN\x12(\xe7\xbd9\xfeĴ9\x9e\xe93I\x93#\x13g\"L\x9c\xe2OH\x174\x19O\xdebd\xd3\xe4\xaf\xfdQ\x17\x84\xad\"\xd2\xcb\v\xb2b܀\xda\xc3\xfeI\xaa>P\xe65\x90\x91c\xf5\b\xe6\tLQ\xdd~\xb7.\x8e\xee\x92`\x99x\xd9\x1f\xec|\xe3\x10A\f\xcd\xf3\f\\\x82\xf12SPc\x1cN\xbe\"6\xbb/\xe8T_\xdf\xff|\x18+\xef\xb7\f\xce;\xd8Ȍйv\xbd\xb7\xa3\xfe\xfa|T\x10~A\x1f(\x06U.\xe7rA(y\x86\x9ds]\xa8 \x96>4tΘ^\x01&\x7f\x90Ϟa\x87`\xd2ٜÖ\xcb\r\xae=C\xc2\xf5O\xb5\x01\x0e\xed\x9a|X\xec\xf0d? \"0\x86\xcfe\x03\u05fc($r'閩KB\v\xb8?a\x9bY\xacҟ\xa3\x9f\xfaD\x0e\xf8I;ZZ\x89\xa9\x98\xcfij@\x99\xc9%\xa8k\xdf(ge\x9c\xc8\xc9ȝ\xb8 \xf7\xd2\xd8\x7f0@\xd3\xc8(?K\xd0\xf7\xd2\xe0\x97\xb3`\xd4-\xfc\x9c\xf8t3\xa0\xa0\t\xa7\xe5-\xc2\xfa9?g\xd3,\xb7E\xdc3M\ue10dW\x1cJ2\xa7\xc2\xf4\xae\x9b\xceMT\xb7\x1a\xd3uB\x8a\x05\xda\xcc\xe4L\x1e\xdfR\r\xd0\xfd\xe2I\xfd\x84_\xad\xb1p\xbf\xb8$3\xa7\x05\x94!\xb2\xc4\xec'5\xb0fE\xe6|5\xa85\x90ƪ\xf0<\x8e\xc8T\xac~7DZO\x9e\xf5\xee\xb7\xef\x8b\xe7\x98/XX\x93\xb3\xf0\x10\x8c\xac3p\xe0uw9\xbf\x9f\x85\x95ٌ^\x81\x13f\xbb\x8e$Gǻ\xe6 \xe5\x05\xe8@+\x8e.\xce,uiY\xe2\x11\x1a\xe5\x0fGX\x94#x\xe1X\xd5\xd0[\xbb3\xc15m\xacZ\xf8okiQ\x9a\xfe\x974\x94)}I\xae\xf1\xa4\x8c\xc3\xe07\x9f\x87\xeb\x81ɘ\xb2\xb1SY\xfe\xd9Pnm\xbfU\xe0\x82\x00w\x9e\x80\\\x1d\xf8E\x17d[I\xed\xcc\xf6\x8a\x01\xc7\xf3\x8a\xf7ϰ{\x7fa\xa7\x9f\x9d\xb2\xafd\xde߉\xf7·8P\x18\xd1ᐂ\xef\xc8{\xfc\xed\xfdK\\\xa9LN\xcd\xec6`њ6y\x1c*\x92\xc9\xfa\xae\r8\xa6\x9f\x9b\xef\x92\xf2\xdeɞ\xdam\x16\x8b6R\x9b\xcf\xe9\xbc\xe1\xc8z\x1e\u0088\xa1g\x9cȱ\xcdF\f>\x8f\x16\xf5\xbdu\"W\x06\x94\xcf%:\x1b\x10\xe2\x8f\x17Ff\xa9S\x99\xfebc2\x90\xc6\xfc\xaeE\xf0\f7\xb9\x83\x9b\x9c%\x1e\xe3\xb0Z\xbc\x1c\xe9\xed\xdf~\xef\xe53\xad\xe4ڿ\xfb\x1bym\x87\xba\x90uM\xf7O5\xb3\x96z\xe3F\x06\x9e\xf6\x80\x1c\xf5պEyε\xc8\x1d\x0f\xe1\xf9喙\x8a\tB\x83\xda\x00\xe5\x19\x8a\x92F\xa6rةVQM\x96\x00\"\xa6\xe8\x7f\x04W\xa2f\xe2\x0e' \x1f\xcf\xe0zDt\x9d\xd3ٽ\x894\x89\x94\x8f\x1f\x9c\xc9jdI\xb6\x15(\x180\xc6a\xde\x1d=U!M/eq\x84C\xda\xc8\xf2'MVLi\xd3_\x82&\xadΥ\xf5\x91\xe4\xb3\xeb\xfe\xcaj\x90\xad9'\x82o\xbbi\x06g\xcd5\xfd\xce\xea\xb6&\xb4\x96\xad3\xe6\x86\xd5\xf1TףwK\x99\x89\xc7V\x98\xbf1Ғ\xa0\xe1`\x80,a\x95>\xefM\xb5B\n\xcdJP\xa1J\xc1\x91\x8dI+\x98+\xcax\x9b:%J\xb5c#`q\xab\xd4I\x01\xf0\x177\xb2\x97w\xac\xe4v\x88\xa0̽\xe3A\x1a\x10\xb6\"\xcc\x10\x10\x85\xc58(\xa7\x92q\n\x8f\fD\r\xcb\xd5sy\n\xdc6\x10m\x9d\x87\x80\x05\n$\x13\x93)\xb7~\xf7O\x94\xf1s\x90\xcdr\xde'\xa9\x1e\x81\x96\xa7\xe4h~\xed\r' t\xab\xf0\xf0\xdf\xe9\x8e-\xe3yk\xb6\x94#\x9c\xb6\xa2\xa8\x00\x95\x90\x18\xea\x06\a\x9e\tm\x80\xe6\xf2\x82\xf5\x8aZ!\x98X\xe7\xd1.;\x11\xda5\x87\ua954\x1c\xe8\xf8)d\xd7,\xae\xdf@\x13\xfd\xdaM\xf3BM\xd4\x11\xc1\x1d\x9b#\x1d\xb2)j\x95\x16\xa1\xc6@\xdd8\x91\x93D\xb5\xa2o]Π\x88\x8e\t\xc3\xfd*^3\xbef\x82e\xd0v@\xd7;\xc1L\xdfy\xb4 \xce\xea<\xda\t\xa2;pJ\x86\xedn\x00\xc0\nh\x88Cp\xed\x91k\x8ep$\x97@hYB\xe9r\x97\xd6\x15\xf1a\x89+|\x1b)nH\xee\xeexO0\x8b\xb2\xa1\r\x82N\xccê\r,Z\xf1,\xe4V,0\x18\xd7G\xeb\x90\x13\xb3T/\x9dޜ\xac\x8c\xe6\xf5K\xbe\x9a\x9e\xd3BC~\xcd\xe7\xa9\xe0?\x9dA\xcbd\xf3\xcdQ\t\x8f).\x98\xd3k\xae\x00{\xe4\xc7\xd9UL\xcd?1\xd8\x1fJ߸b\xe9\x17\x95\xc5ݥA\xf5\x9c\xc2m\x05\xa6\x02\x15J\xb3\x17X\x92^N\x9e\x90v\xc1K\xac\x93\xb3L\x15\\dW\xfe\xb9W9\x87\xd1M\xcb\xf9\x85\xe5m\xda\xf2d8l$\x8a\xd8!geՏ\xa5=\x86\x9c\xea\x8bl<\xf6+-\x86\xf5\x85\xb1\n\"\x14\x18\xca0\xb3\xa7qj\xbfXX\xda;\xdf\x1f\x96S`\xfe/,\xff\x0f/=̨\x94\xc8Gcn\x95fDb\x02V\x82\xc1zh\xec\xea+|?_\xe8\xfbc\xe1\xd4@\xfd\xa5\xf1\x123\xea\xc2f\xa05\x01g\xaf\xde\x04\xadA\xab\x9d+\x10\xed\x80\xcf\x19\xda\xf1ׅ\xbb\x05\x11\xc0\xa4\xf8\xf5k\x05A|}\xf5>\xd3\xe4\x9fI%\xdbDU\xdf\x04\xcaf\xaa;\xe67<(\xf4\xf0\a\n`\xe8\xe6\xe3\xe5\xf0\x17#}\xd9\af\xd1\x12\x800(\xea2\xb3L\x94l\xc3ʖ\xf2 \xb5\xdd\x1d\x02\xc7@\x1d\x9f%\xa0IE\x04\xe3\x8e\x01\xc3\xf8\x01Ñ/\x8d;\x969Z\xc5M\xfb\xa2y\xd5!'ׄ\fk>F\xac\xe1\xb1\xc7\x17\xafR\x05\xfb\x87\xd4z\x1c_\xe1\x91\x13I\xccTs\x9cPÑY,\xf6\xe2\xf3\x96\x9c*\x8dcb\xee\xb3Ud\xbc~\x1dF\x16~\xe6k.\x8e\xc1\xce\xd9\xeb+ް\xaa\xe2mj)2+(^\xaf\x142/\xfa<\xa9\x14`>`\x19\xaf\x82\x98\xad}xQ@sҖfk\x1a\x8e\xa9d\x98\xa5N\x9e\x98\xbdY\xad\u009bU(\xbcm]\xc2$\x17M\xfexL\xe5A\x8c\x93~\xa1M\xc3\xc4\xfa\x90)rYg\x92m\xe6Y\xe6~o!\x03\x9e\xe9\x873]t8\x12\xfa\xba\xeb҉H2\xa4-\x990\xf2\x92\\\x8b\x9d\x87\x9b\x80\xd3\v\x1f\x854\a\x17\xd9첶\x8c\xf3\xfem-\x04;\r\xcaߙԴv\xab\x1a\xf3\xf6\x93t\x95j\xe0\x94\x9f\x148~ك\xd1ώ\xbe\xa5\xe7_\xb7ܰ\x86\x83\xf5\xe86\xacL\xde!3\x15\xec\"\x92\xff&\xf1\x86\xd4r\x87\x90\xbe]\xf4\x9eQ\xec\x9eq\x926\xbf\xc9\x13\xb6\x97Q\xcc~\\\x11{\x06\xcdrE\xf1\r\x8b\xd5߰H\xfd\xad\x8b\xd3g8k\xe6\xe7\xe3\x8a\xd0O>\x81\tG\xfd\xf7\xb2\x84\a\xa9\xcc\\p\xf2\xb0\xdf?q\x92\xda\v\xd8$/\x89\b]\x13\xbb\xc4\x10Ç\x17\xa7m*}\xe8\x19\xdc\xe9_di\xd76w\xc6\xf2\xb8\xd7\xfd\xe0\xae\xf2\n\x14\b\xf7\xcc\xc7\x7f>}\xb9\x8f\xf0S>\xaf\xf7\x8c\xf7\x9e\x97p\x1eL\xe9\x91\xe3\x8f\xe6|1\x93\xc3\x16\xfa\x00\xaf|.B\x1b\xf6\x1f\xf8\xaa\xdb\v\xd2A\xd7\x0fw\b#\xf8i\xf8L\\\xac\xa2\x88'\x96K\xb0\x16+\xa2jT,\xeeV\x03\x88Ê\xdf\xfe3JP\xba'\xb3\x82\xc5d\xa1\xc6\xcb\n\xdeÝ[\xc7\xd8,\x9f\xac\xd3(vD:\x8e\xac\x98*\x17\rUf\x87l\xa3/\x06k\bff*\x9d3\xaaX\x0f\x9f\x01K\xa27\xbc\xfe\x85g\x91\xbbfxڻ\x8f\xbbS\xd61~\xffd\xf6\xe6\xc9+\xaec\xdcb/\x10S\x89\xcf\xc9\x02\x93WK\x93yM\xf4\xf0\xed\xa4\xb4\xcbc\x1c=\xad\xe7l\x14\x1dRM\t0v<\xaa:-h\xa3\xabēK/\xd3u\xf8\x1a\x99\xa1\xa6}\xc9&\x1d\x80\xc1>YQ\xf5\xb4\xd5\x16\x82>\v\xdbFi\xc5a)\xddnm\xb3\xab{a\xfc\xa2\x97)x\x9b#\xe1\xcc\xe7\\N~\xc8šgD\xfd`\xf6˪\xb6CL\x9dp\x18<\xeb\xdae\x14\x19O;\xb1\x99π\xe4\x19\x8c\x13\x9e\xfe@|\xe5\xe2\x8a$_\x04\xc9|\xf5\xe3\x0fE\xf4\x84V\xd3E\x05e\xcb\xe1\xd47\xff\x9ez\xe3\xe7_\xfd\v\xb3e\xbc\xfbg\x91\xdd3\xd0\xd6g\x1e\xbe/\xe8)\xe1!\xf7)9\xe6\xf0ap\xe0\x9e\x17+\xdcK\x94E\x01Z\xafZ\x1e\xaa\x94\n\x05\xd4@\x19\xba3\x1dW|T\x9dM\xdbpIKP7R\xacX\xe2\x84d\x80\xd6\xff\x1at\xde\xe3\xd9\x02?\xb6\xaa{\xdaq\xf2Y\xbc\x17i\xae\x86*\xca9\xf0O\x8c\x83\xfeYn\x85]W\x86@>\xa4\xc6\xf5\xeee\x15\xad\xb2f}GD[/\xad\x93\vƌ\a\x8b+\xa9\xa6+\xa4\x1dޙ0\xb0\x86T|\xbdU\xcc\xc0SC\x95\x06\\Q\xc6\x0e~\xdd\x1b\xe2\xa2\xcf\x15\xa7kW\nW\xb2\x82\x1a\x88\x06\x18g\x18[>\x8e\xd7\b\x8b\xef\xb02I\x8e$\xbd\xb2\x85z\xecJƨX\x8f=/\x9a0\xd5\xc9\aF\x9dE.hc\xf0\x02\f\xd2\x11\x89h<\f|\xb4w\xef\x8d\xd1\x01\xd8qN\xf3e̾`N\x1bZ'\xa2\x84y\xbdss\b\x06\x9f\x05Ve\xaf\xee\xae\xff\xc0b,\xb0#[\xaac1u\xd2\xf7\xee`;0\xe8\xaa[\xd0P\x12\u0600 V\x14)\xe3PNq\xeaWL$\xab\r\xa8\x9ft\x84\x83\x95\x80\x96ş\fU&.\xfdЏYIUSsEJj`aG\x9f溥\x9fIU\xea\xc4\xe3@\xbc\xd9\xe6ţ\b\xd7n\xac\xf5s\xf7\xd1jК\xaeC\x10\xba\x05\x05d\r\xc2\xe2=\xe6\x16\x93\x1eS\xb8\xd2\xe7\x8dE,-\xb5(\xa4\x85i\xa9\x9f\xc0\xb9p\xf1\xf44\xbcO\x8cQ\xeczTE\xa7U\x85\xbf<\xf8\bT\xef?w}\x80\x8bO\xfd\xbe>I\xecv\xec\xceF\xa8+\xf0\xc4\a\x8f\r\x8b\x91uJ\xa6\x8dę\x8f2'\x95\x94\xcfYn\xf6\xe7رK'1\xe1X\t\xafL.ekz~\x8eGxb\x99\xf8\xfc\xe7+\xdb\x17\x84y\xed.P\x8d\xe5V\xf3<\xbd\xcf\x03H1\xbc\x95\x86\xf2`d,_\xc6\x0e\xd5\xc4\x03\x02O\xe1\xf1d\xcew\x17\xfb\x90\xf7^e\xef`W\xddS\x9e^\x13t\xd7\xc7\xc7Ҫ>\xeb\x97\x04\x12_\x01\xed|\x92\xb17\x17\xe7\xec\x1fB\xfd\x84\x8b\xca\xc0\xf1\xe7\xae\xf7\x18\x1e\xdd2\x9d\xc3\f\"\x1di\x12\f>L\x15%ㄥOx\xa9ME\xf5\x9c{\xfa`\xfbD\xb7\xa3g\xae\xa2\x13\xfa8\"\x95\xe9{\xae\vr\x0f\xdb\xc4W\x87,<\xfdB\xa9Jt\xb9\x13\x0fJ\xae\x15\xe8C\xa6[\xe0}F&֟\xa4z\xe0횉/\xe3\x95\xdfS\x9d\x1f\xa82\xcc2\xad[Ob\xecM\xb0q\x89\xdf\xe6G\x8f\xff\xc0\x04\xe5\xec\xf7\x94.\xef\xff87Ä\xbek<\xf2N\xb1P\x01\xf1s\n\xd0k\xe8\x9ft\xcf\xfc\x84y/ɽL\x8a\xb1? fC\xa0L\x93%h\xb3\x80\xd5J*\xe3\xf2\xf7\x8b\x05a\xab\xe0 Y\r\x81q\xa2{͞\xb0T\xe2=\x1e\xbd\x05\x87e\xe5S\x89\n\xad\x0e\x86\x9c5ݹ\x8c$-\n\x1b\x13\xc0\amh*6y\x91\x9e\xc6P\xd5\xcbJ\x8e\n\xb9\xeb\xf7\x8f9\xbe\xa8>\x10\x9cC\x1d^gw\x06\x9d\x8f\x9di\r^\xcb \xdab\xef\x14eB\x9c\x1a\xbb\x1b\x0f\xbb\xf3L\xcd\xd7\beL=\xfa\xfd\r\x1e\xe2\xf6\a\xac\xbe\x93%[QQ\xb1\x1e\xbd\xd0V)ٮ\xab\xc0\x9bc\x0e\x11)[\x8c\x9c\x1bT\x05:\xfc\xc7!\xa6U\xa2wh\xe7k,ƴt\\\uee0f\xf2\x02E\xad\xba\x8b-\x9d\xaa\x9a\xb0\xf9\xd9Y\xc2\x11\x88\xb3\xb6?\x01\x91\xea\x9d(&\xaf\xe0\xf8@\x9bM\xdc՝\xc2P\x12\tQ\x1b\xbf\x1a\x12\"\xc41$\xf4}\x89.\xe2\xf9a02棜\x88\x8ei'\x06\xb78\rj~\xd3}'h\xe8\xee\x1c\x87\x0e=\b\xfeNJ\xbb\r \x1c\x13\xf9\xe2\xdc\xe9\xb8\xf7ǍX7\xd1ۺ=9v\xfd\xb6\ac\xef\n\xa4\x8db\xbbiB\xbc\xf9\xf7l\x95\x92\x17\xf7\xbf3-9\xfc\xc3\xc1\xafo|\x95qK\x95`b}\x12F~\xf5c\x13\xf1\xbc\a{Έ>\xac\xfc\xd5b\xfa\xa4Y:\xf8\x88\f^\xf6\xf0\xecg\xf2_\xfe/\x00\x00\xff\xffP\a\xb5\x16Cm\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xec=]s\x1c)\x92\xef\xfa\x15\x84\xeea?B\xdd^\xc7}ą\xde|\xb2gO\xb1\x1e[ai\xf4\xbctU\xb6\x9aQ\x15\xd4\x00\xd5r\xdf\xde\xfe\xf7\x8dL\xa0\xbe\xba\xe8\xa2Z-ygǼت\x86$\xc9L\xf2\x03\x12X,\x16g\xbc\x12\xf7\xa0\x8dP\xf2\x92\xf1J\xc0W\v\x12\xff2\xcb\xc7\xff6K\xa1\xdelߞ=\n\x99_\xb2\xab\xdaXU~\x01\xa3j\x9d\xc1{X\v)\xacP\xf2\xac\x04\xcbsn\xf9\xe5\x19c\\Je9~6\xf8'c\x99\x92V\xab\xa2\x00\xbdx\x00\xb9|\xacW\xb0\xaaE\x91\x83&\xe0\xa1\xebퟖo\xffk\xf9\x9fg\x8cI^\xc2%3\xd9\x06\xf2\xba\x00\xb3\xdcB\x01Z-\x85:3\x15d\b\xf4A\xab\xba\xbad\xed\x0f\xae\x91\xef\xd0!{\xeb\xdbӧB\x18\xfb\x97\xde\xe7\x8f\xc2X\xfa\xa9*j͋N\x7f\xf4\xd5\b\xf9P\x17\\\xb7\xdf\xcf\x183\x99\xaa\xe0\x92}®*\x9eA~Ƙǟ\xba^0\x9e\xe7D\x11^\xdch!-\xe8+U\xd4e\xa0Ă\xe5`2-*K#\xbe\xb5\xdcֆ\xa95\xb3\x1b\xe8\xf6\x83\xe5g\xa3\xe4\r\xb7\x9bK\xb64ToYm\xb8\t\xbf:\x129\x00\xfe\x93\xdd!n\xc6j!\x1f\xc6z{Ǯ\xb4\x92\f\xbeV\x1a\f\xa2\xccrb\xa0|`O\x1b\x90\xcc*\xa6kI\xa8\xfc\x0f\xcf\x1e\xebj\x04\x91\n\xb2\xe5\x00O\x8fI\xff\xe3\x14.w\x1b`\x057\x96YQ\x02\xe3\xbeC\xf6\xc4\r\xe1\xb0V\x9aٍ0\xd34A =l\x1d:\x1f\x87\x9f\x1dB9\xb7\xe0\xd1\xe9\x80\n»\xcc4\x90\xdcމ\x12\x8c\xe5e\x1f\xe6\xbb\aH\x00F$\xaaxmH8\xda\xd67\xddO\x0e\xc0J\xa9\x02\xb8\x80vX4\xb6\nu%\xa0\x80\xe6\f\xddN\x8d\x16FH\xb6\xae\xd1#]2\xd4\x12Q\x19\x11\xd2X\xe0\x11a>\x01\xef\xe0kV\xd49\xe4WEm,\xe8\xdbLU\x90\x87E\xa6Q͜\xca\xc3\x0f\a!\xfb\xf8\xa5\x10\x19 \x1f2WiA\x8b<1\xd1nC\x99]\x05n\xcd\tY\xed\x87\xd0\xc6(\x93\xbaŀņ\xe7\x7f<\xbf \t\xe8\xf7\xde\xef\xc70\xae\xa1!\xd3,\xddL\x16\x7f\xbc\x85\xb0PF\xa8;\xa9\xa3f\xf0\x9dk\xcdw\a\xb8\xde,\xa6\xbd\x00\xdfc\xb0\a\x9c\x97\xa1\xda7\xe2\xfd\xb0\xff\xdf\"\xf7O\xcboC\x8b\xce\\H\xe4s!\x8c\xed\xb1ٸU,$\xebX\b\xe9\t$\x1dLT\x93S\\\xfd'!\xe6I\xe7Nl\xb24\xb2\xe9'\xc0\xbf\x14%7J=\xa6P\xef\x7f\xb1^\xbb\x84\xc52\xda\x18a+\xd8\xf0\xadP\xda\f\x97I\xe1+d\xb5\x8dj\x16nY.\xd6k\xd0\b\x8b\x96\xf9\x9b]\x81C\xc4:\x1c\xbe\xb0\x8eʊV\x18\x8c\xabe:\xb2\x94\xa8\x11\x1b\n\x05\xa8Q\xa8\xce\xc1\xc1Ђ\x1c\x88\\lE^\xf3\x82|\t.37>\xde\xe0\x17\xd3j\x13\x02\xb1\x87\x7fT\xaa]q\x0eM\x18$2\xb1\xb7\xea\xa5$\xa0\x8f_bl\xb4_5N\x89\xb0\x94p\xb0od\xa6\xae\v0\xbe\xbb\x9c\xdc\xe4V']\xb4\xccrk\f\x05_A\xc1\f\x14\x90Y\xa5\xe3\x14J\x91\x03WR\x95n\x84\xb8#Z\xb6\x1fm\xb5\x83\x99\x00\xcb(\xc4݈l\xe3\xdcW\x144\x82\xc5r\x05\x86VExU\x15\x11\xd3ՖI\xe1\xf0\x9dM鍶$h\x90!ܘ.iK\xa2~n\xcb(\xd9۹٧\xfa\xf8:\xff(\xbe\xbf%\xa2\a\xabs\xa4\xb0Oh\x12F\xfb\x05\xc9\xf3!Jz\xa4\xb8\x00\xb3\xec\xac\xce\t\x1b\xbe\xa60\xb4\xe7?\xeem\xa5\xec\x11\xe5\xd7Ż\xe3&\xcc\f\xd6MΩ\x97e\\\xd3Ϳ\b\xdf\xc8d\xddz\x8b5\x8bg\x1f\xbb-/hW\xc03$\xbf`kQX \xa7j\nQ6\x83s\xa7$P\xaa\x05f\xb4Il\xb3͇f\xef(\xa1ŀVC\x00\xceA\x0fQ\x0e\xf1 \x01$k\\\v\xda4\x15\x1aJڌ\xa5H\xb2\xfb\x85\\\xc1w\x9f\xde\xc7c\xcfnI\x94ԽA%LZW\xde\r\x1c\xa3.\xae>T\t\xbf\x90\xbf\xd6\x04\x82n\x13\xfe\x82q\xf6\b;\xe7bqɐo)\x8b\xff|\xf8*\fv,s\xf6^\x81\xf9\xa4,}yQ*\xbbA\xbc\x06\x8d]O4A\xa5\xb3$H\xc4n\U00088ce5(\xa8\r?\x84a\xd7\x12C2G\xa2\x19\xddQ\xae\x90\xeb\xd2uVֆ\xb6Z\xa5\x92\v\xb7,6֛\xe7\x81\xd2=\x16\x9c\xa4c\xdf\xe9\x1d\x1a#\xf7\x8b\xcbZ*x\x06yآ\xa3t\x1an\xe1Ad3\xfa,A?\x00\xab\xd0,\xa4K\xcb\fE\xedG6_\xbc\xd2=\x87n\xf9\xbax\xacW\xa0%X0\v4k\v\x0fŪ2\x91.\xde&\x8c䜌\x95\x05\xce\xf5ĚAZ\x92\xaaG2r\x0eWO%\xd63\xc9D^\x04\xb9]IR\xd0Ml\x9dg\xbdf\xca\xcd1*\xa63\x16\xe7\x02\x94\x9c\xb6\xd6\xfe\x86\x96\x9ef\xe3\xdfYŅ6K\xf6\x8e2{\v\xe8\xfd\xe6\x17&;`\x12\xbb\xadh\x95\xfd\x97Zly\x81\xfe\a\x1a\bɠpވZ\xef\xf9j\x17\xeci\xa3\x8cs\x1b\x9aM\xbb\xf3Gع\x1d\xe5\xa4n\xbb\n\xeb\xfcZ\x9e;_fO\xf14\x8e\x8f\x92Ŏ\x9d\xd3o\xe7\xcfu\xeffH\xf4\x8c\xaa=Q.y\x95.ɔ7;'\xd0\xc0`=8DظI \xc5\x00a\x8a\x02ɢ\\)\x13I\x16\x89\xa0\x95 \xe87\xcaX\xb7\x0e\xd9\xf3\xf7G\x17*UX\x9cd|mA3c\x95\x0e)\x99\xa8\xf8S\x96\xe2\xbb\xe5n\x03\x06\xfc>\x94_\xf4t\x801\x8a=ou\x83\xb3*\xe7n/\x8c:\xe2\x19yOԶ\xd2*\x03\x13͋hK\xa2m\xeaQp\x9f\x0eͺ.w\xd1\xdf:Ik\xa7,J\x872ϑG\xd2\x1d\x11\x19}\xf8\xdaY\xa2F\xed\x82\x7f\xa7H\xeb182:\xafQ\x96|\x98\x0e\x9c\x8c\xee\x95k\x1d\xe6\x98\a\xe6\xc2-\xfdP\x93Ι\xe3u4\xa2\xfc\xcf\xe6ڔB^SG\xec\xed\v\xbaC^\x8b\xc7ң\xc6\xca\xf1N\xfaU\xe8\xac\xe5^\xf3\xc1\xe7\xd4)\xda\xf8\xd1\xd0c\xee\xfe\x9e\by\xd7R\xd9\xce2\xceL'\xbaR\xf9\xef\f[\vml\x17\rs \xb1j\x14\xd4\x11\xa1\xa7\xfc\xa0\xf5ё\xe7g\u05fa\xb3\xa0\xb8QO>qzN\xbc\x1dH\xba\xe1[\xf0\x99\xab 3UKZ\nC=\x80\xdd̀\xe8X\xe3\xac@\xa2\xbd\xeb4\x96u\x99N\x90\x05I\x92\x90\x93\xebf\xdd&?p\x91\xb6nŎc\xab=\x94\xc39V\x8e\x9fG!\xc1\xb3\x9bN_\U000af8acK\xc6K\xe4!\xb9\x1d\xa2\x84&\xa3ޱ\xbbI\xfb\xc4\x16d\xb4\xac\xc2YV\x15`\xc1\xa7m\xce\xc0#S҈\x1c\x1a\xd3\xefE@I\xc6ٚ\x8b\xa2\xd63\xb4\xeal\x92\xcf\r¼69}d\x95\x8eȂH\x94\xb8\xce>\xc3\v\x9e\xd6\xf8\x95\x9e\xe7Ǧ8\x8c\x1a\xe6\xfb\x8b\x95\x16\xca\x1d\x068\xbd\xcb\xe8ӎ\xb9\xdc}\xf7\x19\xbf\xfb\x8c\xdf}\xc69\x1d}\xf7\x19'\xcaw\x9f\xf1\xbb\xcfx\xb8|\xf7\x19S\xcaw\x9fq&\"\xdf\xcagL\xc1pAk\x9c\a*$a\x95\x98\n1\x85\xf6D_>\xe9ǟ\xd58I.\xf3\xf58ȑC<\x91\xe3\x171\xaf\xa35^Mr3\xce\xc00w\xdc)\xca\x04\x87\xf9\x04\xa7g\x02\x02\xa7?=s}\x10\xf2\tO\xcf\xf8!\xa4E\x18G\x9d\x9d\tD\x9a\x7fz\xe2\xc2'\x11\x95\xc0\xc3V\x8aK\xff\x88\x8d1&I\tx|\xe3\xe4\xf7\xbd\x8c\xc9\x17\x90\xa5W9\x913K\x9eFY\x7f\xfe\xc7\xf3_\a\x8bN˔(\x1b\xf6i\xeb\xd4xL?b,\xdfM\x8d\xecg\xa9\xfez\xa6\xc2Ie?\xf5DMC\xe4\b\xbc\xbeX\x0f\xa8\xfck\xd27\x16\xcaϕ\xb7\x96'8a\x7f=\x02/\xe9\x8c=7;\x99m\xb4\x92\xaa6~M\ba\xbd\xcbܽ\x03\x01dL\xd8G5\xc8\x7f\xb0\x8d\xaa#\xa76&H\x9b\x90E\x9bF\x90^R\xadO\x8c\x00˷o\x97\xfd_\xac\xf2)\xb6\xecI\xd8M\x04\x18\xddG\xc1\xf3\x1c\xe3\x82\u0381\x1e\xaf\a\xc2UIC\xa1\x8c\x00S\x9aIQ8\x89\r\x10z\xf2\xca>Wnu\xf0h\xbfiz\r+=\x11wn\xfam\x93-9\xed\xbe?#\xe9\xf6\xa4G\xa3\xbeYZ\xedqɴ\xa9+\x94\t\x89\xb3\xe9\xe9\xb2)lu%=I69BNM\x88\x9d\xbb\x02\xf1\xa2ɯ/\x93\xf2\x9aL\xb3\xb4\xf4ֹ\x14{\x95T\xd6WN`}\xbd\xb4\xd5\x19ɪ\xa7?\xf5\x92\xbe\x96~tveڲ\xcc\xe1\x84Ӥ4Ӥ\xa5\x9b\x94\x01\x1f5Ԥ\xf4ѹI\xa3I\x9cL\x9f\xae\xaf\x9a\x16\xfa\xaaɠ\xaf\x9f\x02:)m\x93\x15\xe6&y\x8e_r\x18ʴ\x03P|\v\xe1|.\x99\x94\xee\xb9\xe6ϊ;?\x0f`\xa1\xb0\x047\xf5\x15〲.\xac\xa8\x8a\xf6>\xb6X\xc0\xb9\x81]sY\xd1ϊ\x8e\xc8\xfb\x9b\xba>\x7fi$~9\x88j\xb8aOP\x14\x8c\xc7\xe6\xe6\x1e\x152w\x0fh\xa6\x16\x80\xb6\x11g\xb9\xbf\x8c\xc9_\x1ez\xe1\xa6\v\xdd\x06@\x16\xb6\x8c-\xf5qy\xf8\xa6\xaf\x83\x06,U\x8f\xedy\xe6.ޠo\xbfԠw\x8c\xee\x1dk|\xb3\xf6P\xa9\x9f\xe8\x06\x03Ӡ~\xbc:<\xb4g\xb2\x17\xe0\xb4ꁽ\x93\xce#\x18\xe2DmP\xef\xb4\x01\x1d*U\x8cӢ\xfdD@H\xd5@\x884Mq\xfe眲|\x89\xf0\xee\x14\x01^\x92\a4\xcf{\xfd\x86\xa7'\x8f=5\x99\x9e\x8c\x92tJ\xf2%½9\x01\xdf,\x7f5\xfd\x14\xe4\xfc\x8d\xe7\x17>\xf5\xf8R\xa7\x1dgP/\xf5t\xe3|ڽ\xd2i\xc6W?\xc5\xf8\x9a\xa7\x17g\x9dZLNϚ\x95q0'\xb5\xea\x19\xc7\xed\xd2r\t\xa6O!&\x9e>L\xcc4H\x1b\xfc\x91\xc3N<]8\xffTa\"\x7f\xe7L\xe9W>=\xf8ʧ\x06\xbf\xc5i\xc1\x04\tL\xa82\xffT\u0cf7\xa4\x94\xceAOn\xfb͑\xdaIyM\x8d\xe5\xfa\x88\r\xf6\xb5\xc2m\xb2X\xab\x17\x03\x90Y\xf2\x17\xf9ӣ\r\x87\xb6\xc1Q2;\x1eQo_\xb2u\xd7\xfa\x0e\xb1\x7f\xcd\xc1m]\x1a\xa88\x1a\x00\n\xdc(5+\xea*|\xe0\xd9f\xd0Æ\x1b\xb6V\xba䖝7\x9b\xc5o\\\a\xf8\xf7\xf9\x92\xb1\x1fT\x93\xabӽ/͈\xb2*v\x18\x89\xb1\xf3n\x83\xe7IIT:C\xcf7\xaa\x10Y\xc4\xe7\x1c\xbdW\xcf5ػl\x88n\xfe\xcb:\xd9\"\xb1\xc0\a\x9b\x8bp\xebb\xffJfw\x9f\xfb\x91k%\xbc\x12\x7f\xa6'\x95N\xb0\xea\xf6\xee\xe6\x9a`\x051\xa2\xb7\x9a\x9a\x04ņ\xe5+@\x97\xa1\x1d\xfb!}r\xbd\xeeA\xed\xe7\bw\x1f\xab\x80ܽL\x12\xdc\x16\xaf\x9a3\x85Z\xeb\xe6\xda\xe1r\xa8'\x94/.wL\xf9\xa7'\x84\xce\x17\x15\xd7v璉.zx\x04\xbb>\xb5jv\xd0Z\xed\xbf\xbc\xd2-=\xb2\x87GWh'{W\xf5\x93\a\x86\xf4|\x0eN\x87OUO\x9e\xa7~\x01\x9c\x0e\xbbP\v\xa2b\xe4\xa7h\x06\xe4\xc9W,\x8d\xbf\xa1\xffG\xb5\x85\xf7ѕ\xcb\xfe\xeb+\x83&#\xa9\x89\x01*]2\x1f\xa1`\x9b\x8fHw|?O\xed\xc5s\r\x03*\xfe\x8e\xf0\xe7,N\xde\xf6A\x8d?HB7\xa8\x87Nc^\x15=\xf5\xb4c7\xf7\x14\xb76\xaa\xd4O}\x1f\xb7\x86\xe5ɐ`\x10\x81%\xe4\xc17ZNEF\xab4\x7f\x80\x8fʽ\xad\x93\"&\xfd\x16\xbd\x97\x97\xbc\xe7\x16\xf2\xb5\xfd$\x8c)z?\xb6!\xc0\xf6|\xc6\xdeE\xff\x88\xed\x91O\x19X[\xba\x91ғ&\xef\xfd\xeb$\xa8\x8f\r \v\x02\x05\x1c\xb4\x15\xfew\xa3\x9e\xe8\x02\xfc\xf8\x1asx@\xa4\xf3\x86\x19\xd0A\x11J\xe1=j\x98uU(\x9e\x83\xbe\xa2GT\x12F\xfcS\xaf\xc1\xc0\x1d\xe8?\xc5\xe2\xedfd<\xa1\xe7\x17̒A\x8f\xae(\xa0\xf8A\x14`\x1c≦\xe1f\xbfec)\xear\xe5<\xd55\xfe\xd8tr\xc02\xbb\xa1\xd2\x06C\x05\x1a\xfdD\xb7\x15Q\x9b \xf9\x87\x89\xc1\x1a>\ni\xe1\x01\xc6c\xe8\t\x9b\xe0\xdeh \a (0\x8a\xf8\xfe\x12[y\xec\x11\xe4>\xdez \x03\xcdbdL\x8e\x95w\xabn\xee\xaf\f\xabeN\x1b\x00\xf7\x7f\xbe=J~\xb7\xbd\xf7e\x82NHQ\xef\xf7\xe3-;!BG;\x91O\x1fW\xe21X\xdc\x18\x95\t\x8a*\x9e\x84\xf5\xd79\xbe\xdc\x1d\xe2\x87\x02\xc4\x03\xd2Q\x1b\xf8\xfc$A\x7f\t\x16\xc8\\\xcbػ-\xd3\xda\xef\xa7=h\xd1\xf7Z\xac¾G`\f\x000\x15\xf6\xb9\x8c{\t(l\xaf\t\xd3H\x1c\xc4>\x01\xdcyG\xc8ik\x85\xb4\xe3\x9c!n\x9bVt\xd8tDCN\x8b\xed\xfd\x00\xc6 \x93\x9d\x1e}j\xaa\xb8Ӧ\x86\xfd^\x8cy\xa3\xb4c\x96\xe1@\xff\xb0\xf7kT\x83\x1f\xd4\xde1\xcd=\xaaF\xf6>\xd2CxyGr\xbc\x97\xde\xfdR\xaf\xda\a\x15\xd8\xdf\xfe~\xf6\x8f\x00\x00\x00\xff\xff)\x00\x87w>{\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcV\xcfo+5\x10\xbe\xe7\xaf\x18\x89+\xbb\xa1B \x94\x1b*\x1c*\xe0\xa9j\x9ezw\xbc\x93d\xa8\xd7^f\xc6)A\xfc\xf1\xc8\xf6n\x9b\xee:\xb4\x8f\x03\xbe\xad\xed\xf9\xe6\x9bo~x\x9b\xa6Y\x99\x81\x1e\x91\x85\x82߀\x19\b\xffT\xf4\xe9Kڧ\x1f\xa4\xa5\xb0>ݬ\x9e\xc8w\x1b\xb8\x8d\xa2\xa1\x7f@\t\x91-\xfe\x84{\xf2\xa4\x14\xfc\xaaG5\x9dQ\xb3Y\x01\x18\uf0da\xb4-\xe9\x13\xc0\x06\xaf\x1c\x9cCn\x0e\xe8ۧ\xb8\xc3]$\xd7!g\xf0\xc9\xf5\xe9\x9b\xf6\xe6\xfb\xf6\xbb\x15\x807=n@\x90ә\x1a\x8d\xc2\xf8GDQiO\xe8\x90CKa%\x03ڄ\x7f\xe0\x10\x87\r\xbc\x1e\x14\xfb\xd1w\xe1\xbd\xcdP\xdb\f\xf5P\xa0\xf2\xa9#\xd1_\xae\xdd\xf8\x95\xc6[\x83\x8bl\\\x9dP\xbe \xc7\xc0\xfa\xe9\xd5i\x03\"\\N\xc8\x1f\xa23\\5^\x01\x88\r\x03n \xdb\x0e\xc6b\xb7\x02\x18\x05\xc9Xͨ\xc5\xe9\xa6\xc0\xd9#\xf6\xa68\x01\b\x03\xfa\x1f\xef\xef\x1e\xbfݾ\xd9\x06\xe8P,ӠYֿ\x9b\x97}\xa8\x85\t$``\xa4\x04\x1a\xc0X\x8b\"`#3z\x85B\x19\xc8\xef\x03\xf79\xad`v!\xea\x05\xaa\x1e\x11\x1e\xb3\xfec\x98\xed\xcb\xe1\xc0a@V\x9a\xa4)\xeb\xa2\xe2.v\xff\x8dxZ)\xd6b\x05]*=\x94\xecy\xd4\v\xbbQ\x1e\b{\xd0#\t0\x0e\x8c\x82\xbe\x14c\xda6\x1e\xc2\xeew\xb4\xdaΠ\x8b.\x922\x19]\x97*\xf6\x84\xac\xc0h\xc3\xc1\xd3_/ؒ\x04JN\x9dѬ\x9dWdo\x1c\x9c\x8c\x8b\xf85\x18\xdf͐{s\x06\xc6\xe4\x13\xa2\xbf\xc0\xcb\x062\xe7\xf1[`\xccRo\xe0\xa8:\xc8f\xbd>\x90N}hC\xdfGOz^疢]\xd4\xc0\xb2\xee\xf0\x84n-th\f\xdb#)Z\x8d\x8ck3P\x93\x03\xf1\xb9\x17۾\xfb\x8a\xc7Ε7n\xf5\x9cjP\x94\xc9\x1f.\x0er\xeb|AzR#\x95b*P%\xc4\xd7,\xa4\xad$\xdd\xc3\xcf\xdb\xcf01)\x99*Iy\xbd\xba\xd0e\xcaOR\x93\xfc\x1e\xb9\xd8\xed9\xf4\x19\x13}7\x04\xf2\x9a?\xac\xa3\\\xb8qד\xcaT\xda)us\xd8\xdb<\xab`\x87\x10\x87\xce(v\xf3\vw\x1enM\x8f\xee\xd6\b\xfeϹJY\x91&%\xe1Cٺ\x9c\xc0\xf3\xcbEދ\x83iv^ImeJl\a\xb4)\xb9I\xdfdM{\xb2\xa5\xad\xf6\x81\xc1\xd4L\xda\x0f1\xc9\x16_\xc8e\x9cH\x85\xcdlN\xa5.\x7f\x9fM},哣\x11\x9co\xce8ݧ;s\xff\x8e\xf6h\xcf\xd6a\x81(S\bߧ\x92\x16\xfa\xd8/}6\xf0\t\x9f+\xbb\xf7\x1c҄\xc6\xf9\xa8\xb9Z\x1bP\x1e\xb1\x03\xf9E\xb8\xf3\xc8ʭ\xfc0.G~\x0eh\x04\x02\x8eާ\x96\x0e~\x01Yy\x11\x16wH\xb1\xaf\xb0\xa9\xf2\xb9\xf3\xfb\x90\xff\"Lrl\xb4\xb4\x13\x8e\xc9\x1e\xfd\x14^\x15\xc0\xeb\xb9.k9\xe7>$hY\xf9y\xfeo\xc6i.\x11c\xd5w\x93YU\x0f\x92ǚ\xe2\xf5\xfe\x1aYF\xe7\xcc\xce\xe1\x06\x94\xe3Һ\xd8\x1afs\x9eW\xcdTj\x9f\xa9GQ\xd3\x0f\xef\x14\xd0\xe2UH\xeb~\x81\x92\x9a\xe7\xf9\x88\xfeZ\x8b\xc0\xb3\x91W\xe7\x15\xc8\xdd\xf9\x9a\xe9\xed\xcb\xdf\xe6\xb2\xcfJ=o \xcd\xfaF\xa9\"䇔\xaa\xa6\xb4\xd4y\xf5\xb7f\xa1\xd2\xf6\xf2\xee4H\xde\xf4\xcb\xf4W\xb3\x8c\xe1*\x85j\x05,63|w\x11\x9eh`s\x98\x02\xfe'\x00\x00\xff\xff\xef\xf8\xa6>\x10\f\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcVM\x8f\xdb6\x10\xbd\xfbW\f\x92kd7(Z\x14\xbe\x05\xdb\x1e\x82&\xc5\"N\xf7N\x93#{j\x8ad\x87C9.\xfa\xe3\v\x92Ү-\xcb\xc9nQT\x17\xc3\xe4\xf0q>\u07bca\xd34\v\x15\xe8\x019\x92wkP\x81\xf0\x8b\xa0\xcb\xff\xe2\xf2\xf0S\\\x92_\xf5o\x17\arf\rw)\x8a\xef>a\xf4\x895\xfe\x8c-9\x12\xf2nѡ(\xa3D\xad\x17\x00\xca9/*/\xc7\xfc\x17@{'\xec\xadEnv薇\xb4\xc5m\"k\x90\v\xf8xu\xff\xdd\xf2\xed\x8f\xcb\x1f\x16\x00Nu\xb8\x86\xde\xdb\xd4at*Ľ\x17\xebu\xc5\\\xf6h\x91\xfd\x92\xfc\"\x06\xd4\xf9\x8a\x1d\xfb\x14\xd6\xf0\xb4Q!\x86\xeb\xab\xeb\x0f\x05m3\xa0}\x18Њ\x81\xa5(\xbf~\xc5\xe8\x03E)\x86\xc1&V\xf6\xa6g\xc5&\xee=\xcboO\xb77\xd0G[w\xc8\xed\x92U|\xeb\xfc\x02 j\x1fp\r\xe5xP\x1a\xcd\x02`\xc8O\x81k\xc6Լ\xad\x88z\x8f\x9d\xaa\xf7\x00\xf8\x80\xee\xdd\xfd\xfb\x87\xef7\x17\xcb\x00\x06\xa3f\nR\xb2<\x1f\"P\x04\x05\xa3'p\xdc##<\x94|B\x14\xcf\x18\a\xa7\x1fA\x01F\xff\xe3\xf2q1\xb0\x0f\xc8Bc\xf0\xf5;\xe3\xd7\xd9\xeaį\xbf\x9b\x8b=\x80\x1cJ=\x05&\x13\r#\xc8\x1e\xc7t\xa0\x19\xa2\a߂\xec)\x02c`\x8c\xe8*\xf5\xf2\xb2r\xe0\xb7\x7f\xa0\x96\xe5\x04z\x83\x9car\xad\x925\x99\x9f=\xb2\x00\xa3\xf6;G\x7f=bG\x10_.\xb5J0\n\x90\x13d\xa7,\xf4\xca&|\x03ʙ\tr\xa7N\xc0\x98\xef\x84\xe4\xce\xf0ʁ8\xf5\xe3\xa3g\x04r\xad_\xc3^$\xc4\xf5j\xb5#\x19\xbbN\xfb\xaeK\x8e\xe4\xb4*\rD\xdb$\x9e\xe3\xca`\x8fv\x15i\xd7(\xd6{\x12Ԓ\x18W*PS\x02q\xb5K:\xf3\x9a\x87>\x8d\x17\xd7\xca)S,\n\x93\u06ddm\x94.yAyr\xc3T\xd6T\xa8\x1a\xe2S\x15\xf2RNݧ_6\x9fa\xf4\xa4V\xaa\x16\xe5\xc9\xf4*/c}r6ɵ\xc8\xf5\\˾+\x98\xe8L\xf0\xe4\xa4\xfcі\xd0\tĴ\xedH2\r\xfeL\x18%\x97n\n{W\x94\t\xb6\b)\x18%h\xa6\x06\xef\x1dܩ\x0e흊\xf8?\xd7*W%6\xb9\bϪֹ\xdeN\x8dkz\xcf\x1bu\x90\xc9\x1b\xa5\x9dW\x84M@}\xd1x\x19\x85Z\x1a\x14\xa2\xf5i\x8b\x15\x10|;ý\x17\xb9\x9c?t\xa9\x9b#\xe2\xbb^\x91U[{-\t\r\xfc\xee\xd4\xcdݛş\xad\xe7\xd5b̏=\xb3\x06\xe1T\xb1\a\x96\r+\xff\x04\x00\x00\xff\xffNy\xc1Q\xa1\x0e\x00\x00"), } var CRDs = crds() func crds() []*apiextv1.CustomResourceDefinition { apiextinstall.Install(scheme.Scheme) decode := scheme.Codecs.UniversalDeserializer().Decode var objs []*apiextv1.CustomResourceDefinition for _, crd := range rawCRDs { gzr, err := gzip.NewReader(bytes.NewReader(crd)) if err != nil { panic(err) } bytes, err := io.ReadAll(gzr) if err != nil { panic(err) } gzr.Close() obj, _, err := decode(bytes, nil, nil) if err != nil { panic(err) } objs = append(objs, obj.(*apiextv1.CustomResourceDefinition)) } return objs } ================================================ FILE: config/crd/v1/crds/doc.go ================================================ // Package crds embeds the controller-tools generated CRD manifests package crds //go:generate go run ../../../../hack/crd-gen/v1/main.go ================================================ FILE: config/crd/v2alpha1/bases/velero.io_datadownloads.yaml ================================================ --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.16.5 name: datadownloads.velero.io spec: group: velero.io names: kind: DataDownload listKind: DataDownloadList plural: datadownloads singular: datadownload scope: Namespaced versions: - additionalPrinterColumns: - description: DataDownload status such as New/InProgress jsonPath: .status.phase name: Status type: string - description: Time duration since this DataDownload was started jsonPath: .status.startTimestamp name: Started type: date - description: Completed bytes format: int64 jsonPath: .status.progress.bytesDone name: Bytes Done type: integer - description: Total bytes format: int64 jsonPath: .status.progress.totalBytes name: Total Bytes type: integer - description: Name of the Backup Storage Location where the backup data is stored jsonPath: .spec.backupStorageLocation name: Storage Location type: string - description: Time duration since this DataDownload was created jsonPath: .metadata.creationTimestamp name: Age type: date - description: Name of the node where the DataDownload is processed jsonPath: .status.node name: Node type: string name: v2alpha1 schema: openAPIV3Schema: description: DataDownload acts as the protocol between data mover plugins and data mover controller for the datamover restore operation properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: DataDownloadSpec is the specification for a DataDownload. properties: backupStorageLocation: description: |- BackupStorageLocation is the name of the backup storage location where the backup repository is stored. type: string cancel: description: |- Cancel indicates request to cancel the ongoing DataDownload. It can be set when the DataDownload is in InProgress phase type: boolean dataMoverConfig: additionalProperties: type: string description: DataMoverConfig is for data-mover-specific configuration fields. type: object datamover: description: |- DataMover specifies the data mover to be used by the backup. If DataMover is "" or "velero", the built-in data mover will be used. type: string nodeOS: description: NodeOS is OS of the node where the DataDownload is processed. enum: - auto - linux - windows type: string operationTimeout: description: |- OperationTimeout specifies the time used to wait internal operations, before returning error as timeout. type: string snapshotID: description: SnapshotID is the ID of the Velero backup snapshot to be restored from. type: string snapshotSize: description: SnapshotSize is the logical size in Bytes of the snapshot. format: int64 type: integer sourceNamespace: description: |- SourceNamespace is the original namespace where the volume is backed up from. It may be different from SourcePVC's namespace if namespace is remapped during restore. type: string targetVolume: description: TargetVolume is the information of the target PVC and PV. properties: namespace: description: Namespace is the target namespace type: string pv: description: PV is the name of the target PV that is created by Velero restore type: string pvc: description: PVC is the name of the target PVC that is created by Velero restore type: string required: - namespace - pv - pvc type: object required: - backupStorageLocation - operationTimeout - snapshotID - sourceNamespace - targetVolume type: object status: description: DataDownloadStatus is the current status of a DataDownload. properties: acceptedByNode: description: Node is name of the node where the DataUpload is prepared. type: string acceptedTimestamp: description: |- AcceptedTimestamp records the time the DataUpload is to be prepared. The server's time is used for AcceptedTimestamp format: date-time nullable: true type: string completionTimestamp: description: |- CompletionTimestamp records the time a restore was completed. Completion time is recorded even on failed restores. The server's time is used for CompletionTimestamps format: date-time nullable: true type: string message: description: Message is a message about the DataDownload's status. type: string node: description: Node is name of the node where the DataDownload is processed. type: string phase: description: Phase is the current state of the DataDownload. enum: - New - Accepted - Prepared - InProgress - Canceling - Canceled - Completed - Failed type: string progress: description: |- Progress holds the total number of bytes of the snapshot and the current number of restored bytes. This can be used to display progress information about the restore operation. properties: bytesDone: format: int64 type: integer totalBytes: format: int64 type: integer type: object startTimestamp: description: |- StartTimestamp records the time a restore was started. The server's time is used for StartTimestamps format: date-time nullable: true type: string type: object type: object served: true storage: true subresources: {} ================================================ FILE: config/crd/v2alpha1/bases/velero.io_datauploads.yaml ================================================ --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.16.5 name: datauploads.velero.io spec: group: velero.io names: kind: DataUpload listKind: DataUploadList plural: datauploads singular: dataupload scope: Namespaced versions: - additionalPrinterColumns: - description: DataUpload status such as New/InProgress jsonPath: .status.phase name: Status type: string - description: Time duration since this DataUpload was started jsonPath: .status.startTimestamp name: Started type: date - description: Completed bytes format: int64 jsonPath: .status.progress.bytesDone name: Bytes Done type: integer - description: Total bytes format: int64 jsonPath: .status.progress.totalBytes name: Total Bytes type: integer - description: Incremental bytes format: int64 jsonPath: .status.incrementalBytes name: Incremental Bytes priority: 10 type: integer - description: Name of the Backup Storage Location where this backup should be stored jsonPath: .spec.backupStorageLocation name: Storage Location type: string - description: Time duration since this DataUpload was created jsonPath: .metadata.creationTimestamp name: Age type: date - description: Name of the node where the DataUpload is processed jsonPath: .status.node name: Node type: string name: v2alpha1 schema: openAPIV3Schema: description: DataUpload acts as the protocol between data mover plugins and data mover controller for the datamover backup operation properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: DataUploadSpec is the specification for a DataUpload. properties: backupStorageLocation: description: |- BackupStorageLocation is the name of the backup storage location where the backup repository is stored. type: string cancel: description: |- Cancel indicates request to cancel the ongoing DataUpload. It can be set when the DataUpload is in InProgress phase type: boolean csiSnapshot: description: If SnapshotType is CSI, CSISnapshot provides the information of the CSI snapshot. nullable: true properties: driver: description: Driver is the driver used by the VolumeSnapshotContent type: string snapshotClass: description: SnapshotClass is the name of the snapshot class that the volume snapshot is created with type: string storageClass: description: StorageClass is the name of the storage class of the PVC that the volume snapshot is created from type: string volumeSnapshot: description: VolumeSnapshot is the name of the volume snapshot to be backed up type: string required: - storageClass - volumeSnapshot type: object dataMoverConfig: additionalProperties: type: string description: DataMoverConfig is for data-mover-specific configuration fields. nullable: true type: object datamover: description: |- DataMover specifies the data mover to be used by the backup. If DataMover is "" or "velero", the built-in data mover will be used. type: string operationTimeout: description: |- OperationTimeout specifies the time used to wait internal operations, before returning error as timeout. type: string snapshotType: description: SnapshotType is the type of the snapshot to be backed up. type: string sourceNamespace: description: |- SourceNamespace is the original namespace where the volume is backed up from. It is the same namespace for SourcePVC and CSI namespaced objects. type: string sourcePVC: description: SourcePVC is the name of the PVC which the snapshot is taken for. type: string required: - backupStorageLocation - operationTimeout - snapshotType - sourceNamespace - sourcePVC type: object status: description: DataUploadStatus is the current status of a DataUpload. properties: acceptedByNode: description: AcceptedByNode is name of the node where the DataUpload is prepared. type: string acceptedTimestamp: description: |- AcceptedTimestamp records the time the DataUpload is to be prepared. The server's time is used for AcceptedTimestamp format: date-time nullable: true type: string completionTimestamp: description: |- CompletionTimestamp records the time a backup was completed. Completion time is recorded even on failed backups. Completion time is recorded before uploading the backup object. The server's time is used for CompletionTimestamps format: date-time nullable: true type: string dataMoverResult: additionalProperties: type: string description: DataMoverResult stores data-mover-specific information as a result of the DataUpload. nullable: true type: object incrementalBytes: description: IncrementalBytes holds the number of bytes new or changed since the last backup format: int64 type: integer message: description: Message is a message about the DataUpload's status. type: string node: description: Node is name of the node where the DataUpload is processed. type: string nodeOS: description: NodeOS is OS of the node where the DataUpload is processed. enum: - auto - linux - windows type: string path: description: Path is the full path of the snapshot volume being backed up. type: string phase: description: Phase is the current state of the DataUpload. enum: - New - Accepted - Prepared - InProgress - Canceling - Canceled - Completed - Failed type: string progress: description: |- Progress holds the total number of bytes of the volume and the current number of backed up bytes. This can be used to display progress information about the backup operation. properties: bytesDone: format: int64 type: integer totalBytes: format: int64 type: integer type: object snapshotID: description: SnapshotID is the identifier for the snapshot in the backup repository. type: string startTimestamp: description: |- StartTimestamp records the time a backup was started. Separate from CreationTimestamp, since that value changes on restores. The server's time is used for StartTimestamps format: date-time nullable: true type: string type: object type: object served: true storage: true subresources: {} ================================================ FILE: config/crd/v2alpha1/crds/crds.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ // Code generated by crds_generate.go; DO NOT EDIT. package crds import ( "bytes" "compress/gzip" "io" apiextinstall "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/client-go/kubernetes/scheme" ) var rawCRDs = [][]byte{ []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcYK\x93\xe3\xb8\r\xbe\xf7\xaf@M\x0es\x19\xbb3yl\xa5|\x9bq'U]\xd9\xe9q\xad;}\xa7$X\xe6\x0eE2|\xd8\xebM\xf2\xdfS %\x99\x92\xe8\xe7>|3\t\x82\x1f\x01\x10\xf8@\xcdf\xb3\a\xa6\xf9\x1b\x1a˕\\\x00\xd3\x1c\x7fr(韝\x7f\xfb\x9b\x9ds\xf5\xb8\xfb\xf8\xf0\x8d\xcbj\x01Ko\x9dj~@\xab\xbc)\xf1\t7\\rǕ|hб\x8a9\xb6x\x00`R*\xc7h\xd8\xd2_\x80RIg\x94\x10hf5\xca\xf97_`Ṩ\xd0\x04\xe5\xddֻ?\xce?~7\xff\xeb\x03\x80d\r.\x80\xf4Uj/\x85b\x95\x9d\xefP\xa0Qs\xae\x1e\xacƒ\x14\xd7Fy\xbd\x80\xe3D\\\xd8n\x1a\x01?1ǞZ\x1daXp\xeb\xfe9\x99\xfa\x9e[\x17\xa6\xb5\xf0\x86\x89\xd1\xdea\xc6rY{\xc1\xccp\xee\x01\xc0\x96J\xe3\x02^hk\xcdJ\xa4\xb1\xf6L\x01\xca\fXU\x05+1\xb12\\:4K%|\xd3Yg\x06\x15\xda\xd2p\xed\x82\x15RX`\x1dsނ\xf5\xe5\x16\x98\x85\x17\xdc?>˕Q\xb5A\x1ba\x01\xfch\x95\\1\xb7]\xc0<\x8a\xcf\xf5\x96Ylg\xa3)\xd7a\xa2\x1dr\a\xc2k\x9d\xe1\xb2\xce!x\xe5\rB\xe5Mp!\x9d\xbbDp[n\x87\xd0\xf6\xcc\x12<\xe3\xb0:\t$̓:\xebX\xa3Lj\x92\xa5\x11R\xc5\x1c\xe6\x00-U\xa3\x05:\xac\xa088쎱Q\xa6an\x01\\\xba\xef\xfer\xda\x16\xad\xb1\xe6a铒C\xc3|\xa6QH\x86#\x12\xf2R\x8d&k\x1d\xe5\x98\xf8%@\x1c)\xf8\x9c\xac\x8fH\xa2\xdet\xfc\"\x14\n9P\x1bp[\x84Ϭ\xfc\xe65\xac\x9d2\xacF\xf8^\x95\xd1}\xfb-\x1a\f\x12E\x94\xa0\xe8\x05N\xbeS&\xeb:\x8d\xe5<ʶ\xca:]#\xff\r7\xfa\xd5c\xab4Ȳ\xb1ե\x9ay\x90\xe0J\xe6\x03\xecS\x8dW\x05WjD\xa9*L,6\xc0\xc4-h\xa3J\xb4\xf6L\xc0\x93\x82\x01\x8a\x97\xe3\xc0\xc44Qb\xf7'&\xf4\x96}\x8cI\xa6\xdcb\xc3\x16\xed\n\xa5Q~Z=\xbf\xfdy=\x18\x863\t\x83\x95\xceR\xa6 \xf8\xda(\xa7J%\xa0@\xb7G\x94\xd1\xf5\x8dڡ\xa1\x9a\x8d\x93\x06C\xf4\x10@\x93z\x1fhO\x8d\xc6\xf1.\v\xb7\xba\x8f\x05&\x19\x1d\x9d㿳\xc1\x1c\x00\x1d=\xae\x82\x8a*\r\xc6c\xb5\xb9\x15\xab\xd6Z\xd1y܂AmТ\x8c\xb5\x87\x86\x99\x04U\xfc\x88\xa5\x9b\x8fT\xafѐ\x1a\xb0[\xe5EE\x87ݡq`\xb0T\xb5\xe4?\xf7\xba-8\x156\x15̡u\xe12\x1a\xc9\x04\xec\x98\xf0\xf8\x81\x8c6\xd2ܰ\x03\x18\xa4=\xc1\xcbD_X`\xc78\xbe\x90\x15\xb9ܨ\x05l\x9d\xd3v\xf1\xf8Xsו\xddR5\x8d\x97\xdc\x1d\x1e\x837x\xe1\x9d2\xf6\xb1\xc2\x1d\x8aG\xcb\xeb\x193\xe5\x96;,\x9d7\xf8\xc84\x9f\x85\x83\xc8Pz\xe7M\xf5\a\xd3\x16j;\xd8v\x12\x88\xf1\x17\n\xe6\r\xee\xa1*J\xb7\x82\xb5\xaa\xe2\x11\x8f^\xa0!2\xdd\x0f\x7f_\xbfB\x87$z*:\xe5(:\xb1K\xe7\x1f\xb2&\x97\x1b4q\xddƨ&\xe8DYiť\v\x7fJ\xc1Q:\xb0\xbeh\xb8\xa30\xf8\xb7G\xeb\xc8uc\xb5\xcb@M\xa0@\xf0\x9a\xf2A5\x16x\x96\xb0d\r\x8a%\xb3\xf8;\xfb\x8a\xbcbg䄫\xbc\x95\x12\xae\xb1p4o2\xd11\xa6\x13\xaeM3\xc8ZcI^%\xc3\xd22\xbe\xe1m%\xa14\xc0\x06\xb2C\v\xe5\xaf>\xfd\xb2\xd5d,t)\xdc\xe8\xf79\xa7\xa8C+\x93D\xde\xd6:\xdb\x16)1,R\xe9oR\x1f\rje\xb9S\xe6p\xac\x92\xe3P8\xe9\x15\xfa\x95L\x96(\xee9\xde2\xac\x04.+\xb29\xf6\xa1LI(j\r@\x95\xac\x15]\xae\x81+\xe0ّ\fŶE\x97?\xa8\xccV5.\xe1\xc8)!\xe5\x8e\xe3\xe3\x16J\tdc+R\x14~\xa1\xb2\xb0Tr\xc3\xeb\xe9\xc1S\xfa{*D.\xd84\x13\xb0ɖt\n\x8aNB2\v\x15jօ.\xa5\xf6\r\xaf\xbd9\xe5\xff\rGQM\xf2\xcfɛ\xd4\x1d8\xecr\x8f\x8f{\xe8\xdd\xedj\xabZRz\x9d\n\x19\xca\x06\xbe\x9b\x84\xe6\x14$\xc0\xf3&\xd1\xc8-\xbc{\a\xca\xc0\xbb\xd8\x13\xbd\xfb\x10W{.܌\x0f\xea\xff\x9e\v\xd1\xedrSt\x13\xc3\xf9\xba\xbep\xf2\x97 Dx\xbe\xaeo\xe5VS4(}3\xddp\x06\xcc;\x95\x19\x16\\\xfa\x9f2\xe3{.+\xb5\xb7\xb7\x1c\xb6\xe77D1\x95w\xf78\xfc\xebH\xc7\xc8\xef\x8e\bq\xf0\xb5S\xb0g<\xe1\x18\xfd\xee\xf6CFo\x81\x1b*H\x06\x9d7\x92\xd2\x01\x1aC\x19\xda\x06\x95\xcaO8\xcfٓZɴ\xdd*\xf7\xfct\xe1\x8c\xeb^\xb0˻\xcfO\x9d\x8b\xdfB\xd4\xf5ɷ\x95\x84\x8c\x97\b~\xc7\"\xabP\xd6\xefB\xbb\xe6?\xe3\x95xI\xb4C,T\xcdK&\xc0\x861\xd96\x81\xed!:\xddS@\xb9>o\f7\xed\xd6\x12\xbc\x81\xfb\xf4/\x04\xf7\x84\xd1z\xa8\xa2;\x8a2\xbc\xe6\x14,\xb2\x9f9ޱ\x9d\x12\xbe\t\xa2\xe4\x12\xac\xc0\xeb\x13\xb6\x06*\x1fD\xb6\n\x84\x8ao6h\x88Q\x05\xba\x157^\xbd-\xdf\xdbd\x13\xbeI\xffP\xa5j\x98\xd6XQoG\xc1\xd8\xfa\xf6&\xaf:fjto\x01\xf4\x05\x13\xbd&\xa2\x9d)\x88\x9a\x91\x83Z\xee\x1f.W\x10\x83\xd5\xdb2\xc3\xd4\xe9\xb7z\x9b\"<\xcdc\xa0m\xdaN8q\x82r\xe2\xad\x16O\xaf#\xab\xe2l\x19\x04л+v^\xbd\xe5XQo\x0ep[\xe6H\xa2m\xb2\xa18duBw\xa5[wއ\xb7\xbc\n\xf0\xf2,\xe2\xe5\x18\xf2\t\xbc\xc5\xe1\x17C&\xd2\xc5\rV\xb9\x92s\xdas3л\xec`y=\xb5\xc8\xef<\xcb\xf3\xe7\x91̸T\x8d\xa6\x8f\xf9}<1\xcc+\xa3\xd9\xf4J^\xd5h\x84g\x90k[\x8d\xf8\xb8ٺ\xbd\xf4&$\x9d\xf6ɓ\xba\xf7\xbb\x9a\rV\x96\xa8\x1dV\x9f\x0f\xc4B\xae *\x04@\x9e\x7f\x04\xfa\x97>\xd2\x14\xd4\xec֎\xa0\x83\xd4?T\xddS\x00>\x8d\x95\x84\xd7\nS%4b\n7R\xc9Ӡ\x01^\xa9\xe4\x85n\xfb}d\x0e\xb4,\xf0\x11bԓMO\x16Ej\xa7g\xb4~\"!\xbd\x10\xac\x10\xb8\x00g\xfc\xa9\xd6\"\xdfI\xc5w\xdf\xf4\x89﮶j\xaafj;\xd6?j\x85\xc7\xc7\xee\xc59g\xb2\xa3\xbe\xde`Q\x1dV\x80;\x94@\xcd2\xe3\x02\xabNg\xa6\xbf\xb8d\xf9\f\xe8)u\xfd-\x8dߠ\xb5\xac\xbet\x81\xbeD\xa9\xf8\x0e\xd4.\x01V\x10\xcf\x1d\xb3\xfc\xf7\xb6\xbd\xdb7\xf7\x1b\xbf\xce%\xbe\xb2\xdb8\x83%\xf4\xc6\x17\xc0\xacH&\x97\xd3zh\xa7\x93\x1a\x9civ^p\x9f\x19\xed\xeegfj\xd5^\xfa\xcc\xd4\xe4\x13R:\x19\x1f!r\x85\xb1\x9b\xcb\xea\xec\xbf\xd1d\xe6\xfe\x11.\xc3M\x96n\xf1\xdds\xdd\xfb\xa7\x8c\xad\x12\xdd\r\x0f\xdfV\xa4o\n4\xe4\x86\"G\xf8\xc3\vx\xe2\xb5\x1c\xf9\xeb5\xf4\xbdKP5\x87\xd7-Q\x93\xf8\xfe\xd2us\x15\xb7Z\xb0C\x7f\x98\x94\xa1f\x94\x1fo\xcd\xe4y\xfdV\x92\xda\x7f\xeb\xca3\xaf\xf3\x8d\f\\hf\xc2|\xff\r\xeb\xb7\xd9\xe1\xcc\xeb\xcb\xf0\x9b\xe2]\xad\xd4@åR\xd0~\xe3\xbc=\x83\x0f\xb7\xf9=\x93w\xd6z\x93\xc1\x80\xbcJt\xb7\xaf\xa5\xe9\x88/\xfaO\b\v\xf8\xcf\xff\x1e\xfe\x1f\x00\x00\xff\xff73Hq. \x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcZIs\xe3\xb8\x15\xbe\xfbW\xbc\xea\x1c\xe6Ғ\xa7\xb3L\xa5tk\xcbI\x95*3nW\xcb\xf1\x1d\"\x9fD\x8cA\x80\xc1\"\x8d\xb3\xfc\xf7\xd4\x03\b\n$!Q\xd2\xf4\f\x0f]-,\x0fo\xc3\xf7\x16x6\x9bݱ\x86\xbf\xa26\\\xc9\x05\xb0\x86\xe3/\x16%\xfd2\U000f7fda9W\xf7\xfbOwo\\\x96\vX:cU\xfd\x15\x8dr\xba\xc0G\xdcr\xc9-W\xf2\xaeF\xcbJf\xd9\xe2\x0e\x80I\xa9,\xa3aC?\x01\n%\xadVB\xa0\x9e\xedP\xce\xdf\xdc\x067\x8e\x8b\x12\xb5'\x1e\x8f\xde\x7f?\xff\xf4\xc3\xfc/w\x00\x92ո\x00\xa2\xe7\x1a\xa1Xi\xe6{\x14\xa8՜\xab;\xd3`AdwZ\xb9f\x01lj\xb0\xad=2\xb0\xfb\xc8,\xfb\xa7\xa7\xe0\a\x057\xf6\x1f\x83\x89\x1f\xb9\xb1~\xb2\x11N3\xd1;Տ\x1b.wN0\x9d\xce\xdc\x01\x98B5\xb8\x80':\xb2a\x05\xd2X+\x89ga\x06\xac,\xbdn\x98x\xd6\\Z\xd4K%\\\x1du2\x83\x12M\xa1yc\xbd\xecG\x86\xc0Xf\x9d\x01\xe3\x8a\n\x98\x81'<ܯ\xe4\xb3V;\x8d&\xb0\x04\xf0\xb3Q\xf2\x99\xd9j\x01\xf3\xb0|\xdeT\xcc`;\x1bԷ\xf6\x13\xed\x90}'n\x8d\xd5\\\xeer\xe7\xbf\xf0\x1a\xa1tڛ\x8dd.\x10l\xc5M\xca\u0601\x19bN[,O\xb2\xe1牘\xb1\xacn\x86\xfc$[\x03C%\xb3\x98cg\xa9\xeaF\xa0\xc5\x126\xef\x16\xa3\x10[\xa5kf\x17\xc0\xa5\xfd\xe1ϧ5Ѫj\xee\xb7>*\xd9W\xcb\x03\x8dB2\x1c8!\v\xedPgu\xa3,\x13\xbf\x86\x11K\x04\x1e\x92\xfd\x81\x93@7\x1d\x9fde%\v\x8d5\xca\xdb\x18\xe2\xc7\xddcnR\xd2\xe9l\xa3\xb9\xd2ܾ/\xe0\xd3\xf7\x97\xb2I\xb7\x02\xd4\x16l\x85\xf0\xc0\x8a7\xd7\xc0\xda*\xcdv\b?\xaa\"\xf8ءB\xdd\xfa\xd8&,1\x95r\xa2\x84M4\f\x80\xb1Jg\x9d\xad\xc1b\x1ev\xb5t#ف\xc7\xf5\xcf\xfc\xc6w\xa1\xd0Ȳw!\x82\xe1ܯ\xe0J\xe6/\xc4\xe7\x1d^t\x19RmJUb\xa7:L9\xe2\x06\x1a\xad\n4\xe6\xcc\xf5\xa4\xed=\x1e\x9e\x8e\x03#\xb5\x84\x15\xfb?2\xd1T\xecS\x00â\u009a-\xda\x1d\xaaA\xf9\xf9y\xf5\xfa\xa7uo\x18NB\x1b+\xac!L#\xd6\x1b\xad\xac*\x94\x80\r\xda\x03\xa2\xf4\xf0\n\xb5ڣ&,\xdeqi\x80ɲ\xa3\t\xe9\x82cD!\xd7\xf7\xf4h6L\xb6\xee\xa4\x1aԩ\xd9ɕi\xcc\xf2\x18$\u0097D\xbfdt \xc4\x7fg\xbd9\x00\x92;삒\xc2 \x06\xa9\xda\x10\x80e\xab\xaa`7n@c\xa3\xd1\xd0\xf5\xf2^\xa5\xb6\xc0$\xa8\xcd\xcfX\xd8\xf9\x80\xf4\x1a5\x91\x89\xf7\xa1Pr\x8fڂ\xc6B\xed$\xffwGۀU\xfeP\xc1,\x1a\xeb/\xa4\x96L\xc0\x9e\t\x87\x1f\aڣ\xaff\uf811\xce\x04'\x13z~\x83\x19\xf2\xf1\x93\xd2\b\\n\xd5\x02*k\x1b\xb3\xb8\xbf\xdfq\x1bs\x82Bյ\x93ܾ\xdf{c\xf0\x8d\xb3J\x9b\xfb\x12\xf7(\xee\r\xdf͘.*n\xb1\xb0N\xe3=k\xf8\xcc\v\"}^0\xaf\xcb?\xe86\x8b0\xbdcG^\x18>\x1fϯ0\x0f\x85y\xba\x12\xac%\x15D\xa4M.\xb7\xa8þ\xadV\xb5\xa7\x89\xb2l\x14\x97\xd6\xff(\x04Gi\xc1\xb8M\xcd-\xb9\xc1\xbf\x1c\x1aK\xa6\x1b\x92]\xfa\xbc\t6\b\xae!((\x87\vV\x12\x96\xacF\xb1d\x06\x7fg[\x91Ǔ\x8cp\x91\xb5\xd2lp\xb88\xa87\x99\x88\t\xdd\t\xd3\x1e\xe1c\xdd`A6%\xb5\xd2&\xbe\xe5m,!\f`\xc9ʾv\xf2מ\xbel\b\x19.\x9ar5\xfa\x1er\x84\"\xaf2\xc1\xef\x18\xea\xda\xc8$\xfa\x91)\xfd\x8e \xdf\xee\xd1\xd8(í\xd2\xefD8\x84ơ\x1b\x9c\xb4\b}\x05\x93\x05\x8a[\xc4[\xfa\x9d\xc0eI\x1a\xc7\u038d\t\x80\x02UϨ\x92;E\x17+1\x04\xac,\xad \xaf6h\xf3b\xcaL(\xe3\x12\x8eI/\xa4\xc9\xedPԍR\x02\xd9P\x83\x85\xe1k\xc9\x1aS);!\xf0j\vq\xe5\xcb{\x83t\xf8r\xbd\xfaH\xff\xc4q\xf2\xa0=/[\x88\xa7[F\xd9V\xdel\xad\x9d\x97\xeb\x15\x98v\xfb\xd8H\xd2\t\xc16\x02\x17`\xb5\x1b\vv\xdaa=\xf7\x9a\xefQ\xe7f\x867\xc7/\x8c^\x18\xb6\x813>\xa9\xf6C\xafT\x90`\x94r\xa9\xa4E\x99\xb3\xd1Y\xaf\xa2/J\xba\x14\xccdy\x1ep\xb6N\xd7\xe7\xaeI$\b\x85_a+\x96\xe7\vB\xd0\xf5r\x1c7\xf1.7\x83\x03\xb7\xd5M\x12\x85\vz\xb1@\xc9\xf2\xac<\xed}\x0f\xe2\xa8\xed\x19a\x9e_\x97^\xde)\xc9(\xdc\xdc\"پg\xf4\vd\xeb{IN\xba\x01\x97\xa7\x84S\x84\x02\x04fX\x82k\xae\xe7\x9d@\x87k,\xc7<\xcfz\xf6\xcaL\xf7\x85>\x81$\xa3\xc8\x04m\xd2\xf9\x13\xa5\x95K%\xb7|7>;-\xf3\xcf]۳\xa2\x8d\"^r$i\x9c\x02\x1cq2\xf3\x19\xee,F?\xca\r\xb7|\xe7\xf4)4\xdar\x14\xe5(\x81\x99\x04\xa0\t}x&n\x89#\x9dd1~\xb7\x90\x9ad\xf6\xc1KR\x94\n\xe1o,\x03\x10t\x1f)r\x03\x1f>\x80\xd2\xf0!\xb4\x84>|\f\xbb\x1d\x17v\xc6{\xe5Ł\v\x11O\xb9*\x82v%\x05\x15t\xcaM\x85\x96\xac\x0e\xbe\fh\fTa\xa9\xf8\xf4\xe2[\x05\aƓ\xb4\xbe;\xdd|\xcc\xd0\xdd\xe0\x96r@\x8d\xd6iIQ\x18\xb5\xa6\xb4\xc8x\x92\xcae\xc2\xd0\x19IM\x12\x12'\xa4\x1cFO/\x05\xfd\x7f\x88\xe5)\x00d\x04\xc8\xd9\xf8\x1c\x87>e\xef\xfao\xb7\x98b\xdd'\x11\x99W\x9a\xef8)\\v3\xc7d\xacź\xb6k\xe1\x91\xccCq\xd6?;\xb44\x84\x96Grt\x9d\xc3\xe1\x84\xf6L\x96>_\xe8\xe6\xcb\xf6\xeae.\xee\xa4B\x9e_\x97S\xf6\xea\x0e\xce@9\r\x1f*^T}\xd3\xf11\xa8\x02X\xf6\x86>\xf7\xbe\x82\xcd<\x86\xcf\xf2\x99\xf8`\xcd\xf0\xf6\r\xa6S\x97\x1dN\xf5\r\x9d\x9d}~]^T\xad\xf8F\xcae\xf5Jh\xe4\xb6Z.\x9c־\x12\f\xa3j{S\xc5\u008a\x02\x1b\x8b\xe5\xc3\xfb\x93*\xa7\x9c\xfeso11\"/i%eL\xed\x9bKذkK\x8e\xc8n\xd7\x00\xbb\xe5\x9a~\x1e\x12\xf1\xad\x10]&\x809. \x02\u061cf\x1a\xe0\x85\x1cܗ\xf2\xdf\x05\x8c\xa4m\x1ey\xe9z\x8e\x0e\x1dQ\x88=W\xaa\xd5g\xb4\xff\xb6(\x9b/\xd5B\xff;m\x1d\xdeT\xb7\x8dɌu\xc7b\x81\xe9{\x9a\xb1\xf1\x9e\xd3ؑ\\\xa7\xaf@\rK\xc0=J\xa0R\x9cqA\xb1ۓ\xcc\x00\xd8y*m\x10\v\xaf,\xb1G\x13\xfby\xd9fٴ%3J\x18\xa3\xd9oi\xcc.\x85\xfc\x8aƉL\xd2\xf0\x1b\xa6\x90\xe1\xc8\xd0-0\xd9\x14\xf2|9\xcb\f0ЁH\x8b\x1b\xa7@\xebb%e\xf3\xca\xe1\xdb\xc4T\xd5>X\x0e\x95\x12\xadSKWoP\x13\xb7\xfe\x85\x04$\x1e(-,*&w\xd9\xc4#v\xf8\x11\x043\xb6u\xb7\x93\x1e\x92>\xb1\f%K\x9fD\x8e_\x8dư\xdd\x14X\xff\x14V\x85\xa6e\xbb\x05؆2ľֿ3m\f\xb9\n\x89\xe5t\xb8\xb8*H\xf4\xde\x1b\xae\xe6\xe4\xcb\xfa\x02^\xbe\xac\xe9\x90/\xeb_\xcb\vJW\xe7jF\xe6\xac\xca\f\v.\xdd/\x99\xf1\x03\x97\xa5:\x8c\xa1㌨\r\xb3Մ\xa0\xcf\xccV1E\xd8:!\xfc\x9eQ\xea\xdcf\x9d\x1b$L\xfcV\x19\xb4\xef\xaaM\xb1Gkr)\f^\x02\a\xa74\xff\x84\x87\xcch\f\xb9\x99\xa9\xe76\x8eg\xa6Fo\xe3\xe9dh\\\xe6\xe02\xceeiv\xcfϙ\xb9\xbf\xfb\x00w\x95\x9e[\xfen\x89\xe0]\v\xf4\x88o\xfe5y\x84r\xfdV\f\x95\x14\x89\xc52\x84\x93\xfd]\x1d\xe3)\xcd\xe1\xa5\xe2&6mc%Zr\xd3\b\xf6\xde\xc92\x156:\xdc\x1a>ƍ\x9d\xe4|\xb7\xb3{\xc4\xcfw\xaaΣ2L \xb3\x9fW\xa7Cη8\xe1L̋\xd7{\xf5xa\x89\xbdz\x8cW\x91\x97(-\xdf\xf2\xe4\x01\xf4X\xac\xf9\x86zN\x97Ç\x84\xeb\xea\xcbޟv\xdcTo\xf7(Ld\xa2\xed_\x9a\xe4\xf2\xbd5\x81\x01A\x90\x7fr[\x0e\x1f\xd9?v\x11\x9d\xd9\xf6\xdd/\x04\xff\\\x11\xab$\xa57>=\xba>\xb5\xec\v\xf4{f\x95Y\xaf\x1a\rz\xce˄v\xdb&MGܦ{\x88]\xc0\x7f\xfew\xf7\xff\x00\x00\x00\xff\xff\x12=\xc7\xe9\x11&\x00\x00"), } var CRDs = crds() func crds() []*apiextv1.CustomResourceDefinition { apiextinstall.Install(scheme.Scheme) decode := scheme.Codecs.UniversalDeserializer().Decode var objs []*apiextv1.CustomResourceDefinition for _, crd := range rawCRDs { gzr, err := gzip.NewReader(bytes.NewReader(crd)) if err != nil { panic(err) } bytes, err := io.ReadAll(gzr) if err != nil { panic(err) } gzr.Close() obj, _, err := decode(bytes, nil, nil) if err != nil { panic(err) } objs = append(objs, obj.(*apiextv1.CustomResourceDefinition)) } return objs } ================================================ FILE: config/crd/v2alpha1/crds/doc.go ================================================ // Package crds embeds the controller-tools generated CRD manifests package crds //go:generate go run ../../../../hack/crd-gen/v1/main.go ================================================ FILE: config/rbac/role.yaml ================================================ --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: velero-perms rules: - apiGroups: - "" resources: - persistentvolumerclaims - persistentvolumes - pods verbs: - get - apiGroups: - velero.io resources: - backuprepositories - backups - backupstoragelocations - datadownloads - datauploads - deletebackuprequests - downloadrequests - podvolumebackups - podvolumerestores - restores - schedules - serverstatusrequests - volumesnapshotlocations verbs: - create - delete - get - list - patch - update - watch - apiGroups: - velero.io resources: - backuprepositories/status - backups/status - backupstoragelocations/status - datadownloads/status - datauploads/status - deletebackuprequests/status - downloadrequests/status - podvolumebackups/status - podvolumerestores/status - restores/status - schedules/status - serverstatusrequests/status verbs: - get - patch - update ================================================ FILE: design/2082-bsl-delete-associated-resources_design.md ================================================ # Delete Backup and Restic Repo Resources when BSL is Deleted ## Abstract Issue #2082 requested that with the command `velero backup-location delete ` (implemented in Velero 1.6 with #3073), the following will be deleted: - associated Velero backups (to be clear, these are custom Kubernetes resources called "backups" that are stored in the API server) - associated Restic repositories (custom Kubernetes resources called "resticrepositories") This design doc explains how the request will be implemented. ## Background When a BSL resource is deleted from its Velero namespace, the associated custom Kubernetes resources, backups and Restic repositories, can no longer be used. It makes sense to clean those resources up when a BSL is deleted. ## Goals Update the `velero backup-location delete ` command to delete associated backup and Restic repository resources in the same Velero namespace. ## Non Goals [It was suggested](https://github.com/vmware-tanzu/velero/issues/2082#issuecomment-827951311) to fix bug #2697 alongside this issue. However, I think that should be fixed separately because although it is similar (restore objects are not being deleted), it is also quite different. One is adding a command feature update (this issue) and the other is a bug fix and each affect different parts of the code base. ## High-Level Design Update the `velero backup-location delete ` command to do the following: - find in the same Velero namespace from which the BSL was deleted the associated backup resources and Restic repositories, called "backups.velero.io" and "resticrepositories.velero.io" respectively - delete the resources found The above logic will be added to [where BSLs are deleted](https://github.com/vmware-tanzu/velero/blob/main/pkg/cmd/cli/backuplocation/delete.go). ## Alternative Considered I had considered deleting the backup files (the ones in json format and tarballs) in the BSL itself. However, a standard use case is to back up a cluster and then restore into a new cluster. Deleting the backup storage location in either location is not expected to remove all of the backups in the backup storage location and should not be done. ================================================ FILE: design/CLI/PoC/base/CRDs.yaml ================================================ --- apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: (unknown) labels: component: velero name: backups.velero.io spec: group: velero.io names: kind: Backup listKind: BackupList plural: backups singular: backup scope: "" validation: openAPIV3Schema: description: Backup is a Velero resource that represents the capture of Kubernetes cluster state at a point in time (API objects and associated volume state). properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: BackupSpec defines the specification for a Velero backup. properties: excludedNamespaces: description: ExcludedNamespaces contains a list of namespaces that are not included in the backup. items: type: string nullable: true type: array excludedResources: description: ExcludedResources is a slice of resource names that are not included in the backup. items: type: string nullable: true type: array hooks: description: Hooks represent custom behaviors that should be executed at different phases of the backup. properties: resources: description: Resources are hooks that should be executed when backing up individual instances of a resource. items: description: BackupResourceHookSpec defines one or more BackupResourceHooks that should be executed based on the rules defined for namespaces, resources, and label selector. properties: excludedNamespaces: description: ExcludedNamespaces specifies the namespaces to which this hook spec does not apply. items: type: string nullable: true type: array excludedResources: description: ExcludedResources specifies the resources to which this hook spec does not apply. items: type: string nullable: true type: array includedNamespaces: description: IncludedNamespaces specifies the namespaces to which this hook spec applies. If empty, it applies to all namespaces. items: type: string nullable: true type: array includedResources: description: IncludedResources specifies the resources to which this hook spec applies. If empty, it applies to all resources. items: type: string nullable: true type: array labelSelector: description: LabelSelector, if specified, filters the resources to which this hook spec applies. nullable: true properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object name: description: Name is the name of this hook. type: string post: description: PostHooks is a list of BackupResourceHooks to execute after storing the item in the backup. These are executed after all "additional items" from item actions are processed. items: description: BackupResourceHook defines a hook for a resource. properties: exec: description: Exec defines an exec hook. properties: command: description: Command is the command and arguments to execute. items: type: string minItems: 1 type: array container: description: Container is the container in the pod where the command should be executed. If not specified, the pod's first container is used. type: string onError: description: OnError specifies how Velero should behave if it encounters an error executing this hook. items: enum: - Continue - Fail type: string timeout: description: Timeout defines the maximum amount of time Velero should wait for the hook to complete before considering the execution a failure. type: string required: - command type: object required: - exec type: object type: array pre: description: PreHooks is a list of BackupResourceHooks to execute prior to storing the item in the backup. These are executed before any "additional items" from item actions are processed. items: description: BackupResourceHook defines a hook for a resource. properties: exec: description: Exec defines an exec hook. properties: command: description: Command is the command and arguments to execute. items: type: string minItems: 1 type: array container: description: Container is the container in the pod where the command should be executed. If not specified, the pod's first container is used. type: string onError: description: OnError specifies how Velero should behave if it encounters an error executing this hook. items: enum: - Continue - Fail type: string timeout: description: Timeout defines the maximum amount of time Velero should wait for the hook to complete before considering the execution a failure. type: string required: - command type: object required: - exec type: object type: array required: - name type: object nullable: true type: array type: object includeClusterResources: description: IncludeClusterResources specifies whether cluster-scoped resources should be included for consideration in the backup. nullable: true type: boolean includedNamespaces: description: IncludedNamespaces is a slice of namespace names to include objects from. If empty, all namespaces are included. items: type: string nullable: true type: array includedResources: description: IncludedResources is a slice of resource names to include in the backup. If empty, all resources are included. items: type: string nullable: true type: array labelSelector: description: LabelSelector is a metav1.LabelSelector to filter with when adding individual objects to the backup. If empty or nil, all objects are included. Optional. nullable: true properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object snapshotVolumes: description: SnapshotVolumes specifies whether to take cloud snapshots of any PV's referenced in the set of objects included in the Backup. nullable: true type: boolean storageLocation: description: StorageLocation is a string containing the name of a BackupStorageLocation where the backup should be stored. type: string ttl: description: TTL is a time.Duration-parseable string describing how long the Backup should be retained for. type: string volumeSnapshotLocations: description: VolumeSnapshotLocations is a list containing names of VolumeSnapshotLocations associated with this backup. items: type: string type: array type: object status: description: BackupStatus captures the current status of a Velero backup. properties: completionTimestamp: description: CompletionTimestamp records the time a backup was completed. Completion time is recorded even on failed backups. Completion time is recorded before uploading the backup object. The server's time is used for CompletionTimestamps format: date-time nullable: true type: string errors: description: Errors is a count of all error messages that were generated during execution of the backup. The actual errors are in the backup's log file in object storage. type: integer expiration: description: Expiration is when this Backup is eligible for garbage-collection. format: date-time nullable: true type: string phase: description: Phase is the current state of the Backup. items: enum: - New - FailedValidation - InProgress - Completed - PartiallyFailed - Failed - Deleting type: string startTimestamp: description: StartTimestamp records the time a backup was started. Separate from CreationTimestamp, since that value changes on restores. The server's time is used for StartTimestamps format: date-time nullable: true type: string validationErrors: description: ValidationErrors is a slice of all validation errors (if applicable). items: type: string nullable: true type: array version: description: Version is the backup format version. type: integer volumeSnapshotsAttempted: description: VolumeSnapshotsAttempted is the total number of attempted volume snapshots for this backup. type: integer volumeSnapshotsCompleted: description: VolumeSnapshotsCompleted is the total number of successfully completed volume snapshots for this backup. type: integer warnings: description: Warnings is a count of all warning messages that were generated during execution of the backup. The actual warnings are in the backup's log file in object storage. type: integer type: object type: object version: v1beta1 versions: - name: v1beta1 served: true storage: true --- apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: (unknown) labels: component: velero name: deletebackuprequests.velero.io spec: group: velero.io names: kind: DeleteBackupRequest listKind: DeleteBackupRequestList plural: deletebackuprequests singular: deletebackuprequest scope: "" validation: openAPIV3Schema: description: DeleteBackupRequest is a request to delete one or more backups. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: DeleteBackupRequestSpec is the specification for which backups to delete. properties: backupName: type: string required: - backupName type: object status: description: DeleteBackupRequestStatus is the current status of a DeleteBackupRequest. properties: errors: description: Errors contains any errors that were encountered during the deletion process. items: type: string nullable: true type: array phase: description: Phase is the current state of the DeleteBackupRequest. items: enum: - New - InProgress - Processed type: string type: object type: object version: v1beta1 versions: - name: v1beta1 served: true storage: true --- apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: (unknown) labels: component: velero name: downloadrequests.velero.io spec: group: velero.io names: kind: DownloadRequest listKind: DownloadRequestList plural: downloadrequests singular: downloadrequest scope: "" validation: openAPIV3Schema: description: DownloadRequest is a request to download an artifact from backup object storage, such as a backup log file. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: DownloadRequestSpec is the specification for a download request. properties: target: description: Target is what to download (e.g. logs for a backup). properties: kind: description: Kind is the type of file to download. items: enum: - BackupLog - BackupContents - BackupVolumeSnapshot - BackupResourceList - RestoreLog - RestoreResults - CSIBackupVolumeSnapshots - CSIBackupVolumeSnapshotContents type: string name: description: Name is the name of the Kubernetes resource with which the file is associated. type: string required: - kind - name type: object required: - target type: object status: description: DownloadRequestStatus is the current status of a DownloadRequest. properties: downloadURL: description: DownloadURL contains the pre-signed URL for the target file. type: string expiration: description: Expiration is when this DownloadRequest expires and can be deleted by the system. format: date-time nullable: true type: string phase: description: Phase is the current state of the DownloadRequest. items: enum: - New - Processed type: string type: object type: object version: v1beta1 versions: - name: v1beta1 served: true storage: true --- apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: (unknown) labels: component: velero name: restores.velero.io spec: group: velero.io names: kind: Restore listKind: RestoreList plural: restores singular: restore scope: "" validation: openAPIV3Schema: description: Restore is a Velero resource that represents the application of resources from a Velero backup to a target Kubernetes cluster. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: RestoreSpec defines the specification for a Velero restore. properties: backupName: description: BackupName is the unique name of the Velero backup to restore from. type: string excludedNamespaces: description: ExcludedNamespaces contains a list of namespaces that are not included in the restore. items: type: string nullable: true type: array excludedResources: description: ExcludedResources is a slice of resource names that are not included in the restore. items: type: string nullable: true type: array includeClusterResources: description: IncludeClusterResources specifies whether cluster-scoped resources should be included for consideration in the restore. If null, defaults to true. nullable: true type: boolean includedNamespaces: description: IncludedNamespaces is a slice of namespace names to include objects from. If empty, all namespaces are included. items: type: string nullable: true type: array includedResources: description: IncludedResources is a slice of resource names to include in the restore. If empty, all resources in the backup are included. items: type: string nullable: true type: array labelSelector: description: LabelSelector is a metav1.LabelSelector to filter with when restoring individual objects from the backup. If empty or nil, all objects are included. Optional. nullable: true properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object namespaceMapping: additionalProperties: type: string description: NamespaceMapping is a map of source namespace names to target namespace names to restore into. Any source namespaces not included in the map will be restored into namespaces of the same name. type: object restorePVs: description: RestorePVs specifies whether to restore all included PVs from snapshot (via the cloudprovider). nullable: true type: boolean preserveNodePorts: description: PreserveNodePorts specifies whether to restore old nodePorts from backup. nullable: true type: boolean scheduleName: description: ScheduleName is the unique name of the Velero schedule to restore from. If specified, and BackupName is empty, Velero will restore from the most recent successful backup created from this schedule. type: string required: - backupName type: object status: description: RestoreStatus captures the current status of a Velero restore properties: errors: description: Errors is a count of all error messages that were generated during execution of the restore. The actual errors are stored in object storage. type: integer failureReason: description: FailureReason is an error that caused the entire restore to fail. type: string phase: description: Phase is the current state of the Restore items: enum: - New - FailedValidation - InProgress - Completed - PartiallyFailed - Failed type: string validationErrors: description: ValidationErrors is a slice of all validation errors (if applicable) items: type: string nullable: true type: array warnings: description: Warnings is a count of all warning messages that were generated during execution of the restore. The actual warnings are stored in object storage. type: integer type: object type: object version: v1beta1 versions: - name: v1beta1 served: true storage: true --- apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: (unknown) labels: component: velero name: schedules.velero.io spec: group: velero.io names: kind: Schedule listKind: ScheduleList plural: schedules singular: schedule scope: "" validation: openAPIV3Schema: description: Schedule is a Velero resource that represents a pre-scheduled or periodic Backup that should be run. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: ScheduleSpec defines the specification for a Velero schedule properties: schedule: description: Schedule is a Cron expression defining when to run the Backup. type: string template: description: Template is the definition of the Backup to be run on the provided schedule properties: excludedNamespaces: description: ExcludedNamespaces contains a list of namespaces that are not included in the backup. items: type: string nullable: true type: array excludedResources: description: ExcludedResources is a slice of resource names that are not included in the backup. items: type: string nullable: true type: array hooks: description: Hooks represent custom behaviors that should be executed at different phases of the backup. properties: resources: description: Resources are hooks that should be executed when backing up individual instances of a resource. items: description: BackupResourceHookSpec defines one or more BackupResourceHooks that should be executed based on the rules defined for namespaces, resources, and label selector. properties: excludedNamespaces: description: ExcludedNamespaces specifies the namespaces to which this hook spec does not apply. items: type: string nullable: true type: array excludedResources: description: ExcludedResources specifies the resources to which this hook spec does not apply. items: type: string nullable: true type: array includedNamespaces: description: IncludedNamespaces specifies the namespaces to which this hook spec applies. If empty, it applies to all namespaces. items: type: string nullable: true type: array includedResources: description: IncludedResources specifies the resources to which this hook spec applies. If empty, it applies to all resources. items: type: string nullable: true type: array labelSelector: description: LabelSelector, if specified, filters the resources to which this hook spec applies. nullable: true properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object name: description: Name is the name of this hook. type: string post: description: PostHooks is a list of BackupResourceHooks to execute after storing the item in the backup. These are executed after all "additional items" from item actions are processed. items: description: BackupResourceHook defines a hook for a resource. properties: exec: description: Exec defines an exec hook. properties: command: description: Command is the command and arguments to execute. items: type: string minItems: 1 type: array container: description: Container is the container in the pod where the command should be executed. If not specified, the pod's first container is used. type: string onError: description: OnError specifies how Velero should behave if it encounters an error executing this hook. items: enum: - Continue - Fail type: string timeout: description: Timeout defines the maximum amount of time Velero should wait for the hook to complete before considering the execution a failure. type: string required: - command type: object required: - exec type: object type: array pre: description: PreHooks is a list of BackupResourceHooks to execute prior to storing the item in the backup. These are executed before any "additional items" from item actions are processed. items: description: BackupResourceHook defines a hook for a resource. properties: exec: description: Exec defines an exec hook. properties: command: description: Command is the command and arguments to execute. items: type: string minItems: 1 type: array container: description: Container is the container in the pod where the command should be executed. If not specified, the pod's first container is used. type: string onError: description: OnError specifies how Velero should behave if it encounters an error executing this hook. items: enum: - Continue - Fail type: string timeout: description: Timeout defines the maximum amount of time Velero should wait for the hook to complete before considering the execution a failure. type: string required: - command type: object required: - exec type: object type: array required: - name type: object nullable: true type: array type: object includeClusterResources: description: IncludeClusterResources specifies whether cluster-scoped resources should be included for consideration in the backup. nullable: true type: boolean includedNamespaces: description: IncludedNamespaces is a slice of namespace names to include objects from. If empty, all namespaces are included. items: type: string nullable: true type: array includedResources: description: IncludedResources is a slice of resource names to include in the backup. If empty, all resources are included. items: type: string nullable: true type: array labelSelector: description: LabelSelector is a metav1.LabelSelector to filter with when adding individual objects to the backup. If empty or nil, all objects are included. Optional. nullable: true properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object snapshotVolumes: description: SnapshotVolumes specifies whether to take cloud snapshots of any PV's referenced in the set of objects included in the Backup. nullable: true type: boolean storageLocation: description: StorageLocation is a string containing the name of a BackupStorageLocation where the backup should be stored. type: string ttl: description: TTL is a time.Duration-parseable string describing how long the Backup should be retained for. type: string volumeSnapshotLocations: description: VolumeSnapshotLocations is a list containing names of VolumeSnapshotLocations associated with this backup. items: type: string type: array type: object required: - schedule - template type: object status: description: ScheduleStatus captures the current state of a Velero schedule properties: lastBackup: description: LastBackup is the last time a Backup was run for this Schedule schedule format: date-time nullable: true type: string phase: description: Phase is the current phase of the Schedule items: enum: - New - Enabled - FailedValidation type: string validationErrors: description: ValidationErrors is a slice of all validation errors (if applicable) items: type: string type: array type: object type: object version: v1beta1 versions: - name: v1beta1 served: true storage: true --- apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: (unknown) labels: component: velero name: serverstatusrequests.velero.io spec: group: velero.io names: kind: ServerStatusRequest listKind: ServerStatusRequestList plural: serverstatusrequests singular: serverstatusrequest scope: "" validation: openAPIV3Schema: description: ServerStatusRequest is a request to access current status information about the Velero server. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: ServerStatusRequestSpec is the specification for a ServerStatusRequest. type: object status: description: ServerStatusRequestStatus is the current status of a ServerStatusRequest. properties: phase: description: Phase is the current lifecycle phase of the ServerStatusRequest. items: enum: - New - Processed type: string plugins: description: Plugins list information about the plugins running on the Velero server items: description: PluginInfo contains attributes of a Velero plugin properties: kind: type: string name: type: string required: - kind - name type: object nullable: true type: array processedTimestamp: description: ProcessedTimestamp is when the ServerStatusRequest was processed by the ServerStatusRequestController. format: date-time nullable: true type: string serverVersion: description: ServerVersion is the Velero server version. type: string type: object type: object version: v1beta1 versions: - name: v1beta1 served: true storage: true ================================================ FILE: design/CLI/PoC/base/backupstoragelocations.yaml ================================================ --- apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: (unknown) labels: component: velero name: backupstoragelocations.velero.io spec: group: velero.io names: kind: BackupStorageLocation listKind: BackupStorageLocationList plural: backupstoragelocations singular: backupstoragelocation scope: "" validation: openAPIV3Schema: description: BackupStorageLocation is a location where Velero stores backup objects. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: BackupStorageLocationSpec defines the specification for a Velero BackupStorageLocation. properties: accessMode: description: AccessMode defines the permissions for the backup storage location. enum: - ReadOnly - ReadWrite type: string backupSyncPeriod: description: BackupSyncPeriod defines how frequently to sync backup API objects from object storage. A value of 0 disables sync. nullable: true type: string config: additionalProperties: type: string description: Config is for provider-specific configuration fields. type: object objectStorage: description: ObjectStorageLocation specifies the settings necessary to connect to a provider's object storage. properties: bucket: description: Bucket is the bucket to use for object storage. type: string prefix: description: Prefix is the path inside a bucket to use for Velero storage. Optional. type: string required: - bucket type: object provider: description: Provider is the provider of the backup storage. type: string required: - objectStorage - provider type: object status: description: BackupStorageLocationStatus describes the current status of a Velero BackupStorageLocation. properties: accessMode: description: "AccessMode is an unused field. \n Deprecated: there is now an AccessMode field on the Spec and this field will be removed entirely as of v2.0." enum: - ReadOnly - ReadWrite type: string lastSyncedRevision: description: "LastSyncedRevision is the value of the `metadata/revision` file in the backup storage location the last time the BSL's contents were synced into the cluster. \n Deprecated: this field is no longer updated or used for detecting changes to the location's contents and will be removed entirely in v2.0." type: string lastSyncedTime: description: LastSyncedTime is the last time the contents of the location were synced into the cluster. format: date-time nullable: true type: string phase: description: Phase is the current state of the BackupStorageLocation. enum: - Available - Unavailable type: string type: object type: object version: v1 versions: - name: v1 served: true storage: true --- apiVersion: velero.io/v1 kind: BackupStorageLocation metadata: creationTimestamp: null labels: component: velero name: default namespace: velero spec: config: region: minio s3ForcePathStyle: "true" s3Url: http://minio.velero.svc:9000 objectStorage: bucket: velero provider: aws ================================================ FILE: design/CLI/PoC/base/deployment.yaml ================================================ --- apiVersion: apps/v1 kind: Deployment metadata: labels: component: velero name: velero namespace: velero spec: selector: matchLabels: deploy: velero strategy: {} template: metadata: annotations: prometheus.io/path: /metrics prometheus.io/port: "8085" prometheus.io/scrape: "true" labels: component: velero deploy: velero spec: containers: - args: - server command: - /velero env: - name: VELERO_SCRATCH_DIR value: /scratch - name: VELERO_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - name: LD_LIBRARY_PATH value: /plugins name: velero image: velero/velero:latest imagePullPolicy: Always ports: - containerPort: 8085 name: metrics resources: limits: cpu: "1" memory: 256Mi requests: cpu: 500m memory: 128Mi volumeMounts: - mountPath: /scratch name: scratch restartPolicy: Always serviceAccountName: velero volumes: - emptyDir: {} name: scratch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: component: velero name: velero roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: cluster-admin subjects: - kind: ServiceAccount name: velero namespace: velero --- apiVersion: v1 kind: ServiceAccount metadata: labels: component: velero name: velero namespace: velero --- apiVersion: v1 kind: Namespace metadata: labels: component: velero name: velero spec: {} ================================================ FILE: design/CLI/PoC/base/kustomization.yaml ================================================ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - deployment.yaml - CRDs.yaml - backupstoragelocations.yaml - volumesnapshotlocations.yaml # including so the velero server can run - resticrepository.yaml # including so the velero server can runl - podvolumes.yaml # including so the velero server can runl - minio.yaml ================================================ FILE: design/CLI/PoC/base/minio.yaml ================================================ # Copyright 2017 the Velero contributors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. --- apiVersion: apps/v1 kind: Deployment metadata: namespace: velero name: minio labels: component: minio spec: strategy: type: Recreate selector: matchLabels: component: minio template: metadata: labels: component: minio spec: volumes: - name: storage emptyDir: {} - name: config emptyDir: {} containers: - name: minio image: minio/minio:latest imagePullPolicy: IfNotPresent args: - server - /storage - --config-dir=/config env: - name: MINIO_ACCESS_KEY value: "minio" - name: MINIO_SECRET_KEY value: "minio123" ports: - containerPort: 9000 volumeMounts: - name: storage mountPath: "/storage" - name: config mountPath: "/config" --- apiVersion: v1 kind: Service metadata: namespace: velero name: minio labels: component: minio spec: # ClusterIP is recommended for production environments. # Change to NodePort if needed per documentation, # but only if you run Minio in a test/trial environment, for example with Minikube. type: ClusterIP ports: - port: 9000 targetPort: 9000 protocol: TCP selector: component: minio --- apiVersion: batch/v1 kind: Job metadata: namespace: velero name: minio-setup labels: component: minio spec: template: metadata: name: minio-setup spec: restartPolicy: OnFailure volumes: - name: config emptyDir: {} containers: - name: mc image: minio/mc:latest imagePullPolicy: IfNotPresent command: - /bin/sh - -c - "mc --config-dir=/config config host add velero http://minio:9000 minio minio123 && mc --config-dir=/config mb -p velero/velero" volumeMounts: - name: config mountPath: "/config" ================================================ FILE: design/CLI/PoC/base/podvolumes.yaml ================================================ --- apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: (unknown) creationTimestamp: null labels: component: velero name: podvolumebackups.velero.io spec: group: velero.io names: kind: PodVolumeBackup listKind: PodVolumeBackupList plural: podvolumebackups singular: podvolumebackup scope: "" validation: openAPIV3Schema: properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: PodVolumeBackupSpec is the specification for a PodVolumeBackup. properties: backupStorageLocation: description: BackupStorageLocation is the name of the backup storage location where the restic repository is stored. type: string node: description: Node is the name of the node that the Pod is running on. type: string pod: description: Pod is a reference to the pod containing the volume to be backed up. properties: apiVersion: description: API version of the referent. type: string fieldPath: description: 'If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: "spec.containers{name}" (where "name" refers to the name of the container that triggered the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. TODO: this design is not final and this field is subject to change in the future.' type: string kind: description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' type: string namespace: description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' type: string resourceVersion: description: 'Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency' type: string uid: description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' type: string type: object repoIdentifier: description: RepoIdentifier is the restic repository identifier. type: string tags: additionalProperties: type: string description: Tags are a map of key-value pairs that should be applied to the volume backup as tags. type: object volume: description: Volume is the name of the volume within the Pod to be backed up. type: string required: - backupStorageLocation - node - pod - repoIdentifier - volume type: object status: description: PodVolumeBackupStatus is the current status of a PodVolumeBackup. properties: completionTimestamp: description: CompletionTimestamp records the time a backup was completed. Completion time is recorded even on failed backups. Completion time is recorded before uploading the backup object. The server's time is used for CompletionTimestamps format: date-time nullable: true type: string message: description: Message is a message about the pod volume backup's status. type: string path: description: Path is the full path within the controller pod being backed up. type: string phase: description: Phase is the current state of the PodVolumeBackup. enum: - New - InProgress - Completed - Failed type: string progress: description: Progress holds the total number of bytes of the volume and the current number of backed up bytes. This can be used to display progress information about the backup operation. properties: bytesDone: format: int64 type: integer totalBytes: format: int64 type: integer type: object snapshotID: description: SnapshotID is the identifier for the snapshot of the pod volume. type: string startTimestamp: description: StartTimestamp records the time a backup was started. Separate from CreationTimestamp, since that value changes on restores. The server's time is used for StartTimestamps format: date-time nullable: true type: string type: object type: object version: v1 versions: - name: v1 served: true storage: true --- apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: (unknown) creationTimestamp: null labels: component: velero name: podvolumerestores.velero.io spec: group: velero.io names: kind: PodVolumeRestore listKind: PodVolumeRestoreList plural: podvolumerestores singular: podvolumerestore scope: "" validation: openAPIV3Schema: properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: PodVolumeRestoreSpec is the specification for a PodVolumeRestore. properties: backupStorageLocation: description: BackupStorageLocation is the name of the backup storage location where the restic repository is stored. type: string pod: description: Pod is a reference to the pod containing the volume to be restored. properties: apiVersion: description: API version of the referent. type: string fieldPath: description: 'If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: "spec.containers{name}" (where "name" refers to the name of the container that triggered the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. TODO: this design is not final and this field is subject to change in the future.' type: string kind: description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' type: string namespace: description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' type: string resourceVersion: description: 'Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency' type: string uid: description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' type: string type: object repoIdentifier: description: RepoIdentifier is the restic repository identifier. type: string snapshotID: description: SnapshotID is the ID of the volume snapshot to be restored. type: string volume: description: Volume is the name of the volume within the Pod to be restored. type: string required: - backupStorageLocation - pod - repoIdentifier - snapshotID - volume type: object status: description: PodVolumeRestoreStatus is the current status of a PodVolumeRestore. properties: completionTimestamp: description: CompletionTimestamp records the time a restore was completed. Completion time is recorded even on failed restores. The server's time is used for CompletionTimestamps format: date-time nullable: true type: string message: description: Message is a message about the pod volume restore's status. type: string phase: description: Phase is the current state of the PodVolumeRestore. enum: - New - InProgress - Completed - Failed type: string progress: description: Progress holds the total number of bytes of the snapshot and the current number of restored bytes. This can be used to display progress information about the restore operation. properties: bytesDone: format: int64 type: integer totalBytes: format: int64 type: integer type: object startTimestamp: description: StartTimestamp records the time a restore was started. The server's time is used for StartTimestamps format: date-time nullable: true type: string type: object type: object version: v1 versions: - name: v1 served: true storage: true ================================================ FILE: design/CLI/PoC/base/resticrepository.yaml ================================================ --- apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: (unknown) creationTimestamp: null labels: component: velero name: resticrepositories.velero.io spec: group: velero.io names: kind: ResticRepository listKind: ResticRepositoryList plural: resticrepositories singular: resticrepository scope: "" validation: openAPIV3Schema: properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: ResticRepositorySpec is the specification for a ResticRepository. properties: backupStorageLocation: description: BackupStorageLocation is the name of the BackupStorageLocation that should contain this repository. type: string maintenanceFrequency: description: MaintenanceFrequency is how often maintenance should be run. type: string resticIdentifier: description: ResticIdentifier is the full restic-compatible string for identifying this repository. type: string volumeNamespace: description: VolumeNamespace is the namespace this restic repository contains pod volume backups for. type: string required: - backupStorageLocation - maintenanceFrequency - resticIdentifier - volumeNamespace type: object status: description: ResticRepositoryStatus is the current status of a ResticRepository. properties: lastMaintenanceTime: description: LastMaintenanceTime is the last time maintenance was run. format: date-time nullable: true type: string message: description: Message is a message about the current status of the ResticRepository. type: string phase: description: Phase is the current state of the ResticRepository. enum: - New - Ready - NotReady type: string type: object type: object version: v1 versions: - name: v1 served: true storage: true ================================================ FILE: design/CLI/PoC/base/volumesnapshotlocations.yaml ================================================ --- apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: (unknown) labels: component: velero name: volumesnapshotlocations.velero.io spec: group: velero.io names: kind: VolumeSnapshotLocation listKind: VolumeSnapshotLocationList plural: volumesnapshotlocations singular: volumesnapshotlocation scope: "" validation: openAPIV3Schema: description: VolumeSnapshotLocation is a location where Velero stores volume snapshots. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: VolumeSnapshotLocationSpec defines the specification for a Velero VolumeSnapshotLocation. properties: config: additionalProperties: type: string description: Config is for provider-specific configuration fields. type: object provider: description: Provider is the provider of the volume storage. type: string required: - provider type: object status: description: VolumeSnapshotLocationStatus describes the current status of a Velero VolumeSnapshotLocation. properties: phase: description: VolumeSnapshotLocationPhase is the lifecycle phase of a Velero VolumeSnapshotLocation. enum: - Available - Unavailable type: string type: object type: object version: v1 versions: - name: v1 served: true storage: true --- apiVersion: velero.io/v1 kind: VolumeSnapshotLocation metadata: creationTimestamp: null labels: component: velero name: default namespace: velero spec: config: region: us-east-2 provider: aws ================================================ FILE: design/CLI/PoC/overlays/plugins/aws-plugin.yaml ================================================ --- apiVersion: apps/v1 kind: Deployment metadata: name: velero spec: selector: matchLabels: deploy: velero template: metadata: labels: component: velero deploy: velero spec: containers: - args: - server name: velero env: - name: AWS_SHARED_CREDENTIALS_FILE value: /credentials/cloud volumeMounts: - mountPath: /plugins name: plugins - mountPath: /credentials name: cloud-credential-aws initContainers: - image: velero/velero-plugin-for-aws:v1.0.1 imagePullPolicy: Always name: velero-plugin-for-aws volumeMounts: - mountPath: /target name: plugins volumes: - emptyDir: {} name: plugins - name: cloud-credential-aws secret: secretName: cloud-credential-aws ================================================ FILE: design/CLI/PoC/overlays/plugins/azure-plugin.yaml ================================================ --- apiVersion: apps/v1 kind: Deployment metadata: name: velero spec: selector: matchLabels: deploy: velero template: metadata: labels: component: velero deploy: velero spec: containers: - args: - server name: velero env: - name: AZURE_SHARED_CREDENTIALS_FILE value: /credentials/cloud volumeMounts: - mountPath: /plugins name: plugins - mountPath: /credentials name: cloud-credential-azure initContainers: - image: velero/velero-plugin-for-microsoft-azure:v1.0.1 imagePullPolicy: Always name: velero-plugin-for-microsoft-azure volumeMounts: - mountPath: /target name: plugins volumes: - emptyDir: {} name: plugins - name: cloud-credential-azure secret: secretName: cloud-credential-azure ================================================ FILE: design/CLI/PoC/overlays/plugins/cloud ================================================ [default] aws_access_key_id = minio aws_secret_access_key = minio123 ================================================ FILE: design/CLI/PoC/overlays/plugins/kustomization.yaml ================================================ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization bases: - ../../base patchesStrategicMerge: - aws-plugin.yaml # this patches the Velero deployment # - azure-plugin.yaml # this patches the Velero deployment generatorOptions: disableNameSuffixHash: true labels: component: velero secretGenerator: - name: cloud-credentials files: - "cloud" ================================================ FILE: design/CLI/PoC/overlays/plugins/node-agent.yaml ================================================ --- apiVersion: apps/v1 kind: DaemonSet metadata: creationTimestamp: null labels: component: velero name: node-agent namespace: velero spec: selector: matchLabels: name: node-agent template: metadata: creationTimestamp: null labels: component: velero name: node-agent spec: containers: - args: - node-agent - server command: - /velero env: - name: NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName - name: VELERO_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - name: VELERO_SCRATCH_DIR value: /scratch - name: GOOGLE_APPLICATION_CREDENTIALS value: /credentials/cloud - name: AWS_SHARED_CREDENTIALS_FILE value: /credentials/cloud - name: AZURE_CREDENTIALS_FILE value: /credentials/cloud image: velero/velero:latest imagePullPolicy: Always name: node-agent resources: {} volumeMounts: - mountPath: /host_pods mountPropagation: HostToContainer name: host-pods - mountPath: /var/lib/kubelet/plugins mountPropagation: HostToContainer name: host-plugins - mountPath: /scratch name: scratch - mountPath: /credentials name: cloud-credentials securityContext: runAsUser: 0 serviceAccountName: velero volumes: - hostPath: path: /var/lib/kubelet/pods name: host-pods - hostPath: path: /var/lib/kubelet/plugins name: host-plugins - emptyDir: {} name: scratch - name: cloud-credentials secret: secretName: cloud-credentials updateStrategy: {} ================================================ FILE: design/Implemented/Extend-VolumePolicies-to-support-more-actions.md ================================================ # Extend VolumePolicies to support more actions ## Abstract Currently, the [VolumePolicies feature](https://github.com/vmware-tanzu/velero/blob/main/design/Implemented/handle-backup-of-volumes-by-resources-filters.md) which can be used to filter/handle volumes during backup only supports the skip action on matching conditions. Users need more actions to be supported. ## Background The `VolumePolicies` feature was introduced in Velero 1.11 as a flexible way to handle volumes. The main agenda of introducing the VolumePolicies feature was to improve the overall user experience when performing backup operations for volume resources, the feature enables users to group volumes according the `conditions` (criteria) specified and also lets you specify the `action` that velero needs to take for these grouped volumes during the backup operation. The limitation being that currently `VolumePolicies` only supports `skip` as an action, We want to extend the `action` functionality to support more usable options like `fs-backup` (File system backup) and `snapshot` (VolumeSnapshots). ## Goals - Extending the VolumePolicies to support more actions like `fs-backup` (File system backup) and `snapshot` (VolumeSnapshots). - Improve user experience when backing up Volumes via Velero ## Non-Goals - No changes to existing approaches to opt-in/opt-out annotations for volumes - No changes to existing `VolumePolicies` functionalities - No additions or implementations to support more granular actions like `snapshot-csi` and `snapshot-datamover`. These actions can be implemented as a future enhancement ## Use-cases/Scenarios **Use-case 1:** - A user wants to use `snapshot` (volumesnapshots) backup option for all the csi supported volumes and `fs-backup` for the rest of the volumes. - Currently, velero supports this use-case but the user experience is not that great. - The user will have to individually annotate the volume mounting pod with the annotation "backup.velero.io/backup-volumes" for `fs-backup` - This becomes cumbersome at scale. - Using `VolumePolicies`, the user can just specify 2 simple `VolumePolicies` like for csi supported volumes as `snapshot` action and rest can be backed up`fs-backup` action: ```yaml version: v1 volumePolicies: - conditions: storageClass: - gp2 action: type: snapshot - conditions: {} action: type: fs-backup ``` **Use-case 2:** - A user wants to use `fs-backup` for nfs volumes pertaining to a particular server - In such a scenario the user can just specify a `VolumePolicy` like: ```yaml version: v1 volumePolicies: - conditions: nfs: server: 192.168.200.90 action: type: fs-backup ``` ## High-Level Design - When the VolumePolicy action is set as `fs-backup` the backup workflow modifications would be: - We call [backupItem() -> backupItemInternal()](https://github.com/vmware-tanzu/velero/blob/main/pkg/backup/item_backupper.go#L95) on all the items that are to be backed up - Here when we encounter [Pod as an item ](https://github.com/vmware-tanzu/velero/blob/main/pkg/backup/item_backupper.go#L195) - We will have to modify the backup workflow to account for the `fs-backup` VolumePolicy action - When the VolumePolicy action is set as `snapshot` the backup workflow modifications would be: - Once again, We call [backupItem() -> backupItemInternal()](https://github.com/vmware-tanzu/velero/blob/main/pkg/backup/item_backupper.go#L95) on all the items that are to be backed up - Here when we encounter [Persistent Volume as an item](https://github.com/vmware-tanzu/velero/blob/d4128542590470b204a642ee43311921c11db880/pkg/backup/item_backupper.go#L253) - And we call the [takePVSnapshot func](https://github.com/vmware-tanzu/velero/blob/d4128542590470b204a642ee43311921c11db880/pkg/backup/item_backupper.go#L508) - We need to modify the takePVSnapshot function to account for the `snapshot` VolumePolicy action. - In case of csi snapshots for PVC objects, these snapshot actions are taken by the velero-plugin-for-csi, we need to modify the [executeActions()](https://github.com/vmware-tanzu/velero/blob/512fe0dabdcb3bbf1ca68a9089056ae549663bcf/pkg/backup/item_backupper.go#L232) function to account for the `snapshot` VolumePolicy action. **Note:** `Snapshot` action can either be a native snapshot or a csi snapshot, as is the case with the current flow where velero itself makes the decision based on the backup CR. ## Detailed Design - Update VolumePolicy action type validation to account for `fs-backup` and `snapshot` as valid VolumePolicy actions. - Modifications needed for `fs-backup` action: - Now based on the specification of volume policy on backup request we will decide whether to go via legacy pod annotations approach or the newer volume policy based fs-backup action approach. - If there is a presence of volume policy(fs-backup/snapshot) on the backup request that matches as an action for a volume we use the newer volume policy approach to get the list of the volumes for `fs-backup` action - Else continue with the annotation based legacy approach workflow. - Modifications needed for `snapshot` action: - In the [takePVSnapshot function](https://github.com/vmware-tanzu/velero/blob/d4128542590470b204a642ee43311921c11db880/pkg/backup/item_backupper.go#L508) we will check the PV fits the volume policy criteria and see if the associated action is `snapshot` - If it is not snapshot then we skip the further workflow and avoid taking the snapshot of the PV - Similarly, For csi snapshot of PVC object, we need to do similar changes in [executeAction() function](https://github.com/vmware-tanzu/velero/blob/512fe0dabdcb3bbf1ca68a9089056ae549663bcf/pkg/backup/item_backupper.go#L348). we will check the PVC fits the volume policy criteria and see if the associated action is `snapshot` via csi plugin - If it is not snapshot then we skip the csi BIA execute action and avoid taking the snapshot of the PVC by not invoking the csi plugin action for the PVC **Note:** - When we are using the `VolumePolicy` approach for backing up the volumes then the volume policy criteria and action need to be specific and explicit, there is no default behavior, if a volume matches `fs-backup` action then `fs-backup` method will be used for that volume and similarly if the volume matches the criteria for `snapshot` action then the snapshot workflow will be used for the volume backup. - Another thing to note is the workflow proposed in this design uses the legacy `opt-in/opt-out` approach as a fallback option. For instance, the user specifies a VolumePolicy but for a particular volume included in the backup there are no actions(fs-backup/snapshot) matching in the volume policy for that volume, in such a scenario the legacy approach will be used for backing up the particular volume. - The relation between the `VolumePolicy` and the backup's legacy parameter `SnapshotVolumes`: - The `VolumePolicy`'s `snapshot` action matching for volume has higher priority. When there is a `snapshot` action matching for the selected volume, it will be backed by the snapshot way, no matter of the `backup.Spec.SnapshotVolumes` setting. - If there is no `snapshot` action matching the selected volume in the `VolumePolicy`, then the volume will be backed up by `snapshot` way, if the `backup.Spec.SnapshotVolumes` is not set to false. - The relation between the `VolumePolicy` and the backup's legacy filesystem `opt-in/opt-out` approach: - The `VolumePolicy`'s `fs-backup` action matching for volume has higher priority. When there is a `fs-backup` action matching for the selected volume, it will be backed by the fs-backup way, no matter of the `backup.Spec.DefaultVolumesToFsBackup` setting and the pod's `opt-in/opt-out` annotation setting. - If there is no `fs-backup` action matching the selected volume in the `VolumePolicy`, then the volume will be backed up by the legacy `opt-in/opt-out` way. ## Implementation - The implementation should be included in velero 1.14 - We will introduce a `VolumeHelper` interface. It will consist of two methods: ```go type VolumeHelper interface { ShouldPerformSnapshot(obj runtime.Unstructured, groupResource schema.GroupResource) (bool, error) ShouldPerformFSBackup(volume corev1api.Volume, pod corev1api.Pod) (bool, error) } ``` - The `VolumeHelperImpl` struct will implement the `VolumeHelper` interface and will consist of the functions that we will use through the backup workflow to accommodate volume policies for PVs and PVCs. ```go type volumeHelperImpl struct { volumePolicy *resourcepolicies.Policies snapshotVolumes *bool logger logrus.FieldLogger client crclient.Client defaultVolumesToFSBackup bool backupExcludePVC bool } ``` - We will create an instance of the structure `volumeHelperImpl` in `item_backupper.go` ```go itemBackupper := &itemBackupper{ ... volumeHelperImpl: volumehelper.NewVolumeHelperImpl( resourcePolicy, backupRequest.Spec.SnapshotVolumes, log, kb.kbClient, boolptr.IsSetToTrue(backupRequest.Spec.DefaultVolumesToFsBackup), !backupRequest.ResourceIncludesExcludes.ShouldInclude(kuberesource.PersistentVolumeClaims.String()), ), } ``` #### FS-Backup - Regarding `fs-backup` action to decide whether to use legacy annotation based approach or volume policy based approach: - We will use the `vh.ShouldPerformFSBackup()` function from the `volumehelper` package - Functions involved in processing `fs-backup` volume policy action will somewhat look like: ```go func (v volumeHelperImpl) ShouldPerformFSBackup(volume corev1api.Volume, pod corev1api.Pod) (bool, error) { if !v.shouldIncludeVolumeInBackup(volume) { v.logger.Debugf("skip fs-backup action for pod %s's volume %s, due to not pass volume check.", pod.Namespace+"/"+pod.Name, volume.Name) return false, nil } if v.volumePolicy != nil { pvc, err := kubeutil.GetPVCForPodVolume(&volume, &pod, v.client) if err != nil { v.logger.WithError(err).Errorf("fail to get PVC for pod %s", pod.Namespace+"/"+pod.Name) return false, err } pv, err := kubeutil.GetPVForPVC(pvc, v.client) if err != nil { v.logger.WithError(err).Errorf("fail to get PV for PVC %s", pvc.Namespace+"/"+pvc.Name) return false, err } action, err := v.volumePolicy.GetMatchAction(pv) if err != nil { v.logger.WithError(err).Errorf("fail to get VolumePolicy match action for PV %s", pv.Name) return false, err } if action != nil { if action.Type == resourcepolicies.FSBackup { v.logger.Infof("Perform fs-backup action for volume %s of pod %s due to volume policy match", volume.Name, pod.Namespace+"/"+pod.Name) return true, nil } else { v.logger.Infof("Skip fs-backup action for volume %s for pod %s because the action type is %s", volume.Name, pod.Namespace+"/"+pod.Name, action.Type) return false, nil } } } if v.shouldPerformFSBackupLegacy(volume, pod) { v.logger.Infof("Perform fs-backup action for volume %s of pod %s due to opt-in/out way", volume.Name, pod.Namespace+"/"+pod.Name) return true, nil } else { v.logger.Infof("Skip fs-backup action for volume %s of pod %s due to opt-in/out way", volume.Name, pod.Namespace+"/"+pod.Name) return false, nil } } ``` - The main function from the above will be called when we encounter Pods during the backup workflow: ```go for _, volume := range pod.Spec.Volumes { shouldDoFSBackup, err := ib.volumeHelperImpl.ShouldPerformFSBackup(volume, *pod) if err != nil { backupErrs = append(backupErrs, errors.WithStack(err)) } ... } ``` #### Snapshot (PV) - Making sure that `snapshot` action is skipped for PVs that do not fit the volume policy criteria, for this we will use the `vh.ShouldPerformSnapshot` from the `VolumeHelperImpl(vh)` receiver. ```go func (v *volumeHelperImpl) ShouldPerformSnapshot(obj runtime.Unstructured, groupResource schema.GroupResource) (bool, error) { // check if volume policy exists and also check if the object(pv/pvc) fits a volume policy criteria and see if the associated action is snapshot // if it is not snapshot then skip the code path for snapshotting the PV/PVC pvc := new(corev1api.PersistentVolumeClaim) pv := new(corev1api.PersistentVolume) var err error if groupResource == kuberesource.PersistentVolumeClaims { if err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), &pvc); err != nil { return false, err } pv, err = kubeutil.GetPVForPVC(pvc, v.client) if err != nil { return false, err } } if groupResource == kuberesource.PersistentVolumes { if err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), &pv); err != nil { return false, err } } if v.volumePolicy != nil { action, err := v.volumePolicy.GetMatchAction(pv) if err != nil { return false, err } // If there is a match action, and the action type is snapshot, return true, // or the action type is not snapshot, then return false. // If there is no match action, go on to the next check. if action != nil { if action.Type == resourcepolicies.Snapshot { v.logger.Infof(fmt.Sprintf("performing snapshot action for pv %s", pv.Name)) return true, nil } else { v.logger.Infof("Skip snapshot action for pv %s as the action type is %s", pv.Name, action.Type) return false, nil } } } // If this PV is claimed, see if we've already taken a (pod volume backup) // snapshot of the contents of this PV. If so, don't take a snapshot. if pv.Spec.ClaimRef != nil { pods, err := podvolumeutil.GetPodsUsingPVC( pv.Spec.ClaimRef.Namespace, pv.Spec.ClaimRef.Name, v.client, ) if err != nil { v.logger.WithError(err).Errorf("fail to get pod for PV %s", pv.Name) return false, err } for _, pod := range pods { for _, vol := range pod.Spec.Volumes { if vol.PersistentVolumeClaim != nil && vol.PersistentVolumeClaim.ClaimName == pv.Spec.ClaimRef.Name && v.shouldPerformFSBackupLegacy(vol, pod) { v.logger.Infof("Skipping snapshot of pv %s because it is backed up with PodVolumeBackup.", pv.Name) return false, nil } } } } if !boolptr.IsSetToFalse(v.snapshotVolumes) { // If the backup.Spec.SnapshotVolumes is not set, or set to true, then should take the snapshot. v.logger.Infof("performing snapshot action for pv %s as the snapshotVolumes is not set to false", pv.Name) return true, nil } v.logger.Infof(fmt.Sprintf("skipping snapshot action for pv %s possibly due to no volume policy setting or snapshotVolumes is false", pv.Name)) return false, nil } ``` - The function `ShouldPerformSnapshot` will be used as follows in `takePVSnapshot` function of the backup workflow: ```go snapshotVolume, err := ib.volumeHelperImpl.ShouldPerformSnapshot(obj, kuberesource.PersistentVolumes) if err != nil { return err } if !snapshotVolume { log.Info(fmt.Sprintf("skipping volume snapshot for PV %s as it does not fit the volume policy criteria specified by the user for snapshot action", pv.Name)) ib.trackSkippedPV(obj, kuberesource.PersistentVolumes, volumeSnapshotApproach, "does not satisfy the criteria for volume policy based snapshot action", log) return nil } ``` #### Snapshot (PVC) - Making sure that `snapshot` action is skipped for PVCs that do not fit the volume policy criteria, for this we will again use the `vh.ShouldPerformSnapshot` from the `VolumeHelperImpl(vh)` receiver. - We will pass the `VolumeHelperImpl(vh)` instance in `executeActions` method so that it is available to use. ```go ``` - The above function will be used as follows in the `executeActions` function of backup workflow. - Considering the vSphere plugin doesn't support the VolumePolicy yet, don't use the VolumePolicy for vSphere plugin by now. ```go if groupResource == kuberesource.PersistentVolumeClaims { if actionName == csiBIAPluginName { snapshotVolume, err := ib.volumeHelperImpl.ShouldPerformSnapshot(obj, kuberesource.PersistentVolumeClaims) if err != nil { return nil, itemFiles, errors.WithStack(err) } if !snapshotVolume { log.Info(fmt.Sprintf("skipping csi volume snapshot for PVC %s as it does not fit the volume policy criteria specified by the user for snapshot action", namespace+"/"+name)) ib.trackSkippedPV(obj, kuberesource.PersistentVolumeClaims, volumeSnapshotApproach, "does not satisfy the criteria for volume policy based snapshot action", log) continue } } } ``` ## Future Implementation It makes sense to add more specific actions in the future, once we deprecate the legacy opt-in/opt-out approach to keep things simple. Another point of note is, csi related action can be easier to implement once we decide to merge the csi plugin into main velero code flow. In the future, we envision the following actions that can be implemented: - `snapshot-native`: only use volume snapshotter (native cloud provider snapshots), do nothing if not present/not compatible - `snapshot-csi`: only use csi-plugin, don't use volume snapshotter(native cloud provider snapshots), don't use datamover even if snapshotMoveData is true - `snapshot-datamover`: only use csi with datamover, don't use volume snapshotter (native cloud provider snapshots), use datamover even if snapshotMoveData is false **Note:** The above actions are just suggestions for future scope, we may not use/implement them as is. We could definitely merge these suggested actions as `Snapshot` actions and use volume policy parameters and criteria to segregate them instead of making the user explicitly supply the action names to such granular levels. ## Related to Design [Handle backup of volumes by resources filters](https://github.com/vmware-tanzu/velero/blob/main/design/Implemented/handle-backup-of-volumes-by-resources-filters.md) ## Alternatives Considered Same as the earlier design as this is an extension of the original VolumePolicies design ================================================ FILE: design/Implemented/apply-flag.md ================================================ # Apply flag for install command ## Abstract Add an `--apply` flag to the install command that enables applying existing resources rather than creating them. This can be useful as part of the upgrade process for existing installations. ## Background The current Velero install command creates resources but doesn't provide a direct way to apply updates to an existing installation. Users attempting to run the install command on an existing installation receive "already exists" messages. Upgrade steps for existing installs typically involve a three (or more) step process to apply updated CRDs (using `--dry-run` and piping to `kubectl apply`) and then updating/setting images on the Velero deployment and node-agent. ## Goals - Provide a simple flag to enable applying resources on an existing Velero installation. - Use server-side apply to update existing resources rather than attempting to create them. - Maintain consistency with the regular install flow. ## Non Goals - Implement special logic for specific version-to-version upgrades (i.e. resource deletion, etc). - Add complex upgrade validation or pre/post-upgrade hooks. - Provide rollback capabilities. ## High-Level Design The `--apply` flag will be added to the Velero install command. When this flag is set, the installation process will use server-side apply to update existing resources instead of using create on new resources. This flag can be used as _part_ of the upgrade process, but will not always fully handle an upgrade. ## Detailed Design The implementation adds a new boolean flag `--apply` to the install command. This flag will be passed through to the underlying install functions where the resource creation logic resides. When the flag is set to true: - The `createOrApplyResource` function will use server-side apply with field manager "velero-cli" and `force=true` to update resources. - Resources will be applied in the same order as they would be created during installation. - Custom Resource Definitions will still be processed first, and the system will wait for them to be established before continuing. The server-side apply approach with `force=true` ensures that resources are updated even if there are conflicts with the last applied state. This provides a best-effort mechanism to apply resources that follows the same flow as installation but updates resources instead of creating them. No special handling is added for specific versions or resource structures, making this a general-purpose mechanism for applying resources. ## Alternatives Considered 1. Creating a separate `upgrade` command that would duplicate much of the install command logic. - Rejected due to code duplication and maintenance overhead. 2. Implementing version-specific upgrade logic to handle breaking changes between versions. - Rejected as overly complex and difficult to maintain across multiple version paths. - This could be considered again in the future, but is not in the scope of the current design. 3. Adding automatic detection of existing resources and switching to apply mode. - Rejected as it could lead to unexpected behavior and confusion if users unintentionally apply changes to existing resources. ## Security Considerations The apply flag maintains the same security profile as the install command. No additional permissions are required beyond what is needed for resource creation. The use of `force=true` with server-side apply could potentially override manual changes made to resources, but this is a necessary trade-off to ensure apply is successful. ## Compatibility This enhancement is compatible with all existing Velero installations as it is a new opt-in flag. It does not change any resource formats or API contracts. The apply process is best-effort and does not guarantee compatibility between arbitrary versions of Velero. Users should still consult release notes for any breaking changes that may require manual intervention. This flag could be adopted by the helm chart, specifically for CRD updates, to simplify the CRD update job. ## Implementation The implementation involves: 1. Adding support for `Apply` to the existing Kubernetes client code. 1. Adding the `--apply` flag to the install command options. 1. Changing `createResource` to `createOrApplyResource` and updating it to use server-side apply when the `apply` boolean is set. The implementation is straightforward and follows existing code patterns. No migration of state or special handling of specific resources is required. ================================================ FILE: design/Implemented/backup-performance-improvements.md ================================================ # Velero Backup performance Improvements and VolumeGroupSnapshot enablement There are two different goals here, linked by a single primary missing feature in the Velero backup workflow. The first goal is to enhance backup performance by allowing the primary backup controller to run in multiple threads, enabling Velero to back up multiple items at the same time for a given backup. The second goal is to enable Velero to eventually support VolumeGroupSnapshots. For both of these goals, Velero needs a way to determine which items should be backed up together. This design proposal will include two development phases: - Phase 1 will refactor the backup workflow to identify blocks of related items that should be backed up together, and then coordinate backup hooks among items in the block. - Phase 2 will add multiple worker threads for backing up item blocks, so instead of backing up each block as it identified, the velero backup workflow will instead add the block to a channel and one of the workers will pick it up. - Actual support for VolumeGroupSnapshots is out-of-scope here and will be handled in a future design proposal, but the item block refactor introduced in Phase 1 is a primary building block for this future proposal. ## Background Currently, during backup processing, the main Velero backup controller runs in a single thread, completely finishing the primary backup processing for one resource before moving on to the next one. We can improve the overall backup performance by backing up multiple items for a backup at the same time, but before we can do this we must first identify resources that need to be backed up together. Generally speaking, resources that need to be backed up together are resources with interdependencies -- pods with their PVCs, PVCs with their PVs, groups of pods that form a single application, CRs, pods, and other resources that belong to the same operator, etc. As part of this initial refactoring, once these "Item Blocks" are identified, an additional change will be to move pod hook processing up to the ItemBlock level. If there are multiple pods in the ItemBlock, pre-hooks for all pods will be run before backing up the items, followed by post-hooks for all pods. This change to hook processing is another prerequisite for future VolumeGroupSnapshot support, since supporting this will require backing up the pods and volumes together for any volumes which belong to the same group. Once we are backing up items by block, the next step will be to create multiple worker threads to process and back up ItemBlocks, so that we can back up multiple ItemBlocks at the same time. In looking at the different kinds of large backups that Velero must deal with, two obvious scenarios come to mind: 1. Backups with a relatively small number of large volumes 2. Backups with a large number of relatively small volumes. In case 1, the majority of the time spent on the backup is in the asynchronous phases -- CSI snapshot creation actions after the snaphandle exists, and DataUpload processing. In that case, parallel item processing will likely have a minimal impact on overall backup completion time. In case 2, the majority of time spent on the backup will likely be during the synchronous actions. Especially as regards CSI snapshot creation, the waiting for the VSC snaphandle to exist will result in significant passage of time with thousands of volumes. This is the sort of use case which will benefit the most from parallel item processing. ## Goals - Identify groups of related items to back up together (ItemBlocks). - Manage backup hooks at the ItemBlock level rather than per-item. - Using worker threads, back up ItemBlocks at the same time. ## Non Goals - Support VolumeGroupSnapshots: this is a future feature, although certain prerequisites for this enhancement are included in this proposal. - Process multiple backups in parallel: this is a future feature, although certain prerequisites for this enhancement are included in this proposal. - Refactoring plugin infrastructure to avoid RPC calls for internal plugins. - Restore performance improvements: this is potentially a future feature ## High-Level Design ### ItemBlock concept The updated design is based on a new struct/type called `ItemBlock`. Essentially, an `ItemBlock` is a group of items that must be backed up together in order to guarantee backup integrity. When we eventually split item backup across multiple worker threads, `ItemBlocks` will be kept together as the basic unit of backup. To facilitate this, a new plugin type, `ItemBlockAction` will allow relationships between items to be identified by velero -- any resources that must be backed up with other resources will need IBA plugins defined for them. Examples of `ItemBlocks` include: 1. A pod, its mounted PVCs, and the bound PVs for those PVCs. 2. A VolumeGroup (related PVCs and PVs) along with any pods mounting these volumes. 3. For a ReadWriteMany PVC, the PVC, its bound PV, and all pods mounting this PVC. ### Phase 1: ItemBlock processing - A new plugin type, `ItemBlockAction`, will be created - `ItemBlockAction` will contain the API method `GetRelatedItems`, which will be needed for determining which items to group together into `ItemBlocks`. - When processing the list of items returned from the item collector, instead of simply calling `BackupItem` on each in turn, we will use the `GetRelatedItems` API call to determine other items to include with the current item in an ItemBlock. Repeat recursively on each item returned. - Don't include an item in more than one ItemBlock -- if the next item from the item collector is already in a block, skip it. - Once ItemBlock is determined, call new func `BackupItemBlock` instead of `BackupItem`. - New func `BackupItemBlock` will call pre hooks for any pods in the block, then back up the items in the block (`BackupItem` will no longer run hooks directly), then call post hooks for any pods in the block. - The finalize phase will not be affected by the ItemBlock design, since this is just updating resources after async operations are completed on the items and there is no need to run these updates in parallel. ### Phase 2: Process ItemBlocks for a single backup in multiple threads - Concurrent `BackupItemBlock` operations will be executed by worker threads invoked by the backup controller, which will communicate with the backup controller operation via a shared channel. - The ItemBlock processing loop implemented in Phase 1 will be modified to send each newly-created ItemBlock to the shared channel rather than calling `BackupItemBlock` inline. - Users will be able to configure the number of workers available for concurrent `BackupItemBlock` operations. - Access to the BackedUpItems map must be synchronized ## Detailed Design ### Phase 1: ItemBlock processing #### New ItemBlockAction plugin type In order for Velero to identify groups of items to back up together in an ItemBlock, we need a way to identify items which need to be backed up along with the current item. While the current `Execute` BackupItemAction method does return a list of additional items which are required by the current item, we need to know this *before* we start the item backup. To support this, we need a new plugin type, `ItemBlockAction` (IBA) with an API method, `GetRelatedItems` which Velero will call on each item as it processes. The expectation is that the registered IBA plugins will return the same items as returned as additional items by the BIA `Execute` method, with the exception that items which are not created until calling `Execute` should not be returned here, as they don't exist yet. #### Proto definition (compiled into golang by protoc) The ItemBlockAction plugin type is defined as follows: ``` service ItemBlockAction { rpc AppliesTo(ItemBlockActionAppliesToRequest) returns (ItemBlockActionAppliesToResponse); rpc GetRelatedItems(ItemBlockActionGetRelatedItemsRequest) returns (ItemBlockActionGetRelatedItemsResponse); } message ItemBlockActionAppliesToRequest { string plugin = 1; } message ItemBlockActionAppliesToResponse { ResourceSelector ResourceSelector = 1; } message ItemBlockActionGetRelatedItemsRequest { string plugin = 1; bytes item = 2; bytes backup = 3; } message ItemBlockActionGetRelatedItemsResponse { repeated generated.ResourceIdentifier relatedItems = 1; } ``` A new PluginKind, `ItemBlockAction`, will be created, and the backup process will be modified to use this plugin kind. For any BIA plugins which return additional items from `Execute()` that need to be backed up at the same time or sequentially in the same worker thread as the current items should add a new IBA plugin to return these same items (minus any which won't exist before BIA `Execute()` is called). This mainly applies to plugins that operate on pods which reference resources which must be backed up along with the pod and are potentially affected by pod hooks or for plugins which connect multiple pods whose volumes should be backed up at the same time. ### Changes to processing item list from the Item Collector #### New structs BackupItemBlock, ItemBlock, and ItemBlockItem ```go package backup type BackupItemBlock struct { itemblock.ItemBlock // This is a reference to the shared itemBackupper for the backup itemBackupper *itemBackupper } package itemblock type ItemBlock struct { Log logrus.FieldLogger Items []ItemBlockItem } type ItemBlockItem struct { Gr schema.GroupResource Item *unstructured.Unstructured PreferredGVR schema.GroupVersionResource } ``` #### Current workflow In the `BackupWithResolvers` func, the current Velero implementation iterates over the list of items for backup returned by the Item Collector. For each item, Velero loads the item from the file created by the Item Collector, we call `backupItem`, update the GR map if successful, remove the (temporary) file containing item metadata, and update progress for the backup. #### Modifications to the loop over ItemCollector results The `kubernetesResource` struct used by the item collector will be modified to add an `orderedResource` bool which will be set true for all of the resources moved to the beginning for each GroupResource as a result of being ordered resources. In addition, an `inItemBlock` bool is added to the struct which will be set to true later when processing the list when each item is added to an ItemBlock. While the item collector already puts ordered resources first for each GR, there is no indication in the list which of these initial items are from the ordered resources list and which are the remaining (unordered) items. Velero needs to know which resources are ordered because when we process them later, the ordered resources for each GroupResource must be processed sequentially in a single ItemBlock. The current workflow within each iteration of the ItemCollector.items loop will replaced with the following: - (note that some of the below should be pulled out into a helper func to facilitate recursive call to it for items returned from `GetRelatedItems`.) - Before loop iteration, create a pointer to a `BackupItemBlock` which will represent the current ItemBlock being processed. - If `item` has `inItemBlock==true`, continue. This one has already been processed. - If current `itemBlock` is nil, create it. - Add `item` to `itemBlock`. - Load item from ItemCollector file. Close/remove file after loading (on error return or not, possibly with similar anonymous func to current impl) - If other versions of the same item exist (via EnableAPIGroupVersions), add these to the `itemBlock` as well (and load from ItemCollector file) - Get matching IBA plugins for item, call `GetRelatedItems` for each. For each item returned, get full item content from ItemCollector (if present in item list, pulling from file, removing file when done) or from cluster (if not present in item list), add item to the current block, add item to `itemsInBlock` map, and then recursively apply current step to each (i.e. call IBA method, add to block, etc.) - If current item and next item are both ordered items for the same GR, then continue to next item, adding to current `itemBlock`. - Once full ItemBlock list is generated, call `backupItemBlock(block ItemBlock) - Add `backupItemBlock` return values to `backedUpGroupResources` map #### New func `backupItemBlock` Method signature for new func `backupItemBlock` is as follows: ```go func (kb *kubernetesBackupper) backupItemBlock(block BackupItemBlock) []schema.GroupResource ``` The return value is a slice of GRs for resources which were backed up. Velero tracks these to determine which CRDs need to be included in the backup. Note that we need to make sure we include in this not only those resources that were backed up directly, but also those backed up indirectly via additional items BIA execute returns. In order to handle backup hooks, this func will first take the input item list (`block.items`) and get a list of included pods, filtered to include only those not yet backed up (using `block.itemBackupper.backupRequest.BackedUpItems`). Iterate over this list and execute pre hooks (pulled out of `itemBackupper.backupItemInternal`) for each item. Now iterate over the full list (`block.items`) and call `backupItem` for each. After the first, the later items should already have been backed up, but calling a second time is harmless, since the first thing Velero does is check the `BackedUpItems` map, exiting if item is already backed up). We still need this call in case there's a plugin which returns something in `GetAdditionalItems` but forgets to return it in the `Execute` additional items return value. If we don't do this, we could end up missing items. After backing up the items in the block, we now execute post hooks using the same filtered item list we used for pre hooks, again taking the logic from `itemBackupper.backupItemInternal`). #### `itemBackupper.backupItemInternal` cleanup After implementing backup hooks in `backupItemBlock`, hook processing should be removed from `itemBackupper.backupItemInternal`. ### Phase 2: Process ItemBlocks for a single backup in multiple threads #### New input field for number of ItemBlock workers The velero installer and server CLIs will get a new input field `itemBlockWorkerCount`, which will be passed along to the `backupReconciler`. The `backupReconciler` struct will also have this new field added. #### Worker pool for item block processing A new type, `ItemBlockWorker` will be added which will manage a pool of worker goroutines which will process item blocks, a shared input channel for passing blocks to workers, and a WaitGroup to shut down cleanly when the reconciler exits. ```go type ItemBlockWorkerPool struct { itemBlockChannel chan ItemBlockInput wg *sync.WaitGroup logger logrus.FieldLogger } type ItemBlockInput struct { itemBlock *BackupItemBlock returnChan chan ItemBlockReturn } type ItemBlockReturn struct { itemBlock *BackupItemBlock resources []schema.GroupResource err error } func (*p ItemBlockWorkerPool) getInputChannel() chan ItemBlockInput func StartItemBlockWorkerPool(context context.Context, workers int, logger logrus.FieldLogger) ItemBlockWorkerPool func processItemBlockWorker(context context.Context, itemBlockChannel chan ItemBlockInput, logger logrus.FieldLogger, wg *sync.WaitGroup) ``` The worker pool will be started by calling `StartItemBlockWorkerPool` in `NewBackupReconciler()`, passing in the worker count and reconciler context. `backupreconciler.prepareBackupRequest` will also add the input channel to the `backupRequest` so that it will be available during backup processing. The func `StartItemBlockWorkerPool` will create the `ItemBlockWorkerPool` with a shared buffered input channel (fixed buffer size) and start `workers` gororoutines which will each call `processItemBlockWorker`. The `processItemBlockWorker` func (run by the worker goroutines) will read from `itemBlockChannel`, call `BackupItemBlock` on the retrieved `ItemBlock`, and then send the return value to the retrieved `returnChan`, and then process the next block. #### Modify ItemBlock processing loop to send ItemBlocks to the worker pool rather than backing them up directly The ItemBlock processing loop implemented in Phase 1 will be modified to send each newly-created ItemBlock to the shared channel rather than calling `BackupItemBlock` inline, using a WaitGroup to manage in-process items. A separate goroutine will be created to process returns for this backup. After completion of the ItemBlock processing loop, velero will use the WaitGroup to wait for all ItemBlock processing to complete before moving forward. A simplified example of what this response goroutine might look like: ```go // omitting cancel handling, context, etc ret := make(chan ItemBlockReturn) wg := &sync.WaitGroup{} // Handle returns go func() { for { select { case response := <-ret: // process each BackupItemBlock response func() { defer wg.Done() responses = append(responses, response) }() case <-ctx.Done(): return } } }() // Simplified illustration, looping over and assumed already-determined ItemBlock list for _, itemBlock := range itemBlocks { wg.Add(1) inputChan <- ItemBlockInput{itemBlock: itemBlock, returnChan: ret} } done := make(chan struct{}) go func() { defer close(done) wg.Wait() }() // Wait for all the ItemBlocks to be processed select { case <-done: logger.Info("done processing ItemBlocks") } // responses from BackupItemBlock calls are in responses ``` When processing the responses, the main thing is to set `backedUpGroupResources[item.groupResource]=true` for each GR returned, which will give the same result as the current implementation calling items one-by-one and setting that field as needed. The ItemBlock processing loop described above will be split into two separate iterations. For the first iteration, velero will only process those items at the beginning of the loop identified as `orderedResources` -- when the groups generated from these resources are passed to the worker channel, velero will wait for the response before moving on to the next ItemBlock. This is to ensure that the ordered resources are processed in the required order. Once the last ordered resource is processed, the remaining ItemBlocks will be processed and sent to the worker channel without waiting for a response, in order to allow these ItemBlocks to be processed in parallel. The reason we must execute `ItemBlocks` with ordered resources first (and one at a time) is that this is a list of resources identified by the user as resources which must be backed up first, and in a particular order. #### Synchronize access to the BackedUpItems map Velero uses a map of BackedUpItems to track which items have already been backed up. This prevents velero from attempting to back up an item more than once, as well as guarding against creating infinite loops due to circular dependencies in the additional items returns. Since velero will now be accessing this map from the parallel goroutines, access to the map must be synchronized with mutexes. ### Backup Finalize phase The finalize phase will not be affected by the ItemBlock design, since this is just updating resources after async operations are completed on the items and there is no need to run these updates in parallel. ## Alternatives considered ### BackpuItemAction v3 API Instead of adding a new `ItemBlockAction` plugin type, we could add a `GetAdditionalItems` method to BackupItemAction. This was rejected because the new plugin type provides a cleaner interface, and keeps the function of grouping related items separate from the function of modifying item content for the backup. ### Per-backup worker pool The current design makes use of a permanent worker pool, started at backup controller startup time. With this design, when we follow on with running multiple backups in parallel, the same set of workers will take ItemBlock inputs from more than one backup. Another approach that was initially considered was a temporary worker pool, created while processing a backup, and deleted upon backup completion. #### User-visible API differences between the two approaches The main user-visible difference here is in the configuration API. For the permanent worker approach, the worker count represents the total worker count for all backups. The concurrent backup count represents the number of backups running at the same time. At any given time, though, the maximum number of worker threads backing up items concurrently is equal to the worker count. If worker count is 15 and the concurrent backup count is 3, then there will be, at most, 15 items being processed at the same time, split among up to three running backups. For the per-backup worker approach, the worker count represents the worker count for each backup. The concurrent backup count, as before, represents the number of backups running at the same time. If worker count is 15 and the concurrent backup count is 3, then there will be, at most, 45 items being processed at the same time, up to 15 for each of up to three running backups. #### Comparison of the two approaches - Permanent worker pool advantages: - This is the more commonly-followed Kubernetes pattern. It's generally better to follow standard practices, unless there are genuine reasons for the use case to go in a different way. - It's easier for users to understand the maximum number of concurrent items processed, which will have performance impact and impact on the resource requirements for the Velero pod. Users will not have to multiply the config numbers in their heads when working out how many total workers are present. - It will give us more flexibility for future enhancements around concurrent backups. One possible use case: backup priority. Maybe a user wants scheduled backups to have a lower priority than user-generated backups, since a user is sitting there waiting for completion -- a shared worker pool could react to the priority by taking ItemBlocks for the higher priority backup first, which would allow a large lower-priority backup's items to be preempted by a higher-priority backup's items without needing to explicitly stop the main controller flow for that backup. - Per-backup worker pool advantages: - Lower memory consumption than permanent worker pool, but the total memory used by a worker blocked on input will be pretty low, so if we're talking only 10-20 workers, the impact will be minimal. ## Compatibility ### Example IBA implementation for BIA plugins which return additional items Included below is an example of what might be required for a BIA plugin which returns additional items. The code is taken from the internal velero `pod_action.go` which identifies the items required for a given pod. In this particular case, the only function of pod_action is to return additional items, so we can really just convert this plugin to an IBA plugin. If there were other actions, such as modifying the pod content on backup, then we would still need the pod action, and the related items vs. content manipulation functions would need to be separated. ```go // PodAction implements ItemBlockAction. type PodAction struct { log logrus.FieldLogger } // NewPodAction creates a new ItemAction for pods. func NewPodAction(logger logrus.FieldLogger) *PodAction { return &PodAction{log: logger} } // AppliesTo returns a ResourceSelector that applies only to pods. func (a *PodAction) AppliesTo() (velero.ResourceSelector, error) { return velero.ResourceSelector{ IncludedResources: []string{"pods"}, }, nil } // GetRelatedItems scans the pod's spec.volumes for persistentVolumeClaim volumes and returns a // ResourceIdentifier list containing references to all of the persistentVolumeClaim volumes used by // the pod. This ensures that when a pod is backed up, all referenced PVCs are backed up too. func (a *PodAction) GetRelatedItems(item runtime.Unstructured, backup *v1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, error) { pod := new(corev1api.Pod) if err := runtime.DefaultUnstructuredConverter.FromUnstructured(item.UnstructuredContent(), pod); err != nil { return nil, errors.WithStack(err) } var relatedItems []velero.ResourceIdentifier if pod.Spec.PriorityClassName != "" { a.log.Infof("Adding priorityclass %s to relatedItems", pod.Spec.PriorityClassName) relatedItems = append(relatedItems, velero.ResourceIdentifier{ GroupResource: kuberesource.PriorityClasses, Name: pod.Spec.PriorityClassName, }) } if len(pod.Spec.Volumes) == 0 { a.log.Info("pod has no volumes") return relatedItems, nil } for _, volume := range pod.Spec.Volumes { if volume.PersistentVolumeClaim != nil && volume.PersistentVolumeClaim.ClaimName != "" { a.log.Infof("Adding pvc %s to relatedItems", volume.PersistentVolumeClaim.ClaimName) relatedItems = append(relatedItems, velero.ResourceIdentifier{ GroupResource: kuberesource.PersistentVolumeClaims, Namespace: pod.Namespace, Name: volume.PersistentVolumeClaim.ClaimName, }) } } return relatedItems, nil } // API call func (a *PodAction) Name() string { return "PodAction" } ``` ## Implementation Phase 1 and Phase 2 could be implemented within the same Velero release cycle, but they need not be. Phase 1 is expected to be implemented in Velero 1.15. Phase 2 is expected to be implemented in Velero 1.16. ================================================ FILE: design/Implemented/backup-pvc-config.md ================================================ # Backup PVC Configuration Design ## Glossary & Abbreviation **Velero Generic Data Path (VGDP)**: VGDP is the collective modules that is introduced in [Unified Repository design][1]. Velero uses these modules to finish data transfer for various purposes (i.e., PodVolume backup/restore, Volume Snapshot Data Movement). VGDP modules include uploaders and the backup repository. **Exposer**: Exposer is a module that is introduced in [Volume Snapshot Data Movement Design][2]. Velero uses this module to expose the volume snapshots to Velero node-agent pods or node-agent associated pods so as to complete the data movement from the snapshots. **backupPVC**: The intermediate PVC created by the exposer for VGDP to access data from, see [Volume Snapshot Data Movement Design][2] for more details. **backupPod**: The pod consumes the backupPVC so that VGDP could access data from the backupPVC, see [Volume Snapshot Data Movement Design][2] for more details. **sourcePVC**: The PVC to be backed up, see [Volume Snapshot Data Movement Design][2] for more details. ## Background As elaberated in [Volume Snapshot Data Movement Design][2], a backupPVC may be created by the Exposer and the VGDP reads data from the backupPVC. In some scenarios, users may need to configure some advanced settings of the backupPVC so that the data movement could work in best performance in their environments. Specifically: - For some storage providers, when creating a read-only volume from a snapshot, it is very fast; whereas, if a writable volume is created from the snapshot, they need to clone the entire disk data, which is time consuming. If the backupPVC's `accessModes` is set as `ReadOnlyMany`, the volume driver is able to tell the storage to create a read-only volume, which may dramatically shorten the snapshot expose time. On the other hand, `ReadOnlyMany` is not supported by all volumes. Therefore, users should be allowed to configure the `accessModes` for the backupPVC. - Some storage providers create one or more replicas when creating a volume, the number of replicas is defined in the storage class. However, it doesn't make any sense to keep replicas when an intermediate volume used by the backup. Therefore, users should be allowed to configure another storage class specifically used by the backupPVC. ## Goals - Create a mechanism for users to specify various configurations for backupPVC ## Non-Goals ## Solution We will use the ConfigMap specified by `velero node-agent` CLI's parameter `--node-agent-configmap` to host the backupPVC configurations. This configMap is not created by Velero, users should create it manually on demand. The configMap should be in the same namespace where Velero is installed. If multiple Velero instances are installed in different namespaces, there should be one configMap in each namespace which applies to node-agent in that namespace only. Node-agent server checks these configurations at startup time and use it to initiate the related Exposer modules. Therefore, users could edit this configMap any time, but in order to make the changes effective, node-agent server needs to be restarted. Inside the ConfigMap we will add one new kind of configuration as the data in the configMap, the name is ```backupPVC```. Users may want to set different backupPVC configurations for different volumes, therefore, we define the configurations as a map and allow users to specific configurations by storage class. Specifically, the key of the map element is the storage class name used by the sourcePVC and the value is the set of configurations for the backupPVC created for the sourcePVC. The data structure is as below: ```go type Configs struct { // LoadConcurrency is the config for data path load concurrency per node. LoadConcurrency *LoadConcurrency `json:"loadConcurrency,omitempty"` // LoadAffinity is the config for data path load affinity. LoadAffinity []*LoadAffinity `json:"loadAffinity,omitempty"` // BackupPVC is the config for backupPVC of snapshot data movement. BackupPVC map[string]BackupPVC `json:"backupPVC,omitempty"` } type BackupPVC struct { // StorageClass is the name of storage class to be used by the backupPVC. StorageClass string `json:"storageClass,omitempty"` // ReadOnly sets the backupPVC's access mode as read only. ReadOnly bool `json:"readOnly,omitempty"` } ``` ### Sample A sample of the ConfigMap is as below: ```json { "backupPVC": { "storage-class-1": { "storageClass": "snapshot-storage-class", "readOnly": true }, "storage-class-2": { "storageClass": "snapshot-storage-class" }, "storage-class-3": { "readOnly": true } } } ``` To create the configMap, users need to save something like the above sample to a json file and then run below command: ``` kubectl create cm -n velero --from-file= ``` ### Implementation The `backupPVC` is passed to the exposer and the exposer sets the related specification and create the backupPVC. If `backupPVC.storageClass` doesn't exist or set as empty, the sourcePVC's storage class will be used. If `backupPVC.readOnly` is set to true, `ReadOnlyMany` will be the only value set to the backupPVC's `accessModes`, otherwise, `ReadWriteOnce` is used. Once `backupPVC.storageClass` is set, users must make sure that the specified storage class exists in the cluster and can be used the the backupPVC, otherwise, the corresponding DataUpload CR will stay in `Accepted` phase until the prepare timeout (by default 30min). Once `backupPVC.readOnly` is set to true, users must make sure that the storage supports to create a `ReadOnlyMany` PVC from a snapshot, otherwise, the corresponding DataUpload CR will stay in `Accepted` phase until the prepare timeout (by default 30min). Once above problems happen, the DataUpload CR is cancelled after prepare timeout and the backupPVC and backupPod will be deleted, so there is no way to tell the cause is one of the above problems or others. To help the troubleshooting, we can add some diagnostic mechanism to discover the status of the backupPod before deleting it as a result of the prepare timeout. [1]: unified-repo-and-kopia-integration/unified-repo-and-kopia-integration.md [2]: volume-snapshot-data-movement/volume-snapshot-data-movement.md ================================================ FILE: design/Implemented/backup-repo-cache-volume.md ================================================ # Backup Repository Cache Volume Design ## Glossary & Abbreviation **Backup Storage**: The storage to store the backup data. Check [Unified Repository design][1] for details. **Backup Repository**: Backup repository is layered between BR data movers and Backup Storage to provide BR related features that is introduced in [Unified Repository design][1]. **Velero Generic Data Path (VGDP)**: VGDP is the collective of modules that is introduced in [Unified Repository design][1]. Velero uses these modules to finish data transfer for various purposes (i.e., PodVolume backup/restore, Volume Snapshot Data Movement). VGDP modules include uploaders and the backup repository. **Data Mover Pods**: Intermediate pods which hold VGDP and complete the data transfer. See [VGDP Micro Service for Volume Snapshot Data Movement][2] and [VGDP Micro Service For fs-backup][3] for details. **Repository Maintenance Pods**: Pods for [Repository Maintenance Jobs][4], which holds VGDP to run repository maintenance. ## Background According to the [Unified Repository design][1] Velero uses selectable backup repositories for various backup/restore methods, i.e., fs-backup, volume snapshot data movement, etc. Some backup repositories may need to cache data on the client side for various repository operation, so as to accelerate the execution. In the existing [Backup Repository Configuration][5], we allow users to configure the cache data size (`cacheLimitMB`). However, the cache data is still stored in the root file system of data mover pods/repository maintenance pods, so stored in the root file system of the node. This is not good enough, reasons: - In many distributions, the node's system disk size is predefined, non configurable and limit, e.g., the system disk size may be 20G or less - Velero supports concurrent data movements in each node. The cache in each of the concurrent data mover pods could quickly run out of the system disk and cause problems like pod eviction, failure of pod creation, degradation of Kubernetes QoS, etc. We need to allow users to prepare a dedicated location, e.g., a dedictated volume, for the cache. Not all backup repositories or not all backup repository operations require cache, we need to define the details when and how the cache is used. ## Goals - Create a mechanism for users to configure cache volumes for various pods running VGDP - Design the workflow to assign the cache volume pod path to backup repositories - Describe when and how the cache volume is used ## Non-Goals - The solution is based on [Unified Repository design][1], [VGDP Micro Service for Volume Snapshot Data Movement][2] and [VGDP Micro Service For fs-backup][3], legacy data paths are not supported. E.g., when a pod volume restore (PVR) runs with legacy Restic path, if any data is cached, the cache still resides in the root file system. ## Solution ### Cache Data Varying on backup repositoires, cache data may include payload data or repository metadata, e.g., indexes to the payload data chunks. Payload data is highly related to the backup data, and normally take the majority of the repository data as well as the cache data. Repository metadata is related to the backup repository's chunking algorithm, data chunk mapping method, etc, and so the size is not proportional to the backup data size. On the other hand for some backup repository, in extreme cases, the repository metadata may be significantly large. E.g., Kopia's indexes are per chunks, if there are huge number of small files in the repository, Kopia's index data may be in the same level of or even larger than the payload data. However, in the cases that repository metadata data become the majority, other bottlenecks may emerge and concurrency of data movers may be significantly constrained, so the requirement to cache volumes may go away. Therefore, for now we only consider the cache volume requirement for payload data, and leave the consideration for metadata as a future enhancement. ### Scenarios Backup repository cache varies on backup repositories and backup repository operation during VGDP runs. Below are the scenarios when VGDP runs: - Data Upload for Backup: this is the process to upload/write the backup data into the backup repository, e.g., DataUpload or PodVolumeBackup. The pieces of data is almost directly written to the repository, sometimes with a small group staying shortly in the local place. That is to say, there should not be large scale data cached for this scenario, so we don't prepare dedicated cache for this scenario. - Repository Maintenance: Repository maintenance most often visits the backup repository's metadata and sometimes it needs to visit the file system directories from the backed up data. On the other hand, it is not practical to run concurrent maintenance jobs in one node. So the cache data is neither large nor affect the root file system too much. Therefore, we don't need to prepare dedicated cache for this scenario. - Data Download for Restore: this is the process to download/read the backup data from the backup repository during restore, e.g., DataDownload or PodVolumeRestore. For backup repositories for which data are stored in remote backup storages (e.g., Kopia repository stores data in remote object stores), large scale of data are cached locally to accerlerate the restore. Therefore, we need dedicate cache volumes for this scenario. - Backup Deletion: During this scenario, backup repository is connected, metadata is enumerated to find the repository snapshot representing the backup data. That is to say, only metadata is cached if any. Therefore, dedicated cache volumes are not required in this scenario. The above analyses are based on the common behavior of backup repositories and they are not considering the case that backup repository metadata takes majority or siginficant proportion of the cache data. As a conclusion of the analyses, we will create dedicated cache volumes for restore scenarios. For other scenarios, we can add them regarded to the future changes/requirements. The mechanism to expose and connect the cache volumes should work for all scenarios. E.g., if we need to consider the backup repository metadata case, we may need cache volumes for backup and repository maintenance as well, then we can just reuse the same cache volume provision and connection mechanism to backup and repository maintenance scenarios. ### Cache Data and Lifecycle If available, one cache volume is dedicately assigned to one data mover pod. That is, the cached data is destroyed when the data mover pod completes. Then the backup repository instance also closes. Cache data are fully managed by the specific backup repository. So the backup repository may also have its own way to GC the cache data. That is to say, cache data GC may be launched by the backup repository instance during the running of the data mover pod; then the left data are automatically destroyed when the data mover pod and the cache PVC are destroyed (cache PVC's `reclaimPolicy` is always `Deleted`, so once the cache PVC is destroyed, the volume will also be destroyed). So no specially logics are needed for cache data GC. ### Data Size Cache volumes take storage space and cluster resources (PVC, PV), therefore, cache volumes should be created only when necessary and the volumes should be with reasonable size based on the cache data size: - It is not a good bargain to have cache volumes for small backups, small backups will use resident cache location (the cache location in the root file system) - The cache data size has a limit, the existing `cacheLimitMB` is used for this purpose. E.g., it could be set as 1024 for a 1TB backup, which means 1GB of data is cached and the old cache data exceeding this size will be cleared. Therefore, it is meaningless to set the cache volume size much larger than `cacheLimitMB` ### Cache Volume Size The cache volume size is calculated from below factors (for Restore scenarios): - **Limit**: The limit of the cache data, that is represented by `cacheLimitMB`, the default value is 5GB - **backupSize**: The size of the backup as a reference to evaluate whether to create a cache volume. It doesn't mean the backup data really decides the cache data all the time, it is just a reference to evaluate the scale of the backup, small scale backups may need small cache data. Sometimes, backupSize is not irrelevant to the size of cache data, in this case, ResidentThreshold should not be set, Limit will be used directly. It is unlikely that backupSize is unavailable, but once that happens, ResidentThreshold is ignored, Limit will be used directly. - **ResidentThreshold**: The minimum backup size that a cache volume is created - **InflationPercentage**: Considering the overhead of the file system and the possible delay of the cache cleanup, there should be an inflation for the final volume size vs. the logical size, otherwise, the cache volume may be overrun. This inflation percentage is hardcoded, e.g., 20%. A formula is as below: ``` cacheVolumeSize = ((backupSize != 0 ? (backupSize > residentThreshold ? limit : 0) : limit) * (100 + inflationPercentage)) / 100 ``` Finally, the `cacheVolumeSize` will be rounded up to GiB considering the UX friendliness, storage friendliness and management friendliness. ### PVC/PV The PVC for a cache volume is created in Velero namespace and a storage class is required for the cache PVC. The PVC's accessMode is `ReadWriteOnce` and volumeMode is `FileSystem`, so the storage class provided should support this specification. Otherwise, if the storageclass doesn't support either of the specifications, the data mover pod may be hang in `Pending` state until a timeout setting with the data movement (e.g. `prepareTimeout`) and the data movement will finally fail. It is not expected that the cache volume is retained after data mover pod is deleted, so the `reclaimPolicy` for the storageclass must be `Delete`. To detect the problems in the storageclass and fail earlier, a validation is applied to the storageclass and once the validation fails, the cache configuration will be ignored, so the data mover pod will be created without a cache volume. ### Cache Volume Configurations Below configurations are introduced: - **residentThresholdMB**: the minimum data size(in MB) to be processed (if available) that a cache volume is created - **cacheStorageClass**: the name of the storage class to provision the cache PVC Not like `cacheLimitMB` which is set to and affect the backup repository, the above two configurations are actually data mover configurations of how to create cache volumes to data mover pods; and the two configurations don't need to be per backup repository. So we add them to the node-agent Configuration. ### Sample Below are some examples of the node-agent configMap with the configurations: Sample-1: ```json { "cacheVolume": { "storageClass": "sc-1", "residentThresholdMB": 1024 } } ``` Sample-2: ```json { "cacheVolume": { "storageClass": "sc-1", } } ``` Sample-3: ```json { "cacheVolume": { "residentThresholdMB": 1024 } } ``` **sample-1**: This is a valid configuration. Restores with backup data size larger than 1G will be assigned a cache volume using storage class `sc-1`. **sample-2**: This is a valid configuration. Data mover pods are always assigned a cache volume using storage class `sc-1`. **sample-3**: This is not a valid configuration because the storage class is absent. Velero gives up creating a cache volume. To create the configMap, users need to save something like the above sample to a json file and then run below command: ``` kubectl create cm -n velero --from-file= ``` The cache volume configurations will be visited by node-agent server, so they also need to specify the `--node-agent-configmap` to the `velero node-agent` parameters. ## Detailed Design ### Backup and Restore The restore needs to know the backup size so as to calculate the cache volume size, some new fields are added to the DataDownload and PodVolumeRestore CRDs. `snapshotSize` field is also added to DataDownload and PodVolumeRestore's `spec`: ```yaml spec: snapshotID: description: SnapshotID is the ID of the Velero backup snapshot to be restored from. type: string snapshotSize: description: SnapshotSize is the logical size of the snapshot. format: int64 type: integer ``` `snapshotSize` represents the total size of the backup; during restore, the value is transferred from DataUpload/PodVolumeBackup's `Status.Progress.TotalBytes` to DataDownload/PodVolumeRestore. It is unlikely that `Status.Progress.TotalBytes` from DataUpload/PodVolumeBackup is unavailable, but once it happens, according to the above formula, `residentThresholdMB` is ignored, cache volume size is calculated directly from cache limit for the corresponding backup repository. ### Exposer Cache volume configurations are retrieved by node-agent and passed through DataDownload/PodVolumeRestore to GenericRestore exposer/PodVolume exposer. The exposers are responsible to calculate cache volume size, create cache PVCs and mount them to the restorePods. If the calculated cache volume size is 0, or any of the critical parameters is missing (e.g., cache volume storage class), the exposers ignore the cache volume configuration and continue with creating restorePods without cache volumes, so no impact to the result of the restore. Exposers mount the cache volume to a predefined directory and pass the directory to the data mover pods through the `cache-volume-path` parameter. Below data structure is added to the exposers' expose parameters: ```go type GenericRestoreExposeParam struct { // RestoreSize specifies the data size for the volume to be restored RestoreSize int64 // CacheVolume specifies the info for cache volumes CacheVolume *CacheVolumeInfo } type PodVolumeExposeParam struct { // RestoreSize specifies the data size for the volume to be restored RestoreSize int64 // CacheVolume specifies the info for cache volumes CacheVolume *repocache.CacheConfigs } type CacheConfigs struct { // StorageClass specifies the storage class for cache volumes StorageClass string // Limit specifies the maximum size of the cache data Limit int64 // ResidentThreshold specifies the minimum size of the cache data to create a cache volume ResidentThreshold int64 } ``` ### Data Mover Pods Data mover pods retrieve the cache volume directory from `cache-volume-path` parameter and pass it to Unified Repository. If the directory is empty, Unified Repository uses the resident location for data cache, that is, the root file system. ### Kopia Repository Kopia repository supports cache directory configuration for both metadata and data. The existing `SetupConnectOptions` is modified to customize the `CacheDirectory`: ```go func SetupConnectOptions(ctx context.Context, repoOptions udmrepo.RepoOptions) repo.ConnectOptions { ... return repo.ConnectOptions{ CachingOptions: content.CachingOptions{ CacheDirectory: cacheDir, ... }, ... } } ``` [1]: Implemented/unified-repo-and-kopia-integration/unified-repo-and-kopia-integration.md [2]: Implemented/vgdp-micro-service/vgdp-micro-service.md [3]: Implemented/vgdp-micro-service-for-fs-backup/vgdp-micro-service-for-fs-backup.md [4]: Implemented/repo_maintenance_job_config.md [5]: Implemented/backup-repo-config.md ================================================ FILE: design/Implemented/backup-repo-config.md ================================================ # Backup Repository Configuration Design ## Glossary & Abbreviation **Backup Storage**: The storage to store the backup data. Check [Unified Repository design][1] for details. **Backup Repository**: Backup repository is layered between BR data movers and Backup Storage to provide BR related features that is introduced in [Unified Repository design][1]. ## Background According to the [Unified Repository design][1] Velero uses selectable backup repositories for various backup/restore methods, i.e., fs-backup, volume snapshot data movement, etc. To achieve the best performance, backup repositories may need to be configured according to the running environments. For example, if there are sufficient CPU and memory resources in the environment, users may enable compression feature provided by the backup repository, so as to achieve the best backup throughput. As another example, if the local disk space is not sufficient, users may want to constraint the backup repository's cache size, so as to prevent the repository from running out of the disk space. Therefore, it is worthy to allow users to configure some essential parameters of the backup repsoitories, and the configuration may vary from backup repositories. ## Goals - Create a mechanism for users to specify configurations for backup repositories ## Non-Goals ## Solution ### BackupRepository CRD After a backup repository is initialized, a BackupRepository CR is created to represent the instance of the backup repository. The BackupRepository's spec is a core parameter used by Unified Repo modules when interactive with the backup repsoitory. Therefore, we can add the configurations into the BackupRepository CR called ```repositoryConfig```. The configurations may be different varying from backup repositories, therefore, we will not define each of the configurations explicitly. Instead, we add a map in the BackupRepository's spec to take any configuration to be set to the backup repository. During various operations to the backup repository, the Unified Repo modules will retrieve from the map for the specific configuration that is required at that time. So even though it is specified, a configuration may not be visited/hornored if the operations don't require it for the specific backup repository, this won't bring any issue. When and how a configuration is hornored is decided by the configuration itself and should be clarified in the configuration's specification. Below is the new BackupRepository's spec after adding the configuration map: ```yaml spec: description: BackupRepositorySpec is the specification for a BackupRepository. properties: backupStorageLocation: description: |- BackupStorageLocation is the name of the BackupStorageLocation that should contain this repository. type: string maintenanceFrequency: description: MaintenanceFrequency is how often maintenance should be run. type: string repositoryConfig: additionalProperties: type: string description: RepositoryConfig contains configurations for the specific repository. type: object repositoryType: description: RepositoryType indicates the type of the backend repository enum: - kopia - restic - "" type: string resticIdentifier: description: |- ResticIdentifier is the full restic-compatible string for identifying this repository. type: string volumeNamespace: description: |- VolumeNamespace is the namespace this backup repository contains pod volume backups for. type: string required: - backupStorageLocation - maintenanceFrequency - resticIdentifier - volumeNamespace type: object ``` ### BackupRepository configMap The BackupRepository CR is not created explicitly by a Velero CLI, but created as part of the backup/restore/maintenance operation if the CR doesn't exist. As a result, users don't have any way to specify the configurations before the BackupRepository CR is created. Therefore, a BackupRepository configMap is introduced as a template of the configurations to be applied to the backup repository CR. When the backup repository CR is created by the BackupRepository controller, the configurations in the configMap are copied to the ```repositoryConfig``` field. For an existing BackupRepository CR, the configMap is never visited, if users want to modify the configuration value, they should directly edit the BackupRepository CR. The BackupRepository configMap is created by users in velero installation namespace. The configMap name must be specified in the velero server parameter ```--backup-repository-configmap```, otherwise, it won't effect. If the configMap name is specified but the configMap doesn't exist by the time of a backup repository is created, the configMap name is ignored. For any reason, if the configMap doesn't effect, nothing is specified to the backup repository CR, so the Unified Repo modules use the hard-coded values to configure the backup repository. The BackupRepository configMap supports backup repository type specific configurations, even though users can only specify one configMap. So in the configMap struct, multiple entries are supported, indexed by the backup repository type. During the backup repository creation, the configMap is searched by the repository type. ### Configurations With the above mechanisms, any kind of configuration could be added. Here list the configurations defined at present: ```cacheLimitMB```: specifies the size limit(in MB) for the local data cache. The more data is cached locally, the less data may be downloaded from the backup storage, so the better performance may be achieved. Practically, users can specify any size that is smaller than the free space so that the disk space won't run out. This parameter is for each repository connection, that is, users could change it before connecting to the repository. If a backup repository doesn't use local cache, this parameter will be ignored. For Kopia repository, this parameter is supported. ```enableCompression```: specifies to enable/disable compression for a backup repsotiory. Most of the backup repositories support the data compression feature, if it is not supported by a backup repository, this parameter is ignored. Most of the backup repositories support to dynamically enable/disable compression, so this parameter is defined to be used whenever creating a write connection to the backup repository, if the dynamically changing is not supported, this parameter will be hornored only when initializing the backup repository. For Kopia repository, this parameter is supported and can be dynamically modified. ### Sample Below is an example of the BackupRepository configMap with the configurations: ```yaml apiVersion: v1 kind: ConfigMap metadata: name: namespace: velero data: : | { "cacheLimitMB": 2048, "enableCompression": true } : | { "cacheLimitMB": 1, "enableCompression": false } ``` To create the configMap, users need to save something like the above sample to a file and then run below commands: ``` kubectl apply -f ``` [1]: unified-repo-and-kopia-integration/unified-repo-and-kopia-integration.md ================================================ FILE: design/Implemented/backup-resource-list.md ================================================ # Expose list of backed up resources in backup details Status: Accepted To increase the visibility of what a backup might contain, this document proposes storing metadata about backed up resources in object storage and adding a new section to the detailed backup description output to list them. ## Goals - Include a list of backed up resources as metadata in the bucket - Enable users to get a view of what resources are included in a backup using the Velero CLI ## Non Goals - Expose the full manifests of the backed up resources ## Background As reported in [#396](https://github.com/heptio/velero/issues/396), the information reported in a `velero backup describe --details` command is fairly limited, and does not easily describe what resources a backup contains. In order to see what a backup might contain, a user would have to download the backup tarball and extract it. This makes it difficult to keep track of different backups in a cluster. ## High-Level Design After performing a backup, a new file will be created that contains the list of the resources that have been included in the backup. This file will be persisted in object storage alongside the backup contents and existing metadata. A section will be added to the output of `velero backup describe --details` command to view this metadata. ## Detailed Design ### Metadata file This metadata will be in JSON (or YAML) format so that it can be easily inspected from the bucket outside of Velero tooling, and will contain the API resource and group, namespaces and names of the resources: ``` apps/v1/Deployment: - default/database - default/wordpress v1/Service: - default/database - default/wordpress v1/Secret: - default/database-root-password - default/database-user-password v1/ConfigMap: - default/database v1/PersistentVolume: - my-pv ``` The filename for this metadata will be `-resource-list.json.gz`. The top-level key is the string form of the `schema.GroupResource` type that we currently keep track of in the backup controller code path. ### Changes in Backup controller The Backupper currently initialises a map to track the `backedUpItems` (https://github.com/heptio/velero/blob/1594bdc8d0132f548e18ffcc1db8c4cd2b042726/pkg/backup/backup.go#L269), this is passed down through GroupBackupper, ResourceBackupper and ItemBackupper where ItemBackupper records each backed up item. This property will be moved to the [Backup request struct](https://github.com/heptio/velero/blob/16910a6215cbd8f0bde385dba9879629ebcbcc28/pkg/backup/request.go#L11), allowing the BackupController to access it after a successful backup. `backedUpItems` currently uses the `schema.GroupResource` as a key for the resource. In order to record the API group, version and kind for the resource, this key will be constructed from the object's `schema.GroupVersionKind` in the format `{group}/{version}/{kind}` (e.g. `apps/v1/Deployment`). The `backedUpItems` map is kept as a flat structure internally for quick lookup. When the backup is ready to upload, `backedUpItems` will be converted to a nested structure representing the metadata file above, grouped by `schema.GroupVersionKind`. After converting to the right format, it can be passed to the `persistBackup` function to persist the file in object storage. ### Changes to DownloadRequest CRD and processing A new `DownloadTargetKind` "BackupResourceList" will be added to the DownloadRequest CR. The `GetDownloadURL` function in the `persistence` package will be updated to handle this new DownloadTargetKind to enable the Velero client to fetch the metadata from the bucket. ### Changes to `velero backup describe --details` This command will need to be updated to fetch the metadata from the bucket using the `Stream` method used in other commands. The file will be read in memory and displayed in the output of the command. Depending on the format the metadata is stored in, it may need processing to print in a more human-readable format. If we choose to store the metadata in YAML, it can likely be directly printed out. If the metadata file does not exist, this is an older backup and we cannot display the list of resources that were backed up. ## Open Questions ## Alternatives Considered ### Fetch backup contents archive and walkthrough to list contents Instead of recording new metadata about what resources have been backed up, we could simply download the backup contents archive and walkthrough it to list the contents every time `velero backup describe --details` is run. The advantage of this approach is that we don't need to change any backup procedures as we already have this content, and we will also be able to list resources for older backups. Additionally, if we wanted to expose more information about the backed up resources, we can do so without having to update what we store in the metadata. The disadvantages are: - downloading the whole backup archive will be larger than just downloading a smaller file with metadata - reduces the metadata available in the bucket that users might want to inspect outside of Velero tooling (though this is not an explicit requirement) ## Security Considerations ================================================ FILE: design/Implemented/backup-resources-order.md ================================================ ## Backup Resources Order This document proposes a solution that allows user to specify a backup order for resources of specific resource type. ## Background During backup process, user may need to back up resources of specific type in some specific order to ensure the resources were backup properly because these resources are related and ordering might be required to preserve the consistency for the apps to recover itself from the backup image (Ex: primary-secondary database pods in a cluster). ## Goals - Enable user to specify an order of backup resources belong to specific resource type ## Alternatives Considered - Use a plugin to backup an resources and all the sub resources. For example use a plugin for StatefulSet and backup pods belong to the StatefulSet in specific order. This plugin solution is not generic and requires plugin for each resource type. ## High-Level Design User will specify a map of resource type to list resource names (separate by semicolons). Each name will be in the format "namespaceName/resourceName" to enable ordering across namespaces. Based on this map, the resources of each resource type will be sorted by the order specified in the list of resources. If a resource instance belong to that specific type but its name is not in the order list, then it will be put behind other resources that are in the list. ### Changes to BackupSpec Add new field to BackupSpec type BackupSpec struct { ... // OrderedResources contains a list of key-value pairs that represent the order // of backup of resources that belong to specific resource type // +optional // +nullable OrderedResources map[string]string } ### Changes to itemCollector Function getResourceItems collects all items belong to a specific resource type. This function will be enhanced to check with the map to see whether the OrderedResources has specified the order for this resource type. If such order exists, then sort the items by such order being process before return. ### Changes to velero CLI Add new flag "--ordered-resources" to Velero backup create command which takes a string of key-values pairs which represents the map between resource type and the order of the items of such resource type. Key-value pairs are separated by semicolon, items in the value are separated by commas. Example: >velero backup create mybackup --ordered-resources "pod=ns1/pod1,ns1/pod2;persistentvolumeclaim=n2/slavepod,ns2/primarypod" ## Open Issues - In the CLI, the design proposes to use commas to separate items of a resource type and semicolon to separate key-value pairs. This follows the convention of using commas to separate items in a list (For example: --include-namespaces ns1,ns2). However, the syntax for map in labels and annotations use commas to separate key-value pairs. So it introduces some inconsistency. - For pods that managed by Deployment or DaemonSet, this design may not work because the pods' name is randomly generated and if pods are restarted, they would have different names so the Backup operation may not consider the restarted pods in the sorting algorithm. This problem will be addressed when we enhance the design to use regular expression to specify the OrderResources instead of exact match. ================================================ FILE: design/Implemented/biav2-design.md ================================================ # Design for BackupItemAction v2 API ## Abstract This design includes the changes to the BackupItemAction (BIA) api design as required by the [Item Action Progress Monitoring](general-progress-monitoring.md) feature. The BIA v2 interface will have two new methods, and the Execute() return signature will be modified. If there are any additional BIA API changes that are needed in the same Velero release cycle as this change, those can be added here as well. ## Background This API change is needed to facilitate long-running plugin actions that may not be complete when the Execute() method returns. It is an optional feature, so plugins which don't need this feature can simply return an empty operation ID and the new methods can be no-ops. This will allow long-running plugin actions to continue in the background while Velero moves on to the next plugin, the next item, etc. ## Goals - Allow for BIA Execute() to optionally initiate a long-running operation and report on operation status. ## Non Goals - Allowing velero control over when the long-running operation begins. ## High-Level Design As per the [Plugin Versioning](plugin-versioning.md) design, a new BIAv2 plugin `.proto` file will be created to define the GRPC interface. v2 go files will also be created in `plugin/clientmgmt/backupitemaction` and `plugin/framework/backupitemaction`, and a new PluginKind will be created. The velero Backup process will be modified to reference v2 plugins instead of v1 plugins. An adapter will be created so that any existing BIA v1 plugin can be executed as a v2 plugin when executing a backup. ## Detailed Design ### proto changes (compiled into golang by protoc) The v2 BackupItemAction.proto will be like the current v1 version with the following changes: ExecuteResponse gets a new field: ``` message ExecuteResponse { bytes item = 1; repeated generated.ResourceIdentifier additionalItems = 2; string operationID = 3; repeated generated.ResourceIdentifier itemsToUpdate = 4; } ``` The BackupItemAction service gets two new rpc methods: ``` service BackupItemAction { rpc AppliesTo(BackupItemActionAppliesToRequest) returns (BackupItemActionAppliesToResponse); rpc Execute(ExecuteRequest) returns (ExecuteResponse); rpc Progress(BackupItemActionProgressRequest) returns (BackupItemActionProgressResponse); rpc Cancel(BackupItemActionCancelRequest) returns (google.protobuf.Empty); } ``` To support these new rpc methods, we define new request/response message types: ``` message BackupItemActionProgressRequest { string plugin = 1; string operationID = 2; bytes backup = 3; } message BackupItemActionProgressResponse { generated.OperationProgress progress = 1; } message BackupItemActionCancelRequest { string plugin = 1; string operationID = 2; bytes backup = 3; } ``` One new shared message type will be added, as this will also be needed for v2 RestoreItemAction and VolmeSnapshotter: ``` message OperationProgress { bool completed = 1; string err = 2; int64 nCompleted = 3; int64 nTotal = 4; string operationUnits = 5; string description = 6; google.protobuf.Timestamp started = 7; google.protobuf.Timestamp updated = 8; } ``` In addition to the two new rpc methods added to the BackupItemAction interface, there is also a new `Name()` method. This one is only actually used internally by Velero to get the name that the plugin was registered with, but it still must be defined in a plugin which implements BackupItemActionV2 in order to implement the interface. It doesn't really matter what it returns, though, as this particular method is not delegated to the plugin via RPC calls. The new (and modified) interface methods for `BackupItemAction` are as follows: ``` type BackupItemAction interface { ... Name() string ... Execute(item runtime.Unstructured, backup *api.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) Progress(operationID string, backup *api.Backup) (velero.OperationProgress, error) Cancel(operationID string, backup *api.Backup) error ... } ``` A new PluginKind, `BackupItemActionV2`, will be created, and the backup process will be modified to use this plugin kind. See [Plugin Versioning](plugin-versioning.md) for more details on implementation plans, including v1 adapters, etc. ## Compatibility The included v1 adapter will allow any existing BackupItemAction plugin to work as expected, with an empty operation ID returned from Execute() and no-op Progress() and Cancel() methods. ## Implementation This will be implemented during the Velero 1.11 development cycle. ================================================ FILE: design/Implemented/bsl-certificate-support_design.md ================================================ # Design for BSL Certificate Support Enhancement ## Abstract This design document describes the enhancement of BackupStorageLocation (BSL) certificate management in Velero, introducing a Secret-based certificate reference mechanism (`caCertRef`) alongside the existing inline certificate field (`caCert`). This enhancement provides a more secure, Kubernetes-native approach to certificate management while enabling future CLI improvements for automatic certificate discovery. ## Background Currently, Velero supports TLS certificate verification for object storage providers through an inline `caCert` field in the BSL specification. While functional, this approach has several limitations: - **Security**: Certificates are stored directly in the BSL YAML, potentially exposing sensitive data - **Management**: Certificate rotation requires updating the BSL resource itself - **CLI Usability**: Users must manually specify certificates when using CLI commands - **Size Limitations**: Large certificate bundles can make BSL resources unwieldy Issue #9097 and PR #8557 highlight the need for improved certificate management that addresses these concerns while maintaining backward compatibility. ## Goals - Provide a secure, Secret-based certificate storage mechanism - Maintain full backward compatibility with existing BSL configurations - Enable future CLI enhancements for automatic certificate discovery - Simplify certificate rotation and management - Provide clear migration path for existing users ## Non-Goals - Removing support for inline certificates immediately - Changing the behavior of existing BSL configurations - Implementing client-side certificate validation - Supporting certificates from ConfigMaps or other resource types ## High-Level Design ### API Changes #### New Field: CACertRef ```go type ObjectStorageLocation struct { // Existing field (now deprecated) // +optional // +kubebuilder:deprecatedversion:warning="caCert is deprecated, use caCertRef instead" CACert []byte `json:"caCert,omitempty"` // New field for Secret reference // +optional CACertRef *corev1api.SecretKeySelector `json:"caCertRef,omitempty"` } ``` The `SecretKeySelector` follows standard Kubernetes patterns: ```go type SecretKeySelector struct { // Name of the Secret Name string `json:"name"` // Key within the Secret Key string `json:"key"` } ``` ### Certificate Resolution Logic The system follows a priority-based resolution: 1. If `caCertRef` is specified, retrieve certificate from the referenced Secret 2. If `caCert` is specified (and `caCertRef` is not), use the inline certificate 3. If neither is specified, no custom CA certificate is used ### Validation BSL validation ensures mutual exclusivity: ```go func (bsl *BackupStorageLocation) Validate() error { if bsl.Spec.ObjectStorage != nil && bsl.Spec.ObjectStorage.CACert != nil && bsl.Spec.ObjectStorage.CACertRef != nil { return errors.New("cannot specify both caCert and caCertRef in objectStorage") } return nil } ``` ## Detailed Design ### BSL Controller Changes The BSL controller incorporates validation during reconciliation: ```go func (r *backupStorageLocationReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { // ... existing code ... // Validate BSL configuration if err := location.Validate(); err != nil { r.logger.WithError(err).Error("BSL validation failed") return ctrl.Result{}, err } // ... continue reconciliation ... } ``` ### Repository Provider Integration All repository providers implement consistent certificate handling: ```go func configureCACert(bsl *velerov1api.BackupStorageLocation, credGetter *credentials.CredentialGetter) ([]byte, error) { if bsl.Spec.ObjectStorage == nil { return nil, nil } // Prefer caCertRef (new method) if bsl.Spec.ObjectStorage.CACertRef != nil { certString, err := credGetter.FromSecret.Get(bsl.Spec.ObjectStorage.CACertRef) if err != nil { return nil, errors.Wrap(err, "error getting CA certificate from secret") } return []byte(certString), nil } // Fall back to caCert (deprecated) if bsl.Spec.ObjectStorage.CACert != nil { return bsl.Spec.ObjectStorage.CACert, nil } return nil, nil } ``` ### CLI Certificate Discovery Integration #### Background: PR #8557 Implementation PR #8557 ("CLI automatically discovers and uses cacert from BSL") was merged in August 2025, introducing automatic CA certificate discovery from BackupStorageLocation for Velero CLI download operations. This eliminated the need for users to manually specify the `--cacert` flag when performing operations like `backup describe`, `backup download`, `backup logs`, and `restore logs`. #### Current Implementation (Post PR #8557) The CLI now automatically discovers certificates from BSL through the `pkg/cmd/util/cacert/bsl_cacert.go` module: ```go // Current implementation only supports inline caCert func GetCACertFromBSL(ctx context.Context, client kbclient.Client, namespace, bslName string) (string, error) { // ... fetch BSL ... if bsl.Spec.ObjectStorage != nil && len(bsl.Spec.ObjectStorage.CACert) > 0 { return string(bsl.Spec.ObjectStorage.CACert), nil } return "", nil } ``` #### Enhancement with caCertRef Support This design extends the existing CLI certificate discovery to support the new `caCertRef` field: ```go // Enhanced implementation supporting both caCert and caCertRef func GetCACertFromBSL(ctx context.Context, client kbclient.Client, namespace, bslName string) (string, error) { // ... fetch BSL ... // Prefer caCertRef over inline caCert if bsl.Spec.ObjectStorage.CACertRef != nil { secret := &corev1api.Secret{} key := types.NamespacedName{ Name: bsl.Spec.ObjectStorage.CACertRef.Name, Namespace: namespace, } if err := client.Get(ctx, key, secret); err != nil { return "", errors.Wrap(err, "error getting certificate secret") } certData, ok := secret.Data[bsl.Spec.ObjectStorage.CACertRef.Key] if !ok { return "", errors.Errorf("key %s not found in secret", bsl.Spec.ObjectStorage.CACertRef.Key) } return string(certData), nil } // Fall back to inline caCert (deprecated) if bsl.Spec.ObjectStorage.CACert != nil { return string(bsl.Spec.ObjectStorage.CACert), nil } return "", nil } ``` #### Certificate Resolution Priority The CLI follows this priority order for certificate resolution: 1. **`--cacert` flag** - Manual override, highest priority 2. **`caCertRef`** - Secret-based certificate (recommended) 3. **`caCert`** - Inline certificate (deprecated) 4. **System certificate pool** - Default fallback #### User Experience Improvements With both PR #8557 and this enhancement: ```bash # Automatic discovery - works with both caCert and caCertRef velero backup describe my-backup velero backup download my-backup velero backup logs my-backup velero restore logs my-restore # Manual override still available velero backup describe my-backup --cacert /custom/ca.crt # Debug output shows certificate source velero backup download my-backup --log-level=debug # [DEBUG] Resolved CA certificate from BSL 'default' Secret 'storage-ca-cert' key 'ca-bundle.crt' ``` #### RBAC Considerations for CLI CLI users need read access to Secrets when using `caCertRef`: ```yaml apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: velero-cli-user namespace: velero rules: - apiGroups: ["velero.io"] resources: ["backups", "restores", "backupstoragelocations"] verbs: ["get", "list"] - apiGroups: [""] resources: ["secrets"] verbs: ["get"] # Limited to secrets referenced by BSLs ``` ### Migration Strategy #### Phase 1: Introduction (Current) - Add `caCertRef` field - Mark `caCert` as deprecated - Both fields supported, mutual exclusivity enforced #### Phase 2: Migration Period - Documentation and tools to help users migrate - Warning messages for `caCert` usage - CLI enhancements to leverage `caCertRef` #### Phase 3: Future Removal - Remove `caCert` field in major version update - Provide migration tool for automatic conversion ## User Experience ### Creating a BSL with Certificate Reference 1. Create a Secret containing the CA certificate: ```yaml apiVersion: v1 kind: Secret metadata: name: storage-ca-cert namespace: velero type: Opaque data: ca-bundle.crt: ``` 2. Reference the Secret in BSL: ```yaml apiVersion: velero.io/v1 kind: BackupStorageLocation metadata: name: default namespace: velero spec: provider: aws objectStorage: bucket: my-bucket caCertRef: name: storage-ca-cert key: ca-bundle.crt ``` ### Certificate Rotation With Secret-based certificates: ```bash # Update the Secret with new certificate kubectl create secret generic storage-ca-cert \ --from-file=ca-bundle.crt=new-ca.crt \ --dry-run=client -o yaml | kubectl apply -f - # No BSL update required - changes take effect on next use ``` ### CLI Usage Examples #### Immediate Benefits - No change required for existing workflows - Certificate validation errors include helpful context #### Future CLI Enhancements ```bash # Automatic certificate discovery velero backup download my-backup # Manual override still available velero backup download my-backup --cacert /custom/ca.crt # Debug certificate resolution velero backup download my-backup --log-level=debug # [DEBUG] Resolved CA certificate from BSL 'default' Secret 'storage-ca-cert' ``` ## Security Considerations ### Advantages of Secret-based Storage 1. **Encryption at Rest**: Secrets are encrypted in etcd 2. **RBAC Control**: Fine-grained access control via Kubernetes RBAC 3. **Audit Trail**: Secret access is auditable 4. **Separation of Concerns**: Certificates separate from configuration ### Required Permissions The Velero server requires additional RBAC permissions: ```yaml - apiGroups: [""] resources: ["secrets"] verbs: ["get"] # Scoped to secrets referenced by BSLs ``` ## Compatibility ### Backward Compatibility - Existing BSLs with `caCert` continue to function unchanged - No breaking changes to API - Gradual migration path ### Forward Compatibility - Design allows for future enhancements: - Multiple certificate support - Certificate chain validation - Automatic certificate discovery from cloud providers ## Implementation Phases ### Phase 1: Core Implementation ✓ (Current PR) - API changes with new `caCertRef` field - Controller validation - Repository provider updates - Basic testing ### Phase 2: CLI Enhancement (Future) - Automatic certificate discovery in CLI - Enhanced error messages - Debug logging for certificate resolution ### Phase 3: Migration Tools (Future) - Automated migration scripts - Validation tools - Documentation updates ## Testing ### Unit Tests - BSL validation logic - Certificate resolution in providers - Controller behavior ### Integration Tests - End-to-end backup/restore with `caCertRef` - Certificate rotation scenarios - Migration from `caCert` to `caCertRef` ### Manual Testing Scenarios 1. Create BSL with `caCertRef` 2. Perform backup/restore operations 3. Rotate certificate in Secret 4. Verify continued operation ## Documentation ### User Documentation - Migration guide from `caCert` to `caCertRef` - Examples for common cloud providers - Troubleshooting guide ### API Documentation - Updated API reference - Deprecation notices - Field descriptions ## Alternatives Considered ### ConfigMap-based Storage - Pros: Similar to Secrets, simpler API - Cons: Not designed for sensitive data, no encryption at rest - Decision: Secrets are the Kubernetes-standard for sensitive data ### External Certificate Management - Pros: Integration with cert-manager, etc. - Cons: Additional complexity, dependencies - Decision: Keep it simple, allow users to manage certificates as needed ### Immediate Removal of Inline Certificates - Pros: Cleaner API, forces best practices - Cons: Breaking change, migration burden - Decision: Gradual deprecation respects existing users ## Conclusion This design provides a secure, Kubernetes-native approach to certificate management in Velero while maintaining backward compatibility. It establishes the foundation for enhanced CLI functionality and improved user experience, addressing the concerns raised in issue #9097 and enabling the features proposed in PR #8557. The phased approach ensures smooth migration for existing users while delivering immediate security benefits for new deployments. ================================================ FILE: design/Implemented/clean_artifacts_in_csi_flow.md ================================================ # Design to clean the artifacts generated in the CSI backup and restore workflows ## Terminology * VSC: VolumeSnapshotContent * VS: VolumeSnapshot ## Abstract * The design aims to delete the unnecessary VSs and VSCs generated during CSI backup and restore process. * The design stop creating related VSCs during backup syncing. ## Background In the current CSI backup and restore workflows, please notice the CSI B/R workflows means only using the CSI snapshots in the B/R, not including the CSI snapshot data movement workflows, some generated artifacts are kept after the backup or the restore process completion. Some of them are kept due to design, for example, the VolumeSnapshotContents generated during the backup are kept to make sure the backup deletion can clean the snapshots in the storage providers. Some of them are kept by accident, for example, after restore, two VolumeSnapshotContents are generated for the same VolumeSnapshot. One is from the backup content, and one is dynamically generated from the restore's VolumeSnapshot. The design aims to clean the unnecessary artifacts, and make the CSI B/R workflow more concise and reliable. ## Goals - Clean the redundant VSC generated during CSI backup and restore. - Remove the VSCs in the backup sync process. ## Non Goals - There were some discussion about whether Velero backup should include VSs and VSCs not generated in during the backup. By far, the conclusion is not including them is a better option. Although that is a useful enhancement, that is not included this design. - Delete all the CSI-related metadata files in the BSL is not the aim of this design. ## Detailed Design ### Backup During backup, the main change is the backup-generated VSCs should not kept anymore. The reasons is we don't need them to ensure the snapshots clean up during backup deletion. Please reference to the [Backup Deletion section](#backup-deletion) section for detail. As a result, we can simplify the VS deletion logic in the backup. Before, we need to not only delete the VS, but also recreate a static VSC pointing a non-exiting VS. The deletion code in VS BackupItemAction can be simplify to the following: ``` go if backup.Status.Phase == velerov1api.BackupPhaseFinalizing || backup.Status.Phase == velerov1api.BackupPhaseFinalizingPartiallyFailed { p.log. WithField("Backup", fmt.Sprintf("%s/%s", backup.Namespace, backup.Name)). WithField("BackupPhase", backup.Status.Phase).Debugf("Cleaning VolumeSnapshots.") if vsc == nil { vsc = &snapshotv1api.VolumeSnapshotContent{} } csi.DeleteReadyVolumeSnapshot(*vs, *vsc, p.crClient, p.log) return item, nil, "", nil, nil } func DeleteReadyVolumeSnapshot( vs snapshotv1api.VolumeSnapshot, vsc snapshotv1api.VolumeSnapshotContent, client crclient.Client, logger logrus.FieldLogger, ) { logger.Infof("Deleting Volumesnapshot %s/%s", vs.Namespace, vs.Name) if vs.Status == nil || vs.Status.BoundVolumeSnapshotContentName == nil || len(*vs.Status.BoundVolumeSnapshotContentName) <= 0 { logger.Errorf("VolumeSnapshot %s/%s is not ready. This is not expected.", vs.Namespace, vs.Name) return } if vs.Status != nil && vs.Status.BoundVolumeSnapshotContentName != nil { // Patch the DeletionPolicy of the VolumeSnapshotContent to set it to Retain. // This ensures that the volume snapshot in the storage provider is kept. if err := SetVolumeSnapshotContentDeletionPolicy( vsc.Name, client, snapshotv1api.VolumeSnapshotContentRetain, ); err != nil { logger.Warnf("Failed to patch DeletionPolicy of volume snapshot %s/%s", vs.Namespace, vs.Name) return } if err := client.Delete(context.TODO(), &vsc); err != nil { logger.Warnf("Failed to delete the VSC %s: %s", vsc.Name, err.Error()) } } if err := client.Delete(context.TODO(), &vs); err != nil { logger.Warnf("Failed to delete volumesnapshot %s/%s: %v", vs.Namespace, vs.Name, err) } else { logger.Infof("Deleted volumesnapshot with volumesnapshotContent %s/%s", vs.Namespace, vs.Name) } } ``` ### Restore #### Restore the VolumeSnapshotContent The current behavior of VSC restoration is that the VSC from the backup is restore, and the restored VS also triggers creating a new VSC dynamically. Two VSCs created for the same VS in one restore seems not right. Skip restore the VSC from the backup is not a viable alternative, because VSC may reference to a [snapshot create secret](https://kubernetes-csi.github.io/docs/secrets-and-credentials-volume-snapshot-class.html?highlight=snapshotter-secret-name#createdelete-volumesnapshot-secret). If the `SkipRestore` is set true in the restore action's result, the secret returned in the additional items is ignored too. As a result, restore the VSC from the backup, and setup the VSC and the VS's relation is a better choice. Another consideration is the VSC name should not be the same as the backed-up VSC's, because the older version Velero's restore and backup keep the VSC after completion. There's high possibility that the restore will fail due to the VSC already exists in the cluster. Multiple restores of the same backup will also meet the same problem. The proposed solution is using the restore's UID and the VS's name to generate sha256 hash value as the new VSC name. Both the VS and VSC RestoreItemAction can access those UIDs, and it will avoid the conflicts issues. The restored VS name also shares the same generated name. The VS-referenced VSC name and the VSC's snapshot handle name are in their status. Velero restore process purges the restore resources' metadata and status before running the RestoreItemActions. As a result, we cannot read these information in the VS and VSC RestoreItemActions. Fortunately, RestoreItemAction input parameters includes the `ItemFromBackup`. The status is intact in `ItemFromBackup`. ``` go func (p *volumeSnapshotRestoreItemAction) Execute( input *velero.RestoreItemActionExecuteInput, ) (*velero.RestoreItemActionExecuteOutput, error) { p.log.Info("Starting VolumeSnapshotRestoreItemAction") if boolptr.IsSetToFalse(input.Restore.Spec.RestorePVs) { p.log.Infof("Restore %s/%s did not request for PVs to be restored.", input.Restore.Namespace, input.Restore.Name) return &velero.RestoreItemActionExecuteOutput{SkipRestore: true}, nil } var vs snapshotv1api.VolumeSnapshot if err := runtime.DefaultUnstructuredConverter.FromUnstructured( input.Item.UnstructuredContent(), &vs); err != nil { return &velero.RestoreItemActionExecuteOutput{}, errors.Wrapf(err, "failed to convert input.Item from unstructured") } var vsFromBackup snapshotv1api.VolumeSnapshot if err := runtime.DefaultUnstructuredConverter.FromUnstructured( input.ItemFromBackup.UnstructuredContent(), &vsFromBackup); err != nil { return &velero.RestoreItemActionExecuteOutput{}, errors.Wrapf(err, "failed to convert input.Item from unstructured") } // If cross-namespace restore is configured, change the namespace // for VolumeSnapshot object to be restored newNamespace, ok := input.Restore.Spec.NamespaceMapping[vs.GetNamespace()] if !ok { // Use original namespace newNamespace = vs.Namespace } if csiutil.IsVolumeSnapshotExists(newNamespace, vs.Name, p.crClient) { p.log.Debugf("VolumeSnapshot %s already exists in the cluster. Return without change.", vs.Namespace+"/"+vs.Name) return &velero.RestoreItemActionExecuteOutput{UpdatedItem: input.Item}, nil } newVSCName := generateSha256FromRestoreAndVsUID(string(input.Restore.UID), string(vsFromBackup.UID)) // Reset Spec to convert the VolumeSnapshot from using // the dynamic VolumeSnapshotContent to the static one. resetVolumeSnapshotSpecForRestore(&vs, &newVSCName) // Reset VolumeSnapshot annotation. By now, only change // DeletionPolicy to Retain. resetVolumeSnapshotAnnotation(&vs) vsMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&vs) if err != nil { p.log.Errorf("Fail to convert VS %s to unstructured", vs.Namespace+"/"+vs.Name) return nil, errors.WithStack(err) } p.log.Infof(`Returning from VolumeSnapshotRestoreItemAction with no additionalItems`) return &velero.RestoreItemActionExecuteOutput{ UpdatedItem: &unstructured.Unstructured{Object: vsMap}, AdditionalItems: []velero.ResourceIdentifier{}, }, nil } // generateSha256FromRestoreAndVsUID Use the restore UID and the VS UID to generate the new VSC name. // By this way, VS and VSC RIA action can get the same VSC name. func generateSha256FromRestoreAndVsUID(restoreUID string, vsUID string) string { sha256Bytes := sha256.Sum256([]byte(restoreUID + "/" + vsUID)) return "vsc-" + hex.EncodeToString(sha256Bytes[:]) } ``` #### Restore the VolumeSnapshot ``` go // Execute restores a VolumeSnapshotContent object without modification // returning the snapshot lister secret, if any, as additional items to restore. func (p *volumeSnapshotContentRestoreItemAction) Execute( input *velero.RestoreItemActionExecuteInput, ) (*velero.RestoreItemActionExecuteOutput, error) { if boolptr.IsSetToFalse(input.Restore.Spec.RestorePVs) { p.log.Infof("Restore did not request for PVs to be restored %s/%s", input.Restore.Namespace, input.Restore.Name) return &velero.RestoreItemActionExecuteOutput{SkipRestore: true}, nil } p.log.Info("Starting VolumeSnapshotContentRestoreItemAction") var vsc snapshotv1api.VolumeSnapshotContent if err := runtime.DefaultUnstructuredConverter.FromUnstructured( input.Item.UnstructuredContent(), &vsc); err != nil { return &velero.RestoreItemActionExecuteOutput{}, errors.Wrapf(err, "failed to convert input.Item from unstructured") } var vscFromBackup snapshotv1api.VolumeSnapshotContent if err := runtime.DefaultUnstructuredConverter.FromUnstructured( input.ItemFromBackup.UnstructuredContent(), &vscFromBackup); err != nil { return &velero.RestoreItemActionExecuteOutput{}, errors.Errorf(err.Error(), "failed to convert input.ItemFromBackup from unstructured") } // If cross-namespace restore is configured, change the namespace // for VolumeSnapshot object to be restored newNamespace, ok := input.Restore.Spec.NamespaceMapping[vsc.Spec.VolumeSnapshotRef.Namespace] if ok { // Update the referenced VS namespace to the mapping one. vsc.Spec.VolumeSnapshotRef.Namespace = newNamespace } // Reset VSC name to align with VS. vsc.Name = generateSha256FromRestoreAndVsUID(string(input.Restore.UID), string(vscFromBackup.Spec.VolumeSnapshotRef.UID)) // Reset the ResourceVersion and UID of referenced VolumeSnapshot. vsc.Spec.VolumeSnapshotRef.ResourceVersion = "" vsc.Spec.VolumeSnapshotRef.UID = "" // Set the DeletionPolicy to Retain to avoid VS deletion will not trigger snapshot deletion vsc.Spec.DeletionPolicy = snapshotv1api.VolumeSnapshotContentRetain if vscFromBackup.Status != nil && vscFromBackup.Status.SnapshotHandle != nil { vsc.Spec.Source.VolumeHandle = nil vsc.Spec.Source.SnapshotHandle = vscFromBackup.Status.SnapshotHandle } else { p.log.Errorf("fail to get snapshot handle from VSC %s status", vsc.Name) return nil, errors.Errorf("fail to get snapshot handle from VSC %s status", vsc.Name) } additionalItems := []velero.ResourceIdentifier{} if csi.IsVolumeSnapshotContentHasDeleteSecret(&vsc) { additionalItems = append(additionalItems, velero.ResourceIdentifier{ GroupResource: schema.GroupResource{Group: "", Resource: "secrets"}, Name: vsc.Annotations[velerov1api.PrefixedSecretNameAnnotation], Namespace: vsc.Annotations[velerov1api.PrefixedSecretNamespaceAnnotation], }, ) } vscMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&vsc) if err != nil { return nil, errors.WithStack(err) } p.log.Infof("Returning from VolumeSnapshotContentRestoreItemAction with %d additionalItems", len(additionalItems)) return &velero.RestoreItemActionExecuteOutput{ UpdatedItem: &unstructured.Unstructured{Object: vscMap}, AdditionalItems: additionalItems, }, nil } ``` ### Backup Sync csi-volumesnapshotclasses.json, csi-volumesnapshotcontents.json, and csi-volumesnapshots.json are CSI-related metadata files in the BSL for each backup. csi-volumesnapshotcontents.json and csi-volumesnapshots.json are not needed anymore, but csi-volumesnapshotclasses.json is still needed. One concrete scenario is that a backup is created in cluster-A, then the backup is synced to cluster-B, and the backup is deleted in the cluster-B. In this case, we don't have a chance to create the VS and VSC needed VolumeSnapshotClass. The VSC deletion workflow proposed by this design needs to create the VSC first. If the VSC's referenced VolumeSnapshotClass doesn't exist in cluster, the creation of VSC will fail. As a result, the VolumeSnapshotClass should still be synced in the backup sync process. ### Backup Deletion Two factors are worthy for consideration for the backup deletion change: * Because the VSCs generated by the backup are not synced anymore, and the VSCs generated during the backup will not be kept too. The backup deletion needs to generate a VSC, then deletes it to make sure the snapshots in the storage provider are clean too. * The VSs generated by the backup are already deleted in the backup process, we don't need a DeleteItemAction for the VS anymore. As a result, the `velero.io/csi-volumesnapshot-delete` plugin is unneeded. For the VSC DeleteItemAction, we need to generate a VSC. Because we only care about the snapshot deletion, we don't need to create a VS associated with the VSC. Create a static VSC, then point it to a pseudo VS, and reference to the snapshot handle should be enough. To avoid the created VSC conflict with older version Velero B/R generated ones, the VSC name is set to `vsc-uuid`. The following is an example of the implementation. ``` go uuid, err := uuid.NewRandom() if err != nil { p.log.WithError(err).Errorf("Fail to generate the UUID to create VSC %s", snapCont.Name) return errors.Wrapf(err, "Fail to generate the UUID to create VSC %s", snapCont.Name) } snapCont.Name = "vsc-" + uuid.String() snapCont.Spec.DeletionPolicy = snapshotv1api.VolumeSnapshotContentDelete snapCont.Spec.Source = snapshotv1api.VolumeSnapshotContentSource{ SnapshotHandle: snapCont.Status.SnapshotHandle, } snapCont.Spec.VolumeSnapshotRef = corev1api.ObjectReference{ APIVersion: snapshotv1api.SchemeGroupVersion.String(), Kind: "VolumeSnapshot", Namespace: "ns-" + string(snapCont.UID), Name: "name-" + string(snapCont.UID), } snapCont.ResourceVersion = "" if err := p.crClient.Create(context.TODO(), &snapCont); err != nil { return errors.Wrapf(err, "fail to create VolumeSnapshotContent %s", snapCont.Name) } // Read resource timeout from backup annotation, if not set, use default value. timeout, err := time.ParseDuration( input.Backup.Annotations[velerov1api.ResourceTimeoutAnnotation]) if err != nil { p.log.Warnf("fail to parse resource timeout annotation %s: %s", input.Backup.Annotations[velerov1api.ResourceTimeoutAnnotation], err.Error()) timeout = 10 * time.Minute } p.log.Debugf("resource timeout is set to %s", timeout.String()) interval := 5 * time.Second // Wait until VSC created and ReadyToUse is true. if err := wait.PollUntilContextTimeout( context.Background(), interval, timeout, true, func(ctx context.Context) (bool, error) { tmpVSC := new(snapshotv1api.VolumeSnapshotContent) if err := p.crClient.Get(ctx, crclient.ObjectKeyFromObject(&snapCont), tmpVSC); err != nil { return false, errors.Wrapf( err, "failed to get VolumeSnapshotContent %s", snapCont.Name, ) } if tmpVSC.Status != nil && boolptr.IsSetToTrue(tmpVSC.Status.ReadyToUse) { return true, nil } return false, nil }, ); err != nil { return errors.Wrapf(err, "fail to wait VolumeSnapshotContent %s becomes ready.", snapCont.Name) } ``` ## Security Considerations Security is not relevant to this design. ## Compatibility In this design, no new information is added in backup and restore. As a result, this design doesn't have any compatibility issue. ## Open Issues Please notice the CSI snapshot backup and restore mechanism not supporting all file-store-based volume, e.g. Azure Files, EFS or vSphere CNS File Volume. Only block-based volumes are supported. Refer to [this comment](https://github.com/vmware-tanzu/velero/issues/3151#issuecomment-2623507686) for more details. ================================================ FILE: design/Implemented/cluster-scope-resource-filter.md ================================================ # Proposal to add resource filters for backup can distinguish whether resource is cluster-scoped or namespace-scoped. - [Proposal to add resource filters for backup can distinguish whether resource is cluster-scoped or namespace-scoped.](#proposal-to-add-resource-filters-for-backup-can-distinguish-whether-resource-is-cluster-scoped-or-namespace-scoped) - [Abstract](#abstract) - [Background](#background) - [Goals](#goals) - [Non Goals](#non-goals) - [High-Level Design](#high-level-design) - [Parameters Rules](#parameters-rules) - [Using scenarios:](#using-scenarios) - [no namespace-scoped resources + some cluster-scoped resources](#no-namespace-scoped-resources--some-cluster-scoped-resources) - [no namespace-scoped resources + all cluster-scoped resources](#no-namespace-scoped-resources--all-cluster-scoped-resources) - [some namespace-scoped resources + no cluster-scoped resources](#some-namespace-scoped-resources--no-cluster-scoped-resources) - [scenario 1](#scenario-1) - [scenario 2](#scenario-2) - [scenario 3](#scenario-3) - [scenario 4](#scenario-4) - [some namespace-scoped resources + only related cluster-scoped resources](#some-namespace-scoped-resources--only-related-cluster-scoped-resources) - [scenario 1](#scenario-1-1) - [scenario 2](#scenario-2-1) - [scenario 3](#scenario-3-1) - [some namespace-scoped resources + some additional cluster-scoped resources](#some-namespace-scoped-resources--some-additional-cluster-scoped-resources) - [scenario 1](#scenario-1-2) - [scenario 2](#scenario-2-2) - [scenario 3](#scenario-3-2) - [scenario 4](#scenario-4-1) - [some namespace-scoped resources + all cluster-scoped resources](#some-namespace-scoped-resources--all-cluster-scoped-resources) - [scenario 1](#scenario-1-3) - [scenario 2](#scenario-2-3) - [scenario 3](#scenario-3-3) - [all namespace-scoped resources + no cluster-scoped resources](#all-namespace-scoped-resources--no-cluster-scoped-resources) - [all namespace-scoped resources + some additional cluster-scoped resources](#all-namespace-scoped-resources--some-additional-cluster-scoped-resources) - [all namespace-scoped resources + all cluster-scoped resources](#all-namespace-scoped-resources--all-cluster-scoped-resources) - [describe command change](#describe-command-change) - [Detailed Design](#detailed-design) - [Alternatives Considered](#alternatives-considered) - [Security Considerations](#security-considerations) - [Compatibility](#compatibility) - [Implementation](#implementation) - [Open Issues](#open-issues) ## Abstract The current filter (IncludedResources/ExcludedResources + IncludeClusterResources flag) is not enough for some special cases, e.g. all namespace-scoped resources + some kind of cluster-scoped resource and all namespace-scoped resources + cluster-scoped resource excludes. Propose to add a new group of resource filtering parameters, which can distinguish cluster-scoped and namespace-scoped resources. ## Background There are two sets of resource filters for Velero: `IncludedNamespaces/ExcludedNamespaces` and `IncludedResources/ExcludedResources`. `IncludedResources` means only including the resource types specified in the parameter. Both cluster-scoped and namespace-scoped resources are handled in this parameter by now. The k8s resources are separated into cluster-scoped and namespace-scoped. As a result, it's hard to include all resources in one group and only including specified resource in the other group. ## Goals - Make Velero can support more complicated namespace-scoped and cluster-scoped resources filtering scenarios in backup. ## Non Goals - Enrich the resource filtering rules, for example, advanced PV filtering and filtering by resource names. ## High-Level Design Four new parameters are added into command `velero backup create`: `--include-cluster-scoped-resources`, `--exclude-cluster-scoped-resources`, `--include-namespace-scoped-resources` and `--exclude-namespace-scoped-resources`. `--include-cluster-scoped-resources` and `--exclude-cluster-scoped-resources` are used to filter cluster-scoped resources included or excluded in backup per resource type. `--include-namespace-scoped-resources` and `--exclude-namespace-scoped-resources` are used to filter namespace-scoped resources included or excluded in backup per resource type. Restore and other code pieces also use resource filtering will be handled in future releases. ### Parameters Rules * `--include-cluster-scoped-resources`, `--include-namespace-scoped-resources`, `--exclude-cluster-scoped-resources` and `--exclude-namespace-scoped-resources` valid value include `*` and comma separated string. Each element of the CSV string should a k8s resource name. The format should be `resource.group`, such as `storageclasses.storage.k8s.io.`. * `--include-cluster-scoped-resources`, `--include-namespace-scoped-resources`, `--exclude-cluster-scoped-resources` and `--exclude-namespace-scoped-resources` parameters are mutual exclusive with `--include-cluster-resources`, `--include-resources` and `--exclude-resources` parameters. If both sets of parameters are provisioned, validation failure should be returned. * `--include-cluster-scoped-resources` and `--exclude-cluster-scoped-resources` should only contain cluster-scoped resource type names. If namespace-scoped resource type names are included, they are ignored. * If there are conflicts between `--include-cluster-scoped-resources` and `--exclude-cluster-scoped-resources` specified resources type lists, `--exclude-cluster-scoped-resources` parameter has higher priority. * `--include-namespace-scoped-resources` and `--exclude-namespace-scoped-resources` should only contain namespace-scoped resource type names. If cluster-scoped resource type names are included, they are ignored. * If there are conflicts between `--include-namespace-scoped-resources` and `--exclude-namespace-scoped-resources` specified resources type lists, `--exclude-namespace-scoped-resources` parameter has higher priority. * If `--include-namespace-scoped-resources` is not present, it means all namespace-scoped resources are included per resource type. * If both `--include-cluster-scoped-resources` and `--exclude-cluster-scoped-resources` are not present, it means no additional cluster-scoped resource is included per resource type, just as the existing `--include-cluster-resources` parameter not setting value. Cluster-scoped resources are related to the namespace-scoped resources, which means those are returned in the namespace-scoped resources' BackupItemAction's result AdditionalItems array, are still included in backup by default. Taking backing up PVC scenario as an example, PVC is namespace-scoped, PV is cluster-scoped. PVC's BIA will include PVC related PV into backup too. ### Using scenarios: Please notice, if the scenario give the example of using old filtering parameters (`--include-cluster-resources`, `--include-resources` and `--exclude-resources`), that means the old parameters also work for this case. If old parameters example is not given, that means they don't work for this scenario, only new parameters (`--include-cluster-scoped-resources`, `--include-namespace-scoped-resources`, `--exclude-cluster-scoped-resources` and `--exclude-namespace-scoped-resources`) work. #### no namespace-scoped resources + some cluster-scoped resources The following command means backup no namespace-scoped resources and some cluster-scoped resources. ``` bash velero backup create --exclude-namespace-scoped-resources=* --include-cluster-scoped-resources=storageclass ``` #### no namespace-scoped resources + all cluster-scoped resources The following command means backup no namespace-scoped resources and all cluster-scoped resources. ``` bash velero backup create --exclude-namespace-scoped-resources=* --include-cluster-scoped-resources=* ``` #### some namespace-scoped resources + no cluster-scoped resources ##### scenario 1 The following commands mean backup all resources in namespaces default and kube-system, and no cluster-scoped resources. Example of new parameters: ``` bash velero backup create --include-namespaces=default,kube-system --exclude-cluster-scoped-resources=* ``` Example of old parameters: ``` bash velero backup create --include-namespaces=default,kube-system --include-cluster-resources=false ``` ##### scenario 2 The following commands mean backup PVC, Deployment, Service, Endpoint, Pod and ReplicaSet resources in all namespaces, and no cluster-scoped resources. Although PVC's related PV should be included, due to no cluster-scoped resources are included, so they are ruled out too. Example of new parameters: ``` bash velero backup create --include-namespace-scoped-resources=persistentvolumeclaim,deployment,service,endpoint,pod,replicaset --exclude-cluster-scope-resources=* ``` Example of old parameters: ``` bash velero backup create --include-resources=persistentvolumeclaim,deployment,service,endpoint,pod,replicaset --include-cluster-resources=false ``` ##### scenario 3 The following commands mean backup PVC, Deployment, Service, Endpoint, Pod and ReplicaSet resources in namespace default and kube-system, and no cluster-scoped resources. Although PVC's related PV should be included, due to no cluster-scoped resources are included, so they are ruled out too. Example of new parameters: ``` bash velero backup create --include-namespaces=default,kube-system --include-namespace-scoped-resources=persistentvolumeclaim,deployment,service,endpoint,pod,replicaset --exclude-cluster-scope-resources=* ``` Example of old parameters: ``` bash velero backup create --include-namespaces=default,kube-system --include-resources=persistentvolumeclaim,deployment,service,endpoint,pod,replicaset --include-cluster-resources=false ``` ##### scenario 4 The following commands mean backup all resources except Ingress type resources in all namespaces, and no cluster-scoped resources. Example of new parameters: ``` bash velero backup create --exclude-namespace-scoped-resources=ingress --exclude-cluster-scoped-resources=* ``` Example of old parameters: ``` bash velero backup create --exclude-resources=ingress --include-cluster-resources=false ``` #### some namespace-scoped resources + only related cluster-scoped resources ##### scenario 1 This means backup all resources in namespaces default and kube-system, and related cluster-scoped resources. ``` bash velero backup create --include-namespaces=default,kube-system ``` ##### scenario 2 This means backup pods and configmaps in namespaces default and kube-system, and related cluster-scoped resources. ``` bash velero backup create --include-namespaces=default,kube-system --include-namespace-scoped-resources=pods,configmaps ``` ##### scenario 3 This means backup all resources except Ingress type resources in all namespaces, and related cluster-scoped resources. Example of new parameters: ``` bash velero backup create --exclude-namespace-scoped-resources=ingress ``` Example of old parameters: ``` bash velero backup create --exclude-resources=ingress ``` #### some namespace-scoped resources + some additional cluster-scoped resources ##### scenario 1 This means backup all resources in namespace in default, kube-system, and related cluster-scoped resources, plus all StorageClass resources. ``` bash velero backup create --include-namespaces=default,kube-system --include-cluster-scoped-resources=storageclass ``` ##### scenario 2 This means backup PVC, Deployment, Service, Endpoint, Pod and ReplicaSet resources in all namespaces, and related cluster-scoped resources, plus all StorageClass resources, and PVC related PV. ``` bash velero backup create --include-namespace-scoped-resources=persistentvolumeclaim,deployment,service,endpoint,pod,replicaset --include-cluster-scoped-resources=storageclass ``` ##### scenario 3 This means backup PVC, Deployment, Service, Endpoint, Pod and ReplicaSet resources in default and kube-system namespaces, and related cluster-scoped resources, plus all StorageClass resources, and PVC related PV. ``` bash velero backup create --include-namespace-scoped-resources=persistentvolumeclaim,deployment,service,endpoint,pod,replicaset --include-namespaces=default,kube-system --include-cluster-scoped-resources=storageclass ``` ##### scenario 4 This means backup PVC, Deployment, Service, Endpoint, Pod and ReplicaSet resources in default and kube-system namespaces, and related cluster-scoped resources, plus all cluster-scoped resources except StorageClass type resources. ``` bash velero backup create --include-namespace-scoped-resources=persistentvolumeclaim,deployment,service,endpoint,pod,replicaset --include-namespaces=default,kube-system --exclude-cluster-scoped-resources=storageclass ``` #### some namespace-scoped resources + all cluster-scoped resources ##### scenario 1 The following commands mean backup all resources in namespace in default, kube-system, and all cluster-scoped resources. Example of new parameters: ``` bash velero backup create --include-namespaces=default,kube-system --include-cluster-scoped-resources=* ``` Example of old parameters: ``` bash velero backup create --include-namespaces=default,kube-system --include-cluster-resources=true ``` ##### scenario 2 This means backup Deployment, Service, Endpoint, Pod and ReplicaSet resources in all namespaces, and all cluster-scoped resources. ``` bash velero backup create --include-namespace-scoped-resources=deployment,service,endpoint,pod,replicaset --include-cluster-scoped-resources=* ``` ##### scenario 3 This means backup Deployment, Service, Endpoint, Pod and ReplicaSet resources in default and kube-system namespaces, and all cluster-scoped resources. ``` bash velero backup create --include-namespaces=default,kube-system --include-namespace-scoped-resources=deployment,service,endpoint,pod,replicaset --include-cluster-scoped-resources=* ``` #### all namespace-scoped resources + no cluster-scoped resources The following commands all mean backup all namespace-scoped resources and no cluster-scoped resources. Example of new parameters: ``` bash velero backup create --exclude-cluster-scoped-resources=* ``` Example of old parameters: ``` bash velero backup create --include-cluster-resources=false ``` #### all namespace-scoped resources + some additional cluster-scoped resources This command means backup all namespace-scoped resources, and related cluster-scoped resources, plus all PersistentVolume resources. ``` bash velero backup create --include-namespaces=* --include-cluster-scoped-resources=persistentvolume ``` #### all namespace-scoped resources + all cluster-scoped resources The following commands have the same meaning: backup all namespace-scoped resources, and all cluster-scoped resources. ``` bash velero backup create --include-cluster-scoped-resources=* ``` ``` bash velero backup create --include-cluster-resources=true ``` #### describe command change In `velero backup describe` command, the four new parameters should be outputted too. ``` bash velero backup describe ...... Namespaces: Included: ns2 Excluded: Resources: Included cluster-scoped: StorageClass,PersistentVolume Excluded cluster-scoped: Included namespace-scoped: default Excluded namespace-scoped: ...... ``` **Note:** `velero restore` command doesn't support those four new parameter in Velero v1.11, but `velero schedule` supports the four new parameters through backup specification. ## Detailed Design With adding `IncludedNamespaceScopedResources`, `ExcludedNamespaceScopedResources`, `IncludedClusterScopedResources` and `ExcludedClusterScopedResources`, the `BackupSpec` looks like: ``` go type BackupSpec struct { ...... // IncludedResources is a slice of resource names to include // in the backup. If empty, all resources are included. // +optional // +nullable IncludedResources []string `json:"includedResources,omitempty"` // ExcludedResources is a slice of resource names that are not // included in the backup. // +optional // +nullable ExcludedResources []string `json:"excludedResources,omitempty"` // IncludeClusterResources specifies whether cluster-scoped resources // should be included for consideration in the backup. // +optional // +nullable IncludeClusterResources *bool `json:"includeClusterResources,omitempty"` // IncludedClusterScopedResources is a slice of cluster-scoped // resource type names to include in the backup. // If set to "*", all cluster scope resource types are included. // The default value is empty, which means only related cluster // scope resources are included. // +optional // +nullable IncludedClusterScopedResources []string `json:"includedClusterScopedResources,omitempty"` // ExcludedClusterScopedResources is a slice of cluster-scoped // resource type names to exclude from the backup. // If set to "*", all cluster scope resource types are excluded. // +optional // +nullable ExcludedClusterScopedResources []string `json:"excludedClusterScopedResources,omitempty"` // IncludedNamespaceScopedResources is a slice of namespace-scoped // resource type names to include in the backup. // The default value is "*". // +optional // +nullable IncludedNamespaceScopedResources []string `json:"includedNamespaceScopedResources,omitempty"` // ExcludedNamespaceScopedResources is a slice of namespace-scoped // resource type names to exclude from the backup. // If set to "*", all namespace scope resource types are excluded. // +optional // +nullable ExcludedNamespaceScopedResources []string `json:"excludedNamespaceScopedResources,omitempty"` ...... } ``` ## Alternatives Considered Proposal from Jibu Data [Issue 5120](https://github.com/vmware-tanzu/velero/issues/5120#issue-1304534563) ## Security Considerations No security impact. ## Compatibility The four new parameters cannot be mixed with existing resource filter parameters: `IncludedResources`, `ExcludedResources` and `IncludeClusterResources`. If the new parameters and old parameters both appears in command line, or are specified in backup spec, the command line and the backup should fail. ## Implementation This change should be included into Velero v1.11. New parameters will coexist with `IncludedResources`, `ExcludedResources` and `IncludeClusterResources`. Plan to deprecate `IncludedResources`, `ExcludedResources` and `IncludeClusterResources` in future releases, but also open to the community's feedback. ## Open Issues `LabelSelector/OrLabelSelectors` apply to namespace-scoped resources. It may be reasonable to make them also working on cluster-scoped resources. An issue is created to trace this topic [resource label selector not work for cluster-scoped resources](https://github.com/vmware-tanzu/velero/issues/5787) ================================================ FILE: design/Implemented/concurrent-backup-processing.md ================================================ # Concurrent Backup Processing This enhancement will enable Velero to process multiple backups at the same time. This is largely a usability enhancement rather than a performance enhancement, since the overall backup throughput may not be significantly improved over the current implementation, since we are already processing individual backup items in parallel. It is a significant usability improvement, though, as with the current design, a user who submits a small backup may have to wait significantly longer than expected if the backup is submitted immediately after a large backup. ## Background With the current implementation, only one backup may be `InProgress` at a time. A second backup created will not start processing until the first backup moves on to `WaitingForPluginOperations` or `Finalizing`. This is a usability concern, especially in clusters when multiple users are initiating backups. With this enhancement, we intend to allow multiple backups to be processed concurrently. This will allow backups to start processing immediately, even if a large backup was just submitted by another user. This enhancement will build on top of the prior parallel item processing feature by creating a dedicatede ItemBlock worker pool for each running backup. The pool will be created at the beginning of the backup reconcile, and the input channel will be passed to the Kubernetes backupper just like it is in the current release. The primary challenge is to make sure that the same workload in multiple backups is not backed up concurrently. If that were to happen, we would risk data corruption, especially around the processing of pod hooks and volume backup. For this first release we will take a conservative, high-level approach to overlap detection. Two backups will not run concurrently if there is any overlap in included namespaces. For example, if a backup that includes `ns1` and `ns2` is running, then a second backup for `ns2` and `ns3` will not be started. If a backup which does not filter namespaces is running (either a whole cluster backup or a non-namespace-limited backup with a label selector) then no other backups will be started, since a backup across all namespaces overlaps with any other backup. Calculating item-level overlap for queued backups is problematic since we don't know which items are included in a backup until backup processing has begun. A future release may add ItemBlock overlap detection, where at the item block worker level, the same item will not be processed by two different workers at the same time. This works together with workload conflict detection to further detect conflicts in a more granular level for shared resources between backups. Eventually, with a more complete understanding of individual workloads (either via ItemBlocks or some higher level model), the namespace level overlap detection may be relaxed in future versions. ## Goals - Process multiple backups concurrently - Detect namespace overlap to avoid conflicts - For queued backups (not yet runnable due to concurrency limits or overlap), indicate the queue position in status ## Non Goals - Handling NFS PVs when more than one PV point to the same underlying NFS share - Handling VGDP cancellation for failed backups on restart - Mounting a PVC for scenarios in which /tmp is too small for the number of concurrent backups - Providing a mechanism to identify high priority backups which get preferential treatment in terms of ItemBlock worker availability - Item-level overlap detection (future feature) - Providing the ability to disable namespace-level overlap detection once Item-level overlap detection is in place (although this may be supported in a future version). ## High-Level Design ### Backup CRD changes Two new backup phases will be added: `Queued` and `ReadyToStart`. In the Backup workflow, new backups will be moved to the Queued phase when they are added to the backup queue. When a backup is removed from the queue because it is now able to run, it will be moved to the `ReadyToStart` phase, which will allow the backup controller to start processing it. In addition, a new Status field, `QueuePosition`, will be added to track the backup's current position in the queue. ### New Controller: `backupQueueReconciler` A new reconciler will be added, `backupQueueReconciler` which will use the current `backupReconciler` logic for reconciling `New` backups but instead of running the backup, it will move the Backup to the `Queued` phase and set `QueuePosition`. In addition, this reconciler will periodically reconcile all queued backups (on some configurable time interval) and if there is a runnable backup, remove it from the queue, update `QueuePosition` for any queued backups behind it, and update its phase to `ReadyToStart`. Queued backups will be reconciled in order based on `QueuePosition`, so the first runnable backup found will be processed. A backup is runnable if both of the following conditions are true: 1) The total number of backups either `InProgress` or `ReadyToStart` is less than the configured number of concurrent backups. 2) The backup has no overlap with any backups currently `InProgress` or `ReadyToStart` or with any `Queued` backups with a higher (i.e. closer to 1) queue position than this backup. ### Updates to Backup controller The current `backupReconciler` will change its reconciling rules. Instead of watching and reconciling New backups, it will reconcile `ReadyToStart` backups. In addition, it will be configured to run in parallel by setting `MaxConcurrentReconciles` based on the `concurrent-backups` server arg. The startup (and shutdown) of the ItemBlock worker pool will be moved from reconciler startup to the backup reconcile, which will give each running backup its own dedicated worker pool. The per-backup worker pool will will use the existing `--item-block-worker-count` installer/server arg. This means that the maximum number of ItemBlock workers for the entire Velero pod will be the ItemBlock worker count multiplied by concurrentBackups. For example, if concurrentBackups is 5, and itemBlockWorkerCount is 6, then there will be, at most, 30 worker threads active, 5 dedicated to each InProgress backup, but this maximum will only be achieved when the maximum number of backups are InProgress. This also means that each InProgress backup will have a dedicated ItemBlock input channel with the same fixed buffer size. ## Detailed Design ### New Install/Server configuration args A new install/server arg, `concurrent-backups` will be added. This will be an int-valued field specifying the number of backups which may be processed concurrently (with phase `InProgress`). If not specified, the default value of 1 will be used. ### Consideration of backup overlap and concurrent backup processing The primary consideration for running additional backups concurrently is the configured `concurrent-backups` parameter. If the total number of `InProgress` and `ReadyToStart` backups is equal to `concurrent-backups` then any `Queued` backups will remain in the queue. The second consideration is backup overlap. In order to prevent interaction between running backups (particularly around volume backup and pod hooks), we cannot allow two overlapping backups to run at the same time. For now, we will define overlap broadly -- requiring that two concurrent backups don't include any of the same namespaces. A backup for `ns1` can run concurrently with a backup for `ns2`, but a backup for `[ns1,ns2]` cannot run concurrently with a backup for `ns1`. One consequence of this approach is that a backup which includes all namespaces (even if further filtered by resource or label) cannot run concurrently with *any other backup*. When determining which queued backup to run next, velero will look for the next queued backup which has no overlap with any InProgress backup or any Queued backup ahead of it. The reason we need to consider queued as well as running backups for overlap detection is as follows. Consider the following scenario. These are the current not-completed backups (ordered from oldest to newest) 1. backup1, includedNamespaces: [ns1, ns2], phase: InProgress 2. backup2, includedNamespaces: [ns2, ns3, ns5], phase: Queued, QueuePosition: 1 3. backup3, includedNamespaces: [ns4, ns3], phase: Queued, QueuePosition: 2 4. backup4, includedNamespaces: [ns5, ns6], phase: Queued, QueuePosition: 2 5. backup5, includedNamespaces: [ns8, ns9], phase: Queued, QueuePosition: 3 Assuming `concurrent-backups` is 2, on the next reconcile, Velero will be able to start a second backup if there is one with no overlap. `backup2` cannot run, since `ns2` overlaps between it and the running `backup1`. If we only considered running overlap (and not queued overlap), then `backup3` could run now. It conflicts with the queued `backup2` on `ns3` but it does not conflict with the running backup. However, if it runs now, then when `backup1` completes, then `backup2` still can't run (since it now overlaps with running `backup3`on `ns3`), so `backup4` starts instead. Now when `backup3` completes, `backup2` still can't run (since it now conflicts with `backup4` on `ns5`). This means that even though it was the second backup created, it's the fourth to run -- providing worse time to completion than without parallel backups. If a queued backup has a large number of namespaces (a full-cluster backup for example), it would never run as long as new single-namespace backups keep being added to the queue. To resolve this problem we consider both running backups as well as backups ahead in the queue when resolving overlap conflicts. In the above scenario, `backup2` can't run yet since it overlaps with the running backup on `ns2`. In addition, `backup3` and `backup4` also can't run yet since they overlap with queued `backup2`. Therefore, `backup5` will run now. Once `backup1` completes, `backup2` will be free to run. ### Backup CRD changes New Backup phases: ```go const ( // BackupPhaseQueued means the backup has been added to the // queue by the BackupQueueReconciler. BackupPhaseQueued BackupPhase = "Queued" // BackupPhaseReadyToStart means the backup has been removed from the // queue by the BackupQueueReconciler and is ready to start. BackupPhaseReadyToStart BackupPhase = "ReadyToStart" ) ``` In addition, a new Status field, `queuePosition`, will be added to track the backup's current position in the queue. ```go // QueuePosition is the position held by the backup in the queue. // QueuePosition=1 means this backup is the next to be considered. // Only relevant when Phase is "Queued" // +optional QueuePosition int `json:"queuePosition,omitempty"` ``` ### New Controller: `backupQueueReconciler` A new reconciler will be added, `backupQueueReconciler` which will reconcile backups under these conditions: 1) Watching Create/Update for backups in `New` (or empty) phase 2) Watching for Backup phase transition from `InProgress` to something else to reconcile all `Queued` backups 2) Watching for Backup phase transition from `New` (or empty) to `Queued` to reconcile all `Queued` backups 2) Periodic reconcile of `Queued` backups to handle backups queued at server startup as well as to make sure we never have a situation where backups are queued indefinitely because of a race condition or was otherwise missed in the reconcile on prior backup completion. The reconciler will be set up as follows -- note that New backups are reconciled on Create/Update, while Queued backups are reconciled when an InProgress backup moves on to another state or when a new backup moves to the Queued state. We also reconcile Queued backups periodically to handle the case of a Velero pod restart with Queued backups, as well as to handle possible edge cases where a queued backup doesn't get moved out of the queue at the point of backup completion or an error occurs during a prior Queued backup reconcile. ```go func (c *backupOperationsReconciler) SetupWithManager(mgr ctrl.Manager) error { // only consider Queued backups, order by QueuePosition gp := kube.NewGenericEventPredicate(func(object client.Object) bool { backup := object.(*velerov1api.Backup) return (backup.Status.Phase == velerov1api.BackupPhaseQueued) }) s := kube.NewPeriodicalEnqueueSource(c.logger.WithField("controller", constant.ControllerBackupOperations), mgr.GetClient(), &velerov1api.BackupList{}, c.frequency, kube.PeriodicalEnqueueSourceOption{ Predicates: []predicate.Predicate{gp}, OrderFunc: queuePositionOrderFunc, }) return ctrl.NewControllerManagedBy(mgr). For(&velerov1api.Backup{}, builder.WithPredicates(predicate.Funcs{ UpdateFunc: func(ue event.UpdateEvent) bool { backup := ue.ObjectNew.(*velerov1api.Backup) return backup.Status.Phase == "" || backup.status.Phase == velerov1api.BackupPhaseNew }, CreateFunc: func(event.CreateEvent) bool { return backup.Status.Phase == "" || backup.status.Phase == velerov1api.BackupPhaseNew }, DeleteFunc: func(de event.DeleteEvent) bool { return false }, GenericFunc: func(ge event.GenericEvent) bool { return false }, })). Watch( &source.Kind{Type: &velerov1api.Backup{}}, &handler.EnqueueRequestsFromMapFunc{ ToRequests: handler.ToRequestsFunc(func(a handler.MapObject) []reconcile.Request { backupList := velerov1api.BackupList{} if err := p.List(ctx, backupList); err != nil { p.logger.WithError(err).Error("error listing backups") return } requests = []reconcile.request{} // filter backup list by Phase=queued // sort backup list by queuePosition return requests }), }, builder.WithPredicates(predicate.Funcs{ UpdateFunc: func(ue event.UpdateEvent) bool { oldBackup := ue.ObjectOld.(*velerov1api.Backup) newBackup := ue.ObjectNew.(*velerov1api.Backup) return oldBackup.Status.Phase == velerov1api.BackupPhaseInProgress && newBackup.Status.Phase != velerov1api.BackupPhaseInProgress || oldBackup.Status.Phase != velerov1api.BackupPhaseQueued && newBackup.Status.Phase == velerov1api.BackupPhaseQueued }, CreateFunc: func(event.CreateEvent) bool { return false }, DeleteFunc: func(de event.DeleteEvent) bool { return false }, GenericFunc: func(ge event.GenericEvent) bool { return false }, }). WatchesRawSource(s). Named(constant.ControllerBackupQueue). Complete(c) } ``` New backups will be queued: Phase will be set to `Queued`, and `QueuePosition` will be set to a int value incremented from the highest current `QueuePosition` value among Queued backups. Queued backups will be removed from the queue if runnable: 1) If the total number of backups either InProgress or ReadyToStart is greater than or equal to the concurrency limit, then exit without removing from the queue. 2) If the current backup overlaps with any InProgress, ReadyToStart, or Queued backup with `QueuePosition < currentBackup.QueuePosition` then exit without removing from the queue. 3) If we get here, the backup is runnable. To resolve a potential race condition where an InProgress backup completes between reconciling the backup with QueuePosition `n-1` and reconciling the current backup with QueuePosition `n`, we also check to see whether there are any runnable backups in the queue ahead of this one. The only time this will happen is if a backup completes immediately before reconcile starts which either frees up a concurrency slot or removes a namespace conflict. In this case, we don't want to run the current backup since the one ahead of this one in the queue (which was recently passed over before the InProgress backup completed) must run first. In this case, exit without removing from the queue. 4) If we get here, remove the backup from the queue by setting Phase to `ReadyToStart` and `QueuePosition` to zero. Decrement the `QueuePosition` of any other Queued backups with a `QueuePosition` higher than the current backup's queue position prior to dequeuing. At this point, the backup reconciler will start the backup. `if len(inProgressBackups)+len(pendingStartBackups) >= concurrentBackups` ``` switch original.Status.Phase { case "", velerov1api.BackupPhaseNew: // enqueue backup -- set phase=Queued, set queuePosition=maxCurrentQueuePosition+1 } // We should only ever get these events when added in order by the periodical enqueue source // so as long as the current backup has not conflicts ahead of it or running, we should be good to // dequeue case "", velerov1api.BackupPhaseQueued: // list backups, filter on Queued, ReadyToStart, and InProgress // if number of InProgress backups + number of ReadyToStart backups >= concurrency limit, exit // generate list of all namespaces included in InProgress, ReadyToStart, and Queued backups with // queuePosition < backup.Status.QueuePosition // if overlap found, exit // check backups ahead of this one in the queue for runnability. If any are runnable, exit // dequeue backup: set Phase to ReadyToStart, QueuePosition to 0, and decrement QueuePosition // for all QueuedBackups behind this one in the queue } ``` The queue controller will run as a single reconciler thread, so we will not need to deal with concurrency issues when moving backups from New to Queued or from Queued to ReadyToStart, and all of the updates to QueuePosition will be from a single thread. ### Updates to Backup controller The Reconcile logic will be updated to respond to ReadyToStart backups instead of New backups: ``` @@ -234,8 +234,8 @@ func (b *backupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr // InProgress, we still need this check so we can return nil to indicate we've finished processing // this key (even though it was a no-op). switch original.Status.Phase { - case "", velerov1api.BackupPhaseNew: - // only process new backups + case velerov1api.BackupPhaseReadyToStart: + // only process ReadyToStart backups default: b.logger.WithFields(logrus.Fields{ "backup": kubeutil.NamespaceAndName(original), ``` In addition, it will be configured to run in parallel by setting `MaxConcurrentReconciles` based on the `concurrent-backups` server arg. ``` @@ -149,6 +149,9 @@ func NewBackupReconciler( func (b *backupReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&velerov1api.Backup{}). + WithOptions(controller.Options{ + MaxConcurrentReconciles: concurrentBackups, + }). Named(constant.ControllerBackup). Complete(b) } ``` The controller-runtime core reconciler logic already prevents the same resource from being reconciled by two different reconciler threads, so we don't need to worry about concurrency issues at the controller level. The workerPool reference will be moved from the backupReconciler to the backupRequest, since this will now be backup-specific, and the initialization code for the worker pool will be moved from the reconciler init into the backup reconcile. This worker pool will be shut down upon exiting the Reconcile method. ### Resilience to restart of velero pod The new backup phases (`Queued` and `ReadyToStart`) will be resilient to velero pod restarts. If the velero pod crashes or is restarted, only backups in the `InProgress` phase will be failed, so there is no change to current behavior. Queued backups will retain their queue position on restart, and ReadyToStart backups will move to InProgress when reconciled. ### Observability #### Logging When a backup is dequeued, an info log message will also include the wait time, calculated as `now - creationTimestamp`. When a backup is passed over due to overlap, an info log message will indicate which namespaces were in conflict. #### Velero CLI The `velero backup describe` output will include the current queue position for queued backups. ================================================ FILE: design/Implemented/csi-snapshots.md ================================================ # CSI Snapshot Support The Container Storage Interface (CSI) [introduced an alpha snapshot API in Kubernetes v1.12][1]. It will reach beta support in Kubernetes v1.17, scheduled for release in December 2019. This proposal documents an approach for integrating support for this snapshot API within Velero, augmenting its existing capabilities. ## Goals - Enable Velero to backup and restore CSI-backed volumes using the Kubernetes CSI CustomResourceDefinition API ## Non Goals - Replacing Velero's existing [VolumeSnapshotter][7] API - Replacing Velero's Restic support ## Background Velero has had support for performing persistent volume snapshots since its inception. However, support has been limited to a handful of providers. The plugin API introduced in Velero v0.7 enabled the community to expand the number of supported providers. In the meantime, the Kubernetes sig-storage advanced the CSI spec to allow for a generic storage interface, opening up the possibility of moving storage code out of the core Kubernetes code base. The CSI working group has also developed a generic snapshotting API that any CSI driver developer may implement, giving users the ability to snapshot volumes from a standard interface. By supporting the CSI snapshot API, Velero can extend its support to any CSI driver, without requiring a Velero-specific plugin be written, easing the development burden on providers while also reaching more end users. ## High-Level Design In order to support CSI's snapshot API, Velero must interact with the [`VolumeSnapshot`][2] and [`VolumeSnapshotContent`][3] CRDs. These act as requests to the CSI driver to perform a snapshot on the underlying provider's volume. This can largely be accomplished with Velero `BackupItemAction` and `RestoreItemAction` plugins that operate on these CRDs. Additionally, changes to the Velero server and client code are necessary to track `VolumeSnapshot`s that are associated with a given backup, similarly to how Velero tracks its own [`volume.Snapshot`][4] type. Tracking these is important for allowing users to see what is in their backup, and provides parity for the existing `volume.Snapshot` and [`PodVolumeBackup`][5] types. This is also done to retain the object store as Velero's source of truth, without having to query the Kubernetes API server for associated `VolumeSnapshot`s. `velero backup describe --details` will use the stored VolumeSnapshots to list CSI snapshots included in the backup to the user. ## Detailed Design ### Resource Plugins A set of [prototype][6] plugins was developed that informed this design. The plugins will be as follows: #### A `BackupItemAction` for `PersistentVolumeClaim`s, named `velero.io/csi-pvc` This plugin will act directly on PVCs, since an implementation of Velero's VolumeSnapshotter does not have enough information about the StorageClass to properly create the `VolumeSnapshot` objects. The associated PV will be queried and checked for the presence of `PersistentVolume.Spec.PersistentVolumeSource.CSI`. (See the "Snapshot Mechanism Selection" section below). If this field is `nil`, then the plugin will return early without taking action. If the `Backup.Spec.SnapshotVolumes` value is `false`, the plugin will return early without taking action. Additionally, to prevent creating CSI snapshots for volumes backed up by restic, the plugin will query for all pods in the `PersistentVolumeClaim`'s namespace. It will then filter out the pods that have the PVC mounted, and inspect the `backup.velero.io/backup-volumes` annotation for the associated volume's name. If the name is found in the list, then the plugin will return early without taking further action. Create a `VolumeSnapshot.snapshot.storage.k8s.io` object from the PVC. Label the `VolumeSnapshot` object with the [`velero.io/backup-name`][10] label for ease of lookup later. Also set an ownerRef on the `VolumeSnapshot` so that cascading deletion of the Velero `Backup` will delete associated `VolumeSnapshots`. The CSI controllers will create a `VolumeSnapshotContent.snapshot.storage.k8s.io` object associated with the `VolumeSnapshot`. Associated `VolumeSnapshotContent` objects will be retrieved and updated with the [`velero.io/backup-name`][10] label for ease of lookup later. `velero.io/volume-snapshot-name` will be applied as a label to the PVC so that the `VolumeSnapshot` can be found easily for restore. `VolumeSnapshot`, `VolumeSnapshotContent`, and `VolumeSnapshotClass` objects would be returned as additional items to be backed up. GitHub issue [1566][18] represents this work. The `VolumeSnapshotContent.Spec.VolumeSnapshotSource.SnapshotHandle` field is the link to the underlying platform's on-disk snapshot, and must be preserved for restoration. The plugin will _not_ wait for the `VolumeSnapshot.Status.readyToUse` field to be `true` before returning. This field indicates that the snapshot is ready to use for restoration, and for different vendors can indicate that the snapshot has been made durable. However, the applications can proceed as soon as `VolumeSnapshot.Status.CreationTime` is set. This also maintains current Velero behavior, which allows applications to quiesce and resume quickly, with minimal interruption. Any sort of monitoring or waiting for durable snapshots, either Velero-native or CSI snapshots, are not covered by this proposal. ``` K8s object relationships inside of the backup tarball +-----------------------+ +-----------------------+ | PersistentVolumeClaim +-------------->+ PersistentVolume | +-----------+-----------+ +-----------+-----------+ ^ ^ | | | | | | +-----------+-----------+ +-----------+-----------+ | VolumeSnapshot +<------------->+ VolumeSnapshotContent | +-----------------------+ +-----------------------+ ``` #### A `RestoreItemAction` for `VolumeSnapshotContent` objects, named `velero.io/csi-vsc` On restore, `VolumeSnapshotContent` objects are cleaned so that they may be properly associated with IDs assigned by the target cluster. Only `VolumeSnapshotContent` objects with the `velero.io/backup-name` label will be processed, using the plugin's `AppliesTo` function. The metadata (excluding labels), `PersistentVolumeClaim.UUID`, and `VolumeSnapshotRef.UUID` fields will be cleared. The reference fields are cleared because the associated objects will get new UUIDs in the cluster. This also maps to the "import" case of [the snapshot API][1]. This means the relationship between the `VolumeSnapshot` and `VolumeSnapshotContent` is one way until the CSI controllers rebind them. ``` K8s objects after the velero.io/csi-vsc plugin has run +-----------------------+ +-----------------------+ | PersistentVolumeClaim +-------------->+ PersistentVolume | +-----------------------+ +-----------------------+ +-----------------------+ +-----------------------+ | VolumeSnapshot +-------------->+ VolumeSnapshotContent | +-----------------------+ +-----------------------+ ``` #### A `RestoreItemAction` for `VolumeSnapshot` objects, named `velero.io/csi-vs` `VolumeSnapshot` objects must be prepared for importing into the target cluster by removing IDs and metadata associated with their origin cluster. Only `VolumeSnapshot` objects with the `velero.io/backup-name` label will be processed, using the plugin's `AppliesTo` function. Metadata (excluding labels) and `Source` (that is, the pointer to the `PersistentVolumeClaim`) fields on the object will be cleared. The `VolumeSnapshot.Spec.SnapshotContentName` is the link back to the `VolumeSnapshotContent` object, and thus the actual snapshot. The `Source` field indicates that a new CSI snapshot operation should be performed, which isn't relevant on restore. This follows the "import" case of [the snapshot API][1]. The `Backup` associated with the `VolumeSnapshot` will be queried, and set as an ownerRef on the `VolumeSnapshot` so that deletion can cascade. ``` +-----------------------+ +-----------------------+ | PersistentVolumeClaim +-------------->+ PersistentVolume | +-----------------------+ +-----------------------+ +-----------------------+ +-----------------------+ | VolumeSnapshot +-------------->+ VolumeSnapshotContent | +-----------------------+ +-----------------------+ ``` #### A `RestoreItemAction` for `PersistentVolumeClaim`s named `velero.io/csi-pvc` On restore, `PersistentVolumeClaims` will need to be created from the snapshot, and thus will require editing before submission. Only `PersistentVolumeClaim` objects with the `velero.io/volume-snapshot-name` label will be processed, using the plugin's `AppliesTo` function. Metadata (excluding labels) will be cleared, and the `velero.io/volume-snapshot-name` label will be used to find the relevant `VolumeSnapshot`. A reference to the `VolumeSnapshot` will be added to the `PersistentVolumeClaim.DataSource` field. ``` +-----------------------+ | PersistentVolumeClaim | +-----------------------+ +-----------------------+ +-----------------------+ | VolumeSnapshot +-------------->+ VolumeSnapshotContent | +-----------------------+ +-----------------------+ ``` #### VolumeSnapshotClasses No special logic is required to restore `VolumeSnapshotClass` objects. These plugins should be provided with Velero, as there will also be some changes to core Velero code to enable association of a `Backup` to the included `VolumeSnapshot`s. ### Velero server changes Any non-plugin code changes must be behind a `EnableCSI` feature flag and the behavior will be opt-in until it's exited beta status. This will allow the development to continue on the feature while it's in pre-production state, while also reducing the need for long-lived feature branches. [`persistBackup`][8] will be extended to query for all `VolumeSnapshot`s associated with the backup, and persist the list to JSON. [`BackupStore.PutBackup`][9] will receive an additional argument, `volumeSnapshots io.Reader`, that contains the JSON representation of `VolumeSnapshots`. This will be written to a file named `csi-snapshots.json.gz`. [`defaultRestorePriorities`][11] should be rewritten to the following to accommodate proper association between the CSI objects and PVCs. `CustomResourceDefinition`s are moved up because they're necessary for creating the CSI CRDs. The CSI CRDs are created before `PersistentVolume`s and `PersistentVolumeClaim`s so that they may be used as data sources. GitHub issue [1565][17] represents this work. ```go var defaultRestorePriorities = []string{ "namespaces", "storageclasses", "customresourcedefinitions", "volumesnapshotclass.snapshot.storage.k8s.io", "volumesnapshotcontents.snapshot.storage.k8s.io", "volumesnapshots.snapshot.storage.k8s.io", "persistentvolumes", "persistentvolumeclaims", "secrets", "configmaps", "serviceaccounts", "limitranges", "pods", "replicaset", } ``` ### Restic and CSI interaction Volumes found in a `Pod`'s `backup.velero.io/backup-volumes` list will use Velero's current Restic code path. This also means Velero will continue to offer Restic as an option for CSI volumes. The `velero.io/csi-pvc` BackupItemAction plugin will inspect pods in the namespace to ensure that it does not act on PVCs already being backed up by restic. This is preferred to modifying the PVC due to the fact that Velero's current backup process backs up PVCs and PVs mounted to pods at the same time as the pod. A drawback to this approach is that we're querying all pods in the namespace per PVC, which could be a large number. In the future, the plugin interface could be improved to have some sort of context argument, so that additional data such as our existing `resticSnapshotTracker` could be passed to plugins and reduce work. ### Garbage collection and deletion To ensure that all created resources are deleted when a backup expires or is deleted, `VolumeSnapshot`s will have an `ownerRef` defined pointing to the Velero backup that created them. In order to fully delete these objects, each `VolumeSnapshotContent`s object will need to be edited to ensure the associated provider snapshot is deleted. This will be done by editing the object and setting `VolumeSnapshotContent.Spec.DeletionPolicy` to `Delete`, regardless of whether or not the default policy for the class is `Retain`. See the Deletion Policies section below. The edit will happen before making Kubernetes API deletion calls to ensure that the cascade works as expected. Deleting a Velero `Backup` or any associated CSI object via `kubectl` is unsupported; data will be lost or orphaned if this is done. ### Other snapshots included in the backup Since `VolumeSnapshot` and `VolumeSnapshotContent` objects are contained within a Velero backup tarball, it is possible that all CRDs and on-disk provider snapshots have been deleted, yet the CRDs are still within other Velero backup tarballs. Thus, when a Velero backup that contains these CRDs is restored, the `VolumeSnapshot` and `VolumeSnapshotContent` objects are restored into the cluster, the CSI controllers will attempt to reconcile their state, and there are two possible states when the on-disk snapshot has been deleted: 1) If the driver _does not_ support the `ListSnapshots` gRPC method, then the CSI controllers have no way of knowing how to find it, and sets the `VolumeSnapshot.Status.readyToUse` field to `true`. 2) If the driver _does_ support the `ListSnapshots` gRPC method, then the CSI controllers will query the state of the on-disk snapshot, see it is missing, and set `VolumeSnapshot.Status.readyToUse` and `VolumeSnapshotContent.Status.readyToUse` fields to `false`. ## Velero client changes To use CSI features, the Velero client must use the `EnableCSI` feature flag. [`DescribeBackupStatus`][13] will be extended to download the `csi-snapshots.json.gz` file for processing. GitHub Issue [1568][19] captures this work. A new `describeCSIVolumeSnapshots` function should be added to the [output][12] package that knows how to render the included `VolumeSnapshot` names referenced in the `csi-snapshots.json.gz` file. ### Snapshot selection mechanism The most accurate, reliable way to detect if a PersistentVolume is a CSI volume is to check for a non-`nil` [`PersistentVolume.Spec.PersistentVolumeSource.CSI`][16] field. Using the [`volume.beta.kubernetes.io/storage-provisioner`][14] is not viable, since the usage is for any PVC that should be dynamically provisioned, and is _not_ limited to CSI implementations. It was [introduced with dynamic provisioning support][15] in 2016, predating CSI. In the `BackupItemAction` for PVCs, the associated PV will be queried and checked for the presence of `PersistentVolume.Spec.PersistentVolumeSource.CSI`. Volumes with any other `PersistentVolumeSource` set will use Velero's current VolumeSnapshotter plugin code path. ### VolumeSnapshotLocations and VolumeSnapshotClasses Velero uses its own `VolumeSnapshotLocation` CRDs to specify configuration options for a given storage system. In Velero, this often includes topology information such as regions or availability zones, as well as credential information. CSI volume snapshotting has a `VolumeSnapshotClass` CRD which also contains configuration options for a given storage system, but these options are not the same as those that Velero would use. Since CSI volume snapshotting is operating within the same storage system that manages the volumes already, it does not need the same topology or credential information that Velero does. As such, when used with CSI volumes, Velero's `VolumeSnapshotLocation` CRDs are not relevant, and could be omitted. This will create a separate path in our documentation for the time being, and should be called out explicitly. ## Alternatives Considered * Implementing similar logic in a Velero VolumeSnapshotter plugin was considered. However, this is inappropriate given CSI's data model, which requires a PVC/PV's StorageClass. Given the arguments to the VolumeSnapshotter interface, the plugin would have to instantiate its own client and do queries against the Kubernetes API server to get the necessary information. This is unnecessary given the fact that the `BackupItemAction` and `RestoreItemAction` APIs can act directly on the appropriate objects. Additionally, the VolumeSnapshotter plugins and CSI volume snapshot drivers overlap - both produce a snapshot on backup and a PersistentVolume on restore. Thus, there's not a logical place to fit the creation of VolumeSnapshot creation in the VolumeSnapshotter interface. * Implement CSI logic directly in Velero core code. The plugins could be packaged separately, but that doesn't necessarily make sense with server and client changes being made to accommodate CSI snapshot lookup. * Implementing the CSI logic entirely in external plugins. As mentioned above, the necessary plugins for `PersistentVolumeClaim`, `VolumeSnapshot`, and `VolumeSnapshotContent` could be hosted out-out-of-tree from Velero. In fact, much of the logic for creating the CSI objects will be driven entirely inside of the plugin implementation. However, Velero currently has no way for plugins to communicate that some arbitrary data should be stored in or retrieved from object storage, such as list of all `VolumeSnapshot` objects associated with a given `Backup`. This is important, because to display snapshots included in a backup, whether as native snapshots or Restic backups, separate JSON-encoded lists are stored within the backup on object storage. Snapshots are not listed directly on the `Backup` to fit within the etcd size limitations. Additionally, there are no client-side Velero plugin mechanisms, which means that the `velero describe backup --details` command would have no way of displaying the objects to the user, even if they were stored. ## Deletion Policies In order for underlying, provider-level snapshots to be retained similarly to Velero's current functionality, the `VolumeSnapshotContent.Spec.DeletionPolicy` field must be set to `Retain`. This is most easily accomplished by setting the `VolumeSnapshotClass.DeletionPolicy` field to `Retain`, which will be inherited by all `VolumeSnapshotContent` objects associated with the `VolumeSnapshotClass`. The current default for dynamically provisioned `VolumeSnapshotContent` objects is `Delete`, which will delete the provider-level snapshot when the `VolumeSnapshotContent` object representing it is deleted. Additionally, the `Delete` policy will cascade a deletion of a `VolumeSnapshot`, removing the associated `VolumeSnapshotContent` object. It is not currently possible to define a deletion policy on a `VolumeSnapshot` that gets passed to a `VolumeSnapshotContent` object on an individual basis. ## Security Considerations This proposal does not significantly change Velero's security implications within a cluster. If a deployment is using solely CSI volumes, Velero will no longer need privileges to interact with volumes or snapshots, as these will be handled by the CSI driver. This reduces the provider permissions footprint of Velero. Velero must still be able to access cluster-scoped resources in order to back up `VolumeSnapshotContent` objects. Without these objects, the provider-level snapshots cannot be located in order to re-associate them with volumes in the event of a restore. [1]: https://kubernetes.io/blog/2018/10/09/introducing-volume-snapshot-alpha-for-kubernetes/ [2]: https://github.com/kubernetes-csi/external-snapshotter/blob/master/client/apis/volumesnapshot/v1/types.go#L42 [3]: https://github.com/kubernetes-csi/external-snapshotter/blob/master/client/apis/volumesnapshot/v1/types.go#L262 [4]: https://github.com/heptio/velero/blob/main/pkg/volume/snapshot.go#L21 [5]: https://github.com/heptio/velero/blob/main/pkg/apis/velero/v1/pod_volume_backup.go#L88 [6]: https://github.com/heptio/velero-csi-plugin/ [7]: https://github.com/heptio/velero/blob/main/pkg/plugin/velero/volume_snapshotter.go#L26 [8]: https://github.com/heptio/velero/blob/main/pkg/controller/backup_controller.go#L560 [9]: https://github.com/heptio/velero/blob/main/pkg/persistence/object_store.go#L46 [10]: https://github.com/heptio/velero/blob/main/pkg/apis/velero/v1/labels_annotations.go#L21 [11]: https://github.com/heptio/velero/blob/main/pkg/cmd/server/server.go#L471 [12]: https://github.com/heptio/velero/blob/main/pkg/cmd/util/output/backup_describer.go [13]: https://github.com/heptio/velero/blob/main/pkg/cmd/util/output/backup_describer.go#L214 [14]: https://github.com/kubernetes/kubernetes/blob/8ea9edbb0290e9de1e6d274e816a4002892cca6f/pkg/controller/volume/persistentvolume/util/util.go#L69 [15]: https://github.com/kubernetes/kubernetes/pull/30285 [16]: https://github.com/kubernetes/kubernetes/blob/master/pkg/apis/core/types.go#L237 [17]: https://github.com/heptio/velero/issues/1565 [18]: https://github.com/heptio/velero/issues/1566 [19]: https://github.com/heptio/velero/issues/1568 ================================================ FILE: design/Implemented/custom-ca-support.md ================================================ # Custom CA Bundle Support for S3 Object Storage It is desired that Velero performs SSL verification on the Object Storage endpoint (BackupStorageLocation), but it is not guaranteed that the Velero container has the endpoints' CA bundle in it's system store. Velero needs to support the ability for a user to specify custom CA bundles at installation time and Velero needs to support a mechanism in the BackupStorageLocation Custom Resource to allow a user to specify a custom CA bundle. This mechanism needs to also allow Restic to access and use this custom CA bundle. ## Goals - Enable Velero to be configured with a custom CA bundle at installation - Enable Velero support for custom CA bundles with S3 API BackupStorageLocations - Enable Restic to use the custom CA bundles whether it is configured at installation time or on the BackupStorageLocation - Enable Velero client to take a CA bundle as an argument ## Non Goals - Support non-S3 providers ## Background Currently, in order for Velero to perform SSL verification of the object storage endpoint the user must manually set the `AWS_CA_BUNDLE` environment variable on the Velero deployment. If the user is using Restic, the user has to either: 1. Add the certs to the Restic container's system store 1. Modify Velero to pass in the certs as a CLI parameter to Restic - requiring a custom Velero deployment ## High-Level Design There are really 2 methods of using Velero with custom certificates: 1. Including a custom certificate at Velero installation 1. Specifying a custom certificate to be used with a `BackupStorageLocation` ### Specifying a custom cert at installation On the Velero deployment at install time, we can set the AWS environment variable `AWS_CA_BUNDLE` which will allow Velero to communicate over https with the proper certs when communicating with the S3 bucket. This means we will add the ability to specify a custom CA bundle at installation time. For more information, see "Install Command Changes". On the Restic daemonset, we will want to also mount this secret at a pre-defined location. In the `restic` pkg, the command to invoke restic will need to be updated to pass the path to the cert file that is mounted if it is specified in the config. This is good, but doesn't allow us to specify different certs when `BackupStorageLocation` resources are created. ### Specifying a custom cert on BSL In order to support custom certs for object storage, Velero will add an additional field to the `BackupStorageLocation`'s provider `Config` resource to provide a secretRef which will contain the coordinates to a secret containing the relevant cert file for object storage. In order for Restic to be able to consume and use this cert, Velero will need the ability to write the CA bundle somewhere in memory for the Restic pod to consume it. To accomplish this, we can look at the code for managing restic repository credentials. The way this works today is that the key is stored in a secret in the Velero namespace, and each time Velero executes a restic command, the contents of the secret are read and written out to a temp file. The path to this file is then passed to restic and removed afterwards. pass the path of the temp file to restic, and then remove the temp file afterwards. See ref #1 and #2. This same approach can be taken for CA bundles. The bundle can be stored in a secret which is referenced on the BSL and written to a temp file prior to invoking Restic. [1](https://github.com/vmware-tanzu/velero/blob/main/pkg/restic/repository_manager.go#L238-L245) [2](https://github.com/vmware-tanzu/velero/blob/main/pkg/restic/common.go#L168-L203) ## Detailed Design The `AWS_CA_BUNDLE` environment variable works for the Velero deployment because this environment variable is passed into the AWS SDK which is used in the [plugin][1] to build up the config object. This means that a user can simply define the CA bundle in the deployment as an env var. This can be utilized for the installation of Velero with a custom cert by simply setting this env var to the contents of the CA bundle, or the env var can be mapped to a secret which is controlled at installation time. I recommend using a secret as it makes the Restic integration easier as well. At installation time, if a user has specified a custom cert then the Restic daemonset should be updated to include the secret mounted at a predefined path. We could optionally use the system store for all custom certs added at installation time. Restic supports using the custom certs [in addition][3] to the root certs. In the case of the BSL being created with a secret reference, then at runtime the secret will need to be consumed. This secret will be read and applied to the AWS `session` object. The `getSession()` function will need to be updated to take in the custom CA bundle so it can be passed [here][4]. The Restic controller will need to be updated to write the contents of the CA bundle secret out to a temporary file inside of the restic pod.The restic [command invocation][2] will need to be updated to include the path to the file as an argument to the restic server using `--cacert`. For the path when a user defines a custom cert on the BSL, Velero will be responsible for updating the daemonset to include the secret mounted as a volume at a predefined path. Where we mount the secret is a fine detail, but I recommend mounting the certs to `/certs` to keep it in line with the other volume mount paths being used. ### Install command changes The installation flags should be updated to include the ability to pass in a cert file. Then the install command would do the heavy lifting of creating a secret and updating the proper fields on the deployment and daemonset to mount the secret at a well defined path. ### Velero client changes Since the Velero client is responsible for gathering logs and information about the Object Storage, this implementation should include a new flag `--cacert` which can be used when communicating with the Object Storage. Additionally, the user should be able to set this in their client configuration. The command would look like: ``` $ velero client config set cacert PATH ``` [1]: https://github.com/vmware-tanzu/velero-plugin-for-aws/blob/main/velero-plugin-for-aws/object_store.go#L135 [2]: https://github.com/vmware-tanzu/velero/blob/main/pkg/restic/command.go#L47 [3]: https://github.com/restic/restic/blob/main/internal/backend/http_transport.go#L81 [4]: https://github.com/vmware-tanzu/velero-plugin-for-aws/blob/main/velero-plugin-for-aws/object_store.go#L154 ================================================ FILE: design/Implemented/delete-item-action.md ================================================ # Delete Item Action Plugins ## Abstract Velero should provide a way to delete items created during a backup, with a model and interface similar to that of BackupItemAction and RestoreItemAction plugins. These plugins would be invoked when a backup is deleted, and would receive items from within the backup tarball. ## Background As part of Container Storage Interface (CSI) snapshot support, Velero added a new pattern for backing up and restoring snapshots via BackupItemAction and RestoreItemAction plugins. When others have tried to use this pattern, however, they encountered issues with deleting the resources made in their own ItemAction plugins, as Velero does not expose any sort of extension at backup deletion time. These plugins largely seek to delete resources that exist outside of Kubernetes. This design seeks to provide the missing extension point. ## Goals - Provide a DeleteItemAction API for plugins to implement - Update Velero backup deletion logic to invoke registered DeleteItemAction plugins. ## Non Goals - Specific implementations of the DeleteItemAction API beyond test cases. - Rollback of DeleteItemAction execution. ## High-Level Design The DeleteItemAction plugin API will closely resemble the RestoreItemAction plugin design, in that plugins will receive the Velero `Backup` Go struct that is being deleted and a matching Kubernetes resource extracted from the backup tarball. The Velero backup deletion process will be modified so that if there are any DeleteItemAction plugins registered, the backup tarball will be downloaded and extracted, similar to how restore logic works now. Then, each item in the backup tarball will be iterated over to see if a DeleteItemAction plugin matches for it. If a DeleteItemAction plugin matches, the `Backup` and relevant item will be passed to the DeleteItemAction. The DeleteItemAction plugins will be run _first_ in the backup deletion process, before deleting snapshots from storage or `Restore`s from the Kubernetes API server. DeleteItemAction plugins *cannot* rollback their actions. This is because there is currently no way to recover other deleted components of a backup, such as volume/restic snapshots or other DeleteItemAction resources. DeleteItemAction plugins will be run in alphanumeric order based on their registered names. ## Detailed Design ### New types The `DeleteItemAction` interface is as follows: ```go // DeleteItemAction is an actor that performs an action based on an item in a backup that is being deleted. type DeleteItemAction interface { // AppliesTo returns information about which resources this action should be invoked for. // A DeleteItemAction's Execute function will only be invoked on items that match the returned // selector. A zero-valued ResourceSelector matches all resources. AppliesTo() (ResourceSelector, error) // Execute allows the ItemAction to perform arbitrary logic with the item being deleted. Execute(DeleteItemActionInput) error } ``` The `DeleteItemActionInput` type is defined as follows: ```go type DeleteItemActionInput struct { // Item is the item taken from the pristine backed up version of resource. Item runtime.Unstructured // Backup is the representation of the backup resource processed by Velero. Backup *api.Backup } ``` Both `DeleteItemAction` and `DeleteItemActionInput` will be defined in `pkg/plugin/velero/delete_item_action.go`. ### Generate protobuf definitions and client/servers In `pkg/plugin/proto`, add `DeleteItemAction.proto`. Protobuf definitions will be necessary for: ```protobuf message DeleteItemActionExecuteRequest { ... } message DeleteItemActionExecuteResponse { ... } message DeleteItemActionAppliesToRequest { ... } message DeleteItemActionAppliesToResponse { ... } service DeleteItemAction { rpc AppliesTo(DeleteItemActionAppliesToRequest) returns (DeleteItemActionAppliesToResponse) rpc Execute(DeleteItemActionExecuteRequest) returns (DeleteItemActionExecuteResponse) } ``` Once these are written, then a client and server implementation can be written in `pkg/plugin/framework/delete_item_action_client.go` and `pkg/plugin/framework/delete_item_action_server.go`, respectively. These should be largely the same as the client and server implementations for `RestoreItemAction` and `BackupItemAction` plugins. ### Restartable delete plugins Similar to `RestoreItemAction` and `BackupItemAction` plugins, restartable processes will need to be implemented. In `pkg/plugin/clientmgmt`, add `restartable_delete_item_action.go`, creating the following unexported type: ```go type restartableDeleteItemAction struct { key kindAndName sharedPluginProcess RestartableProcess config map[string]string } // newRestartableDeleteItemAction returns a new restartableDeleteItemAction. func newRestartableDeleteItemAction(name string, sharedPluginProcess RestartableProcess) *restartableDeleteItemAction { // ... } // getDeleteItemAction returns the delete item action for this restartableDeleteItemAction. It does *not* restart the // plugin process. func (r *restartableDeleteItemAction) getDeleteItemAction() (velero.DeleteItemAction, error) { // ... } // getDelegate restarts the plugin process (if needed) and returns the delete item action for this restartableDeleteItemAction. func (r *restartableDeleteItemAction) getDelegate() (velero.DeleteItemAction, error) { // ... } // AppliesTo restarts the plugin's process if needed, then delegates the call. func (r *restartableDeleteItemAction) AppliesTo() (velero.ResourceSelector, error) { // ... } // Execute restarts the plugin's process if needed, then delegates the call. func (r *restartableDeleteItemAction) Execute(input *velero.DeleteItemActionInput) (error) { // ... } ``` This file will be very similar in structure to ### Plugin manager changes Add the following methods to `pkg/plugin/clientmgmt/manager.go`'s `Manager` interface: ```go type Manager interface { ... // Get DeleteItemAction returns a DeleteItemAction plugin for name. GetDeleteItemAction(name string) (DeleteItemAction, error) // GetDeteteItemActions returns the all DeleteItemAction plugins. GetDeleteItemActions() ([]DeleteItemAction, error) } ``` The unexported `manager` type should implement both the `GetDeleteItemAction` and `GetDeleteItemActions`. Both of these methods should have the same exception for `velero.io/`-prefixed plugins that all other types do. `GetDeleteItemAction` and `GetDeleteItemActions` will invoke the `restartableDeleteItemAction` implementations. ### Deletion controller modifications `pkg/controller/backup_deletion_controller.go` will be updated to have plugin management invoked. In `processRequest`, before deleting snapshots, get any registered `DeleteItemAction` plugins. If there are none, proceed as normal. If there are one or more, download the backup tarball from backup storage, untar it to temporary storage, and iterate through the items, matching them to the applicable plugins. ## Alternatives Considered Another proposal for higher level `DeleteItemActions` was initially included, which would require implementers to individually download the backup tarball themselves. While this may be useful long term, it is not a good fit for the current goals as each plugin would be re-implementing a lot of boilerplate. See the deletion-plugins.md file for this alternative proposal in more detail. The `VolumeSnapshotter` interface is not generic enough to meet the requirements here, as it is specifically for taking snapshots of block devices. ## Security Considerations By their nature, `DeleteItemAction` plugins will be deleting data, which would normally be a security concern. However, these will only be invoked in two situations: either when a `BackupDeleteRequest` is sent via a user with the `velero` CLI or some other management system, or when a Velero `Backup` expires by going over its TTL. Because of this, the data deletion is not a concern. ## Compatibility In terms of backwards compatibility, this design should stay compatible with most Velero installations that are upgrading. If not DeleteItemAction plugins are present, then the backup deletion process should proceed the same way it worked prior to their inclusion. ## Implementation The implementation dependencies are, roughly, in the order as they are described in the [Detailed Design](#detailed-design) section. ## Open Issues ================================================ FILE: design/Implemented/deletion-plugins.md ================================================ # Deletion Plugins Status: Alternative Proposal ## Abstract Velero should introduce a new type of plugin that runs when a backup is deleted. These plugins will delete any external resources associated with the backup so that they will not be left orphaned. ## Background With the CSI plugin, Velero developers introduced a pattern of using BackupItemAction and RestoreItemAction plugins tied to PersistentVolumeClaims to create other resources to complete a backup. In the CSI plugin case, Velero does clean up of these other resources, which are Kubernetes Custom Resources, within the core Velero server. However, for external plugins that wish to use this same pattern, this is not a practical solution. Velero's core cannot be extended for all possible Custom Resources, and not external resources that get created are Kubernetes Custom Resources. Therefore, Velero needs some mechanism that allows plugin authors who have created resources within a BackupItemAction or RestoreItemAction plugin to ensure those resources are deleted, regardless of what system those resources reside in. ## Goals - Provide a new plugin type in Velero that is invoked when a backup is deleted. ## Non Goals - Implementations of specific deletion plugins. - Rollback of deletion plugin execution. ## High-Level Design Velero will provide a new plugin type that is similar to its existing plugin architecture. These plugins will be referred to as `DeleteAction` plugins. `DeleteAction` plugins will receive the `Backup` CustomResource being deleted on execution. `DeleteAction` plugins cannot prevent deletion of an item. This is because multiple `DeleteAction` plugins can be registered, and this proposal does not include rollback and undoing of a deletion action. Thus, if multiple `DeleteAction` plugins have already run but another would request the deletion of a backup stopped, the backup that's retained would be inconsistent. `DeleteActions` will apply to `Backup`s based on a label on the `Backup` itself. In order to ensure that `Backup`s don't execute `DeleteAction` plugins that are not relevant to them, `DeleteAction` plugins can register an `AppliesTo` function which will define a label selector on Velero backups. `DeleteActions` will be run in alphanumerical order by plugin name. This order is somewhat arbitrary, but will be used to give authors and users a somewhat predictable order of events. ## Detailed Design The `DeleteAction` plugins will implement the following Go interface, defined in `pkg/plugin/velero/deletion_action.go`: ```go type DeleteAction struct { // AppliesTo will match the DeleteAction plugin against Velero Backups that it should operate against. AppliesTo() // Execute runs the custom plugin logic and may connect to external services. Execute(backup *api.backup) error } ``` The following methods would be added to the `clientmgmt.Manager` interface in `pkg/pluginclientmgmt/manager.go`: ``` type Manager interface { ... // GetDeleteActions returns the registered DeleteActions. //TODO: do we need to get these by name, or can we get them all? GetDeleteActions([]velero.DeleteAction, error) ... ``` ## Alternatives Considered TODO ## Security Considerations TODO ## Compatibility Backwards compatibility should be straight-forward; if there are no installed `DeleteAction` plugins, then the backup deletion process will proceed as it does today. ## Implementation TODO ## Open Issues In order to add a custom label to the backup, the backup must be modifiable inside of the `BackupItemActon` and `RestoreItemAction` plugins, which it currently is not. A work around for now is for the user to apply a label to the backup at creation time, but that is not ideal. ================================================ FILE: design/Implemented/existing-resource-policy_design.md ================================================ # Add support for `ExistingResourcePolicy` to restore API ## Abstract Velero currently does not support any restore policy on Kubernetes resources that are already present in-cluster. Velero skips over the restore of the resource if it already exists in the namespace/cluster irrespective of whether the resource present in the restore is the same or different from the one present on the cluster. It is desired that Velero gives the option to the user to decide whether or not the resource in backup should overwrite the one present in the cluster. ## Background As of Today, Velero will skip over the restoration of resources that already exist in the cluster. The current workflow followed by Velero is (Using a `service` that is backed up for example): - Velero tries to attempt restore of the `service` - Fetches the `service` from the cluster - If the `service` exists then: - Checks whether the `service` instance in the cluster is equal to the `service` instance present in backup - If not equal then skips the restore of the `service` and adds a restore warning (except for [ServiceAccount objects](https://github.com/vmware-tanzu/velero/blob/574baeb3c920f97b47985ec3957debdc70bcd5f8/pkg/restore/restore.go#L1246)) - If equal then skips the restore of the `service` and mentions that the restore of resource `service` is skipped in logs It is desired to add the functionality to specify whether or not to overwrite the instance of resource `service` in cluster with the one present in backup during the restore process. Related issue: https://github.com/vmware-tanzu/velero/issues/4066 ## Goals - Add support for `ExistingResourcePolicy` to restore API for Kubernetes resources. ## Non Goals - Change existing restore workflow for `ServiceAccount` objects - Add support for `ExistingResourcePolicy` as `recreate` for Kubernetes resources. (Future scope feature) ## Unrelated Proposals (Completely different functionalities than the one proposed in the design) - Add support for `ExistingResourcePolicy` to restore API for Non-Kubernetes resources. - Add support for `ExistingResourcePolicy` to restore API for `PersistentVolume` data. ### Use-cases/Scenarios ### A. Production Cluster - Backup Cluster: Let's say you have a Backup Cluster which is identical to the Production Cluster. After some operations/usage/time the Production Cluster had changed itself, there might be new deployments, some secrets might have been updated. Now, this means that the Backup cluster will no longer be identical to the Production Cluster. In order to keep the Backup Cluster up to date/identical to the Production Cluster with respect to Kubernetes resources except PV data we would like to use Velero for scheduling new backups which would in turn help us update the Backup Cluster via Velero restore. Reference: https://github.com/vmware-tanzu/velero/issues/4066#issuecomment-954320686 ### B. Help identify resource delta: Here delta resources mean the resources restored by a previous backup, but they are no longer in the latest backup. Let's follow a sequence of steps to understand this scenario: - Consider there are 2 clusters, Cluster A, which has 3 resources - P1, P2 and P3. - Create a Backup1 from Cluster A which has P1, P2 and P3. - Perform restore on a new Cluster B using Backup1. - Now, Lets say in Cluster A resource P1 gets deleted and resource P2 gets updated. - Create a new Backup2 with the new state of Cluster A, keep in mind Backup1 has P1, P2 and P3 while Backup2 has P2' and P3. - So the Delta here is (|Cluster B - Backup2|), Delete P1 and Update P2. - During Restore time we would want the Restore to help us identify this resource delta. Reference: https://github.com/vmware-tanzu/velero/pull/4613#issuecomment-1027260446 ## High-Level Design ### Approach 1: Add a new spec field `existingResourcePolicy` to the Restore API In this approach we do *not* change existing velero behavior. If the resource to restore in cluster is equal to the one backed up then do nothing following current Velero behavior. For resources that already exist in the cluster that are not equal to the resource in the backup (other than Service Accounts). We add a new optional spec field `existingResourcePolicy` which can have the following values: 1. `none`: This is the existing behavior, if Velero encounters a resource that already exists in the cluster, we simply skip restoration. 2. `update`: This option would provide the following behavior. - Unchanged resources: Velero would update the backup/restore labels on the unchanged resources, if labels patch fails Velero adds a restore error. - Changed resources: Velero will first try to patch the changed resource, Now if the patch: - succeeds: Then the in-cluster resource gets updated with the labels as well as the resource diff - fails: Velero adds a restore warning and tries to just update the backup/restore labels on the resource, if the labels patch also fails then we add restore error. 3. `recreate`: If resource already exists, then Velero will delete it and recreate the resource. *Note:* The `recreate` option is a non-goal for this enhancement proposal, but it is considered as a future scope. Another thing to highlight is that Velero will not be deleting any resources in any of the policy options proposed in this design but Velero will patch the resources in `update` policy option. Example: A. The following Restore will execute the `existingResourcePolicy` restore type `none` for the `services` and `deployments` present in the `velero-protection` namespace. ``` Kind: Restore … includeNamespaces: velero-protection includeResources: - services - deployments existingResourcePolicy: none ``` B. The following Restore will execute the `existingResourcePolicy` restore type `update` for the `secrets` and `daemonsets` present in the `gdpr-application` namespace. ``` Kind: Restore … includeNamespaces: gdpr-application includeResources: - secrets - daemonsets existingResourcePolicy: update ``` ### Approach 2: Add a new spec field `existingResourcePolicyConfig` to the Restore API In this approach we give user the ability to specify which resources are to be included for a particular kind of force update behaviour, essentially a more granular approach where in the user is able to specify a resource:behaviour mapping. It would look like: `existingResourcePolicyConfig`: - `patch:` - `includedResources:` [ ]string - `recreate:` - `includedResources:` [ ]string *Note:* - There is no `none` behaviour in this approach as that would conform to the current/default Velero restore behaviour. - The `recreate` option is a non-goal for this enhancement proposal, but it is considered as a future scope. Example: A. The following Restore will execute the restore type `patch` and apply the `existingResourcePolicyConfig` for `secrets` and `daemonsets` present in the `inventory-app` namespace. ``` Kind: Restore … includeNamespaces: inventory-app existingResourcePolicyConfig: patch: includedResources - secrets - daemonsets ``` ### Approach 3: Combination of Approach 1 and Approach 2 Now, this approach is somewhat a combination of the aforementioned approaches. Here we propose addition of two spec fields to the Restore API - `existingResourceDefaultPolicy` and `existingResourcePolicyOverrides`. As the names suggest ,the idea being that `existingResourceDefaultPolicy` would describe the default velero behaviour for this restore and `existingResourcePolicyOverrides` would override the default policy explicitly for some resources. Example: A. The following Restore will execute the restore type `patch` as the `existingResourceDefaultPolicy` but will override the default policy for `secrets` using the `existingResourcePolicyOverrides` spec as `none`. ``` Kind: Restore … includeNamespaces: inventory-app existingResourceDefaultPolicy: patch existingResourcePolicyOverrides: none: includedResources - secrets ``` ## Detailed Design ### Approach 1: Add a new spec field `existingResourcePolicy` to the Restore API The `existingResourcePolicy` spec field will be an `PolicyType` type field. Restore API: ``` type RestoreSpec struct { . . . // ExistingResourcePolicy specifies the restore behaviour for the Kubernetes resource to be restored // +optional ExistingResourcePolicy PolicyType } ``` PolicyType: ``` type PolicyType string const PolicyTypeNone PolicyType = "none" const PolicyTypePatch PolicyType = "update" ``` ### Approach 2: Add a new spec field `existingResourcePolicyConfig` to the Restore API The `existingResourcePolicyConfig` will be a spec of type `PolicyConfiguration` which gets added to the Restore API. Restore API: ``` type RestoreSpec struct { . . . // ExistingResourcePolicyConfig specifies the restore behaviour for a particular/list of Kubernetes resource(s) to be restored // +optional ExistingResourcePolicyConfig []PolicyConfiguration } ``` PolicyConfiguration: ``` type PolicyConfiguration struct { PolicyTypeMapping map[PolicyType]ResourceList } ``` PolicyType: ``` type PolicyType string const PolicyTypePatch PolicyType = "patch" const PolicyTypeRecreate PolicyType = "recreate" ``` ResourceList: ``` type ResourceList struct { IncludedResources []string } ``` ### Approach 3: Combination of Approach 1 and Approach 2 Restore API: ``` type RestoreSpec struct { . . . // ExistingResourceDefaultPolicy specifies the default restore behaviour for the Kubernetes resource to be restored // +optional existingResourceDefaultPolicy PolicyType // ExistingResourcePolicyOverrides specifies the restore behaviour for a particular/list of Kubernetes resource(s) to be restored // +optional existingResourcePolicyOverrides []PolicyConfiguration } ``` PolicyType: ``` type PolicyType string const PolicyTypeNone PolicyType = "none" const PolicyTypePatch PolicyType = "patch" const PolicyTypeRecreate PolicyType = "recreate" ``` PolicyConfiguration: ``` type PolicyConfiguration struct { PolicyTypeMapping map[PolicyType]ResourceList } ``` ResourceList: ``` type ResourceList struct { IncludedResources []string } ``` The restore workflow changes will be done [here](https://github.com/vmware-tanzu/velero/blob/b40bbda2d62af2f35d1406b9af4d387d4b396839/pkg/restore/restore.go#L1245) ### CLI changes for Approach 1 We would introduce a new CLI flag called `existing-resource-policy` of string type. This flag would be used to accept the policy from the user. The velero restore command would look somewhat like this: ``` velero create restore --existing-resource-policy=update ``` Help message `Restore Policy to be used during the restore workflow, can be - none, update` The CLI changes will go at `pkg/cmd/cli/restore/create.go` We would also add a validation which checks for invalid policy values provided to this flag. Restore describer will also be updated to reflect the policy `pkg/cmd/util/output/restore_describer.go` ### Implementation Decision We have decided to go ahead with the implementation of Approach 1 as: - It is easier to implement - It is also easier to scale and leaves room for improvement and the door open to expanding to approach 3 - It also provides an option to preserve the existing velero restore workflow ================================================ FILE: design/Implemented/feature-flags.md ================================================ # Feature Flags Status: Accepted Some features may take a while to get fully implemented, and we don't necessarily want to have long-lived feature branches A simple feature flag implementation allows code to be merged into main, but not used unless a flag is set. ## Goals - Allow unfinished features to be present in Velero releases, but only enabled when the associated flag is set. ## Non Goals - A robust feature flag library. ## Background When considering the [CSI integration work](https://github.com/heptio/velero/pull/1661), the timelines involved presented a problem in balancing a release and longer-running feature work. A simple implementation of feature flags can help protect unfinished code while allowing the rest of the changes to ship. ## High-Level Design A new command line flag, `--features` will be added to the root `velero` command. `--features` will accept a comma-separated list of features, such as `--features EnableCSI,Replication`. Each feature listed will correspond to a key in a map in `pkg/features/features.go` defining whether a feature should be enabled. Any code implementing the feature would then import the map and look up the key's value. For the Velero client, a `features` key can be added to the `config.json` file for more convenient client invocations. ## Detailed Design A new `features` package will be introduced with these basic structs: ```go type FeatureFlagSet struct { flags map[string]bool } type Flags interface { // Enabled reports whether or not the specified flag is found. Enabled(name string) bool // Enable adds the specified flags to the list of enabled flags. Enable(names ...string) // All returns all enabled features All() []string } // NewFeatureFlagSet initializes and populates a new FeatureFlagSet func NewFeatureFlagSet(flags ...string) FeatureFlagSet ``` When parsing the `--features` flag, the entire `[]string` will be passed to `NewFeatureFlagSet`. Additional features can be added with the `Enable` function. Parsed features will be printed as an `Info` level message on server start up. No verification of features will be done in order to keep the implementation minimal. On the client side, `--features` and the `features` key in `config.json` file will be additive, resulting in the union of both. To disable a feature, the server must be stopped and restarted with a modified `--features` list. Similarly, the client process must be stopped and restarted without features. ## Alternatives Considered Omitted ## Security Considerations Omitted ================================================ FILE: design/Implemented/general-progress-monitoring.md ================================================ # Plugin Progress Monitoring This is intended as a replacement for the previously-approved Upload Progress Monitoring design ([Upload Progress Monitoring](upload-progress.md)) in order to expand the supported use cases beyond snapshot uploads to include what was previously called Async Backup/Restore Item Actions. This updated design should handle the combined set of use cases for those previously separate designs. Volume snapshotter plugin are used by Velero to take snapshots of persistent volume contents. Depending on the underlying storage system, those snapshots may be available to use immediately, they may be uploaded to stable storage internally by the plugin or they may need to be uploaded after the snapshot has been taken. We would like for Velero to continue on to the next part of the backup as quickly as possible but we would also like the backup to not be marked as complete until it is a usable backup. We'd also eventually like to bring the control of upload under the control of Velero and allow the user to make decisions about the ultimate destination of backup data independent of the storage system they're using. We would also like any internal or third party Backup or Restore Item Action to have the option of making use of this same ability to run some external process without blocking the current backup or restore operation. Beyond Volume Snapshotters, this is also needed for data mover operations on both backup and restore, and potentially useful for other third party operations -- for example in-cluster registry image backup or restore could make use of this feature in a third party plugin). ### Glossary - BIA: BackupItemAction - RIA: RestoreItemAction ## Examples - AWS - AWS snapshots return quickly, but are then uploaded in the background and cannot be used until EBS moves the data into S3 internally. - vSphere - The vSphere plugin takes a local snapshot and then the vSphere plugin uploads the data to S3. The local snapshot is usable before the upload completes. - Restic - Does not go through the volume snapshot path. Restic backups will block Velero progress until completed. However, with the more generalized approach in the revised design, restic/kopia backup and restore *could* make use of this framework if their actions are refactored as Backup/RestoreItemActions. - Data Movers - Data movers are asynchronous processes executed inside backup/restore item actions that applies to a specific Kubernetes resources. A common use case for data mover is to backup/restore PVCs whose data we want to move to some form of backup storage outside of using velero kopia/restic implementations. - Workflow - User takes velero backup of PVC A - BIA plugin applies to PVCs with compatible storage driver - BIA plugin triggers data mover - Most common use case would be for the plugin action to create a new CR which would trigger an external controller action - Another possible use case would be for the plugin to run its own async action in a goroutine, although this would be less resilient to plugin container restarts. - BIA plugin returns - Velero backup process continues - Main velero backup process monitors running BIA threads via gRPC to determine if process is done and healthy ## Primary changes from the original Upload Progress Monitoring design The most fundamental change here is that rather than proposing a new special-purpose SnapshotItemAction, the existing BackupItemAction plugin will be modified to accommodate an optional snapshot (or other item operation) ID return. The primary reasons for this change are as follows: 1. The intended scope has moved beyond snapshot processing, so it makes sense to support asynchronous operations in other backup or restore item actions. 2. We expect to have plugin API versioning implemented in Velero 1.10, making it feasible to implement changes in the existing plugin APIs now. 3. We will need this feature on both backup and restore, meaning that if we took the "new plugin type" approach, we'd need two new plugin types. 4. Other than the snapshot/operation ID return, the rest of the plugin processing is identical to Backup/RestoreItemActions. With separate plugin types, we'd have to repeat all of that logic (including dealing with additional items, etc.) twice. The other major change is that we will be applying this to both backups and restores, although the Volume Snapshotter use case only needs this on backup. This means that everything we're doing around backup phase and workflow will also need to be done for restore. Then there are various minor changes around terminology to make things more generic. Instead of "snapshotID", we'll have "operationID" (which for volume snapshotters will be a snapshot ID). ## Goals - Enable monitoring of backup/restore item action operations that continue after snapshotting and other operations have completed - Keep non-usable backups and restores (upload/persistence has not finished, etc.) from appearing as completed - Make use of plugin API versioning functionality to manage changes to Backup/RestoreItemAction interfaces - Enable vendors to plug their own data movers into velero using BIA/RIA plugins ## Non-goals - Today, Velero is unable to recover from an in progress backup when the velero server crashes (pod is deleted). This has an impact on running asynchronous processes, but it’s not something we intend to solve in this design. ## Models ### Internal configuration and management In this model, movement of the snapshot to stable storage is under the control of the snapshot plugin. Decisions about where and when the snapshot gets moved to stable storage are not directly controlled by Velero. This is the model for the current VolumeSnapshot plugins. ### Velero controlled management In this model, the snapshot is moved to external storage under the control of Velero. This enables Velero to move data between storage systems. This also allows backup partners to use Velero to snapshot data and then move the data into their backup repository. ## Backup and Restore phases Velero currently has backup/restore phases "InProgress" and "Completed". A backup moves to the Completed phase when all of the volume snapshots have completed and the Kubernetes metadata has been written into the object store. However, the actual data movement may be happening in the background after the backup has been marked "Completed". The backup is not actually a stable backup until the data has been persisted properly. In some cases (e.g. AWS) the backup cannot be restored from until the snapshots have been persisted. Once the snapshots have been taken, however, it is possible for additional backups or restores (as long as they don't use not-yet-completed backups) to be made without interference. Waiting until all data has been moved before starting the next backup will slow the progress of the system without adding any actual benefit to the user. New backup/restore phases, "WaitingForPluginOperations" and "WaitingForPluginOperationsPartiallyFailed" will be introduced. When a backup or restore has entered one of these phases, Velero is free to start another backup/restore. The backup/restore will remain in the "WaitingForPluginOperations" phase until all BIA/RIA operations have completed (for example, for a volume snapshotter, until all data has been successfully moved to persistent storage). The backup/restore will not fail once it reaches this phase, although an error return from a plugin could cause a backup or restore to move to "PartiallyFailed". If the backup is deleted (cancelled), the plugins will attempt to delete the snapshots and stop the data movement - this may not be possible with all storage systems. In addition, for backups (but not restores), there will also be two additional phases, "Finalizing" and "FinalizingPartiallyFailed", which will handle any steps required after plugin operations have all completed. Initially, this will just include adding any required resources to the backup that might have changed during asynchronous operation execution, although eventually other cleanup actions could be added to this phase. ### State progression ![image](AsyncActionFSM.png) ### New When a backup/restore request is initially created, it is in the "New" phase. The next state is either "InProgress" or "FailedValidation" ### FailedValidation If the backup/restore request is incorrectly formed, it goes to the "FailedValidation" phase and terminates ### InProgress When work on the backup/restore begins, it moves to the "InProgress" phase. It remains in the "InProgress" phase until all pre/post execution hooks have been executed, all snapshots have been taken and the Kubernetes metadata and backup/restore info is safely written to the object store plugin. In the current implementation, Restic backups will move data during the "InProgress" phase. In the future, it may be possible to combine a snapshot with a Restic (or equivalent) backup which would allow for data movement to be handled in the "WaitingForPluginOperations" phase, The next phase would be "WaitingForPluginOperations" for backups or restores which have unfinished asynchronous plugin operations and no errors so far, "WaitingForPluginOperationsPartiallyFailed" for backups or restores which have unfinished asynchronous plugin operations at least one error, "Completed" for restores with no unfinished asynchronous plugin operations and no errors, "PartiallyFailed" for restores with no unfinished asynchronous plugin operations and at least one error, "Finalizing" for backups with no unfinished asynchronous plugin operations and no errors, "FinalizingPartiallyFailed" for backups with no unfinished asynchronous plugin operations and at least one error, or "PartiallyFailed". Backups/restores which would have a final phase of "Completed" or "PartiallyFailed" may move to the "WaitingForPluginOperations" or "WaitingForPluginOperationsPartiallyFailed" state. A backup/restore which will be marked "Failed" will go directly to the "Failed" phase. Uploads may continue in the background for snapshots that were taken by a "Failed" backup/restore, but no progress will not be monitored or updated. If there are any operations in progress when a backup is moved to the "Failed" phase (although with the current workflow, that shouldn't happen), Cancel() should be called on these operations. When a "Failed" backup is deleted, all snapshots will be deleted and at that point any uploads still in progress should be aborted. ### WaitingForPluginOperations (new) The "WaitingForPluginOperations" phase signifies that the main part of the backup/restore, including snapshotting has completed successfully and uploading and any other asynchronous BIA/RIA plugin operations are continuing. In the event of an error during this phase, the phase will change to WaitingForPluginOperationsPartiallyFailed. On success, the phase changes to "Finalizing" for backups and "Completed" for restores. Backups cannot be restored from when they are in the WaitingForPluginOperations state. ### WaitingForPluginOperationsPartiallyFailed (new) The "WaitingForPluginOperationsPartiallyFailed" phase signifies that the main part of the backup/restore, including snapshotting has completed, but there were partial failures either during the main part or during any async operations, including snapshot uploads. Backups cannot be restored from when they are in the WaitingForPluginOperationsPartiallyFailed state. ### Finalizing (new) The "Finalizing" phase signifies that asynchronous backup operations have all completed successfully and Velero is currently backing up any resources indicated by asynchronous plugins as items to back up after operations complete. Once this is done, the phase changes to Completed. Backups cannot be restored from when they are in the Finalizing state. ### FinalizingPartiallyFailed (new) The "FinalizingPartiallyFailed" phase signifies that, for a backup which had errors during initial processing or asynchronous plugin operation, asynchronous backup operations have all completed and Velero is currently backing up any resources indicated by asynchronous plugins as items to back up after operations complete. Once this is done, the phase changes to PartiallyFailed. Backups cannot be restored from when they are in the FinalizingPartiallyFailed state. ### Failed When a backup/restore has had fatal errors it is marked as "Failed" Backups in this state cannot be restored from. ### Completed The "Completed" phase signifies that the backup/restore has completed, all data has been transferred to stable storage (or restored to the cluster) and any backup in this state is ready to be used in a restore. When the Completed phase has been reached for a backup it is safe to remove any of the items that were backed up. ### PartiallyFailed The "PartiallyFailed" phase signifies that the backup/restore has completed and at least part of the backup/restore is usable. Restoration from a PartiallyFailed backup will not result in a complete restoration but pieces may be available. ## Workflow When a Backup or Restore Action is executed, any BackupItemAction, RestoreItemAction, or VolumeSnapshot plugins will return operation IDs (snapshot IDs or other plugin-specific identifiers). The plugin should be able to provide status on the progress for the snapshot and handle cancellation of the operation/upload if the snapshot is deleted. If the plugin is restarted, the operation ID should remain valid. When all snapshots have been taken and Kubernetes resources have been persisted to the ObjectStorePlugin the backup will either have fatal errors or will be at least partially usable. If the backup/restore has fatal errors it will move to the "Failed" state and finish. If a backup/restore fails, the upload or other operation will not be cancelled but it will not be monitored either. For backups in any phase, all snapshots will be deleted when the backup is deleted. Plugins will cancel any data movement or other operations and remove snapshots and other associated resources when the VolumeSnapshotter DeleteSnapshot method or DeleteItemAction Execute method is called. Velero will poll the plugins for status on the operations when the backup/restore exits the "InProgress" phase and has no fatal errors. If any operations are not complete, the backup/restore will move to either WaitingForPluginOperations or WaitingForPluginOperationsPartiallyFailed or Failed. Post-snapshot and other operations may take a long time and Velero and its plugins may be restarted during this time. Once a backup/restore has moved into the WaitingForPluginOperations or WaitingForPluginOperationsPartiallyFailed phase, another backup/restore may be started. While in the WaitingForPluginOperations or WaitingForPluginOperationsPartiallyFailed phase, the snapshots and item actions will be periodically polled. When all of the snapshots and item actions have reported success, restores will move directly to the Completed or PartiallyFailed phase, and backups will move to the Finalizing or FinalizingPartiallyFailed phase, depending on whether the backup/restore was in the WaitingForPluginOperations or WaitingForPluginOperationsPartiallyFailed phase. While in the Finalizing or FinalizingPartiallyFailed phase, Velero will update the backup with any resources indicated by plugins that they must be added to the backup after operations are completed, and then the backup will move to the Completed or PartiallyFailed phase, depending on whether there are any backup errors. The Backup resources will be written to object storage at the time the backup leaves the InProgress phase, but it will not be synced to other clusters (or usable for restores in the current cluster) until the backup has entered a final phase: Completed, Failed or PartiallyFailed. During the Finalizing phases, a the backup resources will be updated with any required resources related to asynchronous plugins. ## Reconciliation of InProgress backups InProgress backups will not have a `velero-backup.json` present in the object store. During reconciliation, backups which do not have a `velero-backup.json` object in the object store will be ignored. ## Plugin API changes ### OperationProgress struct type OperationProgress struct { Completed bool // True when the operation has completed, either successfully or with a failure Err string // Set when the operation has failed NCompleted, NTotal int64 // Quantity completed so far and the total quantity associated with the operation in operationUnits // For data mover and volume snapshotter use cases, this would be in bytes // On successful completion, completed and total should be the same. OperationUnits string // Units represented by completed and total -- for data mover and item // snapshotters, this will usually be bytes. Description string // Optional description of operation progress Started, Updated time.Time // When the upload was started and when the last update was seen. Not all // systems retain when the upload was begun, return Time 0 (time.Unix(0, 0)) // if unknown. } ### VolumeSnapshotter changes Two new methods will be added to the VolumeSnapshotter interface: Progress(snapshotID string) (OperationProgress, error) Cancel(snapshotID string) (error) Progress will report the current status of a snapshot upload. This should be callable at any time after the snapshot has been taken. In the event a plugin is restarted, if the operationID (snapshot ID) continues to be valid it should be possible to retrieve the progress. `error` is set if there is an issue retrieving progress. If the snapshot is has encountered an error during the upload, the error should be returned in OperationProgress and error should be nil. ### BackupItemAction and RestoreItemAction plugin changes Currently CSI snapshots and the Velero Plugin for vSphere are implemented as BackupItemAction plugins. While the majority of BackupItemAction plugins do not take snapshots or upload data, this functionality is useful for any longstanding plugin operation managed by an external process/controller so we will modify BackupItemAction and RestoreItemAction to optionally return an operationID in addition to the modified item. Velero can attempt to cancel an operation by calling the Cancel API call on the BIA/RIA. The plugin can then take any appropriate action as needed. Cancel will be called for unfinished operations on backup deletion, and possibly reaching timeout. Cancel is not intended to be used to delete/remove the results of completed actions and will have no effect on a completed action. Cancel has no return value apart from the standard Error return, but this should only be used for unexpected failures. Under normal operations, Cancel will simply return a nil error (and nothing else), whether or not the plugin is able to cancel the operation. _AsyncOperationsNotSupportedError_ should only be returned (by Progress) if the Backup/RestoreItemAction plugin should not be handling the item. If the Backup/RestoreItemAction plugin should handle the item but, for example, the item/snapshot ID cannot be found to report progress, Progress will return an InvalidOperationIDError error rather than a populated OperationProgress struct. If the item action does not start an asynchronous operation, then operationID will be empty. Three new methods will be added to the BackupItemAction interface, and the Execute() return signature will be modified: // Name returns the name of this BIA. Plugins which implement this interface must defined Name, // but its content is unimportant, as it won't actually be called via RPC. Velero's plugin infrastructure // will implement this directly rather than delegating to the RPC plugin in order to return the name // that the plugin was registered under. The plugins must implement the method to complete the interface. Name() string // Execute allows the BackupItemAction to perform arbitrary logic with the item being backed up, // including mutating the item itself prior to backup. The item (unmodified or modified) // should be returned, along with an optional slice of ResourceIdentifiers specifying // additional related items that should be backed up now, an optional operationID for actions which // initiate asynchronous actions, and a second slice of ResourceIdentifiers specifying related items // which should be backed up after all asynchronous operations have completed. This last field will be // ignored if operationID is empty, and should not be filled in unless the resource must be updated in the // backup after async operations complete (i.e. some of the item's Kubernetes metadata will be updated // during the asynch operation which will be required during restore) Execute(item runtime.Unstructured, backup *api.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) // Progress Progress(operationID string, backup *api.Backup) (velero.OperationProgress, error) // Cancel Cancel(operationID string, backup *api.Backup) error Three new methods will be added to the RestoreItemAction interface, and the RestoreItemActionExecuteOutput struct will be modified: // Name returns the name of this RIA. Plugins which implement this interface must defined Name, // but its content is unimportant, as it won't actually be called via RPC. Velero's plugin infrastructure // will implement this directly rather than delegating to the RPC plugin in order to return the name // that the plugin was registered under. The plugins must implement the method to complete the interface. Name() string // Execute allows the ItemAction to perform arbitrary logic with the item being restored, // including mutating the item itself prior to restore. The item (unmodified or modified) // should be returned, an optional OperationID, along with an optional slice of ResourceIdentifiers // specifying additional related items that should be restored, a warning (which will be // logged but will not prevent the item from being restored) or error (which will be logged // and will prevent the item from being restored) if applicable. If OperationID is specified // then velero will wait for this operation to complete before the restore is marked Completed. Execute(input *RestoreItemActionExecuteInput) (*RestoreItemActionExecuteOutput, error) // Progress Progress(operationID string, restore *api.Restore) (velero.OperationProgress, error) // Cancel Cancel(operationID string, restore *api.Restore) error // RestoreItemActionExecuteOutput contains the output variables for the ItemAction's Execution function. type RestoreItemActionExecuteOutput struct { // UpdatedItem is the item being restored mutated by ItemAction. UpdatedItem runtime.Unstructured // AdditionalItems is a list of additional related items that should // be restored. AdditionalItems []ResourceIdentifier // SkipRestore tells velero to stop executing further actions // on this item, and skip the restore step. When this field's // value is true, AdditionalItems will be ignored. SkipRestore bool // OperationID is an identifier which indicates an ongoing asynchronous action which Velero will // continue to monitor after restoring this item. If left blank, then there is no ongoing operation OperationID string } ## Changes in Velero backup format No changes to the existing format are introduced by this change. As part of the backup workflow changes, a `-itemoperations.json.gz` file will be added that contains the items and operation IDs (snapshotIDs) returned by VolumeSnapshotter and BackupItemAction plugins. Also, the creation of the `velero-backup.json` object will not occur until the backup moves to one of the terminal phases (_Completed_, _PartiallyFailed_, or _Failed_). Reconciliation should ignore backups that do not have a `velero-backup.json` object. The Backup/RestoreItemAction plugin identifier as well as the ItemID and OperationID will be stored in the `-itemoperations.json.gz`. When checking for progress, this info will be used to select the appropriate Backup/RestoreItemAction plugin to query for progress. Here's an example of what a record for a datamover plugin might look like: ``` { "spec": { "backupName": "backup-1", "backupUID": "f8c72709-0f73-46e1-a071-116bc4a76b07", "backupItemAction": "velero.io/volumesnapshotcontent-backup", "resourceIdentifier": { "Group": "snapshot.storage.k8s.io", "Resource": "VolumeSnapshotContent", "Namespace": "my-app", "Name": "my-volume-vsc" }, "operationID": "", "itemsToUpdate": [ { "Group": "velero.io", "Resource": "VolumeSnapshotBackup", "Namespace": "my-app", "Name": "vsb-1" } ] }, "status": { "operationPhase": "Completed", "error": "", "nCompleted": 12345, "nTotal": 12345, "operationUnits": "byte", "description": "", "Created": "2022-12-14T12:00:00Z", "Started": "2022-12-14T12:01:00Z", "Updated": "2022-12-14T12:11:02Z" }, } ``` The cluster that is creating the backup will have the Backup resource present and will be able to manage the backup before the backup completes. If the Backup resource is removed (e.g. Velero is uninstalled) before a backup completes and writes its `velero-backup.json` object, the other objects in the object store for the backup will be effectively orphaned. This can currently happen but the current window is much smaller. ### `-itemoperations.json.gz` The itemoperations file is similar to the existing `-itemsnapshots.json.gz` Each snapshot taken via BackupItemAction will have a JSON record in the file. Exact format TBD. This file will be uploaded to object storage at the end of processing all of the items in the backup, before the phase moves away from `InProgress`. ## Changes to Velero restores A `-itemoperations.json.gz` file will be added that contains the items and operation IDs returned by RestoreItemActions. The format will be the same as the `-itemoperations.json.gz` generated for backups. This file will be uploaded to object storage at the end of processing all of the items in the restore, before the phase moves away from `InProgress`. ## CSI snapshots For systems such as EBS, a snapshot is not available until the storage system has transferred the snapshot to stable storage. CSI snapshots expose the _readyToUse_ state that, in the case of EBS, indicates that the snapshot has been transferred to durable storage and is ready to be used. The CSI BackupItemAction.Progress method will poll that field and when completed, return completion. ## vSphere plugin The vSphere Plugin for Velero uploads snapshots to S3 in the background. This is also a BackupItemAction plugin, it will check the status of the Upload records for the snapshot and return progress. ## Backup workflow changes The backup workflow remains the same until we get to the point where the `velero-backup.json` object is written. At this point, Velero will run across all of the VolumeSnapshotter/BackupItemAction operations and call the _Progress_ method on each of them. If all backup item operations have finished (either successfully or failed), the backup will move to one of the finalize phases. If any of the snapshots or backup items are still being processed, the phase of the backup will be set to the appropriate phase (_WaitingForPluginOperations_ or _WaitingForPluginOperationsPartiallyFailed_), and the async backup operations controller will reconcile periodically and call Progress on any unfinished operations. In the event of any of the progress checks return an error, the phase will move to _WaitingForPluginOperationsPartiallyFailed_. Once all operations have completed, the backup will be moved to one of the finalize phases, and the backup finalizer controller will update the the `velero-backup.json`in the object store with any resources necessary after asynchronous operations are complete and the backup will move to the appropriate terminal phase. ## Restore workflow changes The restore workflow remains the same until velero would currently move the backup into one of the terminal states. At this point, Velero will run across all of the RestoreItemAction operations and call the _Progress_ method on each of them. If all restore item operations have finished (either successfully or failed), the restore will be completed and the restore will move to the appropriate terminal phase and the restore will be complete. If any of the restore items are still being processed, the phase of the restore will be set to the appropriate phase (_WaitingForPluginOperations_ or _WaitingForPluginOperationsPartiallyFailed_), and the async restore operations controller will reconcile periodically and call Progress on any unfinished operations. In the event of any of the progress checks return an error, the phase will move to _WaitingForPluginOperationsPartiallyFailed_. Once all of the operations have completed, the restore will be moved to the appropriate terminal phase. ## Restart workflow On restart, the Velero server will scan all Backup/Restore resources. Any Backup/Restore resources which are in the _InProgress_ phase will be moved to the _Failed_ phase. Any Backup/Restore resources in the _WaitingForPluginOperations_ or _WaitingForPluginOperationsPartiallyFailed_ phase will be treated as if they have been requeued and progress checked and the backup/restore will be requeued or moved to a terminal phase as appropriate. ## Notes on already-merged code which may need updating Since this design is modifying a previously-approved design, there is some preparation work based on the earlier upload progress monitoring design that may need modification as a result of these updates. Here is a list of some of these items: 1. Consts for the "Uploading" and "UploadingPartiallyFailed" phases have already been defined. These will need to be removed when the "WaitingForPluginOperations" and "WaitingForPluginOperationsPartiallyFailed" phases are defined. - https://github.com/vmware-tanzu/velero/pull/3805 1. Remove the ItemSnapshotter plugin APIs (and related code) since the revised design will reuse VolumeSnapshotter and BackupItemAction plugins. - https://github.com/vmware-tanzu/velero/pull/4077 - https://github.com/vmware-tanzu/velero/pull/4417 1. UploadProgressFeatureFlag shouldn't be needed anymore. The current design won't really need a feature flag here -- the new features will be added to V2 of the VolumeSnapshotter, BackupItemAction, and RestoreItemAction plugins, and it will only be used if there are plugins which return operation IDs. - https://github.com/vmware-tanzu/velero/pull/4416 1. Adds -itemsnapshots.gz file to backup (when provided) -- this is still part of the revised design, so it should stay. - https://github.com/vmware-tanzu/velero/pull/4429 1. Upload Progress Monitoring and Item Snapshotter basic support: This PR is not yet merged, so nothing will need to be reverted. While the implementation here will be useful in informing the implementation of the new design, several things have changed in the design proposal since the PR was written. - https://github.com/vmware-tanzu/velero/pull/4467 # Implementation tasks VolumeSnapshotter new plugin APIs BackupItemAction new plugin APIs RestoreItemAction new plugin APIs New backup phases New restore phases Defer uploading `velero-backup.json` AWS EBS plugin Progress implementation Operation monitoring Implementation of `-itemoperations.json.gz` file Implementation of `-itemoperations.json.gz` file Restart logic Change in reconciliation logic to ignore backups/restores that have not completed CSI plugin BackupItemAction Progress implementation vSphere plugin BackupItemAction Progress implementation (vSphere plugin team) # Open Questions 1. Do we need a Cancel operation for VolumeSnapshotter? - From feedback, I'm thinking we probably don't need it. The only real purpose of Cancel here is to tell the plugin that Velero won't be waiting anymore, so if there are any required custom cancellation actions, now would be a good time to perform them. For snapshot uploads that are already in proress, there's not really anything else to cancel. 2. Should we actually write the backup *before* moving to the WaitingForPluginOperations or WaitingForPluginOperationsPartiallyFailed phase rather than waiting until all operations have completed? The operations in question won't affect what gets written to object storage for the backup, and since we've already written the list of operations we're waiting for to object storage, writing the backup now would make the process resilient to Velero restart if it happens during WaitingForPluginOperations or WaitingForPluginOperationsPartiallyFailed ================================================ FILE: design/Implemented/generating-velero-crds-with-structural-schema.md ================================================ # Generating Velero CRDs with structural schema support As the apiextensions.k8s.io API moves to GA, structural schema in Custom Resource Definitions (CRDs) will become required. This document proposes updating the CRD generation logic as part of `velero install` to include structural schema for each Velero CRD. ## Goals - Enable structural schema and validation for Velero Custom Resources. ## Non Goals - Update Velero codebase to use Kubebuilder for controller/code generation. - Solve for keeping CRDs in the Velero Helm chart up-to-date. ## Background Currently, Velero CRDs created by the `velero install` command do not contain any structural schema. The CRD is simply [generated at runtime](https://github.com/heptio/velero/blob/8b0cf3855c2b8aa631cf22e63da0955f7b1d06a8/pkg/install/crd.go#L39) using the name and plurals from the [`velerov1api.CustomResources()`](https://github.com/heptio/velero/blob/8b0cf3855c2b8aa631cf22e63da0955f7b1d06a8/pkg/apis/velero/v1/register.go#L60) info. Updating the info returned by that method would be one way to add support for structural schema when generating the CRDs, but this would require manually describing the schema and would duplicate information from the API structs (e.g. comments describing a field). Instead, the [controller-tools](https://github.com/kubernetes-sigs/controller-tools) project from Kubebuilder provides tooling for generating CRD manifests (YAML) from the Velero API types. This document proposes adding _controller-tools_ to the project to automatically generate CRDs, and use these generated CRDs as part of `velero install`. ## High-Level Design _controller-tools_ works by reading the Go files that contain the API type definitions. It uses a combination of the struct fields, types, tags and comments to build the OpenAPIv3 schema for the CRDs. The tooling makes some assumptions based on conventions followed in upstream Kubernetes and the ecosystem, which involves some changes to the Velero API type definitions, especially around optional fields. In order for _controller-tools_ to read the Go files containing Velero API type definitions, the CRDs need to be generated at build time, as these files are not available at runtime (i.e. the Go files are not accessible by the compiled binary). These generated CRD manifests (YAML) will then need to be available to the `pkg/install` package for it to include when installing Velero resources. ## Detailed Design ### Changes to Velero API type definitions API type definitions need to be updated to correctly identify optional and required fields for each API type. Upstream Kubernetes defines all optional fields using the `omitempty` tag as well as a `// +optional` annotation above the field (e.g. see [PodSpec definition](https://github.com/kubernetes/api/blob/master/core/v1/types.go#L2835-L2838)). _controller-tools_ will mark a field as optional if it sees either the tag or the annotation, but to keep consistent with upstream, optional fields will be updated to use both indicators (as [suggested](https://github.com/kubernetes-sigs/kubebuilder/issues/479) by the Kubebuilder project). Additionally, upstream Kubernetes defines the metav1.ObjectMeta, metav1.ListMeta, Spec and Status as [optional on all types](https://github.com/kubernetes/api/blob/master/core/v1/types.go#L3517-L3531). Some Velero API types set the `omitempty` tag on Status, but not on other fields - these will all need to be updated to be made optional. Below is a list of the Velero API type fields and what changes (if any) will be made. Note that this only includes fields used in the spec, all status fields will become optional. | Type | Field | Changes | |---------------------------------|-------------------------|-------------------------------------------------------------| | BackupSpec | IncludedNamespaces | make optional | | | ExcludedNamespaces | make optional | | | IncludedResources | make optional | | | ExcludedResources | make optional | | | LabelSelector | make optional | | | SnapshotVolumes | make optional | | | TTL | make optional | | | IncludeClusterResources | make optional | | | Hooks | make optional | | | StorageLocation | make optional | | | VolumeSnapshotLocations | make optional | | BackupHooks | Resources | make optional | | BackupResourceHookSpec | Name | none (required) | | | IncludedNamespaces | make optional | | | ExcludedNamespaces | make optional | | | IncludedResources | make optional | | | ExcludedResources | make optional | | | LabelSelector | make optional | | | PreHooks | make optional | | | PostHooks | make optional | | BackupResourceHook | Exec | none (required) | | ExecHook | Container | make optional | | | Command | required, validation: MinItems=1 | | | OnError | make optional | | | Timeout | make optional | | HookErrorMode | | validation: Enum | | BackupStorageLocationSpec | Provider | none (required) | | | Config | make optional | | | StorageType | none (required) | | | AccessMode | make optional | | StorageType | ObjectStorage | make required | | ObjectStorageLocation | Bucket | none (required) | | | Prefix | make optional | | BackupStorageLocationAccessMode | | validation: Enum | | DeleteBackupRequestSpec | BackupName | none (required) | | DownloadRequestSpec | Target | none (required) | | DownloadTarget | Kind | none (required) | | | Name | none (required) | | DownloadTargetKind | | validation: Enum | | PodVolumeBackupSpec | Node | none (required) | | | Pod | none (required) | | | Volume | none (required) | | | BackupStorageLocation | none (required) | | | RepoIdentifier | none (required) | | | Tags | make optional | | PodVolumeRestoreSpec | Pod | none (required) | | | Volume | none (required) | | | BackupStorageLocation | none (required) | | | RepoIdentifier | none (required) | | | SnapshotID | none (required) | | ResticRepositorySpec | VolumeNamespace | none (required) | | | BackupStorageLocation | none (required) | | | ResticIdentifier | none (required) | | | MaintenanceFrequency | none (required) | | RestoreSpec | BackupName | none (required) - should be set to "" if using ScheduleName | | | ScheduleName | make optional | | | IncludedNamespaces | make optional | | | ExcludedNamespaces | make optional | | | IncludedResources | make optional | | | ExcludedResources | make optional | | | NamespaceMapping | make optional | | | LabelSelector | make optional | | | RestorePVs | make optional | | | IncludeClusterResources | make optional | | ScheduleSpec | Template | none (required) | | | Schedule | none (required) | | VolumeSnapshotLocationSpec | Provider | none (required) | | | Config | make optional | ### Build-time generation of CRD manifests The build image will be updated as follows to include the _controller-tool_ tooling: ```diff diff --git a/hack/build-image/Dockerfile b/hack/build-image/Dockerfile index b69a8c8a..07eac9c6 100644 --- a/hack/build-image/Dockerfile +++ b/hack/build-image/Dockerfile @@ -21,6 +21,8 @@ RUN mkdir -p /go/src/k8s.io && \ git clone -b kubernetes-1.15.3 https://github.com/kubernetes/apimachinery && \ # vendor code-generator go modules to be compatible with pre-1.15 cd /go/src/k8s.io/code-generator && GO111MODULE=on go mod vendor && \ + go get -d sigs.k8s.io/controller-tools/cmd/controller-gen && \ + cd /go/src/sigs.k8s.io/controller-tools && GO111MODULE=on go mod vendor && \ go get golang.org/x/tools/cmd/goimports && \ cd /go/src/golang.org/x/tools && \ git checkout 40a48ad93fbe707101afb2099b738471f70594ec && \ ``` To tie in the CRD manifest generation with existing scripts/workflows, the `hack/update-generated-crd-code.sh` script will be updated to use _controller-tools_ to generate CRDs manifests after it generates the client code. The generated CRD manifests will be placed in the `pkg/generated/crds/manifests` folder. Similarly to client code generation, these manifests will be checked-in to the git repo. Checking in these manifests allows including documentation and schema changes to API types as part of code review. ### Updating `velero install` to include generated CRD manifests As described above, CRD generation using _controller-tools_ will happen at build time due to need to inspect Go files. To enable the `velero install` to access the generated CRD manifests at runtime, the `pkg/generated/crds/manifests` folder will be embedded as binary data in the Velero binary (e.g. using a tool like [vfsgen](https://github.com/shurcooL/vfsgen) - see [POC branch](https://github.com/prydonius/velero/commit/4aa7413f97ce9b23e071b6054f600dd0c283351e)). `velero install` will then unmarshal the binary data as `unstructured.Unstructured` types and append them to the [resources list](https://github.com/heptio/velero/blob/8b0cf3855c2b8aa631cf22e63da0955f7b1d06a8/pkg/install/resources.go#L217) in place of the existing CRD generation. ## Alternatives Considered Instead of generating and bundling CRD manifests, it could be possible to instead embed the `pkg/apis` package in the Velero binary. With this, _controller-tools_ could be run at runtime during `velero install` to generate the CRD manifests. However, this would require including _controller-tools_ as a dependency in the project, which might not be desirable as it is a developer tool. Another option, to avoid embedding static files in the binary, would be to generate the CRD manifest as one YAML file in CI and upload it as a release artifact (e.g. using GitHub releases). `velero install` could then download this file for the current version and use it on install. The downside here is that `velero install` becomes dependent on the GitHub network, and we lose visibility on changes to the CRD manifests in the Git history. ## Security Considerations n/a ================================================ FILE: design/Implemented/handle-backup-of-volumes-by-resources-filters.md ================================================ # Handle backup of volumes by resources filters ## Abstract Currently, Velero doesn't have one flexible way to handle volumes. If users want to skip the backup of volumes or only backup some volumes in different namespaces in batch, currently they need to use the opt-in and opt-out approach one by one, or use label-selector but if it has big different labels on each different related pod, which is cumbersome when they have lots of volumes to handle with. it would be convenient if Velero could provide policies to handle the backup of volumes just by `some specific volumes conditions`. ## Background As of Today, Velero has lots of filters to handle (backup or skip backup) resources including resources filters like `IncludedNamespaces, ExcludedNamespaces`, label selectors like `LabelSelector, OrLabelSelectors`, annotation like `backup.velero.io/must-include-additional-items` etc. But it's not enough flexible to handle volumes, we need one generic way to handle volumes. ## Goals - Introducing flexible policies to handle volumes, and do not patch any labels or annotations to the pods or volumes. ## Non-Goals - We only handle volumes for backup and do not support restore. - Currently, only handles volumes, and does not support other resources. - Only environment-unrelated and platform-independent general volumes attributes are supported, do not support volumes attributes related to a specific environment. ## Use-cases/Scenarios ### Skip backup volumes by some attributes Users want to skip PV with the requirements: - option to skip all PV data - option to skip specified PV type (RBD, NFS) - option to skip specified PV size - option to skip specified storage-class ## High-Level Design First, Velero will provide the user with one YAML file template and all supported volume policies will be in. Second, writing your own configuration file by imitating the YAML template, it could be partial volume policies from the template. Third, create one configmap from your own configuration file, and the configmap should be in Velero install namespace. Fourth, create a backup with the command `velero backup create --resource-policies-configmap $policiesConfigmap`, which will reference the current backup to your volume policies. At the same time, Velero will validate all volume policies user imported, the backup will fail if the volume policies are not supported or some items could not be parsed. Fifth, the current backup CR will record the reference of volume policies configmap. Sixth, Velero first filters volumes by other current supported filters, at last, it will apply the volume policies to the filtered volumes to get the final matched volume to handle. ## Detailed Design The volume resources policies should contain a list of policies which is the combination of conditions and related `action`, when target volumes meet the conditions, the related `action` will take effection. Below is the API Design for the user configuration: ### API Design ```go type VolumeActionType string const Skip VolumeActionType = "skip" // Action defined as one action for a specific way of backup type Action struct { // Type defined specific type of action, it could be 'file-system-backup', 'volume-snapshot', or 'skip' currently Type VolumeActionType `yaml:"type"` // Parameters defined map of parameters when executing a specific action // +optional // +nullable Parameters map[string]interface{} `yaml:"parameters,omitempty"` } // VolumePolicy defined policy to conditions to match Volumes and related action to handle matched Volumes type VolumePolicy struct { // Conditions defined list of conditions to match Volumes Conditions map[string]interface{} `yaml:"conditions"` Action Action `yaml:"action"` } // ResourcePolicies currently defined slice of volume policies to handle backup type ResourcePolicies struct { Version string `yaml:"version"` VolumePolicies []VolumePolicy `yaml:"volumePolicies"` // we may support other resource policies in the future, and they could be added separately // OtherResourcePolicies: []OtherResourcePolicy } ``` The policies YAML config file would look like this: ```yaml version: v1 volumePolicies: # it's a list and if the input item matches the first policy, the latters will be ignored # each policy consists of a list of conditions and an action # each key in the object is one condition, and one policy will apply to resources that meet ALL conditions - conditions: # capacity condition matches the volumes whose capacity falls into the range capacity: "0,100Gi" csi: driver: ebs.csi.aws.com fsType: ext4 storageClass: - gp2 - ebs-sc action: type: volume-snapshot parameters: # optional parameters which are custom-defined parameters when doing an action volume-snapshot-timeout: "6h" - conditions: capacity: "0,100Gi" storageClass: - gp2 - ebs-sc action: type: file-system-backup - conditions: nfs: server: 192.168.200.90 action: # type of file-system-backup could be defined a second time type: file-system-backup - conditions: nfs: {} action: type: skip - conditions: csi: driver: aws.efs.csi.driver action: type: skip ``` ### Filter rules #### VolumePolicies The whole resource policies consist of groups of volume policies. For one specific volume policy which is a combination of one action and serval conditions. which means one action and serval conditions are the smallest unit of volume policy. Volume policies are a list and if the target volumes match the first policy, the latter will be ignored, which would reduce the complexity of matching volumes especially when there are multiple complex volumes policies. #### Action `Action` defined one action for a specific way of backup: - if choosing `Kopia` or `Restic`, the action value would be `file-system-backup`. - if choosing volume snapshot, the action value would be `volume-snapshot`. - if choosing skip backup of volume, the action value would be `skip`, and it will skip backup of volume no matter is `file-system-backup` or `volume-snapshot`. The policies could be extended for later other ways of backup, which means it may have some other `Action` value that will be assigned in the future. Both `file-system-backup` `volume-snapshot`, and `skip` could be partially or fully configured in the YAML file. And configuration could take effect only for the related action. #### Conditions The conditions are serials of volume attributes, the matched Volumes should meet all the volume attributes in one conditions configuration. ##### Supported conditions In Velero 1.11, we want to support the volume attributes listed below: - capacity: matching volumes have the capacity that falls within this `capacity` range. - storageClass: matching volumes those with specified `storageClass`, such as `gp2`, `ebs-sc` in eks. - matching volumes that used specified volume sources. ##### Parameters Parameters are optional for one specific action. For example, it could be `csi-snapshot-timeout: 6h` for CSI snapshot. #### Special rule definitions: - One single condition in `Conditions` with a specific key and empty value, which means the value matches any value. For example, if the `conditions.nfs` is `{}`, it means if `NFS` is used as `persistentVolumeSource` in Persistent Volume will be skipped no matter what the NFS server or NFS Path is. - The size of each single filter value should limit to 256 bytes in case of an unfriendly long variable assignment. - For capacity for PV or size for Volume, the value should include the lower value and upper value concatenated by commas. And it has several combinations below: - "0,5Gi" or "0Gi,5Gi" which means capacity or size matches from 0 to 5Gi, including value 0 and value 5Gi - ",5Gi" which is equal to "0,5Gi" - "5Gi," which means capacity or size matches larger than 5Gi, including value 5Gi - "5Gi" which is not supported and will be failed in validating configuration. ### Configmap Reference Currently, resources policies are defined in `BackupSpec` struct, it will be more and more bloated with adding more and more filters which makes the size of `Backup` CR bigger and bigger, so we want to store the resources policies in configmap, and `Backup` CRD reference to current configmap. the `configmap` user created would be like this: ```yaml apiVersion: v1 data: policies.yaml: ---- version: v1 volumePolicies: - conditions: capacity: "0,100Gi" csi: driver: ebs.csi.aws.com fsType: ext4 storageClass: - gp2 - ebs-sc action: type: volume-snapshot parameters: volume-snapshot-timeout: "6h" kind: ConfigMap metadata: creationTimestamp: "2023-01-16T14:08:12Z" name: backup01 namespace: velero resourceVersion: "17891025" uid: b73e7f76-fc9e-4e72-8e2e-79db717fe9f1 ``` A new variable `resourcePolices` would be added into `BackupSpec`, it's value is assigned with the current resources policy configmap ```yaml apiVersion: velero.io/v1 kind: Backup metadata: name: backup-1 spec: resourcePolices: refType: Configmap ref: backup01 ... ``` The configmap only stores those assigned values, not the whole resources policies. The name of the configmap is `$BackupName`, and it's in Velero install namespace. #### Resource policies configmap related The life cycle of resource policies configmap is managed by the user instead of Velero, which could make it more flexible and easy to maintain. - The resource policies configmap will remain in the cluster until the user deletes it. - Unlike backup, the resource policies configmap will not sync to the new cluster. So if the user wants to use one resource policies that do not sync to the new cluster, the backup will fail with resource policies not found. - One resource policies configmap could be used by multiple backups. - If the backup referenced resource policies configmap is been deleted, it won't affect the already existing backups, but if the user wants to reference the deleted configmap to create one new backup, it will fail with resource policies not found. #### Versioning We want to introduce the version field in the YAML data to contain break changes. Therefore, we won't follow a semver paradigm, for example in v1.11 the data look like this: ```yaml version: v1 volumePolicies: .... ``` Hypothetically, in v1.12 we add new fields like clusterResourcePolicies, the version will remain as v1 b/c this change is backward compatible: ```yaml version: v1 volumePolicies: .... clusterResourcePolicies: .... ``` Suppose in v1.13, we have to introduce a break change, at this time we will bump up the version: ```yaml version: v2 # This is just an example, we should try to avoid break change volume-policies: .... ``` We only support one version in Velero, so it won't be recognized if backup using a former version of YAML data. #### Multiple versions supporting To manage the effort for maintenance, we will only support one version of the data in Velero. Suppose that there is one break change for the YAML data in Velero v1.13, we should bump up the config version to v2, and v2 is only supported in v1.13. For the existing data with version: v1, it should migrate them when the Velero startup, this won't hurt the existing backup schedule CR as it only references the configmap. To make the migration easier, the configmap for such resource filter policies should be labeled manually before Velero startup like this, Velero will migrate the labeled configmap. We only support migrating from the previous version to the current version in case of complexity in data format conversion, which users could regenerate configmap in the new YAML data version, and it is easier to do version control. ```yaml apiVersion: v1 kind: ConfigMap metadata: labels: # This label can be optional but if this is not set, the backup will fail after the breaking change and the user will need to update the data manually velero.io/resource-filter-policies: "true" name: example namespace: velero data: ..... ``` ### Display of resources policies As the resource policies configmap is referenced by backup CR, the policies in configmap are not so intuitive, so we need to integrate policies in configmap to the output of the command `velero backup describe`, and make it more readable. ## Compatibility Currently, we have these resources filters: - IncludedNamespaces - ExcludedNamespaces - IncludedResources - ExcludedResources - LabelSelector - OrLabelSelectors - IncludeClusterResources - UseVolumeSnapshots - velero.io/exclude-from-backup=true - backup.velero.io/backup-volumes-excludes - backup.velero.io/backup-volumes - backup.velero.io/must-include-additional-items So it should be careful with the combination of volumes resources policies and the above resources filters. - When volume resource policies conflict with the above resource filters, we should respect the above resource filters. For example, if the user used the opt-out approach to `backup.velero.io/backup-volumes-excludes` annotation on the pod and also defined include volume in volumes resources filters configuration, we should respect the opt-out approach to skip backup of the volume. - If volume resource policies conflict with themselves, the first matched policy will be respect. ## Implementation This implementation should be included in Velero v1.11.0 Currently, in Velero v1.11.0 we only support `Action` `skip`, and support `file-system-backup` and `volume-snapshot` for the later version. And `Parameters` in `Action` is also not supported in v1.11.0, we will support in a later version. In Velero 1.11, we supported Conditions and format listed below: - capacity ```yaml capacity: "10Gi,100Gi" // match volume has the size between 10Gi and 100Gi ``` - storageClass ```yaml storageClass: // match volume has the storage class gp2 or ebs-sc - gp2 - ebs-sc ``` - volume sources (currently only support below format and attributes) 1. Specify the volume source name, the name could be `nfs`, `rbd`, `iscsi`, `csi` etc. ```yaml nfs : {} // match any volume has nfs volume source csi : {} // match any volume has csi volume source ``` 2. Specify details for the related volume source (currently we only support csi driver filter and nfs server or path filter) ```yaml csi: // match volume has nfs volume source and using `aws.efs.csi.driver` driver: aws.efs.csi.driver nfs: // match volume has nfs volume source and using below server and path server: 192.168.200.90 path: /mnt/nfs ``` The conditions also could be extended in later versions, such as we could further supporting filtering other volume source detail not only NFS and CSI. ## Alternatives Considered ### Configmap VS CRD Here we support the user define the YAML config file and storing the resources policies into configmap, also we could define one resource's policies CRD and store policies imported from the user-defined config file in the related CR. But CRD is more like one kind of resource with status, Kubernetes API Server handles the lifecycle of a CR and handles it in different statuses. Compared to CRD, Configmap is more focused to store data. ## Open Issues Should we support more than one version of filter policies configmap? ================================================ FILE: design/Implemented/include-exclude-in-resource-policy.md ================================================ # Proposal to add include exclude policy to resource policy This enhancement will allow the user to set include and exclude filters for resources in a resource policy configmap, so that these filters are reusable and the user will not need to set them each time they create a backup. ## Background As mentioned in issue [#8610](https://github.com/vmware-tanzu/velero/issues/8610). When there's a long list of resources to include or exclude in a backup, it can be cumbersome to set them each time a backup is created. There's a requirement to set these filters in a separate data structure so that they can be reused in multiple backups. ## High-Level Design We may extend the data structure of resource policy to add `includeExcludePolicy`, which include the include and exclude filters in the BackupSpec. When the user creates a backup which references the resource policy config `velero backup create --resource-policies-configmap `, the filters in "includeExcludePolicy" will take effect to filter the resources when velero collects the resources to backup. ## Detailed Design ### Data Structure The map `includeExcludePolicy` contains four fields `includedClusterScopedResources`, `excludedClusterScopedResources`, `includedNamespaceScopedResources`,`excludedNamespaceScopedResources`. These filters work exactly as the filters defined BackupSpec with the same names. An example of the policy looks like: ```yaml #omitted other irrelevant fields like 'version', 'volumePolicies' includeExcludePolicy: includedClusterScopedResources: - "cr" - "crd" - "pv" excludedClusterScopedResources: - "volumegroupsnapshotclass" - "ingressclass" includedNamespaceScopedResources: - "pod" - "service" - "deployment" - "pvc" excludedNamespaceScopedResources: - "configmap" ``` These filters are in the form of scoped include/exclude filters, which by design will not work with the "old" resource filters. Therefore, when a Backup references a resource policy configmap which has `includeExcludePolicy`, and at the same time it has the "old" resource filters, i.e. `includedResources`, `excludedResources`, `includeClusterResources` set in the BackupSpec, the Backup will fail with a validation error. ### Priorities A user may set the include/exclude filters in Backupspec and also in the resource policy configmap. In this case, the filters in both the Backupspec and the resource policy configmap will take effect. When there's a conflict, the filters in the Backupspec will take precedence. For example, if resource X is in the list of `includedNamespaceScopedResources` filter in the Backupspec, but it's also in the list of `excludedClusterScopedResources` in the resource policy configmap, then resource X will be included in the backup. In this way, users can set the filters in the resource policy configmap to cover most of their use cases, and then override them in the Backupspec when needed. ### Implementation In addition to the data structure change, we will need to implement the following changes: 1. A new function `CombineWithPolicy` will be added to the struct `ScopeIncludesExcludes`, which will combine the include/exclude filters in the resource policy configmap with the include/exclude filters in the Backupspec: ```go func (ie *ScopeIncludesExcludes) CombineWithPolicy(policy resourcepolicies.IncludeExcludePolicy) { mapFunc := scopeResourceMapFunc(ie.helper) for _, item := range policy.ExcludedNamespaceScopedResources { resolvedItem := mapFunc(item, true) if resolvedItem == "" { continue } if !ie.ShouldInclude(resolvedItem) && !ie.ShouldExclude(resolvedItem) { // The existing includeExcludes in the struct has higher priority, therefore, we should only add the item to the filter // when the struct does not include this item and this item is not yet in the excludes filter. ie.namespaceScopedResourceFilter.excludes.Insert(resolvedItem) } } ..... ``` This function will be called in the `kubernetesBackupper.BackupWithResolvers` function, to make sure the combined `ScopeIncludesExcludes` filter will be assigned to the `ResourceIncludesExcludes` filter of the Backup request. 2. Extra validation code will be added to the function `prepareBackupRequest` of `BackupReconciler` to check if there are "old" Resource filters in the BackupSpec when the Backup references a resource policy configmap which has `includeExcludePolicy`. ## Alternatives Considered We may put `includeExcludePolicy` in a separate configmap, but it will require adding extra field to BackupSpec to reference the configmap, which is not necessary. ================================================ FILE: design/Implemented/json-substitution-action-design.md ================================================ # Proposal to add support for Resource Modifiers (AKA JSON Substitutions) in Restore Workflow - [Proposal to add support for Resource Modifiers (AKA JSON Substitutions) in Restore Workflow](#proposal-to-add-support-for-resource-modifiers-aka-json-substitutions-in-restore-workflow) - [Abstract](#abstract) - [Goals](#goals) - [Non Goals](#non-goals) - [User Stories](#user-stories) - [Scenario 1](#scenario-1) - [Scenario 2](#scenario-2) - [Detailed Design](#detailed-design) - [Reference in velero API](#reference-in-velero-api) - [ConfigMap Structure](#configmap-structure) - [Operations supported by the JSON Patch library:](#operations-supported-by-the-json-patch-library) - [Advance scenarios](#advance-scenarios) - [Conditional patches using test operation](#conditional-patches-using-test-operation) - [Alternatives Considered](#alternatives-considered) - [Security Considerations](#security-considerations) - [Compatibility](#compatibility) - [Implementation](#implementation) - [Future Enhancements](#future-enhancements) - [Open Issues](#open-issues) ## Abstract Currently velero supports substituting certain values in the K8s resources during restoration like changing the namespace, changing the storage class, etc. This proposal is to add generic support for JSON substitutions in the restore workflow. This will allow the user specify filters for particular resources and then specify a JSON patch (operator, path, value) to apply on a resource. This will allow the user to substitute any value in the K8s resource without having to write a new RestoreItemAction plugin for each kind of substitution. ## Goals - Allow the user to specify a GroupResource, Name(optional), JSON patch for modification. - Allow the user to specify multiple JSON patch. ## Non Goals - Deprecating the existing RestoreItemAction plugins for standard substitutions(like changing the namespace, changing the storage class, etc.) ## User Stories ### Scenario 1 - Alice has a PVC which is encrypted using a DES(Disk Encryption Set - Azure example) mentioned in the PVC YAML through the StorageClass YAML. - Alice wishes to restore this snapshot to a different cluster. The new cluster does not have access to the same DES to provision disk's out of the snapshot. - She wishes to use a different DES for all the PVCs which use the certain DES. - She can use this feature to substitute the DES in all StorageClass YAMLs with the new DES without having to create a fresh storageclass, or understanding the name of the storageclass. ### Scenario 2 - Bob has multi zone cluster where nodes are spread across zones. - Bob has pinned certain pods to a particular zone using nodeSelector/ nodeaffinity on the pod spec. - In case of zone outage of the cloudprovider, Bob wishes to restore the workload to a different namespace in the same cluster, but change the zone pinning of the workload. - Bob can use this feature to substitute the nodeSelector/ nodeaffinity in the pod spec with the new zone pinning to quickly failover the workload to a different zone's nodes. ## Detailed Design - The design and approach is inspired from [kubectl patch command](https://github.com/kubernetes/kubectl/blob/0a61782351a027411b8b45b1443ec3dceddef421/pkg/cmd/patch/patch.go#L102C2-L104C1) ```bash # Update a container's image using a json patch with positional arrays kubectl patch pod valid-pod -type='json' -p='[{"op": "replace", "path": "/spec/containers/0/image", "value":"new image"}]' ``` - The user is expected to create a configmap with the desired Resource Modifications. Then the reference of the configmap will be provided in the RestoreSpec. - The core restore workflow before creating/updating a particular resource in the cluster will be checked against the filters provided and respective substitutions will be applied on it. ### Reference in velero API > Example of Reference to configmap in RestoreSpec ```yaml apiVersion: velero.io/v1 kind: Restore metadata: name: restore-1 spec: resourceModifier: refType: Configmap ref: resourcemodifierconfigmap ``` > Example CLI Command ```bash velero restore create --from-backup backup-1 --resource-modifier-configmap resourcemodifierconfigmap ``` ### Resource Modifier ConfigMap Structure - User first needs to provide details on which resources the JSON Substitutions need to be applied. - For this the user will provide 4 inputs - Namespaces(for NS Scoped resources), GroupResource (resource.group format similar to includeResources field in velero) and Name Regex(optional). - If the user does not provide the Name, the JSON Substitutions will be applied to all the resources of the given Group and Kind under the given namespaces. - Further the use will specify the JSON Patch using the structure of kubectl's "JSON Patch" based inputs. - Sample data in ConfigMap ```yaml version: v1 resourceModifierRules: - conditions: groupResource: persistentvolumeclaims resourceNameRegex: "mysql.*" namespaces: - bar - foo patches: - operation: replace path: "/spec/storageClassName" value: "premium" - operation: remove path: "/metadata/labels/test" ``` - The above configmap will apply the JSON Patch to all the PVCs in the namespaces bar and foo with name starting with mysql. The JSON Patch will replace the storageClassName with "premium" and remove the label "test" from the PVCs. - Note that the Namespace here is the original namespace of the backed up resource, not the new namespace where the resource is going to be restored. - The user can specify multiple JSON Patches for a particular resource. The patches will be applied in the order specified in the configmap. A subsequent patch is applied in order and if multiple patches are specified for the same path, the last patch will override the previous patches. - The user can specify multiple resourceModifierRules in the configmap. The rules will be applied in the order specified in the configmap. > Users need to create one configmap in Velero install namespace from a YAML file that defined resource modifiers. The creating command would be like the below: ```bash kubectl create cm --from-file -n velero ``` ### Operations supported by the JSON Patch library: - add - remove - replace - move - copy - test (covered below) ### Advance scenarios #### **Conditional patches using test operation** The `test` operation can be used to check if a particular value is present in the resource. If the value is present, the patch will be applied. If the value is not present, the patch will not be applied. This can be used to apply a patch only if a particular value is present in the resource. For example, if the user wishes to change the storage class of a PVC only if the PVC is using a particular storage class, the user can use the following configmap. ```yaml version: v1 resourceModifierRules: - conditions: groupResource: persistentvolumeclaims.storage.k8s.io resourceNameRegex: ".*" namespaces: - bar - foo patches: - operation: test path: "/spec/storageClassName" value: "premium" - operation: replace path: "/spec/storageClassName" value: "standard" ``` ## Alternatives Considered 1. JSON Path based addressal of json fields in the resource - This was the initial planned approach, but there is no open source library which gives satisfactory edit functionality with support for all operators supported by the JsonPath RFC. - We attempted modifying the [https://kubernetes.io/docs/reference/kubectl/jsonpath/](https://kubernetes.io/docs/reference/kubectl/jsonpath/) but given the complexity of the code it did not make sense to change it since it would become a long term maintainability problem. 1. RestoreItemAction for each kind of standard substitution - Not an extensible design. If a new kind of substitution is required, a new RestoreItemAction needs to be written. 1. RIA for JSON Substitution: The approach of doing JSON Substitution through a RestoreItemAction plugin was considered. But it is likely to have performance implications as the plugin will be invoked for all the resources. ## Security Considerations No security impact. ## Compatibility Compatibility with existing StorageClass mapping RestoreItemAction and similar plugins needs to be evaluated. ## Implementation - Changes in Restore CRD. Add a new field to the RestoreSpec to reference the configmap. - One example of where code will be modified: https://github.com/vmware-tanzu/velero/blob/eeee4e06d209df7f08bfabda326b27aaf0054759/pkg/restore/restore.go#L1266 On the obj before Creation, we can apply the conditions to check if the resource is filtered out using given parameters. Then using JsonPatch provided, we can update the resource. - For Jsonpatch - https://github.com/evanphx/json-patch library is used. - JSON Patch RFC https://datatracker.ietf.org/doc/html/rfc6902 ## Future enhancements - Additional features such as wildcard support in path, regex match support in value, etc. can be added in future. This would involve forking the https://github.com/evanphx/json-patch library and adding the required features, since those features are not supported by the library currently and are not part of jsonpatch RFC. ## Open Issues NA ================================================ FILE: design/Implemented/merge-patch-and-strategic-in-resource-modifier.md ================================================ # Proposal to Support JSON Merge Patch and Strategic Merge Patch in Resource Modifiers - [Proposal to Support JSON Merge Patch and Strategic Merge Patch in Resource Modifiers](#proposal-to-support-json-merge-patch-and-strategic-merge-patch-in-resource-modifiers) - [Abstract](#abstract) - [Goals](#goals) - [Non Goals](#non-goals) - [User Stories](#user-stories) - [Scenario 1](#scenario-1) - [Scenario 2](#scenario-2) - [Detailed Design](#detailed-design) - [How to choose the right patch type](#how-to-choose-the-right-patch-type) - [New Field MergePatches](#new-field-mergepatches) - [New Field StrategicPatches](#new-field-strategicpatches) - [Conditional Patches in ALL Patch Types](#conditional-patches-in-all-patch-types) - [Wildcard Support for GroupResource](#wildcard-support-for-groupresource) - [Helper Command to Generate Merge Patch and Strategic Merge Patch](#helper-command-to-generate-merge-patch-and-strategic-merge-patch) - [Security Considerations](#security-considerations) - [Compatibility](#compatibility) - [Implementation](#implementation) - [Future Enhancements](#future-enhancements) - [Open Issues](#open-issues) ## Abstract Velero introduced the concept of Resource Modifiers in v1.12.0. This feature allows the user to specify a configmap with a set of rules to modify the resources during restore. The user can specify the filters to select the resources and then specify the JSON Patch to apply on the resource. This feature is currently limited to the operations supported by JSON Patch RFC. This proposal is to add support for JSON Merge Patch and Strategic Merge Patch in the Resource Modifiers. This will allow the user to use the same configmap to apply JSON Merge Patch and Strategic Merge Patch on the resources during restore. ## Goals - Allow the user to specify a JSON patch, JSON Merge Patch or Strategic Merge Patch for modification. - Allow the user to specify multiple JSON Patch, JSON Merge Patch or Strategic Merge Patch. - Allow the user to specify mixed JSON Patch, JSON Merge Patch and Strategic Merge Patch in the same configmap. ## Non Goals - Deprecating the existing RestoreItemAction plugins for standard substitutions(like changing the namespace, changing the storage class, etc.) ## User Stories ### Scenario 1 - Alice has some Pods and part of them have an annotation `{"for": "bar"}`. - Alice wishes to restore these Pods to a different cluster without this annotation. - Alice can use this feature to remove this annotation during restore. ### Scenario 2 - Bob has a Pod with several containers and one container with name nginx has an image `repo1/nginx`. - Bob wishes to restore this Pod to a different cluster, but new cluster can not access repo1, so he pushes the image to repo2. - Bob can use this feature to update the image of container nginx to `repo2/nginx` during restore. ## Detailed Design - The design and approach is inspired by kubectl patch command and [this doc](https://kubernetes.io/docs/tasks/manage-kubernetes-objects/update-api-object-kubectl-patch/). - New fields `MergePatches` and `StrategicPatches` will be added to the `ResourceModifierRule` struct to support all three patch types. - Only one of the three patch types can be specified in a single `ResourceModifierRule`. - Add wildcard support for `groupResource` in `conditions` struct. - The workflow to create Resource Modifier ConfigMap and reference it in RestoreSpec will remain the same as described in document [Resource Modifiers](https://github.com/vmware-tanzu/velero/blob/main/site/content/docs/main/restore-resource-modifiers.md). ### How to choose the right patch type - [JSON Merge Patch](https://datatracker.ietf.org/doc/html/rfc7386) is a naively simple format, with limited usability. Probably it is a good choice if you are building something small, with very simple JSON Schema. - [JSON Patch](https://datatracker.ietf.org/doc/html/rfc6902) is a more complex format, but it is applicable to any JSON documents. For a comparison of JSON patch and JSON merge patch, see [JSON Patch and JSON Merge Patch](https://erosb.github.io/post/json-patch-vs-merge-patch/). - Strategic Merge Patch is a Kubernetes defined patch type, mainly used to process resources of type list. You can replace/merge a list, add/remove items from a list by key, change the order of items in a list, etc. Strategic merge patch is not supported for custom resources. For more details, see [this doc](https://kubernetes.io/docs/tasks/manage-kubernetes-objects/update-api-object-kubectl-patch/). ### New Field MergePatches MergePatches is a list to specify the merge patches to be applied on the resource. The merge patches will be applied in the order specified in the configmap. A subsequent patch is applied in order and if multiple patches are specified for the same path, the last patch will override the previous patches. Example of MergePatches in ResourceModifierRule ```yaml version: v1 resourceModifierRules: - conditions: groupResource: pods namespaces: - ns1 mergePatches: - patchData: | { "metadata": { "annotations": { "foo": null } } } ``` - The above configmap will apply the Merge Patch to all the pods in namespace ns1 and remove the annotation `foo` from the pods. - Both json and yaml format are supported for the patchData. ### New Field StrategicPatches StrategicPatches is a list to specify the strategic merge patches to be applied on the resource. The strategic merge patches will be applied in the order specified in the configmap. A subsequent patch is applied in order and if multiple patches are specified for the same path, the last patch will override the previous patches. Example of StrategicPatches in ResourceModifierRule ```yaml version: v1 resourceModifierRules: - conditions: groupResource: pods resourceNameRegex: "^my-pod$" namespaces: - ns1 strategicPatches: - patchData: | { "spec": { "containers": [ { "name": "nginx", "image": "repo2/nginx" } ] } } ``` - The above configmap will apply the Strategic Merge Patch to the pod with name my-pod in namespace ns1 and update the image of container nginx to `repo2/nginx`. - Both json and yaml format are supported for the patchData. ### Conditional Patches in ALL Patch Types Since JSON Merge Patch and Strategic Merge Patch do not support conditional patches, we will use the `test` operation of JSON Patch to support conditional patches in all patch types by adding it to `Conditions` struct in `ResourceModifierRule`. Example of test in conditions ```yaml version: v1 resourceModifierRules: - conditions: groupResource: persistentvolumeclaims.storage.k8s.io matches: - path: "/spec/storageClassName" value: "premium" mergePatches: - patchData: | { "metadata": { "annotations": { "foo": null } } } ``` - The above configmap will apply the Merge Patch to all the PVCs in all namespaces with storageClassName premium and remove the annotation `foo` from the PVCs. - You can specify multiple rules in the `matches` list. The patch will be applied only if all the matches are satisfied. ### Wildcard Support for GroupResource The user can specify a wildcard for groupResource in the conditions' struct. This will allow the user to apply the patches for all the resources of a particular group or all resources in all groups. For example, `*.apps` will apply to all the resources in the `apps` group, `*` will apply to all the resources in all groups. ### Helper Command to Generate Merge Patch and Strategic Merge Patch The patchData of Strategic Merge Patch is sometimes a bit complex for user to write. We can provide a helper command to generate the patchData for Strategic Merge Patch. The command will take the original resource and the modified resource as input and generate the patchData. It can also be used in JSON Merge Patch. Here is a sample code snippet to achieve this: ```go package main import ( "fmt" corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) func main() { pod := &corev1.Pod{ Spec: corev1.PodSpec{ Containers: []corev1.Container{ { Name: "web", Image: "nginx", }, }, }, } newPod := pod.DeepCopy() patch := client.StrategicMergeFrom(pod) newPod.Spec.Containers[0].Image = "nginx1" data, _ := patch.Data(newPod) fmt.Println(string(data)) // Output: // {"spec":{"$setElementOrder/containers":[{"name":"web"}],"containers":[{"image":"nginx1","name":"web"}]}} } ``` ## Security Considerations No security impact. ## Compatibility Compatible with current Resource Modifiers. ## Implementation - Use "github.com/evanphx/json-patch" to support JSON Merge Patch. - Use "k8s.io/apimachinery/pkg/util/strategicpatch" to support Strategic Merge Patch. - Use glob to support wildcard for `groupResource` in `conditions` struct. - Use `test` operation of JSON Patch to calculate the `matches` in `conditions` struct. ## Future enhancements - add a Velero subcommand to generate/validate the patchData for Strategic Merge Patch and JSON Merge Patch. - add jq support for more complex conditions or patches, to meet the situations that the current conditions or patches can not handle. like [this issue](https://github.com/vmware-tanzu/velero/issues/6344) ## Open Issues N/A ================================================ FILE: design/Implemented/move-gh-org.md ================================================ # Plan for moving the Velero GitHub repo into the VMware GitHub organization Currently, the Velero repository sits under the Heptio GitHub organization. With the acquisition of Heptio by VMware, it is due time that this repo moves to one of the VMware GitHub organizations. This document outlines a plan to move this repo to the VMware Tanzu (https://github.com/vmware-tanzu) organization. ## Goals - List all steps necessary to have this repo fully functional under the new org ## Non Goals - Highlight any step necessary around setting up the new organization and its members ## Action items ### Todo list #### Pre move - [ ] PR: Blog post communicating the move. https://github.com/heptio/velero/issues/1841. Who: TBD. - [ ] PR: Find/replace in all Go, script, yaml, documentation, and website files: `github.com/heptio/velero -> github.com/vmware-tanzu/velero`. Who: a Velero developer; TBD - [ ] PR: Update website with the correct GH links. Who: a Velero developer; TBD - [ ] PR: Change deployment and grpc-push scripts with the new location path. Who: a Velero developer; TBD - [ ] Delete branches not to be carried over (https://github.com/heptio/velero/branches/all). Who: Any of the current repo owners; TBD #### Move - [ ] Use GH UI to transfer the repository to the VMW org; must be accepted within a day. Who: new org owner; TBD - [ ] Make owners of this repo owners of repo in the new org. Who: new org owner; TBD - [ ] Update Travis CI. Who: Any of the new repo owners; TBD - [ ] Add DCO for signoff check (https://probot.github.io/apps/dco/). Who: Any of the new repo owners; TBD #### Post move - [ ] Each individual developer should point their origin to the new location: `git remote set-url origin git@github.com:vmware-tanzu/velero.git` - [ ] Transfer ZenHub. Who: Any of the new repo owners; TBD - [ ] Update Netlify deploy settings. Any of the new repo owners; TBD - [ ] GH app: Netlify integration. Who: Any of the new repo owners; TBD - [ ] GH app: Slack integration. Who: Any of the new repo owners; TBD - [ ] Add webhook: travis CI. Who: Any of the new repo owners; TBD - [ ] Add webhook: zenhub. Who: Any of the new repo owners; TBD - [ ] Move all 3 native provider plugins into their own individual repo. https://github.com/heptio/velero/issues/1537. Who: @carlisia. - [ ] Merge PRs from the "pre move" section - [ ] Create a team for the Velero core members (https://github.com/orgs/vmware-tanzu/teams/). Who: Any of the new repo owners; TBD ### Notes/How-Tos #### Transferring the GH repository All action items needed for the repo transfer are listed in the Todo list above. For details about what gets moved and other info, this is the GH documentation: https://help.github.com/en/articles/transferring-a-repository [Pending] We will find out this week who will be the organization owner(s) who will accept this transfer in the new GH org. This organization owner will make all current owners in this repo owners in the new org Velero repo. #### Updating Travis CI Someone with owner permission on the new repository needs to go to their Travis CI account and authorize Travis CI on the repo. Here are instructions: https://docs.travis-ci.com/user/tutorial/. After this, webhook notifications can be added following these instructions: https://docs.travis-ci.com/user/notifications/#configuring-webhook-notifications. #### Transferring ZenHub Pre-requisite: A new Zenhub account must exist for a vmware or vmware-tanzu organization. This page contains a pre-migration checklist for ensuring a repo migration goes well with Zenhub: https://help.zenhub.com/support/solutions/articles/43000010366-moving-a-repo-cross-organization-or-to-a-new-organization. After this, webhooks can be added by following these instructions: https://github.com/ZenHubIO/API#webhooks. #### Updating Netlify The settings for Netlify should remain the same, except that it now needs to be installed in the new repo. The instructions on how to install Netlify on the new repo are here: https://www.netlify.com/docs/github-permissions/. #### Communication strategy [Pending] We will find out this week how this move will be communicated to the community. In particular, the Velero repository move might be tied to the move of our provider plugins into their own repos, also in the new org: https://github.com/heptio/velero/issues/1814. #### TBD Many items on the todo list must be done by a repository member with owner permission. This doesn't all need to be done by the same person obviously, but we should specify if @skriss wants to split these tasks with any other owner(s). #### Other notes Might want to exclude updating documentation prior to v1.0.0. GH documentation does not specify if branches on the server are also moved. All links to the original repository location are automatically redirected to the new location. ## Alternatives Considered Alternatives such as moving Velero to its own organization, or even not moving at all, were considered. Collectively, however, the open source leadership decided it would be best to move it so it lives alongside other VMware supported cloud native related repositories. ## Security Considerations - Ensure that only the Velero core team has maintainer/owner privileges. ================================================ FILE: design/Implemented/move-plugin-repos.md ================================================ # Plan to extract the provider plugins out of (the Velero) tree Currently, the Velero project contains in-tree plugins for three cloud providers: AWS, Azure, and GCP. The Velero team has decided to extract each of those plugins into their own separate repository. This document details the steps necessary to create the new repositories, as well as a general design for what each plugin project will look like. ## Goals - Have 3 new repositories for each cloud provider plugin currently supported by the Velero team: AWS, Azure, and GCP - Have the currently in-tree cloud provider plugins behave like any other plugin external to Velero ## Non Goals - Extend the Velero plugin framework capability in any way - Create GH repositories for any plugin other then the currently 3 in-tree plugins - Extract out any plugin that is not a cloud provider plugin (ex: item action related plugins) ## Background With more and more providers wanting to support Velero, it gets more difficult to justify excluding those from being in-tree just as with the three original ones. At the same time, if we were to include any more plugins in-tree, it would ultimately become the responsibility of the Velero team to maintain an increasing number of plugins. This move aims to equalize the field so all plugins are treated equally. We also hope that, with time, developers interested in getting involved in the upkeep of those plugins will become active enough to be promoted to maintainers. Lastly, having the plugins live in their own individual repositories allows for iteration on them separately from the core codebase. ## Action items ### Todo list #### Repository creation - [ ] Use GH UI to create each repository in the new VMW org. Who: new org owner; TBD - [ ] Make owners of the Velero repo owners of each repo in the new org. Who: new org owner; TBD - [ ] Add Travis CI. Who: Any of the new repo owners; TBD - [ ] Add webhook: travis CI. Who: Any of the new repo owners; TBD - [ ] Add DCO for signoff check (https://probot.github.io/apps/dco/). Who: Any of the new repo owners; TBD #### Plugin changes - [ ] Modify Velero so it can install any of the provider plugins. https://github.com/heptio/velero/issues/1740 - Who: @nrb - [ ] Extract each provider plugin into their own repo. https://github.com/heptio/velero/issues/1537 - [ ] Create deployment and gcr-push scripts with the new location path. Who: @carlisia - [ ] Add documentation for how to use the plugin. Who: @carlisia - [ ] Update Helm chart to install Velero using any of the provider plugins. https://github.com/heptio/velero/issues/1819 - [ ] Upgrade script. https://github.com/heptio/velero/issues/1889. ### Notes/How-Tos #### Creating the GH repository [Pending] The organization owner will make all current owners in the Velero repo also owners in each of the new org plugin repos. #### Setting up Travis CI Someone with owner permission on the new repository needs to go to their Travis CI account and authorize Travis CI on the repo. Here are instructions: https://docs.travis-ci.com/user/tutorial/. After this, any webhook notifications can be added following these instructions: https://docs.travis-ci.com/user/notifications/#configuring-webhook-notifications. ## High-Level Design Each provider plugin will be an independent project, using the Velero library to implement their specific functionalities. The way Velero is installed will be changed to accommodate installing these plugins at deploy time, namely the Velero `install` command, as well as the Helm chart. Each plugin repository will need to have their respective images built and pushed to the same registry as the Velero images. ## Detailed Design ### Projects Each provider plugin will be an independent GH repository, named: `velero-plugin-aws`, `velero-plugin-azure`, and `velero-plugin-gcp`. Build of the project will be done the same way as with Velero, using Travis. Images for all the plugins will be pushed to the same repository as the Velero image, also using Travis. Releases of each of these plugins will happen in sync with releases of Velero. This will consist of having a tag in the repo and a tagged image build with the same release version as Velero so it makes it easy to identify what versions are compatible, starting at v1.2. Documentation for how to install and use the plugins will be augmented in the existing Plugins section of the Velero documentation. Documentation for how to use each plugin will reside in their respective repos. The navigation on the Velero documentation will be modified for easy discovery of the docs/images for these plugins. #### Version compatibility We will keep the major and minor release points in sync, but the plugins can have multiple minor dot something releases as long as it remains compatible with the corresponding major/minor release of Velero. Ex: | Velero | Plugin | Compatible? | |---|---|---| | v1.2 | v1.2 | ✅ | | v1.2 | v1.2.3 | ✅ | | v1.2 | v1.3 | 🚫 | | v1.3 | v1.2 | 🚫 | | v1.3 | v1.3.3 | ✅ | ### Installation As per https://github.com/heptio/velero/issues/1740, we will add a `plugins` flag to the Velero install command which will accept an array of URLs pointing to +1 images of plugins to be installed. The `velero plugin add` command should continue working as is, in specific, it should also allow the installation of any of the new 3 provider plugins. @nrb will provide specifics about how this change will be tackled, as well as what will be documented. Part of the work of adding the `plugins` flag will be removing the logic that adds `velero.io` name spacing to plugins that are added without it. The Helm chart that allows the installation of Velero will be modified to accept the array of plugin images with an added `plugins` configuration item. ### Design code changes and considerations The naming convention to use for name spacing each plugin will be `velero.io`, since they are currently maintained by the Velero team. Install dep Question: are there any places outside the plugins where we depend on the cloud-provider SDKs? can we eliminate those dependencies too? x - the `restic` package uses the `aws`. SDK to get the bucket region for the AWS object store (https://github.com/carlisia/velero/blob/32d46871ccbc6b03e415d1e3d4ad9ae2268b977b/pkg/restic/config.go#L41) - could not find usage of the cloud provider SDKs anywhere else. Plugins such as the pod -> pvc -> pv backupitemaction ones make sense to stay in the core repo as they provide some important logic that just happens to be implemented in a plugin. ### Upgrade The documentation for how to fresh install the out-of-tree plugin with Velero v1.2 will be specified together with the documentation for the install changes on issue https://github.com/heptio/velero/issues/1740. For upgrades, we will provide a script that will: - change the tag on the Velero deployment yaml for both the main image and any of th three plugins installed. - rename existing aws, azure or gcp plugin names to have the `velero.io/` namespace preceding the name (ex: `velero.io/aws). Alternatively, we could add CLI `velero upgrade` command that would make these changes. Ex: `velero upgrade 1.3` would upgrade from `v1.2` to `v1.3`. For upgrading: - Edit the provider field in the backupstoragelocations and volumesnapshotlocations CRDs to include the new namespace. ## Alternatives Considered We considered having the plugins all live in the same GH repository. The downside of that approach is ending up with a binary and image bigger than necessary, since they would contain the SDKs of all three providers. ## Security Considerations - Ensure that only the Velero core team has maintainer/owner privileges. ================================================ FILE: design/Implemented/multiple-arch-build-with-windows.md ================================================ # Multi-arch Build and Windows Build Support ## Background At present, Velero images could be built for linux-amd64 and linux-arm64. We need to support other platforms, i.e., windows-amd64. At present, for linux image build, we leverage Buildkit's `--platform` option to create the image manifest list in one build call. However, it is a limited way and doesn't fully support all multi-arch scenarios. Specifically, since the build is done in one call with the same parameters, it is impossbile to build images with different configurations (e.g., Windows build requires a different Dockerfile). At present, Velero by default build images locally, or no image or manifest is pushed to registry. However, docker doesn't support multi-arch build locally. We need to clarify the behavior of local build. ## Goals - Refactor the `make container` process to fully support multi-arch build - Add Windows build to the existing build process - Clarify the behavior of local build with multi-arch build capabilities - Don't change the pattern of the final image tag to be used by users ## Non-Goals - There may be some workarounds to make the multi-arch image/manifest fully available locally. These workarounds will not be adopted, so local build always build single-arch images ## Local Build For local build, two values of `--output` parameter for `docker buildx build` are supported: - `docker`: a docker format image is built, but the image is only built for the platform (`/`) as same as the building env. E.g., when building from linux-amd64 env, a single manifest of linux-amd64 is created regardless how the input parameters are configured. - `tar`: one or more images are built as tarballs according to the input platform (`/`) parameters. Specifically, one tarball is generated for each platform. The build process is the same with the `Build Separate Manifests` of `Push Build` as detailed below. Merely, the `--output` parameter diffs, as `type=tar;dest=`. The tarball is generated to the `_output` folder and named with the platform info, e.g., `_output/velero-main-linux-amd64.tar`. ## Push Build For push build, the `--output` parameter for `docker buildx build` is always `registry`. And build will go according to the input parameters and create multi-arch manifest lists. ### Step 1: Build Separate Manifests Instead of specifying multiple platforms (`/`) to `--platform` option, we add multiple `container-%` targets in Makefile and each target builds one platform representively. The goal here is to build multiple manifests through the multiple targets. However, `docker buildx build` by default creates a manifest list even though there is only one element in `--platform`. Therefore, two flags `--provenance=false` and `--sbom=false` will be set additionally to force `docker buildx build` to create manifests. Each manifest has a unique tag, the OS type and arch is added to the tag, in the pattern `$(REGISTRY)/$(BIN):$(VERSION)-$(OS)-$(ARCH)`. For example, `velero/velero:main-linux-amd64`. All the created manifests will be pushed to registry so that the all-in-one manifest list could be created. ### Step 2: Create All-In-One Manifest List The next step is to create a manifest list to include all the created manifests. This could be done by `docker manifest create` command, the tags created and pushed at Step 1 are passed to this command. A tag is also created for the manifest list, in the pattern `$(REGISTRY)/$(BIN):$(VERSION)`. For example, `velero/velero:main`. ### Step 3: Push All-In-One Manifest List The created manifest will be pushed to registry by command `docker manifest push`. ## Input Parameters Below are the input parameters that are configurable to meet different build purposes during Dev and release cycle: - BUILD_OUTPUT_TYPE: the type of output for the build, i.e., `docker`, `tar`, `registry`, while `docker` and `tar` is for local build; `registry` means push build. Default value is `docker` - BUILD_OS: which types of OS should be built for. Multiple values are accepted, e.g., `linux,windows`. Default value is `linux` - BUILD_ARCH: which types of architecture should be built for. Multiple values are accepted, e.g., `amd64,arm64`. Default value is `amd64` - BUILDX_INSTANCE: an existing buildx instance to be used by the build. Default value is which indicates the build to create a new buildx instance ## Windows Build Windows container images vary from Windows OS versions, e.g., `ltsc2022` for Windows server 2022 and `1809` for Windows server 2019. Images for different OS versions should be built separately. Therefore, separate build targets are added for each OS version, like `container-windows-%`. For the same reason, a new input parameter is added, `BUILD_WINDOWS_VERSION`. The default value is `ltsc2022`. Windows server 2022 is the only base image we will deliver officially, Windows server 2019 is not supported. In future, we may need to support Windows server 2025 base image. For local build to tar, the Windows OS version is also added to the name of the tarball, e.g., `_output/velero-main-windows-ltsc2022-amd64.tar`. At present, Windows container image only supports `amd64` as the architecture, so `BUILD_ARCH` is ignored for Windows. The Windows manifests need to be annotated with os type, arch, and os version. This will be done through `docker manifest annotate` command. ## Use Malti-arch Images In order to use the images, the manifest list's tag should be provided to `velero install` command or helm, the individual manifests are covered by the manifest list. During launch time, the container engine will load the right image to the container according to the platform of the running node. ## Build Samples **Local build to docker** ``` make container ``` The built image could be listed by `docker image ls`. **Local build for linux-amd64 and windows-amd64 to tar** ``` BUILD_OUTPUT_TYPE=tar BUILD_OS=linux,windows make container ``` Under `_output` directory, below files are generated: ``` velero-main-linux-amd64.tar velero-main-windows-ltsc2022-amd64.tar ``` **Local build for linux-amd64, linux-arm64 and windows-amd64 to tar** ``` BUILD_OUTPUT_TYPE=tar BUILD_OS=linux,windows BUILD_ARCH=amd64,arm64 make container ``` Under `_output` directory, below files are generated: ``` velero-main-linux-amd64.tar velero-main-linux-arm64.tar velero-main-windows-ltsc2022-amd64.tar ``` **Push build for linux-amd64 and windows-amd64** Prerequisite: login to registry, e.g., through `docker login` ``` BUILD_OUTPUT_TYPE=registry REGISTRY= BUILD_OS=linux,windows make container ``` Nothing is available locally, in the registry 3 tags are available: ``` velero/velero:main velero/velero:main-windows-ltsc2022-amd64 velero/velero:main-linux-amd64 ``` **Push build for linux-amd64, linux-arm64 and windows-amd64** Prerequisite: login to registry, e.g., through `docker login` ``` BUILD_OUTPUT_TYPE=registry REGISTRY= BUILD_OS=linux,windows BUILD_ARCH=amd64,arm64 make container ``` Nothing is available locally, in the registry 4 tags are available: ``` velero/velero:main velero/velero:main-windows-ltsc2022-amd64 velero/velero:main-linux-amd64 velero/velero:main-linux-arm64 ``` ================================================ FILE: design/Implemented/multiple-csi-volumesnapshotclass-support.md ================================================ # Proposal to add support for Multiple VolumeSnapshotClasses in CSI Plugin - [Proposal to add support for Multiple VolumeSnapshotClasses in CSI Plugin](#proposal-to-add-support-for-multiple-volumesnapshotclasses-in-csi-plugin) - [Abstract](#abstract) - [Background](#background) - [Goals](#goals) - [Non Goals](#non-goals) - [User Stories](#user-stories) - [Scenario 1](#scenario-1) - [Scenario 2](#scenario-2) - [Detailed Design](#detailed-design) - [Plugin Inputs Contract Changes](#plugin-inputs-contract-changes) - [Using Plugin Inputs for CSI Plugin](#using-plugin-inputs-for-csi-plugin) - [Annotations overrides on PVC for CSI Plugin](#annotations-overrides-on-pvc-for-csi-plugin) - [Using Plugin Inputs for Other Plugins](#using-plugin-inputs-for-other-plugins) - [Alternatives Considered](#alternatives-considered) - [Security Considerations](#security-considerations) - [Compatibility](#compatibility) - [Implementation](#implementation) - [Open Issues](#open-issues) ## Abstract Currently the Velero CSI plugin chooses the VolumeSnapshotClass in the cluster that has the same driver name and also has the velero.io/csi-volumesnapshot-class label set on it. This global selection is not sufficient for many use cases. This proposal is to add support for multiple VolumeSnapshotClasses in CSI Plugin where the user can specify the VolumeSnapshotClass to use for a particular driver and backup. ## Background The Velero CSI plugin chooses the VolumeSnapshotClass in the cluster that has the same driver name and also has the velero.io/csi-volumesnapshot-class label set on it. This global selection is not sufficient for many use cases. For example, if a cluster has multiple VolumeSnapshotClasses for the same driver, the user may want to use a VolumeSnapshotClass that is different from the default one. The user might also have different schedules set up for backing up different parts of the cluster and might wish to use different VolumeSnapshotClasses for each of these backups. ## Goals - Allow the user to specify the VolumeSnapshotClass to use for a particular driver and backup. ## Non Goals - Deprecating existing VSC selection behaviour. (The current behaviour will remain the default behaviour if the user does not specify the VolumeSnapshotClass to use for a particular driver and backup.) ## User Stories ### Scenario 1 - Consider Alice is a cluster admin and has a cluster with multiple VolumeSnapshotClasses for the same driver. Each VSC stores the snapshots taken in different ResourceGroup(Azure equivalent). - Alice has configured multiple scheduled backups each covering a different set of namespaces, representing different apps owned by different teams. - Alice wants to use a different VolumeSnapshotClass for each backup such that each snapshot goes in it's respective ResourceGroup to simply management of snapshots(COGS, RBAC etc). - In current velero, Alice can't achieve this as the CSI plugin will use the default VolumeSnapshotClass for the driver and all snapshots will go in the same ResourceGroup. - Proposed design will allow Alice to achieve this by specifying the VolumeSnapshotClass to use for a particular driver and backup/schedule. ## Scenario 2 - Bob is a cluster admin has PVCs storing different types of data. - Most of the PVCs are used for storing non-sensitive application data. But certain PVCs store critical financial data. - For such PVCs Bob wants to use a VolumeSnapshotClass with certain encryption related parameters set. - In current velero, Bob can't achieve this as the CSI plugin will use the default VolumeSnapshotClass for the driver and all snapshots will be taken using the same VolumeSnapshotClass. - Proposed design will allow Bob to achieve this by overriding the VolumeSnapshotClass to use for a particular driver and backup/schedule using annotations on those specific PVCs. ## Detailed Design ### Staged Approach: ### Stage 1 Approach #### Through Annotations 1. **Support VolumeSnapshotClass selection at backup/schedule level** The user can annotate the backup/ schedule with driver and VolumeSnapshotClass name. The CSI plugin will use the VolumeSnapshotClass specified in the annotation. If the annotation is not present, the CSI plugin will use the default VolumeSnapshotClass for the driver. *example annotation on backup/schedule:* ```yaml apiVersion: velero.io/v1 kind: Backup metadata: name: backup-1 annotations: velero.io/csi-volumesnapshot-class_csi.cloud.disk.driver: csi-diskdriver-snapclass velero.io/csi-volumesnapshot-class_csi.cloud.file.driver: csi-filedriver-snapclass velero.io/csi-volumesnapshot-class_: csi-snapclass ``` To query the annotations on a backup: "velero.io/csi-volumesnapshot-class_'driver name'" - where driver names comes from the PVC's driver. 2. **Support VolumeSnapshotClass selection at PVC level** The user can annotate the PVCs with driver and VolumeSnapshotClass name. The CSI plugin will use the VolumeSnapshotClass specified in the annotation. If the annotation is not present, the CSI plugin will use the default VolumeSnapshotClass for the driver. If the VolumeSnapshotClass provided is of a different driver, the CSI plugin will use the default VolumeSnapshotClass for the driver. *example annotation on PVC:* ```yaml apiVersion: v1 kind: PersistentVolumeClaim metadata: name: pvc-1 annotations: velero.io/csi-volumesnapshot-class: csi-diskdriver-snapclass ``` Consider this as a override option in conjunction to part 1. **Note**: The user has to annotate the PVCs or backups with the VolumeSnapshotClass to use for each driver. This is not ideal for the user experience. - **Mitigation**: We can extend Velero CLI to also annotate backups/schedules with the VolumeSnapshotClass to use for each driver. This will make it easier for the user to annotate the backups/schedules. This mitigation is not for the PVCs though, since PVCs is anyways a specific use case. Similar to : " kubectl run --image myimage --annotations="foo=bar" --annotations="another=one" mypod" We can add support for - velero backup create my-backup --annotations "velero.io/csi:csi.cloud.disk.driver=csi-diskdriver-snapclass" ### Stage 2 Approach The above annotations route is to get started and for initial design closure/ implementation, north star is to either introduce CSI specific fields (considering that CSI might be a very core part of velero going forward) in the backup/restore CR OR leverage the pluginInputs field as being tracked in: https://github.com/vmware-tanzu/velero/pull/5981 Refer section Alternatives 2. **Through generic property bag in the velero contracts**: in the design doc for more details on the pluginInputs field. ## Alternatives Considered 1. **Through CSI Specific Fields in Velero contracts** **Considerations** - Since CSI snapshotting is done through the plugin, we don't intend to bloat up the Backup Spec with CSI specific fields. - But considering that CSI Snapshotting is the way forward, we can debate if we should add a CSI section to the Backup Spec. **Approach**: Similar to VolumeSnapshotLocation param in the Backup Spec, we can add a VolumeSnapshotClass param in the Backup Spec. This will allow the user to specify the VolumeSnapshotClass to use for the backup. The CSI plugin will use the VolumeSnapshotClass specified in the Backup Spec. If the VolumeSnapshotClass is not specified, the CSI plugin will use the default VolumeSnapshotClass for the driver. *example of VolumeSnapshotClass param in the Backup Spec:* ```yaml apiVersion: velero.io/v1 kind: Backup metadata: name: backup-1 spec: csiParameters: volumeSnapshotClasses: driver: csi.cloud.disk.driver snapClass: csi-diskdriver-snapclass timeout: 10m ``` 1. **Through changes in velero contracts** 1. **Through configmap references.** Currently even the storageclass mapping plugin expects the user to create a configmap which is used globally, and fetched through labels. This behaviour has same issue as the VolumeSnapshotClass selection. We can introduce a field in the velero contracts which allow passing configmap references for each plugin. And then the plugin can honour the configmap passed in as reference. The configmap can be used to pass the VolumeSnapshotClass to use for the backup, and also other parameters to tweak. This can help in making plugins more flexible while not depending on global behaviour. *example of configmap reference in the velero contracts:* ```yaml apiVersion: velero.io/v1 kind: Backup metadata: name: backup-1 spec: configmapRefs: - name: csi-volumesnapshotclass-configmap - namespace: velero - plugin: velero.io/csi ``` 2. **Through generic property bag in the velero contracts**: We can introduce a field in the velero contracts which allow passing a generic property bag for each plugin. And then the plugin can honour the property bag passed in. *example of property bag in the velero contracts:* ```yaml apiVersion: velero.io/v1 kind: Backup metadata: name: backup-1 spec: pluginInputs: - name: velero.io/csi - properties: - key: csi.cloud.disk.driver - value: csi-diskdriver-snapclass - key: csi.cloud.file.driver - value: csi-filedriver-snapclass ``` **Note**: Both these approaches can also be used to tweak other parameters such as CSI Snapshotting Timeout/intervals. And further can be used by other plugins. ## Security Considerations No security impact. ## Compatibility Existing behaviour of csi plugin will be retained where it fetches the VolumeSnapshotClass through the label. This will be the default behaviour if the user does not specify the VolumeSnapshotClass. ## Implementation TBD based on closure of high level design proposals. ## Open Issues NA ================================================ FILE: design/Implemented/multiple-label-selectors_design.md ================================================ # Ensure support for backing up resources based on multiple labels ## Abstract As of today Velero supports filtering of resources based on single label selector per backup. It is desired that Velero support backing up of resources based on multiple labels (OR logic). **Note:** This solution is required because Kubernetes label selectors only allow AND logic of labels. ## Background Currently, Velero's Backup/Restore API has a spec field `LabelSelector` which helps in filtering of resources based on a **single** label value per backup/restore request. For instance, if the user specifies the `Backup.Spec.LabelSelector` as `data-protection-app: true`, Velero will grab all the resources that possess this label and perform the backup operation on them. The `LabelSelector` field does not accept more than one labels, and thus if the user want to take backup for resources consisting of a label from a set of labels (label1 OR label2 OR label3) then the user needs to create multiple backups per label rule. It would be really useful if Velero Backup API could respect a set of labels (OR Rule) for a single backup request. Related Issue: https://github.com/vmware-tanzu/velero/issues/1508 ## Goals - Enable support for backing up resources based on multiple labels (OR Logic) in a single backup config. - Enable support for restoring resources based on multiple labels (OR Logic) in a single restore config. ## Use Case/Scenario Let's say as a Velero user you want to take a backup of secrets, but all these secrets do not have one single consistent label on them. We want to take backup of secrets having any one label in `app=gdpr`, `app=wpa` and `app=ccpa`. Here we would have to create 3 instances of backup for each label rule. This can become cumbersome at scale. ## High-Level Design ### Addition of `OrLabelSelectors` spec to Velero Backup/Restore API For Velero to back up resources if they consist of any one label from a set of labels, we would like to add a new spec field `OrLabelSelectors` which would enable user to specify them. The Velero backup would somewhat look like: ``` apiVersion: velero.io/v1 kind: Backup metadata: name: backup-101 namespace: openshift-adp spec: includedNamespaces: - test storageLocation: velero-sample-1 ttl: 720h0m0s orLabelSelectors: - matchLabels: app=gdpr - matchLabels: app=wpa - matchLabels: app=ccpa ``` **Note:** This approach will **not** be changing any current behavior related to Backup API spec `LabelSelector`. Rather we propose that the label in `LabelSelector` spec and labels in `OrLabelSelectors` should be treated as different Velero functionalities. Both these fields will be treated as separate Velero Backup API specs. If `LabelSelector` (singular) is present then just match that label. And if `OrLabelSelectors` is present then match to any label in the set specified by the user. For backup case, if both the `LabelSelector` and `OrLabelSelectors` are specified (we do not anticipate this as a real world use-case) then the `OrLabelSelectors` will take precedence, `LabelSelector` will only be used to filter only when `OrLabelSelectors` is not specified by the user. This helps to keep both spec behaviour independent and not confuse the users. This way we preserve the existing Velero behaviour and implement the new functionality in a much cleaner way. For instance, let's take a look the following cases: 1. Only `LabelSelector` specified: Velero will create a backup with resources matching label `app=protect-db` ``` apiVersion: velero.io/v1 kind: Backup metadata: name: backup-101 namespace: openshift-adp spec: includedNamespaces: - test storageLocation: velero-sample-1 ttl: 720h0m0s labelSelector: - matchLabels: app=gdpr ``` 2. Only `OrLabelSelectors` specified: Velero will create a backup with resources matching any label from set `{app=gdpr, app=wpa, app=ccpa}` ``` apiVersion: velero.io/v1 kind: Backup metadata: name: backup-101 namespace: openshift-adp spec: includedNamespaces: - test storageLocation: velero-sample-1 ttl: 720h0m0s orLabelSelectors: - matchLabels: app=gdpr - matchLabels: app=wpa - matchLabels: app=ccpa ``` Similar implementation will be done for the Restore API as well. ## Detailed Design With the Introduction of `OrLabelSelectors` the BackupSpec and RestoreSpec will look like: BackupSpec: ``` type BackupSpec struct { [...] // OrLabelSelectors is a set of []metav1.LabelSelector to filter with // when adding individual objects to the backup. Resources matching any one // label from the set of labels will be added to the backup. If empty // or nil, all objects are included. Optional. // +optional OrLabelSelectors []\*metav1.LabelSelector [...] } ``` RestoreSpec: ``` type RestoreSpec struct { [...] // OrLabelSelectors is a set of []metav1.LabelSelector to filter with // when restoring objects from the backup. Resources matching any one // label from the set of labels will be restored from the backup. If empty // or nil, all objects are included from the backup. Optional. // +optional OrLabelSelectors []\*metav1.LabelSelector [...] } ``` The logic to collect resources to be backed up for a particular backup will be updated in the `backup/item_collector.go` around [here](https://github.com/vmware-tanzu/velero/blob/574baeb3c920f97b47985ec3957debdc70bcd5f8/pkg/backup/item_collector.go#L294). And for filtering the resources to be restored, the changes will go [here](https://github.com/vmware-tanzu/velero/blob/d1063bda7e513150fd9ae09c3c3c8b1115cb1965/pkg/restore/restore.go#L1769) **Note:** - This feature will not be exposed via Velero CLI. ================================================ FILE: design/Implemented/node-agent-affinity.md ================================================ # Node-agent Load Affinity Design ## Glossary & Abbreviation **Velero Generic Data Path (VGDP)**: VGDP is the collective modules that is introduced in [Unified Repository design][1]. Velero uses these modules to finish data transfer for various purposes (i.e., PodVolume backup/restore, Volume Snapshot Data Movement). VGDP modules include uploaders and the backup repository. **Exposer**: Exposer is a module that is introduced in [Volume Snapshot Data Movement Design][2]. Velero uses this module to expose the volume snapshots to Velero node-agent pods or node-agent associated pods so as to complete the data movement from the snapshots. ## Background Velero node-agent is a daemonset hosting controllers and VGDP modules to complete the concrete work of backups/restores, i.e., PodVolume backup/restore, Volume Snapshot Data Movement backup/restore. Specifically, node-agent runs DataUpload controllers to watch DataUpload CRs for Volume Snapshot Data Movement backups, so there is one controller instance in each node. One controller instance takes a DataUpload CR and then launches a VGDP instance, which initializes a uploader instance and the backup repository connection, to finish the data transfer. The VGDP instance runs inside a node-agent pod or in a pod associated to the node-agent pod in the same node. Varying from the data size, data complexity, resource availability, VGDP may take a long time and remarkable resources (CPU, memory, network bandwidth, etc.). Technically, VGDP instances are able to run in any node that allows pod schedule. On the other hand, users may want to constrain the nodes where VGDP instances run for various reasons, below are some examples: - Prevent VGDP instances from running in specific nodes because users have more critical workloads in the nodes - Constrain VGDP instances to run in specific nodes because these nodes have more resources than others - Constrain VGDP instances to run in specific nodes because the storage allows volume/snapshot provisions in these nodes only Therefore, in order to improve the compatibility, it is worthy to configure the affinity of VGDP to nodes, especially for backups for which VGDP instances run frequently and centrally. ## Goals - Define the behaviors of node affinity of VGDP instances in node-agent for volume snapshot data movement backups - Create a mechanism for users to specify the node affinity of VGDP instances for volume snapshot data movement backups ## Non-Goals - It is also beneficial to support VGDP instances affinity for PodVolume backup/restore, however, it is not possible since VGDP instances for PodVolume backup/restore should always run in the node where the source/target pods are created. - It is also beneficial to support VGDP instances affinity for data movement restores, however, it is not possible in some cases. For example, when the `volumeBindingMode` in the StorageClass is `WaitForFirstConsumer`, the restore volume must be mounted in the node where the target pod is scheduled, so the VGDP instance must run in the same node. On the other hand, considering the fact that restores may not frequently and centrally run, we will not support data movement restores. - As elaborated in the [Volume Snapshot Data Movement Design][2], the Exposer may take different ways to expose snapshots, i.e., through backup pods (this is the only way supported at present). The implementation section below only considers this approach currently, if a new expose method is introduced in future, the definition of the affinity configurations and behaviors should still work, but we may need a new implementation. ## Solution We will use the ConfigMap specified by `velero node-agent` CLI's parameter `--node-agent-configmap` to host the node affinity configurations. This configMap is not created by Velero, users should create it manually on demand. The configMap should be in the same namespace where Velero is installed. If multiple Velero instances are installed in different namespaces, there should be one configMap in each namespace which applies to node-agent in that namespace only. Node-agent server checks these configurations at startup time and use it to initiate the related VGDP modules. Therefore, users could edit this configMap any time, but in order to make the changes effective, node-agent server needs to be restarted. Inside the ConfigMap we will add one new kind of configuration as the data in the configMap, the name is ```loadAffinity```. Users may want to set different LoadAffinity configurations according to different conditions (i.e., for different storages represented by StorageClass, CSI driver, etc.), so we define ```loadAffinity``` as an array. This is for extensibility consideration, at present, we don't implement multiple configurations support, so if there are multiple configurations, we always take the first one in the array. The data structure is as below: ```go type Configs struct { // LoadConcurrency is the config for load concurrency per node. LoadConcurrency *LoadConcurrency `json:"loadConcurrency,omitempty"` // LoadAffinity is the config for data path load affinity. LoadAffinity []*LoadAffinity `json:"loadAffinity,omitempty"` } type LoadAffinity struct { // NodeSelector specifies the label selector to match nodes NodeSelector metav1.LabelSelector `json:"nodeSelector"` } ``` ### Affinity Affinity configuration means allowing VGDP instances running in the nodes specified. There are two ways to define it: - It could be defined by `MatchLabels` of `metav1.LabelSelector`. The labels defined in `MatchLabels` means a `LabelSelectorOpIn` operation by default, so in the current context, they will be treated as affinity rules. - It could be defined by `MatchExpressions` of `metav1.LabelSelector`. The labels are defined in `Key` and `Values` of `MatchExpressions` and the `Operator` should be defined as `LabelSelectorOpIn` or `LabelSelectorOpExists`. ### Anti-affinity Anti-affinity configuration means preventing VGDP instances running in the nodes specified. Below is the way to define it: - It could be defined by `MatchExpressions` of `metav1.LabelSelector`. The labels are defined in `Key` and `Values` of `MatchExpressions` and the `Operator` should be defined as `LabelSelectorOpNotIn` or `LabelSelectorOpDoesNotExist`. ### Sample A sample of the ConfigMap is as below: ```json { "loadAffinity": [ { "nodeSelector": { "matchLabels": { "beta.kubernetes.io/instance-type": "Standard_B4ms" }, "matchExpressions": [ { "key": "kubernetes.io/hostname", "values": [ "node-1", "node-2", "node-3" ], "operator": "In" }, { "key": "xxx/critial-workload", "operator": "DoesNotExist" } ] } } ] } ``` This sample showcases two affinity configurations: - matchLabels: VGDP instances will run only in nodes with label key `beta.kubernetes.io/instance-type` and value `Standard_B4ms` - matchExpressions: VGDP instances will run in node `node1`, `node2` and `node3` (selected by `kubernetes.io/hostname` label) This sample showcases one anti-affinity configuration: - matchExpressions: VGDP instances will not run in nodes with label key `xxx/critial-workload` To create the configMap, users need to save something like the above sample to a json file and then run below command: ``` kubectl create cm -n velero --from-file= ``` ### Implementation As mentioned in the [Volume Snapshot Data Movement Design][2], the exposer decides where to launch the VGDP instances. At present, for volume snapshot data movement backups, the exposer creates backupPods and the VGDP instances will be initiated in the nodes where backupPods are scheduled. So the loadAffinity will be translated (from `metav1.LabelSelector` to `corev1.Affinity`) and set to the backupPods. It is possible that node-agent pods, as a daemonset, don't run in every worker nodes, users could fulfil this by specify `nodeSelector` or `nodeAffinity` to the node-agent daemonset spec. On the other hand, at present, VGDP instances must be assigned to nodes where node-agent pods are running. Therefore, if there is any node selection for node-agent pods, users must consider this into this load affinity configuration, so as to guarantee that VGDP instances are always assigned to nodes where node-agent pods are available. This is done by users, we don't inherit any node selection configuration from node-agent daemonset as we think daemonset scheduler works differently from plain pod scheduler, simply inheriting all the configurations may cause unexpected result of backupPod schedule. Otherwise, if a backupPod are scheduled to a node where node-agent pod is absent, the corresponding DataUpload CR will stay in `Accepted` phase until the prepare timeout (by default 30min). At present, as part of the expose operations, the exposer creates a volume, represented by backupPVC, from the snapshot. The backupPVC uses the same storageClass with the source volume. If the `volumeBindingMode` in the storageClass is `Immediate`, the volume is immediately allocated from the underlying storage without waiting for the backupPod. On the other hand, the loadAffinity is set to the backupPod's affinity. If the backupPod is scheduled to a node where the snapshot volume is not accessible, e.g., because of storage topologies, the backupPod won't get into Running state, concequently, the data movement won't complete. Once this problem happens, the backupPod stays in `Pending` phase, and the corresponding DataUpload CR stays in `Accepted` phase until the prepare timeout (by default 30min). Below is an example of the backupPod's status when the problem happens: ``` status: conditions: - lastProbeTime: null message: '0/2 nodes are available: 1 node(s) didn''t match Pod''s node affinity/selector, 1 node(s) had volume node affinity conflict. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling..' reason: Unschedulable status: "False" type: PodScheduled phase: Pending ``` On the other hand, the backupPod is deleted after the prepare timeout, so there is no way to tell the cause is one of the above problems or others. To help the troubleshooting, we can add some diagnostic mechanism to discover the status of the backupPod and node-agent in the same node before deleting it as a result of the prepare timeout. [1]: unified-repo-and-kopia-integration/unified-repo-and-kopia-integration.md [2]: volume-snapshot-data-movement/volume-snapshot-data-movement.md ================================================ FILE: design/Implemented/node-agent-concurrency.md ================================================ # Node-agent Concurrency Design ## Glossary & Abbreviation **Velero Generic Data Path (VGDP)**: VGDP is the collective of modules that is introduced in [Unified Repository design][1]. Velero uses these modules to finish data transfer for various purposes (i.e., PodVolume backup/restore, Volume Snapshot Data Movement). VGDP modules include uploaders and the backup repository. ## Background Velero node-agent is a daemonset hosting controllers and VGDP modules to complete the concrete work of backups/restores, i.e., PodVolume backup/restore, Volume Snapshot Data Movement backup/restore. For example, node-agent runs DataUpload controllers to watch DataUpload CRs for Volume Snapshot Data Movement backups, so there is one controller instance in each node. One controller instance takes a DataUpload CR and then launches a VGDP instance, which initializes a uploader instance and the backup repository connection, to finish the data transfer. The VGDP instance runs inside the node-agent pod or in a pod associated to the node-agent pod in the same node. Varying from the data size, data complexity, resource availability, VGDP may take a long time and remarkable resources (CPU, memory, network bandwidth, etc.). Technically, VGDP instances are able to run in concurrent regardless of the requesters. For example, a VGDP instance for a PodVolume backup could run in parallel with another VGDP instance for a DataUpload. Then the two VGDP instances share the same resources if they are running in the same node. Therefore, in order to gain the optimized performance with the limited resources, it is worthy to configure the concurrent number of VGDP per node. When the resources are sufficient in nodes, users can set a large concurrent number, so as to reduce the backup/restore time; otherwise, the concurrency should be reduced, otherwise, the backup/restore may encounter problems, i.e., time lagging, hang or OOM kill. ## Goals - Define the behaviors of concurrent VGDP instances in node-agent - Create a mechanism for users to specify the concurrent number of VGDP per node ## Non-Goals - VGDP instances from different nodes always run in concurrent since in most common cases the resources are isolated. For special cases that some resources are shared across nodes, there is no support at present - In practice, restores run in prioritized scenarios, e.g., disaster recovery. However, the current design doesn't consider this difference, a VGDP instance for a restore is blocked if it reaches to the limit of the concurrency, even though the ones block it are for backups. If users do meet some problems here, they should consider to stop the backups first - Sometimes, users wants to totally block backups/restores from running in a specific node, this is out of the scope the current design. To archive this, more modules need to be considered (i.e., expoers of data movers), simply blocking the VGDP (e.g., by setting its concurrent number to 0) doesn't work. E.g., for a fs backup, VGDP instance must run in the node the source pod is running in, if we simply block from VGDP instance, the PodVolumeBackup CR is still submitted but never processed. ## Solution We introduce a ConfigMap specified by `velero node-agent` CLI's parameter `--node-agent-configmap` for users to specify the node-agent related configurations. This configMap is not created by Velero, users should create it manually on demand. The configMap should be in the same namespace where Velero is installed. If multiple Velero instances are installed in different namespaces, there should be one configMap in each namespace which applies to node-agent in that namespace only. Node-agent server checks these configurations at startup time and use it to initiate the related VGDP modules. Therefore, users could edit this configMap any time, but in order to make the changes effective, node-agent server needs to be restarted. The ConfigMap may be used for other purpose of configuring node-agent in future, at present, there is only one kind of configuration as the data in the configMap, the name is ```loadConcurrency```. The data structure is as below: ```go type Configs struct { // LoadConcurrency is the config for load concurrency per node. LoadConcurrency *LoadConcurrency `json:"loadConcurrency,omitempty"` } type LoadConcurrency struct { // GlobalConfig specifies the concurrency number to all nodes for which per-node config is not specified GlobalConfig int `json:"globalConfig,omitempty"` // PerNodeConfig specifies the concurrency number to nodes matched by rules PerNodeConfig []RuledConfigs `json:"perNodeConfig,omitempty"` } type RuledConfigs struct { // NodeSelector specifies the label selector to match nodes NodeSelector metav1.LabelSelector `json:"nodeSelector"` // Number specifies the number value associated to the matched nodes Number int `json:"number"` } ``` ### Global concurrent number We allow users to specify a concurrent number that will be applied to all nodes if the per-node number is not specified. This number is set through ```globalConfig```. The number starts from 1 which means there is no concurrency, only one instance of VGDP is allowed. There is no roof limit. If this number is not specified or not valid, a hard-coded default value will be used, the value is set to 1. ### Per-node concurrent number We allow users to specify different concurrent number per node, for example, users can set 3 concurrent instances in Node-1, 2 instances in Node-2 and 1 instance in Node-3. This is for below considerations: - The resources may be different among nodes. Then users could specify smaller concurrent number for nodes with less resources while larger number for the ones with more resources - Help users to isolate critical environments. Users may run some critical workloads in some specified nodes, since VGDP instances may take large resource consumption, users may want to run less number of instances in the nodes with critical workloads The range of Per-node concurrent number is the same with Global concurrent number. Per-node concurrent number is preferable to Global concurrent number, so it will overwrite the Global concurrent number for that node. Per-node concurrent number is implemented through ```perNodeConfig``` field. ```perNodeConfig``` is a list of ```RuledConfigs``` each item of which matches one or more nodes by label selectors and specify the concurrent number for the matched nodes. This means, the nodes are identified by labels. For example, the ```perNodeConfig`` could have below elements: ``` "nodeSelector: kubernetes.io/hostname=node1; number: 3" "nodeSelector: beta.kubernetes.io/instance-type=Standard_B4ms; number: 5" ``` The first element means the node with host name ```node1``` gets the Per-node concurrent number of 3. The second element means all the nodes with label ```beta.kubernetes.io/instance-type``` of value ```Standard_B4ms``` get the Per-node concurrent number of 5. At least one node is expected to have a label with the specified ```RuledConfigs``` element (rule). If no node is with this label, the Per-node rule makes no effect. If one node falls into more than one rules, e.g., if node1 also has the label ```beta.kubernetes.io/instance-type=Standard_B4ms```, the smallest number (3) will be used. ### Sample A sample of the ConfigMap is as below: ```json { "loadConcurrency": { "globalConfig": 2, "perNodeConfig": [ { "nodeSelector": { "matchLabels": { "kubernetes.io/hostname": "node1" } }, "number": 3 }, { "nodeSelector": { "matchLabels": { "beta.kubernetes.io/instance-type": "Standard_B4ms" } }, "number": 5 } ] } } ``` To create the configMap, users need to save something like the above sample to a json file and then run below command: ``` kubectl create cm -n velero --from-file= ``` ### Global data path manager As for the code implementation, data path manager is to maintain the total number of the running VGDP instances and ensure the limit is not excceeded. At present, there is one data path manager instance per controller, as a result, the concurrent numbers are calculated separately for each controller. This doesn't help to limit the concurrency among different requesters. Therefore, we need to create one global data path manager instance server-wide, and pass it to different controllers. The instance will be created at node-agent server startup. The concurrent number is required to initiate a data path manager, the number comes from either Per-node concurrent number or Global concurrent number. Below are some prototypes related to data path manager: ```go func NewManager(cocurrentNum int) *Manager func (m *Manager) CreateFileSystemBR(jobName string, requestorType string, ctx context.Context, client client.Client, namespace string, callbacks Callbacks, log logrus.FieldLogger) (AsyncBR, error) ``` [1]: Implemented/unified-repo-and-kopia-integration/unified-repo-and-kopia-integration.md ================================================ FILE: design/Implemented/node-agent-load-soothing.md ================================================ # Node-agent Load Soothing Design ## Glossary & Abbreviation **Velero Generic Data Path (VGDP)**: VGDP is the collective of modules that is introduced in [Unified Repository design][1]. Velero uses these modules to finish data transfer for various purposes (i.e., PodVolume backup/restore, Volume Snapshot Data Movement). VGDP modules include uploaders and the backup repository. ## Background As mentioned in [node-agent Concurrency design][2], [CSI Snapshot Data Movement design][3], [VGDP Micro Service design][4] and [VGDP Micro Service for fs-backup design][5], all data movement activities for CSI snapshot data movement backups/restores and fs-backup respect the `loadConcurrency` settings configured in the `node-agent-configmap`. Once the number of existing loads exceeds the corresponding `loadConcurrency` setting, the loads will be throttled and some loads will be held until VGDP quotas are available. However, this throttling only happens after the data mover pod is started and gets to `running`. As a result, when there are large number of concurrent volume backups, there may be many data mover pods get created but the VGDP instances inside them are actually on hold because of the VGDP throttling. This could cause below problems: - In some environments, there is a pod limit in each node of the cluster or a pod limit throughout the cluster, too many of the inactive data mover pods may block other pods from running - In some environments, the system disk for each node of the cluster is limited, while pods also occupy system disk space, etc., many of the inactive data mover pods also take unnecessary space from system disk and cause other critical pods evicted - For CSI snapshot data movement backup, before creation of the data mover pod, the volume snapshot has also created, this means excessive number of snapshots may also be created and live for longer time since the VGDP won't start until the quota is available. However, in some environments, large number of snapshots is not allowed or may cause degradation of the storage peroformance On the other hand, the VGDP throttling mentioned in [node-agent Concurrency design][2] is an accurate controlling mechanism, that is, exactly the required number of data mover pods are throttled. Therefore, another mechanism is required to soothe the creation of the data mover pods and volume snapshots before the VGDP throttling. It doesn't need to accurately control these creations but should effectively reduce the excessive number of inactive data mover pods and volume snapshots. It is not practical to make an accurate control as it is almost impossible to predict which group of nodes a data mover pod is scheduled to, under the consideration of many complex factors, i.e., selected node, affinity, node OS, etc. ## Goals - Allow users to configure the expected number of loads pending on waiting for VGDP load concurrency quota - Create a soothing mechanism to prevent new loads from starting if the number of existing loads excceds the expected number ## Non-Goals - Accurately controlling the loads from initiation is not a goal ## Solution We introduce a new field `prepareQueueLength` in `loadConcurrency` of `node-agent-configmap` as the allowed number of loads that are under preparing (expose). Specifically, loads are in this situation after its CR is in `Accepted` and `Prepared` phase. The `prepareQueueLength` should be a positive number, negative numbers will be ignored. Once the value is set, the soothing mechanism takes effect, as the best effort, only the allowed number of CRs go into `Accepted` or `Prepared` phase, others will wait and stay as `New` state; and thereby only the allowed number of data mover pods, volume snapshots are created. Otherwise, node-agent works the same as the legacy behavior, CRs go to `Accepted` or `Prepared` state as soon as the controllers process them and data mover pods and volume snapshots are also created without any constraints. If users want to constrain the excessive number of pending data mover pods and volume snapshots, they could set a value by considering the VGDP load concurrency; otherwise, if they don't see constrains for pods or volume snapshots in their environment, they don't need to use this feature, in parallel preparing could also be beneficial for increasing the concurrency. Node-agent server checks this configuration at startup time and use it to initiate the related VGDP modules. Therefore, users could edit this configMap any time, but in order to make the changes effective, node-agent server needs to be restarted. The data structure is as below: ```go type LoadConcurrency struct { // GlobalConfig specifies the concurrency number to all nodes for which per-node config is not specified GlobalConfig int `json:"globalConfig,omitempty"` // PerNodeConfig specifies the concurrency number to nodes matched by rules PerNodeConfig []RuledConfigs `json:"perNodeConfig,omitempty"` // PrepareQueueLength specifies the max number of loads that are under expose PrepareQueueLength int `json:"prepareQueueLength,omitempty"` } ``` ### Sample A sample of the ConfigMap is as below: ```json { "loadConcurrency": { "globalConfig": 2, "perNodeConfig": [ { "nodeSelector": { "matchLabels": { "kubernetes.io/hostname": "node1" } }, "number": 3 }, { "nodeSelector": { "matchLabels": { "beta.kubernetes.io/instance-type": "Standard_B4ms" } }, "number": 5 } ], "prepareQueueLength": 2 } } ``` To create the configMap, users need to save something like the above sample to a json file and then run below command: ``` kubectl create cm -n velero --from-file= ``` ## Detailed Design Changes apply to the DataUpload Controller, DataDownload Controller, PodVolumeBackup Controller and PodVolumeRestore Controller, as below: 1. The soothe happens to data mover CRs (DataUpload, DataDownload, PodVolumeBackup or PodVolumeRestore) that are in `New` state 2. Before starting processing the CR, the corresponding controller counts the existing CRs under or pending for expose in the cluster, that is a total number of existing DataUpload, DataDownload, PodVolumeBackup and PodVolumeRestore that are in either `Accepted` or `Preparing` state 3. If the total number doesn't exceed the allowed number, the controller set the CR's phase to `Accepted` 4. Once the total number exceeds the allowed number, the controller gives up processing the CR and have it requeued later. The delay for the requeue is 5 seconds The count happens for all the controllers in all nodes, to prevent the checks drain out the API server, the count happens to controller client caches for those CRs. And the count result is also cached, so that the count only happens whenever necessary. Below shows how it judges the necessity: - When one or more CRs' phase change to `Accepted` - When one or more CRs' phase change from `Accepted` to one of the terminal phases - When one or more CRs' phase change from `Prepared` to one of the terminal phases - When one or more CRs' phase change from `Prepared` to `InProgress` Ideally, 2~3 in the above steps need to be synchornized among controllers in all nodes. However, this synchronization is not implemented, the consideration is as below: 1. It is impossible to accurately synchronize the count among controllers in different nodes, because the client cache is not coherrent among nodes. 2. It is possible to synchronize the count among controllers in the same node. However, it is too expensive to make this synchronization, because 2~3 are part of the expose workflow, the synchronization impacts the performance and stability of the existing workflow. 3. Even without the synchronization, the soothing mechanism still works eventually -- when the controllers see all the discharged loads (expected ones and over-discharged ones), they will stop creating new loads until the quota is available again. 4. Step 2~3 that need to be synchronized could complete very quickly. This is why we say this mechanism is not an accurate control. Or in another word, it is possible that more loads than the number of `prepareQueueLength` are discharged if controllers make the count and expose in the overlapped time (step 2~3). For example, when multiple controllers of the same type (DataUpload, DataDownload, PodVolumeBackup or PodVolumeRestore) from different nodes make the count: ``` max number of waiting loads = number defined by `prepareQueueLength` + number of nodes in cluster ``` As another example, when hybrid loads are running the count concurrently, e.g., mix of data mover backups, data mover restores, pod volume backups or pod volume restores, more loads may be discharged and the number depends on the number of concurrent hybrid loads. In either case, because step 2~3 is short in time, it is less likely to reach the theoretically worset result. [1]: unified-repo-and-kopia-integration/unified-repo-and-kopia-integration.md [2]: node-agent-concurrency.md [3]: volume-snapshot-data-movement/volume-snapshot-data-movement.md [4]: vgdp-micro-service/vgdp-micro-service.md [5]: vgdp-micro-service-for-fs-backup/vgdp-micro-service-for-fs-backup.md ================================================ FILE: design/Implemented/plugin-backup-and-restore-progress-design.md ================================================ # Progress reporting for backups and restores handled by volume snapshotters Users face difficulty in knowing the progress of backup/restore operations of volume snapshotters. This is very similar to the issues faced by users to know progress for restic backup/restore, like, estimation of operation, operation in-progress/hung etc. Each plugin might be providing a way to know the progress, but, it need not uniform across the plugins. Even though plugins provide the way to know the progress of backup operation, this information won't be available to user during restore time on the destination cluster. So, apart from the issues like progress, status of operation, volume snapshotters have unique problems like - not being uniform across plugins - not knowing the backup information during restore operation - need to be optional as few plugins may not have a way to provide the progress information This document proposes an approach for plugins to follow to provide backup/restore progress, which can be used by users to know the progress. ## Goals - Provide uniform way of visibility into backup/restore operations performed by volume snapshotters ## Non Goals - Plugin implementation for this approach ## Background (Omitted, see introduction) ## High-Level Design ### Progress of backup operation handled by volume snapshotter Progress will be updated by volume snapshotter in VolumePluginBackup CR which is specific to that backup operation. ### Progress of restore operation handled by volume snapshotter Progress will be updated by volume snapshotter in VolumePluginRestore CR which is specific to that restore operation. ## Detailed Design ### Approach 1 Existing `Snapshot` Go struct from `volume` package have most of the details related to backup operation performed by volumesnapshotters. This struct also gets backed up to backup location. But, this struct doesn't get synced on other clusters at regular intervals. It is currently synced only during restore operation, and velero CLI shows few of its contents. At a high level, in this approach, this struct will be converted to a CR by adding new fields (related to Progress tracking) to it, and gets rid of `volume.Snapshot` struct. Instead of backing up of Go struct, proposal is: to backup CRs to backup location, and sync them into other cluster by backupSyncController running in that cluster. #### VolumePluginBackup CR There is one addition to volume.SnapshotSpec, i.e., ProviderName to convert it to CR's spec. Below is the updated VolumePluginBackup CR's Spec: ``` type VolumePluginBackupSpec struct { // BackupName is the name of the Velero backup this snapshot // is associated with. BackupName string `json:"backupName"` // BackupUID is the UID of the Velero backup this snapshot // is associated with. BackupUID string `json:"backupUID"` // Location is the name of the VolumeSnapshotLocation where this snapshot is stored. Location string `json:"location"` // PersistentVolumeName is the Kubernetes name for the volume. PersistentVolumeName string `json:"persistentVolumeName"` // ProviderVolumeID is the provider's ID for the volume. ProviderVolumeID string `json:"providerVolumeID"` // Provider is the Provider field given in VolumeSnapshotLocation Provider string `json:"provider"` // VolumeType is the type of the disk/volume in the cloud provider // API. VolumeType string `json:"volumeType"` // VolumeAZ is the where the volume is provisioned // in the cloud provider. VolumeAZ string `json:"volumeAZ,omitempty"` // VolumeIOPS is the optional value of provisioned IOPS for the // disk/volume in the cloud provider API. VolumeIOPS *int64 `json:"volumeIOPS,omitempty"` } ``` Few fields (except first two) are added to volume.SnapshotStatus to convert it to CR's status. Below is the updated VolumePluginBackup CR's status: ``` type VolumePluginBackupStatus struct { // ProviderSnapshotID is the ID of the snapshot taken in the cloud // provider API of this volume. ProviderSnapshotID string `json:"providerSnapshotID,omitempty"` // Phase is the current state of the VolumeSnapshot. Phase SnapshotPhase `json:"phase,omitempty"` // PluginSpecific are a map of key-value pairs that plugin want to provide // to user to identify plugin properties related to this backup // +optional PluginSpecific map[string]string `json:"pluginSpecific,omitempty"` // Message is a message about the volume plugin's backup's status. // +optional Message string `json:"message,omitempty"` // StartTimestamp records the time a backup was started. // Separate from CreationTimestamp, since that value changes // on restores. // The server's time is used for StartTimestamps // +optional // +nullable StartTimestamp *metav1.Time `json:"startTimestamp,omitempty"` // CompletionTimestamp records the time a backup was completed. // Completion time is recorded even on failed backups. // Completion time is recorded before uploading the backup object. // The server's time is used for CompletionTimestamps // +optional // +nullable CompletionTimestamp *metav1.Time `json:"completionTimestamp,omitempty"` // Progress holds the total number of bytes of the volume and the current // number of backed up bytes. This can be used to display progress information // about the backup operation. // +optional Progress VolumeOperationProgress `json:"progress,omitempty"` } type VolumeOperationProgress struct { TotalBytes int64 BytesDone int64 } type VolumePluginBackup struct { metav1.TypeMeta `json:",inline"` // +optional metav1.ObjectMeta `json:"metadata,omitempty"` // +optional Spec VolumePluginBackupSpec `json:"spec,omitempty"` // +optional Status VolumePluginBackupStatus `json:"status,omitempty"` } ``` For every backup operation of volume, Velero creates VolumePluginBackup CR before calling volumesnapshotter's CreateSnapshot API. In order to know the CR created for the particular backup of a volume, Velero adds following labels to CR: - `velero.io/backup-name` with value as Backup Name, and, - `velero.io/pv-name` with value as volume that is undergoing backup Backup name being unique won't cause issues like duplicates in identifying the CR. Labels will be set with the value returned from `GetValidName` function. (https://github.com/vmware-tanzu/velero/blob/main/pkg/label/label.go#L35). If Plugin supports showing progress of the operation it is performing, it does following: - finds the VolumePluginBackup CR related to this backup operation by using `tags` passed in CreateSnapshot call - updates the CR with the progress regularly. After return from `CreateSnapshot` in `takePVSnapshot`, currently Velero adds `volume.Snapshot` to `backupRequest`. Instead of this, CR will be added to `backupRequest`. During persistBackup call, this CR also will be backed up to backup location. In backupSyncController, it checks for any VolumePluginBackup CRs that need to be synced from backup location, and syncs them to cluster if needed. VolumePluginBackup will be useful as long as backed up data is available at backup location. When the Backup is deleted either by manually or due to expiry, VolumePluginBackup also can be deleted. `processRequest` of `backupDeletionController` will perform deletion of VolumePluginBackup before volumesnapshotter's DeleteSnapshot is called. #### Backward compatibility: Currently `volume.Snapshot` is backed up as `-volumesnapshots.json.gz` file in the backup location. As the VolumePluginBackup CR is backed up instead of `volume.Snapshot`, to provide backward compatibility, CR will be backed as the same file i.e., `-volumesnapshots.json.gz` file in the backup location. For backward compatibility on restore side, consider below possible cases wrt Velero version on restore side and format of json.gz file at object location: - older version of Velero, older json.gz file (backupname-volumesnapshots.json.gz) - older version of Velero, newer json.gz file - newer version of Velero, older json.gz file - newer version of Velero, newer json.gz file First and last should be fine. For second case, decode in `GetBackupVolumeSnapshots` on the restore side should fill only required fields of older version and should work. For third case, after decode, metadata.name will be empty. `GetBackupVolumeSnapshots` decodes older json.gz into the CR which goes fine. It will be modified to return []VolumePluginBackupSpec, and the changes are done accordingly in its caller. If decode fails in second case during implementation, this CR need to be backed up to different file. And, for backward compatibility, newer code should check for old file existence, and follow older code if exists. If it doesn't exists, check for newer file and follow the newer code. `backupSyncController` on restore clusters gets the `-volumesnapshots.json.gz` object from backup location and decodes it to in-memory VolumePluginBackup CR. If its `metadata.name` is populated, controller creates CR. Otherwise, it will not create the CR on the cluster. It can be even considered to create CR on the cluster. #### VolumePluginRestore CR ``` // VolumePluginRestoreSpec is the specification for a VolumePluginRestore CR. type VolumePluginRestoreSpec struct { // SnapshotID is the identifier for the snapshot of the volume. // This will be used to relate with output in 'velero describe backup' SnapshotID string `json:"snapshotID"` // BackupName is the name of the Velero backup from which PV will be // created. BackupName string `json:"backupName"` // Provider is the Provider field given in VolumeSnapshotLocation Provider string `json:"provider"` // VolumeType is the type of the disk/volume in the cloud provider // API. VolumeType string `json:"volumeType"` // VolumeAZ is the where the volume is provisioned // in the cloud provider. VolumeAZ string `json:"volumeAZ,omitempty"` } // VolumePluginRestoreStatus is the current status of a VolumePluginRestore CR. type VolumePluginRestoreStatus struct { // Phase is the current state of the VolumePluginRestore. Phase string `json:"phase"` // VolumeID is the PV name to which restore done VolumeID string `json:"volumeID"` // Message is a message about the volume plugin's restore's status. // +optional Message string `json:"message,omitempty"` // StartTimestamp records the time a restore was started. // Separate from CreationTimestamp, since that value changes // on restores. // The server's time is used for StartTimestamps // +optional // +nullable StartTimestamp *metav1.Time `json:"startTimestamp,omitempty"` // CompletionTimestamp records the time a restore was completed. // Completion time is recorded even on failed restores. // The server's time is used for CompletionTimestamps // +optional // +nullable CompletionTimestamp *metav1.Time `json:"completionTimestamp,omitempty"` // Progress holds the total number of bytes of the snapshot and the current // number of restored bytes. This can be used to display progress information // about the restore operation. // +optional Progress VolumeOperationProgress `json:"progress,omitempty"` // PluginSpecific are a map of key-value pairs that plugin want to provide // to user to identify plugin properties related to this restore // +optional PluginSpecific map[string]string `json:"pluginSpecific,omitempty"` } type VolumePluginRestore struct { metav1.TypeMeta `json:",inline"` // +optional metav1.ObjectMeta `json:"metadata,omitempty"` // +optional Spec VolumePluginRestoreSpec `json:"spec,omitempty"` // +optional Status VolumePluginRestoreStatus `json:"status,omitempty"` } ``` For every restore operation, Velero creates VolumePluginRestore CR before calling volumesnapshotter's CreateVolumeFromSnapshot API. In order to know the CR created for the particular restore of a volume, Velero adds following labels to CR: - `velero.io/backup-name` with value as Backup Name, and, - `velero.io/snapshot-id` with value as snapshot id that need to be restored - `velero.io/provider` with value as `Provider` in `VolumeSnapshotLocation` Labels will be set with the value returned from `GetValidName` function. (https://github.com/vmware-tanzu/velero/blob/main/pkg/label/label.go#L35). Plugin will be able to identify CR by using snapshotID that it received as parameter of CreateVolumeFromSnapshot API, and plugin's Provider name. It updates the progress of restore operation regularly if plugin supports feature of showing progress. Velero deletes VolumePluginRestore CR when it handles deletion of Restore CR. ### Approach 2 This approach is different to approach 1 only with respect to Backup. #### VolumePluginBackup CR ``` // VolumePluginBackupSpec is the specification for a VolumePluginBackup CR. type VolumePluginBackupSpec struct { // Volume is the PV name to be backed up. Volume string `json:"volume"` // Backup name Backup string `json:"backup"` // Provider is the Provider field given in VolumeSnapshotLocation Provider string `json:"provider"` } // VolumePluginBackupStatus is the current status of a VolumePluginBackup CR. type VolumePluginBackupStatus struct { // Phase is the current state of the VolumePluginBackup. Phase string `json:"phase"` // SnapshotID is the identifier for the snapshot of the volume. // This will be used to relate with output in 'velero describe backup' SnapshotID string `json:"snapshotID"` // Message is a message about the volume plugin's backup's status. // +optional Message string `json:"message,omitempty"` // StartTimestamp records the time a backup was started. // Separate from CreationTimestamp, since that value changes // on restores. // The server's time is used for StartTimestamps // +optional // +nullable StartTimestamp *metav1.Time `json:"startTimestamp,omitempty"` // CompletionTimestamp records the time a backup was completed. // Completion time is recorded even on failed backups. // Completion time is recorded before uploading the backup object. // The server's time is used for CompletionTimestamps // +optional // +nullable CompletionTimestamp *metav1.Time `json:"completionTimestamp,omitempty"` // PluginSpecific are a map of key-value pairs that plugin want to provide // to user to identify plugin properties related to this backup // +optional PluginSpecific map[string]string `json:"pluginSpecific,omitempty"` // Progress holds the total number of bytes of the volume and the current // number of backed up bytes. This can be used to display progress information // about the backup operation. // +optional Progress VolumeOperationProgress `json:"progress,omitempty"` } type VolumeOperationProgress struct { TotalBytes int64 BytesDone int64 } type VolumePluginBackup struct { metav1.TypeMeta `json:",inline"` // +optional metav1.ObjectMeta `json:"metadata,omitempty"` // +optional Spec VolumePluginBackupSpec `json:"spec,omitempty"` // +optional Status VolumePluginBackupStatus `json:"status,omitempty"` } ``` For every backup operation of volume, volume snapshotter creates VolumePluginBackup CR in Velero namespace. It keep updating the progress of operation along with other details like Volume name, Backup Name, SnapshotID etc as mentioned in the CR. In order to know the CR created for the particular backup of a volume, volume snapshotters adds following labels to CR: - `velero.io/backup-name` with value as Backup Name, and, - `velero.io/volume-name` with value as volume that is undergoing backup Backup name being unique won't cause issues like duplicates in identifying the CR. Plugin need to sanitize the value that can be set for above labels. Label need to be set with the value returned from `GetValidName` function. (https://github.com/vmware-tanzu/velero/blob/main/pkg/label/label.go#L35). Though no restrictions are required on the name of CR, as a general practice, volume snapshotter can name this CR with the value same as return value of CreateSnapshot. After return from `CreateSnapshot` in `takePVSnapshot`, if VolumePluginBackup CR exists for particular backup of the volume, velero adds this CR to `backupRequest`. During persistBackup call, this CR also will be backed up to backup location. In backupSyncController, it checks for any VolumePluginBackup CRs that need to be synced from backup location, and syncs them to cluster if needed. `processRequest` of `backupDeletionController` will perform deletion of VolumePluginBackup before volumesnapshotter's DeleteSnapshot is called. Another alternative is: Deletion of `VolumePluginBackup` CR can be delegated to plugin. Plugin can perform deletion of VolumePluginBackup using the `snapshotID` passed in volumesnapshotter's DeleteSnapshot request. ### 'core' Velero client/server required changes - Creation of the VolumePluginBackup/VolumePluginRestore CRDs at installation time - Persistence of VolumePluginBackup CRs towards the end of the backup operation - As part of backup synchronization, VolumePluginBackup CRs related to the backup will be synced. - Deletion of VolumePluginBackup when volumeshapshotter's DeleteSnapshot is called - Deletion of VolumePluginRestore as part of handling deletion of Restore CR - In case of approach 1, - converting `volume.Snapshot` struct as CR and its related changes - creation of VolumePlugin(Backup|Restore) CRs before calling volumesnapshotter's API - `GetBackupVolumeSnapshots` and its callers related changes for change in return type from []volume.Snapshot to []VolumePluginBackupSpec. ### Velero CLI required changes In 'velero describe' CLI, required CRs will be fetched from API server and its contents like backupName, PVName (if changed due to label size limitation), size of PV snapshot will be shown in the output. ### API Upgrade When CRs gets upgraded, velero can support older API versions also (till they get deprecated) to identify the CRs that need to be persisted to backup location. However, it can provide preference over latest supported API. If new fields are added without changing API version, it won't cause any problem as these resources are intended to provide information, and, there is no reconciliation on these resources. ### Compatibility of latest plugin with older version of Velero Plugin that supports this CR should handle the situation gracefully when CRDs are not installed. It can handle the errors occurred during creation/updation of the CRs. ## Limitations: Non K8s native plugins will not be able to implement this as they can not create the CRs. ## Open Questions ## Alternatives Considered ### Add another method to VolumeSnapshotter interface Above proposed approach have limitation that plugin need to be K8s native in order to create, update CRs. Instead, a new method for 'Progress' will be added to interface. Velero server regularly polls this 'Progress' method and updates VolumePluginBackup CR on behalf of plugin. But, this involves good amount of changes and needs a way for backward compatibility. As volume plugins are mostly K8s native, its fine to go ahead with current limitation. ### Update Backup CR Instead of creating new CRs, plugins can directly update the status of Backup CR. But, this deviates from current approach of having separate CRs like PodVolumeBackup/PodVolumeRestore to know operations progress. ### Restricting on name rather than using labels Instead of using labels to identify the CR related to particular backup on a volume, restrictions can be placed on the name of VolumePluginBackup CR to be same as the value returned from CreateSnapshot. But, this can cause issue when volume snapshotter just crashed without returning snapshot id to velero. ### Backing up VolumePluginBackup CR to different object If CR is backed up to different object other than `#backup-volumesnapshots.json.gz` in backup location, restore controller need to follow 'fall-back model'. It first need to check for new kind of object, and, if it doesn't exists, follow the old model. To avoid 'fall-back' model which prone to errors, VolumePluginBackup CR is backed to same location as that of `volume.Snapshot` location. ## Security Considerations Currently everything runs under the same `velero` service account so all plugins have broad access, which would include being able to modify CRs created by another plugin. ================================================ FILE: design/Implemented/plugin-versioning.md ================================================ # Plugin Versioning ## Abstract This proposal outlines an approach to support versioning of Velero's plugin APIs to enable changes to those APIs. It will allow for backwards compatible changes to be made, such as the addition of new plugin methods, but also backwards incompatible changes such as method removal or method signature changes. ## Background When changes are made to Velero’s plugin APIs, there is no mechanism for the Velero server to communicate the version of the API that is supported, or for plugins to communicate what version they implement. This means that any modification to a plugin API is a backwards incompatible change as it requires all plugins which implement the API to update and implement the new method. There are several components involved to use plugins within Velero. From the perspective of the core Velero codebase, all plugin kinds (e.g. `ObjectStore`, `BackupItemAction`) are defined by a single API interface and all interactions with plugins are managed by a plugin manager which provides an implementation of the plugin API interface for Velero to use. Velero communicates with plugins via gRPC. The core Velero project provides a framework (using the [go-plugin project](https://github.com/hashicorp/go-plugin)) for plugin authors to use to implement their plugins which manages the creation of gRPC servers and clients. Velero plugins import the Velero plugin library in order to use this framework. When a change is made to a plugin API, it needs to be made to the Go interface used by the Velero codebase, and also to the rpc service definition which is compiled to form part of the framework. As each plugin kind is defined by a single interface, when a plugin imports the latest version of the Velero framework, it will need to implement the new APIs in order to build and run successfully. If a plugin does not use the latest version of the framework, and is used with a newer version of Velero that expects the plugin to implement those methods, this will result in a runtime error as the plugin is incompatible. With this proposal, we aim to break this coupling and introduce plugin API versions. ## Scenarios to Support The following describes interactions between Velero and its plugins that will be supported with the implementation of this proposal. For the purposes of this list, we will refer to existing Velero and plugin versions as `v1` and all following versions as version `n`. Velero client communicating with plugins or plugin client calling other plugins: - Version `n` client will be able to communicate with Version `n` plugin - Version `n` client will be able to communicate with all previous versions of the plugin (Version `n-1` back to `v1`) Velero plugins importing Velero framework: - `v1` plugin built against Version `n` Velero framework - A plugin may choose to only implement a `v1` API, but it must be able to be built using Version `n` of the Velero framework ## Goals - Allow plugin APIs to change without requiring all plugins to implement the latest changes (even if they upgrade the version of Velero that is imported) - Allow plugins to choose which plugin versions they support and enable them to support multiple versions - Support breaking changes in the plugin APIs such as method removal or method signature changes - Establish a design process for modifying plugin APIs such as method addition and removal and signature changes - Establish a process for newer Velero clients to use older versions of a plugin API through adaptation ## Non Goals - Change how plugins are managed or added - Allow older plugin clients to communicate with new versions of plugins ## High-Level Design With each change to a plugin API, a new version of the plugin interface and the proto service definition will be created which describes the new plugin API. The plugin framework will be adapted to allow these new plugin versions to be registered. Plugins can opt to implement any or all versions of an API, however Velero will always attempt to use the latest version, and the plugin management will be modified to adapt earlier versions of a plugin to be compatible with the latest API where possible. Under the existing plugin framework, any new plugin version will be treated as a new plugin with a new kind. The plugin manager (which provides implementations of a plugin to Velero) will include an adapter layer which will manage the different versions and provide the adaptation for versions which do not implement the latest version of the plugin API. Providing an adaptation layer enables Velero and other plugin clients to use an older version of a plugin if it can be safely adapted. As the plugins will be able to introduce backwards incompatible changes, it will _not_ be possible for older version of Velero to use plugins which only support the latest versions of the plugin APIs. Although adding new rpc methods to a service is considered a backwards compatible change within gRPC, due to the way the proto definitions are compiled and included in the framework used by plugins, this will require every plugin to implement the new methods. Instead, we are opting to treat the addition of a method to an API as one requiring versioning. The addition of optional fields to existing structs which are used as parameters to or return values of API methods will not be considered as a change requiring versioning. These kinds of changes do not modify method signatures and have been safely made in the past with no impact on existing plugins. ## Detailed Design The following areas will need to be adapted to support plugin versioning. ### Plugin Interface Definitions To provide versioned plugins, any change to a plugin interface (method addition, removal, or signature change) will require a new versioned interface to be created. Currently, all plugin interface definitions reside in `pkg/plugin/velero` in a file corresponding to their plugin kind. These files will be rearranged to be grouped by kind and then versioned: `pkg/plugin/velero///`. The following are examples of how each change may be treated: #### Complete Interface Change If the entire `ObjectStore` interface is being changed such that no previous methods are being included, a file would be added to `pkg/plugin/velero/objectstore/v2/` and would contain the new interface definition: ``` type ObjectStore interface { // Only include new methods that the new API version will support NewMethod() // ... } ``` #### Method Addition If a method is being added to the `ObjectStore` API, a file would be added to `pkg/plugin/velero/objectstore/v2/` and may contain a new API definition as follows: ``` import "github.com/vmware-tanzu/velero/pkg/plugin/velero/objectstore/v1" type ObjectStore interface { // Import all the methods from the previous version of the API if they are to be included as is v1.ObjectStore // Provide definitions of any new methods NewMethod() ``` #### Method Removal If a method is being removed from the `ObjectStore` API, a file would be added to `pkg/plugin/velero/objectstore/v2/` and may contain a new API definition as follows: ``` type ObjectStore interface { // Methods which are required from the previous API version must be included, for example Init(config) PutObject(bucket, key, body) // ... // Methods which are to be removed are not included ``` #### Method Signature modification If a method signature in the `ObjectStore` API is being modified, a file would be added to `pkg/plugin/velero/objectstore/v2/` and may contain a new API definition as follows: ``` type ObjectStore interface { // Methods which are required from the previous API version must be included, for example Init(config) PutObject(bucket, key, body) // ... // Provide new definitions for methods which are being modified List(bucket, prefix, newParameter) } ``` ### Proto Service Definitions The proto service definitions of the plugins will also be versioned and arranged by their plugin kind. Currently, all the proto definitions reside under `pkg/plugin/proto` in a file corresponding to their plugin kind. These files will be rearranged to be grouped by kind and then versioned: `pkg/plugin/proto//`, except for the current v1 plugins. Those will remain in their current package/location for backwards compatibility. This will allow plugin images built with earlier versions of velero to work with the latest velero (for v1 plugins only). The go_package option will be added to all proto service definitions to allow the proto compilation script to place the generated go code for each plugin api version in the proper go package directory. It is not possible to import an existing proto service into a new one, so any methods will need to be duplicated across versions if they are required by the new version. The message definitions can be shared however, so these could be extracted from the service definition files and placed in a file that can be shared across all versions of the service. ### Plugin Framework To allow plugins to register which versions of the API they implement, the plugin framework will need to be adapted to accept new versions. Currently, the plugin manager stores a [`map[string]RestartableProcess`](https://github.com/vmware-tanzu/velero/blob/main/pkg/plugin/clientmgmt/manager.go#L69), where the string key is the binary name for the plugin process (e.g. "velero-plugin-for-aws"). Each `RestartableProcess` contains a [`map[kindAndName]interface{}`](https://github.com/vmware-tanzu/velero/blob/main/pkg/plugin/clientmgmt/restartable_process.go#L60) which represents each of the unique plugin implementations provided by that binary. [`kindAndName`](https://github.com/vmware-tanzu/velero/blob/main/pkg/plugin/clientmgmt/registry.go#L42) is a struct which combines the plugin kind (`ObjectStore`, `VolumeSnapshotter`) and the plugin name ("velero.io/aws", "velero.io/azure"). Each plugin version registration must be unique (to allow for multiple versions to be implemented within the same plugin binary). This will be achieved by adding a specific registration method for each version to the Server interface in the plugin framework. For example, if adding a V2 `RestoreItemAction` plugin, the Server interface would be modified to add the `RegisterRestoreItemActionV2` method. This would require [adding a new plugin Kind const](https://github.com/vmware-tanzu/velero/blob/main/pkg/plugin/framework/plugin_kinds.go#L28-L46) to represent the new plugin version, e.g. `PluginKindRestoreItemActionV2`. It also requires the creation of a new implementation of the go-plugin interface ([example](https://github.com/vmware-tanzu/velero/blob/main/pkg/plugin/framework/object_store.go)) to support that version and use the generated gRPC code from the proto definition (including a client and server implementation). The Server will also need to be adapted to recognize this new plugin Kind and to serve the new implementation. Existing plugin Kind consts and registration methods will be left unchanged and will correspond to the current version of the plugin APIs (assumed to be v1). ### Plugin Manager The plugin manager is responsible for managing the lifecycle of plugins. It provides an interface which is used by Velero to retrieve an instance of a plugin kind with a specific name (e.g. `ObjectStore` with the name "velero.io/aws"). The manager contains a registry of all available plugins which is populated during the main Velero server startup. When the plugin manager is requested to provide a particular plugin, it checks the registry for that plugin kind and name. If it is available in the registry, the manager retrieves a `RestartableProcess` for the plugin binary, creating it if it does not already exist. That `RestartableProcess` is then used by individual restartable implementations of a plugin kind (e.g. `restartableObjectStore`, `restartableVolumeSnapshotter`). As new plugin versions are added, the plugin manager will be modified to always retrieve the latest version of a plugin kind. This is to allow the remainder of the Velero codebase to assume that it will always interact with the latest version of a plugin. If the latest version of a plugin is not available, it will attempt to fall back to previous versions and use an implementation adapted to the latest version if available. It will be up to the author of new plugin versions to determine whether a previous version of a plugin can be adapted to work with the interface of the new version. For each plugin kind, a new `Restartable` struct will be introduced which will contain the plugin Kind and a function, `Get`, which will instantiate a restartable instance of that plugin kind and perform any adaptation required to make it compatible with the latest version. For example, `RestartableObjectStore` or `RestartableVolumeSnapshotter`. For each restartable plugin kind, a new function will be introduced which will return a slice of `Restartable` objects, sorted by version in descending order. The manager will iterate through the list of `Restartable`s and will check the registry for the given plugin kind and name. If the requested version is not found, it will skip and continue to iterate, attempting to fetch previous versions of the plugin kind. Once the requested version is found, the `Get` function will be called, returning the restartable implementation of the latest version of that plugin Kind. ``` type RestartableObjectStore struct { kind framework.PluginKind // Get returns a restartable ObjectStore for the given name and process, wrapping if necessary Get func(name string, restartableProcess RestartableProcess) v2.ObjectStore } func (m *manager) restartableObjectStores() []RestartableObjectStore { return []RestartableObjectStore{ { kind: framework.PluginKindObjectStoreV2, Get: newRestartableObjectStoreV2, }, { kind: framework.PluginKindObjectStore, Get: func(name string, restartableProcess RestartableProcess) v2.ObjectStore { // Adapt the existing restartable v1 plugin to be compatible with the v2 interface return newAdaptedV1ObjectStore(newRestartableObjectStore(name, restartableProcess)) }, }, } } // GetObjectStore returns a restartableObjectStore for name. func (m *manager) GetObjectStore(name string) (v2.ObjectStore, error) { name = sanitizeName(name) for _, restartableObjStore := range m.restartableObjectStores() { restartableProcess, err := m.getRestartableProcess(restartableObjStore.kind, name) if err != nil { // Check if plugin was not found if errors.Is(err, &pluginNotFoundError{}) { continue } return nil, err } return restartableObjStore.Get(name, restartableProcess), nil } return nil, fmt.Errorf("unable to get valid ObjectStore for %q", name) } ``` If the previous version is not available, or can not be adapted to the latest version, it should not be included in the `restartableObjectStores` slice. This will result in an error being returned as is currently the case when a plugin implementation for a particular kind and provider can not be found. There are situations where it may be beneficial to check at the point where a plugin API call is made whether it implements a specific version of the API. This is something that can be addressed with future amendments to this design, however it does not seem to be necessary at this time. #### Plugin Adaptation When a new plugin API version is being proposed, it will be up to the author and the maintainer team to determine whether older versions of an API can be safely adapted to the latest version. An adaptation will implement the latest version of the plugin API interface but will use the methods from the version that is being adapted. In cases where the methods signatures remain the same, the adaptation layer will call through to the same method in the version being adapted. Examples where an adaptation may be safe: - A method signature is being changed to add a new parameter but the parameter could be optional (for example, adding a context parameter). The adaptation could call through to the method provided in the previous version but omit the parameter. - A method signature is being changed to remove a parameter, but it is safe to pass a default value to the previous version. The adaptation could call through to the method provided in the previous version but use a default value for the parameter. - A new method is being added but does not impact any existing behaviour of Velero (for example, a new method which will allow Velero to [wait for additional items to be ready](https://github.com/vmware-tanzu/velero/blob/main/design/Implemented/wait-for-additional-items.md)). The adaptation would return a value which allows the existing behaviour to be performed. - A method is being deleted as it is no longer used. The adaptation would call through to any methods which are still included but would omit the deleted method in the adaptation. Examples where an adaptation may not be safe: - A new method is added which is used to provide new critical functionality in Velero. If this functionality can not be replicated using existing plugin methods in previous API versions, this should not be adapted and instead the plugin manager should return an error indicating that the plugin implementation can not be found. ### Restartable Plugin Process As new versions of plugins are added, new restartable implementations of plugins will also need to be created. These are currently located within "pkg/plugin/clientmgmt" but will be rearranged to be grouped by kind and version like other plugin files. ## Versioning Considerations It should be noted that if changes are being made to a plugin's API, it will only be necessary to bump the API version once within a release cycle, regardless of how many changes are made within that cycle. This is because the changes will only be available to consumers when they upgrade to the next minor version of the Velero library. New plugin API versions will not be introduced or backported to patch releases. Once a new minor or major version of Velero has been released however, any further changes will need to follow the process above and use a new API version. ## Alternatives Considered ### Relying on gRPC’s backwards compatibility when adding new methods One approach for adapting the plugin APIs would have been to rely on the fact that adding methods to gRPC services is a backwards compatible change. This approach would allow older clients to communicate with newer plugins as the existing interface would still be provided. This was considered but ruled out as our current framework would require any plugin that recompiles using the latest version of the framework to adapt to the new version. Also, without specific versioned interfaces, it would require checking plugin implementations at runtime for the specific methods that are supported. ## Compatibility This design doc aims to allow plugin API changes to be made in a manner that may provide some backwards compatibility. Older versions of Velero will not be able to make use of new plugin versions however may continue to use previous versions of a plugin API if supported by the plugin. All compatibility concerns are addressed earlier in the document. ## Implementation This design document primarily outlines an approach to allow future plugin API changes to be made. However, there are changes to the existing code base that will be made to allow plugin authors to more easily propose and introduce changes to these APIs. * Plugin interface definitions (currently in `pkg/plugin/velero`) will be rearranged to be grouped by kind and then versioned: `pkg/plugin/velero///`. * Proto definitions (currently in `pkg/plugin/proto`) will be rearranged to be grouped by kind and then versioned: `pkg/plugin/proto//`. * This will also require changes to the `make update` build task to correctly find the new proto location and output to the versioned directories. It is anticipated that changes to the plugin APIs will be made as part of the 1.9 release cycle. To assist with this work, an additional follow-up task to the ones listed above would be to prepare a V2 version of each of the existing plugins. These new versions will not yet provide any new API methods but will provide a layout for new additions to be made ## Open Issues ================================================ FILE: design/Implemented/priority-class-name-support_design.md ================================================ # PriorityClass Support Design Proposal ## Abstract This design document outlines the implementation of priority class name support for Velero components, including the Velero server deployment, node agent daemonset, and maintenance jobs. This feature allows users to specify a priority class name for Velero components, which can be used to influence the scheduling and eviction behavior of these components. ## Background Kubernetes allows users to define priority classes, which can be used to influence the scheduling and eviction behavior of pods. Priority classes are defined as cluster-wide resources, and pods can reference them by name. When a pod is created, the priority admission controller uses the priority class name to populate the priority value for the pod. The scheduler then uses this priority value to determine the order in which pods are scheduled. Currently, Velero does not provide a way for users to specify a priority class name for its components. This can be problematic in clusters where resource contention is high, as Velero components may be evicted or not scheduled in a timely manner, potentially impacting backup and restore operations. ## Goals - Add support for specifying priority class names for Velero components - Update the Velero CLI to accept priority class name parameters for different components - Update the Velero deployment, node agent daemonset, maintenance jobs, and data mover pods to use the specified priority class names ## Non Goals - Creating or managing priority classes - Automatically determining the appropriate priority class for Velero components ## High-Level Design The implementation will add new fields to the Velero options struct to store the priority class names for the server deployment and node agent daemonset. The Velero CLI will be updated to accept new flags for these components. For data mover pods and maintenance jobs, priority class names will be configured through existing ConfigMap mechanisms (`node-agent-configmap` for data movers and `repo-maintenance-job-configmap` for maintenance jobs). The Velero deployment, node agent daemonset, maintenance jobs, and data mover pods will be updated to use their respective priority class names. ## Detailed Design ### CLI Changes New flags will be added to the `velero install` command to specify priority class names for different components: ```go flags.StringVar( &o.ServerPriorityClassName, "server-priority-class-name", o.ServerPriorityClassName, "Priority class name for the Velero server deployment. Optional.", ) flags.StringVar( &o.NodeAgentPriorityClassName, "node-agent-priority-class-name", o.NodeAgentPriorityClassName, "Priority class name for the node agent daemonset. Optional.", ) ``` Note: Priority class names for data mover pods and maintenance jobs will be configured through their respective ConfigMaps (`--node-agent-configmap` for data movers and `--repo-maintenance-job-configmap` for maintenance jobs). ### Velero Options Changes The `VeleroOptions` struct in `pkg/install/resources.go` will be updated to include new fields for priority class names: ```go type VeleroOptions struct { // ... existing fields ... ServerPriorityClassName string NodeAgentPriorityClassName string } ``` ### Deployment Changes The `podTemplateConfig` struct in `pkg/install/deployment.go` will be updated to include a new field for the priority class name: ```go type podTemplateConfig struct { // ... existing fields ... priorityClassName string } ``` A new function, `WithPriorityClassName`, will be added to set this field: ```go func WithPriorityClassName(priorityClassName string) podTemplateOption { return func(c *podTemplateConfig) { c.priorityClassName = priorityClassName } } ``` The `Deployment` function will be updated to use the priority class name: ```go deployment := &appsv1api.Deployment{ // ... existing fields ... Spec: appsv1api.DeploymentSpec{ // ... existing fields ... Template: corev1api.PodTemplateSpec{ // ... existing fields ... Spec: corev1api.PodSpec{ // ... existing fields ... PriorityClassName: c.priorityClassName, }, }, }, } ``` ### DaemonSet Changes The `DaemonSet` function will use the priority class name passed via the podTemplateConfig (from the CLI flag): ```go daemonSet := &appsv1api.DaemonSet{ // ... existing fields ... Spec: appsv1api.DaemonSetSpec{ // ... existing fields ... Template: corev1api.PodTemplateSpec{ // ... existing fields ... Spec: corev1api.PodSpec{ // ... existing fields ... PriorityClassName: c.priorityClassName, }, }, }, } ``` ### Maintenance Job Changes The `JobConfigs` struct in `pkg/repository/maintenance/maintenance.go` will be updated to include a field for the priority class name: ```go type JobConfigs struct { // LoadAffinities is the config for repository maintenance job load affinity. LoadAffinities []*kube.LoadAffinity `json:"loadAffinity,omitempty"` // PodResources is the config for the CPU and memory resources setting. PodResources *kube.PodResources `json:"podResources,omitempty"` // PriorityClassName is the priority class name for the maintenance job pod // Note: This is only read from the global configuration, not per-repository PriorityClassName string `json:"priorityClassName,omitempty"` } ``` The `buildJob` function will be updated to use the priority class name from the global job configuration: ```go func buildJob(cli client.Client, ctx context.Context, repo *velerov1api.BackupRepository, bslName string, config *JobConfigs, podResources kube.PodResources, logLevel logrus.Level, logFormat *logging.FormatFlag) (*batchv1.Job, error) { // ... existing code ... // Use the priority class name from the global job configuration if available // Note: Priority class is only read from global config, not per-repository priorityClassName := "" if config != nil && config.PriorityClassName != "" { priorityClassName = config.PriorityClassName } // ... existing code ... job := &batchv1.Job{ // ... existing fields ... Spec: batchv1.JobSpec{ // ... existing fields ... Template: corev1api.PodTemplateSpec{ // ... existing fields ... Spec: corev1api.PodSpec{ // ... existing fields ... PriorityClassName: priorityClassName, }, }, }, } // ... existing code ... } ``` Users will be able to configure the priority class name for all maintenance jobs by creating the repository maintenance job ConfigMap before installation. For example: ```bash # Create the ConfigMap before running velero install cat < repo-maintenance-job-config.json { "global": { podResources: { "cpuRequest": "100m", "cpuLimit": "200m", "memoryRequest": "100Mi", "memoryLimit": "200Mi" }, "loadAffinity": [ { "nodeSelector": { "matchExpressions": [ { "key": "cloud.google.com/machine-family", "operator": "In", "values": [ "e2" ] } ] } }, { "nodeSelector": { "matchExpressions": [ { "key": "topology.kubernetes.io/zone", "operator": "In", "values": [ "us-central1-a", "us-central1-b", "us-central1-c" ] } ] } } ] } } EOF ``` This sample showcases two affinity configurations: - matchLabels: maintenance job runs on nodes with label key `cloud.google.com/machine-family` and value `e2`. - matchLabels: maintenance job runs on nodes located in `us-central1-a`, `us-central1-b` and `us-central1-c`. The nodes matching one of the two conditions are selected. To create the configMap, users need to save something like the above sample to a json file and then run below command: ``` kubectl create cm repo-maintenance-job-config -n velero --from-file=repo-maintenance-job-config.json ``` ### Value assigning rules If the Velero BackupRepositoryController cannot find the introduced ConfigMap, the following default values are used for repository maintenance job: ``` go config := Configs { // LoadAffinity is the config for data path load affinity. LoadAffinity: nil, // Resources is the config for the CPU and memory resources setting. PodResources: &kube.PodResources{ // The repository maintenance job CPU request setting CPURequest: "0m", // The repository maintenance job memory request setting MemoryRequest: "0Mi", // The repository maintenance job CPU limit setting CPULimit: "0m", // The repository maintenance job memory limit setting MemoryLimit: "0Mi", }, } ``` If the Velero BackupRepositoryController finds the introduced ConfigMap with only `global` element, the `global` value is used. If the Velero BackupRepositoryController finds the introduced ConfigMap with only element matches the BackupRepository, the matched element value is used. If the Velero BackupRepositoryController finds the introduced ConfigMap with both `global` element and element matches the BackupRepository, the matched element defined values overwrite the `global` value, and the `global` value is still used for matched element undefined values. For example, the ConfigMap content has two elements. ``` json { "global": { "loadAffinity": [ { "nodeSelector": { "matchExpressions": [ { "key": "cloud.google.com/machine-family", "operator": "In", "values": [ "e2" ] } ] } }, ], "podResources": { "cpuRequest": "100m", "cpuLimit": "200m", "memoryRequest": "100Mi", "memoryLimit": "200Mi" } }, "ns1-default-kopia": { "podResources": { "memoryRequest": "400Mi", "memoryLimit": "800Mi" } } } ``` The config value used for BackupRepository backing up volume data in namespace `ns1`, referencing BSL `default`, and the type is `Kopia`: ``` go config := Configs { // LoadAffinity is the config for data path load affinity. LoadAffinity: []*kube.LoadAffinity{ { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { Key: "cloud.google.com/machine-family", Operator: metav1.LabelSelectorOpIn, Values: []string{"e2"}, }, }, }, }, }, PodResources: &kube.PodResources{ // The repository maintenance job CPU request setting CPURequest: "", // The repository maintenance job memory request setting MemoryRequest: "400Mi", // The repository maintenance job CPU limit setting CPULimit: "", // The repository maintenance job memory limit setting MemoryLimit: "800Mi", } } ``` ### Implementation During the Velero repository controller starts to maintain a repository, it will call the repository manager's `PruneRepo` function to build the maintenance Job. The ConfigMap specified by `velero server` CLI parameter `--repo-maintenance-job-configmap` is get to reinitialize the repository `MaintenanceConfig` setting. ``` go jobConfig, err := getMaintenanceJobConfig( context.Background(), m.client, m.log, m.namespace, m.repoMaintenanceJobConfig, repo, ) if err != nil { log.Infof("Cannot find the ConfigMap %s with error: %s. Use default value.", m.namespace+"/"+m.repoMaintenanceJobConfig, err.Error(), ) } log.Info("Start to maintenance repo") maintenanceJob, err := m.buildMaintenanceJob( jobConfig, param, ) if err != nil { return errors.Wrap(err, "error to build maintenance job") } ``` ## Alternatives Considered An other option is creating each ConfigMap for a BackupRepository. This is not ideal for scenario that has a lot of BackupRepositories in the cluster. ================================================ FILE: design/Implemented/repository-maintenance.md ================================================ # Design for repository maintenance job ## Abstract This design proposal aims to decouple repository maintenance from the Velero server by launching a maintenance job when needed, to mitigate the impact on the Velero server during backups. ## Background During backups, Velero performs periodic maintenance on the repository. This operation may consume significant CPU and memory resources in some cases, leading to potential issues such as the Velero server being killed by OOM. This proposal addresses these challenges by separating repository maintenance from the Velero server. ## Goals 1. **Independent Repository Maintenance**: Decouple maintenance from Velero's main logic to reduce the impact on the Velero server pod. 2. **Configurable Resources Usage**: Make the resources used by the maintenance job configurable. 3. **No API Changes**: Retain existing APIs and workflow in the backup repository controller. ## Non Goals We have lots of concerns over parallel maintenance, which will increase the complexity of our design currently. - Non-blocking maintenance job: it may conflict with updating the same `backuprepositories` CR when parallel maintenance. - Maintenance job concurrency control: there is no one suitable mechanism in Kubernetes to control the concurrency of different jobs. - Parallel maintenance: Maintaining the same repo by multiple jobs at the same time would have some compatible cases that some providers may not support. Unfortunately, parallel maintenance is currently not a priority because of the concerns above, improving maintenance efficiency is not the primary focus at this stage. ## High-Level Design 1. **Add Maintenance Subcommand**: Introduce a new Velero server subcommand for repository maintenance. 2. **Create Jobs by Repository Manager**: Modify the backup repository controller to create a maintenance job instead of directly calling the multiple chain calls for Kopia or Restic maintenance. 3. **Update Maintenance Job Result in BackupRepository CR**: Retrieve the result of the maintenance job and update the status of the `BackupRepository` CR accordingly. 4. **Add Setting for Maintenance Job**: Introduce a configuration option to set maintenance jobs, including resource limits (CPU and memory), keeping the latest N maintenance jobs for each repository. ## Detailed Design ### 1. Add Maintenance sub-command The CLI command will be added to the Velero CLI, the command is designed for use in a pod of maintenance jobs. Our CLI command is designed as follows: ```shell $ velero repo-maintenance --repo-name $repo-name --repo-type $repo-type --backup-storage-location $bsl ``` Compared with other CLI commands, the maintenance command is used in a pod of maintenance jobs not for user use, and the job should show the result of maintenance after finish. Here we will write the error message into one specific file which could be read by the maintenance job. on the whole, we record two kinds of logs: - one is the log output of the intermediate maintenance process: this log could be retrieved via the Kubernetes API server, including the error log. - one is the result of the command which could indicate whether the execution is an error or not: the result could be redirected to a file that the maintenance job itself could read, and the file only contains the error message. we will write the error message into the `/dev/termination-log` file if execution is failed. The main maintenance logic would be using the repository provider to do the maintenance. ```golang func checkError(err error, file *os.File) { if err != nil { if err != context.Canceled { if _, errWrite := file.WriteString(fmt.Sprintf("An error occurred: %v", err)); errWrite != nil { fmt.Fprintf(os.Stderr, "Failed to write error to termination log file: %v\n", errWrite) } file.Close() os.Exit(1) // indicate the command executed failed } } } func (o *Options) Run(f veleroCli.Factory) { logger := logging.DefaultLogger(o.LogLevelFlag.Parse(), o.FormatFlag.Parse()) logger.SetOutput(os.Stdout) errorFile, err := os.Create("/dev/termination-log") if err != nil { fmt.Fprintf(os.Stderr, "Failed to create termination log file: %v\n", err) return } defer errorFile.Close() ... err = o.runRepoPrune(cli, f.Namespace(), logger) checkError(err, errorFile) ... } func (o *Options) runRepoPrune(cli client.Client, namespace string, logger logrus.FieldLogger) error { ... var repoProvider provider.Provider if o.RepoType == velerov1api.BackupRepositoryTypeRestic { repoProvider = provider.NewResticRepositoryProvider(credentialFileStore, filesystem.NewFileSystem(), logger) } else { repoProvider = provider.NewUnifiedRepoProvider( credentials.CredentialGetter{ FromFile: credentialFileStore, FromSecret: credentialSecretStore, }, o.RepoType, cli, logger) } ... err = repoProvider.BoostRepoConnect(context.Background(), para) if err != nil { return errors.Wrap(err, "failed to boost repo connect") } err = repoProvider.PruneRepo(context.Background(), para) if err != nil { return errors.Wrap(err, "failed to prune repo") } return nil } ``` ### 2. Create Jobs by Repository Manager Currently, the backup repository controller will call the repository manager to do the `PruneRepo`, and Kopia or Restic maintenance is then finally called through multiple chain calls. We will keep using the `PruneRepo` function in the repository manager, but we cut off the multiple chain calls by creating a maintenance job. The job definition would be like below: ```yaml apiVersion: v1 items: - apiVersion: batch/v1 kind: Job metadata: # labels or affinity or topology settings would inherit from the velero deployment labels: # label the job name for later list jobs by name job-name: nginx-example-default-kopia-pqz6c name: nginx-example-default-kopia-pqz6c namespace: velero spec: # Not retry it again backoffLimit: 1 # Only have one job one time completions: 1 # Not parallel running job parallelism: 1 template: metadata: labels: job-name: nginx-example-default-kopia-pqz6c name: kopia-maintenance-job spec: containers: # arguments for repo maintenance job - args: - repo-maintenance - --repo-name=nginx-example - --repo-type=kopia - --backup-storage-location=default # inherit from Velero server - --log-level=debug command: - /velero # inherit environment variables from the velero deployment env: - name: AZURE_CREDENTIALS_FILE value: /credentials/cloud # inherit image from the velero deployment image: velero/velero:main imagePullPolicy: IfNotPresent name: kopia-maintenance-container # resource limitation set by Velero server configuration # if not specified, it would apply best effort resources allocation strategy resources: {} # error message would be written to /dev/termination-log terminationMessagePath: /dev/termination-log terminationMessagePolicy: File # inherit volume mounts from the velero deployment volumeMounts: - mountPath: /credentials name: cloud-credentials dnsPolicy: ClusterFirst restartPolicy: Never schedulerName: default-scheduler securityContext: {} # inherit service account from the velero deployment serviceAccount: velero serviceAccountName: velero volumes: # inherit cloud credentials from the velero deployment - name: cloud-credentials secret: defaultMode: 420 secretName: cloud-credentials # ttlSecondsAfterFinished set the job expired seconds ttlSecondsAfterFinished: 86400 status: # which contains the result after maintenance message: "" lastMaintenanceTime: "" ``` Now, the backup repository controller will call the repository manager to create one maintenance job and wait for the job to complete. The Kopia or Restic maintenance multiple chains are called by the job. ### 3. Update the Result of the Maintenance Job into BackupRepository CR The backup repository controller will update the result of the maintenance job into the backup repository CR. For how to get the result of the maintenance job we could refer to [here](https://kubernetes.io/docs/tasks/debug/debug-application/determine-reason-pod-failure/#writing-and-reading-a-termination-message). After the maintenance job is finished, we could get the result of maintenance by getting the terminated message from the related pod: ```golang func GetContainerTerminatedMessage(pod *v1.Pod) string { ... for _, containerStatus := range pod.Status.ContainerStatuses { if containerStatus.LastTerminationState.Terminated != nil { return containerStatus.LastTerminationState.Terminated.Message } } ... return "" } ``` Then we could update the status of backupRepository CR with the message. ### 4. Add Setting for Resource Usage of Maintenance Add one configuration for setting the resource limit of maintenance jobs as below: ```shell velero server --maintenance-job-cpu-request $cpu-request --maintenance-job-mem-request $mem-request --maintenance-job-cpu-limit $cpu-limit --maintenance-job-mem-limit $mem-limit ``` Our default value is 0, which means we don't limit the resources, and the resource allocation strategy would be [best effort](https://kubernetes.io/docs/concepts/workloads/pods/pod-qos/#besteffort). ### 5. Automatic Cleanup for Finished Maintenance Jobs Add configuration for clean up maintenance jobs: - keep-latest-maintenance-jobs: the number of keeping latest maintenance jobs for each repository. ```shell velero server --keep-latest-maintenance-jobs $num ``` We would check and keep the latest N jobs after a new job is finished. ```golang func deleteOldMaintenanceJobs(cli client.Client, repo string, keep int) error { // Get the maintenance job list by label jobList := &batchv1.JobList{} err := cli.List(context.TODO(), jobList, client.MatchingLabels(map[string]string{RepositoryNameLabel: repo})) if err != nil { return err } // Delete old maintenance jobs if len(jobList.Items) > keep { sort.Slice(jobList.Items, func(i, j int) bool { return jobList.Items[i].CreationTimestamp.Before(&jobList.Items[j].CreationTimestamp) }) for i := 0; i < len(jobList.Items)-keep; i++ { err = cli.Delete(context.TODO(), &jobList.Items[i], client.PropagationPolicy(metav1.DeletePropagationBackground)) if err != nil { return err } } } return nil } ``` ### 6 Velero Install with Maintenance Options All the above maintenance options should be supported by Velero install command. ### 7. Observability and Debuggability Some monitoring metrics are added for backup repository maintenance: - repo_maintenance_total - repo_maintenance_success_total - repo_maintenance_failed_total - repo_maintenance_duration_seconds We will keep the latest N maintenance jobs for each repo, and users can get the log from the job. the job log level inherent from the Velero server setting. Also, we would integrate maintenance job logs and `backuprepositories` CRs into `velero debug`. Roughly, the process is as follows: 1. The backup repository controller will check the BackupRepository request in the queue periodically. 2. If the maintenance period of the repository checked by `runMaintenanceIfDue` in `Reconcile` is due, then the backup repository controller will call the Repository manager to execute `PruneRepo` 3. The `PruneRepo` of the Repository manager will create one maintenance job, the resource limitation, environment variables, service account, images, etc. would inherit from the Velero server pod. Also, one clean up TTL would be set to maintenance job. 4. The maintenance job will execute the Velero maintenance command, wait for maintaining to finish and write the maintenance result into the terminationMessagePath file of the related pod. 5. Kubernetes could show the result in the status of the pod by reading the termination message in the pod. 6. The backup repository controller will wait for the maintenance job to finish and read the status of the maintenance job, then update the message field and phase in the status of `backuprepositories` CR accordingly. 6. Clean up old maintenance jobs and keep only N latest for each repository. ### 8. Codes Refinement Once `backuprepositories` CR status is modified, the CR would re-queue to be reconciled, and re-execute logics in reconcile shortly not respecting the re-queue frequency configured by `repoSyncPeriod`. For one abnormal scenario if the maintenance job fails, the status of `backuprepositories` CR would be updated and the CR will re-queue immediately, if the new maintenance job still fails, then it will re-queue again, making the logic of `backuprepositories` CR re-queue like a dead loop. So we change the Predicates logic in Controller manager making it only re-queue if the Spec of `backuprepositories` CR is changed. ```golang ctrl.NewControllerManagedBy(mgr).For(&velerov1api.BackupRepository{}, builder.WithPredicates(kube.SpecChangePredicate{})) ``` This change would bring the behavior different from the previous, errors that occurred in the maintenance job would retry in the next reconciliation period instead of retrying immediately. ## Prospects for Future Work Future work may focus on improving the efficiency of Velero maintenance through non-blocking parallel modes. Potential areas for enhancement include: **Non-blocking Mode**: Explore the implementation of a non-blocking mode for parallel maintenance to enhance overall efficiency. **Concurrency Control**: Investigate mechanisms for better concurrency control of different maintenance jobs. **Provider Support for Parallel Maintenance**: Evaluate the feasibility of parallel maintenance for different providers and address any compatibility issues. **Efficiency Improvements**: Investigate strategies to optimize maintenance efficiency without compromising reliability. By considering these areas, future iterations of Velero may benefit from enhanced parallelization and improved resource utilization during repository maintenance. ================================================ FILE: design/Implemented/resource-status-restore.md ================================================ # Allow Object-Level Resource Status Restore in Velero ## Abstract This design proposes a way to enhance Velero’s restore functionality by enabling object-level resource status restoration through annotations. Currently, Velero allows restoring resource statuses only at a resource type level, which lacks granularity of restoring the status of specific resources. By introducing an annotation that controllers can set on individual resource objects, this design aims to improve flexibility and autonomy for users/resource-controllers, providing a more way to enable resource status restore. ## Background Velero provides the `restoreStatus` field in the Restore API to specify resource types for status restoration. However, this feature is limited to resource types as a whole, lacking the granularity needed to restore specific objects of a resource type. Resource controllers, especially those managing custom resources with external dependencies, may need to restore status on a per-object basis based on internal logic and dependencies. This design adds an annotation-based approach to allow controllers to specify status restoration at the object level, enabling Velero to handle status restores more flexibly. ## Goals - Provide a mechanism to specify the restoration of a resource’s status at an object level. - Maintain backwards compatibility with existing functionality, allowing gradual adoption of this feature. - Integrate the new annotation-based objects-level status restore with Velero’s existing resource-type-level `restoreStatus` configuration. ## Non-Goals - Alter Velero’s existing resource type-level status restoration mechanism for resources without annotations. ## Use-Cases/Scenarios 1. Controller managing specific Resources - A resource controller identifies that a specific object of a resource should have its status restored due to particular dependencies - The controller automatically sets the `velero.io/restore-status: true` annotation on the resource. - During restore, Velero restores the status of this object, while leaving other resources unaffected. - The status for the annotated object will be restored regardless of its inclusion/exclusion in `restoreStatus.includedResources` 2. A specific object must not have its status restored even if its included in `restoreStatus.includedResources` - A user specifies a resource type in the `restoreStatus.includedResources` field within the Restore custom resource. - A particular object of that resource type is annotated with `velero.io/restore-status: false` by the user. - The status of the annotated object will not restored even though its included in `restoreStatus.includedResources` because annotation is `false` and it takes precedence. 4. Default Behavior for objects Without the Annotation - Objects without the `velero.io/restore-status` annotation behave as they currently do: Velero skips their status restoration unless the resource type is specified in the `restoreStatus.includedResources` field. ## High-Level Design - Object-Level Status Restore Annotation: We are introducing the `velero.io/restore-status` annotation at the resource object level to mark specific objects for status restoration. - `true`: Indicates that the status should be restored for this object - `false`: Skip restoring status for this specific object - Invalid or missing annotations defer to the meaning of existing resource type-level logic. - Restore logic precedence: - Annotations take precedence when they exist with valid values (`true` or `false`). - Restore spec `restoreStatus.includedResources` is only used when annotations are invalid or missing. - Velero Restore Logic Update: During a restore operation, Velero will: - Extend the existing restore logic to parse and prioritize annotations introduced in this design. - Update resource objects accordingly based on their annotation values or fallback configuration. ## Detailed Design - Annotation for object-Level Status Restore: The `velero.io/restore-status` annotation will be set on individual resource objects by users/controllers as needed: ```yaml metadata: annotations: velero.io/restore-status: "true" ``` - Restore Logic Modifications: During the restore operation, the restore controller will follow these steps: - Parse the `restoreStatus.includedResources` spec to determine resource types eligible for status restoration. - For each resource object: - Check for the `velero.io/restore-status` annotation. - If the annotation value is: - `true`: Restore the status of the object - `false`: Skip restoring the status of the object - If the annotation is invalid or missing: - Default to the `restoreStatus.includedResources` configuration ## Implementation We are targeting the implementation of this design for Velero 1.16 release. Current restoreStatus logic resides here: https://github.com/vmware-tanzu/velero/blob/32a8c62920ad96c70f1465252c0197b83d5fa6b6/pkg/restore/restore.go#L1652 The modified logic would look somewhat like: ```go // Determine whether to restore status from resource type configuration shouldRestoreStatus := ctx.resourceStatusIncludesExcludes != nil && ctx.resourceStatusIncludesExcludes.ShouldInclude(groupResource.String()) // Check for object-level annotation annotations := obj.GetAnnotations() objectAnnotation := annotations["velero.io/restore-status"] annotationValid := objectAnnotation == "true" || objectAnnotation == "false" // Determine restore behavior based on annotation precedence shouldRestoreStatus = (annotationValid && objectAnnotation == "true") || (!annotationValid && shouldRestoreStatus) ctx.log.Debugf("status field for %s: exists: %v, should restore: %v (by annotation: %v)", newGR, statusFieldExists, shouldRestoreStatus, annotationValid) if shouldRestoreStatus && statusFieldExists { if err := unstructured.SetNestedField(obj.Object, objStatus, "status"); err != nil { ctx.log.Errorf("Could not set status field %s: %v", kube.NamespaceAndName(obj), err) errs.Add(namespace, err) return warnings, errs, itemExists } obj.SetResourceVersion(createdObj.GetResourceVersion()) updated, err := resourceClient.UpdateStatus(obj, metav1.UpdateOptions{}) if err != nil { ctx.log.Infof("Status field update failed %s: %v", kube.NamespaceAndName(obj), err) warnings.Add(namespace, err) } else { createdObj = updated } } ``` ================================================ FILE: design/Implemented/restic-backup-and-restore-progress.md ================================================ # Progress reporting for restic backups and restores Status: Accepted During long-running restic backups/restores, there is no visibility into what (if anything) is happening, making it hard to know if the backup/restore is making progress or hung, how long the operation might take, etc. We should capture progress during restic operations and make it user-visible so that it's easier to reason about. This document proposes an approach for capturing progress of backup and restore operations and exposing this information to users. ## Goals - Provide basic visibility into restic operations to inform users about their progress. ## Non Goals - Capturing progress for non-restic backups and restores. ## Background (Omitted, see introduction) ## High-Level Design ### restic backup progress The `restic backup` command provides progress reporting to stdout in JSON format, which includes the completion percentage of the backup. This progress will be read on some interval and the PodVolumeBackup Custom Resource's (CR) status will be updated with this information. ### restic restore progress The `restic stats` command returns the total size of a backup. This can be compared with the total size the volume periodically to calculate the completion percentage of the restore. The PodVolumeRestore CR's status will be updated with this information. ## Detailed Design ## Changes to PodVolumeBackup and PodVolumeRestore Status type A new `Progress` field will be added to PodVolumeBackupStatus and PodVolumeRestoreStatus of type `PodVolumeOperationProgress`: ``` type PodVolumeOperationProgress struct { TotalBytes int64 BytesDone int64 } ``` ### restic backup progress restic added support for [streaming JSON output for the `restic backup` command](https://github.com/restic/restic/pull/1944) in 0.9.5. Our current images ship restic 0.9.4, and so the Dockerfile will be updated to pull the new version: https://github.com/heptio/velero/blob/af4b9373fc73047f843cd4bc3648603d780c8b74/Dockerfile-velero#L21. With the `--json` flag, `restic backup` outputs single lines of JSON reporting the status of the backup: ``` {"message_type":"status","percent_done":0,"total_files":1,"total_bytes":21424504832} {"message_type":"status","action":"scan_finished","item":"","duration":0.219241873,"data_size":49461329920,"metadata_size":0,"total_files":10} {"message_type":"status","percent_done":0,"total_files":10,"total_bytes":49461329920,"current_files":["/file3"]} {"message_type":"status","percent_done":0.0003815984736061056,"total_files":10,"total_bytes":49461329920,"bytes_done":18874368,"current_files":["/file1","/file3"]} {"message_type":"status","percent_done":0.0011765952936188255,"total_files":10,"total_bytes":49461329920,"bytes_done":58195968,"current_files":["/file1","/file3"]} {"message_type":"status","percent_done":0.0019503921984312064,"total_files":10,"total_bytes":49461329920,"bytes_done":96468992,"current_files":["/file1","/file3"]} {"message_type":"status","percent_done":0.0028089887640449437,"total_files":10,"total_bytes":49461329920,"bytes_done":138936320,"current_files":["/file1","/file3"]} ``` The [command factory for backup](https://github.com/heptio/velero/blob/af4b9373fc73047f843cd4bc3648603d780c8b74/pkg/restic/command_factory.go#L37) will be updated to include the `--json` flag. The code to run the `restic backup` command (https://github.com/heptio/velero/blob/af4b9373fc73047f843cd4bc3648603d780c8b74/pkg/controller/pod_volume_backup_controller.go#L241) will be changed to include a Goroutine that reads from the command's stdout stream. The implementation of this will largely follow [@jmontleon's PoC](https://github.com/fusor/velero/pull/4/files) of this. The Goroutine will periodically read the stream (every 10 seconds) and get the last printed status line, which will be converted to JSON. If `bytes_done` is empty, restic has not finished scanning the volume and hasn't calculated the `total_bytes`. In this case, we will not update the PodVolumeBackup and instead will wait for the next iteration. Once we get a non-zero value for `bytes_done`, the `bytes_done` and `total_bytes` properties will be read and the PodVolumeBackup will be patched to update `status.Progress.BytesDone` and `status.Progress.TotalBytes` respectively. Once the backup has completed successfully, the PodVolumeBackup will be patched to set `status.Progress.BytesDone = status.Progress.TotalBytes`. This is done since the main thread may cause early termination of the Goroutine once the operation has finished, preventing a final update to the `BytesDone` property. ### restic restore progress The `restic stats --json` command provides information about the size of backups: ``` {"total_size":10558111744,"total_file_count":11} ``` Before beginning the restore operation, we can use the output of `restic stats` to get the total size of the backup. The PodVolumeRestore will be patched to set `status.Progress.TotalBytes` to the total size of the backup. The code to run the `restic restore` command will be changed to include a Goroutine that periodically (every 10 seconds) gets the current size of the volume. To get the current size of the volume, we will recursively walkthrough all files in the volume to accumulate the total size. The current total size is the number of bytes transferred so far and the PodVolumeRestore will be patched to update `status.Progress.BytesDone`. Once the restore has completed successfully, the PodVolumeRestore will be patched to set `status.Progress.BytesDone = status.Progress.TotalBytes`. This is done since the main thread may cause early termination of the Goroutine once the operation has finished, preventing a final update to the `BytesDone` property. ### Velero CLI changes The output that describes detailed information about [PodVolumeBackups](https://github.com/heptio/velero/blob/559d62a2ec99f7a522924348fc4a173a0699813a/pkg/cmd/util/output/backup_describer.go#L349) and [PodVolumeRestores](https://github.com/heptio/velero/blob/559d62a2ec99f7a522924348fc4a173a0699813a/pkg/cmd/util/output/restore_describer.go#L160) will be updated to calculate and display a completion percentage from `status.Progress.TotalBytes` and `status.Progress.BytesDone` if available. ## Open Questions - Can we assume that the volume we are restoring in will be empty? Can it contain other artefacts? - Based on discussion in this PR, we are okay making the assumption that the PVC is empty and will proceed with the above proposed approach. ## Alternatives Considered ### restic restore progress If we cannot assume that the volume we are restoring into will be empty, we can instead use the output from `restic snapshot` to get the list of files in the backup. This can then be used to calculate the current total size of just those files in the volume, so that we avoid considering any other files unrelated to the backup. The above proposed approach is simpler than this one, as we don't need to keep track of each file in the backup, but this will be more robust if the volume could contain other files not included in the backup. It's possible that certain volume types may contain hidden files that could attribute to the total size of the volume, though these should be small enough that the BytesDone calculation will only be slightly inflated. Another option is to contribute progress reporting similar to `restic backup` for `restic restore` upstream. This may take more time, but would give us a more native view on the progress of a restore. There are several issues about this already in the restic repo (https://github.com/restic/restic/issues/426, https://github.com/restic/restic/issues/1154), and what looks like an abandoned attempt (https://github.com/restic/restic/pull/2003) which we may be able to pick up. ## Security Considerations N/A ================================================ FILE: design/Implemented/restore-finalizing-phase_design.md ================================================ # Design for Adding Finalization Phase in Restore Workflow ## Abstract This design proposes adding the finalization phase to the restore workflow. The finalization phase would be entered after all item restoration and plugin operations have been completed, similar to the way the backup process proceeds. Its purpose is to perform any wrap-up work necessary before transitioning the restore process to a terminal phase. ## Background Currently, the restore process enters a terminal phase once all item restoration and plugin operations have been completed. However, there are some wrap-up works that need to be performed after item restoration and plugin operations have been fully executed. There is no suitable opportunity to perform them at present. To address this, a new finalization phase should be added to the existing restore workflow. in this phase, all plugin operations and item restoration has been fully completed, which provides a clean opportunity to perform any wrap-up work before termination, improving the overall restore process. Wrap-up tasks in Velero can serve several purposes: - Post-restore modification - Velero can modify the restored data that was temporarily changed for some purpose but required to be changed back finally or data that was newly created but missing some information. For example, [issue6435](https://github.com/vmware-tanzu/velero/issues/6435) indicates that some custom settings(like labels, reclaim policy) on restored PVs was lost because those restored PVs was newly dynamically provisioned. Velero can address it by patching the PVs' custom settings back in the finalization phase. - Clean up unused data - Velero can identify and delete any data that are no longer needed after a successful restore in the finalization phase. - Post-restore validation - Velero can validate the state of restored data and report any errors to help users locate the issue in the finalization phase. The uses of wrap-up tasks are not limited to these examples. Additional needs may be addressed as they develop over time. ## Goals - Add the finalization phase and the corresponding controller to restore workflow. ## Non Goals - Implement the specific wrap-up work. ## High-Level Design - The finalization phase will be added to current restore workflow. - The logic for handling current phase transition in restore and restore operations controller will be modified with the introduction of the finalization phase. - A new restore finalizer controller will be implemented to handle the finalization phase. ## Detailed Design ### phase transition Two new phases related to finalization will be added to restore workflow, which are `FinalizingPartiallyFailed` and `Finalizing`. The new phase transition will be similar to backup workflow, proceeding as follow: ![image](restore-phases-transition.png) ### restore finalizer controller The new restore finalizer controller will be implemented to watch for restores in `FinalizingPartiallyFailed` and `Finalizing` phases. Any wrap-up work that needs to wait for the completion of item restoration and plugin operations will be executed by this controller, and the phase will be set to either `Completed` or `PartiallyFailed` based on the results of these works. Points worth noting about the new restore finalizer controller: A new structure `finalizerContext` will be created to facilitate the implementation of any wrap-up tasks. It includes all the dependencies the tasks require as well as a function `execute()` to orderly implement task logic. ``` // finalizerContext includes all the dependencies required by wrap-up tasks type finalizerContext struct { ....... restore *velerov1api.Restore log logrus.FieldLogger ....... } // execute executes all the wrap-up tasks and return the result func (ctx *finalizerContext) execute() (results.Result, results.Result) { // execute task1 ....... // execute task2 ....... // the task execution logic will be expanded as new tasks are included ....... } // newFinalizerContext returns a finalizerContext object, the parameters will be added as new tasks are included. func newFinalizerContext(restore *velerov1api.Restore, log logrus.FieldLogger, ...) *finalizerContext{ return &finalizerContext{ ....... restore: restore, log: log, ....... } } ``` The finalizer controller is responsible for collecting all dependencies and creating a `finalizerContext` object using those dependencies. It then invokes the `execute` function. ``` func (r *restoreFinalizerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { ....... // collect all dependencies required by wrap-up tasks ....... // create a finalizerContext object and invoke execute() finalizerCtx := newFinalizerContext(restore, log, ...) warnings, errs := finalizerCtx.execute() ....... } ``` After completing all necessary tasks, the result metadata in object storage will be updated if any errors or warnings occur during the execution. This behavior breaks the feature of keeping metadata files in object storage immutable, However, we believe the tradeoff is justified because it provides users with the access to examine the error/warning details when the wrap-up tasks go wrong. ``` // UpdateResults updates the result metadata in object storage if necessary func (r *restoreFinalizerReconciler) UpdateResults(restore *api.Restore, newWarnings *results.Result, newErrs *results.Result, backupStore persistence.BackupStore) error { originResults, err := backupStore.GetRestoreResults(restore.Name) if err != nil { return errors.Wrap(err, "error getting restore results") } warnings := originResults["warnings"] errs := originResults["errors"] warnings.Merge(newWarnings) errs.Merge(newErrs) m := map[string]results.Result{ "warnings": warnings, "errors": errs, } if err := putResults(restore, m, backupStore); err != nil { return errors.Wrap(err, "error putting restore results") } return nil } ``` ## Compatibility The new finalization phases are added without modifying the existing phases in the restore workflow. Both new and ongoing restore processes will continue to eventually transition to a terminal phase from any prior phase, ensuring backward compatibility. ## Implementation This will be implemented during the Velero 1.14 development cycle. ================================================ FILE: design/Implemented/restore-hooks.md ================================================ # Restore Hooks This document proposes a solution that allows a user to specify Restore Hooks, much like Backup Hooks, that can be executed during the restore process. ## Goals - Enable custom commands to be run during a restore in order to mirror the commands that are available to the backup process. - Provide observability into the result of commands run in restored pods. ## Non Goals - Handling any application specific scenarios (postgres, mongo, etc) ## Background Velero supports Backup Hooks to execute commands before and/or after a backup. This enables a user to, among other things, prepare data to be backed up without having to freeze an in-use volume. An example of this would be to attach an empty volume to a Postgres pod, use a backup hook to execute `pg_dump` from the data volume, and back up the volume containing the export. The problem is that there's no easy or automated way to include an automated restore process. After a restore with the example configuration above, the postgres pod will be empty, but there will be a need to manually exec in and run `pg_restore`. ## High-Level Design The Restore spec will have a `spec.hooks` section matching the same section on the Backup spec except no `pre` hooks can be defined - only `post`. Annotations comparable to the annotations used during backup can also be set on pods. For each restored pod, the Velero server will check if there are any hooks applicable to the pod. If a restored pod has any applicable hooks, Velero will wait for the container where the hook is to be executed to reach status Running. The Restore log will include the results of each post-restore hook and the Restore object status will incorporate the results of hooks. The Restore log will include the results of each hook and the Restore object status will incorporate the results of hooks. A new section at `spec.hooks.resources.initContainers` will allow for injecting initContainers into restored pods. Annotations can be set as an alternative to defining the initContainers in the Restore object. ## Detailed Design Post-restore hooks can be defined by annotation and/or by an array of resource hooks in the Restore spec. The following annotations are supported: - post.hook.restore.velero.io/container - post.hook.restore.velero.io/command - post.hook.restore.velero.io/on-error - post.hook.restore.velero.io/exec-timeout - post.hook.restore.velero.io/wait-timeout Init restore hooks can be defined by annotation and/or in the new `initContainers` section in the Restore spec. The initContainers schema is `pod.spec.initContainers`. The following annotations are supported: - init.hook.restore.velero.io/timeout - init.hook.restore.velero.io/initContainers This is an example of defining hooks in the Restore spec. ```yaml apiVersion: velero.io/v1 kind: Restore spec: ... hooks: resources: - name: my-hook includedNamespaces: - '*' excludedNamespaces: - some-namespace includedResources: - pods excludedResources: [] labelSelector: matchLabels: app: velero component: server post: - exec: container: postgres command: - /bin/bash - -c - rm /docker-entrypoint-initdb.d/dump.sql onError: Fail timeout: 10s readyTimeout: 60s init: timeout: 120s initContainers: - name: restore image: postgres:12 command: ["/bin/bash", "-c", "mv /backup/dump.sql /docker-entrypoint-initdb.d/"] volumeMounts: - name: backup mountPath: /backup ``` As with Backups, if an annotation is defined on a pod then no hooks from the Restore spec will be applied. ### Implementation The types and function in pkg/backup/item_hook_handler.go will be moved to a new package (pkg/hooks) and exported so they can be used for both backups and restores. The post-restore hooks implementation will closely follow the design of restoring pod volumes with restic. The pkg/restore.context type will have new fields `hooksWaitGroup` and `hooksErrs` comparable to `resticWaitGroup` and `resticErr`. The pkg/restore.context.execute function will start a goroutine for each pod with applicable hooks and then continue with restoring other items. Each hooks goroutine will create a pkg/util/hooks.ItemHookHandler for each pod and send any error on the context.hooksErrs channel. The ItemHookHandler already includes stdout and stderr and other metadata in the Backup log so the same logs will automatically be added to the Restore log (passed as the first argument to the ItemHookhandler.HandleHooks method.) The pkg/restore.context.execute function will wait for the hooksWaitGroup before returning. Any errors received on context.hooksErrs will be added to errs.Velero. One difference compared to the restic restore design is that any error on the context.hooksErrs channel will cancel the context of all hooks, since errors are only reported on this channel if the hook specified `onError: Fail`. However, canceling the hooks goroutines will not cancel the restic goroutines. In practice the restic goroutines will complete before the hooks since the hooks do not run until a pod is ready, but it's possible a hook will be executed and fail while a different pod is still in the pod volume restore phase. Failed hooks with `onError: Continue` will appear in the Restore log but will not affect the status of the parent Restore. Failed hooks with `onError: Fail` will cause the parent Restore to have status Partially Failed. If initContainers are specified for a pod, Velero will inject the containers into the beginning of the pod's initContainers list. If a restic initContainer is also being injected, the restore initContainers will be injected directly after the restic initContainer. The restore will use a RestoreItemAction to inject the initContainers. Stdout and stderr of the restore initContainers will not be added to the Restore logs. InitContainers that fail will not affect the parent Restore's status. ## Alternatives Considered Wait for all restored Pods to report Ready, then execute the first hook in all applicable Pods simultaneously, then proceed to the next hook, etc. That could introduce deadlock, e.g. if an API pod cannot be ready until the DB pod is restored. Put the restore hooks on the Backup spec as a third lifecycle event named `restore` along with `pre` and `post`. That would be confusing since `pre` and `post` would appear in the Backup log but `restore` would only be in the Restore log. Execute restore hooks in parallel for each Pod. That would not match the behavior of Backups. Wait for PodStatus ready before executing the post-restore hooks in any container. There are cases where the pod should not report itself ready until after the restore hook has run. Include the logs from initContainers in the Restore log. Unlike exec hooks where stdout and stderr are permanently lost if not added to the Restore log, the logs of the injected initContainers are available through the K8s API with kubectl or another client. ## Security Considerations Stdout or stderr in the Restore log may contain sensitive information, but the same risk already exists for Backup hooks. ================================================ FILE: design/Implemented/restore-with-EnableAPIGroupVersions-feature.md ================================================ # Restore API Group Version by Priority Level When EnableAPIGroupVersions Feature is Set Status: Accepted ## Abstract This document proposes a solution to select an API group version to restore from the versions backed up using the feature flag EnableAPIGroupVersions. ## Background It is possible that between the time a backup has been made and a restore occurs that the target Kubernetes version has incremented more than one version. In such a case where at least a versions of Kubernetes was skipped, the preferred source cluster's API group versions for resources may no longer be supported by the target cluster. With [PR#2373](https://github.com/vmware-tanzu/velero/pull/2373), all supported API group versions were backed up if the EnableAPIGroupVersions feature flag was set for Velero. The next step (outlined by this design proposal) will be to see if any of the backed up versions are supported in the target cluster and if so, choose one to restore for each backed up resource. ## Goals - Choose an API group to restore from backups given a priority system or a user-provided prioritization of versions. - Restore resources using the chosen API group version. ## Non Goals - Allow users to restore onto a cluster that is running a Kubernetes version older than the source cluster. The changes proposed here only allow for skipping ahead to a newer Kubernetes version, but not going backward. - Allow restoring from backups created using Velero version 1.3 or older. This proposal will only work on backups created using Velero 1.4+. - Modifying the compressed backup tarball files. We don't want to risk corrupting the backups. - Using plugins to restore a resource when the target supports none of the source cluster's API group versions. The ability to use plugins will hopefully be something added in the future, but not at this time. ## High-Level Design During restore, the proposal is that Velero will determine if the `APIGroupVersionsFeatureFlag` was enabled in the target cluster and `Status.FormatVersion 1.1.0` was used during backup. Only if these two conditions are met will the changes proposed here take effect. The proposed code starts with creating three lists for each backed up resource. The three lists will be created by (1) reading the directory names in the backup tarball file and seeing which API group versions were backed up from the source cluster, (2) looking at the target cluster and determining which API group versions are supported, and (3) getting ConfigMaps from the target cluster in order to get user-defined prioritization of versions. The three lists will be used to create a map of chosen versions for each resource to restore. If there is a user-defined list of priority versions, the versions will be checked against the supported versions lists. The highest user-defined priority version that is/was supported by both target and source clusters will be the chosen version for that resource. If no user specified versions are supported by neither target nor source, the versions will be logged and the restore will continue with other prioritizations. Without a user-defined prioritization of versions, the following version prioritization will be followed, starting from the highest priority: target cluster preferred version, source cluster preferred version, and a common supported version. Should there be multiple common supported versions, the one that will be chosen will be based on the [Kubernetes version priorities](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definition-versioning/#version-priority). Once the version to restore is chosen, the file path to the backed up resource in the tarball will be modified such that it points to the resources' chosen API group version. If no version is found in common between the source and target clusters, the chosen version will default to the source cluster's preferred version (the version being restored currently without the changes proposed here). Restore will be allowed to continue as before. ## Detailed Design There are six objectives to achieve the above stated goals: 1. Determine if the APIGroupVersionsFeatureFlag is enabled and Backup Objects use Status.FormatVersion 1.1.0. 1. List the backed up API group versions. 1. List the API group versions supported by the target cluster. 1. Get the user-defined version priorities. 1. Use a priority system to determine which version to restore. The source preferred version will be the default if the priorities fail. 1. Modify the paths to the backup files in the tarball in the resource restore process. ### Objective 1: Determine if the APIGroupVersionsFeatureFlag is enabled and Backup Objects use Status.FormatVersion 1.1.0 For restore to be able to choose from multiple supported backed up versions, the feature flag must have been enabled during the restore processes. Backup objects must also have [Status.FormatVersion == "1.1.0"](https://github.com/vmware-tanzu/velero/blob/a1e182e723a8c5f6d4175d8db2361233a94d2502/pkg/backup/backup.go#L58). The reason for checking for the feature flag during restore is to ensure the user would like to restore a version that might not be the source cluster preferred version. This check is done via `features.IsEnabled(velerov1api.APIGroupVersionsFeatureFlag)`. The reason for checking `Status.FormatVersion` is to ensure the changes made by this proposed design is backward compatible. Only with Velero version 1.4 and forward was Format Version 1.1.0 used to structure the backup directories. Format Version 1.1.0 is required for the restore process proposed in this design doc to work. Before v1.4, the backed up files were in a directory structure that will not be recognized by the proposed code changes. In this case, restore should not attempt to restore from multiple versions as they will not exist. The [`Status.FormatVersion`](https://github.com/vmware-tanzu/velero/blob/6808acd92e30848056a21faf373af03ddb8a3b71/pkg/apis/velero/v1/backup.go#L235) is stored in a `restoreContext` struct field called [`backup`](https://github.com/vmware-tanzu/velero/blob/6808acd92e30848056a21faf373af03ddb8a3b71/pkg/restore/restore.go#L229). The full chain is `ctx.backup.Status.FormatVersion`. The above two checks can be done inside a new method on the `*restoreContext` object with the method signature `meetsAPIGVRestoreReqs() bool`. This method can remain in the `restore` package, but for organizational purposes, it can be moved to a file called `prioritize_group_version.go`. ### Objective 2: List the backed up API group versions Currently, in `pkg/restore/restore.go`, in the `execute(...)` method, around [line 363](https://github.com/vmware-tanzu/velero/blob/7a103b9eda878769018386ecae78da4e4f8dde83/pkg/restore/restore.go#L363), the resources and their backed up items are saved in a map called `backupResources`. At this point, the feature flag and format versions can be checked (described in Objective #1). If the requirements are met, the `backedupResources` map can be sent to a method (to be created) with the signature `ctx.chooseAPIVersionsToRestore(backupResources)`. The `ctx` object has the type `*restore.Context`. The `chooseAPIVersionsToRestore` method can remain in the `restore` package, but for organizational purposes, it can be moved to a file called `prioritize_group_version.go`. Inside the `chooseAPIVersionsToRestore` method, we can take advantage of the `archive` package's `Parser` type. `ParseGroupVersions(backupDir string) (map[string]metav1.APIGroup, error)`. The `ParseGroupVersions(...)` method will loop through the `resources`, `resource.group`, and group version directories to populate a map called `sourceRGVersions`. The `sourceRGVersions` map's keys will be strings in the format `.`, e.g. "horizontalpodautoscalers.autoscaling". The values will be APIGroup structs. The API Group struct can be imported from k8s.io/apimachinery/pkg/apis/meta/v1. Order the APIGroup.Versions slices using a sort function copied from `k8s.io/apimachinery/pkg/version`. ```go sort.SliceStable(gvs, func(i, j int) bool { return version.CompareKubeAwareVersionStrings(gvs[i].Version, gvs[j].Version) > 0 }) ``` ### Objective 3: List the API group versions supported by the target cluster Still within the `chooseAPIVersionsToRestore` method, the target cluster's resource group versions can now be obtained. ```go targetRGVersions := ctx.discoveryHelper.APIGroups() ``` Order the APIGroup.Versions slices using a sort function copied from `k8s.io/apimachinery/pkg/version`. ```go sort.SliceStable(gvs, func(i, j int) bool { return version.CompareKubeAwareVersionStrings(gvs[i].Version, gvs[j].Version) > 0 }) ``` ### Objective 4: Get the user-defined version priorities Still within the `chooseAPIVersionsToRestore` method, the user-defined version priorities can be retrieved. These priorities are expected to be in a config map named `enableapigroupversions` in the `velero` namespace. An example config map is ```yaml apiVersion: v1 kind: ConfigMap metadata: name: enableapigroupversions   namespace: velero data:   restoreResourcesVersionPriority: | -     rockbands.music.example.io=v2beta1,v2beta2     orchestras.music.example.io=v2,v3alpha1     subscriptions.operators.coreos.com=v2,v1 ``` In the config map, the resources and groups and the user-defined version priorities will be listed in the `data.restoreResourcesVersionPriority` field following the following general format: `.=[, ...]`. A map will be created to store the user-defined priority versions. The map's keys will be strings in the format `.`. The values will be APIGroup structs that will be imported from `k8s.io/apimachinery/pkg/apis/meta/v1`. Within the APIGroup structs will be versions in the order that the user provides in the config map. The PreferredVersion field in APIGroup struct will be left empty. ### Objective 5: Use a priority system to determine which version to restore. The source preferred version will be the default if the priorities fail Determining the priority will also be done in the `chooseAPIVersionsToRestore` method. Once a version is chosen, it will be stored in a new map of the form `map[string]ChosenGRVersion` where the key is the `.` and the values are of the `ChosenGroupVersion` struct type (shown below). The map will be saved to the `restore.Context` object in a field called `chosenGrpVersToRestore`. ```go type ChosenGroupVersion struct { Group string Version string Dir string } ``` The first method called will be `ctx.gatherSTUVersions()` and it will gather the source cluster group resource and versions (`sgvs`), target cluster group versions (`tgvs`), and custom user resource and group versions (`ugvs`). Loop through the source cluster resource and group versions (`sgvs`). Find the versions for the group in the target cluster. An attempt will first be made to `findSupportedUserVersion`. Loop through the resource.groups in the custom user resource and group versions (`ugvs`) map. If a version is supported by both `tgvs` and `sgvs`, that will be set as the chosen version for the corresponding resource in `ctx.chosenGrpVersToRestore` If no three-way match can be made between the versions in `ugvs`, `tgvs`, and `sgvs`, move on to attempting to use the target cluster preferred version. Loop through the `sgvs` versions for the resource and see if any of them match the first item in the `tgvs` version list. Because the versions in `tgvs` have been ordered, the first version in the version slide will be the preferred version. If target preferred version cannot be used, attempt to choose the source cluster preferred version. Loop through the target versions and see if any of them match the first item in the source version slice, which will be the preferred version due to Kubernetes version ordering. If neither clusters' preferred version can be used, look through remaining versions in the target version list and see if there is a match with the remaining versions in the source versions list. If none of the previous checks produce a chosen version, the source preferred version will be the default and the restore process will continue. Here is another way to list the priority versions described above: - **Priority 0** ((User override). Users determine restore version priority using a config map - **Priority 1**. Target preferred version can be used. - **Priority 2**. Source preferred version can be used. - **Priority 3**. A common supported version can be used. This means - target supported version == source supported version - if multiple support versions intersect, choose the version using the [Kubernetes’ version prioritization system](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definition-versioning/#version-priority) If there is no common supported version between target and source clusters, then the default `ChosenGRVersion` will be the source preferred version. This is the version that would have been assumed for restore before the changes proposed here. Note that adding a field to `restore.Context` will mean having to make a map for the field during instantiation. To see example cases with version priorities, see a blog post written by Rafael Brito: https://github.com/brito-rafa/k8s-webhooks/tree/master/examples-for-projectvelero. ### Objective 6: Modify the paths to the backup files in the tarball The method doing the bulk of the restoration work is `ctx.restoreResource(...)`. Inside this method, around [line 714](https://github.com/vmware-tanzu/velero/blob/7a103b9eda878769018386ecae78da4e4f8dde83/pkg/restore/restore.go#L714) in `pkg/restore/restore.go`, the path to backup json file for the item being restored is set. After the groupResource is instantiated at pkg/restore/restore.go:733, and before the `for` loop that ranges through the `items`, the `ctx.chosenGRVsToRestore` map can be checked. If the groupResource exists in the map, the path saved to `resource` variable can be updated. Currently, the item paths look something like ```bash /var/folders/zj/vc4ln5h14djg9svz7x_t1d0r0000gq/T/620385697/resources/horizontalpodautoscalers.autoscaling/namespaces/myexample/php-apache-autoscaler.json ``` This proposal will have the path changed to something like ```bash /var/folders/zj/vc4ln5h14djg9svz7x_t1d0r0000gq/T/620385697/resources/horizontalpodautoscalers.autoscaling/v2beta2/namespaces/myexample/php-apache-autoscaler.json ``` The `horizontalpodautoscalers.autoscaling` part of the path will be updated to `horizontalpodautoscalers.autoscaling/v2beta2` using ```go version, ok := ctx.chosenGVsToRestore[groupResource.String()] if ok { resource = filepath.Join(groupResource.String(), version.VerDir) } ``` The restore can now proceed as normal. ## Alternatives Considered - Look for plugins if no common supported API group version could be found between the target and source clusters. We had considered searching for plugins that could handle converting an outdated resource to a new one that is supported in the target cluster, but it is difficult, will take a lot of time, and currently will not be useful because we are not aware of such plugins. It would be better to keep the initial changes simple to see how it works out and progress to more complex solutions as demand necessitates. - It was considered to modify the backed up json files such that the resources API versions are supported by the target but modifying backups is discouraged for several reasons, including introducing data corruption. ## Security Considerations I can't think of any additional risks in terms of Velero security here. ## Compatibility I have made it such that the changes in code will only affect Velero installations that have `APIGroupVersionsFeatureFlag` enabled during restore and Format Version 1.1.0 was used during backup. If both these requirements are not met, the changes will have no affect on the restore process, making the changes here entirely backward compatible. ## Implementation This first draft of the proposal will be submitted Oct. 30, 2020. Once this proposal is approved, I can have the code and unit tests written within a week and submit a PR that fixes Issue #2551. ## Open Issues At the time of writing this design proposal, I had not seen any of @jenting's work for solving Issue #2551. He had independently covered the first two priorities I mentioned above before I was even aware of the issue. I hope to not let his efforts go to waste and welcome incorporating his ideas here to make this design proposal better. ================================================ FILE: design/Implemented/retry-patching-configuration_design.md ================================================ # Backup Restore Status Patch Retrying Configuration ## Abstract When a backup/restore completes, we want to ensure that the custom resource progresses to the correct status. If a patch call fails to update status to completion, it should be retried up to a certain time limit. This design proposes a way to configure timeout for this retry time limit. ## Background Original Issue: https://github.com/vmware-tanzu/velero/issues/7207 Velero was performing a restore when the API server was rolling out to a new version. It had trouble connecting to the API server, but eventually, the restore was successful. However, since the API server was still in the middle of rolling out, Velero failed to update the restore CR status and gave up. After the connection was restored, it didn't attempt to update, causing the restore CR to be stuck at "In progress" indefinitely. This can lead to incorrect decisions for other components that rely on the backup/restore CR status to determine completion. ## Goals - Make timeout configurable for retry patching by reusing existing [`--resource-timeout` server flag](https://github.com/vmware-tanzu/velero/blob/d9ca14747925630664c9e4f85a682b5fc356806d/pkg/cmd/server/server.go#L245) ## Non Goals - Create a new timeout flag - Refactor backup/restore workflow ## High-Level Design We will add retries with timeout to existing patch calls that moves a backup/restore from InProgress to a different status phase such as - FailedValidation (final) - Failed (final) - WaitingForPluginOperations - WaitingForPluginOperationsPartiallyFailed - Finalizing - FinalizingPartiallyFailed and from above non final phases to - Completed - PartiallyFailed Once backup/restore is in some phase it will already be reconciled again periodically and do not need additional retry - WaitingForPluginOperations - WaitingForPluginOperationsPartiallyFailed ## Detailed Design Relevant reconcilers will have `resourceTimeout time.Duration` added to its struct and to parameters of New[Backup|Restore]XReconciler functions. pkg/cmd/server/server.go in `func (s *server) runControllers(..) error` also update the New[Backup|Restore]XCReconciler with added duration parameters using value from existing `--resource-timeout` server flag. Current calls to kube.PatchResource involving status patch will be replaced with kube.PatchResourceWithRetriesOnErrors added to package `kube` below. Calls where there is a ...client.Patch() will be wrapped with client.RetriesPhasePatchFuncOnErrors() added to package `client` below. pkg/util/kube/client.go ```go // PatchResourceWithRetries patches the original resource with the updated resource, retrying when the provided retriable function returns true. func PatchResourceWithRetries(maxDuration time.Duration, original, updated client.Object, kbClient client.Client, retriable func(error) bool) error { return veleroPkgClient.RetryOnRetriableMaxBackOff(maxDuration, func() error { return PatchResource(original, updated, kbClient) }, retriable) } // PatchResourceWithRetriesOnErrors patches the original resource with the updated resource, retrying when the operation returns an error. func PatchResourceWithRetriesOnErrors(maxDuration time.Duration, original, updated client.Object, kbClient client.Client) error { return PatchResourceWithRetries(maxDuration, original, updated, kbClient, func(err error) bool { // retry using DefaultBackoff to resolve connection refused error that may occur when the server is under heavy load // TODO: consider using a more specific error type to retry, for now, we retry on all errors // specific errors: // - connection refused: https://pkg.go.dev/syscall#:~:text=Errno(0x67)-,ECONNREFUSED,-%3D%20Errno(0x6f return err != nil }) } ``` pkg/client/retry.go ```go // CapBackoff provides a backoff with a set backoff cap func CapBackoff(cap time.Duration) wait.Backoff { if cap < 0 { cap = 0 } return wait.Backoff{ Steps: math.MaxInt, Duration: 10 * time.Millisecond, Cap: cap, Factor: retry.DefaultBackoff.Factor, Jitter: retry.DefaultBackoff.Jitter, } } // RetryOnRetriableMaxBackOff accepts a patch function param, retrying when the provided retriable function returns true. func RetryOnRetriableMaxBackOff(maxDuration time.Duration, fn func() error, retriable func(error) bool) error { return retry.OnError(CapBackoff(maxDuration), func(err error) bool { return retriable(err) }, fn) } // RetryOnErrorMaxBackOff accepts a patch function param, retrying when the error is not nil. func RetryOnErrorMaxBackOff(maxDuration time.Duration, fn func() error) error { return RetryOnRetriableMaxBackOff(maxDuration, fn, func(err error) bool { return err != nil }) } ``` ## Alternatives Considered - Requeuing InProgress backups that is not known by current velero instance to still be in progress as failed (attempted in [#7863](https://github.com/vmware-tanzu/velero/pull/7863)) - It was deemed as making backup restore flow hard to enhance for future reconciler updates such as adding cancel or adding parallel backups. ## Security Considerations None ## Compatibility Retry should only trigger a restore or backup that is already in progress and not patching successfully by current instance. Prior InProgress backups/restores will not be re-processed and will remain stuck InProgress until there is another velero server (re)start. ## Implementation There is a past implementation in [#7845](https://github.com/vmware-tanzu/velero/pull/7845/) where implementation for this design will be based upon. ================================================ FILE: design/Implemented/riav2-design.md ================================================ # Design for RestoreItemAction v2 API ## Abstract This design includes the changes to the RestoreItemAction (RIA) api design as required by the [Item Action Progress Monitoring](general-progress-monitoring.md) feature. It also includes changes as required by the [Wait For Additional Items](wait-for-additional-items.md) feature. The BIA v2 interface will have three new methods, and the RestoreItemActionExecuteOutput() struct in the return from Execute() will have three optional fields added. If there are any additional RIA API changes that are needed in the same Velero release cycle as this change, those can be added here as well. ## Background This API change is needed to facilitate long-running plugin actions that may not be complete when the Execute() method returns. It is an optional feature, so plugins which don't need this feature can simply return an empty operation ID and the new methods can be no-ops. This will allow long-running plugin actions to continue in the background while Velero moves on to the next plugin, the next item, etc. The other change allows Velero to wait until newly-restored AdditionalItems returned by a RIA plugin are ready before moving on to restoring the current item. ## Goals - Allow for RIA Execute() to optionally initiate a long-running operation and report on operation status. - Allow for RIA to allow Velero to call back into the plugin to wait until AdditionalItems are ready before continuing with restore. ## Non Goals - Allowing velero control over when the long-running operation begins. ## High-Level Design As per the [Plugin Versioning](plugin-versioning.md) design, a new RIAv2 plugin `.proto` file will be created to define the GRPC interface. v2 go files will also be created in `plugin/clientmgmt/restoreitemaction` and `plugin/framework/restoreitemaction`, and a new PluginKind will be created. Changes to RestoreItemActionExecuteOutput will be made to the existing struct. Since the new fields are optional elements of the struct, the new enlarged struct will work with both v1 and v2 plugins. The velero Restore process will be modified to reference v2 plugins instead of v1 plugins. An adapter will be created so that any existing RIA v1 plugin can be executed as a v2 plugin when executing a restore. ## Detailed Design ### proto changes (compiled into golang by protoc) The v2 RestoreItemAction.proto will be like the current v1 version with the following changes: RestoreItemActionExecuteOutput gets three new fields (defined in the current (v1) RestoreItemAction.proto file: ``` message RestoreItemActionExecuteResponse { bytes item = 1; repeated ResourceIdentifier additionalItems = 2; bool skipRestore = 3; string operationID = 4; bool waitForAdditionalItems = 5; google.protobuf.Duration additionalItemsReadyTimeout = 6; } ``` The RestoreItemAction service gets three new rpc methods: ``` service RestoreItemAction { rpc AppliesTo(RestoreItemActionAppliesToRequest) returns (RestoreItemActionAppliesToResponse); rpc Execute(RestoreItemActionExecuteRequest) returns (RestoreItemActionExecuteResponse); rpc Progress(RestoreItemActionProgressRequest) returns (RestoreItemActionProgressResponse); rpc Cancel(RestoreItemActionCancelRequest) returns (google.protobuf.Empty); rpc AreAdditionalItemsReady(RestoreItemActionItemsReadyRequest) returns (RestoreItemActionItemsReadyResponse); } ``` To support these new rpc methods, we define new request/response message types: ``` message RestoreItemActionProgressRequest { string plugin = 1; string operationID = 2; bytes restore = 3; } message RestoreItemActionProgressResponse { generated.OperationProgress progress = 1; } message RestoreItemActionCancelRequest { string plugin = 1; string operationID = 2; bytes restore = 3; } message RestoreItemActionItemsReadyRequest { string plugin = 1; bytes restore = 2; repeated ResourceIdentifier additionalItems = 3; } message RestoreItemActionItemsReadyResponse { bool ready = 1; } ``` One new shared message type will be needed, as defined in the v2 BackupItemAction design: ``` message OperationProgress { bool completed = 1; string err = 2; int64 completed = 3; int64 total = 4; string operationUnits = 5; string description = 6; google.protobuf.Timestamp started = 7; google.protobuf.Timestamp updated = 8; } ``` In addition to the three new rpc methods added to the RestoreItemAction interface, there is also a new `Name()` method. This one is only actually used internally by Velero to get the name that the plugin was registered with, but it still must be defined in a plugin which implements RestoreItemActionV2 in order to implement the interface. It doesn't really matter what it returns, though, as this particular method is not delegated to the plugin via RPC calls. The new (and modified) interface methods for `RestoreItemAction` are as follows: ``` type BackupItemAction interface { ... Name() string ... Progress(operationID string, restore *api.Restore) (velero.OperationProgress, error) Cancel(operationID string, backup *api.Restore) error AreAdditionalItemsReady(AdditionalItems []velero.ResourceIdentifier, restore *api.Restore) (bool, error) ... } type RestoreItemActionExecuteOutput struct { UpdatedItem runtime.Unstructured AdditionalItems []ResourceIdentifier SkipRestore bool OperationID string WaitForAdditionalItems bool } ``` A new PluginKind, `RestoreItemActionV2`, will be created, and the restore process will be modified to use this plugin kind. See [Plugin Versioning](plugin-versioning.md) for more details on implementation plans, including v1 adapters, etc. ## Compatibility The included v1 adapter will allow any existing RestoreItemAction plugin to work as expected, with no-op AreAdditionalItemsReady(), Progress(), and Cancel() methods. ## Implementation This will be implemented during the Velero 1.11 development cycle. ================================================ FILE: design/Implemented/schedule-skip-immediately-config_design.md ================================================ # Schedule Skip Immediately Config Design ## Abstract When unpausing schedule, a backup could be due immediately. New Schedules also create new backup immediately. This design allows user to *skip **immediately due** backup run upon unpausing or schedule creation*. ## Background Currently, the default behavior of schedule when `.Status.LastBackup` is nil or is due immediately after unpausing, a backup will be created. This may not be a desired by all users (https://github.com/vmware-tanzu/velero/issues/6517) User want ability to skip the first immediately due backup when schedule is unpaused and or created. If you create a schedule with cron "45 * * * *" and pause it at say the 43rd minute and then unpause it at say 50th minute, a backup gets triggered (since .Status.LastBackup is nil or >60min ago). With this design, user can skip the first immediately due backup when schedule is unpaused and or created. ## Goals - Add an option so user can when unpausing (when immediately due) or creating new schedule, to not create a backup immediately. ## Non Goals - Changing the default behavior ## High-Level Design Add a new field with to the schedule spec and as a new cli flags for install, server, schedule commands; allowing user to skip immediately due backup when unpausing or schedule creation. If CLI flag is specified during schedule unpause, velero will update the schedule spec accordingly and override prior spec for `skipImmediately``. ## Detailed Design ### CLI Changes `velero schedule unpause` will now take an optional bool flag `--skip-immediately` to allow user to override the behavior configured for velero server (see `velero server` below). `velero schedule unpause schedule-1 --skip-immediately=false` will unpause the schedule but not skip the backup if due immediately from `Schedule.Status.LastBackup` timestamp. Backup will be run at the next cron schedule. `velero schedule unpause schedule-1 --skip-immediately=true` will unpause the schedule and skip the backup if due immediately from `Schedule.Status.LastBackup` timestamp. Backup will also be run at the next cron schedule. `velero schedule unpause schedule-1` will check `.spec.SkipImmediately` in the schedule to determine behavior. This field will default to false to maintain prior behavior. `velero server` will add a new flag `--schedule-skip-immediately` to configure default value to patch new schedules created without the field. This flag will default to false to maintain prior behavior if not set. `velero install` will add a new flag `--schedule-skip-immediately` to configure default value to patch new schedules created without the field. This flag will default to false to maintain prior behavior if not set. ### API Changes `pkg/apis/velero/v1/schedule_types.go` ```diff // ScheduleSpec defines the specification for a Velero schedule type ScheduleSpec struct { // Template is the definition of the Backup to be run // on the provided schedule Template BackupSpec `json:"template"` // Schedule is a Cron expression defining when to run // the Backup. Schedule string `json:"schedule"` // UseOwnerReferencesBackup specifies whether to use // OwnerReferences on backups created by this Schedule. // +optional // +nullable UseOwnerReferencesInBackup *bool `json:"useOwnerReferencesInBackup,omitempty"` // Paused specifies whether the schedule is paused or not // +optional Paused bool `json:"paused,omitempty"` + // SkipImmediately specifies whether to skip backup if schedule is due immediately from `Schedule.Status.LastBackup` timestamp when schedule is unpaused or if schedule is new. + // If true, backup will be skipped immediately when schedule is unpaused if it is due based on .Status.LastBackupTimestamp or schedule is new, and will run at next schedule time. + // If false, backup will not be skipped immediately when schedule is unpaused, but will run at next schedule time. + // If empty, will follow server configuration (default: false). + // +optional + SkipImmediately bool `json:"skipImmediately,omitempty"` } ``` **Note:** The Velero server automatically patches the `skipImmediately` field back to `false` after it's been used. This is because `skipImmediately` is designed to be a one-time operation rather than a persistent state. When the controller detects that `skipImmediately` is set to `true`, it: 1. Sets the flag back to `false` 2. Records the current time in `schedule.Status.LastSkipped` This "consume and reset" pattern ensures that after skipping one immediate backup, the schedule returns to normal behavior for subsequent runs. The `LastSkipped` timestamp is then used to determine when the next backup should run. ```go // From pkg/controller/schedule_controller.go if schedule.Spec.SkipImmediately != nil && *schedule.Spec.SkipImmediately { *schedule.Spec.SkipImmediately = false schedule.Status.LastSkipped = &metav1.Time{Time: c.clock.Now()} } ``` `LastSkipped` will be added to `ScheduleStatus` struct to track the last time a schedule was skipped. ```diff // ScheduleStatus captures the current state of a Velero schedule type ScheduleStatus struct { // Phase is the current phase of the Schedule // +optional Phase SchedulePhase `json:"phase,omitempty"` // LastBackup is the last time a Backup was run for this // Schedule schedule // +optional // +nullable LastBackup *metav1.Time `json:"lastBackup,omitempty"` + // LastSkipped is the last time a Schedule was skipped + // +optional + // +nullable + LastSkipped *metav1.Time `json:"lastSkipped,omitempty"` // ValidationErrors is a slice of all validation errors (if // applicable) // +optional ValidationErrors []string `json:"validationErrors,omitempty"` } ``` The `LastSkipped` field is crucial for the schedule controller to determine the next run time. When a backup is skipped, this timestamp is used instead of `LastBackup` to calculate when the next backup should occur, ensuring the schedule maintains its intended cadence even after skipping a backup. When `schedule.spec.SkipImmediately` is `true`, `LastSkipped` will be set to the current time, and `schedule.spec.SkipImmediately` set to nil so it can be used again. The `getNextRunTime()` function below is updated so `LastSkipped` which is after `LastBackup` will be used to determine next run time. ```go func getNextRunTime(schedule *velerov1.Schedule, cronSchedule cron.Schedule, asOf time.Time) (bool, time.Time) { var lastBackupTime time.Time if schedule.Status.LastBackup != nil { lastBackupTime = schedule.Status.LastBackup.Time } else { lastBackupTime = schedule.CreationTimestamp.Time } if schedule.Status.LastSkipped != nil && schedule.Status.LastSkipped.After(lastBackupTime) { lastBackupTime = schedule.Status.LastSkipped.Time } nextRunTime := cronSchedule.Next(lastBackupTime) return asOf.After(nextRunTime), nextRunTime } ``` When schedule is unpaused, and `Schedule.Status.LastBackup` is not nil, if `Schedule.Status.LastSkipped` is recent, a backup will not be created. When schedule is unpaused or created with `Schedule.Status.LastBackup` set to nil or schedule is newly created, normally a backup will be created immediately. If `Schedule.Status.LastSkipped` is recent, a backup will not be created. Backup will be run at the next cron schedule based on LastBackup or LastSkipped whichever is more recent. ## Alternatives Considered N/A ## Security Considerations None ## Compatibility Upon upgrade, the new field will be added to the schedule spec automatically and will default to the prior behavior of running a backup when schedule is unpaused if it is due based on .Status.LastBackup or schedule is new. Since this is a new field, it will be ignored by older versions of velero. ## Implementation TBD ## Open Issues N/A ================================================ FILE: design/Implemented/secrets.md ================================================ # Support for multiple provider credentials Currently, Velero only supports a single credential secret per location provider/plugin. Velero creates and stores the plugin credential secret under the hard-coded key `secret.cloud-credentials.data.cloud`. This makes it so switching from one plugin to another necessitates overriding the existing credential secret with the appropriate one for the new plugin. ## Goals - To allow Velero to create and store multiple secrets for provider credentials, even multiple credentials for the same provider - To improve the UX for configuring the velero deployment with multiple plugins/providers. - Enable use cases such as AWS volume snapshots w/Minio as the object storage - Continue to support use cases where multiple Backup Storage Locations are in use simultaneously - `velero backup logs` while backup/restore is running - Handle changes in configuration while operations are happening as well as they currently are ## Non Goals - To make any change except what's necessary to handle multiple credentials - To allow multiple credentials for or change the UX for node-based authentication (e.g. AWS IAM, GCP Workload Identity, Azure AAD Pod Identity). Node-based authentication will not allow cases such as a mix of AWS snapshots with Minio object storage. ## Design overview Instead of one credential per Velero deployment, multiple credentials can be added and used with different BSLs. There are two aspects to handling multiple credentials: - Modifying how credentials are configured and specified by the user - Modifying how credentials are provided to the plugin processes Each of these aspects will be discussed in turn. ### Credential configuration Currently, Velero creates a secret (`cloud-credentials`) during install with a single entry that contains the contents of the credentials file passed by the user. Instead of adding new CLI options to Velero to create and manage credentials, users will create their own Kubernetes secrets within the Velero namespace and reference these. This approach is being chosen as it allows users to directly manage Kubernetes secrets objects as they wish and it removes the need for wrapper functions to be created within Velero to manage the creation of secrets. Separate credentials rather than combining credentials in a single secret also avoids issues with maximum size of credentials as well as update in place issues. To enable the use of existing Kubernetes secrets, BSLs will be modified to have a new field `Credential`. This field will be a [`SecretKeySelector`](https://godoc.org/k8s.io/api/core/v1#SecretKeySelector) which will enable the user to specify which key within a particular secret the BSL should use. Existing BackupStorageLocationSpec definition: // BackupStorageLocationSpec defines the desired state of a Velero BackupStorageLocation type BackupStorageLocationSpec struct { // Provider is the provider of the backup storage. Provider string `json:"provider"` // Config is for provider-specific configuration fields. // +optional Config map[string]string `json:"config,omitempty"` StorageType `json:",inline"` // Default indicates this location is the default backup storage location. // +optional Default bool `json:"default,omitempty"` // AccessMode defines the permissions for the backup storage location. // +optional AccessMode BackupStorageLocationAccessMode `json:"accessMode,omitempty"` // BackupSyncPeriod defines how frequently to sync backup API objects from object storage. A value of 0 disables sync. // +optional // +nullable BackupSyncPeriod *metav1.Duration `json:"backupSyncPeriod,omitempty"` // ValidationFrequency defines how frequently to validate the corresponding object storage. A value of 0 disables validation. // +optional // +nullable ValidationFrequency *metav1.Duration `json:"validationFrequency,omitempty"` } The following field will be added: Credential *corev1api.SecretKeySelector `json:"credential,omitempty"` The resulting BackupStorageLocationSpec will be this: // BackupStorageLocationSpec defines the desired state of a Velero BackupStorageLocation type BackupStorageLocationSpec struct { // Provider is the provider of the backup storage. Provider string `json:"provider"` // Config is for provider-specific configuration fields. // +optional Config map[string]string `json:"config,omitempty"` // Credential contains the credential information intended to be used with this location // +optional Credential *corev1api.SecretKeySelector `json:"credential,omitempty"` StorageType `json:",inline"` // Default indicates this location is the default backup storage location. // +optional Default bool `json:"default,omitempty"` // AccessMode defines the permissions for the backup storage location. // +optional AccessMode BackupStorageLocationAccessMode `json:"accessMode,omitempty"` // BackupSyncPeriod defines how frequently to sync backup API objects from object storage. A value of 0 disables sync. // +optional // +nullable BackupSyncPeriod *metav1.Duration `json:"backupSyncPeriod,omitempty"` // ValidationFrequency defines how frequently to validate the corresponding object storage. A value of 0 disables validation. // +optional // +nullable ValidationFrequency *metav1.Duration `json:"validationFrequency,omitempty"` } The CLI for managing Backup Storage Locations (BSLs) will be modified to allow the user to set these credentials. Both `velero backup-location (create|set)` will have a new flag (`--credential`) to specify the secret and key within the secret to use. This flag will take a key-value pair in the format `=`. The arguments will be validated to ensure that the secret exists in the Velero namespace. If the Credential field is empty in a BSL, the default credentials from `cloud-credentials` will be used as they are currently. ### Making credentials available to plugins The approach we have chosen is to include the path to the credentials file in the `config` map passed to a plugin. ### Including the credentials file path in the `config` map Prior to using any secret for a BSL, it will need to be serialized to disk. Using the details in the `Credential` field in the BSL, the contents of the Secret will be read and serialized. To achieve this, we will create a new package, `credentials`, which will introduce new types and functions to manage the fetching of credentials based on a `SecretKeySelector`. This will also be responsible for serializing the fetched credentials to a temporary directory on the Velero pod filesystem. The path where a set of credentials will be written to will be a fixed path based on the namespace, name, and key from the secret rather than a randomly named file as is usual with temporary files. The reason for this is that `BackupStore`s are frequently created within the controllers and the credentials must be serialized before any plugin APIs are called, which would result in a quick accumulation of temporary credentials files. For example, the default validation frequency for BackupStorageLocations is one minute. This means that any time a `BackupStore`, or other type which requires credentials, is created, the credentials will be fetched from the API server and may overwrite any existing use of that credential. If we instead wanted to use an unique file each time, we could work around the of multiple files being written by cleaning up the temporary files upon completion of the plugin operations, if this information is known. Once the credentials have been serialized, this path will be made available to the plugins. Instead of setting the necessary environment variable for the plugin process, the `config` map for the BSL will be modified to include an addiitional entry with the path to the credentials file: `credentialsFile`. This will be passed through when [initializing the BSL](https://github.com/vmware-tanzu/velero/blob/main/pkg/plugin/velero/object_store.go#L27-L30) and it will be the responsibility of the plugin to use the passed credentials when starting a session. For an example of how this would affect the AWS plugin, see [this PR](https://github.com/vmware-tanzu/velero-plugin-for-aws/pull/69). The restic controllers will also need to be updated to use the correct credentials. The BackupStorageLocation for a given PVB/PVR will be fetched and the `Credential` field from that BSL will be serialized. The existing setup for the restic commands use the credentials from the environment variables with [some repo provider specific overrides](https://github.com/vmware-tanzu/velero/blob/main/pkg/controller/pod_volume_backup_controller.go#L260-L273). Instead of relying on the existing environment variables, if there are credentials for a particular BSL, the environment will be specifically created for each `RepoIdentifier`. This will use a lot of the existing logic with the exception that it will be modified to work with a serialized secret rather than find the secret file from an environment variable. Currently, GCP is the only provider that relies on the existing environment variables with no specific overrides. For GCP, the environment variable will be overwritten with the path of the serialized secret. ## Split credentials between VolumeSnapshotter and ObjectStore plugins One of the use cases we wish to satisfy is the ability to specify a different object store than the cloud provider offers, for example, using a Minio S3 object store from within AWS. Currently the VolumeSnapshotter and the ObjectStore plugin share the cloud credentials. Each backup/restore has a BackupStorageLocation associated with it. The BackupStorageLocation can optionally specify the credential used by the ObjectStorePlugin and Restic daemons while the cloud credential will always be used for the VolumeSnapshotter. ## Velero Plugin for vSphere compatibility The vSphere plugin is implemented as a BackupItemAction and shares the credentials of the AWS plugin for S3 access. The backup storage location is passed in _Backup.Spec.StorageLocation_. Currently the plugin retrieves the S3 bucket and server from the BSL and creates a BackupRespositoryClaim with that and the credentials retrieved from the cloud credential. The plugin will need to be modified to retrieve the credentials field from the BSL and use that credential in the BackupRepositoryClaim. ## Backwards compatibility For now, regardless of the approaches used above, we will still support the existing workflow. Users will be able to set credentials during install and a secret will be created for them. This secret will still be mounted into the Velero pods and the appropriate environment variables set. This will allow users to use versions of plugins which haven't yet been updated to use credentials directly, such as with many community created plugins. Multiple credential handling will only be used in the case where a particular BSL has been modified to use an existing secret. ## Security Considerations Although the handling of secrets will be similar to how credentials are currently managed within Velero, care must be taken to ensure that any new code does not leak the contents of secrets, for example, including them within logs. ## Parallelism In order to support parallelism, Velero will need to be able to use multiple credentials simultaneously with the ObjectStore. Currently backups are single threaded and a single BSL will be used throughout the entire backup. The only existing points of parallelism are when a user downloads logs for a backup or the BackupStorageLocationReconciler reconciles while a backup or restore is running. In the current code, `download_request_controller.go` and `backup_storage_location_controller.go` create a new plugin manager and hence another ObjectStore plugin in parallel with the ObjectStore plugin servicing a backup or restore (if one is running). ## Alternatives Considered Three different approaches can be taken to provide credentials to plugin processes: 1. Providing the path to the credentials file as an environment variable per plugin. This is how credentials are currently passed. 1. Include the path to the credentials file in the `config` map passed to a plugin. 1. Include the details of the secret in the `config` map passed to a plugin. The last two options require changes to the plugin as the plugin will need to instantiate a client using the provided credentials. The client libraries used by the plugins will not be able to rely on the credentials details being available in the environment as they currently do. We have selected option 2 as the approach to take. The approaches that were not selected are detailed below for reference. #### Providing the credentials via environment variables To continue to provide the credentials via the environment, plugins will need to be invoked differently so that the correct credential is used. Currently, there is a single secret, which is mounted into every pod deployed by Velero (the Velero Deployment and the Restic DaemonSet) at the path `/credentials/cloud`. This path is made known to all plugins through provider specific environment variables and all possible provider environment variables are set to this path. Instead of setting the environment variables for all the pods, we can modify plugin processes are created so that the environment variables are set on a per plugin process basis. Prior to using any secret for a BSL, it will need to be serialized to disk. Using the details in the `Credential` field in the BSL, the contents of the Secret will be read and serialized to a file. Each plugin process would still have the same set of environment variables set, however the value used for each of these variables would instead be the path to the serialized secret. To set the environment variables for a plugin process, the plugin manager must be modified so that when creating an ObjectStore, we pass in the entire BSL object, rather than [just the provider](https://github.com/vmware-tanzu/velero/blob/main/pkg/plugin/clientmgmt/manager.go#L132-L158). The plugin manager currently stores a map of [plugin executables to an associated `RestartableProcess`](https://github.com/vmware-tanzu/velero/blob/main/pkg/plugin/clientmgmt/manager.go#L59-L70). New restartable processes are created only [with the executable that the process would run](https://github.com/vmware-tanzu/velero/blob/main/pkg/plugin/clientmgmt/manager.go#L122). This could be modified to also take the necessary environment variables so that when [underlying go-plugin process is created](https://github.com/vmware-tanzu/velero/blob/main/pkg/plugin/clientmgmt/client_builder.go#L78), these environment variables could be provided and would be set on the plugin process. Taking this approach would not require any changes from plugins as the credentials information would be made available to them in the same way. However, it is quite a significant change in how we initialize and invoke plugins. We would also need to ensure that the restic controllers are updated in the same way so that correct credentials are used (when creating a `ResticRepository` or processing `PodVolumeBackup`/`PodVolumeRestore`). This could be achieved by modifying the existing function to [run a restic command](https://github.com/vmware-tanzu/velero/blob/main/pkg/restic/repository_manager.go#L237-L290). This function already sets environment variables for the restic process depending on which storage provider is being used. #### Include the details of the secret in `config` map passed to a plugin This approach is like the selected approach of passing the credentials file via the `config` map, however instead of the Velero process being responsible for serializing the file to disk prior to invoking the plugin, the `Credential SecretKeySelector` details will be passed through to the plugin. It will be the responsibility of the plugin to fetch the secret from the Kubernetes API and perform the necessary steps to make it available for use when creating a session, for example, serializing the contents to disk, or evaluating the contents and adding to the process environment. This approach has an additional burden on the plugin author over the previous approach as it requires the author to create a client to communicate with the Kubernetes API to retrieve the secret. Although it would be the responsibility of the plugin to serialize the credential and use it directly, Velero would still be responsible for serializing the secret so that it could be used with the restic controllers as in the selected approach. ================================================ FILE: design/Implemented/supporting-volumeattributes-resource-policy.md ================================================ # Adding Support For VolumeAttributes in Resource Policy ## Abstract Currently [Velero Resource policies](https://velero.io/docs/main/resource-filtering/#creating-resource-policies) are only supporting "Driver" to be filtered for [CSI volume conditions](https://github.com/vmware-tanzu/velero/blob/8e23752a6ea83f101bd94a69dcf17f519a805388/internal/resourcepolicies/volume_resources_validator.go#L28) If user want to skip certain CSI volumes based on other volume attributes like protocol or SKU, etc, they can't do it with the current Velero resource policies. It would be convenient if Velero resource policies could be extended to filter on volume attributes along with existing driver filter in the resource policies `conditions` to handle the backup of volumes just by `some specific volumes attributes conditions`. ## Background As of Today, Velero resource policy already provides us the way to filter volumes based on the `driver` name. But it's not enough to handle the volumes based on other volume attributes like protocol, SKU, etc. ## Example: - Provision Azure NFS: Define the Storage class with `protocol: nfs` under storage class parameters to provision [CSI NFS Azure File Shares](https://learn.microsoft.com/en-us/azure/aks/azure-files-csi#nfs-file-shares). - User wants to back up AFS (Azure file shares) but only want to backup `SMB` type of file share volumes and not `NFS` file share volumes. ## Goals - We are only bringing additional support in the resource policy to only handle volumes during backup. - Introducing support for `VolumeAttributes` filter along with `driver` filter in CSI volume conditions to handle volumes. ## Non-Goals - Currently, only handles volumes, and does not support other resources. ## Use-cases/Scenarios ### Skip backup volumes by some volume attributes: Users want to skip PV with the requirements: - option to skip specified PV on volume attributes type (like Protocol as NFS, SMB, etc) ### Sample Storage Class Used to create such Volumes ``` apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: azurefile-csi-nfs provisioner: file.csi.azure.com allowVolumeExpansion: true parameters: protocol: nfs ``` ## High-Level Design Modifying the existing Resource Policies code for [csiVolumeSource](https://github.com/vmware-tanzu/velero/blob/8e23752a6ea83f101bd94a69dcf17f519a805388/internal/resourcepolicies/volume_resources_validator.go#L28C6-L28C22) to add the new `VolumeAttributes` filter for CSI volumes and adding validations in existing [csiCondition](https://github.com/vmware-tanzu/velero/blob/8e23752a6ea83f101bd94a69dcf17f519a805388/internal/resourcepolicies/volume_resources.go#L150) to match with volume attributes in the conditions from Resource Policy config map and original persistent volume. ## Detailed Design The volume resources policies should contain a list of policies which is the combination of conditions and related `action`, when target volumes meet the conditions, the related `action` will take effection. Below is the API Design for the user configuration: ### API Design ```go type csiVolumeSource struct { Driver string `yaml:"driver,omitempty"` // [NEW] CSI volume attributes VolumeAttributes map[string]string `yaml:"volumeAttributes,omitempty"` } ``` The policies YAML config file would look like this: ```yaml version: v1 volumePolicies: - conditions: csi: driver: disk.csi.azure.com action: type: skip - conditions: csi: driver: file.csi.azure.com volumeAttributes: protocol: nfs action: type: skip` ``` ### New Supported Conditions #### VolumeAttributes Existing CSI Volume Condition can now add `volumeAttributes` which will be key and value pairs. Specify details for the related volume source (currently only csi driver is supported filter) ```yaml csi: // match volume using `file.csi.azure.com` and with volumeAttributes protocol as nfs driver: file.csi.azure.com volumeAttributes: protocol: nfs ``` ================================================ FILE: design/Implemented/unified-repo-and-kopia-integration/unified-repo-and-kopia-integration.md ================================================ # Unified Repository & Kopia Integration Design ## Glossary & Abbreviation **BR**: Backup & Restore **Backup Storage**: The storage that meets BR requirements, for example, scalable, durable, cost-effective, etc., therefore, Backup Storage is usually implemented as Object storage or File System storage, it may be on-premises or in cloud. Backup Storage is not BR specific necessarily, so it usually doesn’t provide most of the BR related features. On the other hand, storage vendors may provide BR specific storages that include some BR features like deduplication, compression, encryption, etc. For a standalone BR solution (i.e. Velero), the Backup Storage is not part of the solution, it is provided by users, so the BR solution should not assume the BR related features are always available from the Backup Storage. **Backup Repository**: Backup repository is layered between BR data movers and Backup Storage to provide BR related features. Backup Repository is a part of BR solution, so generally, BR solution by default leverages the Backup Repository to provide the features because Backup Repository is always available; when Backup Storage provides duplicated features, and the latter is more beneficial (i.e., performance is better), BR solution should have the ability to opt to use the Backup Storage’s implementation. **Data Mover**: The BR module to read/write data from/to workloads, the aim is to eliminate the differences of workloads. **TCO**: Total Cost of Ownership. This is a general criteria for products/solutions, but also means a lot for BR solutions. For example, this means what kind of backup storage (and its cost) it requires, the retention policy of backup copies, the ways to remove backup data redundancy, etc. **RTO**: Recovery Time Objective. This is the duration of time that users’ business can recover after a disaster. ## Background As a Kubernetes BR solution, Velero is pursuing the capability to back up data from the volatile and limited production environment into the durable, heterogeneous and scalable backup storage. This relies on two parts: - Move data from various production workloads. The data mover has this role. Depending on the type of workload, Velero needs different data movers. For example, file system data mover, block data mover, and data movers for specific applications. At present, Velero supports moving file system data from PVs through Restic, which plays the role of the File System Data Mover. - Persist data in backup storage. For a BR solution, this is the responsibility of the backup repository. Specifically, the backup repository is required to: - Efficiently save data so as to reduce TCO. For example, deduplicate and compress the data before saving it - Securely save data so as to meet security criteria. For example, encrypt the data on rest, make the data immutable after backup, and detect/protect from ransomware - Efficiently retrieve data during restore so as to meet RTO. For example, restore a small unit of data or data associated with a small span of time - Effectively manage data from all kinds of data movers in all kinds of backup storage. This means 2 things: first, apparently, backup storages are different from each other; second, some data movers may save quite different data from others, for example, some data movers save a portion of the logical object for each backup and need to visit and manage the portions as an entire logic object, aka. incremental backup. The backup repository needs to provide unified functionalities to eliminate the differences from the both ends - Provide scalabilities so that users could assign resources (CPU, memory, network, etc.) in a flexible way to the backup repository since backup repository contains resource consuming modules At present, Velero provides some of these capabilities by leveraging Restic (e.g., deduplication and encryption on rest). This means that in addition to being a data mover for file system level data, Restic also plays the role of a backup repository, albeit one that is incomplete and limited: - Restic is an inseparable unit made up of a file system data mover and a repository. This means that the repository capabilities are only available for Restic file system backup. We cannot provide the same capabilities to other data movers using Restic. - The backup storage Velero supports through our Restic backup path depends on the storage Restic supports. As a result, if there is a requirement to introduce backup storage that Restic doesn’t support, we have no way to make it. - There is no way to enhance or extend the repository capabilities, because of the same reason – Restic is an inseparable unit, we cannot insert one or more customized layers to make the enhancements and extensions. Moreover, as reflected by user-reported issues, Restic seems to have many performance issues on both the file system data mover side and the repository side. On the other hand, based on a previous analysis and testing, we found that Kopia has better performance, with more features and more suitable to fulfill Velero’s repository targets (Kopia’s architecture divides modules more clearly according to their responsibilities, every module plays a complete role with clear interfaces. This makes it easier to take individual modules to Velero without losing critical functionalities). ## Goals - Define a Unified Repository Interface that various data movers could interact with. This is for below purposes: - All kinds of data movers acquire the same set of backup repository capabilities very easily - Provide the possibility to plugin in different backup repositories/backup storages without affecting the upper layers - Provide the possibility to plugin in modules between data mover and backup repository, so as to extend the repository capabilities - Provide the possibility to scale the backup repository without affecting the upper layers - Use Kopia repository to implement the Unified Repository - Use Kopia uploader as the file system data mover for Pod Volume Backup - Have Kopia uploader calling the Unified Repository Interface and save/retrieve data to/from the Unified Repository - Make Kopia uploader generic enough to move any file system data so that other data movement cases could use it - Use the existing logic or add new logic to manage the unified repository and Kopia uploader - Preserve the legacy Restic path, this is for the consideration of backward compatibility ## Non-Goals - The Unified Repository supports all kinds of data movers to save logic objects into it. How these logic objects are organized for a specific data mover (for example, how a volume’s block data is organized and represented by a unified repository object) should be included in the related data mover design. - At present, Velero saves Kubernetes resources, backup metedata, debug logs separately. Eventually, we want to save them in the Unified Repository. How to organize these data into the Unified Repository should be included in a separate design. - For PodVolume BR, this design focuses on the data path only, other parts beyond the data read/write and data persistency are irrelevant and kept unchanged. - Kopia uploader is made generic enough to move any file system data. How it is integrated in other cases, is irrelevant to this design. Take CSI snapshot backup for example, how the snapshot is taken and exposed to Kopia uploader should be included in the related data mover design. - The adanced modes of the Unified Repository, for example, backup repository/storage plugin, backup repository extension, etc. are not included in this design. We will have separate designs to cover them whenever necessary. ## Architecture of Unified Repository Below shows the primary modules and their responsibilities: - Kopia uploader, as been well isolated, could move all file system data either from the production PV (as Velero’s PodVolume BR does), or from any kind of snapshot (i.e., CSI snapshot). - Unified Repository Interface, data movers call the Unified Repository Interface to write/read data to/from the Unified Repository. - Kopia repository layers, CAOS and CABS, work as the backup repository and expose the Kopia Repository interface. - A Kopia Repository Library works as an adapter between Unified Repository Interface and Kopia Repository interface. Specifically, it implements Unified Repository Interface and calls Kopia Repository interface. - At present, there is only one kind of backup repository -- Kopia Repository. If a new backup repository/storage is required, we need to create a new Library as an adapter to the Unified Repository Interface - At present, the Kopia Repository works as a single piece in the same process of the caller, in future, we may run its CABS into a dedicated process or node. - At present, we don’t have a requirement to extend the backup repository, if needed, an extra module could be added as an upper layer into the Unified Repository without changing the data movers. Neither Kopia uploader nor Kopia Repository is invoked through CLI, instead, they are invoked through code interfaces, because we need to do lots of customizations. The Unified Repository takes two kinds of data: - Unified Repository Object: This is the user's logical data, for example, files/directories, blocks of a volume, data of a database, etc. - Unified Repository Manifest: This could include all other data to maintain the object data, for example, snapshot information, etc. For Unified Repository Object/Manifest, a brief guidance to data movers are as below: - Data movers treat the simple unit of data they recognize as an Object. For example, file system data movers treat a file or a directory as an Object; block data movers treat a volume as an Object. However, it is unnecessary that every data mover has a unique data format in the Unified Repository, to the opposite, it is recommended that data movers could share the data formats unless there is any reason not to, in this way, the data generated by one data mover could be used by other data movers. - Data movers don't need to care about the differences between full and incremental backups regarding the data organization. Data movers always have full views of their objects, if an object is partially written, they use the object writer's Seek function to skip the unchanged parts - Unified Repository may divide the data movers' logical Object into sub-objects or slices, or append internal metadata, but they are transparent to data movers - Every Object has an unified identifier, in order to retrieve the Object later, data movers need to save the identifiers into the snapshot information. The snapshot information is saved as a Manifest. - Manifests could hold any kind of small piece data in a K-V manner. Inside the backup repository, these kinds of data may be processed differently from Object data, but it is transparent to data movers. - A Manifest also has an unified identifier, the Unified Repository provides the capabilities to list all the Manifests or a specified Manifest by its identifier, or a specified Manifest by its name, or a set of Manifests by their labels. ![A Unified Repository Architecture](unified-repo.png) Velero by default uses the Unified Repository for all kinds of data movement, it is also able to integrate with other data movement paths from any party, for any purpose. Details are concluded as below: - Built-in Data Path: this is the default data movement path, which uses Velero built-in data movers to backup/restore workloads, the data is written to/read from the Unified Repository. - Data Mover Replacement: Any party could write its own data movers and plug them into Velero. Meanwhile, these plugin data movers could also write/read data to/from Velero’s Unified Repository so that these data movers could expose the same capabilities that provided by the Unified Repository. In order to do this, the data mover providers need to call the Unified Repository Interface from inside their plugin data movers. - Data Path Replacement: Some vendors may already have their own data movers and backup repository and they want to replace Velero’s entire data path (including data movers and backup repository). In this case, the providers only need to implement their plugin data movers, all the things downwards are a black box to Velero and managed by providers themselves (including API call, data transport, installation, life cycle management, etc.). Therefore, this case is out of the scope of Unified Repository. ![A Scope](scope.png) # Detailed Design ## The Unified Repository Interface Below are the definitions of the Unified Repository Interface. All the functions are synchronization functions. ``` // BackupRepoService is used to initialize, open or maintain a backup repository type BackupRepoService interface { // Init creates a backup repository or connect to an existing backup repository. // repoOption: option to the backup repository and the underlying backup storage. // createNew: indicates whether to create a new or connect to an existing backup repository. Init(ctx context.Context, repoOption RepoOptions, createNew bool) error // Open opens an backup repository that has been created/connected. // repoOption: options to open the backup repository and the underlying storage. Open(ctx context.Context, repoOption RepoOptions) (BackupRepo, error) // Maintain is periodically called to maintain the backup repository to eliminate redundant data. // repoOption: options to maintain the backup repository. Maintain(ctx context.Context, repoOption RepoOptions) error // DefaultMaintenanceFrequency returns the defgault frequency of maintenance, callers refer this // frequency to maintain the backup repository to get the best maintenance performance DefaultMaintenanceFrequency() time.Duration } // BackupRepo provides the access to the backup repository type BackupRepo interface { // OpenObject opens an existing object for read. // id: the object's unified identifier. OpenObject(ctx context.Context, id ID) (ObjectReader, error) // GetManifest gets a manifest data from the backup repository. GetManifest(ctx context.Context, id ID, mani *RepoManifest) error // FindManifests gets one or more manifest data that match the given labels FindManifests(ctx context.Context, filter ManifestFilter) ([]*ManifestEntryMetadata, error) // NewObjectWriter creates a new object and return the object's writer interface. // return: A unified identifier of the object on success. NewObjectWriter(ctx context.Context, opt ObjectWriteOptions) ObjectWriter // PutManifest saves a manifest object into the backup repository. PutManifest(ctx context.Context, mani RepoManifest) (ID, error) // DeleteManifest deletes a manifest object from the backup repository. DeleteManifest(ctx context.Context, id ID) error // Flush flushes all the backup repository data Flush(ctx context.Context) error // Time returns the local time of the backup repository. It may be different from the time of the caller Time() time.Time // Close closes the backup repository Close(ctx context.Context) error type ObjectReader interface { io.ReadCloser io.Seeker // Length returns the logical size of the object Length() int64 } type ObjectWriter interface { io.WriteCloser // Seeker is used in the cases that the object is not written sequentially io.Seeker // Checkpoint is periodically called to preserve the state of data written to the repo so far. // Checkpoint returns a unified identifier that represent the current state. // An empty ID could be returned on success if the backup repository doesn't support this. Checkpoint() (ID, error) // Result waits for the completion of the object write. // Result returns the object's unified identifier after the write completes. Result() (ID, error) } ``` Some data structure & constants used by the interfaces: ``` type RepoOptions struct { // StorageType is a repository specific string to identify a backup storage, i.e., "s3", "filesystem" StorageType string // RepoPassword is the backup repository's password, if any RepoPassword string // ConfigFilePath is a custom path to save the repository's configuration, if any ConfigFilePath string // GeneralOptions takes other repository specific options GeneralOptions map[string]string // StorageOptions takes storage specific options StorageOptions map[string]string // Description is a description of the backup repository/backup repository operation. // It is for logging/debugging purpose only and doesn't control any behavior of the backup repository. Description string } // ObjectWriteOptions defines the options when creating an object for write type ObjectWriteOptions struct { FullPath string // Full logical path of the object DataType int // OBJECT_DATA_TYPE_* Description string // A description of the object, could be empty Prefix ID // A prefix of the name used to save the object AccessMode int // OBJECT_DATA_ACCESS_* BackupMode int // OBJECT_DATA_BACKUP_* } const ( // Below consts descrbe the data type of one object. // Metadata: This type describes how the data is organized. // For a file system backup, the Metadata describes a Dir or File. // For a block backup, the Metadata describes a Disk and its incremental link. ObjectDataTypeUnknown int = 0 ObjectDataTypeMetadata int = 1 ObjectDataTypeData int = 2 // Below consts defines the access mode when creating an object for write ObjectDataAccessModeUnknown int = 0 ObjectDataAccessModeFile int = 1 ObjectDataAccessModeBlock int = 2 ObjectDataBackupModeUnknown int = 0 ObjectDataBackupModeFull int = 1 ObjectDataBackupModeInc int = 2 ) // ManifestEntryMetadata is the metadata describing one manifest data type ManifestEntryMetadata struct { ID ID // The ID of the manifest data Length int32 // The data size of the manifest data Labels map[string]string // Labels saved together with the manifest data ModTime time.Time // Modified time of the manifest data } type RepoManifest struct { Payload interface{} // The user data of manifest Metadata *ManifestEntryMetadata // The metadata data of manifest } type ManifestFilter struct { Labels map[string]string } ``` ## Workflow ### Backup & Restore Workflow We preserve the bone of the existing BR workflow, that is: - Still use the Velero Server pod and VeleroNodeAgent daemonSet (originally called Restic daemonset) pods to hold the corresponding controllers and modules - Still use the Backup/Restore CR and BackupRepository CR (originally called ResticRepository CR) to drive the BR workflow The modules in gray color in below diagram are the existing modules and with no significant changes. In the new design, we will have separate and independent modules/logics for backup repository and uploader (data mover), specifically: - Repository Provider provides functionalities to manage the backup repository. For example, initialize a repository, connect to a repository, manage the snapshots in the repository, maintain a repository, etc. - Uploader Provider provides functionalities to run a backup or restore. The Repository Provider and Uploader Provider use options to choose the path --- legacy path vs. new path (Kopia uploader + Unified Repository). Specifically, for legacy path, Repository Provider will manage Restic Repository only, otherwise, it manages Unified Repository only; for legacy path, Uploader Provider calls Restic to do the BR, otherwise, it calls Kopia uploader to do the BR. In order to manage Restic Repository, the Repository Provider calls Restic Repository Provider, the latter invokes the existing Restic CLIs. In order to manage Unified Repository, the Repository Provider calls Unified Repository Provider, the latter calls the Unified Repository module through the udmrepo.BackupRepoService interface. It doesn’t know how the Unified Repository is implemented necessarily. In order to use Restic to do BR, the Uploader Provider calls Restic Uploader Provider, the latter invokes the existing Restic CLIs. In order to use Kopia to do BR, the Uploader Provider calls Kopia Uploader Provider, the latter do the following things: - Call Unified Repository through the udmrepo.BackupRepoService interface to open the unified repository for read/write. Again, it doesn’t know how the Unified Repository is implemented necessarily. It gets a BackupRepo’s read/write handle after the call succeeds - Wrap the BackupRepo handle into a Kopia Shim which implements Kopia Repository interface - Call the Kopia Uploader. Kopia Uploader is a Kopia module without any change, so it only understands Kopia Repository interface - Kopia Uploader starts to backup/restore the corresponding PV’s file system data and write/read data to/from the provided Kopia Repository implementation, that is, Kopia Shim here - When read/write calls go into Kopia Shim, it in turn calls the BackupRepo handle for read/write - Finally, the read/write calls flow to Unified Repository module The Unified Repository provides all-in-one functionalities of a Backup Repository and exposes the Unified Repository Interface. Inside, Kopia Library is an adapter for Kopia Repository to translate the Unified Repository Interface calls to Kopia Repository interface calls. Both Kopia Shim and Kopia Library rely on Kopia Repository interface, so we need to have some Kopia version control. We may need to change Kopia Shim and Kopia Library when upgrading Kopia to a new version and the Kopia Repository interface has some changes in the new version. ![A BR Workflow](br-workflow.png) The modules in blue color in below diagram represent the newly added modules/logics or reorganized logics. The modules in yellow color in below diagram represent the called Kopia modules without changes. ### Delete Snapshot Workflow The Delete Snapshot workflow follows the similar manner with BR workflow, that is, we preserve the upper-level workflows until the calls reach to BackupDeletionController, then: - Leverage Repository Provider to switch between Restic implementation and Unified Repository implementation in the same mechanism as BR - For Restic implementation, the Restic Repository Provider invokes the existing “Forget” Restic CLI - For Unified Repository implementation, the Unified Repository Provider calls udmrepo.BackupRepo’s DeleteManifest to delete a snapshot ![A Snapshot Deletion Workflow](snapshot-deletion-workflow.png) ### Maintenance Workflow Backup Repository/Backup Storage may need to periodically reorganize its data so that it could guarantee its QOS during the long-time service. Some Backup Repository/Backup Storage does this in background automatically, so the user doesn’t need to interfere; some others need the caller to explicitly call their maintenance interface periodically. Restic and Kopia both go with the second way, that is, Velero needs to periodically call their maintenance interface. Velero already has an existing workflow to call Restic maintenance (it is called “Prune” in Restic, so Velero uses the same word). The existing workflow is as follows: - The Prune is triggered at the time of the backup - When a BackupRepository CR (originally called ResticRepository CR) is created by PodVolumeBackup/Restore Controller, the BackupRepository controller checks if it reaches to the Prune Due Time, if so, it calls PruneRepo - In the new design, the Repository Provider implements PruneRepo call, it uses the same way to switch between Restic Repository Provider and Unified Repository Provider, then: - For Restic Repository, Restic Repository Provider invokes the existing “Prune” CLI of Restic - For Unified Repository, Unified Repository Provider calls udmrepo.BackupRepoService’s Maintain function Kopia has two maintenance modes – the full maintenance and quick maintenance. There are many differences between full and quick mode, but briefly speaking, quick mode only processes the hottest data (primarily, it is the metadata and index data), so quick maintenance is much faster than full maintenance. On the other hand, quick maintenance also scatters the burden of full maintenance so that the full maintenance could finish fastly and make less impact. We will also take this quick maintenance into Velero. We will add a new Due Time to Velero, finally, we have two Prune Due Time: - Normal Due Time: For Restic, this will invoke Restic Prune; for Unified Repository, this will invoke udmrepo.BackupRepoService’s Maintain(full) call and finally call Kopia’s full maintenance - Quick Due Time: For Restic, this does nothing; for Unified Repository, this will invoke udmrepo.BackupRepoService’s Maintain(quick) call and finally call Kopia’s quick maintenance We assign different values to Normal Due Time and Quick Due Time, as a result of which, the quick maintenance happens more frequently than full maintenance. ![A Maintenance Workflow](maintenance-workflow.png) ### Progress Update Because Kopia Uploader is an unchanged Kopia module, we need to find a way to get its progress during the BR. Kopia Uploader accepts a Progress interface to update rich information during the BR, so the Kopia Uploader Provider will implement a Kopia’s Progress interface and then pass it to Kopia Uploader during its initialization. In this way, Velero will be able to get the progress as shown in the diagram below. ![A Progress Update](progress-update.png) ### Logs In the current design, Velero is using two unchanged Kopia modules --- the Kopia Uploader and the Kopia Repository. Both will generate debug logs during their run. Velero will collect these logs in order to aid the debug. Kopia’s Uploader and Repository both get the Logger information from the current GO Context, therefore, the Kopia Uploader Provider/Kopia Library could set the Logger interface into the current context and pass the context to Kopia Uploader/Kopia Repository. Velero will set Logger interfaces separately for Kopia Uploader and Kopia Repository. In this way, the Unified Repository could serve other data movers without losing the debug log capability; and the Kopia Uploader could write to any repository without losing the debug log capability. Kopia’s debug logs will be written to the same log file as Velero server or VeleroNodeAgent daemonset, so Velero doesn’t need to upload/download these debug logs separately. ![A Debug Log for Uploader](debug-log-uploader.png) ![A Debug Log for Repository](debug-log-repository.png) ## Path Switch & Coexist As mentioned above, There will be two paths. The related controllers need to identify the path during runtime and adjust its working mode. According to the requirements, path changing is fulfilled at the backup/restore level. In order to let the controllers know the path, we need to add some option values. Specifically, there will be option/mode values for path selection in two places: - Add the “uploader-type” option as a parameter of the Velero server. The parameters will be set by the installation. Currently the option has two values, either "restic" or "kopia" (in future, we may add other file system uploaders, then we will have more values). - Add a "uploaderType" value in the PodVolume Backup/Restore CR and a "repositoryType" value in the BackupRepository CR. "uploaderType" currently has two values , either "restic" or "kopia"; "repositoryType" currently has two values, either "restic" or "kopia" (in future, the Unified Repository could opt among multiple backup repository/backup storage, so there may be more values. This is a good reason that repositoryType is a multivariate flag, however, in which way to opt among the backup repository/backup storage is not covered in this PR). If the values are missing in the CRs, it by default means "uploaderType=restic" and "repositoryType=restic", so the legacy CRs are handled correctly by Restic. The corresponding controllers handle the CRs by checking the CRs' path value. Some examples are as below: - The PodVolume BR controller checks the "uploaderType" value from PodVolume CRs and decide its working path - The BackupRepository controller checks the "repositoryType" value from BackupRepository CRs and decide its working path - The Backup controller that runs in Velero server checks its “uploader-type” parameter to decide the path for the Backup it is going to create and then create the PodVolume Backup CR and BackupRepository CR - The Restore controller checks the Backup, from which it is going to restore, for the path and then create the PodVolume Restore CR and BackupRepository CR As described above, the “uploader-type” parameter of the Velero server is only used to decide the path when creating a new Backup, for other cases, the path selection is driven by the related CRs. Therefore, we only need to add this parameter to the Velero server. ## Velero CR Name Changes We will change below CRs' name to make them more generic: - "ResticRepository" CR to "BackupRepository" CR This means, we add a new CR type and deprecate the old one. As a result, if users upgrade from the old release, the old CRs will be orphaned, Velero will neither refer to it nor manage it, users need to delete these CRs manually. As a side effect, when upgrading from an old release, even though the path is not changed, the BackupRepository gets created all the time, because Velero will not refer to the old CR's status. This seems to cause the repository to initialize more than once, however, it won't happen. In the BackupRepository controller, before initializing a repository, it always tries to connect to the repository first, if it is connectable, it won't do the initialization. When backing up with the new release, Velero always creates BackupRepository CRs instead of ResticRepository CRs. When restoring from an old backup, Velero always creates BackupRepository CRs instead of ResticRepository CRs. When there are already backups or restores running during the upgrade, since after upgrade, the Velero server pods and VeleroNodeAgent daemonset pods are restarted, the existing backups/restores will fail immediately. ## Storage Configuration The backup repository needs some parameters to connect to various backup storage. For example, for a S3 compatible storage, the parameters may include bucket name, region, endpoint, etc. Different backup storage have totally different parameters. BackupRepository CRs, PodVolume Backup CRs and PodVolume Restore CRs save these parameters in their spec, as a string called repoIdentififer. The format of the string is for S3 storage only, it meets Restic CLI's requirements but is not enough for other backup repository. On the other hand, the parameters that are used to generate the repoIdentififer all come from the BackupStorageLocation. The latter has a map structure that could take parameters from any storage kind. Therefore, for the new path, Velero uses the information in the BackupStorageLocation directly. That is, whenever Velero needs to initialize/connect to the Unified Repository, it acquires the storage configuration from the corresponding BackupStorageLocation. Then no more elements will be added in BackupRepository CRs, PodVolume Backup CRs or PodVolume Restore CRs. The legacy path will be kept as is. That is, Velero still sets/gets the repoIdentififer in BackupRepository CRs, PodVolume Backup CRs and PodVolume Restore CRs and then passes to Restic CLI. ## Installation We will add a new flag "--uploader-type" during installation. The flag has 2 meanings: - It indicates the file system uploader to be used by PodVolume BR - It implies the backup repository type manner, Restic if uploader-type=restic, Unified Repository in all other cases The flag has below two values: **"Restic"**: it means Velero will use Restic to do the pod volume backup. Therefore, the Velero server deployment will be created as below: ``` spec: containers: - args: - server - --features= - --uploader-type=restic command: - /velero ``` The BackupRepository CRs and PodVolume Backup/Restore CRs created in this case are as below: ``` spec: backupStorageLocation: default maintenanceFrequency: 168h0m0s repositoryType: restic volumeNamespace: nginx-example ``` ``` spec: backupStorageLocation: default node: aks-agentpool-27359964-vmss000000 pod: kind: Pod name: nginx-stateful-0 namespace: nginx-example uid: 86aaec56-2b21-4736-9964-621047717133 tags: ... uploaderType: restic volume: nginx-log ``` ``` spec: backupStorageLocation: default pod: kind: Pod name: nginx-stateful-0 namespace: nginx-example uid: e56d5872-3d94-4125-bfe8-8a222bf0fcf1 snapshotID: 1741e5f1 uploaderType: restic volume: nginx-log ``` **"Kopia"**: it means Velero will use Kopia uploader to do the pod volume backup (so it will use Unified Repository as the backup target). Therefore, the Velero server deployment will be created as below: ``` spec: containers: - args: - server - --features= - --uploader-type=kopia command: - /velero ``` The BackupRepository CRs created in this case are hard set with "kopia" at present, sice Kopia is the only option as a backup repository. The PodVolume Backup/Restore CRs are created with "kopia" as well: ``` spec: backupStorageLocation: default maintenanceFrequency: 168h0m0s repositoryType: kopia volumeNamespace: nginx-example ``` ``` spec: backupStorageLocation: default node: aks-agentpool-27359964-vmss000000 pod: kind: Pod name: nginx-stateful-0 namespace: nginx-example uid: 86aaec56-2b21-4736-9964-621047717133 tags: ... uploaderType: kopia volume: nginx-log ``` ``` spec: backupStorageLocation: default pod: kind: Pod name: nginx-stateful-0 namespace: nginx-example uid: e56d5872-3d94-4125-bfe8-8a222bf0fcf1 snapshotID: 1741e5f1 uploaderType: kopia volume: nginx-log ``` We will add the flag for both CLI installation and Helm Chart Installation. Specifically: - Helm Chart Installation: add the "--uploaderType" and "--default-volumes-to-fs-backup" flag into its value.yaml and then generate the deployments according to the value. Value.yaml is the user-provided configuration file, therefore, users could set this value at the time of installation. The changes in Value.yaml are as below: ``` command: - /velero args: - server {{- with .Values.configuration }} - --uploader-type={{ default "restic" .uploaderType }} {{- if .defaultVolumesToFsBackup }} - --default-volumes-to-fs-backup {{- end }} ``` - CLI Installation: add the "--uploaderType" and "--default-volumes-to-fs-backup" flag into the installation command line, and then create the two deployments accordingly. Users could change the option at the time of installation. The CLI is as below: ```velero install --uploader-type=restic --default-volumes-to-fs-backup --use-node-agent``` ```velero install --uploader-type=kopia --default-volumes-to-fs-backup --use-node-agent``` ## Upgrade For upgrade, we allow users to change the path by specifying "--uploader-type" flag in the same way as the fresh installation. Therefore, the flag change should be applied to the Velero server after upgrade. Additionally, We need to add a label to Velero server to indicate the current path, so as to provide an easy for querying it. Moreover, if users upgrade from the old release, we need to change the existing Restic Daemonset name to VeleroNodeAgent daemonSet. The name change should be applied after upgrade. The recommended way for upgrade is to modify the related Velero resource directly through kubectl, the above changes will be applied in the same way. We need to modify the Velero doc for all these changes. ## CLI Below Velero CLI or its output needs some changes: - ```Velero backup describe```: the output should indicate the path - ```Velero restore describe```: the output should indicate the path - ```Velero restic repo get```: the name of this CLI should be changed to a generic one, for example, "Velero repo get"; the output of this CLI should print all the backup repository if Restic repository and Unified Repository exist at the same time At present, we don't have a requirement for selecting the path during backup, so we don't change the ```Velero backup create``` CLI for now. If there is a requirement in future, we could simply add a flag similar to "--uploader-type" to select the path. ## CR Example Below sample files demonstrate complete CRs with all the changes mentioned above: - BackupRepository CR: https://gist.github.com/Lyndon-Li/f38ad69dd8c4785c046cd7ed0ef2b6ed#file-backup-repository-sample-yaml - PodVolumeBackup CR: https://gist.github.com/Lyndon-Li/f38ad69dd8c4785c046cd7ed0ef2b6ed#file-pvb-sample-yaml - PodVolumeRestore CR: https://gist.github.com/Lyndon-Li/f38ad69dd8c4785c046cd7ed0ef2b6ed#file-pvr-sample-yaml ## User Perspective This design aims to provide a flexible backup repository layer and a generic file system uploader, which are fundermental for PodVolume and other data movements. Although this will make Velero more capable, at present, we don't pursue to expose differentiated features end to end. Specifically: - For a fresh installation, if the "--uploader-type" is not specified, there is a default value for PodVolume BR. We will keep it as "restic" for at least one release, then we switch the value to "kopia" - Even when changing to the new path, Velero still allows users to restore from the data backed up by Restic - The capability of PodVolume BR under the new path is kept the same as it under Restic path and the same as the existing PodVolume BR - The operational experiences are kept the same as much as possible, the known changes are listed below Below user experiences are changed for this design: - Installation CLI change: a new option is added to the installation CLI, see the Installation section for details - CR change: One or more existing CRs have been renamed, see the Velero CR Changes section for details - Velero CLI name and output change, see the CLI section for details - Velero daemonset name change - Wording Alignment: as the existing situation, many places are using the word of "Restic", for example, "default-volume-to-restic" option, most of them are not accurate anymore, we will change these words and give a detailed list of the changes ================================================ FILE: design/Implemented/velero-debug.md ================================================ # `velero debug` command for gathering troubleshooting information ## Abstract To simplify the communication between velero users and developers, this document proposes the `velero debug` command to generate a tarball including the logs needed for debugging. Github issue: https://github.com/vmware-tanzu/velero/issues/675 ## Background Gathering information to troubleshoot a Velero deployment is currently spread across multiple commands, and is not very efficient. Logs for the Velero server itself are accessed via a kubectl logs command, while information on specific backups or restores are accessed via a Velero subcommand. Restic logs are even more complicated to retrieve, since one must gather logs for every instance of the daemonset, and there’s currently no good mechanism to locate which node a particular restic backup ran against. A dedicated subcommand can lower this effort and reduce back-and-forth between user and developer for collecting the logs. ## Goals - Enable efficient log collection for Velero and associated components, like plugins and restic. ## Non Goals - Collecting logs for components that do not belong to velero such as storage service. - Automated log analysis. ## High-Level Design With the introduction of the new command `velero debug`, the command would download all of the following information: - velero deployment logs - restic DaemonSet logs - plugin logs - All the resources in the group `velero.io` that are created such as: - Backup - Restore - BackupStorageLocation - PodVolumeBackup - PodVolumeRestore - *etc ...* - Log of the backup and restore, if specified in the param A project called `crash-diagnostics` (or `crashd`) (https://github.com/vmware-tanzu/crash-diagnostics) implements the Kubernetes API queries and provides Starlark scripting language to abstract details, and collect the information into a local copy. It can be used as a standalone CLI executing a Starlark script file. With the capabilities of embedding files in Go 1.16, we can define a Starlark script gathering the necessary information, embed the script at build time, then the velero debug command will invoke `crashd`, passing in the script’s text contents. ## Detailed Design ### Triggering the script The Starlark script to be called by crashd: ```python def capture_backup_logs(cmd, namespace): if args.backup: log("Collecting log and information for backup: {}".format(args.backup)) backupDescCmd = "{} --namespace={} backup describe {} --details".format(cmd, namespace, args.backup) capture_local(cmd=backupDescCmd, file_name="backup_describe_{}.txt".format(args.backup)) backupLogsCmd = "{} --namespace={} backup logs {}".format(cmd, namespace, args.backup) capture_local(cmd=backupLogsCmd, file_name="backup_{}.log".format(args.backup)) def capture_restore_logs(cmd, namespace): if args.restore: log("Collecting log and information for restore: {}".format(args.restore)) restoreDescCmd = "{} --namespace={} restore describe {} --details".format(cmd, namespace, args.restore) capture_local(cmd=restoreDescCmd, file_name="restore_describe_{}.txt".format(args.restore)) restoreLogsCmd = "{} --namespace={} restore logs {}".format(cmd, namespace, args.restore) capture_local(cmd=restoreLogsCmd, file_name="restore_{}.log".format(args.restore)) ns = args.namespace if args.namespace else "velero" output = args.output if args.output else "bundle.tar.gz" cmd = args.cmd if args.cmd else "velero" # Working dir for writing during script execution crshd = crashd_config(workdir="./velero-bundle") set_defaults(kube_config(path=args.kubeconfig, cluster_context=args.kubecontext)) log("Collecting velero resources in namespace: {}". format(ns)) kube_capture(what="objects", namespaces=[ns], groups=['velero.io']) capture_local(cmd="{} version -n {}".format(cmd, ns), file_name="version.txt") log("Collecting velero deployment logs in namespace: {}". format(ns)) kube_capture(what="logs", namespaces=[ns]) capture_backup_logs(cmd, ns) capture_restore_logs(cmd, ns) archive(output_file=output, source_paths=[crshd.workdir]) log("Generated debug information bundle: {}".format(output)) ``` The sample command to trigger the script via crashd: ```shell ./crashd run ./velero.cshd --args 'backup=harbor-backup-2nd,namespace=velero,basedir=,restore=,kubeconfig=/home/.kube/minikube-250-224/config,output=' ``` To trigger the script in `velero debug`, in the package `pkg/cmd/cli/debug` a struct `option` will be introduced ```go type option struct { // currCmd the velero command currCmd string // workdir for crashd will be $baseDir/velero-debug baseDir string // the namespace where velero server is installed namespace string // the absolute path for the log bundle to be generated outputPath string // the absolute path for the kubeconfig file that will be read by crashd for calling K8S API kubeconfigPath string // the kubecontext to be used for calling K8S API kubeContext string // optional, the name of the backup resource whose log will be packaged into the debug bundle backup string // optional, the name of the restore resource whose log will be packaged into the debug bundle restore string // optional, it controls whether to print the debug log messages when calling crashd verbose bool } ``` The code will consolidate the input parameters and execution context of the `velero` CLI to form the option struct, which can be transformed into the `argsMap` that can be used when calling the func `exec.Execute` in `crashd`: https://github.com/vmware-tanzu/crash-diagnostics/blob/v0.3.4/exec/executor.go#L17 ## Alternatives Considered The collection could be done via the Kubernetes client-go API, but such integration is not necessarily trivial to implement, therefore, `crashd` is preferred approach ## Security Considerations - The starlark script will be embedded into the velero binary, and the byte slice will be passed to the `exec.Execute` func directly, so there’s little risk that the script will be modified before being executed. ## Compatibility As the `crashd` project evolves the behavior of the internal functions used in the Starlark script may change. We’ll ensure the correctness of the script via regular E2E tests. ## Implementation 1. Bump up to use Go v1.16 to compile velero 2. Embed the starlark script 3. Implement the `velero debug` sub-command to call the script 4. Add E2E test case ## Open Questions - **Command dependencies:** In the Starlark script, for collecting version info and backup logs, it calls the `velero backup logs` and `velero version`, which makes the call stack like velero debug -> crashd -> velero xxx. We need to make sure this works under different PATH settings. - **Progress and error handling:** The log collection may take a relatively long time, log messages should be printed to indicate the progress when different items are being downloaded and packaged. Additionally, when an error happens, `crashd` may omit some errors, so before the script is executed we'll do some validation and make sure the `debug` command fail early if some parameters are incorrect. ================================================ FILE: design/Implemented/velero-uploader-configuration.md ================================================ # Velero Uploader Configuration Integration and Extensibility ## Abstract This design proposal aims to make Velero Uploader configurable by introducing a structured approach for managing Uploader settings. we will define and standardize a data structure to facilitate future additions to Uploader configurations. This enhancement provides a template for extending Uploader-related options. And also includes examples of adding sub-options to the Uploader Configuration. ## Background Velero is widely used for backing up and restoring Kubernetes clusters. In various scenarios, optimizing the backup process is essential, future needs may arise for adding more configuration options related to the Uploader component especially when dealing with large datasets. Therefore, a standardized configuration template is required. ## Goals 1. **Extensible Uploader Configuration**: Provide an extensible approach to manage Uploader configurations, making it easy to add and modify configuration options related to the Velero uploader. 2. **User-friendliness**: Ensure that the new Uploader configuration options are easy to understand and use for Velero users without introducing excessive complexity. ## Non Goals 1. Expanding to other Velero components: The primary focus of this design is Uploader configuration and does not include extending to other components or modules within Velero. Configuration changes for other components may require separate design and implementation. ## High-Level Design To achieve extensibility in Velero Uploader configurations, the following key components and changes are proposed: ### UploaderConfig Structure Two new data structures, `UploaderConfigForBackup` and `UploaderConfigForRestore`, will be defined to store Uploader configurations. These structures will include the configuration options related to backup and restore for Uploader: ```go type UploaderConfigForBackup struct { } type UploaderConfigForRestore struct { } ``` ### Integration with Backup & Restore CRD The Velero CLI will support an uploader configuration-related flag, allowing users to set the value when creating backups or restores. This value will be stored in the `UploaderConfig` field within the `Backup` CRD and `Restore` CRD: ```go type BackupSpec struct { // UploaderConfig specifies the configuration for the uploader. // +optional // +nullable UploaderConfig *UploaderConfigForBackup `json:"uploaderConfig,omitempty"` } type RestoreSpec struct { // UploaderConfig specifies the configuration for the restore. // +optional // +nullable UploaderConfig *UploaderConfigForRestore `json:"uploaderConfig,omitempty"` } ``` ### Configuration Propagated to Different CRDs The configuration specified in `UploaderConfig` needs to be effective for backup and restore both by file system way and data-mover way. Therefore, the `UploaderConfig` field value from the `Backup` CRD should be propagated to `PodVolumeBackup` and `DataUpload` CRDs. We aim for the configurations in PodVolumeBackup to originate not only from UploaderConfig in Backup but also potentially from other sources such as the server or configmap. Simultaneously, to align with the configurations in DataUpload's `DataMoverConfig map[string]string`, we have defined an `UploaderSettings map[string]string` here to record the configurations in PodVolumeBackup. ```go type PodVolumeBackupSpec struct { // UploaderSettings are a map of key-value pairs that should be applied to the // uploader configuration. // +optional // +nullable UploaderSettings map[string]string `json:"uploaderSettings,omitempty"` } ``` `UploaderConfig` will be stored in DataUpload's `DataMoverConfig map[string]string` field. Also the `UploaderConfig` field value from the `Restore` CRD should be propagated to `PodVolumeRestore` and `DataDownload` CRDs: ```go type PodVolumeRestoreSpec struct { // UploaderSettings are a map of key-value pairs that should be applied to the // uploader configuration. // +optional // +nullable UploaderSettings map[string]string `json:"uploaderSettings,omitempty"` } ``` Also `UploaderConfig` will be stored in DataUpload's `DataMoverConfig map[string]string` field. ### Store and Get Configuration We need to store and retrieve configurations in the PodVolumeBackup and DataUpload structs. This involves type conversion based on the configuration type, storing it in a map[string]string, or performing type conversion from this map for retrieval. PodVolumeRestore and DataDownload are also similar. ## Sub-options in UploaderConfig Adding fields above in CRDs can accommodate any future additions to Uploader configurations by adding new fields to the `UploaderConfigForBackup` or `UploaderConfigForRestore` structures. ### Parallel Files Upload This section focuses on enabling the configuration for the number of parallel file uploads during backups. below are the key steps that should be added to support this new feature. #### Velero CLI The Velero CLI will support a `--parallel-files-upload` flag, allowing users to set the `ParallelFilesUpload` value when creating backups. #### UploaderConfig below the sub-option `ParallelFilesUpload` is added into UploaderConfig: ```go // UploaderConfigForBackup defines the configuration for the uploader when doing backup. type UploaderConfigForBackup struct { // ParallelFilesUpload is the number of files parallel uploads to perform when using the uploader. // +optional ParallelFilesUpload int `json:"parallelFilesUpload,omitempty"` } ``` #### Kopia Parallel Upload Policy Velero Uploader can set upload policies when calling Kopia APIs. In the Kopia codebase, the structure for upload policies is defined as follows: ```go // UploadPolicy describes the policy to apply when uploading snapshots. type UploadPolicy struct { ... MaxParallelFileReads *OptionalInt `json:"maxParallelFileReads,omitempty"` } ``` Velero can set the `MaxParallelFileReads` parameter for Kopia's upload policy as follows: ```go curPolicy := getDefaultPolicy() if parallelUpload > 0 { curPolicy.UploadPolicy.MaxParallelFileReads = newOptionalInt(parallelUpload) } ``` #### Restic Parallel Upload Policy As Restic does not support parallel file upload, the configuration would not take effect, so we should output a warning when the user sets the `ParallelFilesUpload` value by using Restic to do a backup. ```go if parallelFilesUpload > 0 { log.Warnf("ParallelFilesUpload is set to %d, but Restic does not support parallel file uploads. Ignoring", parallelFilesUpload) } ``` Roughly, the process is as follows: 1. Users pass the ParallelFilesUpload parameter and its value through the Velero CLI. This parameter and its value are stored as a sub-option within UploaderConfig and then placed into the Backup CR. 2. When users perform file system backups, UploaderConfig is passed to the PodVolumeBackup CR. When users use the Data-mover for backups, it is passed to the DataUpload CR. 3. The configuration will be stored in map[string]string type of field in CR. 3. Each respective controller within the CRs calls the uploader, and the ParallelFilesUpload from map in CRs is passed to the uploader. 4. When the uploader subsequently calls the Kopia API, it can use the ParallelFilesUpload to set the MaxParallelFileReads parameter, and if the uploader calls the Restic command it would output one warning log for Restic does not support this feature. ### Sparse Option For Kopia & Restic Restore In many system files, numerous zero bytes or empty blocks persist, occupying physical storage space. Sparse restore employs a more intelligent approach, including appropriately handling empty blocks, thereby achieving the correct system state. This write sparse files mechanism aims to enhance restore efficiency while maintaining restoration accuracy. Below are the key steps that should be added to support this new feature. #### Velero CLI The Velero CLI will support a `--write-sparse-files` flag, allowing users to set the `WriteSparseFiles` value when creating restores with Restic or Kopia uploader. #### UploaderConfig below the sub-option `WriteSparseFiles` is added into UploaderConfig: ```go // UploaderConfigForRestore defines the configuration for the restore. type UploaderConfigForRestore struct { // WriteSparseFiles is a flag to indicate whether write files sparsely or not. // +optional // +nullable WriteSparseFiles *bool `json:"writeSparseFiles,omitempty"` } ``` ### Enable Sparse in Restic For Restic, it could be enabled by pass the flag `--sparse` in creating restore: ```bash restic restore create --sparse $snapshotID ``` ### Enable Sparse in Kopia For Kopia, it could be enabled this feature by the `WriteSparseFiles` field in the [FilesystemOutput](https://pkg.go.dev/github.com/kopia/kopia@v0.13.0/snapshot/restore#FilesystemOutput). ```go fsOutput := &restore.FilesystemOutput{ WriteSparseFiles: uploaderutil.GetWriteSparseFiles(uploaderCfg), } ``` Roughly, the process is as follows: 1. Users pass the WriteSparseFiles parameter and its value through the Velero CLI. This parameter and its value are stored as a sub-option within UploaderConfig and then placed into the Restore CR. 2. When users perform file system restores, UploaderConfig is passed to the PodVolumeRestore CR. When users use the Data-mover for restores, it is passed to the DataDownload CR. 3. The configuration will be stored in map[string]string type of field in CR. 4. Each respective controller within the CRs calls the uploader, and the WriteSparseFiles from map in CRs is passed to the uploader. 5. When the uploader subsequently calls the Kopia API, it can use the WriteSparseFiles to set the WriteSparseFiles parameter, and if the uploader calls the Restic command it would append `--sparse` flag within the restore command. ### Parallel Restore Setting the parallelism of restore operations can improve the efficiency and speed of the restore process, especially when dealing with large amounts of data. ### Velero CLI The Velero CLI will support a --parallel-files-download flag, allowing users to set the parallelism value when creating restores. when no value specified, the value of it would be the number of CPUs for the node that the node agent pod is running. ```bash velero restore create --parallel-files-download $num ``` ### UploaderConfig below the sub-option parallel is added into UploaderConfig: ```go type UploaderConfigForRestore struct { // ParallelFilesDownload is the number of parallel for restore. // +optional ParallelFilesDownload int `json:"parallelFilesDownload,omitempty"` } ``` #### Kopia Parallel Restore Policy Velero Uploader can set restore policies when calling Kopia APIs. In the Kopia codebase, the structure for restore policies is defined as follows: ```go // first get concurrrency from uploader config restoreConcurrency, _ := uploaderutil.GetRestoreConcurrency(uploaderCfg) // set restore concurrency into restore options restoreOpt := restore.Options{ Parallel: restoreConcurrency, } // do restore with restore option restore.Entry(..., restoreOpt) ``` #### Restic Parallel Restore Policy Configurable parallel restore is not supported by restic, so we would return one error if the option is configured. ```go restoreConcurrency, err := uploaderutil.GetRestoreConcurrency(uploaderCfg) if err != nil { return extraFlags, errors.Wrap(err, "failed to get uploader config") } if restoreConcurrency > 0 { return extraFlags, errors.New("restic does not support parallel restore") } ``` ## Alternatives Considered To enhance extensibility further, the option of storing `UploaderConfig` in a Kubernetes ConfigMap can be explored, this approach would allow the addition and modification of configuration options without the need to modify the CRD. ================================================ FILE: design/Implemented/vgdp-affinity-enhancement.md ================================================ # Velero Generic Data Path Load Affinity Enhancement Design ## Glossary & Abbreviation **Velero Generic Data Path (VGDP)**: VGDP is the collective modules that is introduced in [Unified Repository design][1]. Velero uses these modules to finish data transfer for various purposes (i.e., PodVolume backup/restore, Volume Snapshot Data Movement). VGDP modules include uploaders and the backup repository. **Exposer**: Exposer is a module that is introduced in [Volume Snapshot Data Movement Design][1]. Velero uses this module to expose the volume snapshots to Velero node-agent pods or node-agent associated pods so as to complete the data movement from the snapshots. ## Background The implemented [VGDP LoadAffinity design][3] already defined the a structure `LoadAffinity` in `--node-agent-configmap` parameter. The parameter is used to set the affinity of the backupPod of VGDP. There are still some limitations of this design: * The affinity setting is global. Say there are two StorageClasses and the underlying storage can only provision volumes to part of the cluster nodes. The supported nodes don't have intersection. Then the affinity will definitely not work in some cases. * The old design focuses on the backupPod affinity, but the restorePod also needs the affinity setting. As a result, create this design to address the limitations. ## Goals - Enhance the node affinity of VGDP instances for volume snapshot data movement: add per StorageClass node affinity. - Enhance the node affinity of VGDP instances for volume snapshot data movement: support the or logic between affinity selectors. - Define the behaviors of node affinity of VGDP instances in node-agent for volume snapshot data movement restore, when the PVC restore doesn't require delay binding. ## Non-Goals - It is also beneficial to support VGDP instances affinity for PodVolume backup/restore, this will be implemented after the PodVolume micro service completes. ## Solution This design still uses the ConfigMap specified by `velero node-agent` CLI's parameter `--node-agent-configmap` to host the node affinity configurations. Upon the implemented [VGDP LoadAffinity design][3] introduced `[]*LoadAffinity` structure, this design add a new field `StorageClass`. This field is optional. * If the `LoadAffinity` element's `StorageClass` doesn't have value, it means this element is applied to global, just as the old design. * If the `LoadAffinity` element's `StorageClass` has value, it means this element is applied to the VGDP instances' PVCs use the specified StorageClass. * The `LoadAffinity` element whose `StorageClass` has value has higher priority than the `LoadAffinity` element whose `StorageClass` doesn't have value. ```go type Configs struct { // LoadConcurrency is the config for load concurrency per node. LoadConcurrency *LoadConcurrency `json:"loadConcurrency,omitempty"` // LoadAffinity is the config for data path load affinity. LoadAffinity []*LoadAffinity `json:"loadAffinity,omitempty"` } type LoadAffinity struct { // NodeSelector specifies the label selector to match nodes NodeSelector metav1.LabelSelector `json:"nodeSelector"` } ``` ``` go type LoadAffinity struct { // NodeSelector specifies the label selector to match nodes NodeSelector metav1.LabelSelector `json:"nodeSelector"` // StorageClass specifies the VGDPs the LoadAffinity applied to. If the StorageClass doesn't have value, it applies to all. If not, it applies to only the VGDPs that use this StorageClass. StorageClass string `json:"storageClass"` } ``` ### Decision Tree ```mermaid flowchart TD A[VGDP Pod Needs Scheduling] --> B{Is this a restore operation?} B -->|Yes| C{StorageClass has volumeBindingMode: WaitForFirstConsumer?} B -->|No| D[Backup Operation] C -->|Yes| E{restorePVC.ignoreDelayBinding = true?} C -->|No| F[StorageClass binding mode: Immediate] E -->|No| G[Wait for target Pod scheduling
Use Pod's selected node
⚠️ Affinity rules ignored] E -->|Yes| H[Apply affinity rules
despite WaitForFirstConsumer] F --> I{Check StorageClass in loadAffinity by StorageClass field} H --> I D --> J{Using backupPVC with different StorageClass?} J -->|Yes| K[Use final StorageClass
for affinity lookup] J -->|No| L[Use original PVC StorageClass
for affinity lookup] K --> I L --> I I -->|StorageClass found| N[Filter the LoadAffinity by
the StorageClass
🎯 and apply the LoadAffinity HIGHEST PRIORITY] I -->|StorageClass not found| O{Check loadAffinity element without StorageClass field} O -->|No loadAffinity configured| R[No affinity constraints
Schedule on any available node
🌐 DEFAULT] O --> V[Validate node-agent availability
⚠️ Ensure node-agent pods exist on target nodes] N --> V V --> W{Node-agent available on selected nodes?} W -->|Yes| X[✅ VGDP Pod scheduled successfully] W -->|No| Y[❌ Pod stays in Pending state
Timeout after 30min
Check node-agent DaemonSet coverage] R --> Z[Schedule on any node
✅ Basic scheduling] %% Styling classDef successNode fill:#d4edda,stroke:#155724,color:#155724 classDef warningNode fill:#fff3cd,stroke:#856404,color:#856404 classDef errorNode fill:#f8d7da,stroke:#721c24,color:#721c24 classDef priorityHigh fill:#e7f3ff,stroke:#0066cc,color:#0066cc classDef priorityMedium fill:#f0f8ff,stroke:#4d94ff,color:#4d94ff classDef priorityDefault fill:#f8f9fa,stroke:#6c757d,color:#6c757d class X,Z successNode class G,V,Y warningNode class Y errorNode class N,T,U priorityHigh class P,Q priorityMedium class R priorityDefault ``` ### Examples #### LoadAffinity interacts with LoadAffinityPerStorageClass ``` json { "loadAffinity": [ { "nodeSelector": { "matchLabels": { "beta.kubernetes.io/instance-type": "Standard_B4ms" } } }, { "nodeSelector": { "matchExpressions": [ { "key": "kubernetes.io/os", "values": [ "linux" ], "operator": "In" } ] }, "storageClass": "kibishii-storage-class" }, { "nodeSelector": { "matchLabels": { "beta.kubernetes.io/instance-type": "Standard_B8ms" } }, "storageClass": "kibishii-storage-class" } ] } ``` This sample demonstrates how the `loadAffinity` elements with `StorageClass` field and without `StorageClass` field setting work together. If the VGDP mounting volume is created from StorageClass `kibishii-storage-class`, its pod will run Linux nodes or instance type as `Standard_B8ms`. The other VGDP instances will run on nodes, which instance type is `Standard_B4ms`. #### LoadAffinity interacts with BackupPVC ``` json { "loadAffinity": [ { "nodeSelector": { "matchLabels": { "beta.kubernetes.io/instance-type": "Standard_B4ms" } }, "storageClass": "kibishii-storage-class" }, { "nodeSelector": { "matchLabels": { "beta.kubernetes.io/instance-type": "Standard_B2ms" } }, "storageClass": "worker-storagepolicy" } ], "backupPVC": { "kibishii-storage-class": { "storageClass": "worker-storagepolicy" } } } ``` Velero data mover supports to use different StorageClass to create backupPVC by [design](https://github.com/vmware-tanzu/velero/pull/7982). In this example, if the backup target PVC's StorageClass is `kibishii-storage-class`, its backupPVC should use StorageClass `worker-storagepolicy`. Because the final StorageClass is `worker-storagepolicy`, the backupPod uses the loadAffinity specified by `loadAffinity`'s elements with `StorageClass` field set to `worker-storagepolicy`. backupPod will be assigned to nodes, which instance type is `Standard_B2ms`. #### LoadAffinity interacts with RestorePVC ``` json { "loadAffinity": [ { "nodeSelector": { "matchLabels": { "beta.kubernetes.io/instance-type": "Standard_B4ms" } }, "storageClass": "kibishii-storage-class" } ], "restorePVC": { "ignoreDelayBinding": false } } ``` ##### StorageClass's bind mode is WaitForFirstConsumer ``` yaml apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: kibishii-storage-class parameters: svStorageClass: worker-storagepolicy provisioner: csi.vsphere.vmware.com reclaimPolicy: Delete volumeBindingMode: WaitForFirstConsumer ``` If restorePVC should be created from StorageClass `kibishii-storage-class`, and it's volumeBindingMode is `WaitForFirstConsumer`. Although `loadAffinityPerStorageClass` has a section matches the StorageClass, the `ignoreDelayBinding` is set `false`, the Velero exposer will wait until the target Pod scheduled to a node, and returns the node as SelectedNode for the restorePVC. As a result, the `loadAffinityPerStorageClass` will not take affect. ##### StorageClass's bind mode is Immediate ``` yaml apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: kibishii-storage-class parameters: svStorageClass: worker-storagepolicy provisioner: csi.vsphere.vmware.com reclaimPolicy: Delete volumeBindingMode: Immediate ``` Because the StorageClass volumeBindingMode is `Immediate`, although `ignoreDelayBinding` is set to `false`, restorePVC will not be created according to the target Pod. The restorePod will be assigned to nodes, which instance type is `Standard_B4ms`. [1]: Implemented/unified-repo-and-kopia-integration/unified-repo-and-kopia-integration.md [2]: Implemented/volume-snapshot-data-movement/volume-snapshot-data-movement.md [3]: Implemented/node-agent-affinity.md ================================================ FILE: design/Implemented/vgdp-micro-service/vgdp-micro-service.md ================================================ # VGDP Micro Service For Volume Snapshot Data Movement ## Glossary & Abbreviation **VGDP**: Velero Generic Data Path. The collective of modules that is introduced in [Unified Repository design][1]. Velero uses these modules to finish data transmission for various purposes. It includes uploaders and the backup repository. **Volume Snapshot Data Movement**: The backup/restore method introduced in [Volume Snapshot Data Movement design][2]. It backs up snapshot data from the volatile and limited production environment into the durable, heterogeneous and scalable backup storage. **VBDM**: Velero Built-in Data Mover as introduced in [Volume Snapshot Data Movement design][2], it is the built-in data mover shipped along with Velero. **Exposer**: Exposer is introduced in [Volume Snapshot Data Movement design][2] and is used to expose the volume snapshots/target volumes for VGDP to access locally. ## Background As the architecture introduced in [Volume Snapshot Data Movement design][2], VGDP instances are running inside the node-agent pods, however, more and more use cases require to run the VGDP instances in dedicated pods, or in another word, make them as micro services, the benefits are as below: - This avoids VGDP to access volume data through host path, while host path access involves privilege escalations in some environments (e.g., must run under privileged mode), which makes challenge to users. - This enable users to to control resource (i.e., cpu, memory) request/limit in a granular manner, e.g., control them per backup/restore of a volume - This increases the resilience, crash of one VGDP activity won't affect others - In the cases that the backup storage must be represented by a Kubernetes persistent volumes (i.e., nfs storage, [COSI][3]), this avoids to dynamically mount the persistent volumes to node-agent pods and cause node-agent pods to restart (this is not accepted since node-agent lose it current state after its pods restart) - This prevents unnecessary full backup. Velero's fs uploaders support file level incremental backup by comparing the file name and metadata. However, at present the files are visited by host path, while pod and PVC's ID are part of the host path, so once the pod is recreated, the same file is regarded as a different file since the pod's ID has been changed. If the fs uploader is in a dedicated pod and files are visited by pod's volume path, files' full path are not changed after pod restarts, so incremental backups could continue. ## Goals - Create a solution to make VGDP instances as micro services - Modify the VBDM to offload the VGDP work from node-agent to the VGDP micro service - Create the mechanism for VBDM to control and monitor the VGDP micro services in various scenarios ## Non-Goals - The current solution covers Volume Snapshot Data Movement backup/restore type only, even though VGDP is also used by pod volume backup. It is less possible to do this for pod volume backup, since it must run inside the source workload pods. - The current solution covers VBDM only. 3rd data movers still follow the **Replacement** section of [Volume Snapshot Data Movement design][2]. That is, 3rd data movers handle the DUCR/DDCR on their own and they are free to make themselves micro service style or monolith service style. ## Overview The solution is based on [Volume Snapshot Data Movement design][2], the architecture is followed as is and existing components are not changed unless it is necessary. Below lists the changed components, why and how: **Exposer**: Exposer is to expose the snapshot/target volume as a path/device name/endpoint that are recognizable by VGDP. Varying from the type of snapshot/target volume, a pod may be created as part of the expose. Now, since we run the VGDP instance in a separate pod, a pod is created anyway, we assume exposer creates a pod all the time and make the appropriate exposing configurations to the pod so that VGDP instance could access the snapshot/target volume locally inside the pod. The pod is still called as backupPod or restorePod. Then we need to change the command the backupPod/restorePod is running, the command launches VGDP-MS (VGDP Micro Service, see below) when the container starts up. For CSI snapshot, the backupPod/restorePod is created as the result of expose, the only thing left is to change the backupPod/restorePod's image. **VBDM**: VBDM contains the data mover controller, while the controller calls the Exposer and launches the VGDP instances. Now, since the VGDP instance is launched by the backupPod/restorePod, the controller should not launch the VGDP instance again. However, the controller still needs to monitor and control the VGDP instance. Moreover, in order to avoid any contest situations, the controller is still the only place to update DUCRs and DDCRs. Besides the changes to above existing components, we need to add below new components: **VGDP Watcher**: We create a new module to help the data mover controller to watch activities of the VGDP instance in the backupPod/restorePod. VGDP Watcher is a part of VBDM. **VGDP-MS**: VGDP Micro Service is the binary for the command backupPod/restorePod runs. It accepts the parameters and then launches the VGDP instance according to the request type, specifically, backup or restore. VGDP-MS also runs other modules to sync-up with the data mover controller. VGDP-MS is also a part of VBDM. Below diagram shows how these components work together: ![vgdp-ms-1.png](vgdp-ms-1.png) The [Node-agent concurrency][4] is still used to control the concurrency of VGDP micro services. When there are too many volumes in the backup/restore, which takes too much computing resources(CPU, memory, etc.) or Kubernetes resources(pods, PVCs, PVs, etc.), users could set the concurrency in each node so as to control the total number of concurrent VGDP micro services in the cluster. ## Detailed Design ### Exposer At present, the exposer creates backupPod/restorePod and sets ```velero-helper pause``` as the command run by backupPod/restorePod. Now, VGDP-MS command will be used, and the ```velero``` image will be running inside the backupPod/restorePod. The command is like below: ```velero data-mover backup --volume-path xxx --volume-mode xxx --data-upload xxx --resource-timeout xxx --log-format xxx --log-level xxx``` Or: ```velero data-mover restore --volume-path xxx --volume-mode xxx --data-download xxx --resource-timeout xxx --log-format xxx --log-level xxx``` The first one is for backup and the other one is for restore. Below are the parameters of the commands: **volume-path**: Deliver the full path inside the backupPod/restorePod for the volume to be backed up/restored. **volume-mode**: Deliver the mode for the volume be backed up/restored, at present either ```Filesystem``` mode or ```Block``` mode. **data-upload**: DUCR for this backup. **data-download**: DDCR for this backup. **resource-timeout**: resource-timeout is used to control the timeout for operations related to resources. It has the same meaning with the resource-timeout for node-agent. **log-format** and **log-level**: This is to control the behavior of log generation inside VGDP-MS. In order to have the same capability and permission with node-agent, below pod configurations are inherited from node-agent and set to backupPod/restorePod's spec: - Volumes: Some configMaps will be mapped as volumes to node-agent, so we add the same volumes of node-agent to the backupPod/restorePod - Environment Variables - Security Contexts We may not actually need all the capabilities in the VGDP-MS as the node-agent. At present, we just duplicate all of them, if we find any problem in future, we can filter out the capabilities that are not required by VGDP-MS. The backupPod/restorePod is not run in Privileged mode as it is not required since the volumes are visisted by pod path. The root user is still required, especially by the restore (in order to restore the file system attributes, owners, etc.), so we will use root user for backupPod/restorePod. We set backupPod/restorePod's ```RestartPolicy``` to ```RestartPolicyNever```, so that once VGDP-MS terminates in any reason, backupPod/restorePod won't restart and the DUCR/DDCR is marked as one of the terminal phases (Completed/Failed/Cancelled) accordingly. ### VGDP Watcher #### Dual mode event watch The primary task of VGDP Watcher is to watch the status change from backupPod/restorePod or the VGDP instance, so as to inform the data mover controller in below situations: - backupPod/restorePod starts - VGDP instance starts - Progress update - VGDP instance completes/fails/cancelled - backupPod/restorePod stops We use two mechanism to make the watch: **Pod Phases**: VGDP Watcher watches the backupPod/restorePod's phases updated by Kubernetes. That is, VGDP Watcher creates an informer to watch the pod resource for the backupPod/restorePod and detect that the pod reaches to one of the terminated phases (i.e., PodSucceeded, PodFailed). We also check the availability & status of the backupPod/restorePod at the beginning of the watch so as to detect the starting of the backupPod/restorePod. **Custom Kubernetes Events**: VGDP-MS generates Kubernetes events and associates them to the DUCR/DDCR at the time of VGDP instance starting/stopping and progress update, then VGDP Watcher creates another informer to watch the Event resource associated to the DUCR/DDCR. Pod Phases watch covers the entire lifecycle of the backupPod/restorePod, but we don't know the status of the VGDP instance through it; and it can only deliver information by then end of the pod lifecycle. Custom Event watch generates details of the VGDP instances and the events could be generated any time; but it cannot generate notifications before VGDP starts or in the case that VGDP crashes or shutdown abnormally. Therefore, we adopt the both mechanisms to VGDP Watcher. In the end, there will be two sources generating the result of VGDP-MS: - The termination message of backupPod/restorePod - The message along with the VGDP Instance Completes/Fails/Cancelled event On the one hand, in some cases only the backupPod/restorePod's termination message is available, e.g., the backupPod/restorePod crashes or or backupPod/restorePod quits before VGDP instance is started. So we refer to the first mechanism to get the notifications. On the other hand, if they are both available, we have the results from them for mutual verification. Conclusively, under the help of VGDP Watcher, data mover controller starts VGDP-MS controllably and waits until VGDP-MS ends under any circumstances. #### AsyncBR adapter VGDP Watcher needs to notify the data mover controller when one of the watched event happens, so that the controller could do the operations as if it receives the same callbacks from VGDP as the current behavior. In order not to break the existing code logics of data mover controllers, we make VGDP Watcher as an adapter of AsyncBR which is the interface implemented by VGDP and called by the data mover controller. Since the parameters to call VGDP Watcher is different from the ones to call VGDP, we change the AsyncBR interface to hide some parameters from one another, the new interface is as below: ``` type AsyncBR interface { // Init initializes an asynchronous data path instance Init(ctx context.Context, res *exposer.ExposeResult, param interface{}) error // StartBackup starts an asynchronous data path instance for backup StartBackup(dataMoverConfig map[string]string, param interface{}) error // StartRestore starts an asynchronous data path instance for restore StartRestore(snapshotID string, dataMoverConfig map[string]string) error // Cancel cancels an asynchronous data path instance Cancel() // Close closes an asynchronous data path instance Close(ctx context.Context) } ``` Some parameters are hidden into ```param```, but the functions and calling logics are not changed. VGDP Watcher should be launched by the data mover controller before VGDP instance starts, otherwise, multiple corner problems may happen. E.g., VGDP-MS may run the VGDP instance immediately after the backupPod/restorePod is launched and completes it before the data mover controller starts VGDP Watcher, as a result, multiple informs are missed from VGDP Watcher. Therefore, the controller launches VGDP Watcher first and then set the DUCR/DDCR to ```InProgress```; on the other hand, VGDP-MS waits DUCR/DDCR turns to ```InProgress``` before running the VGDP instance. ### VGDP-MS VGDP-MS is represented by ```velero data-mover``` subcommand and has its own subcommand ```backup``` and ```restore```. Below diagram shows the VGDP-MS workflow: ![vgdp-ms-2.png](vgdp-ms-2.png) **Start DUCR/DDCR Watcher**: VGDP-MS needs to watch the corresponding DUCR/DDCR so as to react on some events happening to the DUCR/DDCR. E.g., when the data movement is cancelled, a ```Cancel``` flag is set to the DUCR/DDCR, by watching the DUCR/DDCR, VGDP-MS is able to see it and cancel the VGDP instance. **Wait DUCR/DDCR InProgress**: As mentioned above, VGDP-MS won't start the VGDP instance until DUCR/DDCR turns to ```InProgress```, by which time VGDP Watcher has been started. **Record VGDP Starts**: This generates the VGDP Instance Starts event. **VGDP Callbacks**: When VGDP comes to one of the terminal states (i.e., completed, failed, cancelled), the corresponding callback is called. **Record VGDP Ends**: This generates the VGDP Instance Completes/Fails/Cancelled event, and also generates backupPod/restorePod termination message. **Record VGDP Progress**: This periodically generates/updates the Progress event with totalBytes/bytesDone to indicate the progress of the data movement. **Set VGDP Output**: This writes the termination message to the backupPod/restorePod's termination log (by default, it is written to ```/dev/termination-log```). If VGDP completes, VGDP Instance Completes event and backupPod/restorePod termination shares the same message as below: ``` type BackupResult struct { SnapshotID string `json:"snapshotID"` EmptySnapshot bool `json:"emptySnapshot"` Source exposer.AccessPoint `json:"source,omitempty"` } ``` ``` type RestoreResult struct { Target exposer.AccessPoint `json:"target,omitempty"` } ``` ``` type AccessPoint struct { ByPath string `json:"byPath"` VolMode uploader.PersistentVolumeMode `json:"volumeMode"` } ``` The existing VGDP result structures are actually being reused, we just add the json markers so that they can be marshalled. As mentioned above, once VGDP-MS ends in any way, the backupPod/restorePod terminates and never restarts, so the end of VGDP-MS means the end of DU/DD. For Progress update, the existing Progress structure is being reused: ``` type Progress struct { TotalBytes int64 `json:"totalBytes,omitempty"` BytesDone int64 `json:"doneBytes,omitempty"` } ``` ### Log Collection During the running of VGDP instance, some logs are generated which are important for troubleshooting. This includes all the logs generated by the uploader and repository. Therefore, it is important to collect these logs. On the other hand, the logs are now generated in the backupPod/restorePod, while the backupPod/restorePod is deleted immediately after the data movement completes. Therefore, by default, ```velero debug``` is not able to collect these logs. As a solution, we use logrus's hook mechanism to redirect the backupPod/restorePod's logs into node-agent's log, so that ```velero debug``` could collect VGDP logs as is without any changes. Below diagram shows how VGDP logs are redirected: ![vgdp-ms-3.png](vgdp-ms-3.png) This log redirecting mechanism is thread safe since the hook acquires the write lock before writing the log buffer, so it guarantees that in the node-agent log there is no corruptions after redirecting the log, and the redirected logs and the original node-agent logs are not projected into each other. ### Resource Control The CPU/memory resource of backupPod/restorePod is configurable, which means users are allowed to configure resources per volume backup/restore. By default, the [Best Effort policy][5] is used, and users are allowed to change it through the ConfigMap specified by `velero node-agent` CLI's parameter `--node-agent-configmap`. Specifically, we add below structures to the ConfigMap: ``` type Configs struct { // PodResources is the resource config for various types of pods launched by node-agent, i.e., data mover pods. PodResources *PodResources `json:"podResources,omitempty"` } type PodResources struct { CPURequest string `json:"cpuRequest,omitempty"` MemoryRequest string `json:"memoryRequest,omitempty"` CPULimit string `json:"cpuLimit,omitempty"` MemoryLimit string `json:"memoryLimit,omitempty"` } ``` The string values must mactch Kubernetes Quantity expressions; for each resource, the "request" value must not be larger than the "limit" value. Otherwise, if any one of the values fail, all the resource configurations will be ignored. The configurations are loaded by node-agent at start time, so users can change the values in the configMap any time, but the changes won't effect until node-agent restarts. ## node-agent node-agent is still required. Even though VGDP is now not running inside node-agent, node-agent still hosts the data mover controller which reconciles DUCR/DDCR and operates DUCR/DDCR in other steps before the VGDP instance is started, i.e., Accept, Expose, etc. Privileged mode and root user are not required for node-agent anymore by Volume Snapshot Data Movement, however, they are still required by PVB(PodVolumeBackup) and PVR(PodVolumeRestore). Therefore, we will keep the node-agent deamonset as is, for any users who don't use PVB/PVR and have concern about the privileged mode/root user, they need to manually modify the deamonset spec to remove the dependencies. ## CRD Changes There is no changes to any CRD. ## Installation Changes No changes to installation, the backupPod/restorePod's configurations are all inherited from node-agent. ## Upgrade Upgrade is not impacted. ## CLI CLI is not changed. [1]: ../unified-repo-and-kopia-integration/unified-repo-and-kopia-integration.md [2]: ../volume-snapshot-data-movement/volume-snapshot-data-movement.md [3]: https://kubernetes.io/blog/2022/09/02/cosi-kubernetes-object-storage-management/ [4]: ../node-agent-concurrency.md [5]: https://kubernetes.io/docs/concepts/workloads/pods/pod-qos/ ================================================ FILE: design/Implemented/vgdp-micro-service-for-fs-backup/vgdp-micro-service-for-fs-backup.md ================================================ # VGDP Micro Service For fs-backup ## Glossary & Abbreviation **VGDP**: Velero Generic Data Path. The collective modules that is introduced in [Unified Repository design][1]. Velero uses these modules to finish data transmission for various purposes. It includes uploaders and the backup repository. **fs-backup**: Also known as pod volume backup (PVB)/pod volume restore (PVR). It is one of the primary backup methods built-in with Velero. It has been refactored in [Unified Repository design][1]. **PVB**: Pod Volume Backup, the internal name for backup part of fs-backup. **PVR**: Pod Volume Restore, the internal name for restore part of fs-backup. **Exposer**: Exposer is introduced in [Volume Snapshot Data Movement design][2] and is used to expose the volume snapshots/volumes for VGDP to access locally. **VGDP MS**: VGDP Micro Service, it is introduced in [VGDP Micro Service For Volume Snapshot Data Movement][3]. It hosts VGDP instances in dedicated backup/restore pods, instead of in node-agent pods. ## Background As described in [VGDP Micro Service For Volume Snapshot Data Movement][3], hosting VGDP instances in dedicated pods has solved many major problems and brought significant improvements in scalability. These improvements are also effective for fs-backup. And besides the benefits listed in [VGDP Micro Service For Volume Snapshot Data Movement][3], we can also see below ones specifically for fs-backup: - This enables fs-backup to support Windows workloads. Windows doesn't support propagate mount, so the current fs-backup solution doesn't work for Windows nodes and Windows workloads. However, if the final host-path for the source volume is mounted to the VGDP MS pods, it should work. - This enables fs-backup to reuse the existing VGDP features seamlessly, i.e., concurrency control, node selector, etc. By moving all VGDP instances out of node-agent pods, we would further get prepared for below important features and improvements: - NFS support: NFS volumes are mounted to VGDP MS pods, so node-agent pods don't need to restart when a new BSL is added. - Performance improvement for Kopia uploader restore ([#7725][9]): dedicated cache volumes could be mounted to the VGDP MS pods, without affecting node-agent pods. - Controllable resource usage for node-agent: node-agent pods are long running and so not suitable for data path activities as the OS usually reclaim memory in a lazy reclaim behavior, so the unused memory may be shown as occupied by node-agent pods, which misleads Kubernetes or other related sub system. After this change, node-agent pods no longer require large resource (CPU/memory) usage, so no obvious memory retain will be observed. - Simplify node-agent configuration: host-path mounts, root user and privileged mode are no longer required by node-agent; and the configuration differences of node-agent for linux and Windows nodes could be eliminated. ## Goals - Create a solution to make VGDP instances as micro services for fs-backup - Modify the fs-backup workflow to offload the VGDP work from node-agent to the VGDP MS - Create the mechanism for fs-backup to control and monitor the VGDP MS in various scenarios ## Non-Goals - The current solution covers the VGDP Micro Service for fs-backup itself, the potentional features/improvements that rely on this solution will be covered by further designs and implementations. ## Overview The solution is based on [VGDP Micro Service For Volume Snapshot Data Movement][3], the architecture is followed as is and existing components are not changed unless it is necessary. Below diagram shows how these components work together: ![vgdp-ms-1.png](vgdp-ms-1.png) Below lists the changed components, why and how: **Pod-Volume Exposer**: A new exposer, pod-volume exposer is added. It retrieves the host path of the specific volume and then creates the backupPod/restorePod and mounts the host path to the pod. The command of the backupPod/restorePod is also changed to launch VGDP MS for PVB/PVR. **PVB/PVR Controller**: The PVB/PVR controllers are refactored to work with podVolume exposer, VGDP-MS, etc. The controllers will also support Cancel and resume. So PVB/PVR CRD is also refactored to support these scenarios. **PVB/PVR VGDP-MS**: New commands for PVB/PVR VGDP-MS are added. The VGDP instances are started in the backupPod/restorePod as result of the commands. The VGDP Watcher and its mechanism are fully reused. The [Node-agent concurrency][4] is reused to control the concurrency of VGDP MS for fs-backup. When there are too many volumes in the backup/restore, which takes too much computing resources(CPU, memory, etc.) or Kubernetes resources(pods, PVCs, PVs, etc.), users could set the concurrency in each node so as to control the total number of concurrent VGDP instances in the cluster. ## Detailed Design ### Exposer As the old behavior, the host path (e.g., `/var/lib/kubelet/pods`) for the Kubernetes pods are mounted to node-agent pods, then the VGDP instances running in the same pods access the data through subdir of the host path for a specific volume, e.g., `/var/lib/kubelet/pods//volumes/kubernetes.io~csi//mount`. Therefore, a node-agent pod could access all volumes attached to the same node. For the new implementation, the exposer retrieves the host path for a specific volume directly, and then mount that host path to the backupPod/restorePod. This also means that the backupPod/restorePod could only access the volume to be backed up or restored. The exposer creates backupPod/restorePod and sets ```velero pod-volume``` as the command run by backupPod/restorePod. And `velero` image is used for the backupPod/restorePod. There are sub commands varying from backup and restore: ```velero pod-volume backup --volume-path xxx --pod-volume-backup xxx --resource-timeout xxx --log-format xxx --log-level xxx``` Or: ```velero pod-volume restore --volume-path xxx --pod-volume-restore xxx --resource-timeout xxx --log-format xxx --log-level xxx``` Below are the parameters of the commands: **volume-path**: Deliver the full path inside the backupPod/restorePod for the volume to be backed up/restored. **pod-volume-backup**: PVB CR for this backup. **pod-volume-restore**: PVR CR for this restore. **resource-timeout**: resource-timeout is used to control the timeout for operations related to resources. It has the same meaning with the resource-timeout for node-agent. **log-format** and **log-level**: This is to control the behavior of log generation inside VGDP-MS. Below pod configurations are inherited from node-agent and set to backupPod/restorePod's spec: - Volumes: Some configMaps will be mapped as volumes to node-agent, so we add the same volumes of node-agent to the backupPod/restorePod - Environment Variables - Security Contexts Since the volume data is still accessed by host path, the backupPod/restorePod may still need to run in Privileged mode in some environments. Therefore, the Privileged mode setting which is a part of Security Contexts will be inherited from node-agent. The root user is still required, especially by the restore (in order to restore the file system attributes, owners, etc.), so we will use root user for backupPod/restorePod. As same as [VGDP Micro Service For Volume Snapshot Data Movement][3], the backupPod/restorePods's ```RestartPolicy``` is set to ```RestartPolicyNever```, so that once VGDP-MS terminates for any reason, backupPod/restorePod won't restart and the PVB/PVR is marked as one of the terminal phases (Completed/Failed/Cancelled) accordingly. ### VGDP Watcher The VGDP watcher is fully reused, specifically, we still use the dual mode event watcher to watch the status change from backupPod/restorePod or the VGDP instance. The AsyncBR adapter and its interface is also fully reused. ### VGDP-MS The VGDP-MS that is represented by ```velero pod-volume``` keeps the same workflow as [VGDP Micro Service For Volume Snapshot Data Movement][3]: ![vgdp-ms-2.png](vgdp-ms-2.png) **Start DUCR/DDCR Watcher**: The same as [VGDP Micro Service For Volume Snapshot Data Movement][3], except that it watches PVB/PVR CRs. **Wait DUCR/DDCR InProgress**: The same as The same as [VGDP Micro Service For Volume Snapshot Data Movement][3], VGDP-MS won't start the VGDP instance until PVB/PVR CR turns to ```InProgress```. **Record VGDP Starts**: The same as [VGDP Micro Service For Volume Snapshot Data Movement][3]. **VGDP Callbacks**: The same as [VGDP Micro Service For Volume Snapshot Data Movement][3]. **Record VGDP Ends**: The same as [VGDP Micro Service For Volume Snapshot Data Movement][3]. **Record VGDP Progress**: The same as [VGDP Micro Service For Volume Snapshot Data Movement][3]. **Set VGDP Output**: The same as [VGDP Micro Service For Volume Snapshot Data Movement][3]. The return message for VGDP completion is also reused, except that `VolMode` is always set to `PersistentVolumeFilesystem`: ``` type BackupResult struct { SnapshotID string `json:"snapshotID"` EmptySnapshot bool `json:"emptySnapshot"` Source exposer.AccessPoint `json:"source,omitempty"` } ``` ``` type RestoreResult struct { Target exposer.AccessPoint `json:"target,omitempty"` } ``` ``` type AccessPoint struct { ByPath string `json:"byPath"` VolMode uploader.PersistentVolumeMode `json:"volumeMode"` } ``` And the mechanism and data struct for Progress update is also reused: ``` type Progress struct { TotalBytes int64 `json:"totalBytes,omitempty"` BytesDone int64 `json:"doneBytes,omitempty"` } ``` ### Log Collection The log collection mechanism is the same as [VGDP Micro Service For Volume Snapshot Data Movement][3]. ### Resource Control The resource control mechanism is the same as [VGDP Micro Service For Volume Snapshot Data Movement][3]. ### Restic Restore As the current Restic path deprecation process, restore is still supported. On the other hand, we don't want to support Restic path for this new VGDP MS implementation. Therefore, the legacy PVR controller and workflow is preserved for Restic path restore. The controller watches legacy PVRs only, and then launches the legacy workflow. Meawhile, the new PVR controller should skip legacy PVRs. After Restic path is full deprecated, the code for the legacy controller and workflow should be removed. ### Velero Server Restarts The backup/restore stays in InProgress phase during the running of PVB/PVR, no phase changes between completion of item iteration and completion of PVB/PVR. As a result, on Velero server restarts, there is no way to resume a backup/restore. Therefore, the backup/restore will be be marked as Failed, which is the same as the old behavior. And it is still not as good as CSI snapshot data movement for which the backup/restore could be resumed as long as it has iterated all items. By the meanwhile, there is indeed some improvements. As the old behavior, once the backup/restore is set as Failed on Velero server restart, the running PVB/PVR will be left there, as a result, the VGDP instances may run for a long time and take lots of resource for nothing; for the new implementation, PVB/PVR will be set as Cancel immediately after the backup/restore is set as Failed. ### node-agent Restarts As the old behavior, once a node-agent pod restarts, all the PVBs/PVRs running in the same node will be set as Failed as there is no way to resume the VGDP instances for them. For the new implementation, since the VGDP instances run in dedicated backupPods/restorePods without affected, the PVBs/PVRs will be resumed after node-agent restarts. This includes PVBs/PVRs in all phases. The legacy PVRs handling Restic restore are processed by the old workflow, so they will still be set as Failed on node-agent restart. ### Windows Support Windows nodes and workloads will be supported by following the same changes for CSI snapshot data movement as listed in [Velero Windows Support][7]. There are some additional changes particularly for PVB/PVR. #### Restore Helper PVR requires an init-container, called `restore-wait`, to run in the workload pod. There are default configurations for the container and users could customize them by the `pod-volume-restore` RIA plugin configMap. The `pod-volume-restore` RIA is used to config the init-container, so it should support Windows pods for all the configurations. Meanwhile, the customized options in the configMap should also support Windows pods. If an option is not suitable for Windows pods, it will be ignored by the RIA. By default, the init-container uses `velero` image with a binary called `velero-restore-helper` inside, so that binary should be compiled and assembled to the `velero` image for Windows. #### Privileged mode Privileged pods are implemented by [HostProcess Pods][8] on Windows and need to be specially configured. And there are many constrains for it. As one of the constrains, HostProcess pods supports Windows service accounts only. As a result, restore will not be able to support it until [#8423][10] is fixed, otherwise, the restored files are not usable by workloads which run under genneral container users, e.g., `containerUser` or `containerAdministrator`. Therefore, as the current implementation, fs-backup will not support Windows workloads in the environments where Privileged mode is required. A limitation should be documented. ## node-agent node-agent is required to host the PVB/PVR controller which reconciles PVB/PVR and operates PVB/PVR in other steps before the VGDP instance is started, i.e., Accept, Expose, etc. node-agent still requires host path mount because of two deprecating features [in-tree storage provider support deprecation][5] and [emptyDir volume support deprecation][6]. As a result, Privileged mode and root user are still required in some environments. Therefore, we will keep the node-agent deamonset as is, until the two deprecations complete. ## CRD Changes In order to support the VGDP MS workflow, some elements in the PVB/PVR CRDs are added or extended: - New phases are added for PVB/PVR: `PodVolumeBackupPhaseAccepted`, `PodVolumeBackupPhasePrepared`, `PodVolumeBackupPhaseCanceling`, `PodVolumeBackupPhaseCanceled`; `PodVolumeRestorePhaseAccepted`, `PodVolumeRestorePhasePrepared`, `PodVolumeRestorePhaseCanceling`, `PodVolumeRestorePhaseCanceled`. - New fields are added to PVB/PVR spec to support cancel: `Cancel bool` - New fields are added to PVB/PVR spec to support the accept phase and processing: `AcceptedTimestamp *metav1.Time` - A new field, which records the node the PVR is running, is added to PVR Status: `Node string` New changes happen to Backup/Restore CRDs. Below is the new PVB CRD: ```yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.16.5 name: podvolumebackups.velero.io spec: group: velero.io names: kind: PodVolumeBackup listKind: PodVolumeBackupList plural: podvolumebackups singular: podvolumebackup scope: Namespaced versions: - additionalPrinterColumns: - description: PodVolumeBackup status such as New/InProgress jsonPath: .status.phase name: Status type: string - description: Time duration since this PodVolumeBackup was started jsonPath: .status.startTimestamp name: Started type: date - description: Completed bytes format: int64 jsonPath: .status.progress.bytesDone name: Bytes Done type: integer - description: Total bytes format: int64 jsonPath: .status.progress.totalBytes name: Total Bytes type: integer - description: Name of the Backup Storage Location where this backup should be stored jsonPath: .spec.backupStorageLocation name: Storage Location type: string - description: Time duration since this PodVolumeBackup was created jsonPath: .metadata.creationTimestamp name: Age type: date - description: Name of the node where the PodVolumeBackup is processed jsonPath: .status.node name: Node type: string - description: The type of the uploader to handle data transfer jsonPath: .spec.uploaderType name: Uploader type: string name: v1 schema: openAPIV3Schema: properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: PodVolumeBackupSpec is the specification for a PodVolumeBackup. properties: backupStorageLocation: description: |- BackupStorageLocation is the name of the backup storage location where the backup repository is stored. type: string cancel: description: |- Cancel indicates request to cancel the ongoing PodVolumeBackup. It can be set when the PodVolumeBackup is in InProgress phase type: boolean node: description: Node is the name of the node that the Pod is running on. type: string pod: description: Pod is a reference to the pod containing the volume to be backed up. properties: apiVersion: description: API version of the referent. type: string fieldPath: description: |- If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: "spec.containers{name}" (where "name" refers to the name of the container that triggered the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. type: string kind: description: |- Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: description: |- Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: description: |- Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: description: |- UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object x-kubernetes-map-type: atomic repoIdentifier: description: RepoIdentifier is the backup repository identifier. type: string tags: additionalProperties: type: string description: |- Tags are a map of key-value pairs that should be applied to the volume backup as tags. type: object uploaderSettings: additionalProperties: type: string description: |- UploaderSettings are a map of key-value pairs that should be applied to the uploader configuration. nullable: true type: object uploaderType: description: UploaderType is the type of the uploader to handle the data transfer. enum: - kopia - "" type: string volume: description: |- Volume is the name of the volume within the Pod to be backed up. type: string required: - backupStorageLocation - node - pod - repoIdentifier - volume type: object status: description: PodVolumeBackupStatus is the current status of a PodVolumeBackup. properties: acceptedTimestamp: description: |- AcceptedTimestamp records the time the pod volume backup is to be prepared. The server's time is used for AcceptedTimestamp format: date-time nullable: true type: string completionTimestamp: description: |- CompletionTimestamp records the time a backup was completed. Completion time is recorded even on failed backups. Completion time is recorded before uploading the backup object. The server's time is used for CompletionTimestamps format: date-time nullable: true type: string message: description: Message is a message about the pod volume backup's status. type: string path: description: Path is the full path within the controller pod being backed up. type: string phase: description: Phase is the current state of the PodVolumeBackup. enum: - New - Accepted - Prepared - InProgress - Canceling - Canceled - Completed - Failed type: string progress: description: |- Progress holds the total number of bytes of the volume and the current number of backed up bytes. This can be used to display progress information about the backup operation. properties: bytesDone: format: int64 type: integer totalBytes: format: int64 type: integer type: object snapshotID: description: SnapshotID is the identifier for the snapshot of the pod volume. type: string startTimestamp: description: |- StartTimestamp records the time a backup was started. Separate from CreationTimestamp, since that value changes on restores. The server's time is used for StartTimestamps format: date-time nullable: true type: string type: object type: object served: true storage: true subresources: {} ``` Below is the new PVR CRD: ```yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.16.5 name: podvolumerestores.velero.io spec: group: velero.io names: kind: PodVolumeRestore listKind: PodVolumeRestoreList plural: podvolumerestores singular: podvolumerestore scope: Namespaced versions: - additionalPrinterColumns: - description: PodVolumeRestore status such as New/InProgress jsonPath: .status.phase name: Status type: string - description: Time duration since this PodVolumeRestore was started jsonPath: .status.startTimestamp name: Started type: date - description: Completed bytes format: int64 jsonPath: .status.progress.bytesDone name: Bytes Done type: integer - description: Total bytes format: int64 jsonPath: .status.progress.totalBytes name: Total Bytes type: integer - description: Name of the Backup Storage Location where the backup data is stored jsonPath: .spec.backupStorageLocation name: Storage Location type: string - description: Time duration since this PodVolumeRestore was created jsonPath: .metadata.creationTimestamp name: Age type: date - description: Name of the node where the PodVolumeRestore is processed jsonPath: .status.node name: Node type: string - description: The type of the uploader to handle data transfer jsonPath: .spec.uploaderType name: Uploader Type type: string name: v1 schema: openAPIV3Schema: properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: PodVolumeRestoreSpec is the specification for a PodVolumeRestore. properties: backupStorageLocation: description: |- BackupStorageLocation is the name of the backup storage location where the backup repository is stored. type: string cancel: description: |- Cancel indicates request to cancel the ongoing PodVolumeRestore. It can be set when the PodVolumeRestore is in InProgress phase type: boolean pod: description: Pod is a reference to the pod containing the volume to be restored. properties: apiVersion: description: API version of the referent. type: string fieldPath: description: |- If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: "spec.containers{name}" (where "name" refers to the name of the container that triggered the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. type: string kind: description: |- Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: description: |- Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: description: |- Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: description: |- UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object x-kubernetes-map-type: atomic repoIdentifier: description: RepoIdentifier is the backup repository identifier. type: string snapshotID: description: SnapshotID is the ID of the volume snapshot to be restored. type: string sourceNamespace: description: SourceNamespace is the original namespace for namespace mapping. type: string uploaderSettings: additionalProperties: type: string description: |- UploaderSettings are a map of key-value pairs that should be applied to the uploader configuration. nullable: true type: object uploaderType: description: UploaderType is the type of the uploader to handle the data transfer. enum: - kopia - "" type: string volume: description: Volume is the name of the volume within the Pod to be restored. type: string required: - backupStorageLocation - pod - repoIdentifier - snapshotID - sourceNamespace - volume type: object status: description: PodVolumeRestoreStatus is the current status of a PodVolumeRestore. properties: acceptedTimestamp: description: |- AcceptedTimestamp records the time the pod volume restore is to be prepared. The server's time is used for AcceptedTimestamp format: date-time nullable: true type: string completionTimestamp: description: |- CompletionTimestamp records the time a restore was completed. Completion time is recorded even on failed restores. The server's time is used for CompletionTimestamps format: date-time nullable: true type: string message: description: Message is a message about the pod volume restore's status. type: string node: description: Node is name of the node where the pod volume restore is processed. type: string phase: description: Phase is the current state of the PodVolumeRestore. enum: - New - Accepted - Prepared - InProgress - Canceling - Canceled - Completed - Failed type: string progress: description: |- Progress holds the total number of bytes of the snapshot and the current number of restored bytes. This can be used to display progress information about the restore operation. properties: bytesDone: format: int64 type: integer totalBytes: format: int64 type: integer type: object startTimestamp: description: |- StartTimestamp records the time a restore was started. The server's time is used for StartTimestamps format: date-time nullable: true type: string type: object type: object served: true storage: true subresources: {} ``` ## Installation Changes No changes to installation, the backupPod/restorePod's configurations are either inherited from node-agent or retrieved from node-agent-configmap. ## Upgrade Upgrade is not impacted. ## CLI CLI is not changed. [1]: ../unified-repo-and-kopia-integration/unified-repo-and-kopia-integration.md [2]: ../volume-snapshot-data-movement/volume-snapshot-data-movement.md [3]: ../vgdp-micro-service/vgdp-micro-service.md [4]: ../node-agent-concurrency.md [5]: https://github.com/vmware-tanzu/velero/issues/8955 [6]: https://github.com/vmware-tanzu/velero/issues/8956 [7]: https://github.com/vmware-tanzu/velero/issues/8289 [8]: https://kubernetes.io/docs/tasks/configure-pod-container/create-hostprocess-pod/ [9]: https://github.com/vmware-tanzu/velero/issues/7725 [10]: https://github.com/vmware-tanzu/velero/issues/8423 ================================================ FILE: design/Implemented/volume-group-snapshot.md ================================================ # Add Support for VolumeGroupSnapshots This proposal outlines the design and implementation plan for incorporating VolumeGroupSnapshot support into Velero. The enhancement will allow Velero to perform consistent, atomic snapshots of groups of Volumes using the new Kubernetes [VolumeGroupSnapshot API](https://kubernetes.io/blog/2024/12/18/kubernetes-1-32-volume-group-snapshot-beta/). This capability is especially critical for stateful applications that rely on multiple volumes to ensure data consistency, such as databases and analytics workloads. ## Glossary & Abbreviation Terminology used in this document: - VGS: VolumeGroupSnapshot - VS: VolumeSnapshot - VGSC: VolumeGroupSnapshotContent - VSC: VolumeSnapshotContent - VGSClass: VolumeGroupSnapshotClass - VSClass: VolumeSnapshotClass ## Background Velero currently enables snapshot-based backups on an individual Volume basis through CSI drivers. However, modern stateful applications often require multiple volumes for data, logs, and backups. This distributed data architecture increases the risk of inconsistencies when volumes are captured individually. Kubernetes has introduced the VolumeGroupSnapshot(VGS) API [(KEP-3476)](https://github.com/kubernetes/enhancements/pull/1551), which allows for the atomic snapshotting of multiple volumes in a coordinated manner. By integrating this feature, Velero can offer enhanced disaster recovery for multi-volume applications, ensuring consistency across all related data. ## Goals - Ensure that multiple related volumes are snapshotted simultaneously, preserving consistency for stateful applications via VolumeGroupSnapshots(VGS) API. - Integrate VolumeGroupSnapshot functionality into Velero’s existing backup and restore workflows. - Allow users to opt in to volume group snapshots via specifying the group label. ## Non-Goals - The proposal does not require a complete overhaul of Velero’s CSI integration, it will extend the current mechanism to support group snapshots. - No any changes pertaining to execution of Restore Hooks ## High-Level Design ### Backup workflow: #### Accept the label to be used for VGS from the user: - Accept the label from the user, we will do this in 3 ways: - Firstly, we will have a hard-coded default label key like `velero.io/volume-group-snapshot` that the users can directly use on their PVCs. - Secondly, we will let the users override this default VGS label via a velero server arg, `--volume-group-nsaphot-label-key`, if needed. - And Finally we will have the option to override the default label via Backup API spec, `backup.spec.volumeGroupSnapshotLabelKey` - In all the instances, the VGS label key will be present on the backup spec, this makes the label key accessible to plugins during the execution of backup operation. - This label will enable velero to filter the PVC to be included in the VGS spec. - Users will have to label the PVCs before invoking the backup operation. - This label would act as a group identifier for the PVCs to be grouped under a specific VGS. - It will be used to collect the PVCs to be used for a particular instance of VGS object. **Note:** - Modifying or adding VGS label on PVCs during an active backup operation may lead to unexpected or undesirable backup results. To avoid inconsistencies, ensure PVC labels remain unchanged throughout the backup execution. - Label Key Precedence: When determining which label key to use for grouping PVCs into a VolumeGroupSnapshot, Velero applies overrides in the following order (highest to lowest): - Backup API spec (`backup.spec.volumeGroupSnapshotLabelKey`) - Server flag (`--volume-group-snapshot-label-key`) - Built-in default (`velero.io/volume-group-snapshot`) Whichever key wins this precedence is then injected into the Backup spec so that all Velero plugins can uniformly discover and use it during the backup execution. #### Changes to the Existing PVC ItemBlockAction plugin: - Currently the PVC IBA plugin is applied to PVCs and adds the RelatedItems for the particular PVC into the ItemBlock. - At first it checks whether the PVC is bound and VolumeName is non-empty. - Then it adds the related PV under the list of relatedItems. - Following on, the plugin adds the pods mounting the PVC as relatedItems. - Now we need to extend this PVC IBA plugin to add the PVCs to be grouped for a particular VGS object, so that they are processed together under an ItemBlock by Velero. - First we will check if the PVC that is being processed by the plugin has the user specified VGS label. - If it is present then we will execute a List call in the namespace with the label as a matching criteria and see if this results in any PVCs (other than the current one). - If there are PVCs matching the criteria then we add the PVCs to the relatedItems list. - This helps in building the ItemBlock we need for VGS processing, i.e. we have the relevant pods and PVCs in the ItemBlock. **Note:** The ItemBlock to VGS relationship will not always be 1:1. There might be scenarios when the ItemBlock might have multiple VGS instances associated with it. Lets go over some ItemBlock/VGS scenarios that we might encounter and visualize them for clarity: 1. Pod Mounts: Pod1 mounts both PVC1 and PVC2. Grouping: PVC1 and PVC2 share the same group label (group: A) ItemBlock: The item block includes Pod1, PVC1, and PVC2. VolumeGroupSnapshot (VGS): Because PVC1 and PVC2 are grouped together by their label, they trigger the creation of a single VGS (labeled with group: A). ```mermaid flowchart TD subgraph ItemBlock P1[Pod1] PVC1[PVC1 group: A] PVC2[PVC2 group: A] end P1 -->|mounts| PVC1 P1 -->|mounts| PVC2 PVC1 --- PVC2 PVC1 -- "group: A" --> VGS[VGS group: A] PVC2 -- "group: A" --> VGS ``` 2. Pod Mounts: Pod1 mounts each of the four PVCs. Grouping: Group A: PVC1 and PVC2 share the same grouping label (group: A). Group B: PVC3 and PVC4 share the grouping label (group: B) ItemBlock: All objects (Pod1, PVC1, PVC2, PVC3, and PVC4) are collected into a single item block. VolumeGroupSnapshots: PVC1 and PVC2 (group A) point to the same VGS (VGS (group: A)). PVC3 and PVC4 (group B) point to a different VGS (VGS (group: B)). ```mermaid flowchart TD subgraph ItemBlock P1[Pod1] PVC1[PVC1 group: A] PVC2[PVC2 group: A] PVC3[PVC3 group: B] PVC4[PVC4 group: B] end %% Pod mounts all PVCs P1 -->|mounts| PVC1 P1 -->|mounts| PVC2 P1 -->|mounts| PVC3 P1 -->|mounts| PVC4 %% Group A relationships: PVC1 and PVC2 PVC1 --- PVC2 PVC1 -- "group: A" --> VGS_A[VGS-A group: A] PVC2 -- "group: A" --> VGS_A %% Group B relationships: PVC3 and PVC4 PVC3 --- PVC4 PVC3 -- "group: B" --> VGS_B[VGS-B group: B] PVC4 -- "group: B" --> VGS_B ``` 3. Pod Mounts: Pod1 mounts both PVC1 and PVC2, Pod2 mounts PVC1 and PVC3. Grouping: Group A: PVC1 and PVC2 Group B: PVC3 ItemBlock: All objects-Pod1, Pod2, PVC1, PVC2, and PVC3, are collected into a single item block. VolumeGroupSnapshots: PVC1 and PVC2 (group A) point to the same VGS (VGS (group: A)). PVC3 (group B) point to a different VGS (VGS (group: B)). ```mermaid flowchart TD subgraph ItemBlock P1[Pod1] P2[Pod2] PVC1[PVC1 group: A] PVC2[PVC2 group: A] PVC3[PVC3 group: B] end %% Pod mount relationships P1 -->|mounts| PVC1 P1 -->|mounts| PVC2 P2 -->|mounts| PVC1 P2 -->|mounts| PVC3 %% Grouping for Group A: PVC1 and PVC2 are grouped into VGS_A PVC1 --- PVC2 PVC1 -- "Group A" --> VGS_A[VGS Group A] PVC2 -- "Group A" --> VGS_A %% Grouping for Group B: PVC3 grouped into VGS_B PVC3 -- "Group B" --> VGS_B[VGS Group B] ``` #### Updates to CSI PVC plugin: The CSI PVC plugin now supports obtaining a VolumeSnapshot (VS) reference for a PVC in three ways, and then applies common branching for datamover and non‑datamover workflows: - Scenario 1: PVC has a VGS label and no VS (created via the VGS workflow) exists for its volume group: - Determine VGSClass: The plugin will pick `VolumeGroupSnapshotClass` by following the same tier based precedence as it does for individual `VolumeSnapshotClasses`: - Default by Label: Use the one VGSClass labeled ```yaml metadata: labels: velero.io/csi-volumegroupsnapshot-class: "true" ``` whose `spec.driver` matches the CSI driver used by the PVCs. - Backup‑level Override: If the Backup CR has an annotation ```yaml metadata: annotations: velero.io/csi-volumegroupsnapshot-class_: ``` (with equal to the PVCs’ CSI driver), use that class. - PVC‑level Override: Finally, if the PVC itself carries an annotation ```yaml metadata: annotations: velero.io/csi-volume-group-snapshot-class: ``` and that class exists, use it. At each step, if the plugin finds zero or multiple matching classes, VGS creation is skipped and backup fails. - Create VGS: The plugin creates a new VolumeGroupSnapshot (VGS) for the PVC’s volume group. This action automatically triggers creation of the corresponding VGSC, VS, and VSC objects. - Wait for VS Status: The plugin waits until each VS (one per PVC in the group) has its `volumeGroupSnapshotName` populated. This confirms that the snapshot controller has completed its work. `CSISnapshotTimeout` will be used here. - Update VS Objects: Once the VS objects are provisioned, the plugin updates them by removing VGS owner references and VGS-related finalizers, and by adding backup metadata labels (including BackupName, BackupUUID, and PVC name). These labels are later used to detect an existing VS when processing another PVC of the same group. - Patch and Cleanup: The plugin patches the deletionPolicy of the VGSC to "Retain" (ensuring that deletion of the VGSC does not remove the underlying VSC objects or storage snapshots) and then deletes the temporary VGS and VGSC objects. - Scenario 2: PVC has a VGS label and a VS created via an earlier VGS workflow already exists: - The plugin lists VS objects in the PVC’s namespace using backup metadata labels (BackupUID, BackupName, and PVCName). - It verifies that at least one VS has a non‑empty `volumeGroupSnapshotName` in its status. - If such a VS exists, the plugin skips creating a new VGS (or VS) and proceeds with the legacy workflow using the existing VS. - If a VS is found but its status does not indicate it was created by the VGS workflow (i.e. its `volumeGroupSnapshotName` is empty), the backup for that PVC is failed, resulting in a partially failed backup. - Scenario 3: PVC does not have a VGS label: - The legacy workflow is followed, and an individual VolumeSnapshot (VS) is created for the PVC. - Common Branching for Datamover and Non‑datamover Workflows: - Once a VS reference (`vsRef`) is determined—whether through the VGS workflow (Scenario 1 or 2) or the legacy workflow (Scenario 3)—the plugin then applies the common branching: - Non‑datamover Case: The VS reference is directly added as an additional backup item. - Datamover Case: The plugin waits until the VS’s associated VSC snapshot handle is ready (using the configured CSISnapshotTimeout), then creates a DataUpload for the VS–PVC pair. The resulting DataUpload is then added as an additional backup item. ```mermaid flowchart TD %% Section 1: Accept VGS Label from User subgraph Accept_Label A1[User sets VGS label key using default velero.io/volume-group-snapshot or via server arg or Backup API spec] A2[User labels PVCs before backup] A1 --> A2 end %% Section 2: PVC ItemBlockAction Plugin Extension subgraph PVC_ItemBlockAction B1[Check PVC is bound and has VolumeName] B2[Add related PV to relatedItems] B3[Add pods mounting PVC to relatedItems] B4[Check if PVC has user-specified VGS label] B5[List PVCs in namespace matching label criteria] B6[Add matching PVCs to relatedItems] B1 --> B2 --> B3 --> B4 B4 -- Yes --> B5 B5 --> B6 end %% Section 3: CSI PVC Plugin Updates subgraph CSI_PVC_Plugin C1[For each PVC, check for VGS label] C1 -- Has VGS label --> C2[Determine scenario] C1 -- No VGS label --> C16[Scenario 3: Legacy workflow - create individual VS] %% Scenario 1: No existing VS via VGS exists subgraph Scenario1[Scenario 1: No existing VS via VGS] S1[List grouped PVCs using VGS label] S2[Determine CSI driver for grouped PVCs] S3[If single CSI driver then select matching VGSClass; else fail backup] S4[Create new VGS triggering VGSC, VS, and VSC creation] S5[Wait for VS objects to have nonempty volumeGroupSnapshotName] S6[Update VS objects; remove VGS owner refs and finalizers; add backup metadata labels] S7[Patch VGSC deletionPolicy to Retain] S8[Delete transient VGS and VGSC] S1 --> S2 --> S3 --> S4 --> S5 --> S6 --> S7 --> S8 end %% Scenario 2: Existing VS via VGS exists subgraph Scenario2[Scenario 2: Existing VS via VGS exists] S9[List VS objects using backup metadata - BackupUID, BackupName, PVCName] S10[Check if any VS has nonempty volumeGroupSnapshotName] S9 --> S10 S10 -- Yes --> S11[Use existing VS] S10 -- No --> S12[Fail backup for PVC] end C2 -- Scenario1 applies --> S1 C2 -- Scenario2 applies --> S9 %% Common Branch: After obtaining a VS reference subgraph Common_Branch[Common Branch] CB1[Obtain VS reference as vsRef] CB2[If non-datamover, add vsRef as additional backup item] CB3[If datamover, wait for VSC handle and create DataUpload; add DataUpload as additional backup item] CB1 --> CB2 CB1 --> CB3 end %% Connect Scenario outcomes and legacy branch to the common branch S8 --> CB1 S11 --> CB1 C16 --> CB1 end %% Overall Flow Connections A2 --> B1 B6 --> C1 ``` Restore workflow: - No changes required for the restore workflow. ## Detailed Design Backup workflow: - Accept the label to be used for VGS from the user as a server argument: - Set a default VGS label key to be used: ```go // default VolumeGroupSnapshot Label defaultVGSLabelKey = "velero.io/volume-group-snapshot" ``` - Add this as a server flag and pass it to backup reconciler, so that we can use it during the backup request execution. ```go flags.StringVar(&c.DefaultVGSLabelKey, "volume-group-snapshot-label-key", c.DefaultVGSLabelKey, "Label key for grouping PVCs into VolumeGroupSnapshot") ``` - Update the Backup CRD to accept the VGS Label Key as a spec value: ```go // VolumeGroupSnapshotLabelKey specifies the label key to be used for grouping the PVCs under // an instance of VolumeGroupSnapshot, if left unspecified velero.io/volume-group-snapshot is used // +optional VolumeGroupSnapshotLabelKey string `json:"volumeGroupSnapshotLabelKey,omitempty"` ``` - Modify the [`prepareBackupRequest` function](https://github.com/openshift/velero/blob/8c8a6cccd78b78bd797e40189b0b9bee46a97f9e/pkg/controller/backup_controller.go#L327) to set the default label key as a backup spec if the user does not specify any value: ```go if len(request.Spec.VolumeGroupSnapshotLabelKey) == 0 { // set the default key value request.Spec.VolumeGroupSnapshotLabelKey = b.defaultVGSLabelKey } ``` - Changes to the Existing [PVC ItemBlockAction plugin](https://github.com/vmware-tanzu/velero/blob/512199723ff95d5016b32e91e3bf06b65f57d608/pkg/itemblock/actions/pvc_action.go#L64) (Update the GetRelatedItems function): ```go // Retrieve the VGS label key from the Backup spec. vgsLabelKey := backup.Spec.VolumeGroupSnapshotLabelKey if vgsLabelKey != "" { // Check if the PVC has the specified VGS label. if groupID, ok := pvc.Labels[vgsLabelKey]; ok { // List all PVCs in the namespace with the same label key and value (i.e. same group). pvcList := new(corev1api.PersistentVolumeClaimList) if err := a.crClient.List(context.Background(), pvcList, crclient.InNamespace(pvc.Namespace), crclient.MatchingLabels{vgsLabelKey: groupID}); err != nil { return nil, errors.Wrap(err, "failed to list PVCs for VGS grouping") } // Add each matching PVC (except the current one) to the relatedItems. for _, groupPVC := range pvcList.Items { if groupPVC.Name == pvc.Name { continue } a.log.Infof("Adding grouped PVC %s to relatedItems for PVC %s", groupPVC.Name, pvc.Name) relatedItems = append(relatedItems, velero.ResourceIdentifier{ GroupResource: kuberesource.PersistentVolumeClaims, Namespace: groupPVC.Namespace, Name: groupPVC.Name, }) } } } else { a.log.Info("No VolumeGroupSnapshotLabelKey provided in backup spec; skipping PVC grouping") } ``` - Updates to [CSI PVC plugin](https://github.com/vmware-tanzu/velero/blob/512199723ff95d5016b32e91e3bf06b65f57d608/pkg/backup/actions/csi/pvc_action.go#L200) (Update the Execute method): ```go func (p *pvcBackupItemAction) Execute( item runtime.Unstructured, backup *velerov1api.Backup, ) ( runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error, ) { p.log.Info("Starting PVCBackupItemAction") // Validate backup policy and PVC/PV if valid := p.validateBackup(*backup); !valid { return item, nil, "", nil, nil } var pvc corev1api.PersistentVolumeClaim if err := runtime.DefaultUnstructuredConverter.FromUnstructured(item.UnstructuredContent(), &pvc); err != nil { return nil, nil, "", nil, errors.WithStack(err) } if valid, item, err := p.validatePVCandPV(pvc, item); !valid { if err != nil { return nil, nil, "", nil, err } return item, nil, "", nil, nil } shouldSnapshot, err := volumehelper.ShouldPerformSnapshotWithBackup( item, kuberesource.PersistentVolumeClaims, *backup, p.crClient, p.log, ) if err != nil { return nil, nil, "", nil, err } if !shouldSnapshot { p.log.Debugf("CSI plugin skip snapshot for PVC %s according to VolumeHelper setting", pvc.Namespace+"/"+pvc.Name) return nil, nil, "", nil, nil } var additionalItems []velero.ResourceIdentifier var operationID string var itemToUpdate []velero.ResourceIdentifier // vsRef will be our common reference to the VolumeSnapshot (VS) var vsRef *corev1api.ObjectReference // Retrieve the VGS label key from the backup spec. vgsLabelKey := backup.Spec.VolumeGroupSnapshotLabelKey // Check if the PVC has the user-specified VGS label. if group, ok := pvc.Labels[vgsLabelKey]; ok && group != "" { p.log.Infof("PVC %s has VGS label with group %s", pvc.Name, group) // --- VGS branch --- // 1. Check if a VS created via a VGS workflow exists for this PVC. existingVS, err := p.findExistingVSForBackup(backup.UID, backup.Name, pvc.Name, pvc.Namespace) if err != nil { return nil, nil, "", nil, err } if existingVS != nil && existingVS.Status.VolumeGroupSnapshotName != "" { p.log.Infof("Existing VS %s found for PVC %s in group %s; skipping VGS creation", existingVS.Name, pvc.Name, group) vsRef = &corev1api.ObjectReference{ Namespace: existingVS.Namespace, Name: existingVS.Name, } } else { // 2. No existing VS via VGS; execute VGS creation workflow. groupedPVCs, err := p.listGroupedPVCs(backup, pvc.Namespace, vgsLabelKey, group) if err != nil { return nil, nil, "", nil, err } pvcNames := extractPVCNames(groupedPVCs) // Determine the CSI driver used by the grouped PVCs. driver, err := p.determineCSIDriver(groupedPVCs) if err != nil { return nil, nil, "", nil, errors.Wrap(err, "failed to determine CSI driver for grouped PVCs") } if driver == "" { return nil, nil, "", nil, errors.New("multiple CSI drivers found for grouped PVCs; failing backup") } // Retrieve the appropriate VGSClass for the CSI driver. vgsClass := p.getVGSClassForDriver(driver) p.log.Infof("Determined CSI driver %s with VGSClass %s for PVC group %s", driver, vgsClass, group) newVGS, err := p.createVolumeGroupSnapshot(backup, pvc, pvcNames, vgsLabelKey, group, vgsClass) if err != nil { return nil, nil, "", nil, err } p.log.Infof("Created new VGS %s for PVC group %s", newVGS.Name, group) // Wait for the VS objects created via VGS to have volumeGroupSnapshotName in status. if err := p.waitForVGSAssociatedVS(newVGS, pvc.Namespace, backup.Spec.CSISnapshotTimeout.Duration); err != nil { return nil, nil, "", nil, err } // Update the VS objects: remove VGS owner references and finalizers; add backup metadata labels. if err := p.updateVGSCreatedVS(newVGS, backup); err != nil { return nil, nil, "", nil, err } // Patch the VGSC deletionPolicy to Retain. if err := p.patchVGSCDeletionPolicy(newVGS, pvc.Namespace); err != nil { return nil, nil, "", nil, err } // Delete the VGS and VGSC if err := p.deleteVGSAndVGSC(newVGS, pvc.Namespace); err != nil { return nil, nil, "", nil, err } // Fetch the VS that was created for this PVC via VGS. vs, err := p.getVSForPVC(backup, pvc, vgsLabelKey, group) if err != nil { return nil, nil, "", nil, err } vsRef = &corev1api.ObjectReference{ Namespace: vs.Namespace, Name: vs.Name, } } } else { // Legacy workflow: PVC does not have a VGS label; create an individual VS. vs, err := p.createVolumeSnapshot(pvc, backup) if err != nil { return nil, nil, "", nil, err } vsRef = &corev1api.ObjectReference{ Namespace: vs.Namespace, Name: vs.Name, } } // --- Common Branch --- // Now we have vsRef populated from one of the above cases. // Branch further based on backup.Spec.SnapshotMoveData. if boolptr.IsSetToTrue(backup.Spec.SnapshotMoveData) { // Datamover case: operationID = label.GetValidName( string(velerov1api.AsyncOperationIDPrefixDataUpload) + string(backup.UID) + "." + string(pvc.UID), ) dataUploadLog := p.log.WithFields(logrus.Fields{ "Source PVC": fmt.Sprintf("%s/%s", pvc.Namespace, pvc.Name), "VolumeSnapshot": fmt.Sprintf("%s/%s", vsRef.Namespace, vsRef.Name), "Operation ID": operationID, "Backup": backup.Name, }) // Retrieve the current VS using vsRef vs := &snapshotv1api.VolumeSnapshot{} if err := p.crClient.Get(context.TODO(), crclient.ObjectKey{Namespace: vsRef.Namespace, Name: vsRef.Name}, vs); err != nil { return nil, nil, "", nil, errors.Wrapf(err, "failed to get VolumeSnapshot %s", vsRef.Name) } // Wait until the VS-associated VSC snapshot handle is ready. _, err := csi.WaitUntilVSCHandleIsReady( vs, p.crClient, p.log, true, backup.Spec.CSISnapshotTimeout.Duration, ) if err != nil { dataUploadLog.Errorf("Failed to wait for VolumeSnapshot to become ReadyToUse: %s", err.Error()) csi.CleanupVolumeSnapshot(vs, p.crClient, p.log) return nil, nil, "", nil, errors.WithStack(err) } dataUploadLog.Info("Starting data upload of backup") dataUpload, err := createDataUpload( context.Background(), backup, p.crClient, vs, &pvc, operationID, ) if err != nil { dataUploadLog.WithError(err).Error("Failed to submit DataUpload") if deleteErr := p.crClient.Delete(context.TODO(), vs); deleteErr != nil && !apierrors.IsNotFound(deleteErr) { dataUploadLog.WithError(deleteErr).Error("Failed to delete VolumeSnapshot") } return item, nil, "", nil, nil } dataUploadLog.Info("DataUpload submitted successfully") itemToUpdate = []velero.ResourceIdentifier{ { GroupResource: schema.GroupResource{ Group: "velero.io", Resource: "datauploads", }, Namespace: dataUpload.Namespace, Name: dataUpload.Name, }, } annotations[velerov1api.DataUploadNameAnnotation] = dataUpload.Namespace + "/" + dataUpload.Name // For the datamover case, add the dataUpload as an additional item directly. vsRef = &corev1api.ObjectReference{ Namespace: dataUpload.Namespace, Name: dataUpload.Name, } additionalItems = append(additionalItems, velero.ResourceIdentifier{ GroupResource: schema.GroupResource{ Group: "velero.io", Resource: "datauploads", }, Namespace: dataUpload.Namespace, Name: dataUpload.Name, }) } else { // Non-datamover case: // Use vsRef for snapshot purposes. additionalItems = append(additionalItems, convertVSToResourceIdentifiersFromRef(vsRef)...) p.log.Infof("VolumeSnapshot additional item added for VS %s", vsRef.Name) } // Update PVC metadata with common labels and annotations. labels := map[string]string{ velerov1api.VolumeSnapshotLabel: vsRef.Name, velerov1api.BackupNameLabel: backup.Name, } annotations := map[string]string{ velerov1api.VolumeSnapshotLabel: vsRef.Name, velerov1api.MustIncludeAdditionalItemAnnotation: "true", } kubeutil.AddAnnotations(&pvc.ObjectMeta, annotations) kubeutil.AddLabels(&pvc.ObjectMeta, labels) p.log.Infof("Returning from PVCBackupItemAction with %d additionalItems to backup", len(additionalItems)) for _, ai := range additionalItems { p.log.Debugf("%s: %s", ai.GroupResource.String(), ai.Name) } pvcMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&pvc) if err != nil { return nil, nil, "", nil, errors.WithStack(err) } return &unstructured.Unstructured{Object: pvcMap}, additionalItems, operationID, itemToUpdate, nil } ``` ## Implementation This design proposal is targeted for velero 1.16. The implementation of this proposed design is targeted for velero 1.17. **Note:** - VGS support isn't a requirement on restore. The design does not have any VGS related elements/considerations in the restore workflow. ## Requirements and Assumptions - Kubernetes Version: - Minimum: v1.32.0 or later, since the VolumeGroupSnapshot API goes beta in 1.32. - Assumption: CRDs for `VolumeGroupSnapshot`, `VolumeGroupSnapshotClass`, and `VolumeGroupSnapshotContent` are already installed. - VolumeGroupSnapshot API Availability: - If the VGS API group (`groupsnapshot.storage.k8s.io/v1beta1`) is not present, Velero backup will fail. - CSI Driver Compatibility - Only CSI drivers that implement the VolumeGroupSnapshot admission and controller support this feature. - Upon VGS creation, we assume the driver will atomically snapshot all matching PVCs; if it does not, the plugin may time out. ## Performance Considerations - Use VGS if you have many similar volumes that must be snapped together and you want to minimize API/server load. - Use individual VS if you have only a few volumes, or want one‐volume failures to be isolated. ## Testing Strategy - Unit tests: We will add targeted unit tests to cover all new code paths—including existing-VS detection, VGS creation, legacy VS fallback, and error scenarios. - E2E tests: For E2E we would need, a Kind cluster with a CSI driver that supports group snapshots, deploy an application with multiple PVCs, execute a Velero backup and restore, and verify that VGS is created, all underlying VS objects reach ReadyToUse, and every PVC is restored successfully. ================================================ FILE: design/Implemented/volume-policy-label-selector-criteria.md ================================================ # Add Label Selector as a criteria for Volume Policy ## Abstract Velero’s volume policies currently support several criteria (such as capacity, storage class, and volume source type) to select volumes for backup. This update extends the design by allowing users to specify required labels on the associated PersistentVolumeClaim (PVC) via a simple key/value map. At runtime, Velero looks up the PVC (when a PV has a ClaimRef), extracts its labels, and compares them with the user-specified map. If all key/value pairs match, the volume qualifies for backup. ## Background PersistentVolumes (PVs) in Kubernetes are typically bound to PersistentVolumeClaims (PVCs) that include labels (for example, indicating environment, application, or region). Basing backup policies on these PVC labels enables more precise control over which volumes are processed. ## Goals - Allow users to specify a simple key/value mapping in the volume policy YAML so that only volumes whose associated PVCs contain those labels are selected. - Support policies that target volumes based on criteria such as environment=production or region=us-west. ## Non-Goals - No changes will be made to the actions (skip, snapshot, fs-backup) of the volume policy engine. This update focuses solely on how volumes are selected. - The design does not support other label selector operations (e.g., NotIn, Exists, DoesNotExist) and only allows for exact key/value matching. ## Use-cases/scenarios 1. Environment-Specific Backup: - A user wishes to back up only those volumes whose associated PVCs have labels such as `environment=production` and `app=database`. - The volume policy specifies a pvcLabels map with those key/value pairs; only volumes whose PVCs match are processed. ```yaml volumePolicies: - conditions: pvcLabels: environment: production app: database action: type: snapshot ``` 2. Region-Specific Backup: - A user operating in multiple regions wants to back up only volumes in the `us-west` region. - The policy includes `pvcLabels: { region: us-west }`, so only PVs bound to PVCs with that label are selected. ```yaml volumePolicies: - conditions: pvcLabels: region: us-west action: type: snapshot ``` 3. Automated Label-Based Backups: - An external system automatically labels new PVCs (for example, `backup: true`). - A volume policy with `pvcLabels: { backup: true }` ensures that any new volume whose PVC contains that label is included in backup operations. ```yaml version: v1 volumePolicies: - conditions: pvcLabels: backup: true action: type: snapshot ``` ## High-Level Design 1. Extend Volume Policy Schema: - The YAML schema for volume conditions is extended to include an optional field pvcLabels of type `map[string]string`. 2. Implement New Condition Type: - A new condition, `pvcLabelsCondition`, is created. It implements the `volumeCondition` interface and simply compares the user-specified key/value pairs with the actual PVC labels (populated at runtime). 3. Update Structured Volume: - The internal representation of a volume (`structuredVolume`) is extended with a new field `pvcLabels map[string]string` to store the labels from the associated PVC. - A new helper function (or an updated parsing function) is used to perform a PVC lookup when a PV has a ClaimRef, populating the pvcLabels field. 4. Integrate with Policy Engine: - The policy builder is updated to create and add a `pvcLabelsCondition` if the policy YAML contains a `pvcLabels` entry. - The matching entry point uses the updated `structuredVolume` (populated with PVC labels) to evaluate all conditions, including the new PVC labels condition. ## Detailed Design 1. Update Volume Conditions Schema: Define the conditions struct with a simple map for PVC labels: ```go // volumeConditions defines the current format of conditions we parse. type volumeConditions struct { Capacity string `yaml:"capacity,omitempty"` StorageClass []string `yaml:"storageClass,omitempty"` NFS *nFSVolumeSource `yaml:"nfs,omitempty"` CSI *csiVolumeSource `yaml:"csi,omitempty"` VolumeTypes []SupportedVolume `yaml:"volumeTypes,omitempty"` // New field: pvcLabels for simple exact-match filtering. PVCLabels map[string]string `yaml:"pvcLabels,omitempty"` } ``` 2. New Condition: `pvcLabelsCondition`: Implement a condition that compares expected labels with those on the PVC: ```go // pvcLabelsCondition defines a condition that matches if the PVC's labels contain all the specified key/value pairs. type pvcLabelsCondition struct { labels map[string]string } func (c *pvcLabelsCondition) match(v *structuredVolume) bool { if len(c.labels) == 0 { return true // No label condition specified; always match. } if v.pvcLabels == nil { return false // No PVC labels found. } for key, expectedVal := range c.labels { if actualVal, exists := v.pvcLabels[key]; !exists || actualVal != expectedVal { return false } } return true } func (c *pvcLabelsCondition) validate() error { // No extra validation needed for a simple map. return nil } ``` 3. Update `structuredVolume`: Extend the internal volume representation with a field for PVC labels: ```go // structuredVolume represents a volume with parsed fields. type structuredVolume struct { capacity resource.Quantity storageClass string // New field: pvcLabels stores labels from the associated PVC. pvcLabels map[string]string nfs *nFSVolumeSource csi *csiVolumeSource volumeType SupportedVolume } ``` 4. Update PVC Lookup – `parsePVWithPVC`: Modify the PV parsing function to perform a PVC lookup: ```go func (s *structuredVolume) parsePVWithPVC(pv *corev1.PersistentVolume, client crclient.Client) error { s.capacity = *pv.Spec.Capacity.Storage() s.storageClass = pv.Spec.StorageClassName if pv.Spec.NFS != nil { s.nfs = &nFSVolumeSource{ Server: pv.Spec.NFS.Server, Path: pv.Spec.NFS.Path, } } if pv.Spec.CSI != nil { s.csi = &csiVolumeSource{ Driver: pv.Spec.CSI.Driver, VolumeAttributes: pv.Spec.CSI.VolumeAttributes, } } s.volumeType = getVolumeTypeFromPV(pv) // If the PV is bound to a PVC, look it up and store its labels. if pv.Spec.ClaimRef != nil { pvc := &corev1.PersistentVolumeClaim{} err := client.Get(context.Background(), crclient.ObjectKey{ Namespace: pv.Spec.ClaimRef.Namespace, Name: pv.Spec.ClaimRef.Name, }, pvc) if err != nil { return errors.Wrap(err, "failed to get PVC for PV") } s.pvcLabels = pvc.Labels } return nil } ``` 5. Update the Policy Builder: Add the new condition to the policy if pvcLabels is provided: ```go func (p *Policies) BuildPolicy(resPolicies *ResourcePolicies) error { for _, vp := range resPolicies.VolumePolicies { con, err := unmarshalVolConditions(vp.Conditions) if err != nil { return errors.WithStack(err) } volCap, err := parseCapacity(con.Capacity) if err != nil { return errors.WithStack(err) } var volP volPolicy volP.action = vp.Action volP.conditions = append(volP.conditions, &capacityCondition{capacity: *volCap}) volP.conditions = append(volP.conditions, &storageClassCondition{storageClass: con.StorageClass}) volP.conditions = append(volP.conditions, &nfsCondition{nfs: con.NFS}) volP.conditions = append(volP.conditions, &csiCondition{csi: con.CSI}) volP.conditions = append(volP.conditions, &volumeTypeCondition{volumeTypes: con.VolumeTypes}) // If a pvcLabels map is provided, add the pvcLabelsCondition. if con.PVCLabels != nil && len(con.PVCLabels) > 0 { volP.conditions = append(volP.conditions, &pvcLabelsCondition{labels: con.PVCLabels}) } p.volumePolicies = append(p.volumePolicies, volP) } p.version = resPolicies.Version return nil } ``` 6. Update the Matching Entry Point: Use the updated PV parsing that performs a PVC lookup: ```go func (p *Policies) GetMatchAction(res interface{}, client crclient.Client) (*Action, error) { volume := &structuredVolume{} switch obj := res.(type) { case *corev1.PersistentVolume: if err := volume.parsePVWithPVC(obj, client); err != nil { return nil, errors.Wrap(err, "failed to parse PV with PVC lookup") } case *corev1.Volume: volume.parsePodVolume(obj) default: return nil, errors.New("failed to convert object") } return p.match(volume), nil } ``` Note: The matching loop (p.match(volume)) iterates over all conditions (including our new pvcLabelsCondition) and returns the corresponding action if all conditions match. ================================================ FILE: design/Implemented/volume-snapshot-data-movement/volume-snapshot-data-movement.md ================================================ # Volume Snapshot Data Movement Design ## Glossary & Abbreviation **BR**: Backup & Restore **Backup Storage**: See the same definition in [Unified Repository design][1]. **Backup Repository**: See the same definition in [Unified Repository design][1]. **BIA/RIA V2**: Backup Item Action/Restore Item Action V2 that supports asynchronized operations, see the [general progress monitoring design][2] for details. ## Background As a Kubernetes BR solution, Velero is pursuing the capability to back up data from the volatile and limited production environment into the durable, heterogeneous and scalable backup storage. This relies on two parts: - Data Movement: Move data from various production workloads, including the snapshots of the workloads or volumes of the workloads - Data Persistency and Management: Persistent the data in backup storage and manage its security, redundancy, accessibility, etc. through backup repository. This has been covered by the [Unified Repository design][1] At present, Velero supports moving file system data from PVs through Pod Volume Backup (a.k.a. file system backup). However, it backs up the data from the live file system, so it should be the last option when more consistent data movement (i.e., moving data from snapshot) is not available. Moreover, we would like to create a general workflow to variations during the data movement, e.g., data movement plugins, different snapshot types, different snapshot accesses and different data accesses. ## Goals - Create components and workflows for Velero to move data based on volume snapshots - Create components and workflows for Velero built-in data mover - Create the mechanism to support data mover plugins from third parties - Implement CSI snapshot data movement on file system level - Support different data accesses, i.e., file system level and block level - Support different snapshot types, i.e., CSI snapshot, volume snapshot API from storage vendors - Support different snapshot accesses, i.e., through PV generated from snapshots, and through direct access API from storage vendors - Reuse the existing Velero generic data path as created in [Unified Repository design][1] ## Non-Goals - The current support for block level access is through file system uploader, so it is not aimed to deliver features of an ultimate block level backup. Block level backup will be included in a future design - Most of the components are generic, but the Exposer is snapshot type specific or snapshot access specific. The current design covers the implementation details for exposing CSI snapshot to host path access only, for other types or accesses, we may need a separate design - The current workflow focuses on snapshot-based data movements. For some application/SaaS level data sources, snapshots may not be taken explicitly. We don’t take them into consideration, though we believe that some workflows or components may still be reusable. ## Architecture of Volume Snapshot Data Movement ### Workflows Here are the diagrams that illustrate components and workflows for backup and restore respectively. For backup, we intend to create an extensive architecture for various snapshot types, snapshot accesses and various data accesses. For example, the snapshot specific operations are isolated in Data Mover Plugin and Exposer. In this way, we only need to change the two modules for variations. Likely, the data access details are isolated into uploaders, so different uploaders could be plugged into the workflow seamlessly. For restore, we intend to create a generic workflow that could for all backups. This means the restore is backup source independent. Therefore, for example, we can restore a CSI snapshot backup to another cluster with no CSI facilities or with CSI facilities different from the source cluster. We still have the Exposer module for restore and it is to expose the target volume to the data path. Therefore, we still have the flexibility to introduce different ways to expose the target volume. Likely, the data downloading details are isolated in uploaders, so we can still create multiple types of uploaders. Below is the backup workflow: ![backup-workflow.png](backup-workflow.png) Below is the restore workflow: ![restore-workflow.png](restore-workflow.png) ### Components Below are the generic components in the data movement workflow: **Velero**: Velero controls the backup/restore workflow, it calls BIA/RIA V2 to backup/restore an object that involves data movement, specifically, a PVC or a PV. **BIA/RIA V2**: BIA/RIA V2 are the protocols between Velero and the data mover plugins. They support asynchronized operations so that Velero backup/restore is not marked as completion until the data movement is done and in the meantime, Velero is free to process other backups during the data movement. **Data Mover Plugin (DMP)**: DMP implement BIA/RIA V2 and it invokes the corresponding data mover by creating the DataUpload/DataDownload CRs. DMP is also responsible to take snapshot of the source volume, so it is a snapshot type specific module. For CSI snapshot data movement, the CSI plugin could be extended as a DMP, this also means that the CSI plugin will fully implement BIA/RIA V2 and support some more methods like Progress, Cancel, etc. **DataUpload CR (DUCR)/ DataDownload CR (DDCR)**: DUCR/DDCR are Kubernetes CRs that act as the protocol between data mover plugins and data movers. The parties who want to provide a data mover need to watch and process these CRs. **Data Mover (DM)**: DM is a collective of modules to finish the data movement, specifically, data upload and data download. The modules may include the data mover controllers to reconcile DUCR/DDCR and the data path to transfer data. DMs take the responsibility to handle DUCR/DDCRs, Velero provides a built-in DM and meanwhile Velero supports plugin DMs. Below shows the components for the built-in DM: **Velero Built-in Data Mover (VBDM)**: VBDM is the built-in data mover shipped along with Velero, it includes Velero data mover controllers and Velero generic data path. **Node-Agent**: Node-Agent is an existing Velero module that will be used to host VBDM. **Exposer**: Exposer is to expose the snapshot/target volume as a path/device name/endpoint that are recognizable by Velero generic data path. For different snapshot types/snapshot accesses, the Exposer may be different. This isolation guarantees that when we want to support other snapshot types/snapshot accesses, we only need to replace with a new Exposer and keep other components as is. **Velero Generic Data Path (VGDP)**: VGDP is the collective of modules that is introduced in [Unified Repository design][1]. Velero uses these modules to finish data transmission for various purposes. In includes uploaders and the backup repository. **Uploader**: Uploader is the module in VGDP that reads data from the source and writes to backup repository for backup; while read data from backup repository and write to the restore target for restore. At present, only file system uploader is supported. In future, the block level uploader will be added. For file system and basic block uploader, only Kopia uploader will be used, Restic will not be integrated with VBDM. ### Replacement 3rd parties could integrate their own data movement into Velero by replacing VBDM with their own DMs. The DMs should process DUCR/DDCRs appropriately and finally put them into one of the terminal states as shown in the DataUpload CRD and DataDownload CRD sections. Theoretically, replacing the DMP is also allowed. In this way, the entire workflow is customized, so this is out of the scope of this design. ## Detailed Design ### Backup Sequence Below are the data movement actions and sequences during backup: ![backup-sequence.png](backup-sequence.png) Below are actions from Velero and DMP: **BIA Execute** This is the existing logic in Velero. For a source PVC/PV, Velero delivers it to the corresponding BackupItemAction plugin, the plugin then takes the related actions to back it up. For example, the existing CSI plugin takes a CSI snapshot to the volume represented by the PVC and then returns additional items (i.e., VolumeSnapshot, VolumeSnapshotContent and VolumeSnapshotClass) for Velero to further backup. To support data movement, we will use BIA V2 which supports asynchronous operation management. Here is the Execute method from BIA V2: ``` Execute(item runtime.Unstructured, backup *api.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) ``` Besides ```additionalItem``` (as the 2nd return value), Execute method will return one more resource list called ```itemToUpdate```, which means the items to be updated and persisted when the async operation completes. For details, visit [general progress monitoring design][2]. Specifically, this mechanism will be used to persist DUCR into the persisted backup data, in another words, DUCR will be returned as ```itemToUpdate``` from Execute method. DUCR contains all the information the restore requires, so during restore, DUCR will be extracted from the backup data. Additionally, in the same way, a DMP could add any other items into the persisted backup data. Execute method also returns the ```operationID``` which uniquely identifies the asynchronous operation. This ```operationID``` is generated by plugins. The [general progress monitoring design][2] doesn't restrict the format of the ```operationID```, for Velero CSI plugin, the ```operationID``` is a combination of the backup CR UID and the source PVC (represented by the ```item``` parameter) UID. **Create Snapshot** The DMP creates a snapshot of the requested volume and deliver it to DM through DUCR. After that, the DMP leaves the snapshot and its related objects (e.g., VolumeSnapshot and VolumeSnapshotContent for CSI snapshot) to the DM, DM then has full control of the snapshot and its related objects, i.e., deciding whether to delete the snapshot or its related objects and when to do it. This also indicates that the DUCR should contain the snapshot type specific information because different snapshot types may have their unique information. For Velero built-in implementation, the existing logics to create the snapshots will be reused, specifically, for CSI snapshot, the related logics in CSI plugin are fully reused. **Create DataUpload CR** A DUCR is created for as the result of each Execute call, then Execute method will return and leave DUCR being processed asynchronously. **Set Backup As WaitForAsyncOperations** **Persist Backup** After ```Execute``` returns, the backup is set to ```WaitingForPluginOperations```, and then Velero is free to process other items or backups. Before Velero moves to other items/backups, it will persist the backup data. This is the same as the existing behavior. The backup then is left as ```WaitForAsyncOperations``` until the DM completes or timeout. **BIA Progress** Velero keeps monitoring the status of the backup by calling BIA V2’s Progress method. Below is the Progress method from BIA V2: ``` Progress(operationID string, backup *api.Backup) (velero.OperationProgress, error) ``` On the call of this method, DMP will query the DUCR’s status. Some critical progress data is transferred from DUCR to the ```OperationProgress``` which is the return value of BIA V2’s Progress method. For example, NCompleted indicates the size/number of data that have been completed and NTotal indicates the total size/number of data. When the async operation completes, the Progress method returns an OperationProgress with ```Completed``` set as true. Then Velero will persist DUCR as well as any other items returned by DUP as ```itemToUpdate```. Finally, then backup is as ```Completed```. To help BIA Progress find the corresponding DUCR, the ```operationID``` is saved along with the DUCR as a label ```velero.io/async-operation-id```. DUCRs are handled by the data movers, so how to handle them are totally decided by the data movers. Below covers the details of VBDM, plugging data movers should have their own actions and workflows. **Persist DataUpload CR** As mentioned above, the DUCR will be persisted when it is completed under the help of BIA V2 async operation finalizing mechanism. This means the backup tarball will be uploaded twice, this is as the designed behavior of [general progress monitoring design][2]. Conclusively, as a result of the above executions: - A DataUpload CR is created and persisted to the backup tarball. The CR will be left there after the backup completes because the CR includes many information connecting to the backup that may be useful to end users or upper level modules. - A snapshot as well as the objects representing it are created. For CSI snapshot, a VolumeSnapshot object and a VolumeSnapshotContent object is created. The DMP leaves the snapshot as well as its related objects to DM for further processing. VBDM creates a Data Uploader Controller to handle the DUCRs in node-agent daemonset, therefore, on each node, there will be an instance of this controller. The controller connects to the backup repository and calls the uploader. Below are the VBDM actions. **Acquire Object Lock** **Release Object Lock** There are multiple instances of Data Uploader Controllers and when a DUCR is created, there should be only one of the instances handle the CR. Therefore, an operation called “Acquired Object Lock” is used to reach a consensus among the controller instances so that only one controller instance takes over the CR and tries the next action – Expose for the CR. After the CR is completed in the Expose phase, the CR is released with the operation of “Release Object Lock”. We fulfil the “Acquired Object Lock” and “Release Object Lock” under the help of Kubernetes API server and the etcd in the background, which guarantees strong write consistency among all the nodes. **Expose** For some kinds of snapshot, it may not be usable directly after it is taken. For example, a CSI snapshot is represented by the VolumeSnapshot and VolumeSnapshotContent object, if we don’t do anything, we don’t see any PV really exists in the cluster, so VGDP has no way to access it. Meanwhile, when we have a PV representing the snapshot data, we still need a way to make it accessible by the VGDP. The details of the expose process are snapshot specific, and for one kind of snapshot, we may have different methods to expose it to VGDP. Later, we will have a specific section to explain the current design of the Exposer. **Backup From Data Path** After a snapshot is exposed, VGDP will be able to access the snapshot data, so the controller calls the uploader to start the data backup. To support cancellation and concurrent backup, the call to the VGDP is done asynchronously. How this asynchronization is implemented may be related to the Exposer. as the current design of Exposer, the asynchronization is implemented by the controller with go routines. We keep VGDP reused for VBDM, so everything inside VGDP are kept as is. For details of VGDP, refer to the [Unified Repository design][1]. **Update Repo Snapshot ID** When VGDP completes backup, it returns an ID that represent the root object saved into the backup repository for this backup, through the root object, we will be able to enumerate the entire backup data. This Repo Snapshot ID will be saved along with the DUCR. ### DataUpload CRD Below are the essential fields of DataUpload CRD. The CRD covers below information: - The information to manipulate the specified snapshot - The information to manipulate the specified data mover - The information to manipulate the specified backup repository - The progress of the current data upload - The result of the current data upload once it finishes For snapshot manipulation: - ```snapshotType``` indicates the type of the snapshot, at present, the only valid value is ```CSI```. - If ```snapshotType``` is ```CSI```, ```csiSnapshot``` which is a pointer to a ```CSISnapshotSpec``` must not be absent. - ```CSISnapshotSpec``` specifies the information of the CSI snapshot, e.g., ```volumeSnapshot``` is the name of VolumeSnapshot object representing the CSI snapshot; ```storageClass``` specifies the name of the StorageClass of the source PVC, which will be used to create the backupPVC during the data upload. For data mover manipulation: - ```datamover``` indicates the name of the data mover, if it is empty or ```velero```, it means the built-in data mover will be used for this data upload For backup repository manipulation, ```backupStorageLocation``` is the name of the related BackupStorageLocation, where we can find all the required information. For the progress, it includes the ```totalBytes``` and ```doneBytes``` so that other modules could easily cuclulate a progress. For data upload result, ```snapshotID``` in the ```status``` field is the Repo Snapshot ID. Data movers may have their private outputs as a result of the DataUpload, they will be put in the ```dataMoverResult``` map of the ```status``` field. Here are the statuses of DataUpload CRD and their descriptions: - New: The DUCR has been created but not processed by a controller - Accepted: the Object lock has been acquired for this DUCR and the elected controller is trying to expose the snapshot - Prepared: the snapshot has been exposed, the related controller is starting to process the upload - InProgress: the data upload is in progress - Canceling: the data upload is being canceled - Canceled: the data upload has been canceled - Completed: the data upload has completed - Failed: the data upload has failed Below is the full spec of DataUpload CRD: ``` apiVersion: apiextensions.k8s.io/v1alpha1 kind: CustomResourceDefinition metadata: labels: component: velero name: datauploads.velero.io spec: conversion: strategy: None group: velero.io names: kind: DataUpload listKind: DataUploadList plural: datauploads singular: dataupload scope: Namespaced versions: - additionalPrinterColumns: - description: DataUpload status such as New/InProgress jsonPath: .status.phase name: Status type: string - description: Time duration since this DataUpload was started jsonPath: .status.startTimestamp name: Started type: date - description: Completed bytes format: int64 jsonPath: .status.progress.bytesDone name: Bytes Done type: integer - description: Total bytes format: int64 jsonPath: .status.progress.totalBytes name: Total Bytes type: integer - description: Name of the Backup Storage Location where this backup should be stored jsonPath: .spec.backupStorageLocation name: Storage Location type: string - description: Time duration since this DataUpload was created jsonPath: .metadata.creationTimestamp name: Age type: date name: v1 schema: openAPIV3Schema: properties: spec: description: DataUploadSpec is the specification for a DataUpload. properties: backupStorageLocation: description: BackupStorageLocation is the name of the backup storage location where the backup repository is stored. type: string csiSnapshot: description: If SnapshotType is CSI, CSISnapshot provides the information of the CSI snapshot. properties: snapshotClass: description: SnapshotClass is the name of the snapshot class that the volume snapshot is created with type: string storageClass: description: StorageClass is the name of the storage class of the PVC that the volume snapshot is created from type: string volumeSnapshot: description: VolumeSnapshot is the name of the volume snapshot to be backed up type: string required: - storageClass - volumeSnapshot type: object datamover: description: DataMover specifies the data mover to be used by the backup. If DataMover is "" or "velero", the built-in data mover will be used. type: string operationTimeout: description: OperationTimeout specifies the time used to wait internal operations, e.g., wait the CSI snapshot to become readyToUse. type: string snapshotType: description: SnapshotType is the type of the snapshot to be backed up. type: string sourceNamespace: description: SourceNamespace is the original namespace where the volume is backed up from. type: string required: - backupStorageLocation - csiSnapshot - snapshotType - sourceNamespace type: object status: description: DataUploadStatus is the current status of a DataUpload. properties: completionTimestamp: description: CompletionTimestamp records the time a backup was completed. Completion time is recorded even on failed backups. Completion time is recorded before uploading the backup object. The server's time is used for CompletionTimestamps format: date-time nullable: true type: string dataMoverResult: additionalProperties: type: string description: DataMoverResult stores data-mover-specific information as a result of the DataUpload. nullable: true type: object message: description: Message is a message about the DataUpload's status. type: string node: description: Node is the name of the node where the DataUpload is running. type: string path: description: Path is the full path of the snapshot volume being backed up. type: string phase: description: Phase is the current state of the DataUpload. enum: - New - Accepted - Prepared - InProgress - Canceling - Canceled - Completed - Failed type: string progress: description: Progress holds the total number of bytes of the volume and the current number of backed up bytes. This can be used to display progress information about the backup operation. properties: bytesDone: format: int64 type: integer totalBytes: format: int64 type: integer type: object snapshotID: description: SnapshotID is the identifier for the snapshot in the backup repository. type: string startTimestamp: description: StartTimestamp records the time a backup was started. Separate from CreationTimestamp, since that value changes on restores. The server's time is used for StartTimestamps format: date-time nullable: true type: string type: object type: object ``` ### Restore Sequence Below are the data movement actions sequences during restore: ![restore-sequence.png](restore-sequence.png) Many of the actions are the same with backup, here are the different ones. **Query Backup Result** The essential information to be filled into DataDownload all comes from the DataUpload CR. For example, the Repo Snapshot ID is stored in the status fields of DataUpload CR. However, we don't want to restore the DataUpload CR and leave it in the cluster since it is useless after the restore. Therefore, we will retrieve the necessary information from DataUpload CR and store it in a temporary ConfigMap for the DM to use. There is one ConfigMap for each DataDownload CR and the ConfigMaps belong to a restore will be deleted when the restore finishes. **Prepare Volume Readiness** As the current pattern, Velero delivers an object representing a volume, either a PVC or a PV, to DMP and Velero will create the object after DMP's Execute call returns. However, by this time, DM should have not finished the restore, so the volume is not ready for use. In this step, DMP needs to mark the object as unready to use so as to prevent others from using it, i.e., a pod mounts the volume. Additionlly, DMP needs to provide an approach for DM to mark it as ready when the data movement finishes. How to mark the volume as unready or ready varying from the type of the object, specifically, a PVC or a PV; and there are more than one ways to achieve this. Below show the details of how to do this for CSI snapshot data movement. After the DMP submits the DataDownload CR, it does below modifications to the PVC spec: - Set spec.VolumeName to empty ("") - Add a selector with a matchLabel ```velero.io/dynamic-pv-restore``` With these two steps, it tells Kubernetes that the PVC is not bound and it only binds a PV with the ```velero.io/dynamic-pv-restore``` label. As a result, even after the PVC object is created by Velero later and is used by other resources, it is not usable until the DM creates the target PV. **Expose** The purpose of expose process for restore is to create the target PV and make the PV accessible by VGDP. Later the Expose section will cover the details. **Finish Volume Readiness** By the data restore finishes, the target PV is ready for use but it is not delivered to the outside world. This step is the follow up of Prepare Volume Readiness, which does necessary work to mark the volume ready to use. For CSI snapshot restore, DM does below steps: - Set the target PV's claim reference (the ```claimRef``` filed) to the target PVC - Add the ```velero.io/dynamic-pv-restore``` label to the target PV By the meantime, the target PVC should have been created in the source user namespace and waiting for binding. When the above steps are done, the target PVC will be bound immediately by Kubernetes. This also means that Velero should not restore the PV if a data movement restore is involved, this follows the existing CSI snapshot behavior. For restore, VBDM doesn’t need to persist anything. ### DataDownload CRD Below are the essential fields of DataDownload CRD. The CRD covers below information: - The information to manipulate the target volume - The information to manipulate the specified data mover - The information to manipulate the specified backup repository Target volume information includes PVC and PV that represents the volume and the target namespace. The data mover information and backup repository information are the same with DataUpload CRD. DataDownload CRD defines the same status as DataUpload CRD with nearly the same meanings. Below is the full spec of DataDownload CRD: ``` apiVersion: apiextensions.k8s.io/v1alpha1 kind: CustomResourceDefinition metadata: labels: component: velero name: datadownloads.velero.io spec: conversion: strategy: None group: velero.io names: kind: DataDownload listKind: DataDownloadList plural: datadownloads singular: datadownload scope: Namespaced versions: - DataDownload: - description: DataDownload status such as New/InProgress jsonPath: .status.phase name: Status type: string - description: Time duration since this DataDownload was started jsonPath: .status.startTimestamp name: Started type: date - description: Completed bytes format: int64 jsonPath: .status.progress.bytesDone name: Bytes Done type: integer - description: Total bytes format: int64 jsonPath: .status.progress.totalBytes name: Total Bytes type: integer - description: Time duration since this DataDownload was created jsonPath: .metadata.creationTimestamp name: Age type: date name: v1 schema: openAPIV3Schema: properties: spec: description: SnapshotDownloadSpec is the specification for a SnapshotDownload. properties: backupStorageLocation: description: BackupStorageLocation is the name of the backup storage location where the backup repository is stored. type: string datamover: description: DataMover specifies the data mover to be used by the backup. If DataMover is "" or "velero", the built-in data mover will be used. type: string operationTimeout: description: OperationTimeout specifies the time used to wait internal operations, before returning error as timeout. type: string snapshotID: description: SnapshotID is the ID of the Velero backup snapshot to be restored from. type: string sourceNamespace: description: SourceNamespace is the original namespace where the volume is backed up from. type: string targetVolume: description: TargetVolume is the information of the target PVC and PV. properties: namespace: description: Namespace is the target namespace type: string pv: description: PV is the name of the target PV that is created by Velero restore type: string pvc: description: PVC is the name of the target PVC that is created by Velero restore type: string required: - namespace - pv - pvc type: object required: - backupStorageLocation - restoreName - snapshotID - sourceNamespace - targetVolume type: object status: description: SnapshotRestoreStatus is the current status of a SnapshotRestore. properties: completionTimestamp: description: CompletionTimestamp records the time a restore was completed. Completion time is recorded even on failed restores. The server's time is used for CompletionTimestamps format: date-time nullable: true type: string message: description: Message is a message about the snapshot restore's status. type: string node: description: Node is the name of the node where the DataDownload is running. type: string phase: description: Phase is the current state of theSnapshotRestore. enum: - New - Accepted - Prepared - InProgress - Canceling - Canceled - Completed - Failed type: string progress: description: Progress holds the total number of bytes of the snapshot and the current number of restored bytes. This can be used to display progress information about the restore operation. properties: bytesDone: format: int64 type: integer totalBytes: format: int64 type: integer type: object startTimestamp: description: StartTimestamp records the time a restore was started. The server's time is used for StartTimestamps format: date-time nullable: true type: string type: object type: object ``` ## Expose ### Expose for DataUpload At present, for a file system backup, VGDP accepts a string representing the root path of the snapshot to be backed up, the path should be accessible from the process/pod that VGDP is running. In future, VGDP may accept different access parameters. Anyway, the snapshot should be accessible local. Therefore, the first phase for Expose is to expose the snapshot to be locally accessed. This is a snapshot specific operation. For CSI snapshot, the final target is to create below 3 objects in Velero namespace: - backupVSC: This is the Volume Snapshot Content object represents the CSI snapshot - backupVS: This the Volume Snapshot object for BackupVSC in Velero namespace - backupPVC: This is the PVC created from the backupVS in Velero namespace. Specifically, backupPVC’s data source points to backupVS - backupPod: This is a pod attaching backupPVC in Velero namespace. As Kubernetes restriction, the PV is not provisioned until the PVC is attached to a pod and the pod is scheduled to a node. Therefore, after the backupPod is running, the backupPV which represents the data of the snapshot will be provisioned - backupPV: This is the PV provisioned as a result of backupPod schedule, it has the same data of the snapshot Initially, the CSI VS object is created in the source user namespace (we call it sourceVS), after the Expose, all the objects will be in Velero namespace, so all the data upload activities happen in the Velero namespace only. As you can see, we have duplicated some objects (sourceVS and sourceVSC), this is due to Kubernetes restriction – the data source reference cannot across namespaces. After the duplication completes, the objects related to the source user namespace will be deleted. Below diagram shows the relationships of the objects: ![expose-objects.png](expose-objects.png) After the first phase, we will see a backupPod attaching a backupPVC/backupPV which data is the same as the snapshot data. Then the second phase could start, this phase is related to the uploader. For file system uploader, the target of this phase is to get a path that is accessible locally by the uploader. There are some alternatives: - Get the path in the backupPod, so that VGDP runs inside the backupPod - Get the path on the host, so that VGDP runs inside node-agent, this is similar to the existing PodVolumeBackup Each option has their pros and cons, in the current design, we will use the second way because it is simpler in implementation and more controllable in workflow. ### Expose for DataDownload The Expose operation for DataDownload still takes two phases, The first phase creates below objects: - restorePVC: It is a PVC in Velero namespace with the same specification, it is used to provision the restorePV - restorePod: It is used to attach the restorePVC so that the restorePV could be provisioned by Kubernetes - restorePV: It is provisioned by Kubernetes and bound to restorePVC Data will be downloaded to the restorePV. No object is created in user source namespace and no activity is done there either. The second phase is the same as DataUpload, that is, we still use the host path to access restorePV and run VGDP in node-agent. ### Expose cleanup Some internal objects are created during the expose. Therefore, we need to clean them up to prevent internal objects from rampant growth. The cleanup happens in two cases: - When the controller finishes processing the DUCR/DDCR, this includes the cases that the DUCR/DDCR is completed, failed and cancelled. - When the DM restarts and the DM doesn't support restart recovery. When the DM comes back, it should detect all the ongoing DUCR/DDCR and clean up the expose. Specifically, VBDM should follow this rule since it doesn't support restart recovery. ## Cancellation We will leverage on BIA/RIA V2's Cancel method to implement the cancellation, below are the prototypes from BIA/RIA V2: ``` Cancel(operationID string, backup *api.Backup) error Cancel(operationID string, restore *api.Restore) error ``` At present, Velero doesn’t support canceling an ongoing backup, the current version of BIA/RIA V2 framework has some problems to support the end to end cancellation as well. Therefore, the current design doesn’t aim to deliver an end-to-end cancellation workflow but to implement the cancellation workflow inside the data movement, in future, when the other two parts are ready for cancellation, the data movement cancellation workflow could be directly used. Additionally, at present, the data movement cancellation will be used in the below scenarios: - When a backup is deleted, the backup deletion controller will call DMP’s Cancel method, so that the ongoing data movement will not run after the backup is deleted. - In the restart case, the ongoing backups will be marked as ```Failed``` when Velero restarts, at this time, DMP’s Cancel method will also be called when Velero server comes back because Velero will never process these backups. For data movement implementation, a ```Cancel``` field is included in the DUCR/DDCR. DMP patches the DUCR/DDCR with ```Cancel``` field set to true, then it keeps querying the status of DUCR/DDCR until it comes to Canceled status or timeout, by which time, DMP returns the Cancel call to Velero. Then DM needs to handle the cancel request, e.g., stop the data transition. For VBDM, it sets a signal to the uploader and the uploader will abort in a short time. The cancelled DUCR/DDCR is marked as ```Canceled```. Below diagram shows VBDM’s cancel workflow (take backup for example, restore is the same). ![cancel-sequence.png](cancel-sequence.png) It is possible that a DM doesn’t support cancellation at all or only support in a specific phase (e.g., during InProgress phase), if the cancellation is requested at an unexpected time or to an unexpecting DM, the behavior is decided by the DMP and DM, below are some recommendations: - If a DM doesn't support cancellation at all, DMP should be aware of this, so the DMP could return an error and fail early - If the cancellation is requested at an unexpected time, DMP is possibly not aware of this, it could still deliver it to the DM, so both Velero and DMP wait there until DM completes the cancel request or timeout VBDM's cancellation exactly follows the above rules. ## Parallelism Velero uses BIA/RIA V2 to launch data movement tasks, so from Velero’s perspective, the DataUpload/DataDownload CRs from the running backups will be submitted in parallel. Then how these CRs are handled is decided by data movers, in another words, the specific data mover decides whether to handle them sequentially or in parallel, as well what the parallelism is like. Velero makes no restriction to data movers regarding to this. Next, let’s focus on the parallelism of VBDM, which could also be a reference of the plugin data movers. VBDM is hosted by Velero node-agent, so there is one data movement controller instance on each Kubernetes node, which also means that these instances could handle the DataUpload/DataDownload CRs in parallel. On the other hand, a volume/volume snapshot may be accessed from only one or multiple nodes varying from its location, the backend storage architecture, etc. Therefore, the first decisive factor of the parallelism is the accessibility of a volume/volume snapshot. Therefore, we have below principles: - We should spread the data movement activities equally to all the nodes in the cluster. This requires a load balance design from Velero - In one node, it doesn’t mean the more concurrency the better, because the data movement activities are high in resource consumption, i.e., CPU, memory, and network throughput. For the same consideration, we should make this configurable because the best number should be set by users according to the bottleneck they detect We will address the two principles step by step. As the first step, VBDM’s parallelism is designed as below: - We don’t create the load balancing mechanism for the first step, we don’t detect the accessibility of the volume/volume snapshot explicitly. Instead, we create the backupPod/restorePod under the help of Kubernetes, Kubernetes schedules the backupPod/restorePod to the appropriate node, then the data movement controller on that node will handle the DataUpload/DataDownload CR there, so the resource will be consumed from that node. - We expose the configurable concurrency value per node, for details of how the concurrency number constraints various backups and restores which share VGDP, check the [node-agent concurrency design][3]. As for the resource consumption, it is related to the data scale of the data movement activity and it is charged to node-agent pods, so users should configure enough resource to node-agent pods. ## Progress Report When a DUCR/DDCR is in InProgress phase, users could check the progress. In DUCR/DDCR’s status, we have fields like ```totalBytes``` and ```doneBytes```, the same values will be displayed as a result of below querires: - Call ```kubectl get dataupload -n velero xxx or kubectl get datadownload -n velero xxx```. - Call ```velero backup describe –details```. This is implemented as part of BIA/RIA V2, the above values are transferred to async operation and this command retrieve them from the async operation instead of DUCR/DDCR. See [general progress monitoring design][2] for details ## Backup Sync DUCR contains the information that is required during restore but as mentioned above, it will not be synced because during restore its information is retrieved dynamically. Therefore, we have no change to Backup Sync. ## Backup Deletion Once a backup is deleted, the data in the backup repository should be deleted as well. On the other hand, the data is created by the specific DM, Velero doesn't know how to delete the data. Therefore, Velero relies on the DM to delete the backup data. As the current workflow, when ```velero backup delete``` CLI is called, a ```deletebackuprequests``` CR is submitted; after the backup delete controller finishes all the work, the ```deletebackuprequests``` CR will be deleted. In order to give an opportunity for the DM to delete the backup data, we remedy the workflow as below: - During the backup deletion, the backup delete controller retrieves all the DUCRs belong to the backup - The backup delete controller then creates the DUCRs into the cluster - Before deleting the ```deletebackuprequests``` CR, the backup delete controller adds a ```velero.io/dm-delete-backup``` finalizer to the CR - As a result, the ```deletebackuprequests``` CR will not be deleted until the finalizer is removed - The DM needs to watch the ```deletebackuprequests``` CRs with the ```velero.io/dm-delete-backup``` finalizer - Once the DM finds one, it collects a list of DUCRs that belong to the backup indicating by the ```deletebackuprequests``` CR's spec - If the list is not empty, the DM delete the backup data for each of the DUCRs in the list as well as the DUCRs themselves - Finally, when all the items in the list are processed successfully, the DM removes the ```velero.io/dm-delete-backup``` finalizer - Otherwise, if any error happens during the processing, the ```deletebackuprequests``` CR will be left there with the ```velero.io/dm-delete-backup``` finalizer, as well as the failed DUCRs - DMs may use a periodical manner to retry the failed delete requests ## Restarts If Velero restarts during a data movement activity, the backup/restore will be marked as failed when Velero server comes back, by this time, Velero will request a cancellation to the ongoing data movement. If DM restarts, Velero has no way to detect this, DM is expected to: - Either recover from the restart and continue the data movement - Or if DM doesn’t support recovery, it should cancel the data movement and mark the DUCR/DDCR as failed. DM should also clear any internal objects created during the data movement before and after the restart At present, VBDM doesn't support recovery, so it will follow the second rule. ## Kopia For Block Device To work with block devices, VGDP will be updated. Today, when Kopia attempts to create a snapshot of the block device, it will error because kopia does not support this file type. Kopia does have a nice set of interfaces that are able to be extended though. **Notice** The Kopia block mode uploader only supports non-Windows platforms, because the block mode code invokes some system calls that are not present in the Windows platform. To achieve the necessary information to determine the type of volume that is being used, we will need to pass in the volume mode in provider interface. ```go type PersistentVolumeMode string const ( // PersistentVolumeBlock means the volume will not be formatted with a filesystem and will remain a raw block device. PersistentVolumeBlock PersistentVolumeMode = "Block" // PersistentVolumeFilesystem means the volume will be or is formatted with a filesystem. PersistentVolumeFilesystem PersistentVolumeMode = "Filesystem" ) // Provider which is designed for one pod volume to do the backup or restore type Provider interface { // RunBackup which will do backup for one specific volume and return snapshotID, isSnapshotEmpty, error // updater is used for updating backup progress which implement by third-party RunBackup( ctx context.Context, path string, realSource string, tags map[string]string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg shared.UploaderConfig, updater uploader.ProgressUpdater) (string, bool, error) RunRestore( ctx context.Context, snapshotID string, volumePath string, volMode uploader.PersistentVolumeMode, updater uploader.ProgressUpdater) error ``` In this case, we will extend the default kopia uploader to add the ability, when a given volume is for a block mode and is mapped as a device, we will use the [StreamingFile](https://pkg.go.dev/github.com/kopia/kopia@v0.13.0/fs#StreamingFile) to stream the device and backup to the kopia repository. ```go func getLocalBlockEntry(sourcePath string) (fs.Entry, error) { source, err := resolveSymlink(sourcePath) if err != nil { return nil, errors.Wrap(err, "resolveSymlink") } fileInfo, err := os.Lstat(source) if err != nil { return nil, errors.Wrapf(err, "unable to get the source device information %s", source) } if (fileInfo.Sys().(*syscall.Stat_t).Mode & syscall.S_IFMT) != syscall.S_IFBLK { return nil, errors.Errorf("source path %s is not a block device", source) } device, err := os.Open(source) if err != nil { if os.IsPermission(err) || err.Error() == ErrNotPermitted { return nil, errors.Wrapf(err, "no permission to open the source device %s, make sure that node agent is running in privileged mode", source) } return nil, errors.Wrapf(err, "unable to open the source device %s", source) } sf := virtualfs.StreamingFileFromReader(source, device) return virtualfs.NewStaticDirectory(source, []fs.Entry{sf}), nil } ``` In the `pkg/uploader/kopia/snapshot.go` this is used in the Backup call like ```go if volMode == uploader.PersistentVolumeFilesystem { // to be consistent with restic when backup empty dir returns one error for upper logic handle dirs, err := os.ReadDir(source) if err != nil { return nil, false, errors.Wrapf(err, "Unable to read dir in path %s", source) } else if len(dirs) == 0 { return nil, true, nil } } source = filepath.Clean(source) ... var sourceEntry fs.Entry if volMode == uploader.PersistentVolumeBlock { sourceEntry, err = getLocalBlockEntry(source) if err != nil { return nil, false, errors.Wrap(err, "unable to get local block device entry") } } else { sourceEntry, err = getLocalFSEntry(source) if err != nil { return nil, false, errors.Wrap(err, "unable to get local filesystem entry") } } ... snapID, snapshotSize, err := SnapshotSource(kopiaCtx, repoWriter, fsUploader, sourceInfo, sourceEntry, forceFull, parentSnapshot, tags, log, "Kopia Uploader") ``` To handle restore, we need to extend the [Output](https://pkg.go.dev/github.com/kopia/kopia@v0.13.0/snapshot/restore#Output) interface and specifically the [FilesystemOutput](https://pkg.go.dev/github.com/kopia/kopia@v0.13.0/snapshot/restore#FilesystemOutput). We only need to extend two functions the rest will be passed through. ```go type BlockOutput struct { *restore.FilesystemOutput targetFileName string } var _ restore.Output = &BlockOutput{} const bufferSize = 128 * 1024 func (o *BlockOutput) WriteFile(ctx context.Context, relativePath string, remoteFile fs.File) error { remoteReader, err := remoteFile.Open(ctx) if err != nil { return errors.Wrapf(err, "failed to open remote file %s", remoteFile.Name()) } defer remoteReader.Close() targetFile, err := os.Create(o.targetFileName) if err != nil { return errors.Wrapf(err, "failed to open file %s", o.targetFileName) } defer targetFile.Close() buffer := make([]byte, bufferSize) readData := true for readData { bytesToWrite, err := remoteReader.Read(buffer) if err != nil { if err != io.EOF { return errors.Wrapf(err, "failed to read data from remote file %s", o.targetFileName) } readData = false } if bytesToWrite > 0 { offset := 0 for bytesToWrite > 0 { if bytesWritten, err := targetFile.Write(buffer[offset:bytesToWrite]); err == nil { bytesToWrite -= bytesWritten offset += bytesWritten } else { return errors.Wrapf(err, "failed to write data to file %s", o.targetFileName) } } } } return nil } func (o *BlockOutput) BeginDirectory(ctx context.Context, relativePath string, e fs.Directory) error { var err error o.targetFileName, err = filepath.EvalSymlinks(o.TargetPath) if err != nil { return errors.Wrapf(err, "unable to evaluate symlinks for %s", o.targetFileName) } fileInfo, err := os.Lstat(o.targetFileName) if err != nil { return errors.Wrapf(err, "unable to get the target device information for %s", o.TargetPath) } if (fileInfo.Sys().(*syscall.Stat_t).Mode & syscall.S_IFMT) != syscall.S_IFBLK { return errors.Errorf("target file %s is not a block device", o.TargetPath) } return nil } ``` Additional mount is required in the node-agent specification to resolve symlinks to the block devices from /host_pods/POD_ID/volumeDevices/kubernetes.io~csi directory. ```yaml - mountPath: /var/lib/kubelet/plugins mountPropagation: HostToContainer name: host-plugins .... - hostPath: path: /var/lib/kubelet/plugins name: host-plugins ``` Privileged mode is required to access the block devices in /var/lib/kubelet/plugins/kubernetes.io/csi/volumeDevices/publish directory as confirmed by testing on EKS and Minikube. ```yaml SecurityContext: &corev1.SecurityContext{ Privileged: &c.privilegedNodeAgent, }, ``` ## Plugin Data Movers There should be only one DM to handle a specific DUCR/DDCR in all cases. If more than one DMs process a DUCR/DDCR at the same time, there will be a disaster. Therefore, a DM should check the dataMover field of DUCR/DDCR and process the CRs belong to it only. For example, VBDM reconciles DUCR/DDCR with their ```dataMover``` field set to "" or "velero", it will skip all others. This means during the installation, users are allowed to install more than one DMs, but the DMs should follow the above rule. When creating a backup, we should allow users to specify the data mover, so a new backup CLI option is required. For restore, we should retrieve the same information from the corresponding backup, so that the data mover selection is consistent. At present, Velero doesn't have the capability to verify the existence of the specified data mover. As a result, if a wrong data mover name is specified for the backup or the specified data mover is not installed, nothing will fail early, DUCR/DDCR is still created and Velero will wait there until timeout. Plugin DMs may need some private configurations, the plugin DM providers are recommended to create a self-managed configMap to take the information. Velero doesn't maintain the lifecycle of the configMap. Besides, the configMap is recommended to named as the DM's name, in this way, if Velero or DMP recognizes some generic options that varies between DMs, the options could be added into the configMap and visited by Velero or DMP. Conclusively, below are the steps plugin DMs need to do in order to integrate to Velero volume snapshot data movement. ### Backup - Handle and only handle DUCRs with the matching ```dataMover``` value - Maintain the phases and progresses of DUCRs correctly - If supported, response to the Cancel request of DUCRs - Dispose the volume snapshots as well as their related objects after the snapshot data is transferred ### Restore - Handle and only handle DDCRs with the matching ```dataMover``` value - Maintain the phases and progresses of DDCRs correctly - If supported, response to the Cancel request of DDCRs - Create the PV with data restored to it - Set PV's ```claimRef``` to the provided PVC and set ```velero.io/dynamic-pv-restore``` label ## Working Mode It doesn’t mean that once the data movement feature is enabled users must move every snapshot. We will support below two working modes: - Don’t move snapshots. This is same with the existing CSI snapshot feature, that is, native snapshots are taken and kept - Move snapshot data and delete native snapshots. This means that once the data movement completes, the native snapshots will be deleted. For this purpose, we need to add a new option in the backup command as well as the Backup CRD. The same option for restore will be retrieved from the specified backup, so that the working mode is consistent. ## Backup and Restore CRD Changes We add below new fields in the Backup CRD: ``` // SnapshotMoveData specifies whether snapshot data should be moved // +optional // +nullable SnapshotMoveData *bool `json:"snapshotMoveData,omitempty"` // DataMover specifies the data mover to be used by the backup. // If DataMover is "" or "velero", the built-in data mover will be used. // +optional DataMover string `json:"datamover,omitempty"` ``` SnapshotMoveData will be used to decide the Working Mode. DataMover will be used to decide the data mover to handle the DUCR. DUCR's DataMover value is derived from this value. As mentioned in the Plugin Data Movers section, the data movement information for a restore should be the same with the backup. Therefore, the working mode for restore should be decided by checking the corresponding Backup CR; when creating a DDCR, the DataMover value should be retrieved from the corresponding Backup Result. ## Logging The logs during the data movement are categorized as below: - Logs generated by Velero - Logs generated by DMPs - Logs generated by DMs For 1 and 2, the existing plugin mechanism guarantees that the logs could be saved into the Velero server log as well as backup/restore persistent log. For 3, Velero leverage on DMs to decide how to save the log, but they will not go to Velero server log or backup/restore persistent log. For VBDM, the logs are saved in the node-agent server log. ## Installation DMs need to be configured during installation so that they can be installed. Plugin DMs may have their own configuration, for VGDM, the only requirement is to install Velero node-agent. Moreover, the DMP is also required during the installation. From release-1.14, the `github.com/vmware-tanzu/velero-plugin-for-csi` repository, which is the Velero CSI plugin, is merged into the `github.com/vmware-tanzu/velero` repository. The reason to merge the CSI plugin is: * The VolumeSnapshot data mover depends on the CSI plugin, it's reasonabe to integrate them. * This change reduces the Velero deploying complexity. * This makes performance tuning easier in the future. As a result, no need to install Velero CSI plugin anymore. For example, to move CSI snapshot through VBDM, below is the installation script: ``` velero install \ --provider \ --image \ --features=EnableCSI \ --use-node-agent \ ``` ## Upgrade For VBDM, no new installation option is introduced, so upgrade is not affected. If plugin DMs require new options and so the upgrade is affected, they should explain them in their own documents. ## CLI As explained in the Working Mode section, we add one more flag ```snapshot-move-data``` to indicate whether the snapshot data should be moved. As explained in the Plugin Data Movers section, we add one more flag ```data-mover``` for users to configure the data mover to move the snapshot data. Example of backup command are as below. Below CLI means to create a backup with volume snapshot data movement enabled and with VBDM as the data mover: ``` velero backup create xxx --include-namespaces --snapshot-move-data ``` Below CLI has the same meaning as the first one: ``` velero backup create xxx --include-namespaces --snapshot-move-data --data-mover velero ``` Below CLI means to create a backup with volume snapshot data movement enabled and with "xxx-plugin-dm" as the data mover: ``` velero backup create xxx --include-namespaces --snapshot-move-data --data-mover xxx-plugin-dm ``` Restore command is kept as is. [1]: ../unified-repo-and-kopia-integration/unified-repo-and-kopia-integration.md [2]: ../general-progress-monitoring.md [3]: ../node-agent-concurrency.md ================================================ FILE: design/Implemented/wait-for-additional-items.md ================================================ # Wait for AdditionalItems to be ready on Restore When a velero `RestoreItemAction` plugin returns a list of resources via `AdditionalItems`, velero restores these resources before restoring the current resource. There is a race condition here, as it is possible that after running the restore on these returned items, the current item's restore might execute before the additional items are available. Depending on the nature of the dependency between the current item and the additional items, this could cause the restore of the current item to fail. ## Goals - Enable Velero to ensure that Additional items returned by a restore plugin's `Execute` func are ready before restoring the current item - Augment the RestoreItemAction plugin interface to allow the plugins to determine when an additional item is ready, since doing so requires knowledge specific to the resource type. ## Background Because Velero does not wait after restoring additional items to restore the current item, in some cases the current item restore will fail if the additional items are not yet ready. Velero (and the `RestoreItemAction` plugins) need to implement this "wait until ready" functionality. ## High-Level Design After each RestoreItemAction execute call (and following restore of any returned additional items) , we need to wait for these returned Additional Items to be ready before restoring the current item. In order to do this, we also need to extend the `RestoreItemActionExecuteOutput` struct to allow the plugin which returned additional items to determine whether they are ready. ## Detailed Design ### `restoreItem` Changes When each `RestoreItemAction` `Execute()` call returns, the `RestoreItemActionExecuteOutput` struct contains a slice of `AdditionalItems` which must be restored before this item can be restored. After restoring these items, Velero needs to be able to wait for them to be ready before moving on to the next item. Right after looping over the additional items at https://github.com/vmware-tanzu/velero/blob/main/pkg/restore/restore.go#L960-L991 we still have a reference to the additional items (`GroupResource` and namespaced name), as well as a reference to the `RestoreItemAction` plugin which required it. At this point, if the `RestoreItemActionExecuteOutput` `WaitForAdditionalItems` field is set to `true` we need to call a func similar to `crdAvailable` which we will call `itemsAvailable` https://github.com/vmware-tanzu/velero/blob/main/pkg/restore/restore.go#L623 This func should also be defined within restore.go Instead of the one minute CRD timeout, we'll use a timeout specific to waiting for additional items. There will be a new field added to serverConfig, `additionalItemsReadyTimeout`, with a `defaultAdditionalItemsReadyTimeout` const set to 10 minutes. In addition, each plugin will be able to define an override for the global server-level value, which will be added as another optional field in the `RestoreItemActionExecuteOutput` struct. Instead of the `IsUnstructuredCRDReady` call, we'll call `AreAdditionalItemsReady` on the plugin, passing in the same `AdditionalItems` slice as an argument (with items which failed to restore filtered out). If this func returns an error, then `itemsAvailable` will propagate the error, and `restoreItem` will handle it the same way it handles an error return on restoring an additional item. If the timeout is reached without ready returning true, velero will continue on to attempt restore of the current item. ### `RestoreItemAction` plugin interface changes In order to implement the `AreAdditionalItemsReady` plugin func, a new function will be added to the `RestoreItemAction` interface. ``` type RestoreItemAction interface { // AppliesTo returns information about which resources this action should be invoked for. // A RestoreItemAction's Execute function will only be invoked on items that match the returned // selector. A zero-valued ResourceSelector matches all resources. AppliesTo() (ResourceSelector, error) // Execute allows the ItemAction to perform arbitrary logic with the item being restored, // including mutating the item itself prior to restore. The item (unmodified or modified) // should be returned, along with an optional slice of ResourceIdentifiers specifying additional // related items that should be restored, a warning (which will be logged but will not prevent // the item from being restored) or error (which will be logged and will prevent the item // from being restored) if applicable. Execute(input *RestoreItemActionExecuteInput) (*RestoreItemActionExecuteOutput, error) // AreAdditionalItemsReady allows the ItemAction to communicate whether the passed-in // slice of AdditionalItems (previously returned by Execute()) // are ready. Returns true if all items are ready, and false // otherwise. The second return value is an error string if an // error occurred. AreAdditionalItemsReady(restore *api.Restore, AdditionalItems []ResourceIdentifier) (bool, string) } ``` ### `RestoreItemActionExecuteOutput` changes Two new fields will be added to `RestoreItemActionExecuteOutput`, both optional. `AdditionalItemsReadyTimeout`, if non-zero, will override `serverConfig.additionalItemsReadyTimeout`. If `WaitForAdditionalItems` is true, then `restoreItem` will call `itemsAvailable` which will invoke the plugin func `AreAdditionalItemsReady` and wait until the func returns true or the timeout is reached. If `WaitForAdditionalItems` is false (the default case), then current velero behavior will be followed. Existing plugins which do not need to signal to wait for `AdditionalItems` won't need to change their `Execute()` functions. In addition, a new func, `WithItemsWait()` will be added to `RestoreItemActionExecuteOutput` similar to `WithoutRestore()` which will set `WaitForAdditionalItems` to true. This will allow a plugin to include waiting for AdditionalItems like this: ``` func AreAdditionalItemsReady (restore *api.Restore, additionalItems []ResourceIdentifier) (bool, string) { ... return true, "" } func (p *RestorePlugin) Execute(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { ... return velero.NewRestoreItemActionExecuteOutput(input.Item).WithItemsWait(), nil } ``` ``` // RestoreItemActionExecuteOutput contains the output variables for the ItemAction's Execution function. type RestoreItemActionExecuteOutput struct { // UpdatedItem is the item being restored mutated by ItemAction. UpdatedItem runtime.Unstructured // AdditionalItems is a list of additional related items that should // be restored. AdditionalItems []ResourceIdentifier // SkipRestore tells velero to stop executing further actions // on this item, and skip the restore step. When this field's // value is true, AdditionalItems will be ignored. SkipRestore bool // WaitForAdditionalItems determines whether velero will wait // until AreAdditionalItemsReady returns true before restoring // this item. If this field's value is true, then after restoring // the returned AdditionalItems, velero will not restore this item // until AreAdditionalItemsReady returns true or the timeout is // reached. Otherwise, AreAdditionalItemsReady is not called. WaitForAdditionalItems bool // AdditionalItemsReadyTimeout will override serverConfig.additionalItemsReadyTimeout // if specified. This value specifies how long velero will wait // for additional items to be ready before moving on. AdditionalItemsReadyTimeout time.Duration } // WithItemsWait returns RestoreItemActionExecuteOutput with WaitForAdditionalItems set to true. func (r *RestoreItemActionExecuteOutput) WithItemsWait() ) *RestoreItemActionExecuteOutput { r.WaitForAdditionalItems = true return r } ``` ## New design iteration (Feb 2021) In starting the implementation based on the originally approved design, I've run into an unexpected snag. When adding the wait func pointer to the `RestoreItemActionExecuteOutput` struct, I had forgotten about the protocol buffer message format that's used for passing args to the plugin methods. Funcs are predefined RPC calls with autogenerated go code, so we can't just pass a regular golang func pointer in the struct. I've modified the above design to instead use an explicit `AreAdditionalItemsReady` func. Since this will break backwards compatibility with current `RestoreItemAction` plugins, implementation of this feature should wait until Velero plugin versioning, as described in https://github.com/vmware-tanzu/velero/issues/3285 is implemented. With plugin versioning in place, existing (non-versioned or 1.0-versioned) `RestoreItemAction` plugins which do not define `AreAdditionalItemsReady` would be able to coexist with a to-be-implemented `RestoreItemAction` plugin version 2.0 (or 1.1, etc.) which defines this new interface method. Without plugin versioning, implementing this feature would break all existing plugins until they define `AreAdditionalItemsReady`. Also note that when moving to the new plugin version, the vast majority of plugins will probably not need to wait for additional items. All they will need to do to react to this plugin interface change would be to define the following in the plugin: ``` func AreAdditionalItemsReady (restore *api.Restore, additionalItems []ResourceIdentifier) (bool, string) { return true, "" } ``` As long as they never set `WaitForAdditionalItems` to true, this function won't be called anyway, but if it is called, there will be no waiting, since it will always return true. ================================================ FILE: design/Implemented/wildcard-namespace-support-design.md ================================================ # Wildcard Namespace Support ## Abstract Velero currently treats namespace patterns with glob characters as literal strings. This design adds wildcard expansion to support flexible namespace selection using patterns like `app-*` or `test-{dev,staging}`. ## Background Requested in [#1874](https://github.com/vmware-tanzu/velero/issues/1874) for more flexible namespace selection. ## Goals - Support glob pattern expansion in namespace includes/excludes - Maintain backward compatibility with existing `*` behavior ## Non-Goals - Complex regex patterns beyond basic globs ## High-Level Design Wildcard expansion occurs early in both backup and restore flows, converting patterns to literal namespace lists before normal processing. ### Backup Flow Expansion happens in `getResourceItems()` before namespace collection: 1. Check if wildcards exist using `ShouldExpandWildcards()` 2. Expand patterns against active cluster namespaces 3. Replace includes/excludes with expanded literal namespaces 4. Continue with normal backup processing ### Restore Flow Expansion occurs in `execute()` after parsing backup contents: 1. Extract available namespaces from backup tar 2. Expand patterns against backup namespaces (not cluster namespaces) 3. Update restore context with expanded namespaces 4. Continue with normal restore processing This ensures restore wildcards match actual backup contents, not current cluster state. ## Detailed Design ### Status Fields Add wildcard expansion tracking to backup and restore CRDs: ```go type WildcardNamespaceStatus struct { // IncludeWildcardMatches records namespaces that matched include patterns // +optional IncludeWildcardMatches []string `json:"includeWildcardMatches,omitempty"` // ExcludeWildcardMatches records namespaces that matched exclude patterns // +optional ExcludeWildcardMatches []string `json:"excludeWildcardMatches,omitempty"` // WildcardResult records final namespaces after wildcard processing // +optional WildcardResult []string `json:"wildcardResult,omitempty"` } // Added to both BackupStatus and RestoreStatus type BackupStatus struct { // WildcardNamespaces contains wildcard expansion results // +optional WildcardNamespaces *WildcardNamespaceStatus `json:"wildcardNamespaces,omitempty"` } ``` ### Wildcard Expansion Package New `pkg/util/wildcard/expand.go` package provides: - `ShouldExpandWildcards()` - Skip expansion for simple "*" case - `ExpandWildcards()` - Main expansion function using `github.com/gobwas/glob` - Pattern validation rejecting unsupported regex symbols **Supported patterns**: `*`, `?`, `[abc]`, `{a,b,c}` **Unsupported**: `|()`, `**` ### Implementation Details #### Backup Integration (`pkg/backup/item_collector.go`) Expansion in `getResourceItems()`: - Call `wildcard.ExpandWildcards()` with cluster namespaces - Update `NamespaceIncludesExcludes` with expanded results - Populate status fields with expansion results #### Restore Integration (`pkg/restore/restore.go`) Expansion in `execute()`: ```go if wildcard.ShouldExpandWildcards(includes, excludes) { availableNamespaces := extractNamespacesFromBackup(backupResources) expandedIncludes, expandedExcludes, err := wildcard.ExpandWildcards( availableNamespaces, includes, excludes) // Update context and status } ``` ## Alternatives Considered 1. **Client-side expansion**: Rejected because it wouldn't work for scheduled backups 2. **Expansion in `collectNamespaces`**: Rejected because these functions expect literal namespaces ## Compatibility Maintains full backward compatibility - existing "*" behavior unchanged. ## Implementation Target: Velero 1.18 ================================================ FILE: design/_template.md ================================================ # Design proposal template `` _Note_: The preferred style for design documents is one sentence per line. *Do not wrap lines*. This aids in review of the document as changes to a line are not obscured by the reflowing those changes caused and has a side effect of avoiding debate about one or two space after a period. _Note_: The name of the file should follow the name pattern `_design.md`, e.g: `listener-design.md`. ## Abstract One to two sentences that describes the goal of this proposal and the problem being solved by the proposed change. The reader should be able to tell by the title, and the opening paragraph, if this document is relevant to them. ## Background One to two paragraphs of exposition to set the context for this proposal. ## Goals - A short list of things which will be accomplished by implementing this proposal. - Two things is ok. - Three is pushing it. - More than three goals suggests that the proposal's scope is too large. ## Non Goals - A short list of items which are: - a. out of scope - b. follow on items which are deliberately excluded from this proposal. ## High-Level Design One to two paragraphs that describe the high level changes that will be made to implement this proposal. ## Detailed Design A detailed design describing how the changes to the product should be made. The names of types, fields, interfaces, and methods should be agreed on here, not debated in code review. The same applies to changes in CRDs, YAML examples, and so on. Ideally the changes should be made in sequence so that the work required to implement this design can be done incrementally, possibly in parallel. ## Alternatives Considered If there are alternative high level or detailed designs that were not pursued they should be called out here with a brief explanation of why they were not pursued. ## Security Considerations If this proposal has an impact to the security of the product, its users, or data stored or transmitted via the product, they must be addressed here. ## Compatibility A discussion of any compatibility issues that need to be considered ## Implementation A description of the implementation, timelines, and any resources that have agreed to contribute. ## Open Issues A discussion of issues relating to this proposal for which the author does not know the solution. This section may be omitted if there are none. ================================================ FILE: design/cli-install-changes.md ================================================ # Proposal for a more intuitive CLI to install and configure Velero Currently, the Velero CLI tool has a `install` command that configures numerous major and minor aspects of Velero. As a result, the combined set of flags for this `install` command makes it hard to intuit and reason about the different Velero components. This document proposes changes to improve the UX for installation and configuration in a way that would make it easier for the user to discover what needs to be configured by looking at what is available in the CLI rather then having to rely heavily on our documentation for the usage. At the same time, it is expected that the documentation update to reflect these changes will also make the documentation flow easier to follow. This proposal prioritizes discoverability and self-documentation over minimizing length or number of commands and flags. ## Goals - Split flags currently under the `velero install` command into multiple commands, and group flags under commands in a way that allows a good level of discovery and self-documentation - Maintain compatibility with gitops practices (i.e. ability to generate a full set of yaml for install that can be stored in source control) - Have a clear path for deprecating commands ## Non Goals - Introduce new CLI features - Propose changes to the CLI that go beyond the functionality of install and configure - Optimize for shorter length or number of commands/flags ## Background This document proposes users could benefit from a more intuitive and self-documenting CLI setup as compared to our existing CLI UX. Ultimately, it is proposed that a recipe-style CLI flow for installation, configuration and use would greatly contribute to this purpose. Also, the `install` command currently can be reused to update Velero deployment configurations. For server and restic related install and configurations, settings will be moved to under `velero config`. ## High-Level Design The naming and organization of the proposed new CLI commands below have been inspired on the `kubectl` commands, particularly `kubectl set` and `kubectl config`. #### General CLI improvements These are improvements that are part of this proposal: - Go over all flags and document what is optional, what is required, and default values. - Capitalize all help messages #### Commands The organization of the commands follows this format: ``` velero [resource] [operation] [flags] ``` To conform with Velero's current practice: - commands will also work by swapping the operation/resource. - the "object" of a command is an argument, and flags are strictly for modifiers (example: `backup get my-backup` and not `backup get --name my-backup`) All commands will include the `--dry-run` flag, which can be used to output yaml files containing the commands' configuration for resource creation or patching. `--dry-run generate resources, but don't send them to the cluster. Use with -o. Optional.` The `--help` and `--output` flags will also be included for all commands, omitted below for brevity. Below is the proposed set of new commands to setup and configure Velero. 1) `velero config` ``` server Configure up the namespace, RBAC, deployment, etc., but does not add any external plugins, BSL/VSL definitions. This would be the minimum set of commands to get the Velero server up and running and ready to accept other configurations. --label-columns stringArray a comma-separated list of labels to be displayed as columns --show-labels show labels in the last column --image string image to use for the Velero and restic server pods. Optional. (default "velero/velero:latest") --pod-annotations mapStringString annotations to add to the Velero and restic pods. Optional. Format is key1=value1,key2=value2 --restore-only run the server in restore-only mode. Optional. --pod-cpu-limit string CPU limit for Velero pod. A value of "0" is treated as unbounded. Optional. (default "1000m") --pod-cpu-request string CPU request for Velero pod. A value of "0" is treated as unbounded. Optional. (default "500m") --pod-mem-limit string memory limit for Velero pod. A value of "0" is treated as unbounded. Optional. (default "256Mi") --pod-mem-request string memory request for Velero pod. A value of "0" is treated as unbounded. Optional. (default "128Mi") --client-burst int maximum number of requests by the server to the Kubernetes API in a short period of time (default 30) --client-qps float32 maximum number of requests per second by the server to the Kubernetes API once the burst limit has been reached (default 20) --default-backup-ttl duration how long to wait by default before backups can be garbage collected (default 720h0m0s) --disable-controllers strings list of controllers to disable on startup. Valid values are backup,backup-sync,schedule,gc,backup-deletion,restore,download-request,restic-repo,server-status-request --log-format the format for log output. Valid values are text, json. (default text) --log-level the level at which to log. Valid values are debug, info, warning, error, fatal, panic. (default info) --metrics-address string the address to expose prometheus metrics (default ":8085") --plugin-dir string directory containing Velero plugins (default "/plugins") --profiler-address string the address to expose the pprof profiler (default "localhost:6060") --restore-only run in a mode where only restores are allowed; backups, schedules, and garbage-collection are all disabled. DEPRECATED: this flag will be removed in v2.0. Use read-only backup storage locations instead. --restore-resource-priorities strings desired order of resource restores; any resource not in the list will be restored alphabetically after the prioritized resources (default [namespaces,storageclasses,persistentvolumes,persistentvolumeclaims,secrets,configmaps,serviceaccounts,limitranges,pods,replicaset,customresourcedefinitions]) --terminating-resource-timeout duration how long to wait on persistent volumes and namespaces to terminate during a restore before timing out (default 10m0s) restic Configuration for restic operations. --default-prune-frequency duration how often 'restic prune' is run for restic repositories by default. Optional. --pod-annotations mapStringString annotations to add to the Velero and restic pods. Optional. Format is key1=value1,key2=value2 --pod-cpu-limit string CPU limit for restic pod. A value of "0" is treated as unbounded. Optional. (default "0") --pod-cpu-request string CPU request for restic pod. A value of "0" is treated as unbounded. Optional. (default "0") --pod-mem-limit string memory limit for restic pod. A value of "0" is treated as unbounded. Optional. (default "0") --pod-mem-request string memory request for restic pod. A value of "0" is treated as unbounded. Optional. (default "0") --timeout duration how long backups/restores of pod volumes should be allowed to run before timing out (default 1h0m0s) repo get Get restic repositories ``` The `velero config server` command will create the following resources: ``` Namespace Deployment backups.velero.io backupstoragelocations.velero.io deletebackuprequests.velero.io downloadrequests.velero.io podvolumebackups.velero.io podvolumerestores.velero.io resticrepositories.velero.io restores.velero.io schedules.velero.io serverstatusrequests.velero.io volumesnapshotlocations.velero.io ``` Note: Velero will maintain the `velero server` command run by the Velero pod, which starts the Velero server deployment. 2) `velero backup-location` Commands/flags for backup locations. ``` set --default string sets the default backup storage location (default "default") (NEW, -- was `server --default-backup-storage-location; could be set as an annotation on the BSL) --credentials mapStringString sets the name of the corresponding credentials secret for a provider. Format is provider:credentials-secret-name. (NEW) --cacert-file mapStringString configuration to use for creating a secret containing a custom certificate for an S3 location of a plugin provider. Format is provider:path-to-file. (NEW) create NAME [flags] --default Sets this new location to be the new default backup location. Default is false. (NEW) --access-mode access mode for the backup storage location. Valid values are ReadWrite,ReadOnly (default ReadWrite) --backup-sync-period 0s how often to ensure all Velero backups in object storage exist as Backup API objects in the cluster. Optional. Set this to 0s to disable sync --bucket string name of the object storage bucket where backups should be stored. Required. --config mapStringString configuration to use for creating a backup storage location. Format is key1=value1,key2=value2 (was also in `velero install --backup-location-config`). Required for Azure. --provider string provider name for backup storage. Required. --label-columns stringArray a comma-separated list of labels to be displayed as columns --labels mapStringString labels to apply to the backup storage location --prefix string prefix under which all Velero data should be stored within the bucket. Optional. --provider string name of the backup storage provider (e.g. aws, azure, gcp) --show-labels show labels in the last column --credentials mapStringString sets the name of the corresponding credentials secret for a provider. Format is provider:credentials-secret-name. (NEW) --cacert-file mapStringString configuration to use for creating a secret containing a custom certificate for an S3 location of a plugin provider. Format is provider:path-to-file. (NEW) get Display backup storage locations --default displays the current default backup storage location (NEW) --label-columns stringArray a comma-separated list of labels to be displayed as columns -l, --selector string only show items matching this label selector --show-labels show labels in the last column ``` 3) `velero snapshot-location` Commands/flags for snapshot locations. ``` set --default mapStringString sets the list of unique volume providers and default volume snapshot location (provider1:location-01,provider2:location-02,...) (NEW, -- was `server --default-volume-snapshot-locations; could be set as an annotation on the VSL) --credentials mapStringString sets the list of name of the corresponding credentials secret for providers. Format is (provider1:credentials-secret-name1,provider2:credentials-secret-name2,...) (NEW) create NAME [flags] --default Sets these new locations to be the new default snapshot locations. Default is false. (NEW) --config mapStringString configuration to use for creating a volume snapshot location. Format is key1=value1,key2=value2 (was also in `velero install --`snapshot-location-config`). Required. --provider string provider name for volume storage. Required. --label-columns stringArray a comma-separated list of labels to be displayed as columns --labels mapStringString labels to apply to the volume snapshot location --provider string name of the volume snapshot provider (e.g. aws, azure, gcp) --show-labels show labels in the last column --credentials mapStringString sets the list of name of the corresponding credentials secret for providers. Format is (provider1:credentials-secret-name1,provider2:credentials-secret-name2,...) (NEW) get Display snapshot locations --default list of unique volume providers and default volume snapshot location (provider1:location-01,provider2:location-02,...) (NEW -- was `server --default-volume-snapshot-locations`)) ``` 4) `velero plugin` Configuration for plugins. ``` add stringArray IMAGES [flags] - add plugin container images to install into the Velero Deployment get get information for all plugins on the velero server (was `get`) --timeout duration maximum time to wait for plugin information to be reported (default 5s) remove Remove a plugin [NAME | IMAGE] set --credentials-file mapStringString configuration to use for creating a secret containing the AIM credentials for a plugin provider. Format is provider:path-to-file. (was `secret-file`) --no-secret flag indicating if a secret should be created. Must be used as confirmation if create --secret-file is not provided. Optional. (MOVED FROM install -- not sure we need it?) --sa-annotations mapStringString annotations to add to the Velero ServiceAccount for GKE. Add iam.gke.io/gcp-service-account=[GSA_NAME]@[PROJECT_NAME].iam.gserviceaccount.com for workload identity. Optional. Format is key1=value1,key2=value2 ``` #### Example Considering this proposal, let's consider what a high-level documentation for getting Velero ready to do backups could look like for Velero users: After installing the Velero CLI: ``` velero config server [flags] (required) velero config restic [flags] velero plugin add IMAGES [flags] (add/config provider plugins) velero backup-location/snapshot-location create NAME [flags] (run `velero plugin --get` to see what kind of plugins are available; create locations) velero backup/restore/schedule create/get/delete NAME [flags] ``` The above recipe-style documentation should highlight 1) the main components of Velero, and, 2) the relationship/dependency between the main components ### Deprecation #### Timeline In order to maintain compatibility with the current Velero version for a sufficient amount of time, and give users a chance to upgrade any install scripts they might have, we will keep the current `velero install` command in parallel with the new commands until the next major Velero version, which will be Velero 2.0. In the mean time, ia deprecation warning will be added to the `velero install` command. #### Commands/flags deprecated or moved ##### Velero Install `velero install (DEPRECATED)` Flags moved to... ...`velero config server`: ``` --image string image to use for the Velero and restic server pods. Optional. (default "velero/velero:latest") --label-columns stringArray a comma-separated list of labels to be displayed as columns --pod-annotations mapStringString annotations to add to the Velero and restic pods. Optional. Format is key1=value1,key2=value2 --show-labels show labels in the last column --pod-cpu-limit string CPU limit for Velero pod. A value of "0" is treated as unbounded. Optional. (default "1000m") --pod-cpu-request string CPU request for Velero pod. A value of "0" is treated as unbounded. Optional. (default "500m") --pod-mem-limit string memory limit for Velero pod. A value of "0" is treated as unbounded. Optional. (default "256Mi") --pod-mem-request string memory request for Velero pod. A value of "0" is treated as unbounded. Optional. (default "128Mi") ``` ...`velero config restic` ``` --default-prune-frequency duration how often 'restic prune' is run for restic repositories by default. Optional. --pod-cpu-limit string CPU limit for restic pod. A value of "0" is treated as unbounded. Optional. (default "0") --pod-cpu-request string CPU request for restic pod. A value of "0" is treated as unbounded. Optional. (default "0") --pod-mem-limit string memory limit for restic pod. A value of "0" is treated as unbounded. Optional. (default "0") --pod-mem-request string memory request for restic pod. A value of "0" is treated as unbounded. Optional. (default "0") ``` ...`backup-location create` ``` --backup-location-config mapStringString configuration to use for the backup storage location. Format is key1=value1,key2=value2 --bucket string name of the object storage bucket where backups should be stored --prefix string prefix under which all Velero data should be stored within the bucket. Optional. ``` ...`snapshot-location create` ``` --snapshot-location-config mapStringString configuration to use for the volume snapshot location. Format is key1=value1,key2=value2 ``` ...both `backup-location create` and `snapshot-location create` ``` --provider string provider name for backup and volume storage ``` ...`plugin` ``` --plugins stringArray Plugin container images to install into the Velero Deployment --sa-annotations mapStringString annotations to add to the Velero ServiceAccount. Add iam.gke.io/gcp-service-account=[GSA_NAME]@[PROJECT_NAME].iam.gserviceaccount.com for workload identity. Optional. Format is key1=value1,key2=value2 --no-secret flag indicating if a secret should be created. Must be used as confirmation if --secret-file is not provided. Optional. --secret-file string (renamed `credentials-file`) file containing credentials for backup and volume provider. If not specified, --no-secret must be used for confirmation. Optional. ``` Flags to deprecate: ``` --no-default-backup-location flag indicating if a default backup location should be created. Must be used as confirmation if --bucket or --provider are not provided. Optional. --use-volume-snapshots whether or not to create snapshot location automatically. Set to false if you do not plan to create volume snapshots via a storage provider. (default true) --wait wait for Velero deployment to be ready. Optional. --use-restic (obsolete since now we have `velero config restic`) ``` ##### Velero Server These flags will be moved to under `velero config server`: `velero server --default-backup-storage-location (DEPRECATED)` changed to `velero backup-location set --default` `velero server --default-volume-snapshot-locations (DEPRECATED)` changed to `velero snapshot-location set --default` The value for these flags will be stored as annotations. ## Detailed Design #### Handling CA certs In anticipation of a new configuration implementation to handle custom CA certs (as per design doc https://github.com/vmware-tanzu/velero/blob/main/design/custom-ca-support.md), a new flag `velero storage-location create/set --cacert-file mapStringString` is proposed. It sets the configuration to use for creating a secret containing a custom certificate for an S3 location of a plugin provider. Format is provider:path-to-file. See discussion https://github.com/vmware-tanzu/velero/pull/2259#discussion_r384700723 for more clarification. #### Renaming "provider" to "location-plugin" As part of this change, we should change to use the term `location-plugin` instead of `provider`. The reasoning: in practice, we usually have 1 plugin per provider, and if there is an implementation for both object store and volume snapshotter for that provider, it will all be contained in the same plugin. When we handle plugins, we follow this logic. In other words, there's a plugin name (ex: `velero.io/aws`) and it can contain implementations of kind `ObjectStore` and/or `VolumeSnapshotter`. But when we handle BSL or VSL (and the CLI commands/flags that configure them), we use the term `provider`, which can cause ambiguity as if that is a kind of thing different from a plugin. If the plugin is the "thing" that contains the implementation for the desired provider, we should make it easier for the user to guess that and change BackupStorageLocation/VolumeSnapshotLocation `Spec.Provider` field to be called `Spec.Location-Plugin` and all related CLI command flags to `location-plugin`, and update the docs accordingly. This change will require a CRD version bump and deprecation cycle. #### GitOps Compatibility To maintain compatibility with gitops practices, each of the new commands will generate `yaml` output that can be stored in source control. For content examples, please refer to the files here: https://github.com/carlisia/velero/tree/c-cli-design/design/CLI/PoC Note: actual `yaml` file names are defined by the user. `velero config server` - base/deployment.yaml `velero config restic` - overlays/plugins/restic.yaml `velero backup-location create` - base/backupstoragelocations.yaml `velero snapshot-location create` - base/volumasnapshotlocations.yaml `velero plugin add velero/velero-plugin-for-aws:v1.0.1` - overlays/plugins/aws-plugin.yaml `velero plugin add velero/velero-plugin-for-microsoft-azure:v1.0.1` - overlay/plugins/azure-plugin.yaml These resources can be deployed/deleted using the included kustomize setup and running: ``` kubectl apply -k design/CLI/PoC/overlays/plugins/ kubectl delete -k design/CLI/PoC/overlays/plugins/ ``` Note: All CRDs, including the `ResticRepository`, may continue to be deployed at startup as it is now, or together with their respective instantiation. #### Changes to startup behavior To recap, this proposal redesigns the Velero CLI to make `velero install` obsolete, and instead breaks down the installation and configuration into separate commands. These are the major highlights: - Plugins will only be installed separately via `velero plugin add` - BSL/VSL will be continue to be configured separately, and now each will have an associated secret Since each BSL/VSL will have its own association with a secret, the user will no longer need to upload a new secret whenever changing to, or adding, a BSL/VSL for a provider that is different from the one in use. This will be done at setup time. This will make it easier to support any number of BSL/VSL combinations, with different providers each. The user will start up the Velero server on a cluster by using the command `velero config server`. This will create the Velero deployment resource with default values or values overwritten with flags, create the Velero CRDs, and anything else that is not specific to plugins or BSL/VSL. The Velero server will start up, verify that the deployment is running, that all CRDs were found, and log a message that it is waiting for a BSL to be configured. at this point, other operations, such as configuring restic, will be allowed. Velero should keep track of its status, ie, if it is ready to create backups or not. This could be a field `ServerStatus` added to `ServerStatusRequest`. Possible values could be [ready|waiting]. "ready" would mean there is at least 1 valid BSL, and "waiting" would be anything but that. When adding/configuring a BSL or VSL, we will allow creating locations, and continuously verify if there is a corresponding, valid plugin. When a valid match is found, mark the BSL/VSL as "ready". This would require adding a field to the BSL/VSL, or using the existing `Phase` field, and keep track of its status, possibly: [ready|waiting]. With the first approach: the server would transition into "ready" (to create backups) as soon as there is one BSL. It would require a set sequence of actions, ie, first install the plugin, only then the user can successfully configure a BSL. With the second approach, the Velero server would continue looping and checking all existing BSLs for at least 1 with a "ready" status. Once it found that, it would set itself to "ready" also. Another new behavior that must be added: the server needs to identify when there no longer exists a valid BSL. At this point, it should change its status from "ready" to one that indicates it is not ready, maybe "waiting". With the first approach above, this would mean checking if there is still at least one BSL. With the second approach, it would require checking the status of all BSLs to find at least one with the status of "ready". As it is today, a valid VSL would not be required to create backups, unless the backup included a PV. To make it easier for the user to identify if their Velero server is ready to create backups or not, a `velero status` command should be added. This issue has been created some time ago for this purpose: https://github.com/vmware-tanzu/velero/issues/1094. ## Alternatives Considered It seems that the vast majority of tools document their usage with `kubectl` and `yaml` files to install and configure their Kubernetes resources. Many of them also make use of Helm, and to a lesser extent some of them have their own CLI tools. Amongst the tools that have their own CLI, not enough examples were found to establish a clear pattern of usage. It seems the most relevant priority should be to have output in `yaml` format. Any set of `yaml` files can also be arranged to use with Kustomize by creating/updating resources, and patching them using Kustomize functionalities. The way the Velero commands were arranged in this proposal with the ability to output corresponding `yaml` files, and the included Kustomize examples, makes it in line with the widely used practices for installation and configuration. Some CLI tools do not document their usage with Kustomize, one could assume it is because anyone with knowledge of Kustomize and `yaml` files would know how to use it. Here are some examples: https://github.com/jetstack/kustomize-cert-manager-demo https://github.com/istio/installer/tree/master/kustomize https://github.com/weaveworks/flagger/tree/master/kustomize https://github.com/jpeach/contour/tree/1c575c772e9fd747fba72ae41ab99bdae7a01864/kustomize (RFC) ## Security Considerations N/A ================================================ FILE: design/graph-manifest.md ================================================ # Object Graph Manifest for Velero ## Abstract One to two sentences that describes the goal of this proposal and the problem being solved by the proposed change. The reader should be able to tell by the title, and the opening paragraph, if this document is relevant to them. Currently, Velero does not have a complete manifest of everything in the backup, aside from the backup tarball itself. This change introduces a new data structure to be stored with a backup in object storage which will allow for more efficient operations in reporting of what a backup contains. Additionally, this manifest should enable advancements in Velero's features and architecture, enabling dry-run support, concurrent backup and restore operations, and reliable restoration of complex applications. ## Background Right now, Velero backs up items one at a time, sorted by API Group and namespace. It also restores items one at a time, using the restoreResourcePriorities flag to indicate which order API Groups should have their objects restored first. While this does work currently, it presents challenges for more complex applications that have their dependencies in the form of a graph rather than strictly linear. For example, Cluster API clusters are a set of complex Kubernetes objects that require that the "root" objects are restored first, before their "leaf" objects. If a Cluster that a ClusterResourceSetBinding refers to does not exist, then a restore of the CAPI cluster will fail. Additionally, Velero does not have a reliable way to communicate what objects will be affected in a backup or restore operation without actually performing the operation. This complicates dry-run tasks, because a user must simply perform the action without knowing what will be touched. It also complicates allowing backups and restores to run in parallel, because there is currently no way to know if a single Kubernetes object is included in multiple backups or restores, which can lead to unreliability, deadlocking, and race conditions were Velero made to be more concurrent today. ## Goals - Introduce a manifest data structure that defines the contents of a backup. - Store the manifest data into object storage alongside existing backup data. ## Non Goals This proposal seeks to enable, but not define, the following. - Implementing concurrency beyond what already exists in Velero. - Implementing a dry-run feature. - Implementing a new restore ordering procedure. While the data structure should take these scenarios into account, they will not be implemented alongside it. ## High-Level Design To uniquely identify a Kubernetes object within a cluster or backup, the following fields are sufficient: - API Group and Version (example: backup.velero.io/v1) - Namespace - Name - Labels This criteria covers the majority of Velero's inclusion or exclusion logic. However, some additional fields enable further use cases. - Owners, which are other Kubernetes objects that have some relationship to this object. They may be strict or soft dependencies. - Annotations, which provide extra metadata about the object that might be useful for other programs to consume. - UUID generated by Kubernetes. This is useful in defining Owner relationships, providing a single, immutable key to find an object. This is _not_ considered at restore time, only internally for defining links. All of this information already exists within a Velero backup's tarball of resources, but extracting such data is inefficient. The entire tarball must be downloaded and extracted, and then JSON within parsed to read labels, owners, annotations, and a UUID. The rest of the information is encoded in the file system structure within the Velero backup tarball. While doable, this is heavyweight in terms of time and potentially memory. Instead, this proposal suggests adding a new manifest structure that is kept alongside the backup tarball. This structure would contain the above fields only, and could be used to perform inclusion/exclusion logic on a backup, select a resource from within a backup, and do set operations over backup or restore contents to identify overlapping resources. Here are some use cases that this data structure should enable, that have been difficult to implement prior to its existence: - A dry-run operation on backup, informing the user what would be selected if they were to perform the operation. A manifest could be created and saved, allowing for a user to do a dry-run, then accept it to perform the backup. Restore operations can be treated similarly. - Efficient, non-overlapping parallelization of backup and restore operations. By building or reading a manifest before performing a backup or restore, Velero can determine if there are overlapping resources. If there are no overlaps, the operations can proceed in parallel. If there are overlaps, the operations can proveed serially. - Graph-based restores for non-linear dependencies. Not all resources in a Kubernetes cluster can be defined in a strict, linear way. They may have multiple owners, and writing BackupItemActions or RestoreItemActions to simply return a chain of owners is not an efficient way to support the many Kubernetes operators/controllers being written. Instead, by having a manifest with enough information, Velero can build a discrete list that ensures dependencies are restored before their dependents, with less input from plugin authors. ## Detailed Design The Manifest data structure would look like this, in Go type structure: ```golang // NamespacedItems maps a given namespace to all of its contained items. type NamespacedItems map[string]*Item // APIGroupNamespaces maps an API group/version to a map of namespaces and their items. type KindNamespaces map[string]NamespacedItems type Manifest struct { // Kinds holds the top level map of all resources in a manifest. Kinds KindNamespaces // Index is used to look up an individual item quickly based on UUID. // This enables fetching owners out of the maps more efficiently at the cost of memory space. Index map[string]*Item } // Item represents a Kubernetes resource within a backup based on it's selectable criteria. // It is not the whole Kubernetes resource as retrieved from the API server, but rather a collection of important fields needed for filtering. type Item struct { // Kubernetes API group which this Item belongs to. // Could be a core resource, or a CustomResourceDefinition. APIGroup string // Version of the APIGroup that the Item belongs to. APIVersion string // Kubernetes namespace which contains this item. // Empty string for cluster-level resource. Namespace string // Item's given name. Name string // Map of labels that the Item had at backup time. Labels map[string]string // Map of annotations that the Item had at Backup time. // Useful for plugins that may decide to process only Items with specific annotations. Annotations map[string]string // Owners is a list of UUIDs to other items that own or refer to this item. Owners []string // Manifest is a pointer to the Manifest in which this object is contained. // Useful for getting access to things like the Manifest.Index map. Manifest *Manifest } ``` In addition to the new types, the following Go interfaces would be provided for convenience. ```golang type Itermer interface { // Returns the Item as a string, following the current Velero backup version 1.1.0 tarball structure format. // ///.json String() string // Owners returns a slice of realized Items that own or refer to the current Item. // Useful for building out a full graph of Items to restore. // Will use the UUIDs in Item.Owners to look up the owner Items in the Manifest. Owners() []*Item // Kind returns the Kind of an object, which is a combination of the APIGroup and APIVersion. // Useful for verifying the needed CustomResourceDefinition exists before actually restoring this Item. Kind() *Item // Children returns a slice of all Items that refer to this item as an Owner. Children() []*Items } // This error type is being created in order to make reliable sentinel errors. // See https://dave.cheney.net/2019/06/10/constant-time for more details. type ManifestError string func (e ManifestError) Error() string { return string(e) } const ItemAlreadyExists = ManifestError("item already exists in manifest") type Manifester interface { // Set returns the entire list of resources as a set of strings (using Itemer.String). // This is useful for comparing two manifests and determining if they have any overlapping resources. // In the future, when implementing concurrent operations, this can be used as a sanity check to ensure resources aren't being backed up or restored by two operations at once. Set() sets.String // Adds an item to the appropriate APIGroup and Namespace within a Manifest // Returns (true, nil) if the Item is successfully added to the Manifest, // Returns (false, ItemAlreadyExists) if the Item is already in the Manifest. Add(*Item) (bool, error) } ``` ### Serialization The entire `Manifest` should be serialized into the `manifest.json` file within the object storage for a single backup. It is possible that this file could also be compressed for space efficiency. ### Memory Concerns Because the `Manifest` is holding a minimal amount of data, memory sizes should not be a concern for most clusters. TODO: Document known limits on API group name, resource name, and kind name character limits. ## Security Considerations Introducing this manifest does not increase the attack surface of Velero, as this data is already present in the existing backups. Storing the manifest.json file next to the existing backup data in the object storage does not change access patterns. ## Compatibility The introduction of this file should trigger Velero backup version 1.2.0, but it will not interfere with Velero versions that do not support the `Manifest` as the file will be additive. In time, this file will replace the `-resource-list.json.gz` file, but for compatibility the two will appear side by side. When first implemented, Velero should simply build the `Manifest` as it backs up items, and serialize it at the end. Any logic changes that rely on the `Manifest` file must be introduced with their own design document, with their own compatibility concerns. ## Implementation The `Manifest` object will _not_ be implemented as a Kubernetes CustomResourceDefinition, but rather one of Velero's own internal constructs. Implementation for the data structure alone should be minimal - the types will need to be defined in a `manifest` package. Then, the backup process should create a `Manifest`, passing it to the various `*Backuppers` in the `backup` package. These methods will insert individual `Items` into the `Manifest`. Finally, logic should be added to the `persistence` package to ensure that the new `manifest.json` file is uploadable and allowed. ## Alternatives Considered None so far. ## Open Issues - When should compatibility with the `-resource-list.json.gz` file be dropped? - What are some good test case Kubernetes resources and controllers to try this out with? Cluster API seems like an obvious choice, but are there others? - Since it is not implemented as a CustomResourceDefinition, how can a `Manifest` be retained so that users could issue a dry-run command, then perform their actual desire operation? Could it be stored in Velero's temp directories? Note that this is making Velero itself more stateful. ================================================ FILE: design/new-prepost-backuprestore-plugin-hooks.md ================================================ # Pre-Backup, Post-Backup, Pre-Restore, and Post-Restore Action Plugin Hooks ## Abstract Velero should provide a way to trigger actions before and after each backup and restore. **Important**: These proposed plugin hooks are fundamentally different from the existing plugin hooks, BackupItemAction and RestoreItemAction, which are triggered per resource item during backup and restore, respectively. The proposed plugin hooks are to be executed only once: pre-backup (before backup starts), post-backup (after the backup is completed and uploaded to object storage, including volumes snapshots), pre-restore (before restore starts) and post-restore (after the restore is completed, including volumes are restored). ### PreBackup and PostBackup Actions For the backup, the sequence of events of Velero backup are the following (these sequence depicted is prior upcoming changes for [upload progress #3533](https://github.com/vmware-tanzu/velero/issues/3533) ): ``` New Backup Request |--> Validation of the request |--> Set Backup Phase "In Progress" | --> Start Backup | --> Discover all Plugins |--> Check if Backup Exists |--> Backup all K8s Resource Items |--> Perform all Volumes Snapshots |--> Final Backup Phase is determined |--> Persist Backup and Logs on Object Storage ``` We propose the pre-backup and post-backup plugin hooks to be executed in this sequence: ``` New Backup Request |--> Validation of the request |--> Set Backup Phase "In Progress" | --> Start Backup | --> Discover all Plugins |--> Check if Backup Exists |--> **PreBackupActions** are executed, logging actions on existent backup log file |--> Backup all K8s Resource Items |--> Perform all Volumes Snapshots |--> Final Backup Phase is determined |--> Persist Backup and logs on Object Storage |--> **PostBackupActions** are executed, logging to its own file ``` These plugin hooks will be invoked: - PreBackupAction: plugin actions are executed after the backup object is created and validated but before the backup is being processed, more precisely _before_ function [c.backupper.Backup](https://github.com/vmware-tanzu/velero/blob/74476db9d791fa91bba0147eac8ec189820adb3d/pkg/controller/backup_controller.go#L590). If the PreBackupActions return an err, the backup object is not processed and the Backup phase will be set as `FailedPreBackupActions`. - PostBackupAction: plugin actions are executed after the backup is finished and persisted, more precisely _after_ function [c.runBackup](https://github.com/vmware-tanzu/velero/blob/74476db9d791fa91bba0147eac8ec189820adb3d/pkg/controller/backup_controller.go#L274). The proposed plugin hooks will execute actions that will have statuses on their own: `Backup.Status.PreBackupActionsStatuses` and `Backup.Status.PostBackupActionsStatuses` which will be an array of a proposed struct `ActionStatus` with PluginName, StartTimestamp, CompletionTimestamp and Phase. ### PreRestore and PostRestore Actions For the restore, the sequence of events of Velero restore are the following (these sequence depicted is prior upcoming changes for [upload progress #3533](https://github.com/vmware-tanzu/velero/issues/3533) ): ``` New Restore Request |--> Validation of the request |--> Checks if restore is from a backup or a schedule |--> Fetches backup |--> Set Restore Phase "In Progress" |--> Start Restore |--> Discover all Plugins |--> Download backup file to temp |--> Fetch list of volumes snapshots |--> Restore K8s items, including PVs |--> Final Restore Phase is determined |--> Persist Restore logs on Object Storage ``` We propose the pre-restore and post-restore plugin hooks to be executed in this sequence: ``` New Restore Request |--> Validation of the request |--> Checks if restore is from a backup or a schedule |--> Fetches backup |--> Set Restore Phase "In Progress" |--> Start Restore |--> Discover all Plugins |--> Download backup file to temp |--> Fetch list of volumes snapshots |--> **PreRestoreActions** are executed, logging actions on existent backup log file |--> Restore K8s items, including PVs |--> Final Restore Phase is determined |--> Persist Restore logs on Object Storage |--> **PostRestoreActions** are executed, logging to its own file ``` These plugin hooks will be invoked: - PreRestoreAction: plugin actions are executed after the restore object is created and validated and before the backup object is fetched, more precisely in function `runValidatedRestore` _after_ function [info.backupStore.GetBackupVolumeSnapshots](https://github.com/vmware-tanzu/velero/blob/7c75cd6cf854064c9a454e53ba22cc5881d3f1f0/pkg/controller/restore_controller.go#L460). If the PreRestoreActions return an err, the restore object is not processed and the Restore phase will be set a `FailedPreRestoreActions`. - PostRestoreAction: plugin actions are executed after the restore finishes processing all items and volumes snapshots are restored and logs persisted, more precisely in function `processRestore` _after_ setting [`restore.Status.CompletionTimestamp`](https://github.com/vmware-tanzu/velero/blob/7c75cd6cf854064c9a454e53ba22cc5881d3f1f0/pkg/controller/restore_controller.go#L273). The proposed plugin hooks will execute actions that will have statuses on their own: `Restore.Status.PreRestoreActionsStatuses` and `Restore.Status.PostRestoreActionsStatuses` which will be an array of a proposed struct `ActionStatus` with PluginName, StartTimestamp, CompletionTimestamp and Phase. ## Background Increasingly, Velero is employed for workload migrations across different Kubernetes clusters. Using Velero for migrations requires an atomic operation involving a Velero backup on a source cluster followed by a Velero restore on a destination cluster. It is common during these migrations to perform many actions inside and outside Kubernetes clusters. **Attention**: these actions are not per resource item, but they are actions to be executed _once_ before and/or after the migration itself (remember, migration in this context is Velero Backup + Velero Restore). One important use case driving this proposal is migrating stateful workloads at scale across different clusters/storage backends. Today, Velero's Restic integration is the response for such use cases, but there are some limitations: - Quiesce/unquiesce workloads: Pod hooks are useful for quiescing/unquiescing workloads, but platform engineers often do not have the luxury/visibility/time/knowledge to go through each pod in order to add specific commands to quiesce/unquiesce workloads. - Orphan PVC/PV pairs: PVCs/PVs that do not have associated running pods are not backed up and consequently, are not migrated. Aiming to address these two limitations, and separate from this proposal, we would like to write a Velero plugin that takes advantage of the proposed Pre-Backup plugin hook. This plugin will be executed _once_ (not per resource item) prior backup. It will scale down the applications setting `.spec.replicas=0` to all deployments, statefulsets, daemonsets, replicasets, etc. and will start a small-footprint staging pod that will mount all PVC/PV pairs. Similarly, we would like to write another plugin that will utilize the proposed Post-Restore plugin hook. This plugin will unquiesce migrated applications by killing the staging pod and reinstating original `.spec.replicas` values after the Velero restore is completed. Other examples of plugins that can use the proposed plugin hooks are: - PostBackupAction: trigger a Velero Restore after a successful Velero backup (and complete the migration operation). - PreRestoreAction: pre-expand the cluster's capacity via Cluster API to avoid starvation of cluster resources before the restore. - PostRestoreAction: call actions to be performed outside Kubernetes clusters, such as configure a global load balancer (GLB) that enables the new cluster. The post backup actions will be executed after the backup is uploaded (persisted) on the disk. The logs of post-backup actions will be uploaded on the disk once the actions are completed. The post restore actions will be executed after the restore is uploaded (persisted) on the disk. The logs of post-restore actions will be uploaded on the disk once the actions are completed. This design seeks to provide missing extension points. This proposal's scope is to only add the new plugin hooks, not the plugins themselves. ## Goals - Provide PreBackupAction, PostBackupAction, PreRestoreAction, and PostRestoreAction APIs for plugins to implement. - Update Velero backup and restore creation logic to invoke registered PreBackupAction and PreRestoreAction plugins before processing the backup and restore respectively. - Update Velero backup and restore complete logic to invoke registered PostBackupAction and PostRestoreAction plugins the objects are uploaded on disk. - Create one `ActionStatus` struct to keep track of execution of the plugin hooks. This struct has PluginName, StartTimestamp, CompletionTimestamp and Phase. - Add sub statuses for the plugins on Backup object: `Backup.Status.PreBackupActionsStatuses` and `Backup.Status.PostBackupActionsStatuses`. They will be flagged as optional and nullable. They will be populated only each plugin registered for the PreBackup and PostBackup hooks, respectively. - Add sub statuses for the plugins on Restore object: `Backup.Status.PreRestoreActionsStatuses` and `Backup.Status.PostRestoreActionsStatuses`. They will be flagged as optional and nullable. They will be populated only each plugin registered for the PreRestore and PostRestore hooks, respectively. - that will be populated optionally if Pre/Post Backup/Restore. ## Non-Goals - Specific implementations of the PreBackupAction, PostBackupAction, PreRestoreAction and PostRestoreAction API beyond test cases. - For migration specific actions (Velero Backup + Velero Restore), add disk synchronization during the validation of the Restore (making sure the newly created backup will show during restore) ## High-Level Design The Velero backup controller package will be modified for `PreBackupAction` and `PostBackupAction`. The PreBackupAction plugin API will resemble the BackupItemAction plugin hook design, but with the fundamental difference that it will receive only as input the Velero `Backup` object created. It will not receive any resource list items because the backup is not yet running at that stage. In addition, the `PreBackupAction` interface will only have an `Execute()` method since the plugin will be executed once per Backup creation, not per item. The Velero backup controller will be modified so that if there are any PreBackupAction plugins registered, they will be The PostBackupAction plugin API will resemble the BackupItemAction plugin design, but with the fundamental difference that it will receive only as input the Velero `Backup` object without any resource list items. By this stage, the backup has already been executed, with items backed up and volumes snapshots processed and persisted. The `PostBackupAction` interface will only have an `Execute()` method since the plugin will be executed only once per Backup, not per item. If there are any PostBackupAction plugins registered, they will be executed after the backup is finished and persisted, more precisely _after_ function [c.runBackup](https://github.com/vmware-tanzu/velero/blob/74476db9d791fa91bba0147eac8ec189820adb3d/pkg/controller/backup_controller.go#L274). The Velero restore controller package will be modified for `PreRestoreAction` and `PostRestoreAction`. The PreRestoreAction plugin API will resemble the RestoreItemAction plugin design, but with the fundamental difference that it will receive only as input the Velero `Restore` object created. It will not receive any resource list items because the restore has not yet been running at that stage. In addition, the `PreRestoreAction` interface will only have an `Execute()` method since the plugin will be executed only once per Restore creation, not per item. The Velero restore controller will be modified so that if there are any PreRestoreAction plugins registered, they will be executed after the restore object is created and validated and before the backup object is fetched, more precisely in function `runValidatedRestore` _after_ function [info.backupStore.GetBackupVolumeSnapshots](https://github.com/vmware-tanzu/velero/blob/7c75cd6cf854064c9a454e53ba22cc5881d3f1f0/pkg/controller/restore_controller.go#L460). If the PreRestoreActions return an err, the restore object is not processed and the Restore phase will be set a `FailedPreRestoreActions`. The PostRestoreAction plugin API will resemble the RestoreItemAction plugin design, but with the fundamental difference that it will receive only as input the Velero `Restore` object without any resource list items. At this stage, the restore has already been executed. The `PostRestoreAction` interface will only have an `Execute()` method since the plugin will be executed only once per Restore, not per item. If any PostRestoreAction plugins are registered, they will be executed after the restore finishes processing all items and volumes snapshots are restored and logs persisted, more precisely in function `processRestore` _after_ setting [`restore.Status.CompletionTimestamp`](https://github.com/vmware-tanzu/velero/blob/7c75cd6cf854064c9a454e53ba22cc5881d3f1f0/pkg/controller/restore_controller.go#L273). ## Detailed Design ### New Status struct To keep the status of the plugins, we propose the following struct: ```go type ActionStatus struct { // PluginName is the name of the registered plugin // retrieved by the PluginManager as id.Name // +optional // +nullable PluginName string `json:"pluginName,omitempty"` // StartTimestamp records the time the plugin started. // +optional // +nullable StartTimestamp *metav1.Time `json:"startTimestamp,omitempty"` // CompletionTimestamp records the time the plugin was completed. // +optional // +nullable CompletionTimestamp *metav1.Time `json:"completionTimestamp,omitempty"` // Phase is the current state of the Action. // +optional // +nullable Phase ActionPhase `json:"phase,omitempty"` } // ActionPhase is a string representation of the lifecycle phase of an action being executed by a plugin // of a Velero backup. // +kubebuilder:validation:Enum=InProgress;Completed;Failed type ActionPhase string const ( // ActionPhaseInProgress means the action has being executed ActionPhaseInProgress ActionPhase = "InProgress" // ActionPhaseCompleted means the action finished successfully ActionPhaseCompleted ActionPhase = "Completed" // ActionPhaseFailed means the action failed ActionPhaseFailed ActionPhase = "Failed" ) ``` ### Backup Status of the Plugins The `Backup` Status section will have the follow: ```go type BackupStatus struct { (...) // PreBackupActionsStatuses contains information about the pre backup plugins's execution. // Note that this information is will be only populated if there are prebackup plugins actions // registered // +optional // +nullable PreBackupActionsStatuses *[]ActionStatus `json:"preBackupActionsStatuses,omitempty"` // PostBackupActionsStatuses contains information about the post backup plugins's execution. // Note that this information is will be only populated if there are postbackup plugins actions // registered // +optional // +nullable PostBackupActionsStatuses *[]ActionStatus `json:"postBackupActionsStatuses,omitempty"` } ``` ### Restore Status of the Plugins The `Restore` Status section will have the follow: ```go type RestoreStatus struct { (...) // PreRestoreActionsStatuses contains information about the pre Restore plugins's execution. // Note that this information is will be only populated if there are preRestore plugins actions // registered // +optional // +nullable PreRestoreActionsStatuses *[]ActionStatus `json:"preRestoreActionsStatuses,omitempty"` // PostRestoreActionsStatuses contains information about the post restore plugins's execution. // Note that this information is will be only populated if there are postrestore plugins actions // registered // +optional // +nullable PostRestoreActionsStatuses *[]ActionStatus `json:"postRestoreActionsStatuses,omitempty"` } ``` ### New Backup and Restore Phases #### New Backup Phase: FailedPreBackupActions In case the PreBackupActionsStatuses has at least one `ActionPhase` = `Failed`, it means al least one of the plugins returned an error and consequently, the backup will not move forward. The final status of the Backup object will be set as `FailedPreBackupActions`: ```go // BackupPhase is a string representation of the lifecycle phase // of a Velero backup. // +kubebuilder:validation:Enum=New;FailedValidation;FailedPreBackupActions;InProgress;Uploading;UploadingPartialFailure;Completed;PartiallyFailed;Failed;Deleting type BackupPhase string const ( (...) // BackupPhaseFailedPreBackupActions means one or more the Pre Backup Actions has failed // and therefore backup will not run. BackupPhaseFailedPreBackupActions BackupPhase = "FailedPreBackupActions" (...) ) ``` #### New Restore Phase FailedPreRestoreActions In case the PreRestoreActionsStatuses has at least one `ActionPhase` = `Failed`, it means al least one of the plugins returned an error and consequently, the restore will not move forward. The final status of the Restore object will be set as `FailedPreRestoreActions`: ```go // RestorePhase is a string representation of the lifecycle phase // of a Velero restore // +kubebuilder:validation:Enum=New;FailedValidation;FailedPreRestoreActions;InProgress;Completed;PartiallyFailed;Failed type RestorePhase string const ( (...) // RestorePhaseFailedPreRestoreActions means one or more the Pre Restore Actions has failed // and therefore restore will not run. RestorePhaseFailedPreRestoreActions BackupPhase = "FailedPreRestoreActions" (...) ) ``` ### New Interface types #### PreBackupAction The `PreBackupAction` interface is as follows: ```go // PreBackupAction provides a hook into the backup process before it begins. type PreBackupAction interface { // Execute the PreBackupAction plugin providing it access to the Backup that // is being executed Execute(backup *api.Backup) error } ``` `PreBackupAction` will be defined in `pkg/plugin/velero/pre_backup_action.go`. #### PostBackupAction The `PostBackupAction` interface is as follows: ```go // PostBackupAction provides a hook into the backup process after it completes. type PostBackupAction interface { // Execute the PostBackupAction plugin providing it access to the Backup that // has been completed Execute(backup *api.Backup) error } ``` `PostBackupAction` will be defined in `pkg/plugin/velero/post_backup_action.go`. #### PreRestoreAction The `PreRestoreAction` interface is as follows: ```go // PreRestoreAction provides a hook into the restore process before it begins. type PreRestoreAction interface { // Execute the PreRestoreAction plugin providing it access to the Restore that // is being executed Execute(restore *api.Restore) error } ``` `PreRestoreAction` will be defined in `pkg/plugin/velero/pre_restore_action.go`. #### PostRestoreAction The `PostRestoreAction` interface is as follows: ```go // PostRestoreAction provides a hook into the restore process after it completes. type PostRestoreAction interface { // Execute the PostRestoreAction plugin providing it access to the Restore that // has been completed Execute(restore *api.Restore) error } ``` `PostRestoreAction` will be defined in `pkg/plugin/velero/post_restore_action.go`. ### New BackupStore Interface Methods For the persistence of the logs originated from the PostBackup and PostRestore plugins, create two additional methods on `BackupStore` interface: ```go type BackupStore interface { (...) PutPostBackuplog(backup string, log io.Reader) error PutPostRestoreLog(backup, restore string, log io.Reader) error (...) ``` The implementation of these new two methods will go hand-in-hand with the changes of uploading phases rebase. ### Generate Protobuf Definitions and Client/Servers In `pkg/plugin/proto`, add the following: 1. Protobuf definitions will be necessary for PreBackupAction in `pkg/plugin/proto/PreBackupAction.proto`. ```protobuf message PreBackupActionExecuteRequest { ... } service PreBackupAction { rpc Execute(PreBackupActionExecuteRequest) returns (Empty) } ``` Once these are written, then a client and server implementation can be written in `pkg/plugin/framework/pre_backup_action_client.go` and `pkg/plugin/framework/pre_backup_action_server.go`, respectively. 2. Protobuf definitions will be necessary for PostBackupAction in `pkg/plugin/proto/PostBackupAction.proto`. ```protobuf message PostBackupActionExecuteRequest { ... } service PostBackupAction { rpc Execute(PostBackupActionExecuteRequest) returns (Empty) } ``` Once these are written, then a client and server implementation can be written in `pkg/plugin/framework/post_backup_action_client.go` and `pkg/plugin/framework/post_backup_action_server.go`, respectively. 3. Protobuf definitions will be necessary for PreRestoreAction in `pkg/plugin/proto/PreRestoreAction.proto`. ```protobuf message PreRestoreActionExecuteRequest { ... } service PreRestoreAction { rpc Execute(PreRestoreActionExecuteRequest) returns (Empty) } ``` Once these are written, then a client and server implementation can be written in `pkg/plugin/framework/pre_restore_action_client.go` and `pkg/plugin/framework/pre_restore_action_server.go`, respectively. 4. Protobuf definitions will be necessary for PostRestoreAction in `pkg/plugin/proto/PostRestoreAction.proto`. ```protobuf message PostRestoreActionExecuteRequest { ... } service PostRestoreAction { rpc Execute(PostRestoreActionExecuteRequest) returns (Empty) } ``` Once these are written, then a client and server implementation can be written in `pkg/plugin/framework/post_restore_action_client.go` and `pkg/plugin/framework/post_restore_action_server.go`, respectively. ### Restartable Delete Plugins Similar to the `RestoreItemAction` and `BackupItemAction` plugins, restartable processes will need to be implemented (with the difference that there is no `AppliedTo()` method). In `pkg/plugin/clientmgmt/`, add 1. `restartable_pre_backup_action.go`, creating the following unexported type: ```go type restartablePreBackupAction struct { key kindAndName sharedPluginProcess RestartableProcess } func newRestartablePreBackupAction(name string, sharedPluginProcess RestartableProcess) *restartablePreBackupAction { // ... } func (r *restartablePreBackupAction) getPreBackupAction() (velero.PreBackupAction, error) { // ... } func (r *restartablePreBackupAction) getDelegate() (velero.PreBackupAction, error) { // ... } // Execute restarts the plugin's process if needed, then delegates the call. func (r *restartablePreBackupAction) Execute(input *velero.PreBackupActionInput) (error) { // ... } ``` 2. `restartable_post_backup_action.go`, creating the following unexported type: ```go type restartablePostBackupAction struct { key kindAndName sharedPluginProcess RestartableProcess } func newRestartablePostBackupAction(name string, sharedPluginProcess RestartableProcess) *restartablePostBackupAction { // ... } func (r *restartablePostBackupAction) getPostBackupAction() (velero.PostBackupAction, error) { // ... } func (r *restartablePostBackupAction) getDelegate() (velero.PostBackupAction, error) { // ... } // Execute restarts the plugin's process if needed, then delegates the call. func (r *restartablePostBackupAction) Execute(input *velero.PostBackupActionInput) (error) { // ... } ``` 3. `restartable_pre_restore_action.go`, creating the following unexported type: ```go type restartablePreRestoreAction struct { key kindAndName sharedPluginProcess RestartableProcess } func newRestartablePreRestoreAction(name string, sharedPluginProcess RestartableProcess) *restartablePreRestoreAction { // ... } func (r *restartablePreRestoreAction) getPreRestoreAction() (velero.PreRestoreAction, error) { // ... } func (r *restartablePreRestoreAction) getDelegate() (velero.PreRestoreAction, error) { // ... } // Execute restarts the plugin's process if needed, then delegates the call. func (r *restartablePreRestoreAction) Execute(input *velero.PreRestoreActionInput) (error) { // ... } ``` 4. `restartable_post_restore_action.go`, creating the following unexported type: ```go type restartablePostRestoreAction struct { key kindAndName sharedPluginProcess RestartableProcess } func newRestartablePostRestoreAction(name string, sharedPluginProcess RestartableProcess) *restartablePostRestoreAction { // ... } func (r *restartablePostRestoreAction) getPostRestoreAction() (velero.PostRestoreAction, error) { // ... } func (r *restartablePostRestoreAction) getDelegate() (velero.PostRestoreAction, error) { // ... } // Execute restarts the plugin's process if needed, then delegates the call. func (r *restartablePostRestoreAction) Execute(input *velero.PostRestoreActionInput) (error) { // ... } ``` ### Plugin Manager Changes Add the following methods to the `Manager` interface in `pkg/plugin/clientmgmt/manager.go`: ```go type Manager interface { ... // Get PreBackupAction returns a PreBackupAction plugin for name. GetPreBackupAction(name string) (PreBackupAction, error) // Get PreBackupActions returns the all PreBackupAction plugins. GetPreBackupActions() ([]PreBackupAction, error) // Get PostBackupAction returns a PostBackupAction plugin for name. GetPostBackupAction(name string) (PostBackupAction, error) // GetPostBackupActions returns the all PostBackupAction plugins. GetPostBackupActions() ([]PostBackupAction, error) // Get PreRestoreAction returns a PreRestoreAction plugin for name. GetPreRestoreAction(name string) (PreRestoreAction, error) // Get PreRestoreActions returns the all PreRestoreAction plugins. GetPreRestoreActions() ([]PreRestoreAction, error) // Get PostRestoreAction returns a PostRestoreAction plugin for name. GetPostRestoreAction(name string) (PostRestoreAction, error) // GetPostRestoreActions returns the all PostRestoreAction plugins. GetPostRestoreActions() ([]PostRestoreAction, error) } ``` `GetPreBackupAction` and `GetPreBackupActions` will invoke the `restartablePreBackupAction` implementations. `GetPostBackupAction` and `GetPostBackupActions` will invoke the `restartablePostBackupAction` implementations. `GetPreRestoreAction` and `GetPreRestoreActions` will invoke the `restartablePreRestoreAction` implementations. `GetPostRestoreAction` and `GetPostRestoreActions` will invoke the `restartablePostRestoreAction` implementations. ### How to invoke the Plugins #### Getting Pre/Post Backup Actions Getting Actions on `backup_controller.go` in `runBackup`: ```go backupLog.Info("Getting PreBackup actions") preBackupActions, err := pluginManager.GetPreBackupActions() if err != nil { return err } backupLog.Info("Getting PostBackup actions") postBackupActions, err := pluginManager.GetPostBackupActions() if err != nil { return err } ``` #### Pre Backup Actions Plugins Calling the Pre Backup actions: ```go for _, preBackupAction := range preBackupActions { err := preBackupAction.Execute(backup.Backup) if err != nil { backup.Backup.Status.Phase = velerov1api.BackupPhaseFailedPreBackupActions return err } } ``` #### Post Backup Actions Plugins Calling the Post Backup actions: ```go for _, postBackupAction := range postBackupActions { err := postBackupAction.Execute(backup.Backup) if err != nil { postBackupLog.Error(err) } } ``` #### Getting Pre/Post Restore Actions Getting Actions on `restore_controller.go` in `runValidatedRestore`: ```go restoreLog.Info("Getting PreRestore actions") preRestoreActions, err := pluginManager.GetPreRestoreActions() if err != nil { return errors.Wrap(err, "error getting pre-restore actions") } restoreLog.Info("Getting PostRestore actions") postRestoreActions, err := pluginManager.GetPostRestoreActions() if err != nil { return errors.Wrap(err, "error getting post-restore actions") } ``` #### Pre Restore Actions Plugins Calling the Pre Restore actions: ```go for _, preRestoreAction := range preRestoreActions { err := preRestoreAction.Execute(restoreReq.Restore) if err != nil { restoreReq.Restore.Status.Phase = velerov1api.RestorePhaseFailedPreRestoreActions return errors.Wrap(err, "error executing pre-restore action") } } ``` #### Post Restore Actions Plugins Calling the Post Restore actions: ```go for _, postRestoreAction := range postRestoreActions { err := postRestoreAction.Execute(restoreReq.Restore) if err != nil { postRestoreLog.Error(err.Error()) } } ``` ### Giving the User the Option to Skip the Execution of the Plugins Velero plugins are loaded as init containers. If plugins are unloaded, they trigger a restart of the Velero controller. Not mentioning if one plugin does get loaded for any reason (i.e., docker hub image pace limit), Velero does not start. In other words, the constant load/unload of plugins can disrupt the Velero controller, and they cannot be the only method to run the actions from these plugins selectively. As part of this proposal, we want to give the velero user the ability to skip the execution of the plugins via annotations on the Velero CR backup and restore objects. If one of these exists, the given plugin, referenced below as `plugin-name`, will be skipped. Backup Object Annotations: ``` /prebackup=skip /postbackup=skip ``` Restore Object Annotations: ``` /prerestore=skip /postrestore=skip ``` ## Alternatives Considered An alternative to these plugin hooks is to implement all the pre/post backup/restore logic _outside_ Velero. In this case, one would need to write an external controller that works similar to what [Konveyor Crane](https://github.com/konveyor/mig-controller/blob/master/pkg/controller/migmigration/quiesce.go) does today when quiescing applications. We find this a viable way, but we think that Velero users can benefit from Velero having greater embedded capabilities, which will allow users to write or load plugins extensions without relying on an external components. ## Security Considerations The plugins will only be invoked if loaded per a user's discretion. It is recommended to check security vulnerabilities before execution. ## Compatibility In terms of backward compatibility, this design should stay compatible with most Velero installations that are upgrading. If plugins are not present, then the backup/restore process should proceed the same way it worked before their inclusion. ## Implementation The implementation dependencies are roughly in the order as they are described in the [Detailed Design](#detailed-design) section. ## Open Issues ================================================ FILE: design/restore-progress.md ================================================ # Restore progress reporting Velero _Backup_ resource provides real-time progress of an ongoing backup by means of a _Progress_ field in the CR. Velero _Restore_, on the other hand, only shows one of the phases (InProgress, Completed, PartiallyFailed, Failed) of the ongoing restore. In this document, we propose detailed progress reporting for Velero _Restore_. With the introduction of the proposed _Progress_ field, Velero _Restore_ CR will look like: ```yml apiVersion: velero.io/v1 kind: Restore metadata: name: test-restore namespace: velero spec: [...] status: phase: InProgress progress: itemsRestored: 100 totalItems: 140 ``` ## Goals - Enable progress reporting for Velero Restore ## Non Goals - Estimate time to completion ## Background The current _Restore_ CR lets users know whether a restore is in-progress or completed (failed/succeeded). While this basic piece of information is useful to the end user, there seems to be room for improvement in the user experience. The _Restore_ CR can show detailed progress in terms of the number of resources restored so far and the total number of resources to be restored. This will be particularly useful for restores that run for a longer duration of time. Such progress reporting already exists for Velero _Backup_. This document proposes similar implementation for Velero _Restore_. ## High-Level Design We propose to divide the restore process in two steps. The first step will collect all the items to be restored from the backup tarball. It will apply the label selector and include/exclude rules on the resources / items and store them (preserving the priority order) in an in-memory data structure. The second step will read the collected items and restore them. ## Detailed Design ### Progress struct A new struct will be introduced to store progress information: ```go type RestoreProgress struct { TotalItems int `json:"totalItems,omitempty` ItemsRestored int `json:"itemsRestored,omitempty` } ``` `RestoreStatus` will include the above struct: ```go type RestoreStatus struct { [...] Progress *RestoreProgress `json:"progress,omitempty"` } ``` ### Modifications to restore.go Currently, the restore process works by looping through the resources in the backup tarball and restoring them one-by-one in the same pass: ```go func (ctx *context) execute(...) { [...] for _, resource := range getOrderedResources(...) { [...] for namespace, items := range resourceList.ItemsByNamespace { [...] for _, item := range items { [...] // restore item here w, e := restoreItem(...) } } } } ``` We propose to remove the call to `restoreItem()` in the inner most loop and instead store the item in a data structure. Once all the items are collected, we loop through the array of collected items and make a call to `restoreItem()`: ```go func (ctx *context) getOrderedResourceCollection(...) { collectedResources := []restoreResource for _, resource := range getOrderedResources(...) { [...] for namespace, items := range resourceList.ItemsByNamespace { [...] collectedResource := restoreResource{} for _, item := range items { [...] // store item in a data structure collectedResource.itemsByNamespace[originalNamespace] = append(collectedResource.itemsByNamespace[originalNamespace], item) } } collectedResources.append(collectedResources, collectedResource) } return collectedResources } func (ctx *context) execute(...) { [...] // get all items resources := ctx.getOrderedResourceCollection(...) for _, resource := range resources { [...] for _, items := range resource.itemsByNamespace { [...] for _, item := range items { [...] // restore the item w, e := restoreItem(...) } } } [...] } ``` We introduce two new structs to hold the collected items: ```go type restoreResource struct { resource string itemsByNamespace map[string][]restoreItem totalItems int } type restoreItem struct { targetNamespace string name string } ``` Each group resource is represented by `restoreResource`. The map `itemsByNamespace` is indexed by `originalNamespace`, and the values are list of `items` in the original namespace. `totalItems` is simply the count of all items which are present in the nested map of namespace and items. It is updated every time an item is added to the map. Each item represented by `restoreItem` has `name` and the resolved `targetNamespace`. ### Calculating progress The total number of items can be calculated by simply adding the number of total items present in the map of all resources. ```go totalItems := 0 for _, resource := range collectedResources { totalItems += resource.totalItems } ``` The additional items returned by the plugins will still be discovered at the time of plugin execution. The number of `totalItems` will be adjusted to include such additional items. As a result, the number of total items is expected to change whenever plugins execute: ```go i := 0 for _, resource := range resources { [...] for _, items := range resource.itemsByNamespace { [...] for _, item := range items { [...] // restore the item w, e := restoreItem(...) i++ // calculate the actual count of resources actualTotalItems := len(ctx.restoredItems) + (totalItems - i) } } } ``` ### Updating progress The updates to the `progress` field in the CR can be sent on a channel as soon as an item is restored. A goroutine receiving update on that channel can make an `Update()` call to update the _Restore_ CR. This will require us to pass an instance of `RestoresGetter` to the `kubernetesRestorer` struct. ## Alternatives Considered As an alternative, we have considered an approach which doesn't divide the restore process in two steps. With that approach, the total number of items will be read from the Backup CR. We will keep three counters, `totalItems`, `skippedItems` and `restoredItems`: ```yml status: phase: InProgress progress: totalItems: 100 skippedItems: 20 restoredItems: 79 ``` This approach doesn't require us to find the number of total items beforehand. ## Security Considerations Omitted ## Compatibility Omitted ## Implementation TBD ## Open Issues https://github.com/vmware-tanzu/velero/issues/21 ================================================ FILE: design/upload-progress.md ================================================ # Upload Progress Monitoring Volume snapshotter plugin are used by Velero to take snapshots of persistent volume contents. Depending on the underlying storage system, those snapshots may be available to use immediately, they may be uploaded to stable storage internally by the plugin or they may need to be uploaded after the snapshot has been taken. We would like for Velero to continue on to the next part of the backup as quickly as possible but we would also like the backup to not be marked as complete until it is a usable backup. We'd also eventually like to bring the control of upload under the control of Velero and allow the user to make decisions about the ultimate destination of backup data independent of the storage system they're using. ## Examples AWS - AWS snapshots return quickly, but are then uploaded in the background and cannot be used until EBS moves the data into S3 internally. vSphere - The vSphere plugin takes a local snapshot and then the vSphere plugin uploads the data to S3. The local snapshot is usable before the upload completes. Restic - Does not go through the volume snapshot path. Restic backups will block Velero progress until completed. ## Goals - Enable monitoring of operations that continue after snapshotting operations have completed - Keep non-usable backups (upload/persistence has not finished) from appearing as completed - Minimize change to volume snapshot and BackupItemAction plugins ## Non-goals - Unification of BackupItemActions and VolumeSnapshotters ## Models ### Internal configuration and management In this model, movement of the snapshot to stable storage is under the control of the snapshot plugin. Decisions about where and when the snapshot gets moved to stable storage are not directly controlled by Velero. This is the model for the current VolumeSnapshot plugins. ### Velero controlled management In this model, the snapshot is moved to external storage under the control of Velero. This enables Velero to move data between storage systems. This also allows backup partners to use Velero to snapshot data and then move the data into their backup repository. ## Backup phases Velero currently has backup phases "InProgress" and "Completed". The backup moves to the Completed phase when all of the volume snapshots have completed and the Kubernetes metadata has been written into the object store. However, the actual data movement may be happening in the background after the backup has been marked "Completed". The backup is not actually a stable backup until the data has been persisted properly. In some cases (e.g. AWS) the backup cannot be restored from until the snapshots have been persisted. Once the snapshots have been taken, however, it is possible for additional backups to be made without interference. Waiting until all data has been moved before starting the next backup will slow the progress of the system without adding any actual benefit to the user. A new backup phase, "Uploading" will be introduced. When a backup has entered this phase, Velero is free to start another backup. The backup will remain in the "Uploading" phase until all data has been successfully moved to persistent storage. The backup will not fail once it reaches this phase, it will continuously retry moving the data. If the backup is deleted (cancelled), the plugins will attempt to delete the snapshots and stop the data movement - this may not be possible with all storage systems. ### State progression ![image](UploadFSM.png) ### New When a backup request is initially created, it is in the "New" phase. The next state is either "InProgress" or "FailedValidation" ### FailedValidation If the backup request is incorrectly formed, it goes to the "FailedValidation" phase and terminates ### InProgress When work on the backup begins, it moves to the "InProgress" phase. It remains in the "InProgress" phase until all pre/post execution hooks have been executed, all snapshots have been taken and the Kubernetes metadata and backup info is safely written to the object store plugin. In the current implementation, Restic backups will move data during the "InProgress" phase. In the future, it may be possible to combine a snapshot with a Restic (or equivalent) backup which would allow for data movement to be handled in the "Uploading" phase, The next phase is either "Completed", "Uploading", "Failed" or "PartiallyFailed". Backups which would have a final phase of "Completed" or "PartiallyFailed" may move to the "Uploading" state. A backup which will be marked "Failed" will go directly to the "Failed" phase. Uploads may continue in the background for snapshots that were taken by a "Failed" backup, but no progress will not be monitored or updated. When a "Failed" backup is deleted, all snapshots will be deleted and at that point any uploads still in progress should be aborted. ### Uploading (new) The "Uploading" phase signifies that the main part of the backup, including snapshotting has completed successfully and uploading is continuing. In the event of an error during uploading, the phase will change to UploadingPartialFailure. On success, the phase changes to Completed. The backup cannot be restored from when it is in the Uploading state. ### UploadingPartialFailure (new) The "UploadingPartialFailure" phase signifies that the main part of the backup, including snapshotting has completed, but there were partial failures either during the main part or during the uploading. The backup cannot be restored from when it is in the UploadingPartialFailure state. ### Failed When a backup has had fatal errors it is marked as "Failed" This backup cannot be restored from. ### Completed The "Completed" phase signifies that the backup has completed, all data has been transferred to stable storage and the backup is ready to be used in a restore. When the Completed phase has been reached it is safe to remove any of the items that were backed up. ### PartiallyFailed The "PartiallyFailed" phase signifies that the backup has completed and at least part of the backup is usable. Restoration from a PartiallyFailed backup will not result in a complete restoration but pieces may be available. ## Workflow When a BackupAction is executed, any SnapshotItemAction or VolumeSnapshot plugins will return snapshot IDs. The plugin should be able to provide status on the progress for the snapshot and handle cancellation of the upload if the snapshot is deleted. If the plugin is restarted, the snapshot ID should remain valid. When all snapshots have been taken and Kubernetes resources have been persisted to the ObjectStorePlugin the backup will either have fatal errors or will be at least partially usable. If the backup has fatal errors it will move to the "Failed" state and finish. If a backup fails, the upload will not be cancelled but it will not be monitored either. For backups in any phase, all snapshots will be deleted when the backup is deleted. Plugins will cancel any data movement and remove snapshots and other associated resources when the VolumeSnapshotter DeleteSnapshot method or DeleteItemAction Execute method is called. Velero will poll the plugins for status on the snapshots when the backup exits the "InProgress" phase and has no fatal errors. If any snapshots are not complete, the backup will move to either Uploading or UploadingPartialFailure or Failed. Post-snapshot operations may take a long time and Velero and its plugins may be restarted during this time. Once a backup has moved into the Uploading or UploadingPartialFailure phase, another backup may be started. While in the Uploading or UploadingPartialFailure phase, the snapshots and backup items will be periodically polled. When all of the snapshots and backup items have reported success, the backup will move to the Completed or PartiallyFailed phase, depending on whether the backup was in the Uploading or UploadingPartialFailure phase. The Backup resources will not be written to object storage until the backup has entered a final phase: Completed, Failed or PartialFailure ## Reconciliation of InProgress backups InProgress backups will not have a `velero-backup.json` present in the object store. During reconciliation, backups which do not have a `velero-backup.json` object in the object store will be ignored. ## Plugin API changes ### UploadProgress struct type UploadProgress struct { completed bool // True when the operation has completed, either successfully or with a failure err error // Set when the operation has failed itemsCompleted, itemsToComplete int64 // The number of items that have been completed and the items to complete // For a disk, an item would be a byte and itemsToComplete would be the // total size to transfer (may be less than the size of a volume if // performing an incremental) and itemsCompleted is the number of bytes // transferred. On successful completion, itemsCompleted and itemsToComplete // should be the same started, updated time.Time // When the upload was started and when the last update was seen. Not all // systems retain when the upload was begun, return Time 0 (time.Unix(0, 0)) // if unknown. } ### VolumeSnapshotter changes A new method will be added to the VolumeSnapshotter interface (details depending on plugin versioning spec) UploadProgress(snapshotID string) (UploadProgress, error) UploadProgress will report the current status of a snapshot upload. This should be callable at any time after the snapshot has been taken. In the event a plugin is restarted, if the snapshotID continues to be valid it should be possible to retrieve the progress. `error` is set if there is an issue retrieving progress. If the snapshot is has encountered an error during the upload, the error should be return in UploadProgress and error should be nil. ### SnapshotItemAction plugin Currently CSI snapshots and the Velero Plugin for vSphere are implemented as BackupItemAction plugins. The majority of BackupItemAction plugins do not take snapshots or upload data so rather than modify BackupItemAction we introduce a new plugins, SnapshotItemAction. SnapshotItemAction will be used in place of BackupItemAction for the CSI snapshots and the Velero Plugin for vSphere and will return a snapshot ID in addition to the item itself. The SnapshotItemAction plugin identifier as well as the Item and Snapshot ID will be stored in the `-itemsnapshots.json.gz`. When checking for progress, this info will be used to select the appropriate SnapshotItemAction plugin to query for progress. _NotApplicable_ should only be returned if the SnapshotItemAction plugin should not be handling the item. If the SnapshotItemAction plugin should handle the item but, for example, the item/snapshot ID cannot be found to report progress, a UploadProgress struct with the error set appropriately (in this case _NotFound_) should be returned. // SnapshotItemAction is an actor that snapshots an individual item being backed up (it may also do other operations on the item that is returned). type SnapshotItemAction interface { // AppliesTo returns information about which resources this action should be invoked for. // A BackupItemAction's Execute function will only be invoked on items that match the returned // selector. A zero-valued ResourceSelector matches all resources. AppliesTo() (ResourceSelector, error) // Execute allows the ItemAction to perform arbitrary logic with the item being backed up, // including mutating the item itself prior to backup. The item (unmodified or modified) // should be returned, along with an optional slice of ResourceIdentifiers specifying // additional related items that should be backed up. Execute(item runtime.Unstructured, backup *api.Backup) (runtime.Unstructured, snapshotID string, []ResourceIdentifier, error) // Progress Progress(input *SnapshotItemProgressInput) (UploadProgress, error) } // SnapshotItemProgressInput contains the input parameters for the SnapshotItemAction's Progress function. type SnapshotItemProgressInput struct { // Item is the item that was stored in the backup Item runtime.Unstructured // SnapshotID is the snapshot ID returned by SnapshotItemAction SnapshotID string // Backup is the representation of the restore resource processed by Velero. Backup *velerov1api.Backup } ## Changes in Velero backup format No changes to the existing format are introduced by this change. A `-itemsnapshots.json.gz` file will be added that contains the items and snapshot IDs returned by ItemSnapshotAction. Also, the creation of the `velero-backup.json` object will not occur until the backup moves to one of the terminal phases (_Completed_, _PartiallyFailed_, or _Failed_). Reconciliation should ignore backups that do not have a `velero-backup.json` object. The cluster that is creating the backup will have the Backup resource present and will be able to manage the backup before the backup completes. If the Backup resource is removed (e.g. Velero is uninstalled) before a backup completes and writes its `velero-backup.json` object, the other objects in the object store for the backup will be effectively orphaned. This can currently happen but the current window is much smaller. ### `-itemsnapshots.json.gz` The itemsnapshots file is similar to the existing `-itemsnapshots.json.gz` Each snapshot taken via SnapshotItemAction will have a JSON record in the file. Exact format TBD. ## CSI snapshots For systems such as EBS, a snapshot is not available until the storage system has transferred the snapshot to stable storage. CSI snapshots expose the _readyToUse_ state that, in the case of EBS, indicates that the snapshot has been transferred to durable storage and is ready to be used. The CSI BackupItemProgress.Progress method will poll that field and when completed, return completion. ## vSphere plugin The vSphere Plugin for Velero uploads snapshots to S3 in the background. This is also a BackupItemAction plugin, it will check the status of the Upload records for the snapshot and return progress. ## Backup workflow changes The backup workflow remains the same until we get to the point where the `velero-backup.json` object is written. At this point, we will queue the backup to a finalization go-routine. The next backup may then begin. The finalization routine will run across all of the volume snapshots and call the _UploadProgress_ method on each of them. It will then run across all items and call _BackupItemProgress.Progress_ for any that match with a BackupItemProgress. If all snapshots and backup items have finished uploading (either successfully or failed), the backup will be completed and the backup will move to the appropriate terminal phase and upload the `velero-backup.json` object to the object store and the backup will be complete. If any of the snapshots or backup items are still being processed, the phase of the backup will be set to the appropriate phase (_Uploading_ or _UploadingPartialFailure_). In the event of any of the upload progress checks return an error, the phase will move to _UploadingPartialFailure_. The backup will then be requeued and will be rechecked again after some time has passed. ## Restart workflow On restart, the Velero server will scan all Backup resources. Any Backup resources which are in the _InProgress_ phase will be moved to the _Failed_ phase. Any Backup resources in the _Oploading_ or _OploadingPartialFailure_ phase will be treated as if they have been requeued and progress checked and the backup will be requeued or moved to a terminal phase as appropriate. # Implementation tasks VolumeSnapshotter new plugin APIs BackupItemProgress new plugin interface New backup phases Defer uploading `velero-backup.json` AWS EBS plugin UploadProgress implementation Upload monitoring Implementation of `-itemsnapshots.json.gz` file Restart logic Change in reconciliation logic to ignore backups that have not completed CSI plugin BackupItemProgress implementation vSphere plugin BackupItemProgress implementation (vSphere plugin team) # Future Fragile/Durable snapshot tracking Futures are here for reference, they may change radically when actually implemented. Some storage systems have the ability to provide different levels of protection for snapshots. These are termed "Fragile" and "Durable". Currently, Velero expects snapshots to be Durable (they should be able to survive the destruction of the cluster and the storage it is using). In the future we would like the ability to take advantage of snapshots that are Fragile. For example, vSphere snapshots are Fragile (they reside in the same datastore as the virtual disk). The Velero Plugin for vSphere uses a vSphere local/fragile snapshot to get a consistent snapshot, then uploads the data to S3 to make it Durable. In the current design, upload progress will not be complete until the snapshot is ready to use and Durable. It is possible, however, to restore data from a vSphere snapshot before it has been made Durable, and this is a capability we'd like to expose in the future. Other storage systems implement this functionality as well. We will be moving the control of the data movement from the vSphere plugin into Velero. Some storage system, such as EBS, are only capable of creating Durable snapshots. There is no usable intermediate Fragile stage. For a Velero backup, users should be able to specify whether they want a Durable backup or a Fragile backup (Fragile backups may consume less resources, be quicker to restore from and are suitable for things like backing up a cluster before upgrading software). We can introduce three snapshot states - Creating, Fragile and Durable. A snapshot would be created with a desired state, Fragile or Durable. When the snapshot reaches the desired or higher state (e.g. request was for Fragile but snapshot went to Durable as on EBS), then the snapshot would be completed. ================================================ FILE: design/vsv2-design.md ================================================ # Design for VolumeSnapshotter v2 API ## Abstract This design includes the changes to the VolumeSnapshotter api design as required by the [Item Action Progress Monitoring](general-progress-monitoring.md) feature. The VolumeSnapshotter v2 interface will have two new methods. If there are any additional VolumeSnapshotter API changes that are needed in the same Velero release cycle as this change, those can be added here as well. ## Background This API change is needed to facilitate long-running plugin actions that may not be complete when the Execute() method returns. The existing snapshotID returned by CreateSnapshot will be used as the operation ID. This will allow long-running plugin actions to continue in the background while Velero moves on to the next plugin, the next item, etc. ## Goals - Allow for VolumeSnapshotter CreateSnapshot() to initiate a long-running operation and report on operation status. ## Non Goals - Allowing velero control over when the long-running operation begins. ## High-Level Design As per the [Plugin Versioning](plugin-versioning.md) design, a new VolumeSnapshotterv2 plugin `.proto` file will be created to define the GRPC interface. v2 go files will also be created in `plugin/clientmgmt/volumesnapshotter` and `plugin/framework/volumesnapshotter`, and a new PluginKind will be created. The velero Backup process will be modified to reference v2 plugins instead of v1 plugins. An adapter will be created so that any existing VolumeSnapshotter v1 plugin can be executed as a v2 plugin when executing a backup. ## Detailed Design ### proto changes (compiled into golang by protoc) The v2 VolumeSnapshotter.proto will be like the current v1 version with the following changes: The VolumeSnapshotter service gets two new rpc methods: ``` service VolumeSnapshotter { rpc Init(VolumeSnapshotterInitRequest) returns (Empty); rpc CreateVolumeFromSnapshot(CreateVolumeRequest) returns (CreateVolumeResponse); rpc GetVolumeInfo(GetVolumeInfoRequest) returns (GetVolumeInfoResponse); rpc CreateSnapshot(CreateSnapshotRequest) returns (CreateSnapshotResponse); rpc DeleteSnapshot(DeleteSnapshotRequest) returns (Empty); rpc GetVolumeID(GetVolumeIDRequest) returns (GetVolumeIDResponse); rpc SetVolumeID(SetVolumeIDRequest) returns (SetVolumeIDResponse); rpc Progress(VolumeSnapshotterProgressRequest) returns (VolumeSnapshotterProgressResponse); rpc Cancel(VolumeSnapshotterCancelRequest) returns (google.protobuf.Empty); } ``` To support these new rpc methods, we define new request/response message types: ``` message VolumeSnapshotterProgressRequest { string plugin = 1; string snapshotID = 2; } message VolumeSnapshotterProgressResponse { generated.OperationProgress progress = 1; } message VolumeSnapshotterCancelRequest { string plugin = 1; string operationID = 2; } ``` One new shared message type will be needed, as defined in the v2 BackupItemAction design: ``` message OperationProgress { bool completed = 1; string err = 2; int64 completed = 3; int64 total = 4; string operationUnits = 5; string description = 6; google.protobuf.Timestamp started = 7; google.protobuf.Timestamp updated = 8; } ``` A new PluginKind, `VolumeSnapshotterV2`, will be created, and the backup process will be modified to use this plugin kind. See [Plugin Versioning](plugin-versioning.md) for more details on implementation plans, including v1 adapters, etc. ## Compatibility The included v1 adapter will allow any existing VolumeSnapshotter plugin to work as expected, with no-op Progress() and Cancel() methods. ## Implementation This will be implemented during the Velero 1.11 development cycle. ================================================ FILE: examples/README.md ================================================ # Examples This directory contains sample YAML config files that can be used for exploring Velero. * `minio/`: Used in the [Quickstart][0] to set up [Minio][1], a local S3-compatible object storage service. It provides a convenient way to test Velero without tying you to a specific cloud provider. * `nginx-app/`: A sample nginx app that can be used to test backups and restores. [0]: https://velero.io/docs/main/contributions/minio/ [1]: https://github.com/minio/minio ================================================ FILE: examples/minio/00-minio-deployment.yaml ================================================ # Copyright 2017 the Velero contributors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. --- apiVersion: v1 kind: Namespace metadata: name: velero --- apiVersion: apps/v1 kind: Deployment metadata: namespace: velero name: minio labels: component: minio spec: strategy: type: Recreate selector: matchLabels: component: minio template: metadata: labels: component: minio spec: volumes: - name: storage emptyDir: {} - name: config emptyDir: {} containers: - name: minio image: minio/minio:latest imagePullPolicy: IfNotPresent args: - server - /storage - --config-dir=/config env: - name: MINIO_ACCESS_KEY value: "minio" - name: MINIO_SECRET_KEY value: "minio123" ports: - containerPort: 9000 volumeMounts: - name: storage mountPath: "/storage" - name: config mountPath: "/config" --- apiVersion: v1 kind: Service metadata: namespace: velero name: minio labels: component: minio spec: # ClusterIP is recommended for production environments. # Change to NodePort if needed per documentation, # but only if you run Minio in a test/trial environment, for example with Minikube. type: ClusterIP ports: - port: 9000 targetPort: 9000 protocol: TCP selector: component: minio --- apiVersion: batch/v1 kind: Job metadata: namespace: velero name: minio-setup labels: component: minio spec: template: metadata: name: minio-setup spec: restartPolicy: OnFailure volumes: - name: config emptyDir: {} containers: - name: mc image: minio/mc:latest imagePullPolicy: IfNotPresent command: - /bin/sh - -c - "mc --config-dir=/config alias set velero http://minio:9000 minio minio123 && mc --config-dir=/config mb -p velero/velero" volumeMounts: - name: config mountPath: "/config" ================================================ FILE: examples/nginx-app/README.md ================================================ # Files This directory contains manifests for two versions of a sample Nginx app under the `nginx-example` namespace. ## `base.yaml` This is the most basic version of the Nginx app, which can be used to test Velero's backup and restore functionality. *This can be deployed as is.* ## `with-pv.yaml` This sets up an Nginx app that logs to a persistent volume, so that Velero's PV snapshotting functionality can also be tested. *This requires you to first replace the placeholder value ``.* ================================================ FILE: examples/nginx-app/base.yaml ================================================ # Copyright 2017 the Velero contributors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. --- apiVersion: v1 kind: Namespace metadata: name: nginx-example labels: app: nginx --- apiVersion: apps/v1 kind: Deployment metadata: name: nginx-deployment namespace: nginx-example labels: app: nginx spec: replicas: 2 selector: matchLabels: app: nginx template: metadata: labels: app: nginx spec: containers: - image: nginx:1.17.6 name: nginx ports: - containerPort: 80 --- apiVersion: v1 kind: Service metadata: labels: app: nginx name: my-nginx namespace: nginx-example spec: ports: - port: 80 targetPort: 80 selector: app: nginx type: LoadBalancer ================================================ FILE: examples/nginx-app/with-pv.yaml ================================================ # Copyright 2017 the Velero contributors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. --- apiVersion: v1 kind: Namespace metadata: name: nginx-example labels: app: nginx --- kind: PersistentVolumeClaim apiVersion: v1 metadata: name: nginx-logs namespace: nginx-example labels: app: nginx spec: # Optional: # storageClassName: accessModes: - ReadWriteOnce resources: requests: storage: 50Mi --- apiVersion: apps/v1 kind: Deployment metadata: name: nginx-deployment namespace: nginx-example spec: replicas: 1 selector: matchLabels: app: nginx template: metadata: labels: app: nginx annotations: pre.hook.backup.velero.io/container: fsfreeze pre.hook.backup.velero.io/command: '["/sbin/fsfreeze", "--freeze", "/var/log/nginx"]' post.hook.backup.velero.io/container: fsfreeze post.hook.backup.velero.io/command: '["/sbin/fsfreeze", "--unfreeze", "/var/log/nginx"]' spec: volumes: - name: nginx-logs persistentVolumeClaim: claimName: nginx-logs containers: - image: nginx:1.17.6 name: nginx ports: - containerPort: 80 volumeMounts: - mountPath: "/var/log/nginx" name: nginx-logs readOnly: false - image: ubuntu:bionic name: fsfreeze securityContext: privileged: true volumeMounts: - mountPath: "/var/log/nginx" name: nginx-logs readOnly: false command: - "/bin/bash" - "-c" - "sleep infinity" --- apiVersion: v1 kind: Service metadata: labels: app: nginx name: my-nginx namespace: nginx-example spec: ports: - port: 80 targetPort: 80 selector: app: nginx type: LoadBalancer ================================================ FILE: go.mod ================================================ module github.com/vmware-tanzu/velero go 1.25.0 require ( cloud.google.com/go/storage v1.57.2 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.6.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3 github.com/aws/aws-sdk-go-v2 v1.24.1 github.com/aws/aws-sdk-go-v2/config v1.26.3 github.com/aws/aws-sdk-go-v2/credentials v1.16.14 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.11 github.com/aws/aws-sdk-go-v2/service/ec2 v1.143.0 github.com/aws/aws-sdk-go-v2/service/s3 v1.48.0 github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 github.com/bombsimon/logrusr/v3 v3.0.0 github.com/evanphx/json-patch/v5 v5.9.11 github.com/fatih/color v1.18.0 github.com/gobwas/glob v0.2.3 github.com/google/go-cmp v0.7.0 github.com/google/uuid v1.6.0 github.com/hashicorp/go-hclog v0.14.1 github.com/hashicorp/go-plugin v1.6.0 github.com/joho/godotenv v1.3.0 github.com/kopia/kopia v0.16.0 github.com/kubernetes-csi/external-snapshotter/client/v8 v8.2.0 github.com/onsi/ginkgo/v2 v2.22.0 github.com/onsi/gomega v1.36.1 github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.23.2 github.com/prometheus/client_model v0.6.2 github.com/robfig/cron/v3 v3.0.1 github.com/sirupsen/logrus v1.9.3 github.com/spf13/afero v1.10.0 github.com/spf13/cobra v1.8.1 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.11.1 github.com/vmware-tanzu/crash-diagnostics v0.3.7 go.uber.org/zap v1.27.1 golang.org/x/mod v0.30.0 golang.org/x/oauth2 v0.34.0 golang.org/x/sys v0.40.0 golang.org/x/text v0.32.0 google.golang.org/api v0.256.0 google.golang.org/grpc v1.79.3 google.golang.org/protobuf v1.36.10 gopkg.in/yaml.v3 v3.0.1 k8s.io/api v0.33.3 k8s.io/apiextensions-apiserver v0.33.3 k8s.io/apimachinery v0.33.3 k8s.io/cli-runtime v0.33.3 k8s.io/client-go v0.33.3 k8s.io/klog/v2 v2.130.1 k8s.io/kube-aggregator v0.33.3 k8s.io/metrics v0.33.3 k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 sigs.k8s.io/controller-runtime v0.21.0 sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 sigs.k8s.io/yaml v1.4.0 ) require ( cel.dev/expr v0.25.1 // indirect cloud.google.com/go v0.121.6 // indirect cloud.google.com/go/auth v0.17.0 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect cloud.google.com/go/compute/metadata v0.9.0 // indirect cloud.google.com/go/iam v1.5.2 // indirect cloud.google.com/go/monitoring v1.24.2 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.10 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 // indirect github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.10 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.10 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.18.6 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.6 // indirect github.com/aws/smithy-go v1.19.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chmduquesne/rollinghash v4.0.0+incompatible // indirect github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/edsrzf/mmap-go v1.2.0 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/envoyproxy/go-control-plane/envoy v1.36.0 // indirect github.com/envoyproxy/protoc-gen-validate v1.3.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/go-ini/ini v1.67.0 // indirect github.com/go-jose/go-jose/v4 v4.1.3 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.23.0 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/gofrs/flock v0.13.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v5 v5.3.0 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/btree v1.1.3 // indirect github.com/google/gnostic-models v0.6.9 // indirect github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db // indirect github.com/google/s2a-go v0.1.9 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.7 // indirect github.com/googleapis/gax-go/v2 v2.15.0 // indirect github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect github.com/hashicorp/cronexpr v1.1.3 // indirect github.com/hashicorp/yamux v0.1.1 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.18.2 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/klauspost/crc32 v1.3.0 // indirect github.com/klauspost/pgzip v1.2.6 // indirect github.com/klauspost/reedsolomon v1.12.6 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/minio/crc64nvme v1.1.0 // indirect github.com/minio/md5-simd v1.1.2 // indirect github.com/minio/minio-go/v7 v7.0.97 // indirect github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/moby/spdystream v0.5.0 // indirect github.com/moby/term v0.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/mxk/go-vss v1.2.0 // indirect github.com/natefinch/atomic v1.0.1 // indirect github.com/nxadm/tail v1.4.8 // indirect github.com/oklog/run v1.0.0 // indirect github.com/philhofer/fwd v1.2.0 // indirect github.com/pierrec/lz4 v2.6.1+incompatible // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/common v0.67.4 // indirect github.com/prometheus/procfs v0.16.1 // indirect github.com/rs/xid v1.6.0 // indirect github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/tinylib/msgp v1.3.0 // indirect github.com/vladimirvivien/gexe v0.1.1 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/zeebo/blake3 v0.2.4 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/detectors/gcp v1.39.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect go.opentelemetry.io/otel v1.40.0 // indirect go.opentelemetry.io/otel/metric v1.40.0 // indirect go.opentelemetry.io/otel/sdk v1.40.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.40.0 // indirect go.opentelemetry.io/otel/trace v1.40.0 // indirect go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect golang.org/x/crypto v0.46.0 // indirect golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect golang.org/x/net v0.48.0 // indirect golang.org/x/sync v0.19.0 // indirect golang.org/x/term v0.38.0 // indirect golang.org/x/time v0.14.0 // indirect golang.org/x/tools v0.39.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/genproto v0.0.0-20250603155806-513f23925822 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect ) replace github.com/kopia/kopia => github.com/project-velero/kopia v0.0.0-20251230033609-d946b1e75197 ================================================ FILE: go.sum ================================================ al.essio.dev/pkg/shellescape v1.5.1 h1:86HrALUujYS/h+GtqoB26SBEdkWfmMI6FubjXlsXyho= al.essio.dev/pkg/shellescape v1.5.1/go.mod h1:6sIqp7X2P6mThCQ7twERpZTuigpr6KbZWtls1U8I890= cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= cloud.google.com/go v0.121.6 h1:waZiuajrI28iAf40cWgycWNgaXPO06dupuS+sgibK6c= cloud.google.com/go v0.121.6/go.mod h1:coChdst4Ea5vUpiALcYKXEpR1S9ZgXbhEzzMcMR66vI= cloud.google.com/go/auth v0.17.0 h1:74yCm7hCj2rUyyAocqnFzsAYXgJhrG26XCFimrc/Kz4= cloud.google.com/go/auth v0.17.0/go.mod h1:6wv/t5/6rOPAX4fJiRjKkJCvswLwdet7G8+UGXt7nCQ= cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/iam v1.5.2 h1:qgFRAGEmd8z6dJ/qyEchAuL9jpswyODjA2lS+w234g8= cloud.google.com/go/iam v1.5.2/go.mod h1:SE1vg0N81zQqLzQEwxL2WI6yhetBdbNQuTvIKCSkUHE= cloud.google.com/go/logging v1.13.0 h1:7j0HgAp0B94o1YRDqiqm26w4q1rDMH7XNRU34lJXHYc= cloud.google.com/go/logging v1.13.0/go.mod h1:36CoKh6KA/M0PbhPKMq6/qety2DCAErbhXT62TuXALA= cloud.google.com/go/longrunning v0.7.0 h1:FV0+SYF1RIj59gyoWDRi45GiYUMM3K1qO51qoboQT1E= cloud.google.com/go/longrunning v0.7.0/go.mod h1:ySn2yXmjbK9Ba0zsQqunhDkYi0+9rlXIwnoAf+h+TPY= cloud.google.com/go/monitoring v1.24.2 h1:5OTsoJ1dXYIiMiuL+sYscLc9BumrL3CarVLL7dd7lHM= cloud.google.com/go/monitoring v1.24.2/go.mod h1:x7yzPWcgDRnPEv3sI+jJGBkwl5qINf+6qY4eq0I9B4U= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= cloud.google.com/go/storage v1.57.2 h1:sVlym3cHGYhrp6XZKkKb+92I1V42ks2qKKpB0CF5Mb4= cloud.google.com/go/storage v1.57.2/go.mod h1:n5ijg4yiRXXpCu0sJTD6k+eMf7GRrJmPyr9YxLXGHOk= cloud.google.com/go/trace v1.11.6 h1:2O2zjPzqPYAHrn3OKl029qlqG6W8ZdYaOWRyr8NgMT4= cloud.google.com/go/trace v1.11.6/go.mod h1:GA855OeDEBiBMzcckLPE2kDunIpC72N+Pq8WFieFjnI= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 h1:JXg2dwJUmPB9JmtVmdEB16APJ7jurfbY5jnfXpJoRMc= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0/go.mod h1:YD5h/ldMsG0XiIw7PdyNhLxaM317eFh5yNLccNfGdyw= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1/go.mod h1:IYus9qsFobWIc2YVwe/WPjcnyCkPKtnHAqUYeebc8z0= github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY= github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8= github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 h1:9iefClla7iYpfYWdzPCRDozdmndjTm8DXdpCzPajMgA= github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2/go.mod h1:XtLgD3ZD34DAaVIIAyG3objl5DynM3CQ/vMcbBNJZGI= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.6.0 h1:ui3YNbxfW7J3tTFIZMH6LIGRjCngp+J+nIFlnizfNTE= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.6.0/go.mod h1:gZmgV+qBqygoznvqo2J9oKZAFziqhLZ2xE/WVUmzkHA= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0 h1:PTFGRSlMKCQelWwxUyYVEUqseBJVemLyqWJjvMyt0do= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0/go.mod h1:LRr2FzBTQlONPPa5HREE5+RjSCTXl7BwOvYOaWTqCaI= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.1.0 h1:2qsIIvxVT+uE6yrNldntJKlLRgxGbZ85kgtz5SNBhMw= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.1.0/go.mod h1:AW8VEadnhw9xox+VaVd9sP7NjzOAnaZBLRH6Tq3cJ38= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 h1:Dd+RhdJn0OTtVGaeDLZpcumkIVCtA/3/Fo42+eoYvVM= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0/go.mod h1:5kakwfW5CjC9KK+Q4wjXAg+ShuIm2mBMua0ZFj2C8PE= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3 h1:ZJJNFaQ86GVKQ9ehwqyAFE6pIfyicpuJ8IkVaPBc6/4= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3/go.mod h1:URuDvhmATVKqHBH9/0nOiNKk0+YcwfQ3WkK5PqHKxc8= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs= github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5 h1:IEjq88XO4PuBDcvmjQJcQGg+w+UaafSy8G5Kcb5tBhI= github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5/go.mod h1:exZ0C/1emQJAw5tHOaUDyY1ycttqBAPcxuzf7QbY6ec= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 h1:sBEjpZlNHzK1voKq9695PJSX2o5NEXl7/OL3coiIY0c= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 h1:owcC2UnmsZycprQ5RfRgjydWhuoxg71LUfyiQdijZuM= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0/go.mod h1:ZPpqegjbE99EPKsu3iUWV22A04wzGPcAY/ziSIQEEgs= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.53.0 h1:4LP6hvB4I5ouTbGgWtixJhgED6xdf67twf9PoY96Tbg= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.53.0/go.mod h1:jUZ5LYlw40WMd07qxcQJD5M40aUxrfwqQX1g7zxYnrQ= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 h1:Ron4zCA/yk6U7WOBXhTJcDpsUBG9npumK6xw2auFltQ= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0/go.mod h1:cSgYe11MCNYunTnRXrKiR/tHc0eoKjICUuWpNZoVCOo= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go-v2 v1.24.1 h1:xAojnj+ktS95YZlDf0zxWBkbFtymPeDP+rvUQIH3uAU= github.com/aws/aws-sdk-go-v2 v1.24.1/go.mod h1:LNh45Br1YAkEKaAqvmE1m8FUx6a5b/V0oAKV7of29b4= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 h1:OCs21ST2LrepDfD3lwlQiOqIGp6JiEUqG84GzTDoyJs= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4/go.mod h1:usURWEKSNNAcAZuzRn/9ZYPT8aZQkR7xcCtunK/LkJo= github.com/aws/aws-sdk-go-v2/config v1.26.3 h1:dKuc2jdp10y13dEEvPqWxqLoc0vF3Z9FC45MvuQSxOA= github.com/aws/aws-sdk-go-v2/config v1.26.3/go.mod h1:Bxgi+DeeswYofcYO0XyGClwlrq3DZEXli0kLf4hkGA0= github.com/aws/aws-sdk-go-v2/credentials v1.16.14 h1:mMDTwwYO9A0/JbOCOG7EOZHtYM+o7OfGWfu0toa23VE= github.com/aws/aws-sdk-go-v2/credentials v1.16.14/go.mod h1:cniAUh3ErQPHtCQGPT5ouvSAQ0od8caTO9OOuufZOAE= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 h1:c5I5iH+DZcH3xOIMlz3/tCKJDaHFwYEmxvlh2fAcFo8= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11/go.mod h1:cRrYDYAMUohBJUtUnOhydaMHtiK/1NZ0Otc9lIb6O0Y= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.11 h1:I6lAa3wBWfCz/cKkOpAcumsETRkFAl70sWi8ItcMEsM= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.11/go.mod h1:be1NIO30kJA23ORBLqPo1LttEM6tPNSEcjkd1eKzNW0= github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 h1:vF+Zgd9s+H4vOXd5BMaPWykta2a6Ih0AKLq/X6NYKn4= github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10/go.mod h1:6BkRjejp/GR4411UGqkX8+wFMbFbqsUIimfK4XjOKR4= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 h1:nYPe006ktcqUji8S2mqXf9c/7NdiKriOwMvWQHgYztw= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10/go.mod h1:6UV4SZkVvmODfXKql4LCbaZUpF7HO2BX38FgBf9ZOLw= github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 h1:GrSw8s0Gs/5zZ0SX+gX4zQjRnRsMJDJ2sLur1gRBhEM= github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY= github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.10 h1:5oE2WzJE56/mVveuDZPJESKlg/00AaS2pY2QZcnxg4M= github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.10/go.mod h1:FHbKWQtRBYUz4vO5WBWjzMD2by126ny5y/1EoaWoLfI= github.com/aws/aws-sdk-go-v2/service/ec2 v1.143.0 h1:ZAO4y7MSRqU74ZFCA+HC6Ek5fI7dsTdwJg88s72I/gE= github.com/aws/aws-sdk-go-v2/service/ec2 v1.143.0/go.mod h1:hIsHE0PaWAQakLCshKS7VKWMGXaqrAFp4m95s2W9E6c= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 h1:/b31bi3YVNlkzkBrm9LfpaKoaYZUxIAj4sHfOTmLfqw= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4/go.mod h1:2aGXHFmbInwgP9ZfpmdIfOELL79zhdNYNmReK8qDfdQ= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.10 h1:L0ai8WICYHozIKK+OtPzVJBugL7culcuM4E4JOpIEm8= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.10/go.mod h1:byqfyxJBshFk0fF9YmK0M0ugIO8OWjzH2T3bPG4eGuA= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 h1:DBYTXwIGQSGs9w4jKm60F5dmCQ3EEruxdc0MFh+3EY4= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10/go.mod h1:wohMUQiFdzo0NtxbBg0mSRGZ4vL3n0dKjLTINdcIino= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.10 h1:KOxnQeWy5sXyS37fdKEvAsGHOr9fa/qvwxfJurR/BzE= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.10/go.mod h1:jMx5INQFYFYB3lQD9W0D8Ohgq6Wnl7NYOJ2TQndbulI= github.com/aws/aws-sdk-go-v2/service/s3 v1.48.0 h1:PJTdBMsyvra6FtED7JZtDpQrIAflYDHFoZAu/sKYkwU= github.com/aws/aws-sdk-go-v2/service/s3 v1.48.0/go.mod h1:4qXHrG1Ne3VGIMZPCB8OjH/pLFO94sKABIusjh0KWPU= github.com/aws/aws-sdk-go-v2/service/sso v1.18.6 h1:dGrs+Q/WzhsiUKh82SfTVN66QzyulXuMDTV/G8ZxOac= github.com/aws/aws-sdk-go-v2/service/sso v1.18.6/go.mod h1:+mJNDdF+qiUlNKNC3fxn74WWNN+sOiGOEImje+3ScPM= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.6 h1:Yf2MIo9x+0tyv76GljxzqA3WtC5mw7NmazD2chwjxE4= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.6/go.mod h1:ykf3COxYI0UJmxcfcxcVuz7b6uADi1FkiUz6Eb7AgM8= github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 h1:NzO4Vrau795RkUdSHKEwiR01FaGzGOH1EETJ+5QHnm0= github.com/aws/aws-sdk-go-v2/service/sts v1.26.7/go.mod h1:6h2YuIoxaMSCFf5fi1EgZAwdfkGMgDY+DVfa61uLe4U= github.com/aws/smithy-go v1.19.0 h1:KWFKQV80DpP3vJrrA9sVAHQ5gc2z8i4EzrLhLlWXcBM= github.com/aws/smithy-go v1.19.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bombsimon/logrusr/v3 v3.0.0 h1:tcAoLfuAhKP9npBxWzSdpsvKPQt1XV02nSf2lZA82TQ= github.com/bombsimon/logrusr/v3 v3.0.0/go.mod h1:PksPPgSFEL2I52pla2glgCyyd2OqOHAnFF5E+g8Ixco= github.com/bufbuild/protocompile v0.4.0 h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZlaQsNA= github.com/bufbuild/protocompile v0.4.0/go.mod h1:3v93+mbWn/v3xzN+31nwkJfrEpAUwp+BagBSZWx+TP8= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chmduquesne/rollinghash v4.0.0+incompatible h1:hnREQO+DXjqIw3rUTzWN7/+Dpw+N5Um8zpKV0JOEgbo= github.com/chmduquesne/rollinghash v4.0.0+incompatible/go.mod h1:Uc2I36RRfTAf7Dge82bi3RU0OQUmXT9iweIcPqvr8A0= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 h1:6xNmx7iTtyBRev0+D/Tv1FZd4SCg8axKApyNyRsAt/w= github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5/go.mod h1:KdCmV+x/BuvyMxRnYBlmVaq4OLiKW6iRQfvC62cvdkI= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/danieljoos/wincred v1.2.2 h1:774zMFJrqaeYCK2W57BgAem/MLi6mtSE47MB6BOJ0i0= github.com/danieljoos/wincred v1.2.2/go.mod h1:w7w4Utbrz8lqeMbDAK0lkNJUv5sAOkFi7nd/ogr0Uh8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/edsrzf/mmap-go v1.2.0 h1:hXLYlkbaPzt1SaQk+anYwKSRNhufIDCchSPkUD6dD84= github.com/edsrzf/mmap-go v1.2.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= github.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9ZMkUlwlKxtAohpi2wBEU= github.com/envoyproxy/go-control-plane/envoy v1.36.0 h1:yg/JjO5E7ubRyKX3m07GF3reDNEnfOboJ0QySbH736g= github.com/envoyproxy/go-control-plane/envoy v1.36.0/go.mod h1:ty89S1YCCVruQAm9OtKeEkQLTb+Lkz0k8v9W0Oxsv98= github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v1.3.0 h1:TvGH1wof4H33rezVKWSpqKz5NXWg5VPuZ0uONDT6eb4= github.com/envoyproxy/protoc-gen-validate v1.3.0/go.mod h1:HvYl7zwPa5mffgyeTUHA9zHIH36nmrm7oCbo4YKoSWA= github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/frankban/quicktest v1.13.1 h1:xVm/f9seEhZFL9+n5kv5XLrGwy6elc4V9v/XFY2vmd8= github.com/frankban/quicktest v1.13.1/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs= github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw= github.com/gofrs/flock v0.13.0/go.mod h1:jxeyy9R1auM5S6JYDBhDt+E2TCo7DkratH4Pgi8P+Z0= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw= github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.3.7 h1:zrn2Ee/nWmHulBx5sAVrGgAa0f2/R35S4DJwfFaUPFQ= github.com/googleapis/enterprise-certificate-proxy v0.3.7/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.15.0 h1:SyjDc1mGgZU5LncH8gimWo9lW1DtIfPibOG81vgd/bo= github.com/googleapis/gax-go/v2 v2.15.0/go.mod h1:zVVkkxAQHa1RQpg9z2AUCMnKhi0Qld9rcmyfL1OZhoc= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hanwen/go-fuse/v2 v2.9.0 h1:0AOGUkHtbOVeyGLr0tXupiid1Vg7QB7M6YUcdmVdC58= github.com/hanwen/go-fuse/v2 v2.9.0/go.mod h1:yE6D2PqWwm3CbYRxFXV9xUd8Md5d6NG0WBs5spCswmI= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/cronexpr v1.1.3 h1:rl5IkxXN2m681EfivTlccqIryzYJSXRGRNa0xeG7NA4= github.com/hashicorp/cronexpr v1.1.3/go.mod h1:P4wA0KBl9C5q2hABiMO7cp6jcIg96CDh1Efb3g1PWA4= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-hclog v0.14.1 h1:nQcJDQwIAGnmoUWp8ubocEX40cCml/17YkF6csQLReU= github.com/hashicorp/go-hclog v0.14.1/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-plugin v1.6.0 h1:wgd4KxHJTVGGqWBq4QPB1i5BZNEx9BR8+OFmHDmTk8A= github.com/hashicorp/go-plugin v1.6.0/go.mod h1:lBS5MtSSBZk0SHc66KACcjjlU6WzEVP/8pwz68aMkCI= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hashicorp/yamux v0.1.1 h1:yrQxtgseBDrq9Y652vSRDvsKCJKOUD+GzTS4Y0Y8pvE= github.com/hashicorp/yamux v0.1.1/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jhump/protoreflect v1.15.1 h1:HUMERORf3I3ZdX05WaQ6MIpd/NJ434hTp5YiKgfCL6c= github.com/jhump/protoreflect v1.15.1/go.mod h1:jD/2GMKKE6OqX8qTjhADU1e6DShO+gavG9e0Q693nKo= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU= github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/klauspost/reedsolomon v1.12.6 h1:8pqE9aECQG/ZFitiUD1xK/E83zwosBAZtE3UbuZM8TQ= github.com/klauspost/reedsolomon v1.12.6/go.mod h1:ggJT9lc71Vu+cSOPBlxGvBN6TfAS77qB4fp8vJ05NSA= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kopia/htmluibuild v0.0.1-0.20251125011029-7f1c3f84f29d h1:U3VB/cDMsPW4zB4JRFbVRDzIpPytt889rJUKAG40NPA= github.com/kopia/htmluibuild v0.0.1-0.20251125011029-7f1c3f84f29d/go.mod h1:h53A5JM3t2qiwxqxusBe+PFgGcgZdS+DWCQvG5PTlto= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kubernetes-csi/external-snapshotter/client/v8 v8.2.0 h1:Q3jQ1NkFqv5o+F8dMmHd8SfEmlcwNeo1immFApntEwE= github.com/kubernetes-csi/external-snapshotter/client/v8 v8.2.0/go.mod h1:E3vdYxHj2C2q6qo8/Da4g7P+IcwqRZyy3gJBzYybV9Y= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/minio/crc64nvme v1.1.0 h1:e/tAguZ+4cw32D+IO/8GSf5UVr9y+3eJcxZI2WOO/7Q= github.com/minio/crc64nvme v1.1.0/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= github.com/minio/minio-go/v7 v7.0.97 h1:lqhREPyfgHTB/ciX8k2r8k0D93WaFqxbJX36UZq5occ= github.com/minio/minio-go/v7 v7.0.97/go.mod h1:re5VXuo0pwEtoNLsNuSr0RrLfT/MBtohwdaSmPPSRSk= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/spdystream v0.5.0 h1:7r0J1Si3QO/kjRitvSLVVFUjxMEb/YLj6S9FF62JBCU= github.com/moby/spdystream v0.5.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/mxk/go-vss v1.2.0 h1:JpdOPc/P6B3XyRoddn0iMiG/ADBi3AuEsv8RlTb+JeE= github.com/mxk/go-vss v1.2.0/go.mod h1:ZQ4yFxCG54vqPnCd+p2IxAe5jwZdz56wSjbwzBXiFd8= github.com/natefinch/atomic v1.0.1 h1:ZPYKxkqQOx3KZ+RsbnP/YsgvxWQPGxjC0oBt2AhwV0A= github.com/natefinch/atomic v1.0.1/go.mod h1:N/D/ELrljoqDyT3rZrsUmtsuzvHkeB/wWjHV22AZRbM= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/run v1.0.0 h1:Ru7dDtJNOyC66gQ5dQmaCa0qIsAUFY3sFpK1Xk8igrw= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg= github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.36.1 h1:bJDPBO7ibjxcbHMgSCoo4Yj18UWbKDlLwX1x9sybDcw= github.com/onsi/gomega v1.36.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+Tv1WTxkukpXeMlviSxvL7SRgk= github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/project-velero/kopia v0.0.0-20251230033609-d946b1e75197 h1:iGkfuELGvFCqW+zcrhf2GsOwNH1nWYBsC69IOc57KJk= github.com/project-velero/kopia v0.0.0-20251230033609-d946b1e75197/go.mod h1:RL4KehCNKEIDNltN7oruSa3ldwBNVPmQbwmN3Schbjc= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.67.4 h1:yR3NqWO1/UyO1w2PhUvXlGQs/PtFmoveVO0KZ4+Lvsc= github.com/prometheus/common v0.67.4/go.mod h1:gP0fq6YjjNCLssJCQp0yk4M8W6ikLURwkdd/YKtTbyI= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/afero v1.10.0 h1:EaGW2JJh15aKOejeuJ+wpFSHnbd7GE6Wvp3TsNhb6LY= github.com/spf13/afero v1.10.0/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= github.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMpsbLuo= github.com/spiffe/go-spiffe/v2 v2.6.0/go.mod h1:gm2SeUoMZEtpnzPNs2Csc0D/gX33k1xIx7lEzqblHEs= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tg123/go-htpasswd v1.2.4 h1:HgH8KKCjdmo7jjXWN9k1nefPBd7Be3tFCTjc2jPraPU= github.com/tg123/go-htpasswd v1.2.4/go.mod h1:EKThQok9xHkun6NBMynNv6Jmu24A33XdZzzl4Q7H1+0= github.com/tinylib/msgp v1.3.0 h1:ULuf7GPooDaIlbyvgAxBV/FI7ynli6LZ1/nVUNu+0ww= github.com/tinylib/msgp v1.3.0/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/vladimirvivien/gexe v0.1.1 h1:2A0SBaOSKH+cwLVdt6H+KkHZotZWRNLlWygANGw5DxE= github.com/vladimirvivien/gexe v0.1.1/go.mod h1:LHQL00w/7gDUKIak24n801ABp8C+ni6eBht9vGVst8w= github.com/vmware-tanzu/crash-diagnostics v0.3.7 h1:6gbv/3o1FzyRLS7Dz/+yVg1Lk1oRBQLyI3d1YTtlTT8= github.com/vmware-tanzu/crash-diagnostics v0.3.7/go.mod h1:gO8670rd+qdjnJVol674snT/A46GQ27u085kKhZznlM= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/zalando/go-keyring v0.2.6 h1:r7Yc3+H+Ux0+M72zacZoItR3UDxeWfKTcabvkI8ua9s= github.com/zalando/go-keyring v0.2.6/go.mod h1:2TCrxYrbUNYfNS/Kgy/LSrkSQzZ5UPVH85RwfczwvcI= github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY= github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/blake3 v0.2.4 h1:KYQPkhpRtcqh0ssGYcKLG1JYvddkEA8QwCM/yBqhaZI= github.com/zeebo/blake3 v0.2.4/go.mod h1:7eeQ6d2iXWRGF6npfaxl2CU+xy2Fjo2gxeyZGCRUjcE= github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo= github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/detectors/gcp v1.39.0 h1:kWRNZMsfBHZ+uHjiH4y7Etn2FK26LAGkNFw7RHv1DhE= go.opentelemetry.io/contrib/detectors/gcp v1.39.0/go.mod h1:t/OGqzHBa5v6RHZwrDBJ2OirWc+4q/w2fTbLZwAKjTk= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 h1:q4XOmH/0opmeuJtPsbFNivyl7bCt7yRBbeEm2sC/XtQ= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0/go.mod h1:snMWehoOh2wsEwnvvwtDyFCxVeDAODenXHtn5vzrKjo= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms= go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0 h1:rixTyDGXFxRy1xzhKrotaHy3/KXdPhlWARrCgK+eqUY= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0/go.mod h1:dowW6UsM9MKbJq5JTz2AMVp3/5iW5I/TStsk8S+CfHw= go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g= go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc= go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8= go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE= go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw= go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg= go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw= go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA= go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o= go.starlark.net v0.0.0-20201006213952-227f4aabceb5/go.mod h1:f0znQkUKRrkk36XxWbGjMqQM8wGv/xHBVE2qc3B5oFU= go.starlark.net v0.0.0-20230525235612-a134d8f9ddca h1:VdD38733bfYv5tUZwEIskMM93VanwNIi5bIKnDrJdEY= go.starlark.net v0.0.0-20230525235612-a134d8f9ddca/go.mod h1:jxU+3+j+71eXOW14274+SmmuW82qJzl6iZSeqEtTGds= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= google.golang.org/api v0.256.0 h1:u6Khm8+F9sxbCTYNoBHg6/Hwv0N/i+V94MvkOSor6oI= google.golang.org/api v0.256.0/go.mod h1:KIgPhksXADEKJlnEoRa9qAII4rXcy40vfI8HRqcU964= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto v0.0.0-20250603155806-513f23925822 h1:rHWScKit0gvAPuOnu87KpaYtjK5zBMLcULh7gxkCXu4= google.golang.org/genproto v0.0.0-20250603155806-513f23925822/go.mod h1:HubltRL7rMh0LfnQPkMH4NPDFEWp0jw3vixw7jEM53s= google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 h1:fCvbg86sFXwdrl5LgVcTEvNC+2txB5mgROGmRL5mrls= google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:+rXWjjaukWZun3mLfjmVnQi18E1AsFbDN9QdJ5YXLto= google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 h1:gRkg/vSppuSQoDjxyiGfN4Upv/h/DQmIR10ZU8dh4Ww= google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE= google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/api v0.22.2/go.mod h1:y3ydYpLJAaDI+BbSe2xmGcqxiWHmWjkEeIbiwHvnPR8= k8s.io/api v0.33.3 h1:SRd5t//hhkI1buzxb288fy2xvjubstenEKL9K51KBI8= k8s.io/api v0.33.3/go.mod h1:01Y/iLUjNBM3TAvypct7DIj0M0NIZc+PzAHCIo0CYGE= k8s.io/apiextensions-apiserver v0.33.3 h1:qmOcAHN6DjfD0v9kxL5udB27SRP6SG/MTopmge3MwEs= k8s.io/apiextensions-apiserver v0.33.3/go.mod h1:oROuctgo27mUsyp9+Obahos6CWcMISSAPzQ77CAQGz8= k8s.io/apimachinery v0.22.2/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= k8s.io/apimachinery v0.33.3 h1:4ZSrmNa0c/ZpZJhAgRdcsFcZOw1PQU1bALVQ0B3I5LA= k8s.io/apimachinery v0.33.3/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM= k8s.io/cli-runtime v0.22.2/go.mod h1:tkm2YeORFpbgQHEK/igqttvPTRIHFRz5kATlw53zlMI= k8s.io/cli-runtime v0.33.3 h1:Dgy4vPjNIu8LMJBSvs8W0LcdV0PX/8aGG1DA1W8lklA= k8s.io/cli-runtime v0.33.3/go.mod h1:yklhLklD4vLS8HNGgC9wGiuHWze4g7x6XQZ+8edsKEo= k8s.io/client-go v0.22.2/go.mod h1:sAlhrkVDf50ZHx6z4K0S40wISNTarf1r800F+RlCF6U= k8s.io/client-go v0.33.3 h1:M5AfDnKfYmVJif92ngN532gFqakcGi6RvaOF16efrpA= k8s.io/client-go v0.33.3/go.mod h1:luqKBQggEf3shbxHY4uVENAxrDISLOarxpTKMiUuujg= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-aggregator v0.33.3 h1:Pa6hQpKJMX0p0D2wwcxXJgu02++gYcGWXoW1z1ZJDfo= k8s.io/kube-aggregator v0.33.3/go.mod h1:hwvkUoQ8q6gv0+SgNnlmQ3eUue1zHhJKTHsX7BwxwSE= k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUyGcf03XZEP0ZIKgKj35LS4= k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff/go.mod h1:5jIi+8yX4RIb8wk3XwBo5Pq2ccx4FP10ohkbSKCZoK8= k8s.io/metrics v0.33.3 h1:9CcqBz15JZfISqwca33gdHS8I6XfsK1vA8WUdEnG70g= k8s.io/metrics v0.33.3/go.mod h1:Aw+cdg4AYHw0HvUY+lCyq40FOO84awrqvJRTw0cmXDs= k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/controller-runtime v0.21.0 h1:CYfjpEuicjUecRk+KAeyYh+ouUBn4llGyDYytIGcJS8= sigs.k8s.io/controller-runtime v0.21.0/go.mod h1:OSg14+F65eWqIu4DceX7k/+QRAbTTvxeQSNSOQpukWM= sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= sigs.k8s.io/kustomize/api v0.8.11/go.mod h1:a77Ls36JdfCWojpUqR6m60pdGY1AYFix4AH83nJtY1g= sigs.k8s.io/kustomize/kyaml v0.11.0/go.mod h1:GNMwjim4Ypgp/MueD3zXHLRJEjz7RvtPae0AwlvEMFM= sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/structured-merge-diff/v4 v4.6.0 h1:IUA9nvMmnKWcj5jl84xn+T5MnlZKThmUW1TdblaLVAc= sigs.k8s.io/structured-merge-diff/v4 v4.6.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= ================================================ FILE: hack/boilerplate.go.txt ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ ================================================ FILE: hack/build-image/Dockerfile ================================================ # Copyright 2018, 2019, 2020 the Velero contributors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. FROM --platform=$TARGETPLATFORM golang:1.25-bookworm ARG GOPROXY ENV GO111MODULE=on # Use a proxy for go modules to reduce the likelihood of various hosts being down and breaking the build ENV GOPROXY=${GOPROXY} # kubebuilder test bundle is separated from kubebuilder. Need to setup it for CI test. # Using setup-envtest to download envtest binaries RUN go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest && \ mkdir -p /usr/local/kubebuilder/bin && \ ENVTEST_ASSETS_DIR=$(setup-envtest use 1.33.0 --bin-dir /usr/local/kubebuilder/bin -p path) && \ cp -r ${ENVTEST_ASSETS_DIR}/* /usr/local/kubebuilder/bin/ RUN wget --quiet https://github.com/kubernetes-sigs/kubebuilder/releases/download/v3.2.0/kubebuilder_linux_$(go env GOARCH) && \ mv kubebuilder_linux_$(go env GOARCH) /usr/local/kubebuilder/bin/kubebuilder && \ chmod +x /usr/local/kubebuilder/bin/kubebuilder # get controller-tools RUN go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.16.5 # get goimports (the revision is pinned so we don't indiscriminately update, but the particular commit # is not important) RUN go install golang.org/x/tools/cmd/goimports@v0.33.0 # get protoc compiler and golang plugin WORKDIR /root RUN apt-get update && apt-get install -y unzip # protobuf uses bazel cpunames except following # if cpu == "systemz": # cpu = "s390_64" # elif cpu == "aarch64": # cpu = "aarch_64" # elif cpu == "ppc64": # cpu = "ppcle_64" # snippet from: https://github.com/protocolbuffers/protobuf/blob/d445953603e66eb8992a39b4e10fcafec8501f24/protobuf_release.bzl#L18-L24 # cpu names: https://github.com/bazelbuild/platforms/blob/main/cpu/BUILD RUN ARCH=$(go env GOARCH) && \ if [ "$ARCH" = "s390x" ] ; then \ ARCH="s390_64"; \ elif [ "$ARCH" = "arm64" ] ; then \ ARCH="aarch_64"; \ elif [ "$ARCH" = "ppc64le" ] ; then \ ARCH="ppcle_64"; \ elif [ "$ARCH" = "ppc64" ] ; then \ ARCH="ppcle_64"; \ else \ ARCH=$(uname -m); \ fi && echo "ARCH=$ARCH" && \ wget --quiet https://github.com/protocolbuffers/protobuf/releases/download/v25.2/protoc-25.2-linux-$ARCH.zip && \ unzip protoc-25.2-linux-$ARCH.zip; \ rm *.zip && \ mv bin/protoc /usr/bin/protoc && \ mv include/google /usr/include && \ chmod a+x /usr/include/google && \ chmod a+x /usr/include/google/protobuf && \ chmod a+r -R /usr/include/google && \ chmod +x /usr/bin/protoc RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.33.0 \ && go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.3.0 # get goreleaser # goreleaser name template per arch is basically goarch except for amd64 and 386 https://github.com/goreleaser/goreleaser/blob/ec8819a95c5527fae65e5cb41673f5bbc3245fda/.goreleaser.yaml#L167C1-L173C42 # {{- .ProjectName }}_ # {{- title .Os }}_ # {{- if eq .Arch "amd64" }}x86_64 # {{- else if eq .Arch "386" }}i386 # {{- else }}{{ .Arch }}{{ end }} # {{- if .Arm }}v{{ .Arm }}{{ end -}} RUN ARCH=$(go env GOARCH) && \ if [ "$ARCH" = "amd64" ] ; then \ ARCH="x86_64"; \ elif [ "$ARCH" = "386" ] ; then \ ARCH="i386"; \ elif [ "$ARCH" = "ppc64le" ] ; then \ ARCH="ppc64"; \ fi && \ wget --quiet "https://github.com/goreleaser/goreleaser/releases/download/v1.26.2/goreleaser_Linux_$ARCH.tar.gz" && \ tar xvf goreleaser_Linux_$ARCH.tar.gz; \ mv goreleaser /usr/bin/goreleaser && \ chmod +x /usr/bin/goreleaser # get golangci-lint RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.5.0 # install kubectl RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/$(go env GOARCH)/kubectl RUN chmod +x ./kubectl RUN mv ./kubectl /usr/local/bin # Fix the "dubious ownership" issue from git when running goreleaser.sh RUN echo "[safe] \n\t directory = *" > /.gitconfig ================================================ FILE: hack/build-restic.sh ================================================ #!/bin/bash # Copyright 2020 the Velero contributors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -o errexit set -o nounset set -o pipefail # Use /output/usr/bin/ as the default output directory as this # is the path expected by the Velero Dockerfile. output_dir=${OUTPUT_DIR:-/output/usr/bin} restic_bin=${output_dir}/restic build_path=$(dirname "$PWD") if [[ -z "${BIN}" ]]; then echo "BIN must be set" exit 1 fi if [[ "${BIN}" != "velero" ]]; then echo "${BIN} does not need the restic binary" exit 0 fi if [[ -z "${GOOS}" ]]; then echo "GOOS must be set" exit 1 fi if [[ -z "${GOARCH}" ]]; then echo "GOARCH must be set" exit 1 fi if [[ -z "${RESTIC_VERSION}" ]]; then echo "RESTIC_VERSION must be set" exit 1 fi mkdir ${build_path}/restic git clone -b v${RESTIC_VERSION} https://github.com/restic/restic.git ${build_path}/restic pushd ${build_path}/restic git apply /go/src/github.com/vmware-tanzu/velero/hack/fix_restic_cve.txt go run build.go --goos "${GOOS}" --goarch "${GOARCH}" --goarm "${GOARM}" -o ${restic_bin} chmod +x ${restic_bin} popd ================================================ FILE: hack/build.sh ================================================ #!/bin/bash # Copyright 2016 The Kubernetes Authors. # # Modifications Copyright 2020 the Velero contributors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -o errexit set -o nounset set -o pipefail if [[ -z "${PKG}" ]]; then echo "PKG must be set" exit 1 fi if [[ -z "${BIN}" ]]; then echo "BIN must be set" exit 1 fi if [[ -z "${GOOS}" ]]; then echo "GOOS must be set" exit 1 fi if [[ -z "${GOBIN}" ]]; then echo "GOBIN must be set" exit 1 fi if [[ -z "${GOARCH}" ]]; then echo "GOARCH must be set" exit 1 fi if [[ -z "${VERSION}" ]]; then echo "VERSION must be set" exit 1 fi if [[ -z "${REGISTRY}" ]]; then echo "REGISTRY must be set" exit 1 fi if [[ -z "${GIT_SHA}" ]]; then echo "GIT_SHA must be set" exit 1 fi if [[ -z "${GIT_TREE_STATE}" ]]; then echo "GIT_TREE_STATE must be set" exit 1 fi GCFLAGS="" if [[ ${DEBUG:-} = "1" ]]; then GCFLAGS="all=-N -l" fi export CGO_ENABLED=0 LDFLAGS="-X ${PKG}/pkg/buildinfo.Version=${VERSION}" LDFLAGS="${LDFLAGS} -X ${PKG}/pkg/buildinfo.ImageRegistry=${REGISTRY}" LDFLAGS="${LDFLAGS} -X ${PKG}/pkg/buildinfo.GitSHA=${GIT_SHA}" LDFLAGS="${LDFLAGS} -X ${PKG}/pkg/buildinfo.GitTreeState=${GIT_TREE_STATE}" if [[ -z "${OUTPUT_DIR:-}" ]]; then OUTPUT_DIR=. fi OUTPUT=${OUTPUT_DIR}/${BIN} if [[ "${GOOS}" = "windows" ]]; then OUTPUT="${OUTPUT}.exe" fi go build \ -o ${OUTPUT} \ -gcflags "${GCFLAGS}" \ -installsuffix "static" \ -ldflags "${LDFLAGS}" \ ${PKG}/cmd/${BIN} ================================================ FILE: hack/changelog-check.sh ================================================ #!/bin/bash # Copyright 2020 the Velero contributors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set +x if [[ -z "$CI" ]]; then echo "This script is intended to be run only on Github Actions." >&2 exit 1 fi CHANGELOG_PATH='changelogs/unreleased' # https://help.github.com/en/actions/reference/events-that-trigger-workflows#pull-request-event-pull_request # GITHUB_REF is something like "refs/pull/:prNumber/merge" pr_number=$(echo $GITHUB_REF | cut -d / -f 3) change_log_file="${CHANGELOG_PATH}/${pr_number}-*" if ls ${change_log_file} 1> /dev/null 2>&1; then echo "changelog for PR ${pr_number} exists" exit 0 else echo "PR ${pr_number} is missing a changelog. Please refer https://velero.io/docs/main/code-standards/#adding-a-changelog and add a changelog." exit 1 fi ================================================ FILE: hack/ci-check.sh ================================================ #!/usr/bin/env bash # If we're doing push build, as opposed to a PR, always run make ci if [ "$TRAVIS_PULL_REQUEST" == "false" ]; then make ci # Exit script early, returning make ci's error exit $? fi # Only run `make ci` if files outside of the site directory changed in the branch # In a PR build, $TRAVIS_BRANCH is the destination branch. if [[ $(git diff --name-only $TRAVIS_BRANCH | grep --invert-match site/) ]]; then make ci else echo "Skipping make ci since nothing outside of site directory changed." exit 0 fi ================================================ FILE: hack/crd-gen/v1/main.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ // This code embeds the CRD manifests in ../bases in ../crds/crds.go package main import ( "bytes" "compress/gzip" "fmt" "io" "log" "os" "text/template" ) // This is relative to config/crd/crds const goHeaderFile = "../../../../hack/boilerplate.go.txt" const tpl = `{{.GoHeader}} // Code generated by crds_generate.go; DO NOT EDIT. package crds import ( "bytes" "compress/gzip" "io" apiextinstall "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/client-go/kubernetes/scheme" ) var rawCRDs = [][]byte{ {{- range .RawCRDs }} []byte({{ . }}), {{- end }} } var CRDs = crds() func crds() []*apiextv1.CustomResourceDefinition { apiextinstall.Install(scheme.Scheme) decode := scheme.Codecs.UniversalDeserializer().Decode var objs []*apiextv1.CustomResourceDefinition for _, crd := range rawCRDs { gzr, err := gzip.NewReader(bytes.NewReader(crd)) if err != nil { panic(err) } bytes, err := io.ReadAll(gzr) if err != nil { panic(err) } gzr.Close() obj, _, err := decode(bytes, nil, nil) if err != nil { panic(err) } objs = append(objs, obj.(*apiextv1.CustomResourceDefinition)) } return objs } ` type templateData struct { GoHeader string RawCRDs []string } func main() { headerBytes, err := os.ReadFile(goHeaderFile) if err != nil { log.Fatalln(err) } data := templateData{ GoHeader: string(headerBytes), } // This is relative to config/crd/crds manifests, err := os.ReadDir("../bases") if err != nil { log.Fatalln(err) } for _, crd := range manifests { file, err := os.Open("../bases/" + crd.Name()) if err != nil { log.Fatalln(err) } // gzip compress manifest var buf bytes.Buffer gzw := gzip.NewWriter(&buf) if _, err := io.Copy(gzw, file); err != nil { log.Fatalln(err) } file.Close() gzw.Close() data.RawCRDs = append(data.RawCRDs, fmt.Sprintf("%q", buf.Bytes())) } t, err := template.New("crd").Parse(tpl) if err != nil { log.Fatalln(err) } out, err := os.Create("crds.go") if err != nil { log.Fatalln(err) } if err := t.Execute(out, data); err != nil { log.Fatalln(err) } } ================================================ FILE: hack/docker-push.sh ================================================ #!/bin/bash # Copyright 2020 the Velero contributors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # docker-push is invoked by the CI/CD system to deploy docker images to Docker Hub. # It will build images for all commits to main and all git tags. # The highest, non-prerelease semantic version will also be given the `latest` tag. set +x if [[ -z "$CI" ]]; then echo "This script is intended to be run only on Github Actions." >&2 exit 1 fi # Return value is written into HIGHEST HIGHEST="" function highest_release() { # Loop through the tags since pre-release versions come before the actual versions. # Iterate til we find the first non-pre-release # This is not necessarily the most recently made tag; instead, we want it to be the highest semantic version. # The most recent tag could potentially be a lower semantic version, made as a point release for a previous series. # As an example, if v1.3.0 exists and we create v1.2.2, v1.3.0 should still be `latest`. # `git describe --tags $(git rev-list --tags --max-count=1)` would return the most recently made tag. for t in $(git tag -l --sort=-v:refname); do # If the tag has alpha, beta or rc in it, it's not "latest" if [[ "$t" == *"beta"* || "$t" == *"alpha"* || "$t" == *"rc"* ]]; then continue fi HIGHEST="$t" break done } triggeredBy=$(echo $GITHUB_REF | cut -d / -f 2) if [[ "$triggeredBy" == "heads" ]]; then BRANCH=$(echo $GITHUB_REF | cut -d / -f 3) TAG= elif [[ "$triggeredBy" == "tags" ]]; then BRANCH= TAG=$(echo $GITHUB_REF | cut -d / -f 3) fi # if both BRANCH and TAG are empty, then it's triggered by PR. Use target branch instead. # BRANCH is needed in docker buildx command to set as image tag. # When action is triggered by PR, just build container without pushing, so set type to local. # When action is triggered by PUSH, need to push container, so set type to registry. if [[ -z $BRANCH && -z $TAG ]]; then echo "Test Velero container build without pushing, when Dockerfile is changed by PR." BRANCH="${GITHUB_BASE_REF}-container" OUTPUT_TYPE="tar" else OUTPUT_TYPE="registry" fi TAG_LATEST=false if [[ ! -z "$TAG" ]]; then echo "We're building tag $TAG" VERSION="$TAG" # Explicitly checkout tags when building from a git tag. # This is not needed when building from main git fetch --tags # Calculate the latest release if there's a tag. highest_release if [[ "$TAG" == "$HIGHEST" ]]; then TAG_LATEST=true fi else echo "We're on branch $BRANCH" VERSION="$BRANCH" if [[ "$VERSION" == release-* ]]; then VERSION=${VERSION}-dev fi fi if [[ -z "$BUILD_OS" ]]; then BUILD_OS="linux,windows" fi if [[ -z "$BUILD_ARCH" ]]; then BUILD_ARCH="amd64,arm64" fi # Debugging info echo "Highest tag found: $HIGHEST" echo "BRANCH: $BRANCH" echo "TAG: $TAG" echo "TAG_LATEST: $TAG_LATEST" echo "VERSION: $VERSION" echo "BUILD_OS: $BUILD_OS" echo "BUILD_ARCH: $BUILD_ARCH" echo "Building and pushing container images." VERSION="$VERSION" \ TAG_LATEST="$TAG_LATEST" \ BUILD_OS="$BUILD_OS" \ BUILD_ARCH="$BUILD_ARCH" \ BUILD_OUTPUT_TYPE=$OUTPUT_TYPE \ make all-containers ================================================ FILE: hack/fix_restic_cve.txt ================================================ diff --git a/go.mod b/go.mod index 5f939c481..f6205aa3c 100644 --- a/go.mod +++ b/go.mod @@ -24,32 +24,31 @@ require ( github.com/restic/chunker v0.4.0 github.com/spf13/cobra v1.6.1 github.com/spf13/pflag v1.0.5 - golang.org/x/crypto v0.5.0 - golang.org/x/net v0.5.0 - golang.org/x/oauth2 v0.4.0 - golang.org/x/sync v0.1.0 - golang.org/x/sys v0.4.0 - golang.org/x/term v0.4.0 - golang.org/x/text v0.6.0 - google.golang.org/api v0.106.0 + golang.org/x/crypto v0.45.0 + golang.org/x/net v0.47.0 + golang.org/x/oauth2 v0.28.0 + golang.org/x/sync v0.18.0 + golang.org/x/sys v0.38.0 + golang.org/x/term v0.37.0 + golang.org/x/text v0.31.0 + google.golang.org/api v0.114.0 ) require ( - cloud.google.com/go v0.108.0 // indirect - cloud.google.com/go/compute v1.15.1 // indirect - cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v0.10.0 // indirect + cloud.google.com/go v0.110.0 // indirect + cloud.google.com/go/compute/metadata v0.3.0 // indirect + cloud.google.com/go/iam v0.13.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/dnaeon/go-vcr v1.2.0 // indirect github.com/dustin/go-humanize v1.0.0 // indirect github.com/felixge/fgprof v0.9.3 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/google/pprof v0.0.0-20230111200839-76d1ae5aea2b // indirect github.com/google/uuid v1.3.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.2.1 // indirect - github.com/googleapis/gax-go/v2 v2.7.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect + github.com/googleapis/gax-go/v2 v2.7.1 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.2.3 // indirect @@ -63,11 +62,13 @@ require ( go.opencensus.io v0.24.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect - google.golang.org/grpc v1.52.0 // indirect - google.golang.org/protobuf v1.28.1 // indirect + google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect + google.golang.org/grpc v1.56.3 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) -go 1.18 +go 1.24.0 + +toolchain go1.24.11 diff --git a/go.sum b/go.sum index 026e1d2fa..4a37e7ac7 100644 --- a/go.sum +++ b/go.sum @@ -1,23 +1,24 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.108.0 h1:xntQwnfn8oHGX0crLVinvHM+AhXvi3QHQIEcX/2hiWk= -cloud.google.com/go v0.108.0/go.mod h1:lNUfQqusBJp0bgAg6qrHgYFYbTB+dOiob1itwnlD33Q= -cloud.google.com/go/compute v1.15.1 h1:7UGq3QknM33pw5xATlpzeoomNxsacIVvTqTTvbfajmE= -cloud.google.com/go/compute v1.15.1/go.mod h1:bjjoF/NtFUrkD/urWfdHaKuOPDR5nWIs63rR+SXhcpA= -cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= -cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= -cloud.google.com/go/iam v0.10.0 h1:fpP/gByFs6US1ma53v7VxhvbJpO2Aapng6wabJ99MuI= -cloud.google.com/go/iam v0.10.0/go.mod h1:nXAECrMt2qHpF6RZUZseteD6QyanL68reN4OXPw0UWM= -cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs= +cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys= +cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= +cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= +cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= +cloud.google.com/go/iam v0.13.0 h1:+CmB+K0J/33d0zSQ9SlFWUeCCEn5XJA0ZMZ3pHE9u8k= +cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= +cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM= +cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo= cloud.google.com/go/storage v1.28.1 h1:F5QDG5ChchaAVQhINh24U99OWHURqrW8OmQcGKXcbgI= cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5ogcBBKhU86Y= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0 h1:VuHAcMq8pU1IWNT/m5yRaGqbK0BiQKHT8X4DTp9CHdI= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0/go.mod h1:tZoQYdDZNOiIjdSn0dVWVfl0NEPGOJqVLzSrcFk4Is0= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 h1:QkAcEIAKbNL4KoFr4SathZPhDhF4mVwpBMFlYjyAqy8= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0/go.mod h1:bhXu1AjYL+wutSL/kpSq6s7733q2Rb0yuot9Zgfqa/0= github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 h1:+5VZ72z0Qan5Bog5C+ZkgSqUbeVUd9wgtHOrIKuc5b8= github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 h1:BMTdr+ib5ljLa9MxTJK8x/Ds0MbBb4MfuW5BL0zMJnI= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1/go.mod h1:c6WvOhtmjNUWbLfOG1qxM/q0SPvQNSVJvolm+C52dIU= github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1 h1:BWe8a+f/t+7KY7zH2mqygeUD0t8hNFXe08p1Pb3/jKE= +github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1/go.mod h1:Vt9sXTKwMyGcOxSmLDMnGPgqsUg7m8pe215qMLrDXw4= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74= github.com/anacrolix/fuse v0.2.0 h1:pc+To78kI2d/WUjIyrsdqeJQAesuwpGxlI3h1nAv3Do= @@ -54,6 +55,7 @@ github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNu github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c= +github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= @@ -70,8 +72,8 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -82,17 +84,18 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ= +github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= +github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= github.com/google/pprof v0.0.0-20230111200839-76d1ae5aea2b h1:8htHrh2bw9c7Idkb7YNac+ZpTqLMjRpI+FWu51ltaQc= github.com/google/pprof v0.0.0-20230111200839-76d1ae5aea2b/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.2.1 h1:RY7tHKZcRlk788d5WSo/e83gOyyy742E8GSs771ySpg= -github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= -github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ= -github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= +github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k= +github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= +github.com/googleapis/gax-go/v2 v2.7.1 h1:gF4c0zjUP2H/s/hEGyLA3I0fA2ZWjzYiONAD6cvPr8A= +github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= github.com/hashicorp/golang-lru/v2 v2.0.1 h1:5pv5N1lT1fjLg2VQ5KWc7kmucp2x/kvFOnxuVTqZ6x4= github.com/hashicorp/golang-lru/v2 v2.0.1/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= @@ -114,6 +117,7 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kurin/blazer v0.5.4-0.20211030221322-ba894c124ac6 h1:nz7i1au+nDzgExfqW5Zl6q85XNTvYoGnM5DHiQC0yYs= github.com/kurin/blazer v0.5.4-0.20211030221322-ba894c124ac6/go.mod h1:4FCXMUWo9DllR2Do4TtBd377ezyAJ51vB5uTBjt0pGU= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= github.com/minio/minio-go/v7 v7.0.46 h1:Vo3tNmNXuj7ME5qrvN4iadO7b4mzu/RSFdUkUhaPldk= @@ -129,6 +133,7 @@ github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3P github.com/ncw/swift/v2 v2.0.1 h1:q1IN8hNViXEv8Zvg3Xdis4a3c4IlIGezkYz09zQL5J0= github.com/ncw/swift/v2 v2.0.1/go.mod h1:z0A9RVdYPjNjXVo2pDOPxZ4eu3oarO1P91fTItcb+Kg= github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 h1:Qj1ukM4GlMWXNdMBuXcXfz/Kw9s1qm0CLY32QxuSImI= +github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.7.0 h1:hnbDkaNWPCLMO9wGLdBFTIZvzDrDfBM2072E1S9gJkA= @@ -172,8 +177,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE= -golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= +golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= +golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -189,17 +194,17 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw= -golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.4.0 h1:NF0gk8LVPg1Ml7SSbGyySuoxdsXitj7TvgvuRxIMc/M= -golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= +golang.org/x/oauth2 v0.28.0 h1:CrgCKl8PPAVtLnU3c+EDw6x11699EWlsDeWNWKdIOkc= +golang.org/x/oauth2 v0.28.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= +golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -214,17 +219,17 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= -golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.4.0 h1:O7UWfv5+A2qiuulQk30kVinPoMtoIPeVaKLEgLpVkvg= -golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= +golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= +golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k= -golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -237,8 +242,8 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -google.golang.org/api v0.106.0 h1:ffmW0faWCwKkpbbtvlY/K/8fUl+JKvNS5CVzRoyfCv8= -google.golang.org/api v0.106.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= +google.golang.org/api v0.114.0 h1:1xQPji6cO2E2vLiI+C/XiFAnsn1WV3mjaEwGLhi3grE= +google.golang.org/api v0.114.0/go.mod h1:ifYI2ZsFK6/uGddGfAD5BMxlnkBqCmqHSDUVi45N5Yg= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= @@ -246,15 +251,15 @@ google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCID google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w= -google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.52.0 h1:kd48UiU7EHsV4rnLyOJRuP/Il/UHE7gdDAQ+SZI7nZk= -google.golang.org/grpc v1.52.0/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY= +google.golang.org/grpc v1.56.3 h1:8I4C0Yq1EjstUzUJzpcRVbuYA2mODtEmpWiQoN/b2nc= +google.golang.org/grpc v1.56.3/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -266,14 +271,15 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= ================================================ FILE: hack/issue-template-gen/main.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ // This code renders the IssueTemplate string in pkg/cmd/cli/bug/bug.go to // .github/ISSUE_TEMPLATE/bug_report.md via the hack/update-generated-issue-template.sh script. package main import ( "log" "os" "text/template" "github.com/vmware-tanzu/velero/pkg/cmd/cli/bug" ) func main() { outTemplateFilename := os.Args[1] outFile, err := os.OpenFile(outTemplateFilename, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) if err != nil { log.Fatal(err) } defer outFile.Close() tmpl, err := template.New("ghissue").Parse(bug.IssueTemplate) if err != nil { log.Fatal(err) } err = tmpl.Execute(outFile, bug.VeleroBugInfo{}) if err != nil { log.Fatal(err) } } ================================================ FILE: hack/lint.sh ================================================ #!/bin/bash # # Copyright 2020 the Velero contributors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Printing out cache status golangci-lint cache status # Enable GL_DEBUG line below for debug messages for golangci-lint # export GL_DEBUG=loader,gocritic,env CMD="golangci-lint run" echo "Running $CMD" eval $CMD ================================================ FILE: hack/release-tools/brew-update.sh ================================================ #!/usr/bin/env bash # This script assumes that 2 environment variables are defined outside of it: # VELERO_VERSION - a full version version string, starting with v. example: v1.4.2 # HOMEBREW_GITHUB_API_TOKEN - the GitHub API token that the brew command will use to create a PR on the user's behalf. # Check if brew is found on the user's $PATH; exit if not. if [ -z $(which brew) ]; then echo "Homebrew must first be installed to use this script!" exit 1 fi # GitHub URL which contains the source code archive for the tagged release URL=https://github.com/vmware-tanzu/velero/archive/$VELERO_VERSION.tar.gz # Update brew so we're sure we have the latest Velero formula brew update # Invoke brew's helper function, which will run all their tests and end up opening a browser with the resulting PR. brew bump-formula-pr velero --url=$URL ================================================ FILE: hack/release-tools/changelog.sh ================================================ #!/bin/bash # Copyright 2018 the Velero contributors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -o errexit set -o nounset set -o pipefail function join { local IFS="$1"; shift; echo "$*"; } CHANGELOG_PATH='changelogs/unreleased' UNRELEASED=$(ls -t ${CHANGELOG_PATH}) echo -e "Generating CHANGELOG markdown from ${CHANGELOG_PATH}\n" for entry in $UNRELEASED do IFS=$'-' read -ra pruser <<<"$entry" contents=$(cat ${CHANGELOG_PATH}/${entry}) pr=${pruser[0]} user=$(join '-' ${pruser[@]:1}) echo " * ${contents} (#${pr}, @${user})" done echo -e "\nCopy and paste the list above in to the appropriate CHANGELOG file." echo "Be sure to run: git rm ${CHANGELOG_PATH}/*" ================================================ FILE: hack/release-tools/chk_version.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package main import ( "fmt" "os" "regexp" ) // This regex should match both our GA format (example: v1.4.3) and pre-release formats (v1.2.4-beta.2, v1.5.0-rc.1) // The following sub-capture groups are defined: // // major // minor // patch // prerelease (this will be alpha/beta/rc followed by a ".", followed by 1 or more digits (alpha.5) var releaseRegex = regexp.MustCompile(`^v(?P[[:digit:]]+)\.(?P[[:digit:]]+)\.(?P[[:digit:]]+)(-{1}(?P(alpha|beta|rc)\.[[:digit:]]+))*`) // This small program exists because checking the VELERO_VERSION rules in bash is difficult, and difficult to test for correctness. // Calling it with --verify will verify whether or not the VELERO_VERSION environment variable is a valid version string, without parsing for its components. // Calling it without --verify will try to parse the version into its component pieces. func main() { veleroVersion := os.Getenv("VELERO_VERSION") submatches := reSubMatchMap(releaseRegex, veleroVersion) // Didn't match the regex, exit. if len(submatches) == 0 { fmt.Printf("VELERO_VERSION of %s was not valid. Please correct the value and retry.", veleroVersion) os.Exit(1) } if len(os.Args) > 1 && os.Args[1] == "--verify" { os.Exit(0) } // Send these in a bash variable format to stdout, so that they can be consumed by bash scripts that call the go program. fmt.Printf("VELERO_MAJOR=%s\n", submatches["major"]) fmt.Printf("VELERO_MINOR=%s\n", submatches["minor"]) fmt.Printf("VELERO_PATCH=%s\n", submatches["patch"]) fmt.Printf("VELERO_PRERELEASE=%s\n", submatches["prerelease"]) } // reSubMatchMap returns a map with the named submatches within a regular expression populated as keys, and their matched values within a given string as values. // If no matches are found, a nil map is returned func reSubMatchMap(r *regexp.Regexp, s string) map[string]string { match := r.FindStringSubmatch(s) submatches := make(map[string]string) if len(match) == 0 { return submatches } for i, name := range r.SubexpNames() { // 0 will always be empty from the return values of SubexpNames's documentation, so skip it. if i != 0 { submatches[name] = match[i] } } return submatches } ================================================ FILE: hack/release-tools/chk_version_test.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package main import ( "fmt" "testing" ) func TestRegexMatching(t *testing.T) { tests := []struct { version string expectMatch bool }{ { version: "v1.4.0", expectMatch: true, }, { version: "v2.0.0", expectMatch: true, }, { version: "v1.5.0-alpha.1", expectMatch: true, }, { version: "v1.16.1320-beta.14", expectMatch: true, }, { version: "1.0.0", expectMatch: false, }, { // this is true because while the "--" is invalid, v1.0.0 is a valid part of the regex version: "v1.0.0--beta.1", expectMatch: true, }, } for _, test := range tests { name := fmt.Sprintf("Testing version string %s", test.version) t.Run(name, func(t *testing.T) { results := reSubMatchMap(releaseRegex, test.version) if len(results) == 0 && test.expectMatch { t.Fail() } if len(results) > 0 && !test.expectMatch { fmt.Printf("%v", results) t.Fail() } }) } } ================================================ FILE: hack/release-tools/gen-docs.sh ================================================ #!/bin/bash # Copyright 2020 the Velero contributors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # gen-docs.sh is used for the "make gen-docs" target. It generates a new # versioned docs directory under site/content/docs. It follows # the following process: # 1. Copies the contents of the most recently tagged docs directory into the new # directory, to establish a useful baseline to diff against. # 2. Adds all copied content from step 1 to git's staging area via 'git add'. # 3. Replaces the contents of the new docs directory with the contents of the # 'main' docs directory, updating any version-specific links (e.g. to a # specific branch of the GitHub repository) to use the new version # 4. Copies the previous version's ToC file and runs 'git add' to establish # a useful baseline to diff against. # 5. Replaces the content of the new ToC file with the main ToC. # 6. Update site/config.yaml and site/_data/toc-mapping.yml to include entries # for the new version. # # The unstaged changes in the working directory can now easily be diff'ed against the # staged changes using 'git diff' to review all docs changes made since the previous # tagged version. Once the unstaged changes are ready, they can be added to the # staging area using 'git add' and then committed. # # NEW_DOCS_VERSION defines the version that the docs will be tagged with # (i.e. what’s in the URL, what shows up in the version dropdown on the site). # This should be formatted as either v1.4 (for any GA release, including minor), or v1.5.0-beta.1/v1.5.0-rc.1 (for an alpha/beta/RC). # To run gen-docs: "VELERO_VERSION=v1.4.0 NEW_DOCS_VERSION=v1.4 PREVIOUS_DOCS_VERSION= make gen-docs" # Note: if PREVIOUS_DOCS_VERSION is not set, the script will copy from the # latest version. # # **NOTE**: there are additional manual steps required to finalize the process of generating # a new versioned docs site. The full process is documented in site/README-HUGO.md set -o errexit set -o nounset set -o pipefail DOCS_DIRECTORY=site/content/docs DATA_DOCS_DIRECTORY=site/data/docs CONFIG_FILE=site/config.yaml MAIN_BRANCH=main # don't run if there's already a directory for the target docs version if [[ -d $DOCS_DIRECTORY/$NEW_DOCS_VERSION ]]; then echo "ERROR: $DOCS_DIRECTORY/$NEW_DOCS_VERSION already exists" exit 1 fi # get the alphabetically last item in $DOCS_DIRECTORY to use as PREVIOUS_DOCS_VERSION # if not explicitly specified by the user if [[ -z "${PREVIOUS_DOCS_VERSION:-}" ]]; then echo "PREVIOUS_DOCS_VERSION was not specified, getting the latest version" PREVIOUS_DOCS_VERSION=$(ls -1 $DOCS_DIRECTORY/ | tail -n 1) fi # make a copy of the previous versioned docs dir echo "Creating copy of docs directory $DOCS_DIRECTORY/$PREVIOUS_DOCS_VERSION in $DOCS_DIRECTORY/$NEW_DOCS_VERSION" cp -r $DOCS_DIRECTORY/${PREVIOUS_DOCS_VERSION}/ $DOCS_DIRECTORY/${NEW_DOCS_VERSION}/ # 'git add' the previous version's docs as-is so we get a useful diff when we copy the $MAIN_BRANCH docs in echo "Running 'git add' for previous version's doc contents to use as a base for diff" git add -f $DOCS_DIRECTORY/${NEW_DOCS_VERSION} # now copy the contents of $DOCS_DIRECTORY/$MAIN_BRANCH into the same directory so we can get a nice # git diff of what changed since previous version echo "Copying $DOCS_DIRECTORY/$MAIN_BRANCH/ to $DOCS_DIRECTORY/${NEW_DOCS_VERSION}/" rm -rf $DOCS_DIRECTORY/${NEW_DOCS_VERSION}/ && cp -r $DOCS_DIRECTORY/$MAIN_BRANCH/ $DOCS_DIRECTORY/${NEW_DOCS_VERSION}/ # make a copy of the previous versioned ToC NEW_DOCS_TOC="$(echo ${NEW_DOCS_VERSION} | tr . -)-toc" PREVIOUS_DOCS_TOC="$(echo ${PREVIOUS_DOCS_VERSION} | tr . -)-toc" echo "Creating copy of $DATA_DOCS_DIRECTORY/$PREVIOUS_DOCS_TOC.yml at $DATA_DOCS_DIRECTORY/$NEW_DOCS_TOC.yml" cp $DATA_DOCS_DIRECTORY/$PREVIOUS_DOCS_TOC.yml $DATA_DOCS_DIRECTORY/$NEW_DOCS_TOC.yml # 'git add' the previous version's ToC content as-is so we get a useful diff when we copy the $MAIN_BRANCH ToC in echo "Running 'git add' for previous version's ToC to use as a base for diff" git add $DATA_DOCS_DIRECTORY/$NEW_DOCS_TOC.yml # now copy the $MAIN_BRANCH ToC so we can get a nice git diff of what changed since previous version echo "Copying $DATA_DOCS_DIRECTORY/$MAIN_BRANCH-toc.yml to $DATA_DOCS_DIRECTORY/$NEW_DOCS_TOC.yml" rm $DATA_DOCS_DIRECTORY/$NEW_DOCS_TOC.yml && cp $DATA_DOCS_DIRECTORY/$MAIN_BRANCH-toc.yml $DATA_DOCS_DIRECTORY/$NEW_DOCS_TOC.yml # replace known version-specific links -- the sed syntax is slightly different in OS X and Linux, # so check which OS we're running on. if [[ $(uname) == "Darwin" ]]; then echo "[OS X] updating version-specific links" find $DOCS_DIRECTORY/${NEW_DOCS_VERSION} -type f -name "*.md" | xargs sed -i '' "s|https://velero.io/docs/$MAIN_BRANCH|https://velero.io/docs/$VELERO_VERSION|g" find $DOCS_DIRECTORY/${NEW_DOCS_VERSION} -type f -name "*.md" | xargs sed -i '' "s|https://github.com/vmware-tanzu/velero/blob/$MAIN_BRANCH|https://github.com/vmware-tanzu/velero/blob/$VELERO_VERSION|g" find $DOCS_DIRECTORY/${NEW_DOCS_VERSION} -type f -name "_index.md" | xargs sed -i '' "s|version: $MAIN_BRANCH|version: $NEW_DOCS_VERSION|g" echo "[OS X] Updating latest version in $CONFIG_FILE" sed -i '' "s/latest: ${PREVIOUS_DOCS_VERSION}/latest: ${NEW_DOCS_VERSION}/" $CONFIG_FILE # newlines and lack of indentation are requirements for this sed syntax # which is doing an append echo "[OS X] Adding latest version to versions list in $CONFIG_FILE" sed -i '' "/- $MAIN_BRANCH/a\\ \ \ \ \ - ${NEW_DOCS_VERSION} " $CONFIG_FILE echo "[OS X] Adding ToC mapping entry" sed -i '' "/$MAIN_BRANCH: $MAIN_BRANCH-toc/a\\ ${NEW_DOCS_VERSION}: ${NEW_DOCS_TOC} " $DATA_DOCS_DIRECTORY/toc-mapping.yml else echo "[Linux] updating version-specific links" find $DOCS_DIRECTORY/${NEW_DOCS_VERSION} -type f -name "*.md" | xargs sed -i'' "s|https://velero.io/docs/$MAIN_BRANCH|https://velero.io/docs/$VELERO_VERSION|g" find $DOCS_DIRECTORY/${NEW_DOCS_VERSION} -type f -name "*.md" | xargs sed -i'' "s|https://github.com/vmware-tanzu/velero/blob/$MAIN_BRANCH|https://github.com/vmware-tanzu/velero/blob/$VELERO_VERSION|g" echo "[Linux] Updating latest version in $CONFIG_FILE" sed -i'' "s/latest: ${PREVIOUS_DOCS_VERSION}/latest: ${NEW_DOCS_VERSION}/" $CONFIG_FILE echo "[Linux] Adding latest version to versions list in $CONFIG_FILE" sed -i'' "/- $MAIN_BRANCH/a - ${NEW_DOCS_VERSION}" $CONFIG_FILE echo "[Linux] Adding ToC mapping entry" sed -i'' "/$MAIN_BRANCH: $MAIN_BRANCH-toc/a ${NEW_DOCS_VERSION}: ${NEW_DOCS_TOC}" $DATA_DOCS_DIRECTORY/toc-mapping.yml fi echo "Success! $DOCS_DIRECTORY/$NEW_DOCS_VERSION has been created." echo "" echo "The next steps are:" echo " 1. Consult site/README-HUGO.md for further manual steps required to finalize the new versioned docs generation." echo " 2. Run a 'git diff' to review all changes made to the docs since the previous version." echo " 3. Make any manual changes/corrections necessary." echo " 4. Run 'git add' to stage all unstaged changes, then 'git commit'." ================================================ FILE: hack/release-tools/goreleaser.sh ================================================ #!/bin/bash # Copyright 2018 the Velero contributors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -o errexit set -o nounset set -o pipefail if [[ -z "${GITHUB_TOKEN}" ]]; then echo "GITHUB_TOKEN must be set" exit 1 fi # TODO derive this from the major+minor version if [ -z "${RELEASE_NOTES_FILE}" ]; then echo "RELEASE_NOTES_FILE must be set" exit 1 fi if [ -z "${REGISTRY}" ]; then echo "REGISTRY must be set" exit 1 fi GIT_DIRTY=$(git status --porcelain 2> /dev/null) if [[ -z "${GIT_DIRTY}" ]]; then export GIT_TREE_STATE=clean else export GIT_TREE_STATE=dirty fi # Verify .goreleaser.yml format first. echo "Start to verify .goreleaser.yml format" goreleaser check # $PUBLISH must explicitly be set to 'TRUE' for goreleaser # to publish the release to GitHub. if [[ "${PUBLISH:-}" != "TRUE" ]]; then echo "Not set to publish" goreleaser release \ --clean \ --release-notes="${RELEASE_NOTES_FILE}" \ --snapshot # Generate an unversioned snapshot release, skipping all validations and without publishing any artifacts (implies --skip-publish, --skip-announce and --skip-validate) else echo "Getting ready to publish" goreleaser release \ --clean \ --release-notes="${RELEASE_NOTES_FILE}" fi ================================================ FILE: hack/release-tools/tag-release.sh ================================================ #!/usr/bin/env bash # Copyright 2020 the Velero contributors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # This script will do the necessary checks and actions to create a release of Velero. It will: # - validate that all prerequisites are met # - verify the version string is what the user expects. # - create a git tag # - push the created git tag to GitHub # - run GoReleaser # The following variables are needed: # - $VELERO_VERSION: defines the tag of Velero that any https://github.com/vmware-tanzu/velero/... # links in the docs should redirect to. # - $REMOTE: defines the remote that should be used when pushing tags and branches. Defaults to "upstream" # - $publish: TRUE/FALSE value where FALSE (or not including it) will indicate a dry-run, and TRUE, or simply adding 'publish', # will tag the release with the $VELERO_VERSION and push the tag to a remote named 'upstream'. # - $GITHUB_TOKEN: Needed to run the goreleaser process to generate a GitHub release. # Use https://github.com/settings/tokens/new?scopes=repo if you don't already have a token. # Regenerate an existing token: https://github.com/settings/tokens. # You may regenerate the token for every release if you prefer. # See https://goreleaser.com/environment/ for more details. # This script is meant to be a combination of documentation and executable. # If you have questions at any point, please stop and ask! # Fail on any error. set -eo pipefail # Directory in which the script itself resides, so we can use it for calling programs that are in the same directory. DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" # Default to using upstream as the remote remote=${REMOTE:-upstream} # Parse out the branch we're on so we can switch back to it at the end of a dry-run, where we delete the tag. Requires git v1.8.1+ upstream_branch=$(git symbolic-ref --short HEAD) function tag_and_push() { echo "Tagging $VELERO_VERSION" git tag $VELERO_VERSION || true if [[ $publish == "TRUE" ]]; then echo "Pushing $VELERO_VERSION" git push "$remote" $VELERO_VERSION fi } # Default to a dry-run mode publish=FALSE if [[ "$1" = "publish" ]]; then publish=TRUE fi # For now, have the person doing the release pass in the VELERO_VERSION variable as an environment variable. # In the future, we might be able to inspect git via `git describe --abbrev=0` to get a hint for it. if [[ -z "$VELERO_VERSION" ]]; then printf "The \$VELERO_VERSION environment variable is not set. Please set it with\n\texport VELERO_VERSION=v\nthen try again." exit 1 fi # Make sure the user's provided their github token, so we can give it to goreleaser. if [[ -z "$GITHUB_TOKEN" ]]; then printf "The GITHUB_TOKEN environment variable is not set. Please set it with\n\t export GITHUB_TOKEN=\n then try again." exit 1 fi # Ensure that we have a clean working tree before we let any changes happen, especially important for cutting release branches. if [[ -n $(git status --short) ]]; then echo "Your git working directory is dirty! Please clean up untracked files and stash any changes before proceeding." exit 3 fi # Make sure that there's no issue with the environment variable's format before trying to eval the parsed version. if ! go run $DIR/chk_version.go --verify; then exit 2 fi # Since we're past the validation of the VELERO_VERSION, parse the version's individual components. eval $(go run $DIR/chk_version.go) printf "To clarify, you've provided a version string of $VELERO_VERSION.\n" printf "Based on this, the following assumptions have been made: \n" # $VELERO_PATCH gets populated by the chk_version.go script that parses and verifies the given version format # If we've got a patch release, we assume the tag is on release branch. if [[ "$VELERO_PATCH" != 0 ]]; then printf "*\t This is a patch release.\n" ON_RELEASE_BRANCH=TRUE fi # $VELERO_PRERELEASE gets populated by the chk_version.go script that parses and verifies the given version format # If we've got a GA release, we assume the tag is on release branch. # -n is "string is non-empty" [[ -n $VELERO_PRERELEASE ]] && printf "*\t This is a pre-release.\n" # -z is "string is empty" if [[ -z $VELERO_PRERELEASE ]]; then printf "*\t This is a GA release.\n" ON_RELEASE_BRANCH=TRUE fi if [[ "$ON_RELEASE_BRANCH" == "TRUE" ]]; then release_branch_name=release-$VELERO_MAJOR.$VELERO_MINOR printf "*\t The commit to tag is on branch: %s. Please make sure this branch has been created.\n" $release_branch_name fi if [[ $publish == "TRUE" ]]; then echo "If this is all correct, press enter/return to proceed to TAG THE RELEASE and UPLOAD THE TAG TO GITHUB." else echo "If this is all correct, press enter/return to proceed to TAG THE RELEASE and PROCEED WITH THE DRY-RUN." fi echo "Otherwise, press ctrl-c to CANCEL the process without making any changes." read -p "Ready to continue? " echo "Alright, let's go." echo "Pulling down all git tags and branches before doing any work." git fetch "$remote" --tags if [[ -n $release_branch_name ]]; then # Tag on release branch remote_release_branch_name="$remote/$release_branch_name" # Determine whether the local and remote release branches already exist local_branch=$(git branch | { grep "$release_branch_name" || true; }) remote_branch=$(git branch -r | { grep "$remote_release_branch_name" || true;}) if [[ -z $remote_branch ]]; then echo "The branch $remote_release_branch_name must be created before you tag the release." exit 1 fi if [[ -z $local_branch ]]; then # Remote branch exists, but does not exist locally. Checkout and track the remote branch. git checkout --track "$remote_release_branch_name" else # Checkout the local release branch and ensure it is up to date with the remote git checkout "$release_branch_name" git pull --set-upstream "$remote" "$release_branch_name" fi tag_and_push else echo "Checking out $remote/main." git checkout "$remote"/main tag_and_push fi echo "Invoking Goreleaser to create the GitHub release." RELEASE_NOTES_FILE=changelogs/CHANGELOG-$VELERO_MAJOR.$VELERO_MINOR.md \ PUBLISH=$publish \ make release if [[ $publish == "FALSE" ]]; then # Delete the local tag so we don't potentially conflict when it's re-run for real. # This also means we won't have to just ignore existing tags in tag_and_push, which could be a problem if there's an existing tag. echo "Dry run complete. Deleting git tag $VELERO_VERSION" git checkout $upstream_branch git tag -d $VELERO_VERSION fi ================================================ FILE: hack/test.sh ================================================ #!/bin/bash # Copyright 2016 The Kubernetes Authors. # Modifications Copyright 2020 The Velero Contributors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -o errexit set -o nounset set -o pipefail export CGO_ENABLED=0 TARGETS=($(go list ./pkg/... ./internal/...| grep -vE "/pkg/builder|pkg/apis|pkg/test|pkg/generated|pkg/plugin/generated|mocks|internal/restartabletest")) TARGETS+=( ./cmd/... ) if [[ ${#@} -ne 0 ]]; then TARGETS=("$@") fi echo "Running all short tests in:" "${TARGETS[@]}" if [[ -n "${GOFLAGS:-}" ]]; then echo "GOFLAGS: ${GOFLAGS}" fi # After bumping up "sigs.k8s.io/controller-runtime" to v0.10.2, get the error "panic: mkdir /.cache/kubebuilder-envtest: permission denied" # when running this script with "make test" command. This is caused by that "make test" runs inside a container with user and group specified, # but the user and group don't exist inside the container, when the code(https://github.com/kubernetes-sigs/controller-runtime/blob/v0.10.2/pkg/internal/testing/addr/manager.go#L44) # tries to get the cache directory, it gets the directory "/" and then get the permission error when trying to create directory under "/". # Specifying the cache directory by environment variable "XDG_CACHE_HOME" to workaround it XDG_CACHE_HOME=/tmp/ go test -installsuffix "static" -short -timeout 1200s -coverprofile=coverage.out "${TARGETS[@]}" echo "Success!" ================================================ FILE: hack/update-1fmt.sh ================================================ #!/bin/bash # # Copyright 2017 the Velero contributors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -o errexit set -o nounset set -o pipefail if [[ ${1:-} == '--verify' ]]; then # List file diffs that need formatting updates MODE='-d' ACTION='Verifying' else # Write formatting updates to files MODE='-w' ACTION='Updating' fi if ! command -v goimports > /dev/null; then echo 'goimports is missing - please run "go get golang.org/x/tools/cmd/goimports"' exit 1 fi files="$(find . -type f -name '*.go' -not -path './.go/*' -not -path './vendor/*' -not -path './site/*' -not -path './.git/*' -not -path '*/generated/*' -not -name 'zz_generated*' -not -path '*/mocks/*')" echo "${ACTION} gofmt" output=$(gofmt "${MODE}" -s ${files}) if [[ -n "${output}" ]]; then VERIFY_FMT_FAILED=1 echo "${output}" fi if [[ -n "${VERIFY_FMT_FAILED:-}" ]]; then echo "${ACTION} gofmt - failed! Please run 'make update'." else echo "${ACTION} gofmt - done!" fi echo "${ACTION} goimports" output=$(goimports "${MODE}" -local github.com/vmware-tanzu/velero ${files}) if [[ -n "${output}" ]]; then VERIFY_IMPORTS_FAILED=1 echo "${output}" fi if [[ -n "${VERIFY_IMPORTS_FAILED:-}" ]]; then echo "${ACTION} goimports - failed! Please run 'make update'." else echo "${ACTION} goimports - done!" fi if [[ -n "${VERIFY_FMT_FAILED:-}" || -n "${VERIFY_IMPORTS_FAILED:-}" ]]; then exit 1 fi ================================================ FILE: hack/update-2proto.sh ================================================ #!/bin/bash -e # # Copyright 2017, 2019 the Velero contributors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. HACK_DIR=$(dirname "${BASH_SOURCE}") echo "Updating plugin proto" echo protoc --version protoc \ -I pkg/plugin/proto/ \ -I /usr/include \ --go_out=pkg/plugin/generated/ \ --go_opt=module=github.com/vmware-tanzu/velero/pkg/plugin/generated \ --go-grpc_out=pkg/plugin/generated \ --go-grpc_opt=paths=source_relative \ --go-grpc_opt=require_unimplemented_servers=false \ $(find pkg/plugin/proto -name '*.proto') echo "Updating plugin proto - done!" ================================================ FILE: hack/update-3generated-crd-code.sh ================================================ #!/bin/bash # # Copyright the Velero contributors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -o errexit set -o nounset set -o pipefail set -o xtrace # this script expects to be run from the root of the Velero repo. if [[ -z "${GOPATH}" ]]; then GOPATH=~/go fi if ! command -v controller-gen > /dev/null; then echo "controller-gen is missing" exit 1 fi # Generate CRD for v1. controller-gen \ crd:crdVersions=v1 \ paths=./pkg/apis/velero/v1/... \ paths=./pkg/controller/... \ output:crd:artifacts:config=config/crd/v1/bases \ object \ paths=./pkg/apis/velero/v1/... # Generate CRD for v2alpha1. controller-gen \ crd:crdVersions=v1 \ paths=./pkg/apis/velero/v2alpha1/... \ paths=./pkg/controller/... \ output:crd:artifacts:config=config/crd/v2alpha1/bases \ object \ paths=./pkg/apis/velero/v2alpha1/... # Generate RBAC. controller-gen \ paths=./pkg/apis/velero/v1/... \ paths=./pkg/apis/velero/v2alpha1/... \ paths=./pkg/controller/... \ rbac:roleName=velero-perms go generate ./config/crd/v1/crds go generate ./config/crd/v2alpha1/crds ================================================ FILE: hack/update-4generated-issue-template.sh ================================================ #!/bin/bash -e # # Copyright 2018, 2019 the Velero contributors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. VELERO_ROOT=$(dirname ${BASH_SOURCE})/.. BIN=${VELERO_ROOT}/_output/bin mkdir -p ${BIN} echo "Updating generated Github issue template" go build -o ${BIN}/issue-tmpl-gen ./hack/issue-template-gen/main.go if [[ $# -gt 1 ]]; then echo "usage: ${BASH_SOURCE} [OUTPUT_FILE]" exit 1 fi OUTPUT_ISSUE_FILE="$1" if [[ -z "${OUTPUT_ISSUE_FILE}" ]]; then OUTPUT_ISSUE_FILE=${VELERO_ROOT}/.github/ISSUE_TEMPLATE/bug_report.md fi ${BIN}/issue-tmpl-gen ${OUTPUT_ISSUE_FILE} echo "Success!" ================================================ FILE: hack/update-all.sh ================================================ #!/bin/bash -e # # Copyright 2017 the Velero contributors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. HACK_DIR=$(dirname "${BASH_SOURCE}") echo "Running all update scripts" for f in ${HACK_DIR}/update-*.sh; do if [[ $f = "${HACK_DIR}/update-all.sh" ]]; then continue fi $f done ================================================ FILE: hack/verify-all.sh ================================================ #!/bin/bash -e # # Copyright 2017 the Velero contributors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. HACK_DIR=$(dirname "${BASH_SOURCE}") echo "Running all verification scripts" for f in ${HACK_DIR}/verify-*.sh; do if [[ $f = "${HACK_DIR}/verify-all.sh" ]]; then continue fi $f done ================================================ FILE: hack/verify-fmt.sh ================================================ #!/bin/bash # # Copyright 2017 the Velero contributors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. HACK_DIR=$(dirname "${BASH_SOURCE[0]}") "${HACK_DIR}"/update-1fmt.sh --verify ================================================ FILE: hack/verify-generated-crd-code.sh ================================================ #!/bin/bash -e # # Copyright the Velero contributors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. HACK_DIR=$(dirname "${BASH_SOURCE}") ${HACK_DIR}/update-3generated-crd-code.sh # ensure no changes to generated CRDs if ! git diff --exit-code config/crd/v1/crds/crds.go config/crd/v2alpha1/crds/crds.go &> /dev/null; then # revert changes to state before running CRD generation to stay consistent # with code-generator `--verify-only` option which discards generated changes git checkout config/crd echo "CRD verification - failed! Generated CRDs are out-of-date, please run 'make update' and 'git add' the generated file(s)." exit 1 fi ================================================ FILE: hack/verify-generated-issue-template.sh ================================================ #!/bin/bash -e # # Copyright 2018 the Velero contributors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. VELERO_ROOT=$(dirname ${BASH_SOURCE})/.. HACK_DIR=$(dirname "${BASH_SOURCE}") ISSUE_TEMPLATE_FILE=${VELERO_ROOT}/.github/ISSUE_TEMPLATE/bug_report.md OUT_TMP_FILE="$(mktemp -d)"/bug_report.md trap cleanup INT TERM HUP EXIT cleanup() { rm -rf ${TMP_DIR} } echo "Verifying generated Github issue template" ${HACK_DIR}/update-4generated-issue-template.sh ${OUT_TMP_FILE} > /dev/null output=$(echo "`diff ${ISSUE_TEMPLATE_FILE} ${OUT_TMP_FILE}`") if [[ -n "${output}" ]] ; then echo "FAILURE: verification of generated template failed:" echo "${output}" exit 1 fi echo "Success!" ================================================ FILE: internal/credentials/file_store.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package credentials import ( "fmt" "os" "path/filepath" "github.com/pkg/errors" corev1api "k8s.io/api/core/v1" kbclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/pkg/util/filesystem" "github.com/vmware-tanzu/velero/pkg/util/kube" ) // FileStore defines operations for interacting with credentials // that are stored on a file system. type FileStore interface { // Path returns a path on disk where the secret key defined by // the given selector is serialized. Path(selector *corev1api.SecretKeySelector) (string, error) } type namespacedFileStore struct { client kbclient.Client namespace string fsRoot string fs filesystem.Interface } // NewNamespacedFileStore returns a FileStore which can interact with credentials // for the given namespace and will store them under the given fsRoot. func NewNamespacedFileStore(client kbclient.Client, namespace string, fsRoot string, fs filesystem.Interface) (FileStore, error) { fsNamespaceRoot := filepath.Join(fsRoot, namespace) if err := fs.MkdirAll(fsNamespaceRoot, 0755); err != nil { return nil, err } return &namespacedFileStore{ client: client, namespace: namespace, fsRoot: fsNamespaceRoot, fs: fs, }, nil } // Path returns a path on disk where the secret key defined by // the given selector is serialized. func (n *namespacedFileStore) Path(selector *corev1api.SecretKeySelector) (string, error) { creds, err := kube.GetSecretKey(n.client, n.namespace, selector) if err != nil { return "", errors.Wrap(err, "unable to get key for secret") } keyFilePath := filepath.Join(n.fsRoot, fmt.Sprintf("%s-%s", selector.Name, selector.Key)) // owner RW perms, group R perms, no public perms file, err := n.fs.OpenFile(keyFilePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0640) if err != nil { return "", errors.Wrap(err, "unable to open credentials file for writing") } if _, err := file.Write(creds); err != nil { return "", errors.Wrap(err, "unable to write credentials to store") } if err := file.Close(); err != nil { return "", errors.Wrap(err, "unable to close credentials file") } return keyFilePath, nil } ================================================ FILE: internal/credentials/file_store_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package credentials import ( "testing" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" "github.com/vmware-tanzu/velero/pkg/builder" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func TestNamespacedFileStore(t *testing.T) { testCases := []struct { name string namespace string fsRoot string secrets []*corev1api.Secret secretSelector *corev1api.SecretKeySelector wantErr string expectedPath string expectedContents string }{ { name: "returns an error if the secret can't be found", secretSelector: builder.ForSecretKeySelector("non-existent-secret", "secret-key").Result(), wantErr: "unable to get key for secret: secrets \"non-existent-secret\" not found", }, { name: "returns a filepath formed using fsRoot, namespace, secret name and key, with secret contents", namespace: "ns1", fsRoot: "/tmp/credentials", secretSelector: builder.ForSecretKeySelector("credential", "key2").Result(), secrets: []*corev1api.Secret{ builder.ForSecret("ns1", "credential").Data(map[string][]byte{ "key1": []byte("ns1-secretdata1"), "key2": []byte("ns1-secretdata2"), "key3": []byte("ns1-secretdata3"), }).Result(), builder.ForSecret("ns2", "credential").Data(map[string][]byte{ "key2": []byte("ns2-secretdata2"), }).Result(), }, expectedPath: "/tmp/credentials/ns1/credential-key2", expectedContents: "ns1-secretdata2", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { client := velerotest.NewFakeControllerRuntimeClient(t) for _, secret := range tc.secrets { require.NoError(t, client.Create(t.Context(), secret)) } fs := velerotest.NewFakeFileSystem() fileStore, err := NewNamespacedFileStore(client, tc.namespace, tc.fsRoot, fs) require.NoError(t, err) path, err := fileStore.Path(tc.secretSelector) if tc.wantErr != "" { require.EqualError(t, err, tc.wantErr) } else { require.NoError(t, err) } require.Equal(t, tc.expectedPath, path) contents, err := fs.ReadFile(path) require.NoError(t, err) require.Equal(t, []byte(tc.expectedContents), contents) }) } } ================================================ FILE: internal/credentials/getter.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package credentials // CredentialGetter is a collection of interfaces for interacting with credentials // that are stored in different targets type CredentialGetter struct { FromFile FileStore FromSecret SecretStore } ================================================ FILE: internal/credentials/local.go ================================================ package credentials import "os" func DefaultStoreDirectory() string { return os.TempDir() + "/credentials" } ================================================ FILE: internal/credentials/mocks/FileStore.go ================================================ // Code generated by mockery v2.14.0. DO NOT EDIT. package mocks import ( mock "github.com/stretchr/testify/mock" corev1api "k8s.io/api/core/v1" ) // FileStore is an autogenerated mock type for the FileStore type type FileStore struct { mock.Mock } // Path provides a mock function with given fields: selector func (_m *FileStore) Path(selector *corev1api.SecretKeySelector) (string, error) { ret := _m.Called(selector) var r0 string if rf, ok := ret.Get(0).(func(*corev1api.SecretKeySelector) string); ok { r0 = rf(selector) } else { r0 = ret.Get(0).(string) } var r1 error if rf, ok := ret.Get(1).(func(*corev1api.SecretKeySelector) error); ok { r1 = rf(selector) } else { r1 = ret.Error(1) } return r0, r1 } type mockConstructorTestingTNewFileStore interface { mock.TestingT Cleanup(func()) } // NewFileStore creates a new instance of FileStore. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. func NewFileStore(t mockConstructorTestingTNewFileStore) *FileStore { mock := &FileStore{} mock.Mock.Test(t) t.Cleanup(func() { mock.AssertExpectations(t) }) return mock } ================================================ FILE: internal/credentials/mocks/SecretStore.go ================================================ // Code generated by mockery v2.14.0. DO NOT EDIT. package mocks import ( mock "github.com/stretchr/testify/mock" corev1api "k8s.io/api/core/v1" ) // SecretStore is an autogenerated mock type for the SecretStore type type SecretStore struct { mock.Mock } // Get provides a mock function with given fields: selector func (_m *SecretStore) Get(selector *corev1api.SecretKeySelector) (string, error) { ret := _m.Called(selector) var r0 string if rf, ok := ret.Get(0).(func(*corev1api.SecretKeySelector) string); ok { r0 = rf(selector) } else { r0 = ret.Get(0).(string) } var r1 error if rf, ok := ret.Get(1).(func(*corev1api.SecretKeySelector) error); ok { r1 = rf(selector) } else { r1 = ret.Error(1) } return r0, r1 } type mockConstructorTestingTNewSecretStore interface { mock.TestingT Cleanup(func()) } // NewSecretStore creates a new instance of SecretStore. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. func NewSecretStore(t mockConstructorTestingTNewSecretStore) *SecretStore { mock := &SecretStore{} mock.Mock.Test(t) t.Cleanup(func() { mock.AssertExpectations(t) }) return mock } ================================================ FILE: internal/credentials/secret_store.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package credentials import ( "github.com/pkg/errors" corev1api "k8s.io/api/core/v1" kbclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/pkg/util/kube" ) // SecretStore defines operations for interacting with credentials // that are stored in Secret. type SecretStore interface { // Get returns the secret key defined by the given selector Get(selector *corev1api.SecretKeySelector) (string, error) } type namespacedSecretStore struct { client kbclient.Client namespace string } // NewNamespacedSecretStore returns a SecretStore which can interact with credentials // for the given namespace. func NewNamespacedSecretStore(client kbclient.Client, namespace string) (SecretStore, error) { return &namespacedSecretStore{ client: client, namespace: namespace, }, nil } // Buffer returns the secret key defined by the given selector. func (n *namespacedSecretStore) Get(selector *corev1api.SecretKeySelector) (string, error) { creds, err := kube.GetSecretKey(n.client, n.namespace, selector) if err != nil { return "", errors.Wrap(err, "unable to get key for secret") } return string(creds), nil } ================================================ FILE: internal/delete/actions/csi/volumesnapshotcontent_action.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package csi import ( "context" "time" "github.com/google/uuid" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/wait" crclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" plugincommon "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" "github.com/vmware-tanzu/velero/pkg/plugin/velero" "github.com/vmware-tanzu/velero/pkg/util/boolptr" kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube" ) // volumeSnapshotContentDeleteItemAction is a restore item action plugin for Velero type volumeSnapshotContentDeleteItemAction struct { log logrus.FieldLogger crClient crclient.Client } // AppliesTo returns information indicating // VolumeSnapshotContentRestoreItemAction action should be invoked // while restoring VolumeSnapshotContent.snapshot.storage.k8s.io resources func (p *volumeSnapshotContentDeleteItemAction) AppliesTo() (velero.ResourceSelector, error) { return velero.ResourceSelector{ IncludedResources: []string{"volumesnapshotcontents.snapshot.storage.k8s.io"}, }, nil } func (p *volumeSnapshotContentDeleteItemAction) Execute( input *velero.DeleteItemActionExecuteInput, ) error { p.log.Info("Starting VolumeSnapshotContentDeleteItemAction") var snapCont snapshotv1api.VolumeSnapshotContent if err := runtime.DefaultUnstructuredConverter.FromUnstructured( input.Item.UnstructuredContent(), &snapCont, ); err != nil { return errors.Wrapf(err, "failed to convert VolumeSnapshotContent from unstructured") } // We don't want this DeleteItemAction plugin to delete // VolumeSnapshotContent taken outside of Velero. // So skip deleting VolumeSnapshotContent not have the backup name // in its labels. if !kubeutil.HasBackupLabel(&snapCont.ObjectMeta, input.Backup.Name) { p.log.Info( "VolumeSnapshotContent %s was not taken by backup %s, skipping deletion", snapCont.Name, input.Backup.Name, ) return nil } p.log.Infof("Deleting VolumeSnapshotContent %s", snapCont.Name) uuid, err := uuid.NewRandom() if err != nil { p.log.WithError(err).Errorf("Fail to generate the UUID to create VSC %s", snapCont.Name) return errors.Wrapf(err, "Fail to generate the UUID to create VSC %s", snapCont.Name) } snapCont.Name = "vsc-" + uuid.String() snapCont.Spec.DeletionPolicy = snapshotv1api.VolumeSnapshotContentDelete snapCont.Spec.Source = snapshotv1api.VolumeSnapshotContentSource{ SnapshotHandle: snapCont.Status.SnapshotHandle, } snapCont.Spec.VolumeSnapshotRef = corev1api.ObjectReference{ APIVersion: snapshotv1api.SchemeGroupVersion.String(), Kind: "VolumeSnapshot", Namespace: "ns-" + string(snapCont.UID), Name: "name-" + string(snapCont.UID), } snapCont.ResourceVersion = "" if snapCont.Spec.VolumeSnapshotClassName != nil { // Delete VolumeSnapshotClass from the VolumeSnapshotContent. // This is necessary to make the deletion independent of the VolumeSnapshotClass. snapCont.Spec.VolumeSnapshotClassName = nil p.log.Debugf("Deleted VolumeSnapshotClassName from VolumeSnapshotContent %s to make deletion independent of VolumeSnapshotClass", snapCont.Name) } if err := p.crClient.Create(context.TODO(), &snapCont); err != nil { return errors.Wrapf(err, "fail to create VolumeSnapshotContent %s", snapCont.Name) } // Read resource timeout from backup annotation, if not set, use default value. timeout, err := time.ParseDuration( input.Backup.Annotations[velerov1api.ResourceTimeoutAnnotation]) if err != nil { p.log.Warnf("fail to parse resource timeout annotation %s: %s", input.Backup.Annotations[velerov1api.ResourceTimeoutAnnotation], err.Error()) timeout = 10 * time.Minute } p.log.Debugf("resource timeout is set to %s", timeout.String()) interval := 5 * time.Second // Wait until VSC created and ReadyToUse is true. if err := wait.PollUntilContextTimeout( context.Background(), interval, timeout, true, func(ctx context.Context) (bool, error) { return checkVSCReadiness(ctx, &snapCont, p.crClient) }, ); err != nil { // Clean up the VSC we created since it can't become ready if deleteErr := p.crClient.Delete(context.TODO(), &snapCont); deleteErr != nil && !apierrors.IsNotFound(deleteErr) { p.log.WithError(deleteErr).Errorf("Failed to clean up VolumeSnapshotContent %s", snapCont.Name) } return errors.Wrapf(err, "fail to wait VolumeSnapshotContent %s becomes ready.", snapCont.Name) } if err := p.crClient.Delete( context.TODO(), &snapCont, ); err != nil && !apierrors.IsNotFound(err) { p.log.Infof("VolumeSnapshotContent %s not found", snapCont.Name) return err } return nil } var checkVSCReadiness = func( ctx context.Context, vsc *snapshotv1api.VolumeSnapshotContent, client crclient.Client, ) (bool, error) { tmpVSC := new(snapshotv1api.VolumeSnapshotContent) if err := client.Get(ctx, crclient.ObjectKeyFromObject(vsc), tmpVSC); err != nil { return false, errors.Wrapf( err, "failed to get VolumeSnapshotContent %s", vsc.Name, ) } if tmpVSC.Status != nil && boolptr.IsSetToTrue(tmpVSC.Status.ReadyToUse) { return true, nil } // Fail fast on permanent CSI driver errors (e.g., InvalidSnapshot.NotFound) if tmpVSC.Status != nil && tmpVSC.Status.Error != nil && tmpVSC.Status.Error.Message != nil { return false, errors.Errorf( "VolumeSnapshotContent %s has error: %s", vsc.Name, *tmpVSC.Status.Error.Message, ) } return false, nil } func NewVolumeSnapshotContentDeleteItemAction( f client.Factory, ) plugincommon.HandlerInitializer { return func(logger logrus.FieldLogger) (any, error) { crClient, err := f.KubebuilderClient() if err != nil { return nil, err } return &volumeSnapshotContentDeleteItemAction{ log: logger, crClient: crClient, }, nil } } ================================================ FILE: internal/delete/actions/csi/volumesnapshotcontent_action_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package csi import ( "context" "fmt" "testing" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" crclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" factorymocks "github.com/vmware-tanzu/velero/pkg/client/mocks" "github.com/vmware-tanzu/velero/pkg/plugin/velero" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func TestVSCExecute(t *testing.T) { snapshotHandleStr := "test" tests := []struct { name string item runtime.Unstructured vsc *snapshotv1api.VolumeSnapshotContent backup *velerov1api.Backup function func( ctx context.Context, vsc *snapshotv1api.VolumeSnapshotContent, client crclient.Client, ) (bool, error) expectErr bool }{ { name: "VolumeSnapshotContent doesn't have backup label", item: velerotest.UnstructuredOrDie( ` { "apiVersion": "snapshot.storage.k8s.io/v1", "kind": "VolumeSnapshotContent", "metadata": { "namespace": "ns", "name": "foo" } } `, ), backup: builder.ForBackup("velero", "backup").Result(), expectErr: false, }, { name: "Normal case, VolumeSnapshot should be deleted", vsc: builder.ForVolumeSnapshotContent("bar").ObjectMeta(builder.WithLabelsMap(map[string]string{velerov1api.BackupNameLabel: "backup"})).VolumeSnapshotClassName("volumesnapshotclass").Status(&snapshotv1api.VolumeSnapshotContentStatus{SnapshotHandle: &snapshotHandleStr}).Result(), backup: builder.ForBackup("velero", "backup").ObjectMeta(builder.WithAnnotationsMap(map[string]string{velerov1api.ResourceTimeoutAnnotation: "5s"})).Result(), expectErr: false, function: func( ctx context.Context, vsc *snapshotv1api.VolumeSnapshotContent, client crclient.Client, ) (bool, error) { return true, nil }, }, { name: "Error case, deletion fails", vsc: builder.ForVolumeSnapshotContent("bar").ObjectMeta(builder.WithLabelsMap(map[string]string{velerov1api.BackupNameLabel: "backup"})).Status(&snapshotv1api.VolumeSnapshotContentStatus{SnapshotHandle: &snapshotHandleStr}).Result(), backup: builder.ForBackup("velero", "backup").ObjectMeta(builder.WithAnnotationsMap(map[string]string{velerov1api.ResourceTimeoutAnnotation: "5s"})).Result(), expectErr: true, function: func( ctx context.Context, vsc *snapshotv1api.VolumeSnapshotContent, client crclient.Client, ) (bool, error) { return false, errors.Errorf("test error case") }, }, { name: "Error case with CSI error, dangling VSC should be cleaned up", vsc: builder.ForVolumeSnapshotContent("bar").ObjectMeta(builder.WithLabelsMap(map[string]string{velerov1api.BackupNameLabel: "backup"})).Status(&snapshotv1api.VolumeSnapshotContentStatus{SnapshotHandle: &snapshotHandleStr}).Result(), backup: builder.ForBackup("velero", "backup").ObjectMeta(builder.WithAnnotationsMap(map[string]string{velerov1api.ResourceTimeoutAnnotation: "5s"})).Result(), expectErr: true, function: func( ctx context.Context, vsc *snapshotv1api.VolumeSnapshotContent, client crclient.Client, ) (bool, error) { return false, errors.Errorf("VolumeSnapshotContent %s has error: InvalidSnapshot.NotFound", vsc.Name) }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { crClient := velerotest.NewFakeControllerRuntimeClient(t) logger := logrus.StandardLogger() checkVSCReadiness = test.function p := volumeSnapshotContentDeleteItemAction{log: logger, crClient: crClient} if test.vsc != nil { vscMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(test.vsc) require.NoError(t, err) test.item = &unstructured.Unstructured{Object: vscMap} } err := p.Execute( &velero.DeleteItemActionExecuteInput{ Item: test.item, Backup: test.backup, }, ) if test.expectErr == false { require.NoError(t, err) } }) } } func TestVSCAppliesTo(t *testing.T) { p := volumeSnapshotContentDeleteItemAction{ log: logrus.StandardLogger(), } selector, err := p.AppliesTo() require.NoError(t, err) require.Equal( t, velero.ResourceSelector{ IncludedResources: []string{"volumesnapshotcontents.snapshot.storage.k8s.io"}, }, selector, ) } func TestNewVolumeSnapshotContentDeleteItemAction(t *testing.T) { logger := logrus.StandardLogger() crClient := velerotest.NewFakeControllerRuntimeClient(t) f := &factorymocks.Factory{} f.On("KubebuilderClient").Return(nil, fmt.Errorf("")) plugin := NewVolumeSnapshotContentDeleteItemAction(f) _, err := plugin(logger) require.Error(t, err) f1 := &factorymocks.Factory{} f1.On("KubebuilderClient").Return(crClient, nil) plugin1 := NewVolumeSnapshotContentDeleteItemAction(f1) _, err1 := plugin1(logger) require.NoError(t, err1) } func TestCheckVSCReadiness(t *testing.T) { tests := []struct { name string vsc *snapshotv1api.VolumeSnapshotContent createVSC bool expectErr bool ready bool }{ { name: "VSC not exist", vsc: &snapshotv1api.VolumeSnapshotContent{ ObjectMeta: metav1.ObjectMeta{ Name: "vsc-1", Namespace: "velero", }, }, createVSC: false, expectErr: true, ready: false, }, { name: "VSC not ready", vsc: &snapshotv1api.VolumeSnapshotContent{ ObjectMeta: metav1.ObjectMeta{ Name: "vsc-1", Namespace: "velero", }, }, createVSC: true, expectErr: false, ready: false, }, { name: "VSC has error from CSI driver", vsc: &snapshotv1api.VolumeSnapshotContent{ ObjectMeta: metav1.ObjectMeta{ Name: "vsc-1", Namespace: "velero", }, Status: &snapshotv1api.VolumeSnapshotContentStatus{ ReadyToUse: boolPtr(false), Error: &snapshotv1api.VolumeSnapshotError{ Message: stringPtr("InvalidSnapshot.NotFound: The snapshot 'snap-0abc123' does not exist."), }, }, }, createVSC: true, expectErr: true, ready: false, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { crClient := velerotest.NewFakeControllerRuntimeClient(t) if test.createVSC { require.NoError(t, crClient.Create(t.Context(), test.vsc)) } ready, err := checkVSCReadiness(t.Context(), test.vsc, crClient) require.Equal(t, test.ready, ready) if test.expectErr { require.Error(t, err) } }) } } func boolPtr(b bool) *bool { return &b } func stringPtr(s string) *string { return &s } ================================================ FILE: internal/delete/delete_item_action_handler.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package delete import ( "io" "github.com/vmware-tanzu/velero/pkg/plugin/framework" "github.com/pkg/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/sets" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/archive" "github.com/vmware-tanzu/velero/pkg/discovery" "github.com/vmware-tanzu/velero/pkg/plugin/velero" "github.com/vmware-tanzu/velero/pkg/util/filesystem" ) // Context provides the necessary environment to run DeleteItemAction plugins type Context struct { Backup *velerov1api.Backup BackupReader io.Reader Actions []velero.DeleteItemAction Filesystem filesystem.Interface Log logrus.FieldLogger DiscoveryHelper discovery.Helper resolvedActions []framework.DeleteItemResolvedAction } func InvokeDeleteActions(ctx *Context) error { var err error resolver := framework.NewDeleteItemActionResolver(ctx.Actions) ctx.resolvedActions, err = resolver.ResolveActions(ctx.DiscoveryHelper, ctx.Log) // No actions installed and no error means we don't have to continue; // just do the backup deletion without worrying about plugins. if len(ctx.resolvedActions) == 0 && err == nil { ctx.Log.Debug("No delete item actions present, proceeding with rest of backup deletion process") return nil } else if err != nil { return errors.Wrapf(err, "error resolving actions") } // get items out of backup tarball into a temp directory dir, err := archive.NewExtractor(ctx.Log, ctx.Filesystem).UnzipAndExtractBackup(ctx.BackupReader) if err != nil { return errors.Wrapf(err, "error extracting backup") } defer func() { if err := ctx.Filesystem.RemoveAll(dir); err != nil { ctx.Log.Errorf("error removing temporary directory %s: %s", dir, err.Error()) } }() ctx.Log.Debugf("Downloaded and extracted the backup file to: %s", dir) backupResources, err := archive.NewParser(ctx.Log, ctx.Filesystem).Parse(dir) if existErr := errors.Is(err, archive.ErrNotExist); existErr { ctx.Log.Debug("ignore invoking delete item actions: ", err) return nil } else if err != nil { return errors.Wrapf(err, "error parsing backup %q", dir) } processdResources := sets.NewString() for resource := range backupResources { groupResource := schema.ParseGroupResource(resource) // We've already seen this group/resource, so don't process it again. if processdResources.Has(groupResource.String()) { continue } // Get a list of all items that exist for this resource resourceList := backupResources[groupResource.String()] if resourceList == nil { continue } // Iterate over all items, grouped by namespace. for namespace, items := range resourceList.ItemsByNamespace { nsLog := ctx.Log.WithField("namespace", namespace) nsLog.Info("Starting to check for items in namespace") // Filter applicable actions based on namespace only once per namespace. actions := ctx.getApplicableActions(groupResource, namespace) // Process individual items from the backup for _, item := range items { itemPath := archive.GetItemFilePath(dir, resource, namespace, item) // obj is the Unstructured item from the backup obj, err := archive.Unmarshal(ctx.Filesystem, itemPath) if err != nil { return errors.Wrapf(err, "Could not unmarshal item: %v", item) } itemLog := nsLog.WithField("item", obj.GetName()) itemLog.Infof("invoking DeleteItemAction plugins") for _, action := range actions { if !action.Selector.Matches(labels.Set(obj.GetLabels())) { continue } err = action.DeleteItemAction.Execute(&velero.DeleteItemActionExecuteInput{ Item: obj, Backup: ctx.Backup, }) // Since we want to keep looping even on errors, log them instead of just returning. if err != nil { itemLog.WithError(err).Error("plugin error") } } } } } return nil } // getApplicableActions takes resolved DeleteItemActions and filters them for a given group/resource and namespace. func (ctx *Context) getApplicableActions(groupResource schema.GroupResource, namespace string) []framework.DeleteItemResolvedAction { var actions []framework.DeleteItemResolvedAction for _, action := range ctx.resolvedActions { if action.ShouldUse(groupResource, namespace, nil, ctx.Log) { actions = append(actions, action) } } return actions } ================================================ FILE: internal/delete/delete_item_action_handler_test.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package delete import ( "io" "sort" "testing" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/discovery" "github.com/vmware-tanzu/velero/pkg/plugin/velero" "github.com/vmware-tanzu/velero/pkg/test" kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube" ) func TestInvokeDeleteItemActionsRunForCorrectItems(t *testing.T) { // Declare test-singleton objects. fs := test.NewFakeFileSystem() log := logrus.StandardLogger() tests := []struct { name string backup *velerov1api.Backup apiResources []*test.APIResource tarball io.Reader actions map[*recordResourcesAction][]string // recordResourceActions are the plugins that will capture item ids, the []string values are the ids we'll test against. }{ { name: "single action with no selector runs for all items", backup: builder.ForBackup("velero", "velero").Result(), tarball: test.NewTarWriter(t). AddItems("pods", builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result()). AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result()). Done(), apiResources: []*test.APIResource{test.Pods(), test.PVs()}, actions: map[*recordResourcesAction][]string{ new(recordResourcesAction): {"ns-1/pod-1", "ns-2/pod-2", "pv-1", "pv-2"}, }, }, { name: "single action with a resource selector for namespaced resources runs only for matching resources", backup: builder.ForBackup("velero", "velero").Result(), tarball: test.NewTarWriter(t). AddItems("pods", builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result()). AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result()). Done(), apiResources: []*test.APIResource{test.Pods(), test.PVs()}, actions: map[*recordResourcesAction][]string{ new(recordResourcesAction).ForResource("pods"): {"ns-1/pod-1", "ns-2/pod-2"}, }, }, { name: "single action with a resource selector for cluster-scoped resources runs only for matching resources", backup: builder.ForBackup("velero", "velero").Result(), tarball: test.NewTarWriter(t). AddItems("pods", builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result()). AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result()). Done(), apiResources: []*test.APIResource{test.Pods(), test.PVs()}, actions: map[*recordResourcesAction][]string{ new(recordResourcesAction).ForResource("persistentvolumes"): {"pv-1", "pv-2"}, }, }, { name: "single action with a namespace selector runs only for resources in that namespace", backup: builder.ForBackup("velero", "velero").Result(), tarball: test.NewTarWriter(t). AddItems("pods", builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result()). AddItems("persistentvolumeclaims", builder.ForPersistentVolumeClaim("ns-1", "pvc-1").Result(), builder.ForPersistentVolumeClaim("ns-2", "pvc-2").Result()). AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result()). Done(), apiResources: []*test.APIResource{test.Pods(), test.PVCs(), test.PVs()}, actions: map[*recordResourcesAction][]string{ new(recordResourcesAction).ForNamespace("ns-1"): {"ns-1/pod-1", "ns-1/pvc-1"}, }, }, { name: "multiple actions, each with a different resource selector using short name, run for matching resources", backup: builder.ForBackup("velero", "velero").Result(), tarball: test.NewTarWriter(t). AddItems("pods", builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result()). AddItems("persistentvolumeclaims", builder.ForPersistentVolumeClaim("ns-1", "pvc-1").Result(), builder.ForPersistentVolumeClaim("ns-2", "pvc-2").Result()). AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result()). Done(), apiResources: []*test.APIResource{test.Pods(), test.PVCs(), test.PVs()}, actions: map[*recordResourcesAction][]string{ new(recordResourcesAction).ForResource("po"): {"ns-1/pod-1", "ns-2/pod-2"}, new(recordResourcesAction).ForResource("pv"): {"pv-1", "pv-2"}, }, }, { name: "actions with selectors that don't match anything don't run for any resources", backup: builder.ForBackup("velero", "velero").Result(), tarball: test.NewTarWriter(t). AddItems("pods", builder.ForPod("ns-1", "pod-1").Result()). AddItems("persistentvolumeclaims", builder.ForPersistentVolumeClaim("ns-2", "pvc-2").Result()). Done(), apiResources: []*test.APIResource{test.Pods(), test.PVCs(), test.PVs()}, actions: map[*recordResourcesAction][]string{ new(recordResourcesAction).ForNamespace("ns-1").ForResource("persistentvolumeclaims"): nil, new(recordResourcesAction).ForNamespace("ns-2").ForResource("pods"): nil, }, }, { name: "single action with label selector runs only for those items", backup: builder.ForBackup("velero", "velero").Result(), tarball: test.NewTarWriter(t). AddItems("pods", builder.ForPod("ns-1", "pod-1").ObjectMeta(builder.WithLabels("app", "app1")).Result(), builder.ForPod("ns-2", "pod-2").Result()). AddItems("persistentvolumeclaims", builder.ForPersistentVolumeClaim("ns-1", "pvc-1").Result(), builder.ForPersistentVolumeClaim("ns-2", "pvc-2").ObjectMeta(builder.WithLabels("app", "app1")).Result()). Done(), apiResources: []*test.APIResource{test.Pods(), test.PVCs()}, actions: map[*recordResourcesAction][]string{ new(recordResourcesAction).ForLabelSelector("app=app1"): {"ns-1/pod-1", "ns-2/pvc-2"}, }, }, { name: "success if resources dir does not exist", backup: builder.ForBackup("velero", "velero").Result(), tarball: test.NewTarWriter(t). Done(), apiResources: []*test.APIResource{test.Pods(), test.PVCs()}, actions: map[*recordResourcesAction][]string{ new(recordResourcesAction).ForNamespace("ns-1").ForResource("persistentvolumeclaims"): nil, new(recordResourcesAction).ForNamespace("ns-2").ForResource("pods"): nil, }, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { // test harness contains the fake API server/discovery client h := newHarness(t) for _, r := range tc.apiResources { h.addResource(t, r) } // Get the plugins out of the map in order to use them. var actions []velero.DeleteItemAction for action := range tc.actions { actions = append(actions, action) } c := &Context{ Backup: tc.backup, BackupReader: tc.tarball, Filesystem: fs, DiscoveryHelper: h.discoveryHelper, Actions: actions, Log: log, } err := InvokeDeleteActions(c) require.NoError(t, err) // Compare the plugins against the ids that we wanted. for action, want := range tc.actions { sort.Strings(want) sort.Strings(action.ids) assert.Equal(t, want, action.ids) } }) } } // TODO: unify this with the test harness in pkg/restore/restore_test.go type harness struct { *test.APIServer discoveryHelper discovery.Helper } func newHarness(t *testing.T) *harness { t.Helper() apiServer := test.NewAPIServer(t) log := logrus.StandardLogger() discoveryHelper, err := discovery.NewHelper(apiServer.DiscoveryClient, log) require.NoError(t, err) return &harness{ APIServer: apiServer, discoveryHelper: discoveryHelper, } } // addResource adds an APIResource and it's items to a faked API server for testing. func (h *harness) addResource(t *testing.T, resource *test.APIResource) { t.Helper() h.DiscoveryClient.WithAPIResource(resource) require.NoError(t, h.discoveryHelper.Refresh()) for _, item := range resource.Items { obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(item) require.NoError(t, err) unstructuredObj := &unstructured.Unstructured{Object: obj} if resource.Namespaced { _, err = h.DynamicClient.Resource(resource.GVR()).Namespace(item.GetNamespace()).Create(t.Context(), unstructuredObj, metav1.CreateOptions{}) } else { _, err = h.DynamicClient.Resource(resource.GVR()).Create(t.Context(), unstructuredObj, metav1.CreateOptions{}) } require.NoError(t, err) } } // recordResourcesAction is a delete item action that can be configured to run // for specific resources/namespaces and simply record the items that is // executed for. type recordResourcesAction struct { selector velero.ResourceSelector ids []string } func (a *recordResourcesAction) AppliesTo() (velero.ResourceSelector, error) { return a.selector, nil } func (a *recordResourcesAction) Execute(input *velero.DeleteItemActionExecuteInput) error { metadata, err := meta.Accessor(input.Item) if err != nil { return err } a.ids = append(a.ids, kubeutil.NamespaceAndName(metadata)) return nil } func (a *recordResourcesAction) ForResource(resource string) *recordResourcesAction { a.selector.IncludedResources = append(a.selector.IncludedResources, resource) return a } func (a *recordResourcesAction) ForNamespace(namespace string) *recordResourcesAction { a.selector.IncludedNamespaces = append(a.selector.IncludedNamespaces, namespace) return a } func (a *recordResourcesAction) ForLabelSelector(selector string) *recordResourcesAction { a.selector.LabelSelector = selector return a } func TestInvokeDeleteItemActionsWithNoPlugins(t *testing.T) { c := &Context{ Backup: builder.ForBackup("velero", "velero").Result(), Log: logrus.StandardLogger(), // No other fields are set on the assumption that if 0 actions are present, // the backup tarball and file system being empty will produce no errors. } err := InvokeDeleteActions(c) require.NoError(t, err) } ================================================ FILE: internal/hook/hook_tracker.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package hook import ( "fmt" "sync" ) const ( HookSourceAnnotation = "annotation" HookSourceSpec = "spec" ) // hookKey identifies a backup/restore hook type hookKey struct { // PodNamespace indicates the namespace of pod where hooks are executed. // For hooks specified in the backup/restore spec, this field is the namespace of an applicable pod. // For hooks specified in pod annotation, this field is the namespace of pod where hooks are annotated. podNamespace string // PodName indicates the pod where hooks are executed. // For hooks specified in the backup/restore spec, this field is an applicable pod name. // For hooks specified in pod annotation, this field is the pod where hooks are annotated. podName string // HookPhase is only for backup hooks, for restore hooks, this field is empty. hookPhase HookPhase // HookName is only for hooks specified in the backup/restore spec. // For hooks specified in pod annotation, this field is empty or "". hookName string // HookSource indicates where hooks come from. hookSource string // Container indicates the container hooks use. // For hooks specified in the backup/restore spec, the container might be the same under different hookName. container string // hookIndex contains the slice index for the specific hook, in order to track multiple hooks // for the same container hookIndex int } // hookStatus records the execution status of a specific hook. // hookStatus is extensible to accommodate additional fields as needs develop. type hookStatus struct { // HookFailed indicates if hook failed to execute. hookFailed bool // hookExecuted indicates if hook already execute. hookExecuted bool } // HookTracker tracks all hooks' execution status in a single backup/restore. type HookTracker struct { lock *sync.RWMutex // tracker records all hook info for a single backup/restore. tracker map[hookKey]hookStatus // hookAttemptedCnt indicates the number of attempted hooks. hookAttemptedCnt int // hookFailedCnt indicates the number of failed hooks. hookFailedCnt int // HookExecutedCnt indicates the number of executed hooks. hookExecutedCnt int // hookErrs records hook execution errors if any. hookErrs []HookErrInfo } // NewHookTracker creates a hookTracker instance. func NewHookTracker() *HookTracker { return &HookTracker{ lock: &sync.RWMutex{}, tracker: make(map[hookKey]hookStatus), } } // Add adds a hook to the hook tracker // Add must precede the Record for each individual hook. // In other words, a hook must be added to the tracker before its execution result is recorded. func (ht *HookTracker) Add(podNamespace, podName, container, source, hookName string, hookPhase HookPhase, hookIndex int) { ht.lock.Lock() defer ht.lock.Unlock() key := hookKey{ podNamespace: podNamespace, podName: podName, hookSource: source, container: container, hookPhase: hookPhase, hookName: hookName, hookIndex: hookIndex, } if _, ok := ht.tracker[key]; !ok { ht.tracker[key] = hookStatus{ hookFailed: false, hookExecuted: false, } ht.hookAttemptedCnt++ } } // Record records the hook's execution status // Add must precede the Record for each individual hook. // In other words, a hook must be added to the tracker before its execution result is recorded. func (ht *HookTracker) Record(podNamespace, podName, container, source, hookName string, hookPhase HookPhase, hookIndex int, hookFailed bool, hookErr error) error { ht.lock.Lock() defer ht.lock.Unlock() key := hookKey{ podNamespace: podNamespace, podName: podName, hookSource: source, container: container, hookPhase: hookPhase, hookName: hookName, hookIndex: hookIndex, } if _, ok := ht.tracker[key]; !ok { return fmt.Errorf("hook not exist in hook tracker, hook: %+v", key) } if !ht.tracker[key].hookExecuted { ht.tracker[key] = hookStatus{ hookFailed: hookFailed, hookExecuted: true, } ht.hookExecutedCnt++ if hookFailed { ht.hookFailedCnt++ ht.hookErrs = append(ht.hookErrs, HookErrInfo{Namespace: key.podNamespace, Err: hookErr}) } } return nil } // Stat returns the number of attempted hooks and failed hooks func (ht *HookTracker) Stat() (hookAttemptedCnt int, hookFailedCnt int) { ht.lock.RLock() defer ht.lock.RUnlock() return ht.hookAttemptedCnt, ht.hookFailedCnt } // IsComplete returns whether the execution of all hooks has finished or not func (ht *HookTracker) IsComplete() bool { ht.lock.RLock() defer ht.lock.RUnlock() return ht.hookAttemptedCnt == ht.hookExecutedCnt } // HooksErr returns hook execution errors func (ht *HookTracker) HookErrs() []HookErrInfo { ht.lock.RLock() defer ht.lock.RUnlock() return ht.hookErrs } // MultiHookTrackers tracks all hooks' execution status for multiple backups/restores. type MultiHookTracker struct { lock *sync.RWMutex // trackers is a map that uses the backup/restore name as the key and stores a HookTracker as value. trackers map[string]*HookTracker } // NewMultiHookTracker creates a multiHookTracker instance. func NewMultiHookTracker() *MultiHookTracker { return &MultiHookTracker{ lock: &sync.RWMutex{}, trackers: make(map[string]*HookTracker), } } // Add adds a backup/restore hook to the tracker func (mht *MultiHookTracker) Add(name, podNamespace, podName, container, source, hookName string, hookPhase HookPhase, hookIndex int) { mht.lock.Lock() defer mht.lock.Unlock() if _, ok := mht.trackers[name]; !ok { mht.trackers[name] = NewHookTracker() } mht.trackers[name].Add(podNamespace, podName, container, source, hookName, hookPhase, hookIndex) } // Record records a backup/restore hook execution status func (mht *MultiHookTracker) Record(name, podNamespace, podName, container, source, hookName string, hookPhase HookPhase, hookIndex int, hookFailed bool, hookErr error) error { mht.lock.RLock() defer mht.lock.RUnlock() var err error if _, ok := mht.trackers[name]; ok { err = mht.trackers[name].Record(podNamespace, podName, container, source, hookName, hookPhase, hookIndex, hookFailed, hookErr) } else { err = fmt.Errorf("the backup/restore not exist in hook tracker, backup/restore name: %s", name) } return err } // Stat returns the number of attempted hooks and failed hooks for a particular backup/restore func (mht *MultiHookTracker) Stat(name string) (hookAttemptedCnt int, hookFailedCnt int) { mht.lock.RLock() defer mht.lock.RUnlock() if _, ok := mht.trackers[name]; ok { return mht.trackers[name].Stat() } return } // Delete removes the hook data for a particular backup/restore func (mht *MultiHookTracker) Delete(name string) { mht.lock.Lock() defer mht.lock.Unlock() delete(mht.trackers, name) } // IsComplete returns whether the execution of all hooks for a particular backup/restore has finished or not func (mht *MultiHookTracker) IsComplete(name string) bool { mht.lock.RLock() defer mht.lock.RUnlock() if _, ok := mht.trackers[name]; ok { return mht.trackers[name].IsComplete() } return true } // HooksErr returns hook execution errors for a particular backup/restore func (mht *MultiHookTracker) HookErrs(name string) []HookErrInfo { mht.lock.RLock() defer mht.lock.RUnlock() if _, ok := mht.trackers[name]; ok { return mht.trackers[name].HookErrs() } return nil } ================================================ FILE: internal/hook/hook_tracker_test.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package hook import ( "fmt" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestNewHookTracker(t *testing.T) { tracker := NewHookTracker() assert.NotNil(t, tracker) assert.Empty(t, tracker.tracker) } func TestHookTracker_Add(t *testing.T) { tracker := NewHookTracker() tracker.Add("ns1", "pod1", "container1", HookSourceAnnotation, "h1", "", 0) key := hookKey{ podNamespace: "ns1", podName: "pod1", container: "container1", hookPhase: "", hookSource: HookSourceAnnotation, hookName: "h1", } _, ok := tracker.tracker[key] assert.True(t, ok) } func TestHookTracker_Record(t *testing.T) { tracker := NewHookTracker() tracker.Add("ns1", "pod1", "container1", HookSourceAnnotation, "h1", "", 0) err := tracker.Record("ns1", "pod1", "container1", HookSourceAnnotation, "h1", "", 0, true, fmt.Errorf("err")) key := hookKey{ podNamespace: "ns1", podName: "pod1", container: "container1", hookPhase: "", hookSource: HookSourceAnnotation, hookName: "h1", } info := tracker.tracker[key] assert.True(t, info.hookFailed) assert.True(t, info.hookExecuted) require.NoError(t, err) err = tracker.Record("ns2", "pod2", "container1", HookSourceAnnotation, "h1", "", 0, true, fmt.Errorf("err")) require.Error(t, err) err = tracker.Record("ns1", "pod1", "container1", HookSourceAnnotation, "h1", "", 0, false, nil) require.NoError(t, err) assert.True(t, info.hookFailed) } func TestHookTracker_Stat(t *testing.T) { tracker := NewHookTracker() tracker.Add("ns1", "pod1", "container1", HookSourceAnnotation, "h1", "", 0) tracker.Add("ns2", "pod2", "container1", HookSourceAnnotation, "h2", "", 0) tracker.Add("ns2", "pod2", "container1", HookSourceAnnotation, "h2", "", 1) tracker.Record("ns1", "pod1", "container1", HookSourceAnnotation, "h1", "", 0, true, fmt.Errorf("err")) attempted, failed := tracker.Stat() assert.Equal(t, 3, attempted) assert.Equal(t, 1, failed) } func TestHookTracker_IsComplete(t *testing.T) { tracker := NewHookTracker() tracker.Add("ns1", "pod1", "container1", HookSourceAnnotation, "h1", PhasePre, 0) tracker.Record("ns1", "pod1", "container1", HookSourceAnnotation, "h1", PhasePre, 0, true, fmt.Errorf("err")) assert.True(t, tracker.IsComplete()) tracker.Add("ns1", "pod1", "container1", HookSourceAnnotation, "h1", "", 0) assert.False(t, tracker.IsComplete()) } func TestHookTracker_HookErrs(t *testing.T) { tracker := NewHookTracker() tracker.Add("ns1", "pod1", "container1", HookSourceAnnotation, "h1", "", 0) tracker.Record("ns1", "pod1", "container1", HookSourceAnnotation, "h1", "", 0, true, fmt.Errorf("err")) hookErrs := tracker.HookErrs() assert.Len(t, hookErrs, 1) } func TestMultiHookTracker_Add(t *testing.T) { mht := NewMultiHookTracker() mht.Add("restore1", "ns1", "pod1", "container1", HookSourceAnnotation, "h1", "", 0) key := hookKey{ podNamespace: "ns1", podName: "pod1", container: "container1", hookPhase: "", hookSource: HookSourceAnnotation, hookName: "h1", hookIndex: 0, } _, ok := mht.trackers["restore1"].tracker[key] assert.True(t, ok) } func TestMultiHookTracker_Record(t *testing.T) { mht := NewMultiHookTracker() mht.Add("restore1", "ns1", "pod1", "container1", HookSourceAnnotation, "h1", "", 0) err := mht.Record("restore1", "ns1", "pod1", "container1", HookSourceAnnotation, "h1", "", 0, true, fmt.Errorf("err")) key := hookKey{ podNamespace: "ns1", podName: "pod1", container: "container1", hookPhase: "", hookSource: HookSourceAnnotation, hookName: "h1", hookIndex: 0, } info := mht.trackers["restore1"].tracker[key] assert.True(t, info.hookFailed) assert.True(t, info.hookExecuted) require.NoError(t, err) err = mht.Record("restore1", "ns2", "pod2", "container1", HookSourceAnnotation, "h1", "", 0, true, fmt.Errorf("err")) require.Error(t, err) err = mht.Record("restore2", "ns2", "pod2", "container1", HookSourceAnnotation, "h1", "", 0, true, fmt.Errorf("err")) assert.Error(t, err) } func TestMultiHookTracker_Stat(t *testing.T) { mht := NewMultiHookTracker() mht.Add("restore1", "ns1", "pod1", "container1", HookSourceAnnotation, "h1", "", 0) mht.Add("restore1", "ns2", "pod2", "container1", HookSourceAnnotation, "h2", "", 0) mht.Add("restore1", "ns2", "pod2", "container1", HookSourceAnnotation, "h2", "", 1) mht.Record("restore1", "ns1", "pod1", "container1", HookSourceAnnotation, "h1", "", 0, true, fmt.Errorf("err")) mht.Record("restore1", "ns2", "pod2", "container1", HookSourceAnnotation, "h2", "", 0, false, nil) mht.Record("restore1", "ns2", "pod2", "container1", HookSourceAnnotation, "h2", "", 1, false, nil) attempted, failed := mht.Stat("restore1") assert.Equal(t, 3, attempted) assert.Equal(t, 1, failed) } func TestMultiHookTracker_Delete(t *testing.T) { mht := NewMultiHookTracker() mht.Add("restore1", "ns1", "pod1", "container1", HookSourceAnnotation, "h1", "", 0) mht.Delete("restore1") _, ok := mht.trackers["restore1"] assert.False(t, ok) } func TestMultiHookTracker_IsComplete(t *testing.T) { mht := NewMultiHookTracker() mht.Add("backup1", "ns1", "pod1", "container1", HookSourceAnnotation, "h1", PhasePre, 0) mht.Record("backup1", "ns1", "pod1", "container1", HookSourceAnnotation, "h1", PhasePre, 0, true, fmt.Errorf("err")) assert.True(t, mht.IsComplete("backup1")) mht.Add("restore1", "ns1", "pod1", "container1", HookSourceAnnotation, "h1", "", 0) assert.False(t, mht.IsComplete("restore1")) assert.True(t, mht.IsComplete("restore2")) } func TestMultiHookTracker_HookErrs(t *testing.T) { mht := NewMultiHookTracker() mht.Add("restore1", "ns1", "pod1", "container1", HookSourceAnnotation, "h1", "", 0) mht.Record("restore1", "ns1", "pod1", "container1", HookSourceAnnotation, "h1", "", 0, true, fmt.Errorf("err")) hookErrs := mht.HookErrs("restore1") assert.Len(t, hookErrs, 1) hookErrs2 := mht.HookErrs("restore2") assert.Empty(t, hookErrs2) } ================================================ FILE: internal/hook/item_hook_handler.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package hook import ( "encoding/json" "fmt" "strconv" "strings" "time" "github.com/google/uuid" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/podexec" "github.com/vmware-tanzu/velero/pkg/restorehelper" "github.com/vmware-tanzu/velero/pkg/util/boolptr" "github.com/vmware-tanzu/velero/pkg/util/collections" "github.com/vmware-tanzu/velero/pkg/util/kube" ) type HookPhase string const ( PhasePre HookPhase = "pre" PhasePost HookPhase = "post" ) const ( // Backup hook annotations podBackupHookContainerAnnotationKey = "hook.backup.velero.io/container" podBackupHookCommandAnnotationKey = "hook.backup.velero.io/command" podBackupHookOnErrorAnnotationKey = "hook.backup.velero.io/on-error" podBackupHookTimeoutAnnotationKey = "hook.backup.velero.io/timeout" // Restore hook annotations podRestoreHookContainerAnnotationKey = "post.hook.restore.velero.io/container" podRestoreHookCommandAnnotationKey = "post.hook.restore.velero.io/command" podRestoreHookOnErrorAnnotationKey = "post.hook.restore.velero.io/on-error" podRestoreHookTimeoutAnnotationKey = "post.hook.restore.velero.io/exec-timeout" podRestoreHookWaitTimeoutAnnotationKey = "post.hook.restore.velero.io/wait-timeout" podRestoreHookWaitForReadyAnnotationKey = "post.hook.restore.velero.io/wait-for-ready" podRestoreHookInitContainerImageAnnotationKey = "init.hook.restore.velero.io/container-image" podRestoreHookInitContainerNameAnnotationKey = "init.hook.restore.velero.io/container-name" podRestoreHookInitContainerCommandAnnotationKey = "init.hook.restore.velero.io/command" podRestoreHookInitContainerTimeoutAnnotationKey = "init.hook.restore.velero.io/timeout" ) // ItemHookHandler invokes hooks for an item. type ItemHookHandler interface { // HandleHooks invokes hooks for an item. If the item is a pod and the appropriate annotations exist // to specify a hook, that is executed. Otherwise, this looks at the backup context's Backup to // determine if there are any hooks relevant to the item, taking into account the hook spec's // namespaces, resources, and label selector. HandleHooks( log logrus.FieldLogger, groupResource schema.GroupResource, obj runtime.Unstructured, resourceHooks []ResourceHook, phase HookPhase, hookTracker *HookTracker, ) error } // ItemRestoreHookHandler invokes restore hooks for an item type ItemRestoreHookHandler interface { HandleRestoreHooks( log logrus.FieldLogger, groupResource schema.GroupResource, obj runtime.Unstructured, rh []ResourceRestoreHook, ) (runtime.Unstructured, error) } // InitContainerRestoreHookHandler is the restore hook handler to add init containers to restored pods. type InitContainerRestoreHookHandler struct{} // HandleRestoreHooks runs the restore hooks for an item. // If the item is a pod, then hooks are chosen to be run as follows: // If the pod has the appropriate annotations specifying the hook action, then hooks from the annotation are run // Otherwise, the supplied ResourceRestoreHooks are applied. func (i *InitContainerRestoreHookHandler) HandleRestoreHooks( log logrus.FieldLogger, groupResource schema.GroupResource, obj runtime.Unstructured, resourceRestoreHooks []ResourceRestoreHook, namespaceMapping map[string]string, ) (runtime.Unstructured, error) { // We only support hooks on pods right now if groupResource != kuberesource.Pods { return nil, nil } metadata, err := meta.Accessor(obj) if err != nil { return nil, errors.Wrap(err, "unable to get a metadata accessor") } pod := new(corev1api.Pod) if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), pod); err != nil { return nil, errors.WithStack(err) } initContainers := []corev1api.Container{} // If this pod is backed up with data movement, then we want to the pod volumes restored prior to // running the restore hook init containers. This allows the restore hook init containers to prepare the // restored data to be consumed by the application container(s). // So if there is a "restore-wait" init container already on the pod at index 0, we'll preserve that and run // it before running any other init container. if len(pod.Spec.InitContainers) > 0 && pod.Spec.InitContainers[0].Name == restorehelper.WaitInitContainer { initContainers = append(initContainers, pod.Spec.InitContainers[0]) pod.Spec.InitContainers = pod.Spec.InitContainers[1:] } initContainerFromAnnotations := getInitContainerFromAnnotation(kube.NamespaceAndName(pod), metadata.GetAnnotations(), log) if initContainerFromAnnotations != nil { log.Infof("Handling InitRestoreHooks from pod annotations") initContainers = append(initContainers, *initContainerFromAnnotations) } else { log.Infof("Handling InitRestoreHooks from RestoreSpec") // pod did not have the annotations appropriate for restore hooks // running applicable ResourceRestoreHooks supplied. namespace := metadata.GetNamespace() labels := labels.Set(metadata.GetLabels()) // Apply the hook according to the target namespace in which the pod will be restored // more details see https://github.com/vmware-tanzu/velero/issues/4720 if namespaceMapping != nil { if n, ok := namespaceMapping[namespace]; ok { namespace = n } } for _, rh := range resourceRestoreHooks { if !rh.Selector.applicableTo(groupResource, namespace, labels) { continue } for _, hook := range rh.RestoreHooks { if hook.Init != nil { containers := make([]corev1api.Container, 0) for _, raw := range hook.Init.InitContainers { container := corev1api.Container{} err := ValidateContainer(raw.Raw) if err != nil { log.Errorf("invalid Restore Init hook: %s", err.Error()) return nil, err } err = json.Unmarshal(raw.Raw, &container) if err != nil { log.Errorf("fail to Unmarshal hook Init into container: %s", err.Error()) return nil, errors.WithStack(err) } containers = append(containers, container) } initContainers = append(initContainers, containers...) } } } } pod.Spec.InitContainers = append(initContainers, pod.Spec.InitContainers...) log.Infof("Returning pod %s/%s with %d init container(s)", pod.Namespace, pod.Name, len(pod.Spec.InitContainers)) podMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&pod) if err != nil { return nil, errors.WithStack(err) } return &unstructured.Unstructured{Object: podMap}, nil } // DefaultItemHookHandler is the default itemHookHandler. type DefaultItemHookHandler struct { PodCommandExecutor podexec.PodCommandExecutor } func (h *DefaultItemHookHandler) HandleHooks( log logrus.FieldLogger, groupResource schema.GroupResource, obj runtime.Unstructured, resourceHooks []ResourceHook, phase HookPhase, hookTracker *HookTracker, ) error { // We only support hooks on pods right now if groupResource != kuberesource.Pods { return nil } metadata, err := meta.Accessor(obj) if err != nil { return errors.Wrap(err, "unable to get a metadata accessor") } namespace := metadata.GetNamespace() name := metadata.GetName() // If the pod has the hook specified via annotations, that takes priority. hookFromAnnotations := getPodExecHookFromAnnotations(metadata.GetAnnotations(), phase, log) if phase == PhasePre && hookFromAnnotations == nil { // See if the pod has the legacy hook annotation keys (i.e. without a phase specified) hookFromAnnotations = getPodExecHookFromAnnotations(metadata.GetAnnotations(), "", log) } if hookFromAnnotations != nil { hookTracker.Add(namespace, name, hookFromAnnotations.Container, HookSourceAnnotation, "", phase, 0) hookLog := log.WithFields( logrus.Fields{ "hookSource": HookSourceAnnotation, "hookType": "exec", "hookPhase": phase, }, ) hookFailed := false var errExec error if errExec = h.PodCommandExecutor.ExecutePodCommand(hookLog, obj.UnstructuredContent(), namespace, name, "", hookFromAnnotations); errExec != nil { hookLog.WithError(errExec).Error("Error executing hook") hookFailed = true } errTracker := hookTracker.Record(namespace, name, hookFromAnnotations.Container, HookSourceAnnotation, "", phase, 0, hookFailed, errExec) if errTracker != nil { hookLog.WithError(errTracker).Warn("Error recording the hook in hook tracker") } if errExec != nil && hookFromAnnotations.OnError == velerov1api.HookErrorModeFail { return errExec } return nil } labels := labels.Set(metadata.GetLabels()) // Otherwise, check for hooks defined in the backup spec. // modeFailError records the error from the hook with "Fail" error mode var modeFailError error for _, resourceHook := range resourceHooks { if !resourceHook.Selector.applicableTo(groupResource, namespace, labels) { continue } var hooks []velerov1api.BackupResourceHook if phase == PhasePre { hooks = resourceHook.Pre } else { hooks = resourceHook.Post } for i, hook := range hooks { if groupResource == kuberesource.Pods { if hook.Exec != nil { hookTracker.Add(namespace, name, hook.Exec.Container, HookSourceSpec, resourceHook.Name, phase, i) // The remaining hooks will only be executed if modeFailError is nil. // Otherwise, execution will stop and only hook collection will occur. if modeFailError == nil { hookLog := log.WithFields( logrus.Fields{ "hookSource": HookSourceSpec, "hookType": "exec", "hookPhase": phase, }, ) hookFailed := false err := h.PodCommandExecutor.ExecutePodCommand(hookLog, obj.UnstructuredContent(), namespace, name, resourceHook.Name, hook.Exec) if err != nil { hookLog.WithError(err).Error("Error executing hook") hookFailed = true if hook.Exec.OnError == velerov1api.HookErrorModeFail { modeFailError = err } } errTracker := hookTracker.Record(namespace, name, hook.Exec.Container, HookSourceSpec, resourceHook.Name, phase, i, hookFailed, err) if errTracker != nil { hookLog.WithError(errTracker).Warn("Error recording the hook in hook tracker") } } } } } } return modeFailError } // NoOpItemHookHandler is the an itemHookHandler for the Finalize controller where hooks don't run type NoOpItemHookHandler struct{} func (h *NoOpItemHookHandler) HandleHooks( log logrus.FieldLogger, groupResource schema.GroupResource, obj runtime.Unstructured, resourceHooks []ResourceHook, phase HookPhase, hookTracker *HookTracker, ) error { return nil } func phasedKey(phase HookPhase, key string) string { if phase != "" { return fmt.Sprintf("%v.%v", phase, key) } return key } func getHookAnnotation(annotations map[string]string, key string, phase HookPhase) string { return annotations[phasedKey(phase, key)] } // getPodExecHookFromAnnotations returns an ExecHook based on the annotations, as long as the // 'command' annotation is present. If it is absent, this returns nil. // If there is an error in parsing a supplied timeout, it is logged. func getPodExecHookFromAnnotations(annotations map[string]string, phase HookPhase, log logrus.FieldLogger) *velerov1api.ExecHook { commandValue := getHookAnnotation(annotations, podBackupHookCommandAnnotationKey, phase) if commandValue == "" { return nil } container := getHookAnnotation(annotations, podBackupHookContainerAnnotationKey, phase) onError := velerov1api.HookErrorMode(getHookAnnotation(annotations, podBackupHookOnErrorAnnotationKey, phase)) if onError != velerov1api.HookErrorModeContinue && onError != velerov1api.HookErrorModeFail { onError = "" } var timeout time.Duration timeoutString := getHookAnnotation(annotations, podBackupHookTimeoutAnnotationKey, phase) if timeoutString != "" { if temp, err := time.ParseDuration(timeoutString); err == nil { timeout = temp } else { log.Warn(errors.Wrapf(err, "Unable to parse provided timeout %s, using default", timeoutString)) } } return &velerov1api.ExecHook{ Container: container, Command: parseStringToCommand(commandValue), OnError: onError, Timeout: metav1.Duration{Duration: timeout}, } } func parseStringToCommand(commandValue string) []string { var command []string // check for json array if commandValue[0] == '[' { if err := json.Unmarshal([]byte(commandValue), &command); err != nil { command = []string{commandValue} } } else { command = append(command, commandValue) } return command } type ResourceHookSelector struct { Namespaces *collections.IncludesExcludes Resources *collections.IncludesExcludes LabelSelector labels.Selector } // ResourceHook is a hook for a given resource. type ResourceHook struct { Name string Selector ResourceHookSelector Pre []velerov1api.BackupResourceHook Post []velerov1api.BackupResourceHook } func (r ResourceHookSelector) applicableTo(groupResource schema.GroupResource, namespace string, labels labels.Set) bool { if r.Namespaces != nil && !r.Namespaces.ShouldInclude(namespace) { return false } if r.Resources != nil && !r.Resources.ShouldInclude(groupResource.String()) { return false } if r.LabelSelector != nil && !r.LabelSelector.Matches(labels) { return false } return true } // ResourceRestoreHook is a restore hook for a given resource. type ResourceRestoreHook struct { Name string Selector ResourceHookSelector RestoreHooks []velerov1api.RestoreResourceHook } func getInitContainerFromAnnotation(podName string, annotations map[string]string, log logrus.FieldLogger) *corev1api.Container { containerImage := annotations[podRestoreHookInitContainerImageAnnotationKey] containerName := annotations[podRestoreHookInitContainerNameAnnotationKey] command := annotations[podRestoreHookInitContainerCommandAnnotationKey] if containerImage == "" { log.Infof("Pod %s has no %s annotation, no initRestoreHook in annotation", podName, podRestoreHookInitContainerImageAnnotationKey) return nil } if command == "" { log.Infof("RestoreHook init container for pod %s is using container's default entrypoint", podName, containerImage) } if containerName == "" { uid, err := uuid.NewRandom() uuidStr := "deadfeed" if err != nil { log.Errorf("Failed to generate UUID for container name") } else { uuidStr = strings.Split(uid.String(), "-")[0] } containerName = fmt.Sprintf("velero-restore-init-%s", uuidStr) log.Infof("Pod %s has no %s annotation, using generated name %s for initContainer", podName, podRestoreHookInitContainerNameAnnotationKey, containerName) } initContainer := corev1api.Container{ Image: containerImage, Name: containerName, Command: parseStringToCommand(command), } return &initContainer } // GetRestoreHooksFromSpec returns a list of ResourceRestoreHooks from the restore Spec. func GetRestoreHooksFromSpec(hooksSpec *velerov1api.RestoreHooks) ([]ResourceRestoreHook, error) { if hooksSpec == nil { return []ResourceRestoreHook{}, nil } restoreHooks := make([]ResourceRestoreHook, 0, len(hooksSpec.Resources)) for _, rs := range hooksSpec.Resources { rh := ResourceRestoreHook{ Name: rs.Name, Selector: ResourceHookSelector{ Namespaces: collections.NewIncludesExcludes().Includes(rs.IncludedNamespaces...).Excludes(rs.ExcludedNamespaces...), // these hooks ara applicable only to pods. // TODO: resolve the pods resource via discovery? Resources: collections.NewIncludesExcludes().Includes(kuberesource.Pods.Resource), }, // TODO does this work for ExecRestoreHook as well? RestoreHooks: rs.PostHooks, } if rs.LabelSelector != nil { ls, err := metav1.LabelSelectorAsSelector(rs.LabelSelector) if err != nil { return []ResourceRestoreHook{}, errors.WithStack(err) } rh.Selector.LabelSelector = ls } restoreHooks = append(restoreHooks, rh) } return restoreHooks, nil } // getPodExecRestoreHookFromAnnotations returns an ExecRestoreHook based on restore annotations, as // long as the 'command' annotation is present. If it is absent, this returns nil. func getPodExecRestoreHookFromAnnotations(annotations map[string]string, log logrus.FieldLogger) *velerov1api.ExecRestoreHook { commandValue := annotations[podRestoreHookCommandAnnotationKey] if commandValue == "" { return nil } container := annotations[podRestoreHookContainerAnnotationKey] onError := velerov1api.HookErrorMode(annotations[podRestoreHookOnErrorAnnotationKey]) if onError != velerov1api.HookErrorModeContinue && onError != velerov1api.HookErrorModeFail { onError = "" } var execTimeout time.Duration execTimeoutString := annotations[podRestoreHookTimeoutAnnotationKey] if execTimeoutString != "" { if temp, err := time.ParseDuration(execTimeoutString); err == nil { execTimeout = temp } else { log.Warn(errors.Wrapf(err, "Unable to parse exec timeout %s, ignoring", execTimeoutString)) } } var waitTimeout time.Duration waitTimeoutString := annotations[podRestoreHookWaitTimeoutAnnotationKey] if waitTimeoutString != "" { if temp, err := time.ParseDuration(waitTimeoutString); err == nil { waitTimeout = temp } else { log.Warn(errors.Wrapf(err, "Unable to parse wait timeout %s, ignoring", waitTimeoutString)) } } waitForReadyString := annotations[podRestoreHookWaitForReadyAnnotationKey] waitForReady := boolptr.False() if waitForReadyString != "" { var err error *waitForReady, err = strconv.ParseBool(waitForReadyString) if err != nil { log.Warn(errors.Wrapf(err, "Unable to parse wait for ready %s, ignoring", waitForReadyString)) } } return &velerov1api.ExecRestoreHook{ Container: container, Command: parseStringToCommand(commandValue), OnError: onError, ExecTimeout: metav1.Duration{Duration: execTimeout}, WaitTimeout: metav1.Duration{Duration: waitTimeout}, WaitForReady: waitForReady, } } type PodExecRestoreHook struct { HookName string HookSource string Hook velerov1api.ExecRestoreHook executed bool // hookIndex contains the slice index for the specific hook from the restore spec // in order to track multiple hooks. Stored here because restore hook results are recorded // outside of the original slice iteration // for the same container hookIndex int } // GroupRestoreExecHooks returns a list of hooks to be executed in a pod grouped by // container name. If an exec hook is defined in annotation that is used, else applicable exec // hooks from the restore resource are accumulated. func GroupRestoreExecHooks( restoreName string, resourceRestoreHooks []ResourceRestoreHook, pod *corev1api.Pod, log logrus.FieldLogger, hookTrack *MultiHookTracker, ) (map[string][]PodExecRestoreHook, error) { byContainer := map[string][]PodExecRestoreHook{} if pod == nil || len(pod.Spec.Containers) == 0 { return byContainer, nil } metadata, err := meta.Accessor(pod) if err != nil { return nil, errors.WithStack(err) } hookFromAnnotation := getPodExecRestoreHookFromAnnotations(metadata.GetAnnotations(), log) if hookFromAnnotation != nil { // default to first container in pod if unset if hookFromAnnotation.Container == "" { hookFromAnnotation.Container = pod.Spec.Containers[0].Name } hookTrack.Add(restoreName, metadata.GetNamespace(), metadata.GetName(), hookFromAnnotation.Container, HookSourceAnnotation, "", HookPhase(""), 0) byContainer[hookFromAnnotation.Container] = []PodExecRestoreHook{ { HookName: "", HookSource: HookSourceAnnotation, Hook: *hookFromAnnotation, hookIndex: 0, }, } return byContainer, nil } // No hook found on pod's annotations so check for applicable hooks from the restore spec labels := metadata.GetLabels() namespace := metadata.GetNamespace() for _, rrh := range resourceRestoreHooks { if !rrh.Selector.applicableTo(kuberesource.Pods, namespace, labels) { continue } for i, rh := range rrh.RestoreHooks { if rh.Exec == nil { continue } named := PodExecRestoreHook{ HookName: rrh.Name, Hook: *rh.Exec, HookSource: HookSourceSpec, hookIndex: i, } // default to false if attr WaitForReady not set if named.Hook.WaitForReady == nil { named.Hook.WaitForReady = boolptr.False() } // default to first container in pod if unset, without mutating resource restore hook if named.Hook.Container == "" { named.Hook.Container = pod.Spec.Containers[0].Name } hookTrack.Add(restoreName, metadata.GetNamespace(), metadata.GetName(), named.Hook.Container, HookSourceSpec, rrh.Name, HookPhase(""), i) byContainer[named.Hook.Container] = append(byContainer[named.Hook.Container], named) } } return byContainer, nil } // ValidateContainer validate whether a map contains mandatory k8s Container fields. // mandatory fields include name, image and commands. func ValidateContainer(raw []byte) error { container := corev1api.Container{} err := json.Unmarshal(raw, &container) if err != nil { return err } if len(container.Command) <= 0 || len(container.Name) <= 0 || len(container.Image) <= 0 { return fmt.Errorf("invalid InitContainer in restore hook, it doesn't have Command, Name or Image field") } return nil } ================================================ FILE: internal/hook/item_hook_handler_test.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package hook import ( "fmt" "testing" "time" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/kuberesource" velerotest "github.com/vmware-tanzu/velero/pkg/test" "github.com/vmware-tanzu/velero/pkg/util/boolptr" "github.com/vmware-tanzu/velero/pkg/util/collections" ) func TestHandleHooksSkips(t *testing.T) { tests := []struct { name string groupResource string item runtime.Unstructured hooks []ResourceHook }{ { name: "not a pod", groupResource: "widget.group", }, { name: "pod without annotation / no spec hooks", item: velerotest.UnstructuredOrDie( ` { "apiVersion": "v1", "kind": "Pod", "metadata": { "namespace": "ns", "name": "foo" } } `, ), }, { name: "spec hooks not applicable", groupResource: "pods", item: velerotest.UnstructuredOrDie( ` { "apiVersion": "v1", "kind": "Pod", "metadata": { "namespace": "ns", "name": "foo", "labels": { "color": "blue" } } } `, ), hooks: []ResourceHook{ { Name: "ns exclude", Selector: ResourceHookSelector{Namespaces: collections.NewIncludesExcludes().Excludes("ns")}, }, { Name: "resource exclude", Selector: ResourceHookSelector{Resources: collections.NewIncludesExcludes().Includes("widgets.group")}, }, { Name: "label selector mismatch", Selector: ResourceHookSelector{LabelSelector: parseLabelSelectorOrDie("color=green")}, }, { Name: "missing exec hook", Pre: []velerov1api.BackupResourceHook{ {}, {}, }, }, }, }, } hookTracker := NewHookTracker() for _, test := range tests { t.Run(test.name, func(t *testing.T) { podCommandExecutor := &velerotest.MockPodCommandExecutor{} defer podCommandExecutor.AssertExpectations(t) h := &DefaultItemHookHandler{ PodCommandExecutor: podCommandExecutor, } groupResource := schema.ParseGroupResource(test.groupResource) err := h.HandleHooks(velerotest.NewLogger(), groupResource, test.item, test.hooks, PhasePre, hookTracker) require.NoError(t, err) }) } } func TestHandleHooks(t *testing.T) { tests := []struct { name string phase HookPhase groupResource string item runtime.Unstructured hooks []ResourceHook hookErrorsByContainer map[string]error expectedError error expectedPodHook *velerov1api.ExecHook expectedPodHookError error }{ { name: "pod, no annotation, spec (multiple pre hooks) = run spec", phase: PhasePre, groupResource: "pods", item: velerotest.UnstructuredOrDie(` { "apiVersion": "v1", "kind": "Pod", "metadata": { "namespace": "ns", "name": "name" } }`), hooks: []ResourceHook{ { Name: "hook1", Pre: []velerov1api.BackupResourceHook{ { Exec: &velerov1api.ExecHook{ Container: "1a", Command: []string{"pre-1a"}, }, }, { Exec: &velerov1api.ExecHook{ Container: "1b", Command: []string{"pre-1b"}, }, }, }, }, { Name: "hook2", Pre: []velerov1api.BackupResourceHook{ { Exec: &velerov1api.ExecHook{ Container: "2a", Command: []string{"2a"}, }, }, { Exec: &velerov1api.ExecHook{ Container: "2b", Command: []string{"2b"}, }, }, }, }, }, }, { name: "pod, no annotation, spec (multiple post hooks) = run spec", phase: PhasePost, groupResource: "pods", item: velerotest.UnstructuredOrDie(` { "apiVersion": "v1", "kind": "Pod", "metadata": { "namespace": "ns", "name": "name" } }`), hooks: []ResourceHook{ { Name: "hook1", Post: []velerov1api.BackupResourceHook{ { Exec: &velerov1api.ExecHook{ Container: "1a", Command: []string{"pre-1a"}, }, }, { Exec: &velerov1api.ExecHook{ Container: "1b", Command: []string{"pre-1b"}, }, }, }, }, { Name: "hook2", Post: []velerov1api.BackupResourceHook{ { Exec: &velerov1api.ExecHook{ Container: "2a", Command: []string{"2a"}, }, }, { Exec: &velerov1api.ExecHook{ Container: "2b", Command: []string{"2b"}, }, }, }, }, }, }, { name: "pod, annotation (legacy), no spec = run annotation", phase: PhasePre, groupResource: "pods", item: velerotest.UnstructuredOrDie(` { "apiVersion": "v1", "kind": "Pod", "metadata": { "namespace": "ns", "name": "name", "annotations": { "hook.backup.velero.io/container": "c", "hook.backup.velero.io/command": "/bin/ls" } } }`), expectedPodHook: &velerov1api.ExecHook{ Container: "c", Command: []string{"/bin/ls"}, }, }, { name: "pod, annotation (pre), no spec = run annotation", phase: PhasePre, groupResource: "pods", item: velerotest.UnstructuredOrDie(` { "apiVersion": "v1", "kind": "Pod", "metadata": { "namespace": "ns", "name": "name", "annotations": { "pre.hook.backup.velero.io/container": "c", "pre.hook.backup.velero.io/command": "/bin/ls" } } }`), expectedPodHook: &velerov1api.ExecHook{ Container: "c", Command: []string{"/bin/ls"}, }, }, { name: "pod, annotation (post), no spec = run annotation", phase: PhasePost, groupResource: "pods", item: velerotest.UnstructuredOrDie(` { "apiVersion": "v1", "kind": "Pod", "metadata": { "namespace": "ns", "name": "name", "annotations": { "post.hook.backup.velero.io/container": "c", "post.hook.backup.velero.io/command": "/bin/ls" } } }`), expectedPodHook: &velerov1api.ExecHook{ Container: "c", Command: []string{"/bin/ls"}, }, }, { name: "pod, annotation & spec = run annotation", phase: PhasePre, groupResource: "pods", item: velerotest.UnstructuredOrDie(` { "apiVersion": "v1", "kind": "Pod", "metadata": { "namespace": "ns", "name": "name", "annotations": { "hook.backup.velero.io/container": "c", "hook.backup.velero.io/command": "/bin/ls" } } }`), expectedPodHook: &velerov1api.ExecHook{ Container: "c", Command: []string{"/bin/ls"}, }, hooks: []ResourceHook{ { Name: "hook1", Pre: []velerov1api.BackupResourceHook{ { Exec: &velerov1api.ExecHook{ Container: "1a", Command: []string{"1a"}, }, }, }, }, }, }, { name: "pod, annotation, onError=fail = return error", phase: PhasePre, groupResource: "pods", item: velerotest.UnstructuredOrDie(` { "apiVersion": "v1", "kind": "Pod", "metadata": { "namespace": "ns", "name": "name", "annotations": { "hook.backup.velero.io/container": "c", "hook.backup.velero.io/command": "/bin/ls", "hook.backup.velero.io/on-error": "Fail" } } }`), expectedPodHook: &velerov1api.ExecHook{ Container: "c", Command: []string{"/bin/ls"}, OnError: velerov1api.HookErrorModeFail, }, expectedPodHookError: errors.New("pod hook error"), expectedError: errors.New("pod hook error"), }, { name: "pod, annotation, onError=continue = return nil", phase: PhasePre, groupResource: "pods", item: velerotest.UnstructuredOrDie(` { "apiVersion": "v1", "kind": "Pod", "metadata": { "namespace": "ns", "name": "name", "annotations": { "hook.backup.velero.io/container": "c", "hook.backup.velero.io/command": "/bin/ls", "hook.backup.velero.io/on-error": "Continue" } } }`), expectedPodHook: &velerov1api.ExecHook{ Container: "c", Command: []string{"/bin/ls"}, OnError: velerov1api.HookErrorModeContinue, }, expectedPodHookError: errors.New("pod hook error"), expectedError: nil, }, { name: "pod, spec, onError=fail = don't run other hooks", phase: PhasePre, groupResource: "pods", item: velerotest.UnstructuredOrDie(` { "apiVersion": "v1", "kind": "Pod", "metadata": { "namespace": "ns", "name": "name" } }`), hooks: []ResourceHook{ { Name: "hook1", Pre: []velerov1api.BackupResourceHook{ { Exec: &velerov1api.ExecHook{ Container: "1a", Command: []string{"1a"}, OnError: velerov1api.HookErrorModeContinue, }, }, { Exec: &velerov1api.ExecHook{ Container: "1b", Command: []string{"1b"}, }, }, }, }, { Name: "hook2", Pre: []velerov1api.BackupResourceHook{ { Exec: &velerov1api.ExecHook{ Container: "2", Command: []string{"2"}, OnError: velerov1api.HookErrorModeFail, }, }, }, }, { Name: "hook3", Pre: []velerov1api.BackupResourceHook{ { Exec: &velerov1api.ExecHook{ Container: "3", Command: []string{"3"}, }, }, }, }, }, hookErrorsByContainer: map[string]error{ "1a": errors.New("1a error, but continue"), "2": errors.New("2 error, fail"), }, expectedError: errors.New("2 error, fail"), }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { podCommandExecutor := &velerotest.MockPodCommandExecutor{} defer podCommandExecutor.AssertExpectations(t) h := &DefaultItemHookHandler{ PodCommandExecutor: podCommandExecutor, } if test.expectedPodHook != nil { podCommandExecutor.On("ExecutePodCommand", mock.Anything, test.item.UnstructuredContent(), "ns", "name", "", test.expectedPodHook).Return(test.expectedPodHookError) } else { hookLoop: for _, resourceHook := range test.hooks { for _, hook := range resourceHook.Pre { hookError := test.hookErrorsByContainer[hook.Exec.Container] podCommandExecutor.On("ExecutePodCommand", mock.Anything, test.item.UnstructuredContent(), "ns", "name", resourceHook.Name, hook.Exec).Return(hookError) if hookError != nil && hook.Exec.OnError == velerov1api.HookErrorModeFail { break hookLoop } } for _, hook := range resourceHook.Post { hookError := test.hookErrorsByContainer[hook.Exec.Container] podCommandExecutor.On("ExecutePodCommand", mock.Anything, test.item.UnstructuredContent(), "ns", "name", resourceHook.Name, hook.Exec).Return(hookError) if hookError != nil && hook.Exec.OnError == velerov1api.HookErrorModeFail { break hookLoop } } } } groupResource := schema.ParseGroupResource(test.groupResource) hookTracker := NewHookTracker() err := h.HandleHooks(velerotest.NewLogger(), groupResource, test.item, test.hooks, test.phase, hookTracker) if test.expectedError != nil { assert.EqualError(t, err, test.expectedError.Error()) return } require.NoError(t, err) }) } } func TestGetPodExecHookFromAnnotations(t *testing.T) { phases := []HookPhase{"", PhasePre, PhasePost} for _, phase := range phases { tests := []struct { name string annotations map[string]string expectedHook *velerov1api.ExecHook }{ { name: "missing command annotation", expectedHook: nil, }, { name: "malformed command json array", annotations: map[string]string{ phasedKey(phase, podBackupHookCommandAnnotationKey): "[blarg", }, expectedHook: &velerov1api.ExecHook{ Command: []string{"[blarg"}, }, }, { name: "valid command json array", annotations: map[string]string{ phasedKey(phase, podBackupHookCommandAnnotationKey): `["a","b","c"]`, }, expectedHook: &velerov1api.ExecHook{ Command: []string{"a", "b", "c"}, }, }, { name: "command as a string", annotations: map[string]string{ phasedKey(phase, podBackupHookCommandAnnotationKey): "/usr/bin/foo", }, expectedHook: &velerov1api.ExecHook{ Command: []string{"/usr/bin/foo"}, }, }, { name: "hook mode set to continue", annotations: map[string]string{ phasedKey(phase, podBackupHookCommandAnnotationKey): "/usr/bin/foo", phasedKey(phase, podBackupHookOnErrorAnnotationKey): string(velerov1api.HookErrorModeContinue), }, expectedHook: &velerov1api.ExecHook{ Command: []string{"/usr/bin/foo"}, OnError: velerov1api.HookErrorModeContinue, }, }, { name: "hook mode set to fail", annotations: map[string]string{ phasedKey(phase, podBackupHookCommandAnnotationKey): "/usr/bin/foo", phasedKey(phase, podBackupHookOnErrorAnnotationKey): string(velerov1api.HookErrorModeFail), }, expectedHook: &velerov1api.ExecHook{ Command: []string{"/usr/bin/foo"}, OnError: velerov1api.HookErrorModeFail, }, }, { name: "use the specified timeout", annotations: map[string]string{ phasedKey(phase, podBackupHookCommandAnnotationKey): "/usr/bin/foo", phasedKey(phase, podBackupHookTimeoutAnnotationKey): "5m3s", }, expectedHook: &velerov1api.ExecHook{ Command: []string{"/usr/bin/foo"}, Timeout: metav1.Duration{Duration: 5*time.Minute + 3*time.Second}, }, }, { name: "invalid timeout is logged", annotations: map[string]string{ phasedKey(phase, podBackupHookCommandAnnotationKey): "/usr/bin/foo", phasedKey(phase, podBackupHookTimeoutAnnotationKey): "invalid", }, expectedHook: &velerov1api.ExecHook{ Command: []string{"/usr/bin/foo"}, }, }, { name: "use the specified container", annotations: map[string]string{ phasedKey(phase, podBackupHookContainerAnnotationKey): "some-container", phasedKey(phase, podBackupHookCommandAnnotationKey): "/usr/bin/foo", }, expectedHook: &velerov1api.ExecHook{ Container: "some-container", Command: []string{"/usr/bin/foo"}, }, }, } for _, test := range tests { t.Run(fmt.Sprintf("%s (phase=%q)", test.name, phase), func(t *testing.T) { l := velerotest.NewLogger() hook := getPodExecHookFromAnnotations(test.annotations, phase, l) assert.Equal(t, test.expectedHook, hook) }) } } } func TestResourceHookApplicableTo(t *testing.T) { tests := []struct { name string includedNamespaces []string excludedNamespaces []string includedResources []string excludedResources []string labelSelector string namespace string resource schema.GroupResource labels labels.Set expected bool }{ { name: "allow anything", namespace: "foo", resource: schema.GroupResource{Group: "foo", Resource: "bar"}, expected: true, }, { name: "namespace in included list", includedNamespaces: []string{"a", "b"}, excludedNamespaces: []string{"c", "d"}, namespace: "b", expected: true, }, { name: "namespace not in included list", includedNamespaces: []string{"a", "b"}, namespace: "c", expected: false, }, { name: "namespace excluded", excludedNamespaces: []string{"a", "b"}, namespace: "a", expected: false, }, { name: "resource in included list", includedResources: []string{"foo.a", "bar.b"}, excludedResources: []string{"baz.c"}, resource: schema.GroupResource{Group: "a", Resource: "foo"}, expected: true, }, { name: "resource not in included list", includedResources: []string{"foo.a", "bar.b"}, resource: schema.GroupResource{Group: "c", Resource: "baz"}, expected: false, }, { name: "resource excluded", excludedResources: []string{"foo.a", "bar.b"}, resource: schema.GroupResource{Group: "b", Resource: "bar"}, expected: false, }, { name: "label selector matches", labelSelector: "a=b", labels: labels.Set{"a": "b"}, expected: true, }, { name: "label selector doesn't match", labelSelector: "a=b", labels: labels.Set{"a": "c"}, expected: false, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { h := ResourceHook{ Selector: ResourceHookSelector{ Namespaces: collections.NewIncludesExcludes().Includes(test.includedNamespaces...).Excludes(test.excludedNamespaces...), Resources: collections.NewIncludesExcludes().Includes(test.includedResources...).Excludes(test.excludedResources...), }, } if test.labelSelector != "" { selector, err := labels.Parse(test.labelSelector) require.NoError(t, err) h.Selector.LabelSelector = selector } result := h.Selector.applicableTo(test.resource, test.namespace, test.labels) assert.Equal(t, test.expected, result) }) } } func parseLabelSelectorOrDie(s string) labels.Selector { ret, err := labels.Parse(s) if err != nil { panic(err) } return ret } func TestGetPodExecRestoreHookFromAnnotations(t *testing.T) { testCases := []struct { name string inputAnnotations map[string]string expected *velerov1api.ExecRestoreHook }{ { name: "should return nil when command is missing", inputAnnotations: nil, expected: nil, }, { name: "should return nil when command is empty string", inputAnnotations: map[string]string{ podRestoreHookCommandAnnotationKey: "", }, expected: nil, }, { name: "should return a hook when 1 item command is a string", inputAnnotations: map[string]string{ podRestoreHookCommandAnnotationKey: "/usr/bin/foo", }, expected: &velerov1api.ExecRestoreHook{ Command: []string{"/usr/bin/foo"}, WaitForReady: boolptr.False(), }, }, { name: "should return a multi-item hook when command is a json array", inputAnnotations: map[string]string{ podRestoreHookCommandAnnotationKey: `["a","b","c"]`, }, expected: &velerov1api.ExecRestoreHook{ Command: []string{"a", "b", "c"}, WaitForReady: boolptr.False(), }, }, { name: "error mode continue should be in returned hook when set in annotation", inputAnnotations: map[string]string{ podRestoreHookCommandAnnotationKey: "/usr/bin/foo", podRestoreHookOnErrorAnnotationKey: string(velerov1api.HookErrorModeContinue), }, expected: &velerov1api.ExecRestoreHook{ Command: []string{"/usr/bin/foo"}, OnError: velerov1api.HookErrorModeContinue, WaitForReady: boolptr.False(), }, }, { name: "error mode fail should be in returned hook when set in annotation", inputAnnotations: map[string]string{ podRestoreHookCommandAnnotationKey: "/usr/bin/foo", podRestoreHookOnErrorAnnotationKey: string(velerov1api.HookErrorModeFail), }, expected: &velerov1api.ExecRestoreHook{ Command: []string{"/usr/bin/foo"}, OnError: velerov1api.HookErrorModeFail, WaitForReady: boolptr.False(), }, }, { name: "exec and wait timeouts should be in returned hook when set in annotations", inputAnnotations: map[string]string{ podRestoreHookCommandAnnotationKey: "/usr/bin/foo", podRestoreHookTimeoutAnnotationKey: "45s", podRestoreHookWaitTimeoutAnnotationKey: "1h", }, expected: &velerov1api.ExecRestoreHook{ Command: []string{"/usr/bin/foo"}, ExecTimeout: metav1.Duration{Duration: 45 * time.Second}, WaitTimeout: metav1.Duration{Duration: time.Hour}, WaitForReady: boolptr.False(), }, }, { name: "container should be in returned hook when set in annotation", inputAnnotations: map[string]string{ podRestoreHookCommandAnnotationKey: "/usr/bin/foo", podRestoreHookContainerAnnotationKey: "my-app", }, expected: &velerov1api.ExecRestoreHook{ Command: []string{"/usr/bin/foo"}, Container: "my-app", WaitForReady: boolptr.False(), }, }, { name: "bad exec timeout should be discarded", inputAnnotations: map[string]string{ podRestoreHookCommandAnnotationKey: "/usr/bin/foo", podRestoreHookContainerAnnotationKey: "my-app", podRestoreHookTimeoutAnnotationKey: "none", }, expected: &velerov1api.ExecRestoreHook{ Command: []string{"/usr/bin/foo"}, Container: "my-app", ExecTimeout: metav1.Duration{Duration: 0}, WaitForReady: boolptr.False(), }, }, { name: "bad wait timeout should be discarded", inputAnnotations: map[string]string{ podRestoreHookCommandAnnotationKey: "/usr/bin/foo", podRestoreHookContainerAnnotationKey: "my-app", podRestoreHookWaitTimeoutAnnotationKey: "none", }, expected: &velerov1api.ExecRestoreHook{ Command: []string{"/usr/bin/foo"}, Container: "my-app", ExecTimeout: metav1.Duration{Duration: 0}, WaitForReady: boolptr.False(), }, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { l := velerotest.NewLogger() actual := getPodExecRestoreHookFromAnnotations(tc.inputAnnotations, l) assert.Equal(t, tc.expected, actual) }) } } func TestGroupRestoreExecHooks(t *testing.T) { testCases := []struct { name string resourceRestoreHooks []ResourceRestoreHook pod *corev1api.Pod expected map[string][]PodExecRestoreHook }{ { name: "should return empty map when neither spec hooks nor annotations hooks are set", resourceRestoreHooks: nil, pod: builder.ForPod("default", "my-pod").Result(), expected: map[string][]PodExecRestoreHook{}, }, { name: "should return hook from annotation when no spec hooks are set", resourceRestoreHooks: nil, pod: builder.ForPod("default", "my-pod"). ObjectMeta(builder.WithAnnotations( podRestoreHookCommandAnnotationKey, "/usr/bin/foo", podRestoreHookContainerAnnotationKey, "container1", podRestoreHookOnErrorAnnotationKey, string(velerov1api.HookErrorModeContinue), podRestoreHookTimeoutAnnotationKey, "1s", podRestoreHookWaitTimeoutAnnotationKey, "1m", podRestoreHookWaitForReadyAnnotationKey, "true", )). Containers(&corev1api.Container{ Name: "container1", }). Result(), expected: map[string][]PodExecRestoreHook{ "container1": { { HookName: "", HookSource: HookSourceAnnotation, Hook: velerov1api.ExecRestoreHook{ Container: "container1", Command: []string{"/usr/bin/foo"}, OnError: velerov1api.HookErrorModeContinue, ExecTimeout: metav1.Duration{Duration: time.Second}, WaitTimeout: metav1.Duration{Duration: time.Minute}, WaitForReady: boolptr.True(), }, }, }, }, }, { name: "should default to first pod container when not set in annotation", resourceRestoreHooks: nil, pod: builder.ForPod("default", "my-pod"). ObjectMeta(builder.WithAnnotations( podRestoreHookCommandAnnotationKey, "/usr/bin/foo", podRestoreHookOnErrorAnnotationKey, string(velerov1api.HookErrorModeContinue), podRestoreHookTimeoutAnnotationKey, "1s", podRestoreHookWaitTimeoutAnnotationKey, "1m", )). Containers(&corev1api.Container{ Name: "container1", }). Result(), expected: map[string][]PodExecRestoreHook{ "container1": { { HookName: "", HookSource: HookSourceAnnotation, Hook: velerov1api.ExecRestoreHook{ Container: "container1", Command: []string{"/usr/bin/foo"}, OnError: velerov1api.HookErrorModeContinue, ExecTimeout: metav1.Duration{Duration: time.Second}, WaitTimeout: metav1.Duration{Duration: time.Minute}, WaitForReady: boolptr.False(), }, }, }, }, }, { name: "should return hook from spec for pod with no hook annotations", resourceRestoreHooks: []ResourceRestoreHook{ { Name: "hook1", Selector: ResourceHookSelector{}, RestoreHooks: []velerov1api.RestoreResourceHook{ { Exec: &velerov1api.ExecRestoreHook{ Container: "container1", Command: []string{"/usr/bin/foo"}, OnError: velerov1api.HookErrorModeContinue, ExecTimeout: metav1.Duration{Duration: time.Second}, WaitTimeout: metav1.Duration{Duration: time.Minute}, }, }, }, }, }, pod: builder.ForPod("default", "my-pod"). Containers(&corev1api.Container{ Name: "container1", }). Result(), expected: map[string][]PodExecRestoreHook{ "container1": { { HookName: "hook1", HookSource: HookSourceSpec, Hook: velerov1api.ExecRestoreHook{ Container: "container1", Command: []string{"/usr/bin/foo"}, OnError: velerov1api.HookErrorModeContinue, ExecTimeout: metav1.Duration{Duration: time.Second}, WaitTimeout: metav1.Duration{Duration: time.Minute}, WaitForReady: boolptr.False(), }, }, }, }, }, { name: "should default to first container pod when unset in spec hook", resourceRestoreHooks: []ResourceRestoreHook{ { Name: "hook1", Selector: ResourceHookSelector{}, RestoreHooks: []velerov1api.RestoreResourceHook{ { Exec: &velerov1api.ExecRestoreHook{ Command: []string{"/usr/bin/foo"}, OnError: velerov1api.HookErrorModeContinue, ExecTimeout: metav1.Duration{Duration: time.Second}, WaitTimeout: metav1.Duration{Duration: time.Minute}, }, }, }, }, }, pod: builder.ForPod("default", "my-pod"). Containers(&corev1api.Container{ Name: "container1", }). Result(), expected: map[string][]PodExecRestoreHook{ "container1": { { HookName: "hook1", HookSource: HookSourceSpec, Hook: velerov1api.ExecRestoreHook{ Container: "container1", Command: []string{"/usr/bin/foo"}, OnError: velerov1api.HookErrorModeContinue, ExecTimeout: metav1.Duration{Duration: time.Second}, WaitTimeout: metav1.Duration{Duration: time.Minute}, WaitForReady: boolptr.False(), }, }, }, }, }, { name: "should return hook from annotation ignoring hooks in spec", resourceRestoreHooks: []ResourceRestoreHook{ { Name: "hook1", Selector: ResourceHookSelector{}, RestoreHooks: []velerov1api.RestoreResourceHook{ { Exec: &velerov1api.ExecRestoreHook{ Container: "container2", Command: []string{"/usr/bin/bar"}, OnError: velerov1api.HookErrorModeFail, ExecTimeout: metav1.Duration{Duration: time.Hour}, WaitTimeout: metav1.Duration{Duration: time.Hour}, }, }, }, }, }, pod: builder.ForPod("default", "my-pod"). ObjectMeta(builder.WithAnnotations( podRestoreHookCommandAnnotationKey, "/usr/bin/foo", podRestoreHookContainerAnnotationKey, "container1", podRestoreHookOnErrorAnnotationKey, string(velerov1api.HookErrorModeContinue), podRestoreHookTimeoutAnnotationKey, "1s", podRestoreHookWaitTimeoutAnnotationKey, "1m", )). Containers(&corev1api.Container{ Name: "container1", }). Result(), expected: map[string][]PodExecRestoreHook{ "container1": { { HookName: "", HookSource: HookSourceAnnotation, Hook: velerov1api.ExecRestoreHook{ Container: "container1", Command: []string{"/usr/bin/foo"}, OnError: velerov1api.HookErrorModeContinue, ExecTimeout: metav1.Duration{Duration: time.Second}, WaitTimeout: metav1.Duration{Duration: time.Minute}, WaitForReady: boolptr.False(), }, }, }, }, }, { name: "should return empty map when only has init hook and pod has no hook annotations", resourceRestoreHooks: []ResourceRestoreHook{ { Name: "hook1", Selector: ResourceHookSelector{}, RestoreHooks: []velerov1api.RestoreResourceHook{ { Init: &velerov1api.InitRestoreHook{}, }, }, }, }, pod: builder.ForPod("default", "my-pod"). Containers(&corev1api.Container{ Name: "container1", }). Result(), expected: map[string][]PodExecRestoreHook{}, }, { name: "should return empty map when spec has exec hook for pod in different namespace and pod has no hook annotations", resourceRestoreHooks: []ResourceRestoreHook{ { Name: "hook1", Selector: ResourceHookSelector{ Namespaces: collections.NewIncludesExcludes().Includes("other"), }, RestoreHooks: []velerov1api.RestoreResourceHook{ { Exec: &velerov1api.ExecRestoreHook{ Container: "container1", Command: []string{"/usr/bin/foo"}, OnError: velerov1api.HookErrorModeContinue, ExecTimeout: metav1.Duration{Duration: time.Second}, WaitTimeout: metav1.Duration{Duration: time.Minute}, }, }, }, }, }, pod: builder.ForPod("default", "my-pod").Result(), expected: map[string][]PodExecRestoreHook{}, }, { name: "should return map with multiple keys when spec hooks apply to multiple containers in pod and has no hook annotations", resourceRestoreHooks: []ResourceRestoreHook{ { Name: "hook1", Selector: ResourceHookSelector{}, RestoreHooks: []velerov1api.RestoreResourceHook{ { Exec: &velerov1api.ExecRestoreHook{ Container: "container1", Command: []string{"/usr/bin/foo"}, OnError: velerov1api.HookErrorModeFail, ExecTimeout: metav1.Duration{Duration: time.Second}, WaitTimeout: metav1.Duration{Duration: time.Minute}, }, }, { Exec: &velerov1api.ExecRestoreHook{ Container: "container2", Command: []string{"/usr/bin/baz"}, OnError: velerov1api.HookErrorModeContinue, ExecTimeout: metav1.Duration{Duration: time.Second * 3}, WaitTimeout: metav1.Duration{Duration: time.Second * 3}, }, }, { Exec: &velerov1api.ExecRestoreHook{ Container: "container1", Command: []string{"/usr/bin/bar"}, OnError: velerov1api.HookErrorModeContinue, ExecTimeout: metav1.Duration{Duration: time.Second * 2}, WaitTimeout: metav1.Duration{Duration: time.Minute * 2}, }, }, }, }, { Name: "hook2", Selector: ResourceHookSelector{}, RestoreHooks: []velerov1api.RestoreResourceHook{ { Exec: &velerov1api.ExecRestoreHook{ Container: "container1", Command: []string{"/usr/bin/aaa"}, OnError: velerov1api.HookErrorModeContinue, ExecTimeout: metav1.Duration{Duration: time.Second * 4}, WaitTimeout: metav1.Duration{Duration: time.Minute * 4}, WaitForReady: boolptr.True(), }, }, }, }, }, pod: builder.ForPod("default", "my-pod"). Containers(&corev1api.Container{ Name: "container1", }). Result(), expected: map[string][]PodExecRestoreHook{ "container1": { { HookName: "hook1", HookSource: HookSourceSpec, Hook: velerov1api.ExecRestoreHook{ Container: "container1", Command: []string{"/usr/bin/foo"}, OnError: velerov1api.HookErrorModeFail, ExecTimeout: metav1.Duration{Duration: time.Second}, WaitTimeout: metav1.Duration{Duration: time.Minute}, WaitForReady: boolptr.False(), }, hookIndex: 0, }, { HookName: "hook1", HookSource: HookSourceSpec, Hook: velerov1api.ExecRestoreHook{ Container: "container1", Command: []string{"/usr/bin/bar"}, OnError: velerov1api.HookErrorModeContinue, ExecTimeout: metav1.Duration{Duration: time.Second * 2}, WaitTimeout: metav1.Duration{Duration: time.Minute * 2}, WaitForReady: boolptr.False(), }, hookIndex: 2, }, { HookName: "hook2", HookSource: HookSourceSpec, Hook: velerov1api.ExecRestoreHook{ Container: "container1", Command: []string{"/usr/bin/aaa"}, OnError: velerov1api.HookErrorModeContinue, ExecTimeout: metav1.Duration{Duration: time.Second * 4}, WaitTimeout: metav1.Duration{Duration: time.Minute * 4}, WaitForReady: boolptr.True(), }, hookIndex: 0, }, }, "container2": { { HookName: "hook1", HookSource: HookSourceSpec, Hook: velerov1api.ExecRestoreHook{ Container: "container2", Command: []string{"/usr/bin/baz"}, OnError: velerov1api.HookErrorModeContinue, ExecTimeout: metav1.Duration{Duration: time.Second * 3}, WaitTimeout: metav1.Duration{Duration: time.Second * 3}, WaitForReady: boolptr.False(), }, hookIndex: 1, }, }, }, }, } hookTracker := NewMultiHookTracker() for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { actual, err := GroupRestoreExecHooks("restore1", tc.resourceRestoreHooks, tc.pod, velerotest.NewLogger(), hookTracker) require.NoError(t, err) assert.Equal(t, tc.expected, actual) }) } } func TestGetInitContainerFromAnnotations(t *testing.T) { testCases := []struct { name string inputAnnotations map[string]string expected *corev1api.Container expectNil bool }{ { name: "should return nil when container image is empty", expectNil: true, inputAnnotations: map[string]string{ podRestoreHookInitContainerImageAnnotationKey: "", podRestoreHookInitContainerNameAnnotationKey: "restore-init", podRestoreHookInitContainerCommandAnnotationKey: "/usr/bin/data-populator", }, }, { name: "should return nil when container image is missing", expectNil: true, inputAnnotations: map[string]string{ podRestoreHookInitContainerNameAnnotationKey: "restore-init", podRestoreHookInitContainerCommandAnnotationKey: "/usr/bin/data-populator", }, }, { name: "should generate container name when container name is empty", expectNil: false, inputAnnotations: map[string]string{ podRestoreHookInitContainerImageAnnotationKey: "busy-box", podRestoreHookInitContainerNameAnnotationKey: "", podRestoreHookInitContainerCommandAnnotationKey: "/usr/bin/data-populator /user-data full", }, expected: builder.ForContainer("restore-init1", "busy-box"). Command([]string{"/usr/bin/data-populator /user-data full"}).Result(), }, { name: "should generate container name when container name is missing", expectNil: false, inputAnnotations: map[string]string{ podRestoreHookInitContainerImageAnnotationKey: "busy-box", podRestoreHookInitContainerCommandAnnotationKey: "/usr/bin/data-populator /user-data full", }, expected: builder.ForContainer("restore-init1", "busy-box"). Command([]string{"/usr/bin/data-populator /user-data full"}).Result(), }, { name: "should return expected init container when all annotations are specified", expectNil: false, expected: builder.ForContainer("restore-init1", "busy-box"). Command([]string{"/usr/bin/data-populator /user-data full"}).Result(), inputAnnotations: map[string]string{ podRestoreHookInitContainerImageAnnotationKey: "busy-box", podRestoreHookInitContainerNameAnnotationKey: "restore-init", podRestoreHookInitContainerCommandAnnotationKey: "/usr/bin/data-populator /user-data full", }, }, { name: "should return expected init container when all annotations are specified with command as a JSON array", expectNil: false, expected: builder.ForContainer("restore-init1", "busy-box"). Command([]string{"a", "b", "c"}).Result(), inputAnnotations: map[string]string{ podRestoreHookInitContainerImageAnnotationKey: "busy-box", podRestoreHookInitContainerNameAnnotationKey: "restore-init", podRestoreHookInitContainerCommandAnnotationKey: `["a","b","c"]`, }, }, { name: "should return expected init container when all annotations are specified with command as malformed a JSON array", expectNil: false, expected: builder.ForContainer("restore-init1", "busy-box"). Command([]string{"[foobarbaz"}).Result(), inputAnnotations: map[string]string{ podRestoreHookInitContainerImageAnnotationKey: "busy-box", podRestoreHookInitContainerNameAnnotationKey: "restore-init", podRestoreHookInitContainerCommandAnnotationKey: "[foobarbaz", }, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { actualInitContainer := getInitContainerFromAnnotation("test/pod1", tc.inputAnnotations, velerotest.NewLogger()) if tc.expectNil { assert.Nil(t, actualInitContainer) return } assert.NotEmpty(t, actualInitContainer.Name) assert.Equal(t, tc.expected.Image, actualInitContainer.Image) assert.Equal(t, tc.expected.Command, actualInitContainer.Command) }) } } func TestGetRestoreHooksFromSpec(t *testing.T) { testCases := []struct { name string hookSpec *velerov1api.RestoreHooks expected []ResourceRestoreHook expectedError error }{ { name: "should return empty hooks and no error when hookSpec is nil", hookSpec: nil, expected: []ResourceRestoreHook{}, expectedError: nil, }, { name: "should return empty hooks and no error when hookSpec resources is nil", hookSpec: &velerov1api.RestoreHooks{ Resources: nil, }, expected: []ResourceRestoreHook{}, expectedError: nil, }, { name: "should return empty hooks and no error when hookSpec resources is empty", hookSpec: &velerov1api.RestoreHooks{ Resources: []velerov1api.RestoreResourceHookSpec{}, }, expected: []ResourceRestoreHook{}, expectedError: nil, }, { name: "should return hooks specified in the hookSpec initContainer hooks only", hookSpec: &velerov1api.RestoreHooks{ Resources: []velerov1api.RestoreResourceHookSpec{ { Name: "h1", IncludedNamespaces: []string{"ns1", "ns2", "ns3"}, ExcludedNamespaces: []string{"ns4", "ns5", "ns6"}, IncludedResources: []string{kuberesource.Pods.Resource}, PostHooks: []velerov1api.RestoreResourceHook{ { Init: &velerov1api.InitRestoreHook{ InitContainers: []runtime.RawExtension{ builder.ForContainer("restore-init1", "busy-box"). Command([]string{"foobarbaz"}).ResultRawExtension(), builder.ForContainer("restore-init2", "busy-box"). Command([]string{"foobarbaz"}).ResultRawExtension(), }, }, }, }, }, }, }, expected: []ResourceRestoreHook{ { Name: "h1", Selector: ResourceHookSelector{ Namespaces: collections.NewIncludesExcludes().Includes("ns1", "ns2", "ns3").Excludes("ns4", "ns5", "ns6"), Resources: collections.NewIncludesExcludes().Includes(kuberesource.Pods.Resource), }, RestoreHooks: []velerov1api.RestoreResourceHook{ { Init: &velerov1api.InitRestoreHook{ InitContainers: []runtime.RawExtension{ builder.ForContainer("restore-init1", "busy-box"). Command([]string{"foobarbaz"}).ResultRawExtension(), builder.ForContainer("restore-init2", "busy-box"). Command([]string{"foobarbaz"}).ResultRawExtension(), }, }, }, }, }, }, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { actual, err := GetRestoreHooksFromSpec(tc.hookSpec) assert.Equal(t, tc.expected, actual) assert.Equal(t, tc.expectedError, err) }) } } func TestHandleRestoreHooks(t *testing.T) { testCases := []struct { name string podInput corev1api.Pod restoreHooks []ResourceRestoreHook namespaceMapping map[string]string expectedPod *corev1api.Pod expectedError error }{ { name: "should handle hook from annotation no hooks in spec on pod with no init containers", podInput: corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "app1", Namespace: "default", Annotations: map[string]string{ podRestoreHookInitContainerImageAnnotationKey: "nginx", podRestoreHookInitContainerNameAnnotationKey: "restore-init-container", podRestoreHookInitContainerCommandAnnotationKey: `["a", "b", "c"]`, }, }, }, expectedError: nil, expectedPod: &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "app1", Namespace: "default", Annotations: map[string]string{ podRestoreHookInitContainerImageAnnotationKey: "nginx", podRestoreHookInitContainerNameAnnotationKey: "restore-init-container", podRestoreHookInitContainerCommandAnnotationKey: `["a", "b", "c"]`, }, }, Spec: corev1api.PodSpec{ InitContainers: []corev1api.Container{ *builder.ForContainer("restore-init-container", "nginx"). Command([]string{"a", "b", "c"}).Result(), }, }, }, }, { name: "should handle hook from annotation no hooks in spec on pod with init containers", podInput: corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "app1", Namespace: "default", Annotations: map[string]string{ podRestoreHookInitContainerImageAnnotationKey: "nginx", podRestoreHookInitContainerNameAnnotationKey: "restore-init-container", podRestoreHookInitContainerCommandAnnotationKey: `["a", "b", "c"]`, }, }, Spec: corev1api.PodSpec{ InitContainers: []corev1api.Container{ *builder.ForContainer("init-app-step1", "busy-box"). Command([]string{"init-step1"}).Result(), *builder.ForContainer("init-app-step2", "busy-box"). Command([]string{"init-step2"}).Result(), *builder.ForContainer("init-app-step3", "busy-box"). Command([]string{"init-step3"}).Result(), }, }, }, expectedError: nil, expectedPod: &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "app1", Namespace: "default", Annotations: map[string]string{ podRestoreHookInitContainerImageAnnotationKey: "nginx", podRestoreHookInitContainerNameAnnotationKey: "restore-init-container", podRestoreHookInitContainerCommandAnnotationKey: `["a", "b", "c"]`, }, }, Spec: corev1api.PodSpec{ InitContainers: []corev1api.Container{ *builder.ForContainer("restore-init-container", "nginx"). Command([]string{"a", "b", "c"}).Result(), *builder.ForContainer("init-app-step1", "busy-box"). Command([]string{"init-step1"}).Result(), *builder.ForContainer("init-app-step2", "busy-box"). Command([]string{"init-step2"}).Result(), *builder.ForContainer("init-app-step3", "busy-box"). Command([]string{"init-step3"}).Result(), }, }, }, }, { name: "should handle hook from annotation ignoring hooks in spec", podInput: corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "app1", Namespace: "default", Annotations: map[string]string{ podRestoreHookInitContainerImageAnnotationKey: "nginx", podRestoreHookInitContainerNameAnnotationKey: "restore-init-container", podRestoreHookInitContainerCommandAnnotationKey: `["a", "b", "c"]`, }, }, Spec: corev1api.PodSpec{ InitContainers: []corev1api.Container{ *builder.ForContainer("init-app-step1", "busy-box"). Command([]string{"init-step1"}).Result(), *builder.ForContainer("init-app-step2", "busy-box"). Command([]string{"init-step2"}).Result(), *builder.ForContainer("init-app-step3", "busy-box"). Command([]string{"init-step3"}).Result(), }, }, }, expectedError: nil, expectedPod: &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "app1", Namespace: "default", Annotations: map[string]string{ podRestoreHookInitContainerImageAnnotationKey: "nginx", podRestoreHookInitContainerNameAnnotationKey: "restore-init-container", podRestoreHookInitContainerCommandAnnotationKey: `["a", "b", "c"]`, }, }, Spec: corev1api.PodSpec{ InitContainers: []corev1api.Container{ *builder.ForContainer("restore-init-container", "nginx"). Command([]string{"a", "b", "c"}).Result(), *builder.ForContainer("init-app-step1", "busy-box"). Command([]string{"init-step1"}).Result(), *builder.ForContainer("init-app-step2", "busy-box"). Command([]string{"init-step2"}).Result(), *builder.ForContainer("init-app-step3", "busy-box"). Command([]string{"init-step3"}).Result(), }, }, }, restoreHooks: []ResourceRestoreHook{ { Name: "ignore-hook1", Selector: ResourceHookSelector{ Namespaces: collections.NewIncludesExcludes().Includes("default"), Resources: collections.NewIncludesExcludes().Includes(kuberesource.Pods.Resource), }, RestoreHooks: []velerov1api.RestoreResourceHook{ { Init: &velerov1api.InitRestoreHook{ InitContainers: []runtime.RawExtension{ builder.ForContainer("should-not exist", "does-not-matter"). Command([]string{""}).ResultRawExtension(), }, }, }, }, }, }, }, { name: "should handle hook from spec on pod with no init containers", podInput: corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "app1", Namespace: "default", }, Spec: corev1api.PodSpec{ InitContainers: []corev1api.Container{}, }, }, expectedError: nil, expectedPod: &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "app1", Namespace: "default", }, Spec: corev1api.PodSpec{ InitContainers: []corev1api.Container{ *builder.ForContainer("restore-init-container-1", "nginx"). Command([]string{"a", "b", "c"}).Result(), *builder.ForContainer("restore-init-container-2", "nginx"). Command([]string{"a", "b", "c"}).Result(), }, }, }, restoreHooks: []ResourceRestoreHook{ { Name: "hook1", Selector: ResourceHookSelector{ Namespaces: collections.NewIncludesExcludes().Includes("default"), Resources: collections.NewIncludesExcludes().Includes(kuberesource.Pods.Resource), }, RestoreHooks: []velerov1api.RestoreResourceHook{ { Init: &velerov1api.InitRestoreHook{ InitContainers: []runtime.RawExtension{ builder.ForContainer("restore-init-container-1", "nginx"). Command([]string{"a", "b", "c"}).ResultRawExtension(), builder.ForContainer("restore-init-container-2", "nginx"). Command([]string{"a", "b", "c"}).ResultRawExtension(), }, }, }, }, }, }, }, { name: "should handle hook from spec when no restore hook annotation and existing init containers", podInput: corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "app1", Namespace: "default", }, Spec: corev1api.PodSpec{ InitContainers: []corev1api.Container{ *builder.ForContainer("init-app-step1", "busy-box"). Command([]string{"init-step1"}).Result(), *builder.ForContainer("init-app-step2", "busy-box"). Command([]string{"init-step2"}).Result(), *builder.ForContainer("init-app-step3", "busy-box"). Command([]string{"init-step3"}).Result(), }, }, }, expectedError: nil, expectedPod: &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "app1", Namespace: "default", }, Spec: corev1api.PodSpec{ InitContainers: []corev1api.Container{ *builder.ForContainer("restore-init-container-1", "nginx"). Command([]string{"a", "b", "c"}).Result(), *builder.ForContainer("restore-init-container-2", "nginx"). Command([]string{"a", "b", "c"}).Result(), *builder.ForContainer("init-app-step1", "busy-box"). Command([]string{"init-step1"}).Result(), *builder.ForContainer("init-app-step2", "busy-box"). Command([]string{"init-step2"}).Result(), *builder.ForContainer("init-app-step3", "busy-box"). Command([]string{"init-step3"}).Result(), }, }, }, restoreHooks: []ResourceRestoreHook{ { Name: "hook1", Selector: ResourceHookSelector{ Namespaces: collections.NewIncludesExcludes().Includes("default"), Resources: collections.NewIncludesExcludes().Includes(kuberesource.Pods.Resource), }, RestoreHooks: []velerov1api.RestoreResourceHook{ { Init: &velerov1api.InitRestoreHook{ InitContainers: []runtime.RawExtension{ builder.ForContainer("restore-init-container-1", "nginx"). Command([]string{"a", "b", "c"}).ResultRawExtension(), builder.ForContainer("restore-init-container-2", "nginx"). Command([]string{"a", "b", "c"}).ResultRawExtension(), }, }, }, }, }, }, }, { name: "should not apply any restore hook init containers when resource hook selector mismatch", podInput: corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "app1", Namespace: "default", }, }, expectedError: nil, expectedPod: &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "app1", Namespace: "default", }, }, restoreHooks: []ResourceRestoreHook{ { Name: "hook1", Selector: ResourceHookSelector{ Namespaces: collections.NewIncludesExcludes().Excludes("default"), Resources: collections.NewIncludesExcludes().Includes(kuberesource.Pods.Resource), }, RestoreHooks: []velerov1api.RestoreResourceHook{ { Init: &velerov1api.InitRestoreHook{ InitContainers: []runtime.RawExtension{ builder.ForContainer("restore-init-container-1", "nginx"). Command([]string{"a", "b", "c"}).ResultRawExtension(), builder.ForContainer("restore-init-container-2", "nginx"). Command([]string{"a", "b", "c"}).ResultRawExtension(), }, }, }, }, }, }, }, { name: "should preserve restore-wait init container when it is the only existing init container", podInput: corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "app1", Namespace: "default", }, Spec: corev1api.PodSpec{ InitContainers: []corev1api.Container{ *builder.ForContainer("restore-wait", "bus-box"). Command([]string{"pod-volume-restore"}).Result(), }, }, }, expectedError: nil, expectedPod: &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "app1", Namespace: "default", }, Spec: corev1api.PodSpec{ InitContainers: []corev1api.Container{ *builder.ForContainer("restore-wait", "bus-box"). Command([]string{"pod-volume-restore"}).Result(), *builder.ForContainer("restore-init-container-1", "nginx"). Command([]string{"a", "b", "c"}).Result(), *builder.ForContainer("restore-init-container-2", "nginx"). Command([]string{"a", "b", "c"}).Result(), }, }, }, restoreHooks: []ResourceRestoreHook{ { Name: "hook1", Selector: ResourceHookSelector{ Namespaces: collections.NewIncludesExcludes().Includes("default"), Resources: collections.NewIncludesExcludes().Includes(kuberesource.Pods.Resource), }, RestoreHooks: []velerov1api.RestoreResourceHook{ { Init: &velerov1api.InitRestoreHook{ InitContainers: []runtime.RawExtension{ builder.ForContainer("restore-init-container-1", "nginx"). Command([]string{"a", "b", "c"}).ResultRawExtension(), builder.ForContainer("restore-init-container-2", "nginx"). Command([]string{"a", "b", "c"}).ResultRawExtension(), }, }, }, }, }, }, }, { name: "should preserve restore-wait init container when it exits with other init containers", podInput: corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "app1", Namespace: "default", }, Spec: corev1api.PodSpec{ InitContainers: []corev1api.Container{ *builder.ForContainer("restore-wait", "bus-box"). Command([]string{"pod-volume-restore"}).Result(), *builder.ForContainer("init-app-step1", "busy-box"). Command([]string{"init-step1"}).Result(), *builder.ForContainer("init-app-step2", "busy-box"). Command([]string{"init-step2"}).Result(), }, }, }, expectedError: nil, expectedPod: &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "app1", Namespace: "default", }, Spec: corev1api.PodSpec{ InitContainers: []corev1api.Container{ *builder.ForContainer("restore-wait", "bus-box"). Command([]string{"pod-volume-restore"}).Result(), *builder.ForContainer("restore-init-container-1", "nginx"). Command([]string{"a", "b", "c"}).Result(), *builder.ForContainer("restore-init-container-2", "nginx"). Command([]string{"a", "b", "c"}).Result(), *builder.ForContainer("init-app-step1", "busy-box"). Command([]string{"init-step1"}).Result(), *builder.ForContainer("init-app-step2", "busy-box"). Command([]string{"init-step2"}).Result(), }, }, }, restoreHooks: []ResourceRestoreHook{ { Name: "hook1", Selector: ResourceHookSelector{ Namespaces: collections.NewIncludesExcludes().Includes("default"), Resources: collections.NewIncludesExcludes().Includes(kuberesource.Pods.Resource), }, RestoreHooks: []velerov1api.RestoreResourceHook{ { Init: &velerov1api.InitRestoreHook{ InitContainers: []runtime.RawExtension{ builder.ForContainer("restore-init-container-1", "nginx"). Command([]string{"a", "b", "c"}).ResultRawExtension(), builder.ForContainer("restore-init-container-2", "nginx"). Command([]string{"a", "b", "c"}).ResultRawExtension(), }, }, }, }, }, }, }, { name: "should not apply any restore hook init containers when resource hook is nil", podInput: corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "app1", Namespace: "default", }, }, expectedError: nil, expectedPod: &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "app1", Namespace: "default", }, }, restoreHooks: nil, }, { name: "should not apply any restore hook init containers when resource hook is empty", podInput: corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "app1", Namespace: "default", }, }, expectedError: nil, expectedPod: &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "app1", Namespace: "default", }, }, restoreHooks: []ResourceRestoreHook{}, }, { name: "should not apply init container when the namespace mapping is provided and the hook points to the original namespace", podInput: corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "app1", Namespace: "default", }, Spec: corev1api.PodSpec{}, }, expectedError: nil, expectedPod: &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "app1", Namespace: "default", }, Spec: corev1api.PodSpec{}, }, restoreHooks: []ResourceRestoreHook{ { Name: "hook1", Selector: ResourceHookSelector{ Namespaces: collections.NewIncludesExcludes().Includes("default"), Resources: collections.NewIncludesExcludes().Includes(kuberesource.Pods.Resource), }, RestoreHooks: []velerov1api.RestoreResourceHook{ { Init: &velerov1api.InitRestoreHook{ InitContainers: []runtime.RawExtension{ builder.ForContainer("restore-init-container-1", "nginx"). Command([]string{"a", "b", "c"}).ResultRawExtension(), }, }, }, }, }, }, namespaceMapping: map[string]string{"default": "new"}, }, { name: "should apply init container when the namespace mapping is provided and the hook points to the new namespace", podInput: corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "app1", Namespace: "default", }, Spec: corev1api.PodSpec{}, }, expectedError: nil, expectedPod: &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "app1", Namespace: "default", }, Spec: corev1api.PodSpec{ InitContainers: []corev1api.Container{ *builder.ForContainer("restore-init-container-1", "nginx"). Command([]string{"a", "b", "c"}).Result(), }, }, }, restoreHooks: []ResourceRestoreHook{ { Name: "hook1", Selector: ResourceHookSelector{ Namespaces: collections.NewIncludesExcludes().Includes("new"), Resources: collections.NewIncludesExcludes().Includes(kuberesource.Pods.Resource), }, RestoreHooks: []velerov1api.RestoreResourceHook{ { Init: &velerov1api.InitRestoreHook{ InitContainers: []runtime.RawExtension{ builder.ForContainer("restore-init-container-1", "nginx"). Command([]string{"a", "b", "c"}).ResultRawExtension(), }, }, }, }, }, }, namespaceMapping: map[string]string{"default": "new"}, }, { name: "Invalid InitContainer in Restore hook should return nil as pod, and error.", podInput: corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "app1", Namespace: "new", }, Spec: corev1api.PodSpec{}, }, expectedError: fmt.Errorf("invalid InitContainer in restore hook, it doesn't have Command, Name or Image field"), expectedPod: nil, restoreHooks: []ResourceRestoreHook{ { Name: "hook1", Selector: ResourceHookSelector{ Namespaces: collections.NewIncludesExcludes().Includes("new"), Resources: collections.NewIncludesExcludes().Includes(kuberesource.Pods.Resource), }, RestoreHooks: []velerov1api.RestoreResourceHook{ { Init: &velerov1api.InitRestoreHook{ InitContainers: []runtime.RawExtension{ builder.ForContainer("restore-init-container-1", "nginx"). ResultRawExtension(), }, }, }, }, }, }, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { handler := InitContainerRestoreHookHandler{} podMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&tc.podInput) require.NoError(t, err) actual, err := handler.HandleRestoreHooks(velerotest.NewLogger(), kuberesource.Pods, &unstructured.Unstructured{Object: podMap}, tc.restoreHooks, tc.namespaceMapping) assert.Equal(t, tc.expectedError, err) if actual != nil { actualPod := new(corev1api.Pod) err = runtime.DefaultUnstructuredConverter.FromUnstructured(actual.UnstructuredContent(), actualPod) require.NoError(t, err) assert.Equal(t, tc.expectedPod, actualPod) } }) } } func TestValidateContainer(t *testing.T) { valid := `{"name": "test", "image": "busybox", "command": ["pwd"]}` noName := `{"image": "busybox", "command": ["pwd"]}` noImage := `{"name": "test", "command": ["pwd"]}` noCommand := `{"name": "test", "image": "busybox"}` expectedError := fmt.Errorf("invalid InitContainer in restore hook, it doesn't have Command, Name or Image field") // valid string should return nil as result. require.NoError(t, ValidateContainer([]byte(valid))) // noName string should return expected error as result. assert.Equal(t, expectedError, ValidateContainer([]byte(noName))) // noImage string should return expected error as result. assert.Equal(t, expectedError, ValidateContainer([]byte(noImage))) // noCommand string should return expected error as result. assert.Equal(t, expectedError, ValidateContainer([]byte(noCommand))) } func TestBackupHookTracker(t *testing.T) { type podWithHook struct { item runtime.Unstructured hooks []ResourceHook hookErrorsByContainer map[string]error expectedPodHook *velerov1api.ExecHook expectedPodHookError error expectedError error } test1 := []struct { name string phase HookPhase groupResource string pods []podWithHook hookTracker *HookTracker expectedHookAttempted int expectedHookFailed int }{ { name: "a pod with spec hooks, no error", phase: PhasePre, groupResource: "pods", hookTracker: NewHookTracker(), expectedHookAttempted: 2, expectedHookFailed: 0, pods: []podWithHook{ { item: velerotest.UnstructuredOrDie(` { "apiVersion": "v1", "kind": "Pod", "metadata": { "namespace": "ns", "name": "name" } }`), hooks: []ResourceHook{ { Name: "hook1", Pre: []velerov1api.BackupResourceHook{ { Exec: &velerov1api.ExecHook{ Container: "1a", Command: []string{"pre-1a"}, }, }, { Exec: &velerov1api.ExecHook{ Container: "1b", Command: []string{"pre-1b"}, }, }, }, }, }, }, }, }, { name: "a pod with spec hooks and same container under different hook name, no error", phase: PhasePre, groupResource: "pods", hookTracker: NewHookTracker(), expectedHookAttempted: 4, expectedHookFailed: 0, pods: []podWithHook{ { item: velerotest.UnstructuredOrDie(` { "apiVersion": "v1", "kind": "Pod", "metadata": { "namespace": "ns", "name": "name" } }`), hooks: []ResourceHook{ { Name: "hook1", Pre: []velerov1api.BackupResourceHook{ { Exec: &velerov1api.ExecHook{ Container: "1a", Command: []string{"pre-1a"}, }, }, { Exec: &velerov1api.ExecHook{ Container: "1b", Command: []string{"pre-1b"}, }, }, }, }, { Name: "hook2", Pre: []velerov1api.BackupResourceHook{ { Exec: &velerov1api.ExecHook{ Container: "1a", Command: []string{"2a"}, }, }, { Exec: &velerov1api.ExecHook{ Container: "2b", Command: []string{"2b"}, }, }, }, }, }, }, }, }, { name: "a pod with spec hooks, on error=fail", phase: PhasePre, groupResource: "pods", hookTracker: NewHookTracker(), expectedHookAttempted: 4, expectedHookFailed: 2, pods: []podWithHook{ { item: velerotest.UnstructuredOrDie(` { "apiVersion": "v1", "kind": "Pod", "metadata": { "namespace": "ns", "name": "name" } }`), hooks: []ResourceHook{ { Name: "hook1", Pre: []velerov1api.BackupResourceHook{ { Exec: &velerov1api.ExecHook{ Container: "1a", Command: []string{"1a"}, OnError: velerov1api.HookErrorModeContinue, }, }, { Exec: &velerov1api.ExecHook{ Container: "1b", Command: []string{"1b"}, }, }, }, }, { Name: "hook2", Pre: []velerov1api.BackupResourceHook{ { Exec: &velerov1api.ExecHook{ Container: "2", Command: []string{"2"}, OnError: velerov1api.HookErrorModeFail, }, }, }, }, { Name: "hook3", Pre: []velerov1api.BackupResourceHook{ { Exec: &velerov1api.ExecHook{ Container: "3", Command: []string{"3"}, }, }, }, }, }, hookErrorsByContainer: map[string]error{ "1a": errors.New("1a error, but continue"), "2": errors.New("2 error, fail"), }, }, }, }, { name: "a pod with annotation and spec hooks", phase: PhasePre, groupResource: "pods", hookTracker: NewHookTracker(), expectedHookAttempted: 1, expectedHookFailed: 0, pods: []podWithHook{ { item: velerotest.UnstructuredOrDie(` { "apiVersion": "v1", "kind": "Pod", "metadata": { "namespace": "ns", "name": "name", "annotations": { "hook.backup.velero.io/container": "c", "hook.backup.velero.io/command": "/bin/ls" } } }`), expectedPodHook: &velerov1api.ExecHook{ Container: "c", Command: []string{"/bin/ls"}, }, hooks: []ResourceHook{ { Name: "hook1", Pre: []velerov1api.BackupResourceHook{ { Exec: &velerov1api.ExecHook{ Container: "1a", Command: []string{"1a"}, OnError: velerov1api.HookErrorModeContinue, }, }, { Exec: &velerov1api.ExecHook{ Container: "1b", Command: []string{"1b"}, }, }, }, }, }, }, }, }, { name: "a pod with annotation, on error=fail", phase: PhasePre, groupResource: "pods", hookTracker: NewHookTracker(), expectedHookAttempted: 1, expectedHookFailed: 1, pods: []podWithHook{ { item: velerotest.UnstructuredOrDie(` { "apiVersion": "v1", "kind": "Pod", "metadata": { "namespace": "ns", "name": "name", "annotations": { "hook.backup.velero.io/container": "c", "hook.backup.velero.io/command": "/bin/ls", "hook.backup.velero.io/on-error": "Fail" } } }`), expectedPodHook: &velerov1api.ExecHook{ Container: "c", Command: []string{"/bin/ls"}, OnError: velerov1api.HookErrorModeFail, }, expectedPodHookError: errors.New("pod hook error"), }, }, }, { name: "two pods, one with annotation, the other with spec", phase: PhasePre, groupResource: "pods", hookTracker: NewHookTracker(), expectedHookAttempted: 3, expectedHookFailed: 1, pods: []podWithHook{ { item: velerotest.UnstructuredOrDie(` { "apiVersion": "v1", "kind": "Pod", "metadata": { "namespace": "ns", "name": "name", "annotations": { "hook.backup.velero.io/container": "c", "hook.backup.velero.io/command": "/bin/ls", "hook.backup.velero.io/on-error": "Fail" } } }`), expectedPodHook: &velerov1api.ExecHook{ Container: "c", Command: []string{"/bin/ls"}, OnError: velerov1api.HookErrorModeFail, }, expectedPodHookError: errors.New("pod hook error"), }, { item: velerotest.UnstructuredOrDie(` { "apiVersion": "v1", "kind": "Pod", "metadata": { "namespace": "ns", "name": "name" } }`), hooks: []ResourceHook{ { Name: "hook1", Pre: []velerov1api.BackupResourceHook{ { Exec: &velerov1api.ExecHook{ Container: "1a", Command: []string{"pre-1a"}, }, }, { Exec: &velerov1api.ExecHook{ Container: "1b", Command: []string{"pre-1b"}, }, }, }, }, }, }, }, }, } for _, test := range test1 { t.Run(test.name, func(t *testing.T) { podCommandExecutor := &velerotest.MockPodCommandExecutor{} defer podCommandExecutor.AssertExpectations(t) h := &DefaultItemHookHandler{ PodCommandExecutor: podCommandExecutor, } groupResource := schema.ParseGroupResource(test.groupResource) hookTracker := test.hookTracker for _, pod := range test.pods { if pod.expectedPodHook != nil { podCommandExecutor.On("ExecutePodCommand", mock.Anything, pod.item.UnstructuredContent(), "ns", "name", "", pod.expectedPodHook).Return(pod.expectedPodHookError) } else { hookLoop: for _, resourceHook := range pod.hooks { for _, hook := range resourceHook.Pre { hookError := pod.hookErrorsByContainer[hook.Exec.Container] podCommandExecutor.On("ExecutePodCommand", mock.Anything, pod.item.UnstructuredContent(), "ns", "name", resourceHook.Name, hook.Exec).Return(hookError) if hookError != nil && hook.Exec.OnError == velerov1api.HookErrorModeFail { break hookLoop } } for _, hook := range resourceHook.Post { hookError := pod.hookErrorsByContainer[hook.Exec.Container] podCommandExecutor.On("ExecutePodCommand", mock.Anything, pod.item.UnstructuredContent(), "ns", "name", resourceHook.Name, hook.Exec).Return(hookError) if hookError != nil && hook.Exec.OnError == velerov1api.HookErrorModeFail { break hookLoop } } } } h.HandleHooks(velerotest.NewLogger(), groupResource, pod.item, pod.hooks, test.phase, hookTracker) } actualAtemptted, actualFailed := hookTracker.Stat() assert.Equal(t, test.expectedHookAttempted, actualAtemptted) assert.Equal(t, test.expectedHookFailed, actualFailed) }) } } func TestRestoreHookTrackerAdd(t *testing.T) { testCases := []struct { name string resourceRestoreHooks []ResourceRestoreHook pod *corev1api.Pod hookTracker *MultiHookTracker expectedCnt int }{ { name: "neither spec hooks nor annotations hooks are set", resourceRestoreHooks: nil, pod: builder.ForPod("default", "my-pod").Result(), hookTracker: NewMultiHookTracker(), expectedCnt: 0, }, { name: "a hook specified in pod annotation", resourceRestoreHooks: nil, pod: builder.ForPod("default", "my-pod"). ObjectMeta(builder.WithAnnotations( podRestoreHookCommandAnnotationKey, "/usr/bin/foo", podRestoreHookContainerAnnotationKey, "container1", podRestoreHookOnErrorAnnotationKey, string(velerov1api.HookErrorModeContinue), podRestoreHookTimeoutAnnotationKey, "1s", podRestoreHookWaitTimeoutAnnotationKey, "1m", podRestoreHookWaitForReadyAnnotationKey, "true", )). Containers(&corev1api.Container{ Name: "container1", }). Result(), hookTracker: NewMultiHookTracker(), expectedCnt: 1, }, { name: "two hooks specified in restore spec", resourceRestoreHooks: []ResourceRestoreHook{ { Name: "hook1", Selector: ResourceHookSelector{}, RestoreHooks: []velerov1api.RestoreResourceHook{ { Exec: &velerov1api.ExecRestoreHook{ Container: "container1", Command: []string{"/usr/bin/foo"}, OnError: velerov1api.HookErrorModeContinue, ExecTimeout: metav1.Duration{Duration: time.Second}, WaitTimeout: metav1.Duration{Duration: time.Minute}, }, }, { Exec: &velerov1api.ExecRestoreHook{ Container: "container2", Command: []string{"/usr/bin/foo"}, OnError: velerov1api.HookErrorModeContinue, ExecTimeout: metav1.Duration{Duration: time.Second}, WaitTimeout: metav1.Duration{Duration: time.Minute}, }, }, }, }, }, pod: builder.ForPod("default", "my-pod"). Containers(&corev1api.Container{ Name: "container1", }, &corev1api.Container{ Name: "container2", }). Result(), hookTracker: NewMultiHookTracker(), expectedCnt: 2, }, { name: "both spec hooks and annotations hooks are set", resourceRestoreHooks: []ResourceRestoreHook{ { Name: "hook1", Selector: ResourceHookSelector{}, RestoreHooks: []velerov1api.RestoreResourceHook{ { Exec: &velerov1api.ExecRestoreHook{ Container: "container1", Command: []string{"/usr/bin/foo2"}, OnError: velerov1api.HookErrorModeContinue, ExecTimeout: metav1.Duration{Duration: time.Second}, WaitTimeout: metav1.Duration{Duration: time.Minute}, }, }, }, }, }, pod: builder.ForPod("default", "my-pod"). ObjectMeta(builder.WithAnnotations( podRestoreHookCommandAnnotationKey, "/usr/bin/foo", podRestoreHookContainerAnnotationKey, "container1", podRestoreHookOnErrorAnnotationKey, string(velerov1api.HookErrorModeContinue), podRestoreHookTimeoutAnnotationKey, "1s", podRestoreHookWaitTimeoutAnnotationKey, "1m", podRestoreHookWaitForReadyAnnotationKey, "true", )). Containers(&corev1api.Container{ Name: "container1", }). Result(), hookTracker: NewMultiHookTracker(), expectedCnt: 1, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { _, _ = GroupRestoreExecHooks("restore1", tc.resourceRestoreHooks, tc.pod, velerotest.NewLogger(), tc.hookTracker) if _, ok := tc.hookTracker.trackers["restore1"]; !ok { return } tracker := tc.hookTracker.trackers["restore1"].tracker assert.Len(t, tracker, tc.expectedCnt) }) } } ================================================ FILE: internal/hook/wait_exec_hook_handler.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package hook import ( "context" "fmt" "time" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/cache" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/podexec" "github.com/vmware-tanzu/velero/pkg/util/boolptr" "github.com/vmware-tanzu/velero/pkg/util/kube" ) type WaitExecHookHandler interface { HandleHooks( ctx context.Context, log logrus.FieldLogger, pod *corev1api.Pod, byContainer map[string][]PodExecRestoreHook, multiHookTracker *MultiHookTracker, restoreName string, ) []error } type ListWatchFactory interface { NewListWatch(namespace string, selector fields.Selector) cache.ListerWatcher } type DefaultListWatchFactory struct { PodsGetter cache.Getter } type HookErrInfo struct { Namespace string Err error } func (d *DefaultListWatchFactory) NewListWatch(namespace string, selector fields.Selector) cache.ListerWatcher { return cache.NewListWatchFromClient(d.PodsGetter, "pods", namespace, selector) } var _ ListWatchFactory = &DefaultListWatchFactory{} type DefaultWaitExecHookHandler struct { ListWatchFactory ListWatchFactory PodCommandExecutor podexec.PodCommandExecutor } var _ WaitExecHookHandler = &DefaultWaitExecHookHandler{} func (e *DefaultWaitExecHookHandler) HandleHooks( ctx context.Context, log logrus.FieldLogger, pod *corev1api.Pod, byContainer map[string][]PodExecRestoreHook, multiHookTracker *MultiHookTracker, restoreName string, ) []error { if pod == nil { return nil } // If hooks are defined for a container that does not exist in the pod log a warning and discard // those hooks to avoid waiting for a container that will never become ready. After that if // there are no hooks left to be executed return immediately. for containerName := range byContainer { if !podHasContainer(pod, containerName) { log.Warningf("Pod %s does not have container %s: discarding post-restore exec hooks", kube.NamespaceAndName(pod), containerName) delete(byContainer, containerName) } } if len(byContainer) == 0 { return nil } // Every hook in every container can have its own wait timeout. Rather than setting up separate // contexts for each, find the largest wait timeout for any hook that should be executed in // the pod and watch the pod for up to that long. Before executing any hook in a container, // check if that hook has a timeout and skip execution if expired. ctx, cancel := context.WithCancel(ctx) maxWait := maxHookWait(byContainer) // If no hook has a wait timeout then this function will continue waiting for containers to // become ready until the shared hook context is canceled. if maxWait > 0 { ctx, cancel = context.WithTimeout(ctx, maxWait) } waitStart := time.Now() var errors []error // The first time this handler is called after a container starts running it will execute all // pending hooks for that container. Subsequent invocations of this handler will never execute // hooks in that container. It uses the byContainer map to keep track of which containers have // not yet been observed to be running. It relies on the Informer not to be called concurrently. // When a container is observed running and its hooks are executed, the container is deleted // from the byContainer map. When the map is empty the watch is ended. handler := func(newObj any) { newPod, ok := newObj.(*corev1api.Pod) if !ok { return } podLog := log.WithFields( logrus.Fields{ "pod": kube.NamespaceAndName(newPod), }, ) if newPod.Status.Phase == corev1api.PodSucceeded || newPod.Status.Phase == corev1api.PodFailed { err := fmt.Errorf("pod entered phase %s before some post-restore exec hooks ran", newPod.Status.Phase) podLog.Warning(err) cancel() return } for containerName, hooks := range byContainer { if !isContainerUp(newPod, containerName, hooks) { podLog.Infof("Container %s is not up: post-restore hooks will not yet be executed", containerName) continue } podMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(newPod) if err != nil { podLog.WithError(err).Error("error unstructuring pod") cancel() return } // Sequentially run all hooks for the ready container. The container's hooks are not // removed from the byContainer map until all have completed so that if one fails // remaining unexecuted hooks can be handled by the outer function. for i, hook := range hooks { // This indicates to the outer function not to handle this hook as unexecuted in // case of terminating before deleting this container's slice of hooks from the // byContainer map. byContainer[containerName][i].executed = true hookLog := podLog.WithFields( logrus.Fields{ "hookSource": hook.HookSource, "hookType": "exec", "hookPhase": "post", }, ) // Check the individual hook's wait timeout is not expired if hook.Hook.WaitTimeout.Duration != 0 && time.Since(waitStart) > hook.Hook.WaitTimeout.Duration { err := fmt.Errorf("hook %s in container %s expired before executing", hook.HookName, hook.Hook.Container) hookLog.Error(err) errors = append(errors, err) errTracker := multiHookTracker.Record(restoreName, newPod.Namespace, newPod.Name, hook.Hook.Container, hook.HookSource, hook.HookName, HookPhase(""), hook.hookIndex, true, err) if errTracker != nil { hookLog.WithError(errTracker).Warn("Error recording the hook in hook tracker") } if hook.Hook.OnError == velerov1api.HookErrorModeFail { cancel() return } } eh := &velerov1api.ExecHook{ Container: hook.Hook.Container, Command: hook.Hook.Command, OnError: hook.Hook.OnError, Timeout: hook.Hook.ExecTimeout, } hookFailed := false var hookErr error if hookErr = e.PodCommandExecutor.ExecutePodCommand(hookLog, podMap, pod.Namespace, pod.Name, hook.HookName, eh); hookErr != nil { hookLog.WithError(hookErr).Error("Error executing hook") hookErr = fmt.Errorf("hook %s in container %s failed to execute, err: %v", hook.HookName, hook.Hook.Container, hookErr) errors = append(errors, hookErr) hookFailed = true } errTracker := multiHookTracker.Record(restoreName, newPod.Namespace, newPod.Name, hook.Hook.Container, hook.HookSource, hook.HookName, HookPhase(""), hook.hookIndex, hookFailed, hookErr) if errTracker != nil { hookLog.WithError(errTracker).Warn("Error recording the hook in hook tracker") } if hookErr != nil && hook.Hook.OnError == velerov1api.HookErrorModeFail { cancel() return } } delete(byContainer, containerName) } if len(byContainer) == 0 { cancel() } } selector := fields.OneTermEqualSelector("metadata.name", pod.Name) lw := e.ListWatchFactory.NewListWatch(pod.Namespace, selector) _, podWatcher := cache.NewInformerWithOptions(cache.InformerOptions{ ListerWatcher: lw, ObjectType: pod, ResyncPeriod: 0, Handler: cache.ResourceEventHandlerFuncs{ AddFunc: handler, UpdateFunc: func(_, newObj any) { handler(newObj) }, DeleteFunc: func(obj any) { err := fmt.Errorf("pod %s deleted before all hooks were executed", kube.NamespaceAndName(pod)) log.Error(err) cancel() }, }, }, ) podWatcher.Run(ctx.Done()) // There are some cases where this function could return with unexecuted hooks: the pod may // be deleted, a hook could fail, or it may timeout waiting for // containers to become ready. // Each unexecuted hook is logged as an error and this error will be returned from this function. for _, hooks := range byContainer { for _, hook := range hooks { if hook.executed { continue } err := fmt.Errorf("hook %s in container %s in pod %s not executed: %v", hook.HookName, hook.Hook.Container, kube.NamespaceAndName(pod), ctx.Err()) hookLog := log.WithFields( logrus.Fields{ "hookSource": hook.HookSource, "hookType": "exec", "hookPhase": "post", }, ) errTracker := multiHookTracker.Record(restoreName, pod.Namespace, pod.Name, hook.Hook.Container, hook.HookSource, hook.HookName, HookPhase(""), hook.hookIndex, true, err) if errTracker != nil { hookLog.WithError(errTracker).Warn("Error recording the hook in hook tracker") } hookLog.Error(err) errors = append(errors, err) } } return errors } func podHasContainer(pod *corev1api.Pod, containerName string) bool { if pod == nil { return false } for _, c := range pod.Spec.Containers { if c.Name == containerName { return true } } return false } func isContainerUp(pod *corev1api.Pod, containerName string, hooks []PodExecRestoreHook) bool { if pod == nil { return false } var waitForReady bool for _, hook := range hooks { if boolptr.IsSetToTrue(hook.Hook.WaitForReady) { waitForReady = true break } } for _, cs := range pod.Status.ContainerStatuses { if cs.Name != containerName { continue } if waitForReady { return cs.Ready } return cs.State.Running != nil } return false } // maxHookWait returns 0 to mean wait indefinitely. Any hook without a wait timeout will cause this // function to return 0. func maxHookWait(byContainer map[string][]PodExecRestoreHook) time.Duration { var maxWait time.Duration for _, hooks := range byContainer { for _, hook := range hooks { if hook.Hook.WaitTimeout.Duration <= 0 { return 0 } if hook.Hook.WaitTimeout.Duration > maxWait { maxWait = hook.Hook.WaitTimeout.Duration } } } return maxWait } ================================================ FILE: internal/hook/wait_exec_hook_handler_test.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package hook import ( "context" "testing" "time" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/cache" fcache "k8s.io/client-go/tools/cache/testing" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" velerotest "github.com/vmware-tanzu/velero/pkg/test" "github.com/vmware-tanzu/velero/pkg/util/boolptr" ) type fakeListWatchFactory struct { lw cache.ListerWatcher } func (f *fakeListWatchFactory) NewListWatch(ns string, selector fields.Selector) cache.ListerWatcher { return f.lw } var _ ListWatchFactory = &fakeListWatchFactory{} func TestWaitExecHandleHooks(t *testing.T) { type change struct { // delta to wait since last change applied or pod added wait time.Duration updated *corev1api.Pod } type expectedExecution struct { hook *velerov1api.ExecHook name string error error pod *corev1api.Pod } tests := []struct { name string // Used as argument to HandleHooks and first state added to ListerWatcher initialPod *corev1api.Pod groupResource string byContainer map[string][]PodExecRestoreHook expectedExecutions []expectedExecution expectedErrors []error // changes represents the states of the pod over time. It can be used to test a container // becoming ready at some point after it is first observed by the controller. changes []change sharedHooksContextTimeout time.Duration }{ { name: "should return no error when hook from annotation executes successfully", initialPod: builder.ForPod("default", "my-pod"). ObjectMeta(builder.WithAnnotations( podRestoreHookCommandAnnotationKey, "/usr/bin/foo", podRestoreHookContainerAnnotationKey, "container1", podRestoreHookOnErrorAnnotationKey, string(velerov1api.HookErrorModeContinue), podRestoreHookTimeoutAnnotationKey, "1s", podRestoreHookWaitTimeoutAnnotationKey, "1m", )). Containers(&corev1api.Container{ Name: "container1", }). ContainerStatuses(&corev1api.ContainerStatus{ Name: "container1", State: corev1api.ContainerState{ Running: &corev1api.ContainerStateRunning{}, }, }). Result(), groupResource: "pods", byContainer: map[string][]PodExecRestoreHook{ "container1": { { HookName: "", HookSource: HookSourceAnnotation, Hook: velerov1api.ExecRestoreHook{ Container: "container1", Command: []string{"/usr/bin/foo"}, OnError: velerov1api.HookErrorModeContinue, ExecTimeout: metav1.Duration{Duration: time.Second}, WaitTimeout: metav1.Duration{Duration: time.Minute}, }, }, }, }, expectedExecutions: []expectedExecution{ { name: "", hook: &velerov1api.ExecHook{ Container: "container1", Command: []string{"/usr/bin/foo"}, OnError: velerov1api.HookErrorModeContinue, Timeout: metav1.Duration{Duration: time.Second}, }, error: nil, pod: builder.ForPod("default", "my-pod"). ObjectMeta(builder.WithResourceVersion("1")). ObjectMeta(builder.WithAnnotations( podRestoreHookCommandAnnotationKey, "/usr/bin/foo", podRestoreHookContainerAnnotationKey, "container1", podRestoreHookOnErrorAnnotationKey, string(velerov1api.HookErrorModeContinue), podRestoreHookTimeoutAnnotationKey, "1s", podRestoreHookWaitTimeoutAnnotationKey, "1m", )). Containers(&corev1api.Container{ Name: "container1", }). ContainerStatuses(&corev1api.ContainerStatus{ Name: "container1", State: corev1api.ContainerState{ Running: &corev1api.ContainerStateRunning{}, }, }). Result(), }, }, expectedErrors: nil, }, { name: "should return an error when hook from annotation fails with on error mode fail", initialPod: builder.ForPod("default", "my-pod"). ObjectMeta(builder.WithAnnotations( podRestoreHookCommandAnnotationKey, "/usr/bin/foo", podRestoreHookContainerAnnotationKey, "container1", podRestoreHookOnErrorAnnotationKey, string(velerov1api.HookErrorModeFail), podRestoreHookTimeoutAnnotationKey, "1s", podRestoreHookWaitTimeoutAnnotationKey, "1m", )). Containers(&corev1api.Container{ Name: "container1", }). ContainerStatuses(&corev1api.ContainerStatus{ Name: "container1", State: corev1api.ContainerState{ Running: &corev1api.ContainerStateRunning{}, }, }). Result(), groupResource: "pods", byContainer: map[string][]PodExecRestoreHook{ "container1": { { HookName: "", HookSource: HookSourceAnnotation, Hook: velerov1api.ExecRestoreHook{ Container: "container1", Command: []string{"/usr/bin/foo"}, OnError: velerov1api.HookErrorModeFail, ExecTimeout: metav1.Duration{Duration: time.Second}, WaitTimeout: metav1.Duration{Duration: time.Minute}, }, }, }, }, expectedExecutions: []expectedExecution{ { name: "", hook: &velerov1api.ExecHook{ Container: "container1", Command: []string{"/usr/bin/foo"}, OnError: velerov1api.HookErrorModeFail, Timeout: metav1.Duration{Duration: time.Second}, }, error: errors.New("pod hook error"), pod: builder.ForPod("default", "my-pod"). ObjectMeta(builder.WithResourceVersion("1")). ObjectMeta(builder.WithAnnotations( podRestoreHookCommandAnnotationKey, "/usr/bin/foo", podRestoreHookContainerAnnotationKey, "container1", podRestoreHookOnErrorAnnotationKey, string(velerov1api.HookErrorModeFail), podRestoreHookTimeoutAnnotationKey, "1s", podRestoreHookWaitTimeoutAnnotationKey, "1m", )). Containers(&corev1api.Container{ Name: "container1", }). ContainerStatuses(&corev1api.ContainerStatus{ Name: "container1", State: corev1api.ContainerState{ Running: &corev1api.ContainerStateRunning{}, }, }). Result(), }, }, expectedErrors: []error{errors.New("hook in container container1 failed to execute, err: pod hook error")}, }, { name: "should return error when hook from annotation fails with on error mode continue", initialPod: builder.ForPod("default", "my-pod"). ObjectMeta(builder.WithAnnotations( podRestoreHookCommandAnnotationKey, "/usr/bin/foo", podRestoreHookContainerAnnotationKey, "container1", podRestoreHookOnErrorAnnotationKey, string(velerov1api.HookErrorModeContinue), podRestoreHookTimeoutAnnotationKey, "1s", podRestoreHookWaitTimeoutAnnotationKey, "1m", )). Containers(&corev1api.Container{ Name: "container1", }). ContainerStatuses(&corev1api.ContainerStatus{ Name: "container1", State: corev1api.ContainerState{ Running: &corev1api.ContainerStateRunning{}, }, }). Result(), groupResource: "pods", byContainer: map[string][]PodExecRestoreHook{ "container1": { { HookName: "", HookSource: HookSourceAnnotation, Hook: velerov1api.ExecRestoreHook{ Container: "container1", Command: []string{"/usr/bin/foo"}, OnError: velerov1api.HookErrorModeContinue, ExecTimeout: metav1.Duration{Duration: time.Second}, WaitTimeout: metav1.Duration{Duration: time.Minute}, }, }, }, }, expectedExecutions: []expectedExecution{ { name: "", hook: &velerov1api.ExecHook{ Container: "container1", Command: []string{"/usr/bin/foo"}, OnError: velerov1api.HookErrorModeContinue, Timeout: metav1.Duration{Duration: time.Second}, }, error: errors.New("pod hook error"), pod: builder.ForPod("default", "my-pod"). ObjectMeta(builder.WithResourceVersion("1")). ObjectMeta(builder.WithAnnotations( podRestoreHookCommandAnnotationKey, "/usr/bin/foo", podRestoreHookContainerAnnotationKey, "container1", podRestoreHookOnErrorAnnotationKey, string(velerov1api.HookErrorModeContinue), podRestoreHookTimeoutAnnotationKey, "1s", podRestoreHookWaitTimeoutAnnotationKey, "1m", )). Containers(&corev1api.Container{ Name: "container1", }). ContainerStatuses(&corev1api.ContainerStatus{ Name: "container1", State: corev1api.ContainerState{ Running: &corev1api.ContainerStateRunning{}, }, }). Result(), }, }, expectedErrors: []error{errors.New("hook in container container1 failed to execute, err: pod hook error")}, }, { name: "should return no error when hook from annotation executes after 10ms wait for container to start", initialPod: builder.ForPod("default", "my-pod"). ObjectMeta(builder.WithAnnotations( podRestoreHookCommandAnnotationKey, "/usr/bin/foo", podRestoreHookContainerAnnotationKey, "container1", podRestoreHookOnErrorAnnotationKey, string(velerov1api.HookErrorModeContinue), podRestoreHookTimeoutAnnotationKey, "1s", podRestoreHookWaitTimeoutAnnotationKey, "1m", )). Containers(&corev1api.Container{ Name: "container1", }). ContainerStatuses(&corev1api.ContainerStatus{ Name: "container1", State: corev1api.ContainerState{ Waiting: &corev1api.ContainerStateWaiting{}, }, }). Result(), groupResource: "pods", byContainer: map[string][]PodExecRestoreHook{ "container1": { { HookName: "", HookSource: HookSourceAnnotation, Hook: velerov1api.ExecRestoreHook{ Container: "container1", Command: []string{"/usr/bin/foo"}, OnError: velerov1api.HookErrorModeContinue, ExecTimeout: metav1.Duration{Duration: time.Second}, WaitTimeout: metav1.Duration{Duration: time.Minute}, }, }, }, }, expectedExecutions: []expectedExecution{ { name: "", hook: &velerov1api.ExecHook{ Container: "container1", Command: []string{"/usr/bin/foo"}, OnError: velerov1api.HookErrorModeContinue, Timeout: metav1.Duration{Duration: time.Second}, }, error: nil, pod: builder.ForPod("default", "my-pod"). ObjectMeta(builder.WithResourceVersion("2")). ObjectMeta(builder.WithAnnotations( podRestoreHookCommandAnnotationKey, "/usr/bin/foo", podRestoreHookContainerAnnotationKey, "container1", podRestoreHookOnErrorAnnotationKey, string(velerov1api.HookErrorModeContinue), podRestoreHookTimeoutAnnotationKey, "1s", podRestoreHookWaitTimeoutAnnotationKey, "1m", )). Containers(&corev1api.Container{ Name: "container1", }). ContainerStatuses(&corev1api.ContainerStatus{ Name: "container1", State: corev1api.ContainerState{ Running: &corev1api.ContainerStateRunning{}, }, }). Result(), }, }, expectedErrors: nil, changes: []change{ { wait: 10 * time.Millisecond, updated: builder.ForPod("default", "my-pod"). ObjectMeta(builder.WithAnnotations( podRestoreHookCommandAnnotationKey, "/usr/bin/foo", podRestoreHookContainerAnnotationKey, "container1", podRestoreHookOnErrorAnnotationKey, string(velerov1api.HookErrorModeContinue), podRestoreHookTimeoutAnnotationKey, "1s", podRestoreHookWaitTimeoutAnnotationKey, "1m", )). Containers(&corev1api.Container{ Name: "container1", }). ContainerStatuses(&corev1api.ContainerStatus{ Name: "container1", State: corev1api.ContainerState{ Running: &corev1api.ContainerStateRunning{}, }, }). Result(), }, }, }, { name: "should return no error when hook from spec executes successfully", groupResource: "pods", initialPod: builder.ForPod("default", "my-pod"). Containers(&corev1api.Container{ Name: "container1", }). ContainerStatuses(&corev1api.ContainerStatus{ Name: "container1", State: corev1api.ContainerState{ Running: &corev1api.ContainerStateRunning{}, }, }). Result(), expectedErrors: nil, byContainer: map[string][]PodExecRestoreHook{ "container1": { { HookName: "my-hook-1", HookSource: HookSourceSpec, Hook: velerov1api.ExecRestoreHook{ Container: "container1", Command: []string{"/usr/bin/foo"}, }, }, }, }, expectedExecutions: []expectedExecution{ { name: "my-hook-1", hook: &velerov1api.ExecHook{ Container: "container1", Command: []string{"/usr/bin/foo"}, }, pod: builder.ForPod("default", "my-pod"). ObjectMeta(builder.WithResourceVersion("1")). Containers(&corev1api.Container{ Name: "container1", }). ContainerStatuses(&corev1api.ContainerStatus{ Name: "container1", State: corev1api.ContainerState{ Running: &corev1api.ContainerStateRunning{}, }, }). Result(), }, }, }, { name: "should return error when spec hook with wait timeout expires with OnError mode Continue", groupResource: "pods", initialPod: builder.ForPod("default", "my-pod"). Containers(&corev1api.Container{ Name: "container1", }). ContainerStatuses(&corev1api.ContainerStatus{ Name: "container1", State: corev1api.ContainerState{ Waiting: &corev1api.ContainerStateWaiting{}, }, }). Result(), expectedErrors: []error{errors.New("hook my-hook-1 in container container1 in pod default/my-pod not executed: context deadline exceeded")}, byContainer: map[string][]PodExecRestoreHook{ "container1": { { HookName: "my-hook-1", HookSource: HookSourceSpec, Hook: velerov1api.ExecRestoreHook{ Container: "container1", Command: []string{"/usr/bin/foo"}, OnError: velerov1api.HookErrorModeContinue, WaitTimeout: metav1.Duration{Duration: time.Millisecond}, }, }, }, }, expectedExecutions: []expectedExecution{}, }, { name: "should return an error when spec hook with wait timeout expires with OnError mode Fail", groupResource: "pods", initialPod: builder.ForPod("default", "my-pod"). Containers(&corev1api.Container{ Name: "container1", }). ContainerStatuses(&corev1api.ContainerStatus{ Name: "container1", State: corev1api.ContainerState{ Waiting: &corev1api.ContainerStateWaiting{}, }, }). Result(), expectedErrors: []error{errors.New("hook my-hook-1 in container container1 in pod default/my-pod not executed: context deadline exceeded")}, byContainer: map[string][]PodExecRestoreHook{ "container1": { { HookName: "my-hook-1", HookSource: HookSourceSpec, Hook: velerov1api.ExecRestoreHook{ Container: "container1", Command: []string{"/usr/bin/foo"}, OnError: velerov1api.HookErrorModeFail, WaitTimeout: metav1.Duration{Duration: time.Millisecond}, }, }, }, }, expectedExecutions: []expectedExecution{}, }, { name: "should return an error when shared hooks context is canceled before spec hook with OnError mode Fail executes", groupResource: "pods", initialPod: builder.ForPod("default", "my-pod"). Containers(&corev1api.Container{ Name: "container1", }). ContainerStatuses(&corev1api.ContainerStatus{ Name: "container1", State: corev1api.ContainerState{ Waiting: &corev1api.ContainerStateWaiting{}, }, }). Result(), expectedErrors: []error{errors.New("hook my-hook-1 in container container1 in pod default/my-pod not executed: context deadline exceeded")}, byContainer: map[string][]PodExecRestoreHook{ "container1": { { HookName: "my-hook-1", HookSource: HookSourceSpec, Hook: velerov1api.ExecRestoreHook{ Container: "container1", Command: []string{"/usr/bin/foo"}, OnError: velerov1api.HookErrorModeFail, }, }, }, }, expectedExecutions: []expectedExecution{}, sharedHooksContextTimeout: time.Millisecond, }, { name: "should return error when shared hooks context is canceled before spec hook with OnError mode Continue executes", expectedErrors: []error{errors.New("hook my-hook-1 in container container1 in pod default/my-pod not executed: context deadline exceeded")}, groupResource: "pods", initialPod: builder.ForPod("default", "my-pod"). Containers(&corev1api.Container{ Name: "container1", }). ContainerStatuses(&corev1api.ContainerStatus{ Name: "container1", State: corev1api.ContainerState{ Waiting: &corev1api.ContainerStateWaiting{}, }, }). Result(), byContainer: map[string][]PodExecRestoreHook{ "container1": { { HookName: "my-hook-1", HookSource: HookSourceSpec, Hook: velerov1api.ExecRestoreHook{ Container: "container1", Command: []string{"/usr/bin/foo"}, OnError: velerov1api.HookErrorModeContinue, }, }, }, }, expectedExecutions: []expectedExecution{}, sharedHooksContextTimeout: time.Millisecond, }, { name: "should return no error with 2 spec hooks in 2 different containers, 1st container starts running after 10ms, 2nd container after 20ms, both succeed", groupResource: "pods", initialPod: builder.ForPod("default", "my-pod"). Containers(&corev1api.Container{ Name: "container1", }). Containers(&corev1api.Container{ Name: "container2", }). // initially both are waiting ContainerStatuses(&corev1api.ContainerStatus{ Name: "container1", State: corev1api.ContainerState{ Waiting: &corev1api.ContainerStateWaiting{}, }, }). ContainerStatuses(&corev1api.ContainerStatus{ Name: "container2", State: corev1api.ContainerState{ Waiting: &corev1api.ContainerStateWaiting{}, }, }). Result(), expectedErrors: nil, byContainer: map[string][]PodExecRestoreHook{ "container1": { { HookName: "my-hook-1", HookSource: HookSourceSpec, Hook: velerov1api.ExecRestoreHook{ Container: "container1", Command: []string{"/usr/bin/foo"}, }, }, }, "container2": { { HookName: "my-hook-1", HookSource: HookSourceSpec, Hook: velerov1api.ExecRestoreHook{ Container: "container2", Command: []string{"/usr/bin/bar"}, }, }, }, }, expectedExecutions: []expectedExecution{ { name: "my-hook-1", hook: &velerov1api.ExecHook{ Container: "container1", Command: []string{"/usr/bin/foo"}, }, error: nil, pod: builder.ForPod("default", "my-pod"). ObjectMeta(builder.WithResourceVersion("2")). Containers(&corev1api.Container{ Name: "container1", }). Containers(&corev1api.Container{ Name: "container2", }). ContainerStatuses(&corev1api.ContainerStatus{ Name: "container1", State: corev1api.ContainerState{ Running: &corev1api.ContainerStateRunning{}, }, }). // container 2 is still waiting when the first hook executes in container1 ContainerStatuses(&corev1api.ContainerStatus{ Name: "container2", State: corev1api.ContainerState{ Waiting: &corev1api.ContainerStateWaiting{}, }, }). Result(), }, { name: "my-hook-1", hook: &velerov1api.ExecHook{ Container: "container2", Command: []string{"/usr/bin/bar"}, }, error: nil, pod: builder.ForPod("default", "my-pod"). ObjectMeta(builder.WithResourceVersion("3")). Containers(&corev1api.Container{ Name: "container1", }). Containers(&corev1api.Container{ Name: "container2", }). ContainerStatuses(&corev1api.ContainerStatus{ Name: "container1", State: corev1api.ContainerState{ Running: &corev1api.ContainerStateRunning{}, }, }). ContainerStatuses(&corev1api.ContainerStatus{ Name: "container2", State: corev1api.ContainerState{ Running: &corev1api.ContainerStateRunning{}, }, }). Result(), }, }, changes: []change{ // 1st modification: container1 starts running, resourceVersion 2, container2 still waiting { wait: 10 * time.Millisecond, updated: builder.ForPod("default", "my-pod"). ObjectMeta(builder.WithResourceVersion("2")). Containers(&corev1api.Container{ Name: "container1", }). Containers(&corev1api.Container{ Name: "container2", }). ContainerStatuses(&corev1api.ContainerStatus{ Name: "container1", State: corev1api.ContainerState{ Running: &corev1api.ContainerStateRunning{}, }, }). ContainerStatuses(&corev1api.ContainerStatus{ Name: "container2", State: corev1api.ContainerState{ Waiting: &corev1api.ContainerStateWaiting{}, }, }). Result(), }, // 2nd modification: container2 starts running, resourceVersion 3 { wait: 10 * time.Millisecond, updated: builder.ForPod("default", "my-pod"). ObjectMeta(builder.WithResourceVersion("3")). Containers(&corev1api.Container{ Name: "container1", }). Containers(&corev1api.Container{ Name: "container2", }). ContainerStatuses(&corev1api.ContainerStatus{ Name: "container1", State: corev1api.ContainerState{ Running: &corev1api.ContainerStateRunning{}, }, }). ContainerStatuses(&corev1api.ContainerStatus{ Name: "container2", State: corev1api.ContainerState{ Running: &corev1api.ContainerStateRunning{}, }, }). Result(), }, }, }, { name: "Multiple hooks with non-sequential indices (bug #9359)", initialPod: builder.ForPod("default", "my-pod"). Containers(&corev1api.Container{ Name: "container1", }). ContainerStatuses(&corev1api.ContainerStatus{ Name: "container1", State: corev1api.ContainerState{ Running: &corev1api.ContainerStateRunning{}, }, }). Result(), groupResource: "pods", byContainer: map[string][]PodExecRestoreHook{ "container1": { { HookName: "first-hook", HookSource: HookSourceAnnotation, Hook: velerov1api.ExecRestoreHook{ Container: "container1", Command: []string{"/usr/bin/foo"}, OnError: velerov1api.HookErrorModeContinue, ExecTimeout: metav1.Duration{Duration: time.Second}, WaitTimeout: metav1.Duration{Duration: time.Minute}, }, hookIndex: 0, }, { HookName: "second-hook", HookSource: HookSourceAnnotation, Hook: velerov1api.ExecRestoreHook{ Container: "container1", Command: []string{"/usr/bin/bar"}, OnError: velerov1api.HookErrorModeContinue, ExecTimeout: metav1.Duration{Duration: time.Second}, WaitTimeout: metav1.Duration{Duration: time.Minute}, }, hookIndex: 2, }, { HookName: "third-hook", HookSource: HookSourceAnnotation, Hook: velerov1api.ExecRestoreHook{ Container: "container1", Command: []string{"/usr/bin/third"}, OnError: velerov1api.HookErrorModeContinue, ExecTimeout: metav1.Duration{Duration: time.Second}, WaitTimeout: metav1.Duration{Duration: time.Minute}, }, hookIndex: 4, }, }, }, expectedExecutions: []expectedExecution{ { name: "first-hook", hook: &velerov1api.ExecHook{ Container: "container1", Command: []string{"/usr/bin/foo"}, OnError: velerov1api.HookErrorModeContinue, Timeout: metav1.Duration{Duration: time.Second}, }, error: nil, pod: builder.ForPod("default", "my-pod"). ObjectMeta(builder.WithResourceVersion("1")). Containers(&corev1api.Container{ Name: "container1", }). ContainerStatuses(&corev1api.ContainerStatus{ Name: "container1", State: corev1api.ContainerState{ Running: &corev1api.ContainerStateRunning{}, }, }). Result(), }, { name: "second-hook", hook: &velerov1api.ExecHook{ Container: "container1", Command: []string{"/usr/bin/bar"}, OnError: velerov1api.HookErrorModeContinue, Timeout: metav1.Duration{Duration: time.Second}, }, error: nil, pod: builder.ForPod("default", "my-pod"). ObjectMeta(builder.WithResourceVersion("1")). Containers(&corev1api.Container{ Name: "container1", }). ContainerStatuses(&corev1api.ContainerStatus{ Name: "container1", State: corev1api.ContainerState{ Running: &corev1api.ContainerStateRunning{}, }, }). Result(), }, { name: "third-hook", hook: &velerov1api.ExecHook{ Container: "container1", Command: []string{"/usr/bin/third"}, OnError: velerov1api.HookErrorModeContinue, Timeout: metav1.Duration{Duration: time.Second}, }, error: nil, pod: builder.ForPod("default", "my-pod"). ObjectMeta(builder.WithResourceVersion("1")). Containers(&corev1api.Container{ Name: "container1", }). ContainerStatuses(&corev1api.ContainerStatus{ Name: "container1", State: corev1api.ContainerState{ Running: &corev1api.ContainerStateRunning{}, }, }). Result(), }, }, expectedErrors: nil, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { source := fcache.NewFakeControllerSource() go func() { // This is the state of the pod that will be seen by the AddFunc handler. source.Add(test.initialPod) // Changes holds the versions of the pod over time. Each of these states // will be seen by the UpdateFunc handler. for _, change := range test.changes { time.Sleep(change.wait) source.Modify(change.updated) } }() podCommandExecutor := &velerotest.MockPodCommandExecutor{} defer podCommandExecutor.AssertExpectations(t) h := &DefaultWaitExecHookHandler{ PodCommandExecutor: podCommandExecutor, ListWatchFactory: &fakeListWatchFactory{source}, } for _, e := range test.expectedExecutions { obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(e.pod) require.NoError(t, err) podCommandExecutor.On("ExecutePodCommand", mock.Anything, obj, e.pod.Namespace, e.pod.Name, e.name, e.hook).Return(e.error) } ctx := t.Context() if test.sharedHooksContextTimeout > 0 { var ctxCancel context.CancelFunc ctx, ctxCancel = context.WithTimeout(ctx, test.sharedHooksContextTimeout) defer ctxCancel() } hookTracker := NewMultiHookTracker() errs := h.HandleHooks(ctx, velerotest.NewLogger(), test.initialPod, test.byContainer, hookTracker, "restore1") // for i, ee := range test.expectedErrors { require.Len(t, errs, len(test.expectedErrors)) for i, ee := range test.expectedErrors { assert.EqualError(t, errs[i], ee.Error()) } }) } } func TestPodHasContainer(t *testing.T) { tests := []struct { name string pod *corev1api.Pod container string expect bool }{ { name: "has container", expect: true, container: "container1", pod: builder.ForPod("default", "my-pod"). Containers(&corev1api.Container{ Name: "container1", }). Result(), }, { name: "does not have container", expect: false, container: "container1", pod: builder.ForPod("default", "my-pod"). Containers(&corev1api.Container{ Name: "container2", }). Result(), }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { actual := podHasContainer(test.pod, test.container) assert.Equal(t, actual, test.expect) }) } } func TestIsContainerUp(t *testing.T) { tests := []struct { name string pod *corev1api.Pod container string expect bool hooks []PodExecRestoreHook }{ { name: "should return true when running", container: "container1", expect: true, pod: builder.ForPod("default", "my-pod"). ContainerStatuses(&corev1api.ContainerStatus{ Name: "container1", State: corev1api.ContainerState{ Running: &corev1api.ContainerStateRunning{}, }, }). Result(), hooks: []PodExecRestoreHook{}, }, { name: "should return false when running but not ready", container: "container1", expect: false, pod: builder.ForPod("default", "my-pod"). ContainerStatuses(&corev1api.ContainerStatus{ Name: "container1", State: corev1api.ContainerState{ Running: &corev1api.ContainerStateRunning{}, }, Ready: false, }). Result(), hooks: []PodExecRestoreHook{ { Hook: velerov1api.ExecRestoreHook{ WaitForReady: boolptr.True(), }, }, }, }, { name: "should return true when running and ready", container: "container1", expect: true, pod: builder.ForPod("default", "my-pod"). ContainerStatuses(&corev1api.ContainerStatus{ Name: "container1", State: corev1api.ContainerState{ Running: &corev1api.ContainerStateRunning{}, }, Ready: true, }). Result(), hooks: []PodExecRestoreHook{ { Hook: velerov1api.ExecRestoreHook{ WaitForReady: boolptr.True(), }, }, }, }, { name: "should return false when no state is set", container: "container1", expect: false, pod: builder.ForPod("default", "my-pod"). ContainerStatuses(&corev1api.ContainerStatus{ Name: "container1", State: corev1api.ContainerState{}, }). Result(), hooks: []PodExecRestoreHook{}, }, { name: "should return false when waiting", container: "container1", expect: false, pod: builder.ForPod("default", "my-pod"). ContainerStatuses(&corev1api.ContainerStatus{ Name: "container1", State: corev1api.ContainerState{ Waiting: &corev1api.ContainerStateWaiting{}, }, }). Result(), hooks: []PodExecRestoreHook{}, }, { name: "should return true when running and first container is terminated", container: "container1", expect: true, pod: builder.ForPod("default", "my-pod"). ContainerStatuses(&corev1api.ContainerStatus{ Name: "container0", State: corev1api.ContainerState{ Terminated: &corev1api.ContainerStateTerminated{}, }, }, &corev1api.ContainerStatus{ Name: "container1", State: corev1api.ContainerState{ Running: &corev1api.ContainerStateRunning{}, }, }). Result(), hooks: []PodExecRestoreHook{}, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { actual := isContainerUp(test.pod, test.container, test.hooks) assert.Equal(t, actual, test.expect) }) } } func TestMaxHookWait(t *testing.T) { tests := []struct { name string byContainer map[string][]PodExecRestoreHook expect time.Duration }{ { name: "should return 0 for nil map", byContainer: nil, expect: 0, }, { name: "should return 0 if all hooks are 0 or negative", expect: 0, byContainer: map[string][]PodExecRestoreHook{ "container1": { { Hook: velerov1api.ExecRestoreHook{ ExecTimeout: metav1.Duration{Duration: time.Second}, WaitTimeout: metav1.Duration{Duration: 0}, }, }, { Hook: velerov1api.ExecRestoreHook{ WaitTimeout: metav1.Duration{Duration: -1}, }, }, }, }, }, { name: "should return biggest wait timeout from multiple hooks in multiple containers", expect: time.Hour, byContainer: map[string][]PodExecRestoreHook{ "container1": { { Hook: velerov1api.ExecRestoreHook{ WaitTimeout: metav1.Duration{Duration: time.Second}, }, }, { Hook: velerov1api.ExecRestoreHook{ WaitTimeout: metav1.Duration{Duration: time.Second}, }, }, }, "container2": { { Hook: velerov1api.ExecRestoreHook{ WaitTimeout: metav1.Duration{Duration: time.Hour}, }, }, { Hook: velerov1api.ExecRestoreHook{ WaitTimeout: metav1.Duration{Duration: time.Minute}, }, }, }, }, }, { name: "should return 0 if any hook does not have a wait timeout", expect: 0, byContainer: map[string][]PodExecRestoreHook{ "container1": { { Hook: velerov1api.ExecRestoreHook{ ExecTimeout: metav1.Duration{Duration: time.Second}, WaitTimeout: metav1.Duration{Duration: time.Second}, }, }, { Hook: velerov1api.ExecRestoreHook{ WaitTimeout: metav1.Duration{Duration: 0}, }, }, }, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { actual := maxHookWait(test.byContainer) assert.Equal(t, actual, test.expect) }) } } func TestRestoreHookTrackerUpdate(t *testing.T) { type expectedExecution struct { hook *velerov1api.ExecHook name string error error pod *corev1api.Pod } hookTracker1 := NewMultiHookTracker() hookTracker1.Add("restore1", "default", "my-pod", "container1", HookSourceAnnotation, "", HookPhase(""), 0) hookTracker2 := NewMultiHookTracker() hookTracker2.Add("restore1", "default", "my-pod", "container1", HookSourceSpec, "my-hook-1", HookPhase(""), 0) hookTracker3 := NewMultiHookTracker() hookTracker3.Add("restore1", "default", "my-pod", "container1", HookSourceSpec, "my-hook-1", HookPhase(""), 0) hookTracker3.Add("restore1", "default", "my-pod", "container2", HookSourceSpec, "my-hook-2", HookPhase(""), 0) hookTracker4 := NewMultiHookTracker() hookTracker4.Add("restore1", "default", "my-pod", "container1", HookSourceSpec, "my-hook-1", HookPhase(""), 0) tests1 := []struct { name string initialPod *corev1api.Pod groupResource string byContainer map[string][]PodExecRestoreHook expectedExecutions []expectedExecution hookTracker *MultiHookTracker expectedFailed int }{ { name: "a hook executes successfully", initialPod: builder.ForPod("default", "my-pod"). ObjectMeta(builder.WithAnnotations( podRestoreHookCommandAnnotationKey, "/usr/bin/foo", podRestoreHookContainerAnnotationKey, "container1", podRestoreHookOnErrorAnnotationKey, string(velerov1api.HookErrorModeContinue), podRestoreHookTimeoutAnnotationKey, "1s", podRestoreHookWaitTimeoutAnnotationKey, "1m", )). Containers(&corev1api.Container{ Name: "container1", }). ContainerStatuses(&corev1api.ContainerStatus{ Name: "container1", State: corev1api.ContainerState{ Running: &corev1api.ContainerStateRunning{}, }, }). Result(), groupResource: "pods", byContainer: map[string][]PodExecRestoreHook{ "container1": { { HookName: "", HookSource: HookSourceAnnotation, Hook: velerov1api.ExecRestoreHook{ Container: "container1", Command: []string{"/usr/bin/foo"}, OnError: velerov1api.HookErrorModeContinue, ExecTimeout: metav1.Duration{Duration: time.Second}, WaitTimeout: metav1.Duration{Duration: time.Minute}, }, }, }, }, expectedExecutions: []expectedExecution{ { name: "", hook: &velerov1api.ExecHook{ Container: "container1", Command: []string{"/usr/bin/foo"}, OnError: velerov1api.HookErrorModeContinue, Timeout: metav1.Duration{Duration: time.Second}, }, error: nil, pod: builder.ForPod("default", "my-pod"). ObjectMeta(builder.WithResourceVersion("1")). ObjectMeta(builder.WithAnnotations( podRestoreHookCommandAnnotationKey, "/usr/bin/foo", podRestoreHookContainerAnnotationKey, "container1", podRestoreHookOnErrorAnnotationKey, string(velerov1api.HookErrorModeContinue), podRestoreHookTimeoutAnnotationKey, "1s", podRestoreHookWaitTimeoutAnnotationKey, "1m", )). Containers(&corev1api.Container{ Name: "container1", }). ContainerStatuses(&corev1api.ContainerStatus{ Name: "container1", State: corev1api.ContainerState{ Running: &corev1api.ContainerStateRunning{}, }, }). Result(), }, }, hookTracker: hookTracker1, expectedFailed: 0, }, { name: "a hook with OnError mode Fail failed to execute", groupResource: "pods", initialPod: builder.ForPod("default", "my-pod"). Containers(&corev1api.Container{ Name: "container1", }). ContainerStatuses(&corev1api.ContainerStatus{ Name: "container1", State: corev1api.ContainerState{ Waiting: &corev1api.ContainerStateWaiting{}, }, }). Result(), byContainer: map[string][]PodExecRestoreHook{ "container1": { { HookName: "my-hook-1", HookSource: HookSourceSpec, Hook: velerov1api.ExecRestoreHook{ Container: "container1", Command: []string{"/usr/bin/foo"}, OnError: velerov1api.HookErrorModeFail, WaitTimeout: metav1.Duration{Duration: time.Millisecond}, }, }, }, }, hookTracker: hookTracker2, expectedFailed: 1, }, { name: "a hook with OnError mode Continue failed to execute", groupResource: "pods", initialPod: builder.ForPod("default", "my-pod"). Containers(&corev1api.Container{ Name: "container1", }). ContainerStatuses(&corev1api.ContainerStatus{ Name: "container1", State: corev1api.ContainerState{ Waiting: &corev1api.ContainerStateWaiting{}, }, }). Result(), byContainer: map[string][]PodExecRestoreHook{ "container1": { { HookName: "my-hook-1", HookSource: HookSourceSpec, Hook: velerov1api.ExecRestoreHook{ Container: "container1", Command: []string{"/usr/bin/foo"}, OnError: velerov1api.HookErrorModeContinue, WaitTimeout: metav1.Duration{Duration: time.Millisecond}, }, }, }, }, hookTracker: hookTracker4, expectedFailed: 1, }, { name: "two hooks with OnError mode Continue failed to execute", groupResource: "pods", initialPod: builder.ForPod("default", "my-pod"). Containers(&corev1api.Container{ Name: "container1", }). Containers(&corev1api.Container{ Name: "container2", }). // initially both are waiting ContainerStatuses(&corev1api.ContainerStatus{ Name: "container1", State: corev1api.ContainerState{ Waiting: &corev1api.ContainerStateWaiting{}, }, }). ContainerStatuses(&corev1api.ContainerStatus{ Name: "container2", State: corev1api.ContainerState{ Waiting: &corev1api.ContainerStateWaiting{}, }, }). Result(), byContainer: map[string][]PodExecRestoreHook{ "container1": { { HookName: "my-hook-1", HookSource: HookSourceSpec, Hook: velerov1api.ExecRestoreHook{ Container: "container1", Command: []string{"/usr/bin/foo"}, OnError: velerov1api.HookErrorModeContinue, WaitTimeout: metav1.Duration{Duration: time.Millisecond}, }, }, }, "container2": { { HookName: "my-hook-2", HookSource: HookSourceSpec, Hook: velerov1api.ExecRestoreHook{ Container: "container2", Command: []string{"/usr/bin/bar"}, OnError: velerov1api.HookErrorModeContinue, WaitTimeout: metav1.Duration{Duration: time.Millisecond}, }, }, }, }, hookTracker: hookTracker3, expectedFailed: 2, }, { name: "a hook was recorded before added to tracker", groupResource: "pods", initialPod: builder.ForPod("default", "my-pod"). Containers(&corev1api.Container{ Name: "container1", }). ContainerStatuses(&corev1api.ContainerStatus{ Name: "container1", State: corev1api.ContainerState{ Waiting: &corev1api.ContainerStateWaiting{}, }, }). Result(), byContainer: map[string][]PodExecRestoreHook{ "container1": { { HookName: "my-hook-1", HookSource: HookSourceSpec, Hook: velerov1api.ExecRestoreHook{ Container: "container1", Command: []string{"/usr/bin/foo"}, OnError: velerov1api.HookErrorModeContinue, WaitTimeout: metav1.Duration{Duration: time.Millisecond}, }, }, }, }, hookTracker: NewMultiHookTracker(), expectedFailed: 0, }, } for _, test := range tests1 { t.Run(test.name, func(t *testing.T) { source := fcache.NewFakeControllerSource() go func() { // This is the state of the pod that will be seen by the AddFunc handler. source.Add(test.initialPod) }() podCommandExecutor := &velerotest.MockPodCommandExecutor{} defer podCommandExecutor.AssertExpectations(t) h := &DefaultWaitExecHookHandler{ PodCommandExecutor: podCommandExecutor, ListWatchFactory: &fakeListWatchFactory{source}, } for _, e := range test.expectedExecutions { obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(e.pod) require.NoError(t, err) podCommandExecutor.On("ExecutePodCommand", mock.Anything, obj, e.pod.Namespace, e.pod.Name, e.name, e.hook).Return(e.error) } ctx := t.Context() _ = h.HandleHooks(ctx, velerotest.NewLogger(), test.initialPod, test.byContainer, test.hookTracker, "restore1") _, actualFailed := test.hookTracker.Stat("restore1") assert.Equal(t, test.expectedFailed, actualFailed) }) } } ================================================ FILE: internal/resourcemodifiers/json_merge_patch.go ================================================ package resourcemodifiers import ( "fmt" jsonpatch "github.com/evanphx/json-patch/v5" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "sigs.k8s.io/yaml" ) type JSONMergePatch struct { PatchData string `json:"patchData,omitempty"` } type JSONMergePatcher struct { patches []JSONMergePatch } func (p *JSONMergePatcher) Patch(u *unstructured.Unstructured, _ logrus.FieldLogger) (*unstructured.Unstructured, error) { objBytes, err := u.MarshalJSON() if err != nil { return nil, fmt.Errorf("error in marshaling object %s", err) } for _, patch := range p.patches { patchBytes, err := yaml.YAMLToJSON([]byte(patch.PatchData)) if err != nil { return nil, fmt.Errorf("error in converting YAML to JSON %s", err) } objBytes, err = jsonpatch.MergePatch(objBytes, patchBytes) if err != nil { return nil, fmt.Errorf("error in applying JSON Patch: %s", err.Error()) } } updated := &unstructured.Unstructured{} err = updated.UnmarshalJSON(objBytes) if err != nil { return nil, fmt.Errorf("error in unmarshalling modified object %s", err.Error()) } return updated, nil } ================================================ FILE: internal/resourcemodifiers/json_merge_patch_test.go ================================================ package resourcemodifiers import ( "testing" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ) func TestJsonMergePatchFailure(t *testing.T) { tests := []struct { name string data string }{ { name: "patch with bad yaml", data: "a: b:", }, { name: "patch with bad json", data: `{"a"::1}`, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { scheme := runtime.NewScheme() err := clientgoscheme.AddToScheme(scheme) require.NoError(t, err) pt := &JSONMergePatcher{ patches: []JSONMergePatch{{PatchData: tt.data}}, } u := &unstructured.Unstructured{} _, err = pt.Patch(u, logrus.New()) assert.Error(t, err) }) } } ================================================ FILE: internal/resourcemodifiers/json_patch.go ================================================ package resourcemodifiers import ( "errors" "fmt" "strconv" "strings" jsonpatch "github.com/evanphx/json-patch/v5" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) type JSONPatch struct { Operation string `json:"operation"` From string `json:"from,omitempty"` Path string `json:"path"` Value string `json:"value,omitempty"` } func (p *JSONPatch) ToString() string { if addQuotes(&p.Value) { return fmt.Sprintf(`{"op": "%s", "from": "%s", "path": "%s", "value": "%s"}`, p.Operation, p.From, p.Path, p.Value) } return fmt.Sprintf(`{"op": "%s", "from": "%s", "path": "%s", "value": %s}`, p.Operation, p.From, p.Path, p.Value) } func addQuotes(value *string) bool { if *value == "" { return true } // if value is escaped, remove escape and add quotes // this is useful for scenarios where boolean, null and numbers are required to be set as string. if strings.HasPrefix(*value, "\"") && strings.HasSuffix(*value, "\"") { *value = strings.TrimPrefix(*value, "\"") *value = strings.TrimSuffix(*value, "\"") return true } // if value is null, then don't add quotes if *value == "null" { return false } // if value is a boolean, then don't add quotes if strings.ToLower(*value) == "true" || strings.ToLower(*value) == "false" { return false } // if value is a json object or array, then don't add quotes. if strings.HasPrefix(*value, "{") || strings.HasPrefix(*value, "[") { return false } // if value is a number, then don't add quotes if _, err := strconv.ParseFloat(*value, 64); err == nil { return false } return true } type JSONPatcher struct { patches []JSONPatch `yaml:"patches"` } func (p *JSONPatcher) Patch(u *unstructured.Unstructured, logger logrus.FieldLogger) (*unstructured.Unstructured, error) { modifiedObjBytes, err := p.applyPatch(u) if err != nil { if errors.Is(err, jsonpatch.ErrTestFailed) { logger.Infof("Test operation failed for JSON Patch %s", err.Error()) return u.DeepCopy(), nil } return nil, fmt.Errorf("error in applying JSON Patch %s", err.Error()) } updated := &unstructured.Unstructured{} err = updated.UnmarshalJSON(modifiedObjBytes) if err != nil { return nil, fmt.Errorf("error in unmarshalling modified object %s", err.Error()) } return updated, nil } func (p *JSONPatcher) applyPatch(u *unstructured.Unstructured) ([]byte, error) { patchBytes := p.patchArrayToByteArray() jsonPatch, err := jsonpatch.DecodePatch(patchBytes) if err != nil { return nil, fmt.Errorf("error in decoding json patch %s", err.Error()) } objBytes, err := u.MarshalJSON() if err != nil { return nil, fmt.Errorf("error in marshaling object %s", err.Error()) } return jsonPatch.Apply(objBytes) } func (p *JSONPatcher) patchArrayToByteArray() []byte { var patches []string for _, patch := range p.patches { patches = append(patches, patch.ToString()) } patchesStr := strings.Join(patches, ",\n\t") return []byte(fmt.Sprintf(`[%s]`, patchesStr)) } ================================================ FILE: internal/resourcemodifiers/resource_modifiers.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package resourcemodifiers import ( "fmt" "regexp" jsonpatch "github.com/evanphx/json-patch/v5" "github.com/gobwas/glob" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/yaml" "github.com/vmware-tanzu/velero/pkg/util/collections" ) const ( ConfigmapRefType = "configmap" ResourceModifierSupportedVersionV1 = "v1" ) type MatchRule struct { Path string `json:"path,omitempty"` Value string `json:"value,omitempty"` } type Conditions struct { Namespaces []string `json:"namespaces,omitempty"` GroupResource string `json:"groupResource"` ResourceNameRegex string `json:"resourceNameRegex,omitempty"` LabelSelector *metav1.LabelSelector `json:"labelSelector,omitempty"` Matches []MatchRule `json:"matches,omitempty"` } type ResourceModifierRule struct { Conditions Conditions `json:"conditions"` Patches []JSONPatch `json:"patches,omitempty"` MergePatches []JSONMergePatch `json:"mergePatches,omitempty"` StrategicPatches []StrategicMergePatch `json:"strategicPatches,omitempty"` } type ResourceModifiers struct { Version string `json:"version"` ResourceModifierRules []ResourceModifierRule `json:"resourceModifierRules"` } func GetResourceModifiersFromConfig(cm *corev1api.ConfigMap) (*ResourceModifiers, error) { if cm == nil { return nil, fmt.Errorf("could not parse config from nil configmap") } if len(cm.Data) != 1 { return nil, fmt.Errorf("illegal resource modifiers %s/%s configmap", cm.Namespace, cm.Name) } var yamlData string for _, v := range cm.Data { yamlData = v } resModifiers, err := unmarshalResourceModifiers([]byte(yamlData)) if err != nil { return nil, errors.WithStack(err) } return resModifiers, nil } func (p *ResourceModifiers) ApplyResourceModifierRules(obj *unstructured.Unstructured, groupResource string, scheme *runtime.Scheme, log logrus.FieldLogger) []error { var errs []error origin := obj // If there are more than one rules, we need to keep the original object for condition matching if len(p.ResourceModifierRules) > 1 { origin = obj.DeepCopy() } for _, rule := range p.ResourceModifierRules { matched, err := rule.match(origin, groupResource, log) if err != nil { errs = append(errs, err) continue } else if !matched { continue } log.Infof("Applying resource modifier patch on %s/%s", origin.GetNamespace(), origin.GetName()) err = rule.applyPatch(obj, scheme, log) if err != nil { errs = append(errs, err) } } return errs } func (r *ResourceModifierRule) match(obj *unstructured.Unstructured, groupResource string, log logrus.FieldLogger) (bool, error) { ns := obj.GetNamespace() if ns != "" { namespaceInclusion := collections.NewIncludesExcludes().Includes(r.Conditions.Namespaces...) if !namespaceInclusion.ShouldInclude(ns) { return false, nil } } g, err := glob.Compile(r.Conditions.GroupResource, '.') if err != nil { log.Errorf("Bad glob pattern of groupResource in condition, groupResource: %s, err: %s", r.Conditions.GroupResource, err) return false, err } if !g.Match(groupResource) { return false, nil } if r.Conditions.ResourceNameRegex != "" { match, err := regexp.MatchString(r.Conditions.ResourceNameRegex, obj.GetName()) if err != nil { return false, errors.Errorf("error in matching regex %s", err.Error()) } if !match { return false, nil } } if r.Conditions.LabelSelector != nil { selector, err := metav1.LabelSelectorAsSelector(r.Conditions.LabelSelector) if err != nil { return false, errors.Errorf("error in creating label selector %s", err.Error()) } if !selector.Matches(labels.Set(obj.GetLabels())) { return false, nil } } match, err := matchConditions(obj, r.Conditions.Matches, log) if err != nil { return false, err } else if !match { log.Info("Conditions do not match, skip it") return false, nil } return true, nil } func matchConditions(u *unstructured.Unstructured, rules []MatchRule, _ logrus.FieldLogger) (bool, error) { if len(rules) == 0 { return true, nil } var fixed []JSONPatch for _, rule := range rules { if rule.Path == "" { return false, fmt.Errorf("path is required for match rule") } fixed = append(fixed, JSONPatch{ Operation: "test", Path: rule.Path, Value: rule.Value, }) } p := &JSONPatcher{patches: fixed} _, err := p.applyPatch(u) if err != nil { if errors.Is(err, jsonpatch.ErrTestFailed) || errors.Is(err, jsonpatch.ErrMissing) { return false, nil } return false, err } return true, nil } func unmarshalResourceModifiers(yamlData []byte) (*ResourceModifiers, error) { resModifiers := &ResourceModifiers{} err := yaml.UnmarshalStrict(yamlData, resModifiers) if err != nil { return nil, fmt.Errorf("failed to decode yaml data into resource modifiers, err: %s", err) } return resModifiers, nil } type patcher interface { Patch(u *unstructured.Unstructured, logger logrus.FieldLogger) (*unstructured.Unstructured, error) } func (r *ResourceModifierRule) applyPatch(u *unstructured.Unstructured, scheme *runtime.Scheme, logger logrus.FieldLogger) error { var p patcher if len(r.Patches) > 0 { p = &JSONPatcher{patches: r.Patches} } else if len(r.MergePatches) > 0 { p = &JSONMergePatcher{patches: r.MergePatches} } else if len(r.StrategicPatches) > 0 { p = &StrategicMergePatcher{patches: r.StrategicPatches, scheme: scheme} } else { return fmt.Errorf("no patch data found") } updated, err := p.Patch(u, logger) if err != nil { return fmt.Errorf("error in applying patch %s", err) } u.SetUnstructuredContent(updated.Object) return nil } ================================================ FILE: internal/resourcemodifiers/resource_modifiers_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package resourcemodifiers import ( "reflect" "testing" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer/yaml" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ) func TestGetResourceModifiersFromConfig(t *testing.T) { cm1 := &corev1api.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "test-configmap", Namespace: "test-namespace", }, Data: map[string]string{ "sub.yml": "version: v1\nresourceModifierRules:\n- conditions:\n groupResource: persistentvolumeclaims\n resourceNameRegex: \".*\"\n namespaces:\n - bar\n - foo\n patches:\n - operation: replace\n path: \"/spec/storageClassName\"\n value: \"premium\"\n - operation: remove\n path: \"/metadata/labels/test\"\n\n\n", }, } rules1 := &ResourceModifiers{ Version: "v1", ResourceModifierRules: []ResourceModifierRule{ { Conditions: Conditions{ GroupResource: "persistentvolumeclaims", ResourceNameRegex: ".*", Namespaces: []string{"bar", "foo"}, }, Patches: []JSONPatch{ { Operation: "replace", Path: "/spec/storageClassName", Value: "premium", }, { Operation: "remove", Path: "/metadata/labels/test", }, }, }, }, } cm2 := &corev1api.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "test-configmap", Namespace: "test-namespace", }, Data: map[string]string{ "sub.yml": "version: v1\nresourceModifierRules:\n- conditions:\n groupResource: deployments.apps\n resourceNameRegex: \"^test-.*$\"\n namespaces:\n - bar\n - foo\n patches:\n - operation: add\n path: \"/spec/template/spec/containers/0\"\n value: \"{\\\"name\\\": \\\"nginx\\\", \\\"image\\\": \\\"nginx:1.14.2\\\", \\\"ports\\\": [{\\\"containerPort\\\": 80}]}\"\n - operation: copy\n from: \"/spec/template/spec/containers/0\"\n path: \"/spec/template/spec/containers/1\"\n\n\n", }, } rules2 := &ResourceModifiers{ Version: "v1", ResourceModifierRules: []ResourceModifierRule{ { Conditions: Conditions{ GroupResource: "deployments.apps", ResourceNameRegex: "^test-.*$", Namespaces: []string{"bar", "foo"}, }, Patches: []JSONPatch{ { Operation: "add", Path: "/spec/template/spec/containers/0", Value: `{"name": "nginx", "image": "nginx:1.14.2", "ports": [{"containerPort": 80}]}`, }, { Operation: "copy", From: "/spec/template/spec/containers/0", Path: "/spec/template/spec/containers/1", }, }, }, }, } cm3 := &corev1api.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "test-configmap", Namespace: "test-namespace", }, Data: map[string]string{ "sub.yml": "version1: v1\nresourceModifierRules:\n- conditions:\n groupResource: deployments.apps\n resourceNameRegex: \"^test-.*$\"\n namespaces:\n - bar\n - foo\n patches:\n - operation: add\n path: \"/spec/template/spec/containers/0\"\n value: \"{\\\"name\\\": \\\"nginx\\\", \\\"image\\\": \\\"nginx:1.14.2\\\", \\\"ports\\\": [{\\\"containerPort\\\": 80}]}\"\n - operation: copy\n from: \"/spec/template/spec/containers/0\"\n path: \"/spec/template/spec/containers/1\"\n\n\n", }, } cm4 := &corev1api.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "test-configmap", Namespace: "test-namespace", }, Data: map[string]string{ "sub.yml": "version: v1\nresourceModifierRules:\n- conditions:\n groupResource: deployments.apps\n labelSelector:\n matchLabels:\n a: b\n", }, } rules4 := &ResourceModifiers{ Version: "v1", ResourceModifierRules: []ResourceModifierRule{ { Conditions: Conditions{ GroupResource: "deployments.apps", LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "a": "b", }, }, }, }, }, } cm5 := &corev1api.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "test-configmap", Namespace: "test-namespace", }, Data: map[string]string{ "sub.yml": "version: v1\nresourceModifierRules:\n- conditions:\n groupResource: pods\n namespaces:\n - ns1\n matches:\n - path: /metadata/annotations/foo\n value: bar\n mergePatches:\n - patchData: |\n metadata:\n annotations:\n foo: null", }, } rules5 := &ResourceModifiers{ Version: "v1", ResourceModifierRules: []ResourceModifierRule{ { Conditions: Conditions{ GroupResource: "pods", Namespaces: []string{ "ns1", }, Matches: []MatchRule{ { Path: "/metadata/annotations/foo", Value: "bar", }, }, }, MergePatches: []JSONMergePatch{ { PatchData: "metadata:\n annotations:\n foo: null", }, }, }, }, } cm6 := &corev1api.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "test-configmap", Namespace: "test-namespace", }, Data: map[string]string{ "sub.yml": "version: v1\nresourceModifierRules:\n- conditions:\n groupResource: pods\n namespaces:\n - ns1\n strategicPatches:\n - patchData: |\n spec:\n containers:\n - name: nginx\n image: repo2/nginx", }, } rules6 := &ResourceModifiers{ Version: "v1", ResourceModifierRules: []ResourceModifierRule{ { Conditions: Conditions{ GroupResource: "pods", Namespaces: []string{ "ns1", }, }, StrategicPatches: []StrategicMergePatch{ { PatchData: "spec:\n containers:\n - name: nginx\n image: repo2/nginx", }, }, }, }, } cm7 := &corev1api.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "test-configmap", Namespace: "test-namespace", }, Data: map[string]string{ "sub.yml": "version: v1\nresourceModifierRules:\n- conditions:\n groupResource: pods\n namespaces:\n - ns1\n mergePatches:\n - patchData: |\n {\"metadata\":{\"annotations\":{\"foo\":null}}}", }, } rules7 := &ResourceModifiers{ Version: "v1", ResourceModifierRules: []ResourceModifierRule{ { Conditions: Conditions{ GroupResource: "pods", Namespaces: []string{ "ns1", }, }, MergePatches: []JSONMergePatch{ { PatchData: `{"metadata":{"annotations":{"foo":null}}}`, }, }, }, }, } cm8 := &corev1api.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "test-configmap", Namespace: "test-namespace", }, Data: map[string]string{ "sub.yml": "version: v1\nresourceModifierRules:\n- conditions:\n groupResource: pods\n namespaces:\n - ns1\n strategicPatches:\n - patchData: |\n {\"spec\":{\"containers\":[{\"name\": \"nginx\",\"image\": \"repo2/nginx\"}]}}", }, } rules8 := &ResourceModifiers{ Version: "v1", ResourceModifierRules: []ResourceModifierRule{ { Conditions: Conditions{ GroupResource: "pods", Namespaces: []string{ "ns1", }, }, StrategicPatches: []StrategicMergePatch{ { PatchData: `{"spec":{"containers":[{"name": "nginx","image": "repo2/nginx"}]}}`, }, }, }, }, } cm9 := &corev1api.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "test-configmap", Namespace: "test-namespace", }, Data: map[string]string{ "sub.yml": "version: v1\nresourceModifierRules:\n- conditions:\n groupResource: deployments.apps\n resourceNameRegex: \"^test-.*$\"\n namespaces:\n - bar\n - foo\n patches:\n - operation: replace\n path: \"/value/bool\"\n value: \"\\\"true\\\"\"\n\n\n", }, } rules9 := &ResourceModifiers{ Version: "v1", ResourceModifierRules: []ResourceModifierRule{ { Conditions: Conditions{ GroupResource: "deployments.apps", ResourceNameRegex: "^test-.*$", Namespaces: []string{"bar", "foo"}, }, Patches: []JSONPatch{ { Operation: "replace", Path: "/value/bool", Value: `"true"`, }, }, }, }, } cm10 := &corev1api.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "test-configmap", Namespace: "test-namespace", }, Data: map[string]string{ "sub.yml": "version: v1\nresourceModifierRules:\n- conditions:\n groupResource: deployments.apps\n resourceNameRegex: \"^test-.*$\"\n namespaces:\n - bar\n - foo\n patches:\n - operation: replace\n path: \"/value/bool\"\n value: \"true\"\n\n\n", }, } rules10 := &ResourceModifiers{ Version: "v1", ResourceModifierRules: []ResourceModifierRule{ { Conditions: Conditions{ GroupResource: "deployments.apps", ResourceNameRegex: "^test-.*$", Namespaces: []string{"bar", "foo"}, }, Patches: []JSONPatch{ { Operation: "replace", Path: "/value/bool", Value: "true", }, }, }, }, } type args struct { cm *corev1api.ConfigMap } tests := []struct { name string args args want *ResourceModifiers wantErr bool }{ { name: "test 1", args: args{ cm: cm1, }, want: rules1, wantErr: false, }, { name: "complex payload in add and copy operator", args: args{ cm: cm2, }, want: rules2, wantErr: false, }, { name: "invalid payload version1", args: args{ cm: cm3, }, want: nil, wantErr: true, }, { name: "match labels", args: args{ cm: cm4, }, want: rules4, wantErr: false, }, { name: "nil configmap", args: args{ cm: nil, }, want: nil, wantErr: true, }, { name: "complex yaml data with json merge patch", args: args{ cm: cm5, }, want: rules5, wantErr: false, }, { name: "complex yaml data with strategic merge patch", args: args{ cm: cm6, }, want: rules6, wantErr: false, }, { name: "complex json data with json merge patch", args: args{ cm: cm7, }, want: rules7, wantErr: false, }, { name: "complex json data with strategic merge patch", args: args{ cm: cm8, }, want: rules8, wantErr: false, }, { name: "bool value as string", args: args{ cm: cm9, }, want: rules9, wantErr: false, }, { name: "bool value as bool", args: args{ cm: cm10, }, want: rules10, wantErr: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got, err := GetResourceModifiersFromConfig(tt.args.cm) if (err != nil) != tt.wantErr { t.Errorf("GetResourceModifiersFromConfig() error = %v, wantErr %v", err, tt.wantErr) return } if !reflect.DeepEqual(got, tt.want) { t.Errorf("GetResourceModifiersFromConfig() = %v, want %v", got, tt.want) } }) } } func TestResourceModifiers_ApplyResourceModifierRules(t *testing.T) { pvcStandardSc := &unstructured.Unstructured{ Object: map[string]any{ "apiVersion": "v1", "kind": "PersistentVolumeClaim", "metadata": map[string]any{ "name": "test-pvc", "namespace": "foo", }, "spec": map[string]any{ "storageClassName": "standard", }, }, } pvcPremiumSc := &unstructured.Unstructured{ Object: map[string]any{ "apiVersion": "v1", "kind": "PersistentVolumeClaim", "metadata": map[string]any{ "name": "test-pvc", "namespace": "foo", }, "spec": map[string]any{ "storageClassName": "premium", }, }, } pvcGoldSc := &unstructured.Unstructured{ Object: map[string]any{ "apiVersion": "v1", "kind": "PersistentVolumeClaim", "metadata": map[string]any{ "name": "test-pvc", "namespace": "foo", }, "spec": map[string]any{ "storageClassName": "gold", }, }, } deployNginxOneReplica := &unstructured.Unstructured{ Object: map[string]any{ "apiVersion": "apps/v1", "kind": "Deployment", "metadata": map[string]any{ "name": "test-deployment", "namespace": "foo", "labels": map[string]any{ "app": "nginx", }, }, "spec": map[string]any{ "replicas": int64(1), "template": map[string]any{ "metadata": map[string]any{ "labels": map[string]any{ "app": "nginx", }, }, "spec": map[string]any{ "containers": []any{ map[string]any{ "name": "nginx", "image": "nginx:latest", }, }, }, }, }, }, } deployNginxTwoReplica := &unstructured.Unstructured{ Object: map[string]any{ "apiVersion": "apps/v1", "kind": "Deployment", "metadata": map[string]any{ "name": "test-deployment", "namespace": "foo", "labels": map[string]any{ "app": "nginx", }, }, "spec": map[string]any{ "replicas": int64(2), "template": map[string]any{ "metadata": map[string]any{ "labels": map[string]any{ "app": "nginx", }, }, "spec": map[string]any{ "containers": []any{ map[string]any{ "name": "nginx", "image": "nginx:latest", }, }, }, }, }, }, } deployNginxMysql := &unstructured.Unstructured{ Object: map[string]any{ "apiVersion": "apps/v1", "kind": "Deployment", "metadata": map[string]any{ "name": "test-deployment", "namespace": "foo", "labels": map[string]any{ "app": "nginx", }, }, "spec": map[string]any{ "replicas": int64(1), "template": map[string]any{ "metadata": map[string]any{ "labels": map[string]any{ "app": "nginx", }, }, "spec": map[string]any{ "containers": []any{ map[string]any{ "name": "nginx", "image": "nginx:latest", }, map[string]any{ "name": "mysql", "image": "mysql:latest", }, }, }, }, }, }, } cmTrue := &unstructured.Unstructured{ Object: map[string]any{ "apiVersion": "v1", "kind": "ConfigMap", "data": map[string]any{ "test": "true", }, }, } cmFalse := &unstructured.Unstructured{ Object: map[string]any{ "apiVersion": "v1", "kind": "ConfigMap", "data": map[string]any{ "test": "false", }, }, } type fields struct { Version string ResourceModifierRules []ResourceModifierRule } type args struct { obj *unstructured.Unstructured groupResource string } tests := []struct { name string fields fields args args wantErr bool wantObj *unstructured.Unstructured }{ { name: "configmap true false string", fields: fields{ Version: "v1", ResourceModifierRules: []ResourceModifierRule{ { Conditions: Conditions{ GroupResource: "configmaps", ResourceNameRegex: ".*", }, Patches: []JSONPatch{ { Operation: "replace", Path: "/data/test", Value: `"false"`, }, }, }, }, }, args: args{ obj: cmTrue.DeepCopy(), groupResource: "configmaps", }, wantErr: false, wantObj: cmFalse.DeepCopy(), }, { name: "Invalid Regex throws error", fields: fields{ Version: "v1", ResourceModifierRules: []ResourceModifierRule{ { Conditions: Conditions{ GroupResource: "persistentvolumeclaims", ResourceNameRegex: "[a-z", Namespaces: []string{"foo"}, }, Patches: []JSONPatch{ { Operation: "test", Path: "/spec/storageClassName", Value: "standard", }, { Operation: "replace", Path: "/spec/storageClassName", Value: "premium", }, }, }, }, }, args: args{ obj: pvcStandardSc.DeepCopy(), groupResource: "persistentvolumeclaims", }, wantErr: true, wantObj: pvcStandardSc.DeepCopy(), }, { name: "pvc with standard storage class should be patched to premium", fields: fields{ Version: "v1", ResourceModifierRules: []ResourceModifierRule{ { Conditions: Conditions{ GroupResource: "persistentvolumeclaims", ResourceNameRegex: ".*", Namespaces: []string{"foo"}, }, Patches: []JSONPatch{ { Operation: "test", Path: "/spec/storageClassName", Value: "standard", }, { Operation: "replace", Path: "/spec/storageClassName", Value: "premium", }, }, }, }, }, args: args{ obj: pvcStandardSc.DeepCopy(), groupResource: "persistentvolumeclaims", }, wantErr: false, wantObj: pvcPremiumSc.DeepCopy(), }, { name: "pvc with standard storage class should be patched to premium, even when rules are [standard => premium, premium => gold]", fields: fields{ Version: "v1", ResourceModifierRules: []ResourceModifierRule{ { Conditions: Conditions{ GroupResource: "persistentvolumeclaims", ResourceNameRegex: ".*", Matches: []MatchRule{ { Path: "/spec/storageClassName", Value: "standard", }, }, }, Patches: []JSONPatch{ { Operation: "replace", Path: "/spec/storageClassName", Value: "premium", }, }, }, { Conditions: Conditions{ GroupResource: "persistentvolumeclaims", ResourceNameRegex: ".*", Matches: []MatchRule{ { Path: "/spec/storageClassName", Value: "premium", }, }, }, Patches: []JSONPatch{ { Operation: "replace", Path: "/spec/storageClassName", Value: "gold", }, }, }, }, }, args: args{ obj: pvcStandardSc.DeepCopy(), groupResource: "persistentvolumeclaims", }, wantErr: false, wantObj: pvcPremiumSc.DeepCopy(), }, { name: "pvc with standard storage class should be patched to gold, even when rules are [standard => premium, standard => gold]", fields: fields{ Version: "v1", ResourceModifierRules: []ResourceModifierRule{ { Conditions: Conditions{ GroupResource: "persistentvolumeclaims", ResourceNameRegex: ".*", Matches: []MatchRule{ { Path: "/spec/storageClassName", Value: "standard", }, }, }, Patches: []JSONPatch{ { Operation: "replace", Path: "/spec/storageClassName", Value: "premium", }, }, }, { Conditions: Conditions{ GroupResource: "persistentvolumeclaims", ResourceNameRegex: ".*", Matches: []MatchRule{ { Path: "/spec/storageClassName", Value: "standard", }, }, }, Patches: []JSONPatch{ { Operation: "replace", Path: "/spec/storageClassName", Value: "gold", }, }, }, }, }, args: args{ obj: pvcStandardSc.DeepCopy(), groupResource: "persistentvolumeclaims", }, wantErr: false, wantObj: pvcGoldSc.DeepCopy(), }, { name: "nginx deployment: 1 -> 2 replicas", fields: fields{ Version: "v1", ResourceModifierRules: []ResourceModifierRule{ { Conditions: Conditions{ GroupResource: "deployments.apps", ResourceNameRegex: "^test-.*$", Namespaces: []string{"foo"}, }, Patches: []JSONPatch{ { Operation: "test", Path: "/spec/replicas", Value: "1", }, { Operation: "replace", Path: "/spec/replicas", Value: "2", }, }, }, }, }, args: args{ obj: deployNginxOneReplica.DeepCopy(), groupResource: "deployments.apps", }, wantErr: false, wantObj: deployNginxTwoReplica.DeepCopy(), }, { name: "nginx deployment: test operator fails, skips substitution, no error", fields: fields{ Version: "v1", ResourceModifierRules: []ResourceModifierRule{ { Conditions: Conditions{ GroupResource: "deployments.apps", ResourceNameRegex: "^test-.*$", Namespaces: []string{"foo"}, }, Patches: []JSONPatch{ { Operation: "test", Path: "/spec/replicas", Value: "5", }, { Operation: "replace", Path: "/spec/replicas", Value: "2", }, }, }, }, }, args: args{ obj: deployNginxOneReplica.DeepCopy(), groupResource: "deployments.apps", }, wantErr: false, wantObj: deployNginxOneReplica.DeepCopy(), }, { name: "nginx deployment: Empty Resource Regex", fields: fields{ Version: "v1", ResourceModifierRules: []ResourceModifierRule{ { Conditions: Conditions{ GroupResource: "deployments.apps", Namespaces: []string{"foo"}, }, Patches: []JSONPatch{ { Operation: "test", Path: "/spec/replicas", Value: "1", }, { Operation: "replace", Path: "/spec/replicas", Value: "2", }, }, }, }, }, args: args{ obj: deployNginxOneReplica.DeepCopy(), groupResource: "deployments.apps", }, wantErr: false, wantObj: deployNginxTwoReplica.DeepCopy(), }, { name: "nginx deployment: Empty Resource Regex", fields: fields{ Version: "v1", ResourceModifierRules: []ResourceModifierRule{ { Conditions: Conditions{ GroupResource: "deployments.apps", Namespaces: []string{"foo"}, }, Patches: []JSONPatch{ { Operation: "test", Path: "/spec/replicas", Value: "1", }, { Operation: "replace", Path: "/spec/replicas", Value: "2", }, }, }, }, }, args: args{ obj: deployNginxOneReplica.DeepCopy(), groupResource: "deployments.apps", }, wantErr: false, wantObj: deployNginxTwoReplica.DeepCopy(), }, { name: "nginx deployment: Empty Resource Regex and namespaces list", fields: fields{ Version: "v1", ResourceModifierRules: []ResourceModifierRule{ { Conditions: Conditions{ GroupResource: "deployments.apps", }, Patches: []JSONPatch{ { Operation: "test", Path: "/spec/replicas", Value: "1", }, { Operation: "replace", Path: "/spec/replicas", Value: "2", }, }, }, }, }, args: args{ obj: deployNginxOneReplica.DeepCopy(), groupResource: "deployments.apps", }, wantErr: false, wantObj: deployNginxTwoReplica.DeepCopy(), }, { name: "nginx deployment: namespace doesn't match", fields: fields{ Version: "v1", ResourceModifierRules: []ResourceModifierRule{ { Conditions: Conditions{ GroupResource: "deployments.apps", ResourceNameRegex: ".*", Namespaces: []string{"bar"}, }, Patches: []JSONPatch{ { Operation: "test", Path: "/spec/replicas", Value: "1", }, { Operation: "replace", Path: "/spec/replicas", Value: "2", }, }, }, }, }, args: args{ obj: deployNginxOneReplica.DeepCopy(), groupResource: "deployments.apps", }, wantErr: false, wantObj: deployNginxOneReplica.DeepCopy(), }, { name: "add container mysql to deployment", fields: fields{ Version: "v1", ResourceModifierRules: []ResourceModifierRule{ { Conditions: Conditions{ GroupResource: "deployments.apps", ResourceNameRegex: "^test-.*$", Namespaces: []string{"foo"}, }, Patches: []JSONPatch{ { Operation: "add", Path: "/spec/template/spec/containers/1", Value: `{"name": "mysql", "image": "mysql:latest"}`, }, }, }, }, }, args: args{ obj: deployNginxOneReplica, groupResource: "deployments.apps", }, wantErr: false, wantObj: deployNginxMysql, }, { name: "Copy container 0 to container 1 and then modify container 1", fields: fields{ Version: "v1", ResourceModifierRules: []ResourceModifierRule{ { Conditions: Conditions{ GroupResource: "deployments.apps", ResourceNameRegex: "^test-.*$", Namespaces: []string{"foo"}, }, Patches: []JSONPatch{ { Operation: "copy", From: "/spec/template/spec/containers/0", Path: "/spec/template/spec/containers/1", }, { Operation: "test", Path: "/spec/template/spec/containers/1/image", Value: "nginx:latest", }, { Operation: "replace", Path: "/spec/template/spec/containers/1/name", Value: "mysql", }, { Operation: "replace", Path: "/spec/template/spec/containers/1/image", Value: "mysql:latest", }, }, }, }, }, args: args{ obj: deployNginxOneReplica.DeepCopy(), groupResource: "deployments.apps", }, wantErr: false, wantObj: deployNginxMysql.DeepCopy(), }, { name: "nginx deployment: match label selector", fields: fields{ Version: "v1", ResourceModifierRules: []ResourceModifierRule{ { Conditions: Conditions{ GroupResource: "deployments.apps", Namespaces: []string{"foo"}, LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "app": "nginx", }, }, }, Patches: []JSONPatch{ { Operation: "test", Path: "/spec/replicas", Value: "1", }, { Operation: "replace", Path: "/spec/replicas", Value: "2", }, }, }, }, }, args: args{ obj: deployNginxOneReplica.DeepCopy(), groupResource: "deployments.apps", }, wantErr: false, wantObj: deployNginxTwoReplica.DeepCopy(), }, { name: "nginx deployment: mismatch label selector", fields: fields{ Version: "v1", ResourceModifierRules: []ResourceModifierRule{ { Conditions: Conditions{ GroupResource: "deployments.apps", Namespaces: []string{"foo"}, LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "app": "nginx-mismatch", }, }, }, Patches: []JSONPatch{ { Operation: "test", Path: "/spec/replicas", Value: "1", }, { Operation: "replace", Path: "/spec/replicas", Value: "2", }, }, }, }, }, args: args{ obj: deployNginxOneReplica.DeepCopy(), groupResource: "deployments.apps", }, wantErr: false, wantObj: deployNginxOneReplica.DeepCopy(), }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { p := &ResourceModifiers{ Version: tt.fields.Version, ResourceModifierRules: tt.fields.ResourceModifierRules, } got := p.ApplyResourceModifierRules(tt.args.obj, tt.args.groupResource, nil, logrus.New()) assert.Equal(t, tt.wantErr, len(got) > 0) assert.Equal(t, *tt.wantObj, *tt.args.obj) }) } } var podYAMLWithNginxImage = ` apiVersion: v1 kind: Pod metadata: name: pod1 namespace: fake spec: containers: - image: nginx name: nginx ` var podYAMLWithNginx1Image = ` apiVersion: v1 kind: Pod metadata: name: pod1 namespace: fake spec: containers: - image: nginx1 name: nginx ` var podYAMLWithNFSVolume = ` apiVersion: v1 kind: Pod metadata: name: pod1 namespace: fake spec: containers: - image: fake name: fake volumeMounts: - mountPath: /fake1 name: vol1 - mountPath: /fake2 name: vol2 volumes: - name: vol1 nfs: path: /fake2 - name: vol2 emptyDir: {} ` var podYAMLWithPVCVolume = ` apiVersion: v1 kind: Pod metadata: name: pod1 namespace: fake spec: containers: - image: fake name: fake volumeMounts: - mountPath: /fake1 name: vol1 - mountPath: /fake2 name: vol2 volumes: - name: vol1 persistentVolumeClaim: claimName: pvc1 - name: vol2 emptyDir: {} ` var svcYAMLWithPort8000 = ` apiVersion: v1 kind: Service metadata: name: svc1 namespace: fake spec: ports: - name: fake1 port: 8001 protocol: TCP targetPort: 8001 - name: fake port: 8000 protocol: TCP targetPort: 8000 - name: fake2 port: 8002 protocol: TCP targetPort: 8002 ` var svcYAMLWithPort9000 = ` apiVersion: v1 kind: Service metadata: name: svc1 namespace: fake spec: ports: - name: fake1 port: 8001 protocol: TCP targetPort: 8001 - name: fake port: 9000 protocol: TCP targetPort: 9000 - name: fake2 port: 8002 protocol: TCP targetPort: 8002 ` var cmYAMLWithLabelAToB = ` apiVersion: v1 kind: ConfigMap metadata: name: cm1 namespace: fake labels: a: b c: d ` var cmYAMLWithLabelAToC = ` apiVersion: v1 kind: ConfigMap metadata: name: cm1 namespace: fake labels: a: c c: d ` var cmYAMLWithoutLabelA = ` apiVersion: v1 kind: ConfigMap metadata: name: cm1 namespace: fake labels: c: d ` func TestResourceModifiers_ApplyResourceModifierRules_StrategicMergePatch(t *testing.T) { scheme := runtime.NewScheme() utilruntime.Must(clientgoscheme.AddToScheme(scheme)) unstructuredSerializer := yaml.NewDecodingSerializer(unstructured.UnstructuredJSONScheme) o1, _, err := unstructuredSerializer.Decode([]byte(podYAMLWithNFSVolume), nil, nil) require.NoError(t, err) podWithNFSVolume := o1.(*unstructured.Unstructured) o2, _, err := unstructuredSerializer.Decode([]byte(podYAMLWithPVCVolume), nil, nil) require.NoError(t, err) podWithPVCVolume := o2.(*unstructured.Unstructured) o3, _, err := unstructuredSerializer.Decode([]byte(svcYAMLWithPort8000), nil, nil) require.NoError(t, err) svcWithPort8000 := o3.(*unstructured.Unstructured) o4, _, err := unstructuredSerializer.Decode([]byte(svcYAMLWithPort9000), nil, nil) require.NoError(t, err) svcWithPort9000 := o4.(*unstructured.Unstructured) o5, _, err := unstructuredSerializer.Decode([]byte(podYAMLWithNginxImage), nil, nil) require.NoError(t, err) podWithNginxImage := o5.(*unstructured.Unstructured) o6, _, err := unstructuredSerializer.Decode([]byte(podYAMLWithNginx1Image), nil, nil) require.NoError(t, err) podWithNginx1Image := o6.(*unstructured.Unstructured) tests := []struct { name string rm *ResourceModifiers obj *unstructured.Unstructured groupResource string wantErr bool wantObj *unstructured.Unstructured }{ { name: "update image", rm: &ResourceModifiers{ Version: "v1", ResourceModifierRules: []ResourceModifierRule{ { Conditions: Conditions{ GroupResource: "pods", Namespaces: []string{"fake"}, }, StrategicPatches: []StrategicMergePatch{ { PatchData: `{"spec":{"containers":[{"name":"nginx","image":"nginx1"}]}}`, }, }, }, }, }, obj: podWithNginxImage.DeepCopy(), groupResource: "pods", wantErr: false, wantObj: podWithNginx1Image.DeepCopy(), }, { name: "update image with yaml format", rm: &ResourceModifiers{ Version: "v1", ResourceModifierRules: []ResourceModifierRule{ { Conditions: Conditions{ GroupResource: "pods", Namespaces: []string{"fake"}, }, StrategicPatches: []StrategicMergePatch{ { PatchData: `spec: containers: - name: nginx image: nginx1`, }, }, }, }, }, obj: podWithNginxImage.DeepCopy(), groupResource: "pods", wantErr: false, wantObj: podWithNginx1Image.DeepCopy(), }, { name: "replace nfs with pvc in volume", rm: &ResourceModifiers{ Version: "v1", ResourceModifierRules: []ResourceModifierRule{ { Conditions: Conditions{ GroupResource: "pods", Namespaces: []string{"fake"}, }, StrategicPatches: []StrategicMergePatch{ { PatchData: `{"spec":{"volumes":[{"nfs":null,"name":"vol1","persistentVolumeClaim":{"claimName":"pvc1"}}]}}`, }, }, }, }, }, obj: podWithNFSVolume.DeepCopy(), groupResource: "pods", wantErr: false, wantObj: podWithPVCVolume.DeepCopy(), }, { name: "replace any other volume source with pvc in volume", rm: &ResourceModifiers{ Version: "v1", ResourceModifierRules: []ResourceModifierRule{ { Conditions: Conditions{ GroupResource: "pods", Namespaces: []string{"fake"}, }, StrategicPatches: []StrategicMergePatch{ { PatchData: `{"spec":{"volumes":[{"$retainKeys":["name","persistentVolumeClaim"],"name":"vol1","persistentVolumeClaim":{"claimName":"pvc1"}}]}}`, }, }, }, }, }, obj: podWithNFSVolume.DeepCopy(), groupResource: "pods", wantErr: false, wantObj: podWithPVCVolume.DeepCopy(), }, { name: "update a service port", rm: &ResourceModifiers{ Version: "v1", ResourceModifierRules: []ResourceModifierRule{ { Conditions: Conditions{ GroupResource: "services", Namespaces: []string{"fake"}, }, StrategicPatches: []StrategicMergePatch{ { PatchData: `{"spec":{"$setElementOrder/ports":[{"port":8001},{"port":9000},{"port":8002}],"ports":[{"name":"fake","port":9000,"protocol":"TCP","targetPort":9000},{"$patch":"delete","port":8000}]}}`, }, }, }, }, }, obj: svcWithPort8000.DeepCopy(), groupResource: "services", wantErr: false, wantObj: svcWithPort9000.DeepCopy(), }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got := tt.rm.ApplyResourceModifierRules(tt.obj, tt.groupResource, scheme, logrus.New()) assert.Equal(t, tt.wantErr, len(got) > 0) assert.Equal(t, *tt.wantObj, *tt.obj) }) } } func TestResourceModifiers_ApplyResourceModifierRules_JSONMergePatch(t *testing.T) { unstructuredSerializer := yaml.NewDecodingSerializer(unstructured.UnstructuredJSONScheme) o1, _, err := unstructuredSerializer.Decode([]byte(cmYAMLWithLabelAToB), nil, nil) require.NoError(t, err) cmWithLabelAToB := o1.(*unstructured.Unstructured) o2, _, err := unstructuredSerializer.Decode([]byte(cmYAMLWithLabelAToC), nil, nil) require.NoError(t, err) cmWithLabelAToC := o2.(*unstructured.Unstructured) o3, _, err := unstructuredSerializer.Decode([]byte(cmYAMLWithoutLabelA), nil, nil) require.NoError(t, err) cmWithoutLabelA := o3.(*unstructured.Unstructured) tests := []struct { name string rm *ResourceModifiers obj *unstructured.Unstructured groupResource string wantErr bool wantObj *unstructured.Unstructured }{ { name: "update labels", rm: &ResourceModifiers{ Version: "v1", ResourceModifierRules: []ResourceModifierRule{ { Conditions: Conditions{ GroupResource: "configmaps", Namespaces: []string{"fake"}, }, MergePatches: []JSONMergePatch{ { PatchData: `{"metadata":{"labels":{"a":"c"}}}`, }, }, }, }, }, obj: cmWithLabelAToB.DeepCopy(), groupResource: "configmaps", wantErr: false, wantObj: cmWithLabelAToC.DeepCopy(), }, { name: "update labels in yaml format", rm: &ResourceModifiers{ Version: "v1", ResourceModifierRules: []ResourceModifierRule{ { Conditions: Conditions{ GroupResource: "configmaps", Namespaces: []string{"fake"}, }, MergePatches: []JSONMergePatch{ { PatchData: `metadata: labels: a: c`, }, }, }, }, }, obj: cmWithLabelAToB.DeepCopy(), groupResource: "configmaps", wantErr: false, wantObj: cmWithLabelAToC.DeepCopy(), }, { name: "delete labels", rm: &ResourceModifiers{ Version: "v1", ResourceModifierRules: []ResourceModifierRule{ { Conditions: Conditions{ GroupResource: "configmaps", Namespaces: []string{"fake"}, }, MergePatches: []JSONMergePatch{ { PatchData: `{"metadata":{"labels":{"a":null}}}`, }, }, }, }, }, obj: cmWithLabelAToB.DeepCopy(), groupResource: "configmaps", wantErr: false, wantObj: cmWithoutLabelA.DeepCopy(), }, { name: "add labels", rm: &ResourceModifiers{ Version: "v1", ResourceModifierRules: []ResourceModifierRule{ { Conditions: Conditions{ GroupResource: "configmaps", Namespaces: []string{"fake"}, }, MergePatches: []JSONMergePatch{ { PatchData: `{"metadata":{"labels":{"a":"b"}}}`, }, }, }, }, }, obj: cmWithoutLabelA.DeepCopy(), groupResource: "configmaps", wantErr: false, wantObj: cmWithLabelAToB.DeepCopy(), }, { name: "delete non-existing labels", rm: &ResourceModifiers{ Version: "v1", ResourceModifierRules: []ResourceModifierRule{ { Conditions: Conditions{ GroupResource: "configmaps", Namespaces: []string{"fake"}, }, MergePatches: []JSONMergePatch{ { PatchData: `{"metadata":{"labels":{"a":null}}}`, }, }, }, }, }, obj: cmWithoutLabelA.DeepCopy(), groupResource: "configmaps", wantErr: false, wantObj: cmWithoutLabelA.DeepCopy(), }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got := tt.rm.ApplyResourceModifierRules(tt.obj, tt.groupResource, nil, logrus.New()) assert.Equal(t, tt.wantErr, len(got) > 0) assert.Equal(t, *tt.wantObj, *tt.obj) }) } } func TestResourceModifiers_wildcard_in_GroupResource(t *testing.T) { unstructuredSerializer := yaml.NewDecodingSerializer(unstructured.UnstructuredJSONScheme) o1, _, err := unstructuredSerializer.Decode([]byte(cmYAMLWithLabelAToB), nil, nil) require.NoError(t, err) cmWithLabelAToB := o1.(*unstructured.Unstructured) o2, _, err := unstructuredSerializer.Decode([]byte(cmYAMLWithLabelAToC), nil, nil) require.NoError(t, err) cmWithLabelAToC := o2.(*unstructured.Unstructured) tests := []struct { name string rm *ResourceModifiers obj *unstructured.Unstructured groupResource string wantErr bool wantObj *unstructured.Unstructured }{ { name: "match all groups and resources", rm: &ResourceModifiers{ Version: "v1", ResourceModifierRules: []ResourceModifierRule{ { Conditions: Conditions{ GroupResource: "*", Namespaces: []string{"fake"}, }, MergePatches: []JSONMergePatch{ { PatchData: `{"metadata":{"labels":{"a":"c"}}}`, }, }, }, }, }, obj: cmWithLabelAToB.DeepCopy(), groupResource: "configmaps", wantErr: false, wantObj: cmWithLabelAToC.DeepCopy(), }, { name: "match all resources in group apps", rm: &ResourceModifiers{ Version: "v1", ResourceModifierRules: []ResourceModifierRule{ { Conditions: Conditions{ GroupResource: "*.apps", Namespaces: []string{"fake"}, }, MergePatches: []JSONMergePatch{ { PatchData: `{"metadata":{"labels":{"a":"c"}}}`, }, }, }, }, }, obj: cmWithLabelAToB.DeepCopy(), groupResource: "fake.apps", wantErr: false, wantObj: cmWithLabelAToC.DeepCopy(), }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got := tt.rm.ApplyResourceModifierRules(tt.obj, tt.groupResource, nil, logrus.New()) assert.Equal(t, tt.wantErr, len(got) > 0) assert.Equal(t, *tt.wantObj, *tt.obj) }) } } func TestResourceModifiers_conditional_patches(t *testing.T) { unstructuredSerializer := yaml.NewDecodingSerializer(unstructured.UnstructuredJSONScheme) o1, _, err := unstructuredSerializer.Decode([]byte(cmYAMLWithLabelAToB), nil, nil) require.NoError(t, err) cmWithLabelAToB := o1.(*unstructured.Unstructured) o2, _, err := unstructuredSerializer.Decode([]byte(cmYAMLWithLabelAToC), nil, nil) require.NoError(t, err) cmWithLabelAToC := o2.(*unstructured.Unstructured) tests := []struct { name string rm *ResourceModifiers obj *unstructured.Unstructured groupResource string wantErr bool wantObj *unstructured.Unstructured }{ { name: "match conditions and apply patches", rm: &ResourceModifiers{ Version: "v1", ResourceModifierRules: []ResourceModifierRule{ { Conditions: Conditions{ GroupResource: "*", Namespaces: []string{"fake"}, Matches: []MatchRule{ { Path: "/metadata/labels/a", Value: "b", }, }, }, MergePatches: []JSONMergePatch{ { PatchData: `{"metadata":{"labels":{"a":"c"}}}`, }, }, }, }, }, obj: cmWithLabelAToB.DeepCopy(), groupResource: "configmaps", wantErr: false, wantObj: cmWithLabelAToC.DeepCopy(), }, { name: "mismatch conditions and skip patches", rm: &ResourceModifiers{ Version: "v1", ResourceModifierRules: []ResourceModifierRule{ { Conditions: Conditions{ GroupResource: "*", Namespaces: []string{"fake"}, Matches: []MatchRule{ { Path: "/metadata/labels/a", Value: "c", }, }, }, MergePatches: []JSONMergePatch{ { PatchData: `{"metadata":{"labels":{"a":"c"}}}`, }, }, }, }, }, obj: cmWithLabelAToB.DeepCopy(), groupResource: "configmaps", wantErr: false, wantObj: cmWithLabelAToB.DeepCopy(), }, { name: "missing condition path and skip patches", rm: &ResourceModifiers{ Version: "v1", ResourceModifierRules: []ResourceModifierRule{ { Conditions: Conditions{ GroupResource: "*", Namespaces: []string{"fake"}, Matches: []MatchRule{ { Path: "/metadata/labels/a/b", Value: "c", }, }, }, MergePatches: []JSONMergePatch{ { PatchData: `{"metadata":{"labels":{"a":"c"}}}`, }, }, }, }, }, obj: cmWithLabelAToB.DeepCopy(), groupResource: "configmaps", wantErr: false, wantObj: cmWithLabelAToB.DeepCopy(), }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got := tt.rm.ApplyResourceModifierRules(tt.obj, tt.groupResource, nil, logrus.New()) assert.Equal(t, tt.wantErr, len(got) > 0) assert.Equal(t, *tt.wantObj, *tt.obj) }) } } func TestJSONPatch_ToString(t *testing.T) { type fields struct { Operation string From string Path string Value string } tests := []struct { name string fields fields want string }{ { name: "test", fields: fields{ Operation: "test", Path: "/spec/replicas", Value: "1", }, want: `{"op": "test", "from": "", "path": "/spec/replicas", "value": 1}`, }, { name: "replace integer", fields: fields{ Operation: "replace", Path: "/spec/replicas", Value: "2", }, want: `{"op": "replace", "from": "", "path": "/spec/replicas", "value": 2}`, }, { name: "replace array", fields: fields{ Operation: "replace", Path: "/spec/template/spec/containers/0/ports", Value: `[{"containerPort": 80}]`, }, want: `{"op": "replace", "from": "", "path": "/spec/template/spec/containers/0/ports", "value": [{"containerPort": 80}]}`, }, { name: "replace with null", fields: fields{ Operation: "replace", Path: "/spec/template/spec/containers/0/ports", Value: `null`, }, want: `{"op": "replace", "from": "", "path": "/spec/template/spec/containers/0/ports", "value": null}`, }, { name: "add json object", fields: fields{ Operation: "add", Path: "/spec/template/spec/containers/0", Value: `{"name": "nginx", "image": "nginx:1.14.2", "ports": [{"containerPort": 80}]}`, }, want: `{"op": "add", "from": "", "path": "/spec/template/spec/containers/0", "value": {"name": "nginx", "image": "nginx:1.14.2", "ports": [{"containerPort": 80}]}}`, }, { name: "remove", fields: fields{ Operation: "remove", Path: "/spec/template/spec/containers/0", }, want: `{"op": "remove", "from": "", "path": "/spec/template/spec/containers/0", "value": ""}`, }, { name: "move", fields: fields{ Operation: "move", From: "/spec/template/spec/containers/0", Path: "/spec/template/spec/containers/1", }, want: `{"op": "move", "from": "/spec/template/spec/containers/0", "path": "/spec/template/spec/containers/1", "value": ""}`, }, { name: "copy", fields: fields{ Operation: "copy", From: "/spec/template/spec/containers/0", Path: "/spec/template/spec/containers/1", }, want: `{"op": "copy", "from": "/spec/template/spec/containers/0", "path": "/spec/template/spec/containers/1", "value": ""}`, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { p := &JSONPatch{ Operation: tt.fields.Operation, From: tt.fields.From, Path: tt.fields.Path, Value: tt.fields.Value, } if got := p.ToString(); got != tt.want { t.Errorf("JSONPatch.ToString() = %v, want %v", got, tt.want) } }) } } ================================================ FILE: internal/resourcemodifiers/resource_modifiers_validator.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package resourcemodifiers import ( "fmt" "strings" ) func (r *ResourceModifierRule) Validate() error { if err := r.Conditions.Validate(); err != nil { return err } count := 0 for _, size := range []int{ len(r.Patches), len(r.MergePatches), len(r.StrategicPatches), } { if size != 0 { count++ } if count >= 2 { return fmt.Errorf("only one of patches, mergePatches, strategicPatches can be specified") } } for _, patch := range r.Patches { if err := patch.Validate(); err != nil { return err } } return nil } func (p *ResourceModifiers) Validate() error { if !strings.EqualFold(p.Version, ResourceModifierSupportedVersionV1) { return fmt.Errorf("unsupported resource modifier version %s", p.Version) } if len(p.ResourceModifierRules) == 0 { return fmt.Errorf("resource modifier rules cannot be empty") } for _, rule := range p.ResourceModifierRules { if err := rule.Validate(); err != nil { return err } } return nil } func (p *JSONPatch) Validate() error { if p.Operation == "" { return fmt.Errorf("operation cannot be empty") } if operation := strings.ToLower(p.Operation); operation != "add" && operation != "remove" && operation != "replace" && operation != "test" && operation != "move" && operation != "copy" { return fmt.Errorf("unsupported operation %s", p.Operation) } if p.Path == "" { return fmt.Errorf("path cannot be empty") } return nil } func (c *Conditions) Validate() error { if c.GroupResource == "" { return fmt.Errorf("groupkResource cannot be empty") } return nil } ================================================ FILE: internal/resourcemodifiers/resource_modifiers_validator_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package resourcemodifiers import ( "testing" ) func TestResourceModifiers_Validate(t *testing.T) { type fields struct { Version string ResourceModifierRules []ResourceModifierRule } tests := []struct { name string fields fields wantErr bool }{ { name: "correct version, non 0 length ResourceModifierRules", fields: fields{ Version: "v1", ResourceModifierRules: []ResourceModifierRule{ { Conditions: Conditions{ GroupResource: "persistentvolumeclaims", ResourceNameRegex: ".*", Namespaces: []string{"bar", "foo"}, }, Patches: []JSONPatch{ { Operation: "replace", Path: "/spec/storageClassName", Value: "premium", }, }, }, }, }, wantErr: false, }, { name: "incorrect version, non 0 length ResourceModifierRules", fields: fields{ Version: "v2", ResourceModifierRules: []ResourceModifierRule{ { Conditions: Conditions{ GroupResource: "persistentvolumeclaims", ResourceNameRegex: ".*", Namespaces: []string{"bar", "foo"}, }, Patches: []JSONPatch{ { Operation: "replace", Path: "/spec/storageClassName", Value: "premium", }, }, }, }, }, wantErr: true, }, { name: "correct version, 0 length ResourceModifierRules", fields: fields{ Version: "v1", ResourceModifierRules: []ResourceModifierRule{}, }, wantErr: true, }, { name: "patch has invalid operation", fields: fields{ Version: "v1", ResourceModifierRules: []ResourceModifierRule{ { Conditions: Conditions{ GroupResource: "persistentvolumeclaims", ResourceNameRegex: ".*", Namespaces: []string{"bar", "foo"}, }, Patches: []JSONPatch{ { Operation: "invalid", Path: "/spec/storageClassName", Value: "premium", }, }, }, }, }, wantErr: true, }, { name: "Condition has empty GroupResource", fields: fields{ Version: "v1", ResourceModifierRules: []ResourceModifierRule{ { Conditions: Conditions{ GroupResource: "", ResourceNameRegex: ".*", Namespaces: []string{"bar", "foo"}, }, Patches: []JSONPatch{ { Operation: "invalid", Path: "/spec/storageClassName", Value: "premium", }, }, }, }, }, wantErr: true, }, { name: "More than one patch type in a rule", fields: fields{ Version: "v1", ResourceModifierRules: []ResourceModifierRule{ { Conditions: Conditions{ GroupResource: "*", }, Patches: []JSONPatch{ { Operation: "test", Path: "/spec/storageClassName", Value: "premium", }, }, MergePatches: []JSONMergePatch{ { PatchData: `{"metadata":{"labels":{"a":null}}}`, }, }, }, }, }, wantErr: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { p := &ResourceModifiers{ Version: tt.fields.Version, ResourceModifierRules: tt.fields.ResourceModifierRules, } if err := p.Validate(); (err != nil) != tt.wantErr { t.Errorf("ResourceModifiers.Validate() error = %v, wantErr %v", err, tt.wantErr) } }) } } func TestJsonPatch_Validate(t *testing.T) { type fields struct { Operation string Path string Value string } tests := []struct { name string fields fields wantErr bool }{ { name: "not empty operation, path, and new value, valid scenario", fields: fields{ Operation: "replace", Path: "/spec/storageClassName", Value: "premium", }, wantErr: false, }, { name: "empty operation throws error", fields: fields{ Operation: "", Path: "/spec/storageClassName", Value: "premium", }, wantErr: true, }, { name: "empty path throws error", fields: fields{ Operation: "replace", Path: "", Value: "premium", }, wantErr: true, }, { name: "invalid operation throws error", fields: fields{ Operation: "invalid", Path: "/spec/storageClassName", Value: "premium", }, wantErr: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { p := &JSONPatch{ Operation: tt.fields.Operation, Path: tt.fields.Path, Value: tt.fields.Value, } if err := p.Validate(); (err != nil) != tt.wantErr { t.Errorf("JsonPatch.Validate() error = %v, wantErr %v", err, tt.wantErr) } }) } } ================================================ FILE: internal/resourcemodifiers/strategic_merge_patch.go ================================================ package resourcemodifiers import ( "fmt" "net/http" "github.com/sirupsen/logrus" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/mergepatch" "k8s.io/apimachinery/pkg/util/strategicpatch" "k8s.io/apimachinery/pkg/util/validation/field" kubejson "sigs.k8s.io/json" "sigs.k8s.io/yaml" ) type StrategicMergePatch struct { PatchData string `json:"patchData,omitempty"` } type StrategicMergePatcher struct { patches []StrategicMergePatch scheme *runtime.Scheme } func (p *StrategicMergePatcher) Patch(u *unstructured.Unstructured, _ logrus.FieldLogger) (*unstructured.Unstructured, error) { gvk := u.GetObjectKind().GroupVersionKind() schemaReferenceObj, err := p.scheme.New(gvk) if err != nil { return nil, err } origin := u.DeepCopy() updated := u.DeepCopy() for _, patch := range p.patches { patchBytes, err := yaml.YAMLToJSON([]byte(patch.PatchData)) if err != nil { return nil, fmt.Errorf("error in converting YAML to JSON %s", err) } err = strategicPatchObject(origin, patchBytes, updated, schemaReferenceObj) if err != nil { return nil, fmt.Errorf("error in applying Strategic Patch %s", err.Error()) } origin = updated.DeepCopy() } return updated, nil } // strategicPatchObject applies a strategic merge patch of `patchBytes` to // `originalObject` and stores the result in `objToUpdate`. // It additionally returns the map[string]any representation of the // `originalObject` and `patchBytes`. // NOTE: Both `originalObject` and `objToUpdate` are supposed to be versioned. func strategicPatchObject( originalObject runtime.Object, patchBytes []byte, objToUpdate runtime.Object, schemaReferenceObj runtime.Object, ) error { originalObjMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(originalObject) if err != nil { return err } patchMap := make(map[string]any) var strictErrs []error strictErrs, err = kubejson.UnmarshalStrict(patchBytes, &patchMap) if err != nil { return apierrors.NewBadRequest(err.Error()) } if err := applyPatchToObject(originalObjMap, patchMap, objToUpdate, schemaReferenceObj, strictErrs); err != nil { return err } return nil } // applyPatchToObject applies a strategic merge patch of to // and stores the result in . // NOTE: must be a versioned object. func applyPatchToObject( originalMap map[string]any, patchMap map[string]any, objToUpdate runtime.Object, schemaReferenceObj runtime.Object, strictErrs []error, ) error { patchedObjMap, err := strategicpatch.StrategicMergeMapPatch(originalMap, patchMap, schemaReferenceObj) if err != nil { return interpretStrategicMergePatchError(err) } // Rather than serialize the patched map to JSON, then decode it to an object, we go directly from a map to an object converter := runtime.DefaultUnstructuredConverter if err := converter.FromUnstructuredWithValidation(patchedObjMap, objToUpdate, true); err != nil { strictError, isStrictError := runtime.AsStrictDecodingError(err) switch { case !isStrictError: // disregard any sttrictErrs, because it's an incomplete // list of strict errors given that we don't know what fields were // unknown because StrategicMergeMapPatch failed. // Non-strict errors trump in this case. return apierrors.NewInvalid(schema.GroupKind{}, "", field.ErrorList{ field.Invalid(field.NewPath("patch"), fmt.Sprintf("%+v", patchMap), err.Error()), }) //case validationDirective == metav1.FieldValidationWarn: // addStrictDecodingWarnings(requestContext, append(strictErrs, strictError.Errors()...)) default: strictDecodingError := runtime.NewStrictDecodingError(append(strictErrs, strictError.Errors()...)) return apierrors.NewInvalid(schema.GroupKind{}, "", field.ErrorList{ field.Invalid(field.NewPath("patch"), fmt.Sprintf("%+v", patchMap), strictDecodingError.Error()), }) } } else if len(strictErrs) > 0 { return apierrors.NewInvalid(schema.GroupKind{}, "", field.ErrorList{ field.Invalid(field.NewPath("patch"), fmt.Sprintf("%+v", patchMap), runtime.NewStrictDecodingError(strictErrs).Error()), }) } return nil } // interpretStrategicMergePatchError interprets the error type and returns an error with appropriate HTTP code. func interpretStrategicMergePatchError(err error) error { switch err { case mergepatch.ErrBadJSONDoc, mergepatch.ErrBadPatchFormatForPrimitiveList, mergepatch.ErrBadPatchFormatForRetainKeys, mergepatch.ErrBadPatchFormatForSetElementOrderList, mergepatch.ErrUnsupportedStrategicMergePatchFormat: return apierrors.NewBadRequest(err.Error()) case mergepatch.ErrNoListOfLists, mergepatch.ErrPatchContentNotMatchRetainKeys: return apierrors.NewGenericServerResponse(http.StatusUnprocessableEntity, "", schema.GroupResource{}, "", err.Error(), 0, false) default: return err } } ================================================ FILE: internal/resourcemodifiers/strategic_merge_patch_test.go ================================================ package resourcemodifiers import ( "testing" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ) func TestStrategicMergePatchFailure(t *testing.T) { tests := []struct { name string data string kind string }{ { name: "patch with unknown kind", data: "{}", kind: "BadKind", }, { name: "patch with bad yaml", data: "a: b:", kind: "Pod", }, { name: "patch with bad json", data: `{"a"::1}`, kind: "Pod", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { scheme := runtime.NewScheme() err := clientgoscheme.AddToScheme(scheme) require.NoError(t, err) pt := &StrategicMergePatcher{ patches: []StrategicMergePatch{{PatchData: tt.data}}, scheme: scheme, } u := &unstructured.Unstructured{} u.SetGroupVersionKind(schema.GroupVersionKind{Version: "v1", Kind: tt.kind}) _, err = pt.Patch(u, logrus.New()) assert.Error(t, err) }) } } ================================================ FILE: internal/resourcepolicies/resource_policies.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package resourcepolicies import ( "context" "fmt" "strings" "k8s.io/apimachinery/pkg/util/sets" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" crclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) type VolumeActionType string const ( // currently only support configmap type of resource config ConfigmapRefType string = "configmap" // skip action implies the volume would be skipped from the backup operation Skip VolumeActionType = "skip" // fs-backup action implies that the volume would be backed up via file system copy method using the uploader(kopia/restic) configured by the user FSBackup VolumeActionType = "fs-backup" // snapshot action can have 3 different meaning based on velero configuration and backup spec - cloud provider based snapshots, local csi snapshots and datamover snapshots Snapshot VolumeActionType = "snapshot" ) // Action defined as one action for a specific way of backup type Action struct { // Type defined specific type of action, currently only support 'skip' Type VolumeActionType `yaml:"type"` // Parameters defined map of parameters when executing a specific action Parameters map[string]any `yaml:"parameters,omitempty"` } // IncludeExcludePolicy defined policy to include or exclude resources based on the names type IncludeExcludePolicy struct { // The following fields have the same semantics as those from the spec of backup. // Refer to the comment in the velerov1api.BackupSpec for more details. IncludedClusterScopedResources []string `yaml:"includedClusterScopedResources"` ExcludedClusterScopedResources []string `yaml:"excludedClusterScopedResources"` IncludedNamespaceScopedResources []string `yaml:"includedNamespaceScopedResources"` ExcludedNamespaceScopedResources []string `yaml:"excludedNamespaceScopedResources"` } func (p *IncludeExcludePolicy) Validate() error { if err := p.validateIncludeExclude(p.IncludedClusterScopedResources, p.ExcludedClusterScopedResources); err != nil { return err } return p.validateIncludeExclude(p.IncludedNamespaceScopedResources, p.ExcludedNamespaceScopedResources) } func (p *IncludeExcludePolicy) validateIncludeExclude(includesList, excludesList []string) error { includes := sets.NewString(includesList...) excludes := sets.NewString(excludesList...) if includes.Has("*") || excludes.Has("*") { return fmt.Errorf("cannot use '*' in includes or excludes filters in the policy") } for _, itm := range excludes.List() { if includes.Has(itm) { return fmt.Errorf("excludes list cannot contain an item in the includes list: %s", itm) } } return nil } // VolumePolicy defined policy to conditions to match Volumes and related action to handle matched Volumes type VolumePolicy struct { // Conditions defined list of conditions to match Volumes Conditions map[string]any `yaml:"conditions"` Action Action `yaml:"action"` } // ResourcePolicies currently defined slice of volume policies to handle backup type ResourcePolicies struct { Version string `yaml:"version"` VolumePolicies []VolumePolicy `yaml:"volumePolicies"` IncludeExcludePolicy *IncludeExcludePolicy `yaml:"includeExcludePolicy"` // we may support other resource policies in the future, and they could be added separately // OtherResourcePolicies []OtherResourcePolicy } type Policies struct { version string volumePolicies []volPolicy includeExcludePolicy *IncludeExcludePolicy // OtherPolicies } func unmarshalResourcePolicies(yamlData *string) (*ResourcePolicies, error) { resPolicies := &ResourcePolicies{} err := decodeStruct(strings.NewReader(*yamlData), resPolicies) if err != nil { return nil, fmt.Errorf("failed to decode yaml data into resource policies %v", err) } for _, vp := range resPolicies.VolumePolicies { if raw, ok := vp.Conditions["pvcLabels"]; ok { switch raw.(type) { case map[string]any, map[string]string: default: return nil, fmt.Errorf("pvcLabels must be a map of string to string, got %T", raw) } } } return resPolicies, nil } func (p *Policies) BuildPolicy(resPolicies *ResourcePolicies) error { for _, vp := range resPolicies.VolumePolicies { con, err := unmarshalVolConditions(vp.Conditions) if err != nil { return errors.WithStack(err) } volCap, err := parseCapacity(con.Capacity) if err != nil { return errors.WithStack(err) } var volP volPolicy volP.action = vp.Action volP.conditions = append(volP.conditions, &capacityCondition{capacity: *volCap}) volP.conditions = append(volP.conditions, &storageClassCondition{storageClass: con.StorageClass}) volP.conditions = append(volP.conditions, &nfsCondition{nfs: con.NFS}) volP.conditions = append(volP.conditions, &csiCondition{csi: con.CSI}) volP.conditions = append(volP.conditions, &volumeTypeCondition{volumeTypes: con.VolumeTypes}) if len(con.PVCLabels) > 0 { volP.conditions = append(volP.conditions, &pvcLabelsCondition{labels: con.PVCLabels}) } if len(con.PVCPhase) > 0 { volP.conditions = append(volP.conditions, &pvcPhaseCondition{phases: con.PVCPhase}) } p.volumePolicies = append(p.volumePolicies, volP) } // Other resource policies p.version = resPolicies.Version p.includeExcludePolicy = resPolicies.IncludeExcludePolicy return nil } func (p *Policies) match(res *structuredVolume) *Action { for _, policy := range p.volumePolicies { isAllMatch := false for _, con := range policy.conditions { if !con.match(res) { isAllMatch = false break } isAllMatch = true } if isAllMatch { return &policy.action } } return nil } func (p *Policies) GetMatchAction(res any) (*Action, error) { data, ok := res.(VolumeFilterData) if !ok { return nil, errors.New("failed to convert input to VolumeFilterData") } volume := &structuredVolume{} switch { case data.PersistentVolume != nil: volume.parsePV(data.PersistentVolume) if data.PVC != nil { volume.parsePVC(data.PVC) } case data.PodVolume != nil: volume.parsePodVolume(data.PodVolume) if data.PVC != nil { volume.parsePVC(data.PVC) } case data.PVC != nil: // Handle PVC-only scenarios (e.g., unbound PVCs) volume.parsePVC(data.PVC) default: return nil, errors.New("failed to convert object") } return p.match(volume), nil } func (p *Policies) Validate() error { if p.version != currentSupportDataVersion { return fmt.Errorf("incompatible version number %s with supported version %s", p.version, currentSupportDataVersion) } for _, policy := range p.volumePolicies { if err := policy.action.validate(); err != nil { return errors.WithStack(err) } for _, con := range policy.conditions { if err := con.validate(); err != nil { return errors.WithStack(err) } } } if p.GetIncludeExcludePolicy() != nil { if err := p.GetIncludeExcludePolicy().Validate(); err != nil { return errors.WithStack(err) } } return nil } func (p *Policies) GetIncludeExcludePolicy() *IncludeExcludePolicy { return p.includeExcludePolicy } func GetResourcePoliciesFromBackup( backup velerov1api.Backup, client crclient.Client, logger logrus.FieldLogger, ) (resourcePolicies *Policies, err error) { if backup.Spec.ResourcePolicy != nil && strings.EqualFold(backup.Spec.ResourcePolicy.Kind, ConfigmapRefType) { policiesConfigMap := &corev1api.ConfigMap{} err = client.Get( context.Background(), crclient.ObjectKey{Namespace: backup.Namespace, Name: backup.Spec.ResourcePolicy.Name}, policiesConfigMap, ) if err != nil { logger.Errorf("Fail to get ResourcePolicies %s ConfigMap with error %s.", backup.Namespace+"/"+backup.Spec.ResourcePolicy.Name, err.Error()) return nil, fmt.Errorf("fail to get ResourcePolicies %s ConfigMap with error %s", backup.Namespace+"/"+backup.Spec.ResourcePolicy.Name, err.Error()) } resourcePolicies, err = getResourcePoliciesFromConfig(policiesConfigMap) if err != nil { logger.Errorf("Fail to read ResourcePolicies from ConfigMap %s with error %s.", backup.Namespace+"/"+backup.Name, err.Error()) return nil, fmt.Errorf("fail to read the ResourcePolicies from ConfigMap %s with error %s", backup.Namespace+"/"+backup.Name, err.Error()) } else if err = resourcePolicies.Validate(); err != nil { logger.Errorf("Fail to validate ResourcePolicies in ConfigMap %s with error %s.", backup.Namespace+"/"+backup.Name, err.Error()) return nil, fmt.Errorf("fail to validate ResourcePolicies in ConfigMap %s with error %s", backup.Namespace+"/"+backup.Name, err.Error()) } } return resourcePolicies, nil } func getResourcePoliciesFromConfig(cm *corev1api.ConfigMap) (*Policies, error) { if cm == nil { return nil, fmt.Errorf("could not parse config from nil configmap") } if len(cm.Data) != 1 { return nil, fmt.Errorf("illegal resource policies %s/%s configmap", cm.Namespace, cm.Name) } var yamlData string for _, v := range cm.Data { yamlData = v } resPolicies, err := unmarshalResourcePolicies(&yamlData) if err != nil { return nil, errors.WithStack(err) } policies := &Policies{} if err := policies.BuildPolicy(resPolicies); err != nil { return nil, errors.WithStack(err) } return policies, nil } ================================================ FILE: internal/resourcepolicies/resource_policies_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package resourcepolicies import ( "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func TestLoadResourcePolicies(t *testing.T) { testCases := []struct { name string yamlData string wantErr bool }{ { name: "unknown key in yaml", yamlData: `version: v1 volumePolicies: - conditions: capacity: "0,100Gi" unknown: {} storageClass: - gp2 - ebs-sc action: type: skip`, wantErr: true, }, { name: "reduplicated key in yaml", yamlData: `version: v1 volumePolicies: - conditions: capacity: "0,100Gi" capacity: "0,100Gi" storageClass: - gp2 - ebs-sc action: type: skip`, wantErr: true, }, { name: "error format of storageClass", yamlData: `version: v1 volumePolicies: - conditions: capacity: "0,100Gi" storageClass: gp2 action: type: skip`, wantErr: true, }, { name: "error format of csi", yamlData: `version: v1 volumePolicies: - conditions: capacity: "0,100Gi" csi: gp2 action: type: skip`, wantErr: true, }, { name: "error format of nfs", yamlData: `version: v1 volumePolicies: - conditions: capacity: "0,100Gi" csi: {} nfs: abc action: type: skip`, wantErr: true, }, { name: "supported format volume policies", yamlData: `version: v1 volumePolicies: - conditions: capacity: '0,100Gi' csi: driver: aws.efs.csi.driver action: type: skip `, wantErr: false, }, { name: "supported format csi driver with volumeAttributes for volume policies", yamlData: `version: v1 volumePolicies: - conditions: capacity: '0,100Gi' csi: driver: aws.efs.csi.driver volumeAttributes: key1: value1 action: type: skip `, wantErr: false, }, { name: "supported format pvcLabels", yamlData: `version: v1 volumePolicies: - conditions: pvcLabels: environment: production app: database action: type: skip `, wantErr: false, }, { name: "error format of pvcLabels (not a map)", yamlData: `version: v1 volumePolicies: - conditions: pvcLabels: "production" action: type: skip `, wantErr: true, }, { name: "supported format pvcLabels with extra keys", yamlData: `version: v1 volumePolicies: - conditions: pvcLabels: environment: production region: us-west action: type: skip `, wantErr: false, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { _, err := unmarshalResourcePolicies(&tc.yamlData) if (err != nil) != tc.wantErr { t.Fatalf("Expected error %v, but got error %v", tc.wantErr, err) } }) } } func TestGetResourceMatchedAction(t *testing.T) { resPolicies := &ResourcePolicies{ Version: "v1", VolumePolicies: []VolumePolicy{ { Action: Action{Type: "skip"}, Conditions: map[string]any{ "capacity": "0,10Gi", "storageClass": []string{"gp2", "ebs-sc"}, "csi": any( map[string]any{ "driver": "aws.efs.csi.driver", }), }, }, { Action: Action{Type: "skip"}, Conditions: map[string]any{ "csi": any( map[string]any{ "driver": "files.csi.driver", "volumeAttributes": map[string]string{"protocol": "nfs"}, }), }, }, { Action: Action{Type: "snapshot"}, Conditions: map[string]any{ "capacity": "10,100Gi", "storageClass": []string{"gp2", "ebs-sc"}, "csi": any( map[string]any{ "driver": "aws.efs.csi.driver", }), }, }, { Action: Action{Type: "fs-backup"}, Conditions: map[string]any{ "storageClass": []string{"gp2", "ebs-sc"}, "csi": any( map[string]any{ "driver": "aws.efs.csi.driver", }), }, }, { Action: Action{Type: "snapshot"}, Conditions: map[string]any{ "pvcLabels": map[string]string{ "environment": "production", }, }, }, }, } testCases := []struct { name string volume *structuredVolume expectedAction *Action resourcePolicies *ResourcePolicies }{ { name: "match policy", volume: &structuredVolume{ capacity: *resource.NewQuantity(5<<30, resource.BinarySI), storageClass: "ebs-sc", csi: &csiVolumeSource{Driver: "aws.efs.csi.driver"}, }, expectedAction: &Action{Type: "skip"}, }, { name: "match policy AFS NFS", volume: &structuredVolume{ capacity: *resource.NewQuantity(5<<30, resource.BinarySI), storageClass: "afs-nfs", csi: &csiVolumeSource{Driver: "files.csi.driver", VolumeAttributes: map[string]string{"protocol": "nfs"}}, }, expectedAction: &Action{Type: "skip"}, }, { name: "match policy AFS SMB", volume: &structuredVolume{ capacity: *resource.NewQuantity(5<<30, resource.BinarySI), storageClass: "afs-smb", csi: &csiVolumeSource{Driver: "files.csi.driver"}, }, expectedAction: nil, }, { name: "both matches return the first policy", volume: &structuredVolume{ capacity: *resource.NewQuantity(50<<30, resource.BinarySI), storageClass: "ebs-sc", csi: &csiVolumeSource{Driver: "aws.efs.csi.driver"}, }, expectedAction: &Action{Type: "snapshot"}, }, { name: "mismatch all policies", volume: &structuredVolume{ capacity: *resource.NewQuantity(50<<30, resource.BinarySI), storageClass: "ebs-sc", nfs: &nFSVolumeSource{}, }, expectedAction: nil, }, { name: "match pvcLabels condition", volume: &structuredVolume{ capacity: *resource.NewQuantity(5<<30, resource.BinarySI), storageClass: "some-class", pvcLabels: map[string]string{ "environment": "production", "team": "backend", }, }, expectedAction: &Action{Type: "snapshot"}, }, { name: "mismatch pvcLabels condition", volume: &structuredVolume{ capacity: *resource.NewQuantity(5<<30, resource.BinarySI), storageClass: "some-class", pvcLabels: map[string]string{ "environment": "staging", }, }, expectedAction: nil, }, { name: "nil condition always match the action", volume: &structuredVolume{ capacity: *resource.NewQuantity(5<<30, resource.BinarySI), storageClass: "some-class", pvcLabels: map[string]string{ "environment": "staging", }, }, resourcePolicies: &ResourcePolicies{ Version: "v1", VolumePolicies: []VolumePolicy{ { Action: Action{Type: "skip"}, Conditions: map[string]any{}, }, }, }, expectedAction: &Action{Type: "skip"}, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { policies := &Policies{} currentResourcePolicy := resPolicies if tc.resourcePolicies != nil { currentResourcePolicy = tc.resourcePolicies } err := policies.BuildPolicy(currentResourcePolicy) if err != nil { t.Errorf("Failed to build policy with error %v", err) } action := policies.match(tc.volume) if action == nil { if tc.expectedAction != nil { t.Errorf("Expected action %v, but got result nil", tc.expectedAction.Type) } } else { if tc.expectedAction != nil { if action.Type != tc.expectedAction.Type { t.Errorf("Expected action %v, but got result %v", tc.expectedAction.Type, action.Type) } } else { t.Errorf("Expected action nil, but got result %v", action.Type) } } }) } } func TestGetResourcePoliciesFromConfig(t *testing.T) { // Create a test ConfigMap cm := &corev1api.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "test-configmap", Namespace: "test-namespace", }, Data: map[string]string{ "test-data": `version: v1 volumePolicies: - conditions: capacity: '0,10Gi' csi: driver: disks.csi.driver action: type: skip - conditions: csi: driver: files.csi.driver volumeAttributes: protocol: nfs action: type: skip - conditions: pvcLabels: environment: production action: type: skip `, }, } // Call the function and check for errors resPolicies, err := getResourcePoliciesFromConfig(cm) require.NoError(t, err) // Check that the returned resourcePolicies object contains the expected data assert.Equal(t, "v1", resPolicies.version) assert.Len(t, resPolicies.volumePolicies, 3) policies := ResourcePolicies{ Version: "v1", VolumePolicies: []VolumePolicy{ { Conditions: map[string]any{ "capacity": "0,10Gi", "csi": map[string]any{ "driver": "disks.csi.driver", }, }, Action: Action{ Type: Skip, }, }, { Conditions: map[string]any{ "csi": map[string]any{ "driver": "files.csi.driver", "volumeAttributes": map[string]string{"protocol": "nfs"}, }, }, Action: Action{ Type: Skip, }, }, { Conditions: map[string]any{ "pvcLabels": map[string]string{ "environment": "production", }, }, Action: Action{ Type: Skip, }, }, }, } p := &Policies{} err = p.BuildPolicy(&policies) if err != nil { t.Fatalf("failed to build policy: %v", err) } assert.Equal(t, p, resPolicies) } func TestGetMatchAction(t *testing.T) { testCases := []struct { name string yamlData string vol *corev1api.PersistentVolume podVol *corev1api.Volume pvc *corev1api.PersistentVolumeClaim skip bool }{ { name: "empty csi", yamlData: `version: v1 volumePolicies: - conditions: csi: {} action: type: skip`, vol: &corev1api.PersistentVolume{ Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ CSI: &corev1api.CSIPersistentVolumeSource{Driver: "ebs.csi.aws.com"}, }}, }, skip: true, }, { name: "empty csi with pv no csi driver", yamlData: `version: v1 volumePolicies: - conditions: csi: {} action: type: skip`, vol: &corev1api.PersistentVolume{ Spec: corev1api.PersistentVolumeSpec{ Capacity: corev1api.ResourceList{ corev1api.ResourceStorage: resource.MustParse("1Gi"), }}, }, skip: false, }, { name: "Skip AFS CSI condition with Disk volumes", yamlData: `version: v1 volumePolicies: - conditions: csi: driver: files.csi.driver action: type: skip`, vol: &corev1api.PersistentVolume{ Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ CSI: &corev1api.CSIPersistentVolumeSource{Driver: "disks.csi.driver"}, }}, }, skip: false, }, { name: "Skip AFS CSI condition with AFS volumes", yamlData: `version: v1 volumePolicies: - conditions: csi: driver: files.csi.driver action: type: skip`, vol: &corev1api.PersistentVolume{ Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ CSI: &corev1api.CSIPersistentVolumeSource{Driver: "files.csi.driver"}, }}, }, skip: true, }, { name: "Skip AFS NFS CSI condition with Disk volumes", yamlData: `version: v1 volumePolicies: - conditions: csi: driver: files.csi.driver volumeAttributes: protocol: nfs action: type: skip `, vol: &corev1api.PersistentVolume{ Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ CSI: &corev1api.CSIPersistentVolumeSource{Driver: "disks.csi.driver"}, }}, }, skip: false, }, { name: "Skip AFS NFS CSI condition with AFS SMB volumes", yamlData: `version: v1 volumePolicies: - conditions: csi: driver: files.csi.driver volumeAttributes: protocol: nfs action: type: skip `, vol: &corev1api.PersistentVolume{ Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ CSI: &corev1api.CSIPersistentVolumeSource{Driver: "files.csi.driver", VolumeAttributes: map[string]string{"key1": "val1"}}, }}, }, skip: false, }, { name: "Skip AFS NFS CSI condition with AFS NFS volumes", yamlData: `version: v1 volumePolicies: - conditions: csi: driver: files.csi.driver volumeAttributes: protocol: nfs action: type: skip `, vol: &corev1api.PersistentVolume{ Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ CSI: &corev1api.CSIPersistentVolumeSource{Driver: "files.csi.driver", VolumeAttributes: map[string]string{"protocol": "nfs"}}, }}, }, skip: true, }, { name: "Skip Disk and AFS NFS CSI condition with Disk volumes", yamlData: `version: v1 volumePolicies: - conditions: csi: driver: disks.csi.driver action: type: skip - conditions: csi: driver: files.csi.driver volumeAttributes: protocol: nfs action: type: skip`, vol: &corev1api.PersistentVolume{ Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ CSI: &corev1api.CSIPersistentVolumeSource{Driver: "disks.csi.driver", VolumeAttributes: map[string]string{"key1": "val1"}}, }}, }, skip: true, }, { name: "Skip Disk and AFS NFS CSI condition with AFS SMB volumes", yamlData: `version: v1 volumePolicies: - conditions: csi: driver: disks.csi.driver action: type: skip - conditions: csi: driver: files.csi.driver volumeAttributes: protocol: nfs action: type: skip`, vol: &corev1api.PersistentVolume{ Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ CSI: &corev1api.CSIPersistentVolumeSource{Driver: "files.csi.driver", VolumeAttributes: map[string]string{"key1": "val1"}}, }}, }, skip: false, }, { name: "Skip Disk and AFS NFS CSI condition with AFS NFS volumes", yamlData: `version: v1 volumePolicies: - conditions: csi: driver: disks.csi.driver action: type: skip - conditions: csi: driver: files.csi.driver volumeAttributes: protocol: nfs action: type: skip`, vol: &corev1api.PersistentVolume{ Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ CSI: &corev1api.CSIPersistentVolumeSource{Driver: "files.csi.driver", VolumeAttributes: map[string]string{"key1": "val1", "protocol": "nfs"}}, }}, }, skip: true, }, { name: "csi not configured and testing capacity condition", yamlData: `version: v1 volumePolicies: - conditions: capacity: "0,100Gi" action: type: skip`, vol: &corev1api.PersistentVolume{ Spec: corev1api.PersistentVolumeSpec{ Capacity: corev1api.ResourceList{ corev1api.ResourceStorage: resource.MustParse("1Gi"), }, PersistentVolumeSource: corev1api.PersistentVolumeSource{ CSI: &corev1api.CSIPersistentVolumeSource{Driver: "ebs.csi.aws.com"}, }}, }, skip: true, }, { name: "empty nfs", yamlData: `version: v1 volumePolicies: - conditions: nfs: {} action: type: skip`, vol: &corev1api.PersistentVolume{ Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ NFS: &corev1api.NFSVolumeSource{Server: "192.168.1.20"}, }}, }, skip: true, }, { name: "nfs not configured", yamlData: `version: v1 volumePolicies: - conditions: capacity: "0,100Gi" action: type: skip`, vol: &corev1api.PersistentVolume{ Spec: corev1api.PersistentVolumeSpec{ Capacity: corev1api.ResourceList{ corev1api.ResourceStorage: resource.MustParse("1Gi"), }, PersistentVolumeSource: corev1api.PersistentVolumeSource{ NFS: &corev1api.NFSVolumeSource{Server: "192.168.1.20"}, }, }, }, skip: true, }, { name: "empty nfs with pv no nfs volume source", yamlData: `version: v1 volumePolicies: - conditions: capacity: "0,100Gi" nfs: {} action: type: skip`, vol: &corev1api.PersistentVolume{ Spec: corev1api.PersistentVolumeSpec{ Capacity: corev1api.ResourceList{ corev1api.ResourceStorage: resource.MustParse("1Gi"), }, }, }, skip: false, }, { name: "match volume by types", yamlData: `version: v1 volumePolicies: - conditions: capacity: "0,100Gi" volumeTypes: - local - hostPath action: type: skip`, vol: &corev1api.PersistentVolume{ Spec: corev1api.PersistentVolumeSpec{ Capacity: corev1api.ResourceList{ corev1api.ResourceStorage: resource.MustParse("1Gi"), }, PersistentVolumeSource: corev1api.PersistentVolumeSource{ HostPath: &corev1api.HostPathVolumeSource{Path: "/mnt/data"}, }, }, }, skip: true, }, { name: "mismatch volume by types", yamlData: `version: v1 volumePolicies: - conditions: capacity: "0,100Gi" volumeTypes: - local action: type: skip`, vol: &corev1api.PersistentVolume{ Spec: corev1api.PersistentVolumeSpec{ Capacity: corev1api.ResourceList{ corev1api.ResourceStorage: resource.MustParse("1Gi"), }, PersistentVolumeSource: corev1api.PersistentVolumeSource{ HostPath: &corev1api.HostPathVolumeSource{Path: "/mnt/data"}, }, }, }, skip: false, }, { name: "PVC labels match", yamlData: `version: v1 volumePolicies: - conditions: capacity: "0,100Gi" pvcLabels: environment: production action: type: skip`, vol: &corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "pv-1", }, Spec: corev1api.PersistentVolumeSpec{ Capacity: corev1api.ResourceList{ corev1api.ResourceStorage: resource.MustParse("1Gi"), }, PersistentVolumeSource: corev1api.PersistentVolumeSource{}, ClaimRef: &corev1api.ObjectReference{ Namespace: "default", Name: "pvc-1", }, }, }, pvc: &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: "default", Name: "pvc-1", Labels: map[string]string{"environment": "production"}, }, }, skip: true, }, { name: "PVC labels match, criteria label is a subset of the pvc labels", yamlData: `version: v1 volumePolicies: - conditions: capacity: "0,100Gi" pvcLabels: environment: production action: type: skip`, vol: &corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "pv-1", }, Spec: corev1api.PersistentVolumeSpec{ Capacity: corev1api.ResourceList{ corev1api.ResourceStorage: resource.MustParse("1Gi"), }, PersistentVolumeSource: corev1api.PersistentVolumeSource{}, ClaimRef: &corev1api.ObjectReference{ Namespace: "default", Name: "pvc-1", }, }, }, pvc: &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: "default", Name: "pvc-1", Labels: map[string]string{"environment": "production", "app": "backend"}, }, }, skip: true, }, { name: "PVC labels match don't match exactly", yamlData: `version: v1 volumePolicies: - conditions: capacity: "0,100Gi" pvcLabels: environment: production app: frontend action: type: skip`, vol: &corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "pv-1", }, Spec: corev1api.PersistentVolumeSpec{ Capacity: corev1api.ResourceList{ corev1api.ResourceStorage: resource.MustParse("1Gi"), }, PersistentVolumeSource: corev1api.PersistentVolumeSource{}, ClaimRef: &corev1api.ObjectReference{ Namespace: "default", Name: "pvc-1", }, }, }, pvc: &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: "default", Name: "pvc-1", Labels: map[string]string{"environment": "production"}, }, }, skip: false, }, { name: "PVC labels mismatch", yamlData: `version: v1 volumePolicies: - conditions: capacity: "0,100Gi" pvcLabels: environment: production action: type: skip`, vol: &corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "pv-2", }, Spec: corev1api.PersistentVolumeSpec{ Capacity: corev1api.ResourceList{ corev1api.ResourceStorage: resource.MustParse("1Gi"), }, PersistentVolumeSource: corev1api.PersistentVolumeSource{}, ClaimRef: &corev1api.ObjectReference{ Namespace: "default", Name: "pvc-2", }, }, }, pvc: &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: "default", Name: "pvc-1", Labels: map[string]string{"environment": "staging"}, }, }, skip: false, }, { name: "PodVolume case with PVC labels match", yamlData: `version: v1 volumePolicies: - conditions: pvcLabels: environment: production action: type: skip`, vol: nil, podVol: &corev1api.Volume{Name: "pod-vol-1"}, pvc: &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: "default", Name: "pvc-1", Labels: map[string]string{"environment": "production"}, }, }, skip: true, }, { name: "PodVolume case with PVC labels mismatch", yamlData: `version: v1 volumePolicies: - conditions: pvcLabels: environment: production action: type: skip`, vol: nil, podVol: &corev1api.Volume{Name: "pod-vol-2"}, pvc: &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: "default", Name: "pvc-2", Labels: map[string]string{"environment": "staging"}, }, }, skip: false, }, { name: "PodVolume case with PVC labels match with extra keys on PVC", yamlData: `version: v1 volumePolicies: - conditions: pvcLabels: environment: production action: type: skip`, vol: nil, podVol: &corev1api.Volume{Name: "pod-vol-3"}, pvc: &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: "default", Name: "pvc-3", Labels: map[string]string{"environment": "production", "app": "backend"}, }, }, skip: true, }, { name: "PodVolume case with PVC labels don't match exactly", yamlData: `version: v1 volumePolicies: - conditions: pvcLabels: environment: production app: frontend action: type: skip`, vol: nil, podVol: &corev1api.Volume{Name: "pod-vol-4"}, pvc: &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: "default", Name: "pvc-4", Labels: map[string]string{"environment": "production"}, }, }, skip: false, }, { name: "PVC phase matching - Pending phase should skip", yamlData: `version: v1 volumePolicies: - conditions: pvcPhase: ["Pending"] action: type: skip`, vol: nil, podVol: nil, pvc: &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: "default", Name: "pvc-pending", }, Status: corev1api.PersistentVolumeClaimStatus{ Phase: corev1api.ClaimPending, }, }, skip: true, }, { name: "PVC phase matching - Bound phase should not skip", yamlData: `version: v1 volumePolicies: - conditions: pvcPhase: ["Pending"] action: type: skip`, vol: nil, podVol: nil, pvc: &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: "default", Name: "pvc-bound", }, Status: corev1api.PersistentVolumeClaimStatus{ Phase: corev1api.ClaimBound, }, }, skip: false, }, { name: "PVC phase matching - Multiple phases (Pending, Lost)", yamlData: `version: v1 volumePolicies: - conditions: pvcPhase: ["Pending", "Lost"] action: type: skip`, vol: nil, podVol: nil, pvc: &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: "default", Name: "pvc-lost", }, Status: corev1api.PersistentVolumeClaimStatus{ Phase: corev1api.ClaimLost, }, }, skip: true, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { resPolicies, err := unmarshalResourcePolicies(&tc.yamlData) if err != nil { t.Fatalf("got error when get match action %v", err) } require.NoError(t, err) policies := &Policies{} err = policies.BuildPolicy(resPolicies) require.NoError(t, err) vfd := VolumeFilterData{} if tc.pvc != nil { vfd.PVC = tc.pvc } if tc.vol != nil { vfd.PersistentVolume = tc.vol } if tc.podVol != nil { vfd.PodVolume = tc.podVol } action, err := policies.GetMatchAction(vfd) require.NoError(t, err) if tc.skip { if action.Type != Skip { t.Fatalf("Expected action skip but is %v", action.Type) } } else if action != nil && action.Type == Skip { t.Fatalf("Expected action not skip but is %v", action.Type) } }) } } func TestGetMatchAction_Errors(t *testing.T) { p := &Policies{} testCases := []struct { name string input any expectedErr string }{ { name: "invalid input type", input: "invalid input", expectedErr: "failed to convert input to VolumeFilterData", }, { name: "no volume provided", input: VolumeFilterData{ PersistentVolume: nil, PodVolume: nil, PVC: nil, }, expectedErr: "failed to convert object", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { action, err := p.GetMatchAction(tc.input) require.ErrorContains(t, err, tc.expectedErr) assert.Nil(t, action) }) } } func TestParsePVC(t *testing.T) { tests := []struct { name string pvc *corev1api.PersistentVolumeClaim expectedLabels map[string]string expectedPhase string expectErr bool }{ { name: "valid PVC with labels and Pending phase", pvc: &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{"env": "prod"}, }, Status: corev1api.PersistentVolumeClaimStatus{ Phase: corev1api.ClaimPending, }, }, expectedLabels: map[string]string{"env": "prod"}, expectedPhase: "Pending", expectErr: false, }, { name: "valid PVC with Bound phase", pvc: &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{}, }, Status: corev1api.PersistentVolumeClaimStatus{ Phase: corev1api.ClaimBound, }, }, expectedLabels: nil, expectedPhase: "Bound", expectErr: false, }, { name: "valid PVC with Lost phase", pvc: &corev1api.PersistentVolumeClaim{ Status: corev1api.PersistentVolumeClaimStatus{ Phase: corev1api.ClaimLost, }, }, expectedLabels: nil, expectedPhase: "Lost", expectErr: false, }, { name: "nil PVC pointer", pvc: (*corev1api.PersistentVolumeClaim)(nil), expectedLabels: nil, expectedPhase: "", expectErr: false, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { s := &structuredVolume{} s.parsePVC(tc.pvc) assert.Equal(t, tc.expectedLabels, s.pvcLabels) assert.Equal(t, tc.expectedPhase, s.pvcPhase) }) } } func TestPVCPhaseMatch(t *testing.T) { tests := []struct { name string condition *pvcPhaseCondition volume *structuredVolume expectedMatch bool }{ { name: "match Pending phase", condition: &pvcPhaseCondition{phases: []string{"Pending"}}, volume: &structuredVolume{pvcPhase: "Pending"}, expectedMatch: true, }, { name: "match multiple phases - Pending matches", condition: &pvcPhaseCondition{phases: []string{"Pending", "Bound"}}, volume: &structuredVolume{pvcPhase: "Pending"}, expectedMatch: true, }, { name: "match multiple phases - Bound matches", condition: &pvcPhaseCondition{phases: []string{"Pending", "Bound"}}, volume: &structuredVolume{pvcPhase: "Bound"}, expectedMatch: true, }, { name: "no match for different phase", condition: &pvcPhaseCondition{phases: []string{"Pending"}}, volume: &structuredVolume{pvcPhase: "Bound"}, expectedMatch: false, }, { name: "no match for empty phase", condition: &pvcPhaseCondition{phases: []string{"Pending"}}, volume: &structuredVolume{pvcPhase: ""}, expectedMatch: false, }, { name: "match with empty phases list (always match)", condition: &pvcPhaseCondition{phases: []string{}}, volume: &structuredVolume{pvcPhase: "Pending"}, expectedMatch: true, }, { name: "match with nil phases list (always match)", condition: &pvcPhaseCondition{phases: nil}, volume: &structuredVolume{pvcPhase: "Pending"}, expectedMatch: true, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { result := tc.condition.match(tc.volume) assert.Equal(t, tc.expectedMatch, result) }) } } ================================================ FILE: internal/resourcepolicies/volume_filter_data.go ================================================ package resourcepolicies import ( corev1api "k8s.io/api/core/v1" ) // VolumeFilterData bundles the volume data needed for volume policy filtering type VolumeFilterData struct { PersistentVolume *corev1api.PersistentVolume PodVolume *corev1api.Volume PVC *corev1api.PersistentVolumeClaim } // NewVolumeFilterData constructs a new VolumeFilterData instance. func NewVolumeFilterData(pv *corev1api.PersistentVolume, podVol *corev1api.Volume, pvc *corev1api.PersistentVolumeClaim) VolumeFilterData { return VolumeFilterData{ PersistentVolume: pv, PodVolume: podVol, PVC: pvc, } } ================================================ FILE: internal/resourcepolicies/volume_filter_data_test.go ================================================ package resourcepolicies import ( "testing" "github.com/stretchr/testify/assert" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func TestNewVolumeFilterData(t *testing.T) { testCases := []struct { name string pv *corev1api.PersistentVolume podVol *corev1api.Volume pvc *corev1api.PersistentVolumeClaim expectedPVName string expectedPodName string expectedPVCName string }{ { name: "all provided", pv: &corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "pv-test", }, }, podVol: &corev1api.Volume{ Name: "pod-vol-test", }, pvc: &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: "pvc-test", }, }, expectedPVName: "pv-test", expectedPodName: "pod-vol-test", expectedPVCName: "pvc-test", }, { name: "only PV provided", pv: &corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "pv-only", }, }, podVol: nil, pvc: nil, expectedPVName: "pv-only", expectedPodName: "", expectedPVCName: "", }, { name: "only PodVolume provided", pv: nil, podVol: &corev1api.Volume{ Name: "pod-only", }, pvc: nil, expectedPVName: "", expectedPodName: "pod-only", expectedPVCName: "", }, { name: "only PVC provided", pv: nil, podVol: nil, pvc: &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: "pvc-only", }, }, expectedPVName: "", expectedPodName: "", expectedPVCName: "pvc-only", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { vfd := NewVolumeFilterData(tc.pv, tc.podVol, tc.pvc) if tc.expectedPVName != "" { assert.NotNil(t, vfd.PersistentVolume) assert.Equal(t, tc.expectedPVName, vfd.PersistentVolume.Name) } else { assert.Nil(t, vfd.PersistentVolume) } if tc.expectedPodName != "" { assert.NotNil(t, vfd.PodVolume) assert.Equal(t, tc.expectedPodName, vfd.PodVolume.Name) } else { assert.Nil(t, vfd.PodVolume) } if tc.expectedPVCName != "" { assert.NotNil(t, vfd.PVC) assert.Equal(t, tc.expectedPVCName, vfd.PVC.Name) } else { assert.Nil(t, vfd.PVC) } }) } } ================================================ FILE: internal/resourcepolicies/volume_resources.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package resourcepolicies import ( "bytes" "fmt" "strings" "k8s.io/apimachinery/pkg/labels" "github.com/pkg/errors" "gopkg.in/yaml.v3" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" ) type volPolicy struct { action Action conditions []volumeCondition } type volumeCondition interface { match(v *structuredVolume) bool validate() error } // capacity consist of the lower and upper boundary type capacity struct { lower resource.Quantity upper resource.Quantity } type structuredVolume struct { capacity resource.Quantity storageClass string nfs *nFSVolumeSource csi *csiVolumeSource volumeType SupportedVolume pvcLabels map[string]string pvcPhase string } func (s *structuredVolume) parsePV(pv *corev1api.PersistentVolume) { s.capacity = *pv.Spec.Capacity.Storage() s.storageClass = pv.Spec.StorageClassName nfs := pv.Spec.NFS if nfs != nil { s.nfs = &nFSVolumeSource{Server: nfs.Server, Path: nfs.Path} } csi := pv.Spec.CSI if csi != nil { s.csi = &csiVolumeSource{Driver: csi.Driver, VolumeAttributes: csi.VolumeAttributes} } s.volumeType = getVolumeTypeFromPV(pv) } func (s *structuredVolume) parsePVC(pvc *corev1api.PersistentVolumeClaim) { if pvc != nil { if len(pvc.GetLabels()) > 0 { s.pvcLabels = pvc.Labels } s.pvcPhase = string(pvc.Status.Phase) } } func (s *structuredVolume) parsePodVolume(vol *corev1api.Volume) { nfs := vol.NFS if nfs != nil { s.nfs = &nFSVolumeSource{Server: nfs.Server, Path: nfs.Path} } csi := vol.CSI if csi != nil { s.csi = &csiVolumeSource{Driver: csi.Driver, VolumeAttributes: csi.VolumeAttributes} } s.volumeType = getVolumeTypeFromVolume(vol) } // pvcLabelsCondition defines a condition that matches if the PVC's labels contain all the provided key/value pairs. type pvcLabelsCondition struct { labels map[string]string } func (c *pvcLabelsCondition) match(v *structuredVolume) bool { // No labels specified: always match. if len(c.labels) == 0 { return true } if v.pvcLabels == nil { return false } selector := labels.SelectorFromSet(c.labels) return selector.Matches(labels.Set(v.pvcLabels)) } func (c *pvcLabelsCondition) validate() error { return nil } // pvcPhaseCondition defines a condition that matches if the PVC's phase matches any of the provided phases. type pvcPhaseCondition struct { phases []string } func (c *pvcPhaseCondition) match(v *structuredVolume) bool { // No phases specified: always match. if len(c.phases) == 0 { return true } if v.pvcPhase == "" { return false } for _, phase := range c.phases { if v.pvcPhase == phase { return true } } return false } func (c *pvcPhaseCondition) validate() error { return nil } type capacityCondition struct { capacity capacity } func (c *capacityCondition) match(v *structuredVolume) bool { return c.capacity.isInRange(v.capacity) } type storageClassCondition struct { storageClass []string } func (s *storageClassCondition) match(v *structuredVolume) bool { if len(s.storageClass) == 0 { return true } if v.storageClass == "" { return false } for _, sc := range s.storageClass { if v.storageClass == sc { return true } } return false } type nfsCondition struct { nfs *nFSVolumeSource } func (c *nfsCondition) match(v *structuredVolume) bool { if c.nfs == nil { return true } if v.nfs == nil { return false } if c.nfs.Path == "" { if c.nfs.Server == "" { // match nfs: {} return v.nfs != nil } if c.nfs.Server != v.nfs.Server { return false } return true } if c.nfs.Path != v.nfs.Path { return false } if c.nfs.Server == "" { return true } if c.nfs.Server != v.nfs.Server { return false } return true } type csiCondition struct { csi *csiVolumeSource } func (c *csiCondition) match(v *structuredVolume) bool { if c.csi == nil { return true } if c.csi.Driver == "" { // match csi: {} return v.csi != nil } if v.csi == nil { return false } if c.csi.Driver != v.csi.Driver { return false } if len(c.csi.VolumeAttributes) == 0 { return true } if len(v.csi.VolumeAttributes) == 0 { return false } for key, value := range c.csi.VolumeAttributes { if value != v.csi.VolumeAttributes[key] { return false } } return true } // parseCapacity parse string into capacity format func parseCapacity(cap string) (*capacity, error) { if cap == "" { cap = "," } capacities := strings.Split(cap, ",") var quantities []resource.Quantity if len(capacities) != 2 { return nil, fmt.Errorf("wrong format of Capacity %v", cap) } for _, v := range capacities { if strings.TrimSpace(v) == "" { // case similar "10Gi," // if empty, the quantity will assigned with 0 quantities = append(quantities, *resource.NewQuantity(int64(0), resource.DecimalSI)) } else { quantity, err := resource.ParseQuantity(strings.TrimSpace(v)) if err != nil { return nil, fmt.Errorf("wrong format of Capacity %v with err %v", v, err) } quantities = append(quantities, quantity) } } return &capacity{lower: quantities[0], upper: quantities[1]}, nil } // isInRange returns true if the quantity y is in range of capacity, or it returns false func (c *capacity) isInRange(y resource.Quantity) bool { if c.lower.IsZero() && c.upper.Cmp(y) >= 0 { // [0, a] y return true } if c.upper.IsZero() && c.lower.Cmp(y) <= 0 { // [b, 0] y return true } if !c.lower.IsZero() && !c.upper.IsZero() { // [a, b] y return c.lower.Cmp(y) <= 0 && c.upper.Cmp(y) >= 0 } return false } // unmarshalVolConditions parse map[string]any into volumeConditions format // and validate key fields of the map. func unmarshalVolConditions(con map[string]any) (*volumeConditions, error) { volConditons := &volumeConditions{} buffer := new(bytes.Buffer) err := yaml.NewEncoder(buffer).Encode(con) if err != nil { return nil, errors.Wrap(err, "failed to encode volume conditions") } if err := decodeStruct(buffer, volConditons); err != nil { return nil, errors.Wrap(err, "failed to decode volume conditions") } return volConditons, nil } ================================================ FILE: internal/resourcepolicies/volume_resources_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package resourcepolicies import ( "fmt" "strings" "testing" "github.com/stretchr/testify/assert" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" ) func setStructuredVolume(capacity resource.Quantity, sc string, nfs *nFSVolumeSource, csi *csiVolumeSource, pvcLabels map[string]string) *structuredVolume { return &structuredVolume{ capacity: capacity, storageClass: sc, nfs: nfs, csi: csi, pvcLabels: pvcLabels, } } func TestPVCLabelsMatch(t *testing.T) { tests := []struct { name string condition *pvcLabelsCondition volume *structuredVolume expectedMatch bool }{ { name: "match exact label (single)", condition: &pvcLabelsCondition{ labels: map[string]string{"environment": "production"}, }, volume: setStructuredVolume( *resource.NewQuantity(0, resource.BinarySI), "any", nil, nil, map[string]string{"environment": "production", "app": "database"}, ), expectedMatch: true, }, { name: "match exact label (multiple)", condition: &pvcLabelsCondition{ labels: map[string]string{"environment": "production", "app": "database"}, }, volume: setStructuredVolume( *resource.NewQuantity(0, resource.BinarySI), "any", nil, nil, map[string]string{"environment": "production", "app": "database"}, ), expectedMatch: true, }, { name: "mismatch label value", condition: &pvcLabelsCondition{ labels: map[string]string{"environment": "production"}, }, volume: setStructuredVolume( *resource.NewQuantity(0, resource.BinarySI), "any", nil, nil, map[string]string{"environment": "staging", "app": "database"}, ), expectedMatch: false, }, { name: "missing label key", condition: &pvcLabelsCondition{ labels: map[string]string{"environment": "production", "region": "us-west"}, }, volume: setStructuredVolume( *resource.NewQuantity(0, resource.BinarySI), "any", nil, nil, map[string]string{"environment": "production", "app": "database"}, ), expectedMatch: false, }, { name: "empty condition always matches", condition: &pvcLabelsCondition{ labels: map[string]string{}, }, volume: setStructuredVolume( *resource.NewQuantity(0, resource.BinarySI), "any", nil, nil, map[string]string{"environment": "staging"}, ), expectedMatch: true, }, { name: "nil pvcLabels fails non-empty condition", condition: &pvcLabelsCondition{ labels: map[string]string{"environment": "production"}, }, volume: setStructuredVolume( *resource.NewQuantity(0, resource.BinarySI), "any", nil, nil, nil, ), expectedMatch: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { match := tt.condition.match(tt.volume) assert.Equal(t, tt.expectedMatch, match, "expected match %v, got %v", tt.expectedMatch, match) }) } } func TestParseCapacity(t *testing.T) { var emptyCapacity capacity tests := []struct { input string expected capacity expectedErr error }{ {"10Gi,20Gi", capacity{lower: *resource.NewQuantity(10<<30, resource.BinarySI), upper: *resource.NewQuantity(20<<30, resource.BinarySI)}, nil}, {"10Gi,", capacity{lower: *resource.NewQuantity(10<<30, resource.BinarySI), upper: *resource.NewQuantity(0, resource.DecimalSI)}, nil}, {"10Gi", emptyCapacity, fmt.Errorf("wrong format of Capacity 10Gi")}, {"", emptyCapacity, nil}, } for _, test := range tests { t.Run(test.input, func(t *testing.T) { actual, actualErr := parseCapacity(test.input) if test.expected != emptyCapacity { assert.Equal(t, 0, test.expected.lower.Cmp(actual.lower)) assert.Equal(t, 0, test.expected.upper.Cmp(actual.upper)) } assert.Equal(t, test.expectedErr, actualErr) }) } } func TestCapacityIsInRange(t *testing.T) { t.Parallel() tests := []struct { capacity *capacity quantity resource.Quantity isInRange bool }{ {&capacity{*resource.NewQuantity(0, resource.BinarySI), *resource.NewQuantity(10<<30, resource.BinarySI)}, *resource.NewQuantity(5<<30, resource.BinarySI), true}, {&capacity{*resource.NewQuantity(0, resource.BinarySI), *resource.NewQuantity(10<<30, resource.BinarySI)}, *resource.NewQuantity(15<<30, resource.BinarySI), false}, {&capacity{*resource.NewQuantity(20<<30, resource.BinarySI), *resource.NewQuantity(0, resource.DecimalSI)}, *resource.NewQuantity(25<<30, resource.BinarySI), true}, {&capacity{*resource.NewQuantity(20<<30, resource.BinarySI), *resource.NewQuantity(0, resource.DecimalSI)}, *resource.NewQuantity(15<<30, resource.BinarySI), false}, {&capacity{*resource.NewQuantity(10<<30, resource.BinarySI), *resource.NewQuantity(20<<30, resource.BinarySI)}, *resource.NewQuantity(15<<30, resource.BinarySI), true}, {&capacity{*resource.NewQuantity(10<<30, resource.BinarySI), *resource.NewQuantity(20<<30, resource.BinarySI)}, *resource.NewQuantity(5<<30, resource.BinarySI), false}, {&capacity{*resource.NewQuantity(10<<30, resource.BinarySI), *resource.NewQuantity(20<<30, resource.BinarySI)}, *resource.NewQuantity(25<<30, resource.BinarySI), false}, {&capacity{*resource.NewQuantity(0, resource.BinarySI), *resource.NewQuantity(0, resource.BinarySI)}, *resource.NewQuantity(5<<30, resource.BinarySI), true}, } for _, test := range tests { t.Run(fmt.Sprintf("%v with %v", test.capacity, test.quantity), func(t *testing.T) { t.Parallel() actual := test.capacity.isInRange(test.quantity) assert.Equal(t, test.isInRange, actual) }) } } func TestStorageClassConditionMatch(t *testing.T) { tests := []struct { name string condition *storageClassCondition volume *structuredVolume expectedMatch bool }{ { name: "match single storage class", condition: &storageClassCondition{[]string{"gp2"}}, volume: setStructuredVolume(*resource.NewQuantity(0, resource.BinarySI), "gp2", nil, nil, nil), expectedMatch: true, }, { name: "match multiple storage classes", condition: &storageClassCondition{[]string{"gp2", "ebs-sc"}}, volume: setStructuredVolume(*resource.NewQuantity(0, resource.BinarySI), "gp2", nil, nil, nil), expectedMatch: true, }, { name: "mismatch storage class", condition: &storageClassCondition{[]string{"gp2"}}, volume: setStructuredVolume(*resource.NewQuantity(0, resource.BinarySI), "ebs-sc", nil, nil, nil), expectedMatch: false, }, { name: "empty storage class", condition: &storageClassCondition{[]string{}}, volume: setStructuredVolume(*resource.NewQuantity(0, resource.BinarySI), "ebs-sc", nil, nil, nil), expectedMatch: true, }, { name: "empty volume storage class", condition: &storageClassCondition{[]string{"gp2"}}, volume: setStructuredVolume(*resource.NewQuantity(0, resource.BinarySI), "", nil, nil, nil), expectedMatch: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { match := tt.condition.match(tt.volume) if match != tt.expectedMatch { t.Errorf("expected %v, but got %v", tt.expectedMatch, match) } }) } } func TestNFSConditionMatch(t *testing.T) { tests := []struct { name string condition *nfsCondition volume *structuredVolume expectedMatch bool }{ { name: "match nfs condition", condition: &nfsCondition{&nFSVolumeSource{Server: "192.168.10.20"}}, volume: setStructuredVolume(*resource.NewQuantity(0, resource.BinarySI), "", &nFSVolumeSource{Server: "192.168.10.20"}, nil, nil), expectedMatch: true, }, { name: "empty nfs condition", condition: &nfsCondition{nil}, volume: setStructuredVolume(*resource.NewQuantity(0, resource.BinarySI), "", &nFSVolumeSource{Server: "192.168.10.20"}, nil, nil), expectedMatch: true, }, { name: "empty nfs server and path condition", condition: &nfsCondition{&nFSVolumeSource{Server: "", Path: ""}}, volume: setStructuredVolume(*resource.NewQuantity(0, resource.BinarySI), "", &nFSVolumeSource{Server: "192.168.10.20"}, nil, nil), expectedMatch: true, }, { name: "server mismatch", condition: &nfsCondition{&nFSVolumeSource{Server: "192.168.10.20", Path: ""}}, volume: setStructuredVolume(*resource.NewQuantity(0, resource.BinarySI), "", &nFSVolumeSource{Server: ""}, nil, nil), expectedMatch: false, }, { name: "empty nfs server condition", condition: &nfsCondition{&nFSVolumeSource{Path: "/mnt/data"}}, volume: setStructuredVolume(*resource.NewQuantity(0, resource.BinarySI), "", &nFSVolumeSource{Server: "192.168.10.20", Path: "/mnt/data"}, nil, nil), expectedMatch: true, }, { name: "empty nfs volume", condition: &nfsCondition{&nFSVolumeSource{Server: "192.168.10.20"}}, volume: setStructuredVolume(*resource.NewQuantity(0, resource.BinarySI), "", nil, nil, nil), expectedMatch: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { match := tt.condition.match(tt.volume) if match != tt.expectedMatch { t.Errorf("expected %v, but got %v", tt.expectedMatch, match) } }) } } func TestCSIConditionMatch(t *testing.T) { tests := []struct { name string condition *csiCondition volume *structuredVolume expectedMatch bool }{ { name: "match csi driver condition", condition: &csiCondition{&csiVolumeSource{Driver: "test"}}, volume: setStructuredVolume(*resource.NewQuantity(0, resource.BinarySI), "", nil, &csiVolumeSource{Driver: "test"}, nil), expectedMatch: true, }, { name: "empty csi driver condition", condition: &csiCondition{nil}, volume: setStructuredVolume(*resource.NewQuantity(0, resource.BinarySI), "", nil, &csiVolumeSource{Driver: "test"}, nil), expectedMatch: true, }, { name: "empty csi driver volume", condition: &csiCondition{&csiVolumeSource{Driver: "test"}}, volume: setStructuredVolume(*resource.NewQuantity(0, resource.BinarySI), "", nil, &csiVolumeSource{}, nil), expectedMatch: false, }, { name: "match csi volumeAttributes condition", condition: &csiCondition{&csiVolumeSource{Driver: "test", VolumeAttributes: map[string]string{"protocol": "nfs"}}}, volume: setStructuredVolume(*resource.NewQuantity(0, resource.BinarySI), "", nil, &csiVolumeSource{Driver: "test", VolumeAttributes: map[string]string{"protocol": "nfs"}}, nil), expectedMatch: true, }, { name: "empty csi volumeAttributes condition", condition: &csiCondition{&csiVolumeSource{Driver: "test"}}, volume: setStructuredVolume(*resource.NewQuantity(0, resource.BinarySI), "", nil, &csiVolumeSource{Driver: "test", VolumeAttributes: map[string]string{"protocol": "nfs"}}, nil), expectedMatch: true, }, { name: "empty csi volumeAttributes volume", condition: &csiCondition{&csiVolumeSource{Driver: "test", VolumeAttributes: map[string]string{"protocol": "nfs"}}}, volume: setStructuredVolume(*resource.NewQuantity(0, resource.BinarySI), "", nil, &csiVolumeSource{Driver: "test", VolumeAttributes: map[string]string{"protocol": ""}}, nil), expectedMatch: false, }, { name: "empty csi volumeAttributes volume", condition: &csiCondition{&csiVolumeSource{Driver: "test", VolumeAttributes: map[string]string{"protocol": "nfs"}}}, volume: setStructuredVolume(*resource.NewQuantity(0, resource.BinarySI), "", nil, &csiVolumeSource{Driver: "test"}, nil), expectedMatch: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { match := tt.condition.match(tt.volume) if match != tt.expectedMatch { t.Errorf("expected %v, but got %v", tt.expectedMatch, match) } }) } } func TestUnmarshalVolumeConditions(t *testing.T) { testCases := []struct { name string input map[string]any expectedError string }{ { name: "Valid input", input: map[string]any{ "capacity": "1Gi,10Gi", "storageClass": []string{ "gp2", "ebs-sc", }, "csi": &csiVolumeSource{ Driver: "aws.efs.csi.driver", }, }, expectedError: "", }, { name: "Invalid input: invalid capacity filed name", input: map[string]any{ "Capacity": "1Gi,10Gi", }, expectedError: "field Capacity not found", }, { name: "Invalid input: invalid storage class format", input: map[string]any{ "storageClass": "ebs-sc", }, expectedError: "str `ebs-sc` into []string", }, { name: "Invalid input: invalid csi format", input: map[string]any{ "csi": "csi.driver", }, expectedError: "str `csi.driver` into resourcepolicies.csiVolumeSource", }, { name: "Invalid input: unknown field", input: map[string]any{ "unknown": "foo", }, expectedError: "field unknown not found in type", }, { name: "Valid pvcLabels input as map[string]string", input: map[string]any{ "capacity": "1Gi,10Gi", "pvcLabels": map[string]string{ "environment": "production", }, }, expectedError: "", }, { name: "Valid pvcLabels input as map[string]any", input: map[string]any{ "capacity": "1Gi,10Gi", "pvcLabels": map[string]any{ "environment": "production", "app": "database", }, }, expectedError: "", }, { name: "Invalid pvcLabels input: not a map", input: map[string]any{ "capacity": "1Gi,10Gi", "pvcLabels": "production", }, expectedError: "!!str `production` into map[string]string", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { _, err := unmarshalVolConditions(tc.input) if tc.expectedError != "" { if err == nil { t.Errorf("Expected error '%s', but got nil", tc.expectedError) } else if !strings.Contains(err.Error(), tc.expectedError) { t.Errorf("Expected error '%s', but got '%v'", tc.expectedError, err) } } }) } } func TestParsePodVolume(t *testing.T) { // Mock data nfsVolume := corev1api.Volume{} nfsVolume.NFS = &corev1api.NFSVolumeSource{ Server: "nfs.example.com", Path: "/exports/data", } csiVolume := corev1api.Volume{} csiVolume.CSI = &corev1api.CSIVolumeSource{ Driver: "csi.example.com", VolumeAttributes: map[string]string{"protocol": "nfs"}, } emptyVolume := corev1api.Volume{} // Test cases testCases := []struct { name string inputVolume *corev1api.Volume expectedNFS *nFSVolumeSource expectedCSI *csiVolumeSource }{ { name: "NFS volume", inputVolume: &nfsVolume, expectedNFS: &nFSVolumeSource{Server: "nfs.example.com", Path: "/exports/data"}, }, { name: "CSI volume", inputVolume: &csiVolume, expectedCSI: &csiVolumeSource{Driver: "csi.example.com", VolumeAttributes: map[string]string{"protocol": "nfs"}}, }, { name: "Empty volume", inputVolume: &emptyVolume, expectedNFS: nil, expectedCSI: nil, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { // Call the function structuredVolume := &structuredVolume{} structuredVolume.parsePodVolume(tc.inputVolume) // Check the results if tc.expectedNFS != nil { if structuredVolume.nfs == nil { t.Errorf("Expected a non-nil NFS volume source") } else if *tc.expectedNFS != *structuredVolume.nfs { t.Errorf("NFS volume source does not match expected value") } } if tc.expectedCSI != nil { if structuredVolume.csi == nil { t.Errorf("Expected a non-nil CSI volume source") } else if tc.expectedCSI.Driver != structuredVolume.csi.Driver { t.Errorf("CSI volume source does not match expected value") } // Check volumeAttributes if len(tc.expectedCSI.VolumeAttributes) != len(structuredVolume.csi.VolumeAttributes) { t.Errorf("CSI volume attributes does not match expected value") } else { for k, v := range tc.expectedCSI.VolumeAttributes { if structuredVolume.csi.VolumeAttributes[k] != v { t.Errorf("CSI volume attributes does not match expected value") } } } } }) } } func TestParsePV(t *testing.T) { // Mock data nfsVolume := corev1api.PersistentVolume{} nfsVolume.Spec.Capacity = corev1api.ResourceList{corev1api.ResourceStorage: resource.MustParse("1Gi")} nfsVolume.Spec.NFS = &corev1api.NFSVolumeSource{Server: "nfs.example.com", Path: "/exports/data"} csiVolume := corev1api.PersistentVolume{} csiVolume.Spec.Capacity = corev1api.ResourceList{corev1api.ResourceStorage: resource.MustParse("2Gi")} csiVolume.Spec.CSI = &corev1api.CSIPersistentVolumeSource{Driver: "csi.example.com", VolumeAttributes: map[string]string{"protocol": "nfs"}} emptyVolume := corev1api.PersistentVolume{} // Test cases testCases := []struct { name string inputVolume *corev1api.PersistentVolume expectedNFS *nFSVolumeSource expectedCSI *csiVolumeSource }{ { name: "NFS volume", inputVolume: &nfsVolume, expectedNFS: &nFSVolumeSource{Server: "nfs.example.com", Path: "/exports/data"}, expectedCSI: nil, }, { name: "CSI volume", inputVolume: &csiVolume, expectedNFS: nil, expectedCSI: &csiVolumeSource{Driver: "csi.example.com", VolumeAttributes: map[string]string{"protocol": "nfs"}}, }, { name: "Empty volume", inputVolume: &emptyVolume, expectedNFS: nil, expectedCSI: nil, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { // Call the function structuredVolume := &structuredVolume{} structuredVolume.parsePV(tc.inputVolume) // Check the results if structuredVolume.capacity != *tc.inputVolume.Spec.Capacity.Storage() { t.Errorf("capacity does not match expected value") } if structuredVolume.storageClass != tc.inputVolume.Spec.StorageClassName { t.Errorf("Storage class does not match expected value") } if tc.expectedNFS != nil { if structuredVolume.nfs == nil { t.Errorf("Expected a non-nil NFS volume source") } else if *tc.expectedNFS != *structuredVolume.nfs { t.Errorf("NFS volume source does not match expected value") } } if tc.expectedCSI != nil { if structuredVolume.csi == nil { t.Errorf("Expected a non-nil CSI volume source") } else if tc.expectedCSI.Driver != structuredVolume.csi.Driver { t.Errorf("CSI volume source does not match expected value") } // Check volumeAttributes if len(tc.expectedCSI.VolumeAttributes) != len(structuredVolume.csi.VolumeAttributes) { t.Errorf("CSI volume attributes does not match expected value") } else { for k, v := range tc.expectedCSI.VolumeAttributes { if structuredVolume.csi.VolumeAttributes[k] != v { t.Errorf("CSI volume attributes does not match expected value") } } } } }) } } ================================================ FILE: internal/resourcepolicies/volume_resources_validator.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package resourcepolicies import ( "fmt" "io" "github.com/pkg/errors" "gopkg.in/yaml.v3" ) const currentSupportDataVersion = "v1" type csiVolumeSource struct { Driver string `yaml:"driver,omitempty"` // CSI volume attributes VolumeAttributes map[string]string `yaml:"volumeAttributes,omitempty"` } type nFSVolumeSource struct { // Server is the hostname or IP address of the NFS server Server string `yaml:"server,omitempty"` // Path is the exported NFS share Path string `yaml:"path,omitempty"` } // volumeConditions defined the current format of conditions we parsed type volumeConditions struct { Capacity string `yaml:"capacity,omitempty"` StorageClass []string `yaml:"storageClass,omitempty"` NFS *nFSVolumeSource `yaml:"nfs,omitempty"` CSI *csiVolumeSource `yaml:"csi,omitempty"` VolumeTypes []SupportedVolume `yaml:"volumeTypes,omitempty"` PVCLabels map[string]string `yaml:"pvcLabels,omitempty"` PVCPhase []string `yaml:"pvcPhase,omitempty"` } func (c *capacityCondition) validate() error { // [0, a] // [a, b] // [b, 0] // ==> low <= upper or upper is zero if (c.capacity.upper.Cmp(c.capacity.lower) >= 0) || (!c.capacity.lower.IsZero() && c.capacity.upper.IsZero()) { return nil } return errors.Errorf("illegal values for capacity %v", c.capacity) } func (s *storageClassCondition) validate() error { // validate by yamlv3 return nil } func (c *nfsCondition) validate() error { // validate by yamlv3 return nil } func (c *csiCondition) validate() error { if c != nil && c.csi != nil && c.csi.Driver == "" && c.csi.VolumeAttributes != nil { return errors.New("csi driver should not be empty when filtering by volume attributes") } return nil } // decodeStruct restric validate the keys in decoded mappings to exist as fields in the struct being decoded into func decodeStruct(r io.Reader, s any) error { dec := yaml.NewDecoder(r) dec.KnownFields(true) return dec.Decode(s) } // validate check action format func (a *Action) validate() error { // validate Type valid := false if a.Type == Skip || a.Type == Snapshot || a.Type == FSBackup { valid = true } if !valid { return fmt.Errorf("invalid action type %s", a.Type) } // TODO validate parameters return nil } ================================================ FILE: internal/resourcepolicies/volume_resources_validator_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package resourcepolicies import ( "testing" "k8s.io/apimachinery/pkg/api/resource" ) func TestCapacityConditionValidate(t *testing.T) { testCases := []struct { name string capacity *capacity wantErr bool }{ { name: "lower and upper are both zero", capacity: &capacity{lower: *resource.NewQuantity(0, resource.DecimalSI), upper: *resource.NewQuantity(0, resource.DecimalSI)}, wantErr: false, }, { name: "lower is zero and upper is greater than zero", capacity: &capacity{lower: *resource.NewQuantity(0, resource.DecimalSI), upper: *resource.NewQuantity(100, resource.DecimalSI)}, wantErr: false, }, { name: "lower is greater than upper", capacity: &capacity{lower: *resource.NewQuantity(100, resource.DecimalSI), upper: *resource.NewQuantity(50, resource.DecimalSI)}, wantErr: true, }, { name: "lower and upper are equal", capacity: &capacity{lower: *resource.NewQuantity(100, resource.DecimalSI), upper: *resource.NewQuantity(100, resource.DecimalSI)}, wantErr: false, }, { name: "lower is greater than zero and upper is zero", capacity: &capacity{lower: *resource.NewQuantity(100, resource.DecimalSI), upper: *resource.NewQuantity(0, resource.DecimalSI)}, wantErr: false, }, { name: "lower and upper are both not zero and lower is less than upper", capacity: &capacity{lower: *resource.NewQuantity(100, resource.DecimalSI), upper: *resource.NewQuantity(200, resource.DecimalSI)}, wantErr: false, }, { name: "lower and upper are both not zero and lower is equal to upper", capacity: &capacity{lower: *resource.NewQuantity(100, resource.DecimalSI), upper: *resource.NewQuantity(100, resource.DecimalSI)}, wantErr: false, }, { name: "lower and upper are both not zero and lower is greater than upper", capacity: &capacity{lower: *resource.NewQuantity(200, resource.DecimalSI), upper: *resource.NewQuantity(100, resource.DecimalSI)}, wantErr: true, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { c := &capacityCondition{capacity: *tc.capacity} err := c.validate() if (err != nil) != tc.wantErr { t.Fatalf("Expected error %v, but got error %v", tc.wantErr, err) } }) } } func TestValidate(t *testing.T) { testCases := []struct { name string res *ResourcePolicies wantErr bool }{ { name: "unknown key in yaml", res: &ResourcePolicies{ Version: "v1", VolumePolicies: []VolumePolicy{ { Action: Action{Type: "skip"}, Conditions: map[string]any{ "capacity": "0,10Gi", "unknown": "", "storageClass": []string{"gp2", "ebs-sc"}, "csi": any( map[string]any{ "driver": "aws.efs.csi.driver", }), }, }, }, }, wantErr: true, }, { name: "error format of capacity", res: &ResourcePolicies{ Version: "v1", VolumePolicies: []VolumePolicy{ { Action: Action{Type: "skip"}, Conditions: map[string]any{ "capacity": "10Gi", "storageClass": []string{"gp2", "ebs-sc"}, "csi": any( map[string]any{ "driver": "aws.efs.csi.driver", }), }, }, }, }, wantErr: true, }, { name: "error format of storageClass", res: &ResourcePolicies{ Version: "v1", VolumePolicies: []VolumePolicy{ { Action: Action{Type: "skip"}, Conditions: map[string]any{ "capacity": "0,10Gi", "storageClass": "ebs-sc", "csi": any( map[string]any{ "driver": "aws.efs.csi.driver", }), }, }, }, }, wantErr: true, }, { name: "error format of csi", res: &ResourcePolicies{ Version: "v1", VolumePolicies: []VolumePolicy{ { Action: Action{Type: "skip"}, Conditions: map[string]any{ "capacity": "0,10Gi", "storageClass": []string{"gp2", "ebs-sc"}, "csi": "aws.efs.csi.driver", }, }, }, }, wantErr: true, }, { name: "error format of csi driver", res: &ResourcePolicies{ Version: "v1", VolumePolicies: []VolumePolicy{ { Action: Action{Type: "skip"}, Conditions: map[string]any{ "capacity": "0,10Gi", "storageClass": []string{"gp2", "ebs-sc"}, "csi": any( map[string]any{ "driver": []string{"aws.efs.csi.driver"}, }), }, }, }, }, wantErr: true, }, { name: "error format of csi driver volumeAttributes", res: &ResourcePolicies{ Version: "v1", VolumePolicies: []VolumePolicy{ { Action: Action{Type: "skip"}, Conditions: map[string]any{ "capacity": "0,10Gi", "storageClass": []string{"gp2", "ebs-sc"}, "csi": any( map[string]any{ "driver": "aws.efs.csi.driver", "volumeAttributes": "test", }), }, }, }, }, wantErr: true, }, { name: "unsupported version", res: &ResourcePolicies{ Version: "v2", VolumePolicies: []VolumePolicy{ { Action: Action{Type: "skip"}, Conditions: map[string]any{ "capacity": "0,10Gi", "csi": any( map[string]any{ "driver": "aws.efs.csi.driver", }), }, }, }, }, wantErr: true, }, { name: "unsupported action", res: &ResourcePolicies{ Version: "v1", VolumePolicies: []VolumePolicy{ { Action: Action{Type: "unsupported"}, Conditions: map[string]any{ "capacity": "0,10Gi", "csi": any( map[string]any{ "driver": "aws.efs.csi.driver", }), }, }, }, }, wantErr: true, }, { name: "error format of nfs", res: &ResourcePolicies{ Version: "v1", VolumePolicies: []VolumePolicy{ { Action: Action{Type: "skip"}, Conditions: map[string]any{ "capacity": "0,10Gi", "storageClass": []string{"gp2", "ebs-sc"}, "nfs": "aws.efs.csi.driver", }, }, }, }, wantErr: true, }, { name: "supported format volume policies only csi driver", res: &ResourcePolicies{ Version: "v1", VolumePolicies: []VolumePolicy{ { Action: Action{Type: "skip"}, Conditions: map[string]any{ "csi": any( map[string]any{ "driver": "aws.efs.csi.driver", }), }, }, }, }, wantErr: false, }, { name: "unsupported format volume policies only csi volumeattributes", res: &ResourcePolicies{ Version: "v1", VolumePolicies: []VolumePolicy{ { Action: Action{Type: "skip"}, Conditions: map[string]any{ "csi": any( map[string]any{ "volumeAttributes": map[string]string{ "key1": "value1", }, }), }, }, }, }, wantErr: true, }, { name: "supported format volume policies with csi driver and volumeattributes", res: &ResourcePolicies{ Version: "v1", VolumePolicies: []VolumePolicy{ { Action: Action{Type: "skip"}, Conditions: map[string]any{ "csi": any( map[string]any{ "driver": "aws.efs.csi.driver", "volumeAttributes": map[string]string{ "key1": "value1", }, }), }, }, }, }, wantErr: false, }, { name: "supported format volume policies", res: &ResourcePolicies{ Version: "v1", VolumePolicies: []VolumePolicy{ { Action: Action{Type: "skip"}, Conditions: map[string]any{ "capacity": "0,10Gi", "storageClass": []string{"gp2", "ebs-sc"}, "csi": any( map[string]any{ "driver": "aws.efs.csi.driver", }), "nfs": any( map[string]any{ "server": "192.168.20.90", "path": "/mnt/data/", }), }, }, }, }, wantErr: false, }, { name: "supported format volume policies, action type snapshot", res: &ResourcePolicies{ Version: "v1", VolumePolicies: []VolumePolicy{ { Action: Action{Type: "snapshot"}, Conditions: map[string]any{ "capacity": "0,10Gi", "storageClass": []string{"gp2", "ebs-sc"}, "csi": any( map[string]any{ "driver": "aws.efs.csi.driver", }), "nfs": any( map[string]any{ "server": "192.168.20.90", "path": "/mnt/data/", }), }, }, }, }, wantErr: false, }, { name: "supported format volume policies, action type fs-backup", res: &ResourcePolicies{ Version: "v1", VolumePolicies: []VolumePolicy{ { Action: Action{Type: "fs-backup"}, Conditions: map[string]any{ "capacity": "0,10Gi", "storageClass": []string{"gp2", "ebs-sc"}, "csi": any( map[string]any{ "driver": "aws.efs.csi.driver", }), "nfs": any( map[string]any{ "server": "192.168.20.90", "path": "/mnt/data/", }), }, }, }, }, wantErr: false, }, { name: "supported format volume policies, action type fs-backup and snapshot", res: &ResourcePolicies{ Version: "v1", VolumePolicies: []VolumePolicy{ { Action: Action{Type: Snapshot}, Conditions: map[string]any{ "storageClass": []string{"gp2"}, }, }, { Action: Action{Type: FSBackup}, Conditions: map[string]any{ "nfs": any( map[string]any{ "server": "192.168.20.90", "path": "/mnt/data/", }), }, }, }, }, wantErr: false, }, { name: "supported format volume policies with pvcLabels (valid map)", res: &ResourcePolicies{ Version: "v1", VolumePolicies: []VolumePolicy{ { Action: Action{Type: "skip"}, Conditions: map[string]any{ "pvcLabels": map[string]string{ "environment": "production", "app": "database", }, }, }, }, }, wantErr: false, }, { name: "error format volume policies with pvcLabels (not a map)", res: &ResourcePolicies{ Version: "v1", VolumePolicies: []VolumePolicy{ { Action: Action{Type: "skip"}, Conditions: map[string]any{ "pvcLabels": "production", }, }, }, }, wantErr: true, }, { name: " '*' in the filters of include exclude policy - 1", res: &ResourcePolicies{ Version: "v1", VolumePolicies: []VolumePolicy{ { Action: Action{Type: "skip"}, Conditions: map[string]any{ "pvcLabels": map[string]string{ "environment": "production", "app": "database", }, }, }, }, IncludeExcludePolicy: &IncludeExcludePolicy{ IncludedClusterScopedResources: []string{"*"}, ExcludedClusterScopedResources: []string{"crds"}, IncludedNamespaceScopedResources: []string{"pods"}, ExcludedNamespaceScopedResources: []string{"secrets"}, }, }, wantErr: true, }, { name: " '*' in the filters of include exclude policy - 2", res: &ResourcePolicies{ Version: "v1", VolumePolicies: []VolumePolicy{ { Action: Action{Type: "skip"}, Conditions: map[string]any{ "pvcLabels": map[string]string{ "environment": "production", "app": "database", }, }, }, }, IncludeExcludePolicy: &IncludeExcludePolicy{ IncludedClusterScopedResources: []string{"persistentvolumes"}, ExcludedClusterScopedResources: []string{"crds"}, IncludedNamespaceScopedResources: []string{"pods"}, ExcludedNamespaceScopedResources: []string{"*"}, }, }, wantErr: true, }, { name: " dup item in both the include and exclude filters of include exclude policy", res: &ResourcePolicies{ Version: "v1", VolumePolicies: []VolumePolicy{ { Action: Action{Type: "skip"}, Conditions: map[string]any{ "pvcLabels": map[string]string{ "environment": "production", "app": "database", }, }, }, }, IncludeExcludePolicy: &IncludeExcludePolicy{ IncludedClusterScopedResources: []string{"persistentvolumes"}, ExcludedClusterScopedResources: []string{"crds"}, IncludedNamespaceScopedResources: []string{"pods", "configmaps"}, ExcludedNamespaceScopedResources: []string{"secrets", "pods"}, }, }, wantErr: true, }, { name: " valid volume policies and valid include/exclude policy", res: &ResourcePolicies{ Version: "v1", VolumePolicies: []VolumePolicy{ { Action: Action{Type: "skip"}, Conditions: map[string]any{ "pvcLabels": map[string]string{ "environment": "production", "app": "database", }, }, }, }, IncludeExcludePolicy: &IncludeExcludePolicy{ IncludedClusterScopedResources: []string{"persistentvolumes"}, ExcludedClusterScopedResources: []string{"crds"}, IncludedNamespaceScopedResources: []string{"pods", "configmaps"}, ExcludedNamespaceScopedResources: []string{"secrets"}, }, }, wantErr: false, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { policies := &Policies{} err1 := policies.BuildPolicy(tc.res) err2 := policies.Validate() if tc.wantErr { if err1 == nil && err2 == nil { t.Fatalf("Expected error %v, but not get error", tc.wantErr) } } else { if err1 != nil || err2 != nil { t.Fatalf("Expected error %v, but got error %v %v", tc.wantErr, err1, err2) } } }) } } ================================================ FILE: internal/resourcepolicies/volume_types_conditions.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package resourcepolicies import ( corev1api "k8s.io/api/core/v1" ) type volumeTypeCondition struct { volumeTypes []SupportedVolume } type SupportedVolume string const ( AWSAzureDisk SupportedVolume = "awsAzureDisk" AWSElasticBlockStore SupportedVolume = "awsElasticBlockStore" AzureDisk SupportedVolume = "azureDisk" AzureFile SupportedVolume = "azureFile" Cinder SupportedVolume = "cinder" CephFS SupportedVolume = "cephfs" ConfigMap SupportedVolume = "configMap" CSI SupportedVolume = "csi" DownwardAPI SupportedVolume = "downwardAPI" EmptyDir SupportedVolume = "emptyDir" Ephemeral SupportedVolume = "ephemeral" FC SupportedVolume = "fc" Flocker SupportedVolume = "flocker" FlexVolume SupportedVolume = "flexVolume" GitRepo SupportedVolume = "gitRepo" Glusterfs SupportedVolume = "glusterfs" GCEPersistentDisk SupportedVolume = "gcePersistentDisk" HostPath SupportedVolume = "hostPath" ISCSI SupportedVolume = "iscsi" Local SupportedVolume = "local" NFS SupportedVolume = "nfs" PhotonPersistentDisk SupportedVolume = "photonPersistentDisk" PortworxVolume SupportedVolume = "portworxVolume" Projected SupportedVolume = "projected" Quobyte SupportedVolume = "quobyte" RBD SupportedVolume = "rbd" ScaleIO SupportedVolume = "scaleIO" Secret SupportedVolume = "secret" StorageOS SupportedVolume = "storageOS" VsphereVolume SupportedVolume = "vsphereVolume" ) func (v *volumeTypeCondition) match(s *structuredVolume) bool { if len(v.volumeTypes) == 0 { return true } for _, vt := range v.volumeTypes { if vt == s.volumeType { return true } } return false } func (v *volumeTypeCondition) validate() error { // validate by yamlv3 return nil } func getVolumeTypeFromPV(pv *corev1api.PersistentVolume) SupportedVolume { if pv == nil { return "" } if pv.Spec.AWSElasticBlockStore != nil { return AWSElasticBlockStore } if pv.Spec.AzureDisk != nil { return AzureDisk } if pv.Spec.AzureFile != nil { return AzureFile } if pv.Spec.CephFS != nil { return CephFS } if pv.Spec.Cinder != nil { return Cinder } if pv.Spec.CSI != nil { return CSI } if pv.Spec.FC != nil { return FC } if pv.Spec.Flocker != nil { return Flocker } if pv.Spec.FlexVolume != nil { return FlexVolume } if pv.Spec.GCEPersistentDisk != nil { return GCEPersistentDisk } if pv.Spec.Glusterfs != nil { return Glusterfs } if pv.Spec.HostPath != nil { return HostPath } if pv.Spec.ISCSI != nil { return ISCSI } if pv.Spec.Local != nil { return Local } if pv.Spec.NFS != nil { return NFS } if pv.Spec.PhotonPersistentDisk != nil { return PhotonPersistentDisk } if pv.Spec.PortworxVolume != nil { return PortworxVolume } if pv.Spec.Quobyte != nil { return Quobyte } if pv.Spec.RBD != nil { return RBD } if pv.Spec.ScaleIO != nil { return ScaleIO } if pv.Spec.StorageOS != nil { return StorageOS } if pv.Spec.VsphereVolume != nil { return VsphereVolume } return "" } func getVolumeTypeFromVolume(vol *corev1api.Volume) SupportedVolume { if vol == nil { return "" } if vol.AWSElasticBlockStore != nil { return AWSElasticBlockStore } if vol.AzureDisk != nil { return AzureDisk } if vol.AzureFile != nil { return AzureFile } if vol.CephFS != nil { return CephFS } if vol.Cinder != nil { return Cinder } if vol.CSI != nil { return CSI } if vol.FC != nil { return FC } if vol.Flocker != nil { return Flocker } if vol.FlexVolume != nil { return FlexVolume } if vol.GCEPersistentDisk != nil { return GCEPersistentDisk } if vol.GitRepo != nil { return GitRepo } if vol.Glusterfs != nil { return Glusterfs } if vol.ISCSI != nil { return ISCSI } if vol.NFS != nil { return NFS } if vol.Secret != nil { return Secret } if vol.RBD != nil { return RBD } if vol.DownwardAPI != nil { return DownwardAPI } if vol.ConfigMap != nil { return ConfigMap } if vol.Projected != nil { return Projected } if vol.Ephemeral != nil { return Ephemeral } if vol.FC != nil { return FC } if vol.PhotonPersistentDisk != nil { return PhotonPersistentDisk } if vol.PortworxVolume != nil { return PortworxVolume } if vol.Quobyte != nil { return Quobyte } if vol.ScaleIO != nil { return ScaleIO } if vol.StorageOS != nil { return StorageOS } if vol.VsphereVolume != nil { return VsphereVolume } if vol.HostPath != nil { return HostPath } if vol.EmptyDir != nil { return EmptyDir } return "" } ================================================ FILE: internal/resourcepolicies/volume_types_conditions_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package resourcepolicies import ( "testing" corev1api "k8s.io/api/core/v1" ) func TestGetVolumeTypeFromPV(t *testing.T) { testCases := []struct { name string inputPV *corev1api.PersistentVolume expected SupportedVolume }{ { name: "nil PersistentVolume", inputPV: nil, expected: "", }, { name: "Test GCEPersistentDisk", inputPV: &corev1api.PersistentVolume{ Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ GCEPersistentDisk: &corev1api.GCEPersistentDiskVolumeSource{}, }, }, }, expected: GCEPersistentDisk, }, { name: "Test AWSElasticBlockStore", inputPV: &corev1api.PersistentVolume{ Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ AWSElasticBlockStore: &corev1api.AWSElasticBlockStoreVolumeSource{}, }, }, }, expected: AWSElasticBlockStore, }, { name: "Test HostPath", inputPV: &corev1api.PersistentVolume{ Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ HostPath: &corev1api.HostPathVolumeSource{}, }, }, }, expected: HostPath, }, { name: "Test Glusterfs", inputPV: &corev1api.PersistentVolume{ Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ Glusterfs: &corev1api.GlusterfsPersistentVolumeSource{}, }, }, }, expected: Glusterfs, }, { name: "Test NFS", inputPV: &corev1api.PersistentVolume{ Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ NFS: &corev1api.NFSVolumeSource{}, }, }, }, expected: NFS, }, { name: "Test RBD", inputPV: &corev1api.PersistentVolume{ Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ RBD: &corev1api.RBDPersistentVolumeSource{}, }, }, }, expected: RBD, }, { name: "Test ISCSI", inputPV: &corev1api.PersistentVolume{ Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ ISCSI: &corev1api.ISCSIPersistentVolumeSource{}, }, }, }, expected: ISCSI, }, { name: "Test Cinder", inputPV: &corev1api.PersistentVolume{ Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ Cinder: &corev1api.CinderPersistentVolumeSource{}, }, }, }, expected: Cinder, }, { name: "Test CephFS", inputPV: &corev1api.PersistentVolume{ Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ CephFS: &corev1api.CephFSPersistentVolumeSource{}, }, }, }, expected: CephFS, }, { name: "Test FC", inputPV: &corev1api.PersistentVolume{ Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ FC: &corev1api.FCVolumeSource{}, }, }, }, expected: FC, }, { name: "Test Flocker", inputPV: &corev1api.PersistentVolume{ Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ Flocker: &corev1api.FlockerVolumeSource{}, }, }, }, expected: Flocker, }, { name: "Test FlexVolume", inputPV: &corev1api.PersistentVolume{ Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ FlexVolume: &corev1api.FlexPersistentVolumeSource{}, }, }, }, expected: FlexVolume, }, { name: "Test AzureFile", inputPV: &corev1api.PersistentVolume{ Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ AzureFile: &corev1api.AzureFilePersistentVolumeSource{}, }, }, }, expected: AzureFile, }, { name: "Test VsphereVolume", inputPV: &corev1api.PersistentVolume{ Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ VsphereVolume: &corev1api.VsphereVirtualDiskVolumeSource{}, }, }, }, expected: VsphereVolume, }, { name: "Test Quobyte", inputPV: &corev1api.PersistentVolume{ Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ Quobyte: &corev1api.QuobyteVolumeSource{}, }, }, }, expected: Quobyte, }, { name: "Test AzureDisk", inputPV: &corev1api.PersistentVolume{ Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ AzureDisk: &corev1api.AzureDiskVolumeSource{}, }, }, }, expected: AzureDisk, }, { name: "Test PhotonPersistentDisk", inputPV: &corev1api.PersistentVolume{ Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ PhotonPersistentDisk: &corev1api.PhotonPersistentDiskVolumeSource{}, }, }, }, expected: PhotonPersistentDisk, }, { name: "Test PortworxVolume", inputPV: &corev1api.PersistentVolume{ Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ PortworxVolume: &corev1api.PortworxVolumeSource{}, }, }, }, expected: PortworxVolume, }, { name: "Test ScaleIO", inputPV: &corev1api.PersistentVolume{ Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ ScaleIO: &corev1api.ScaleIOPersistentVolumeSource{}, }, }, }, expected: ScaleIO, }, { name: "Test Local", inputPV: &corev1api.PersistentVolume{ Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ Local: &corev1api.LocalVolumeSource{}, }, }, }, expected: Local, }, { name: "Test StorageOS", inputPV: &corev1api.PersistentVolume{ Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ StorageOS: &corev1api.StorageOSPersistentVolumeSource{}, }, }, }, expected: StorageOS, }, { name: "Test CSI", inputPV: &corev1api.PersistentVolume{ Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ CSI: &corev1api.CSIPersistentVolumeSource{}, }, }, }, expected: CSI, }, { name: "Test Unknown Source", inputPV: &corev1api.PersistentVolume{ Spec: corev1api.PersistentVolumeSpec{}, }, expected: "", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { result := getVolumeTypeFromPV(tc.inputPV) if result != tc.expected { t.Errorf("Expected %s, but got %s", tc.expected, result) } }) } } func TestGetVolumeTypeFromVolume(t *testing.T) { testCases := []struct { name string inputVol *corev1api.Volume expected SupportedVolume }{ { name: "nil Volume", inputVol: nil, expected: "", }, { name: "Test Unknown Source", inputVol: &corev1api.Volume{ VolumeSource: corev1api.VolumeSource{}, }, expected: "", }, { name: "Test HostPath", inputVol: &corev1api.Volume{ VolumeSource: corev1api.VolumeSource{ HostPath: &corev1api.HostPathVolumeSource{}, }, }, expected: HostPath, }, { name: "Test EmptyDir", inputVol: &corev1api.Volume{ VolumeSource: corev1api.VolumeSource{ EmptyDir: &corev1api.EmptyDirVolumeSource{}, }, }, expected: EmptyDir, }, { name: "Test GCEPersistentDisk", inputVol: &corev1api.Volume{ VolumeSource: corev1api.VolumeSource{ GCEPersistentDisk: &corev1api.GCEPersistentDiskVolumeSource{}, }, }, expected: GCEPersistentDisk, }, { name: "Test AWSElasticBlockStore", inputVol: &corev1api.Volume{ VolumeSource: corev1api.VolumeSource{ AWSElasticBlockStore: &corev1api.AWSElasticBlockStoreVolumeSource{}, }, }, expected: AWSElasticBlockStore, }, { name: "Test GitRepo", inputVol: &corev1api.Volume{ VolumeSource: corev1api.VolumeSource{ GitRepo: &corev1api.GitRepoVolumeSource{}, }, }, expected: GitRepo, }, { name: "Test Secret", inputVol: &corev1api.Volume{ VolumeSource: corev1api.VolumeSource{ Secret: &corev1api.SecretVolumeSource{}, }, }, expected: Secret, }, { name: "Test NFS", inputVol: &corev1api.Volume{ VolumeSource: corev1api.VolumeSource{ NFS: &corev1api.NFSVolumeSource{}, }, }, expected: NFS, }, { name: "Test ISCSI", inputVol: &corev1api.Volume{ VolumeSource: corev1api.VolumeSource{ ISCSI: &corev1api.ISCSIVolumeSource{}, }, }, expected: ISCSI, }, { name: "Test Glusterfs", inputVol: &corev1api.Volume{ VolumeSource: corev1api.VolumeSource{ Glusterfs: &corev1api.GlusterfsVolumeSource{}, }, }, expected: Glusterfs, }, { name: "Test RBD", inputVol: &corev1api.Volume{ VolumeSource: corev1api.VolumeSource{ RBD: &corev1api.RBDVolumeSource{}, }, }, expected: RBD, }, { name: "Test FlexVolume", inputVol: &corev1api.Volume{ VolumeSource: corev1api.VolumeSource{ FlexVolume: &corev1api.FlexVolumeSource{}, }, }, expected: FlexVolume, }, { name: "Test Cinder", inputVol: &corev1api.Volume{ VolumeSource: corev1api.VolumeSource{ Cinder: &corev1api.CinderVolumeSource{}, }, }, expected: Cinder, }, { name: "Test CephFS", inputVol: &corev1api.Volume{ VolumeSource: corev1api.VolumeSource{ CephFS: &corev1api.CephFSVolumeSource{}, }, }, expected: CephFS, }, { name: "Test Flocker", inputVol: &corev1api.Volume{ VolumeSource: corev1api.VolumeSource{ Flocker: &corev1api.FlockerVolumeSource{}, }, }, expected: Flocker, }, { name: "Test DownwardAPI", inputVol: &corev1api.Volume{ VolumeSource: corev1api.VolumeSource{ DownwardAPI: &corev1api.DownwardAPIVolumeSource{}, }, }, expected: DownwardAPI, }, { name: "Test FC", inputVol: &corev1api.Volume{ VolumeSource: corev1api.VolumeSource{ FC: &corev1api.FCVolumeSource{}, }, }, expected: FC, }, { name: "Test AzureFile", inputVol: &corev1api.Volume{ VolumeSource: corev1api.VolumeSource{ AzureFile: &corev1api.AzureFileVolumeSource{}, }, }, expected: AzureFile, }, { name: "Test ConfigMap", inputVol: &corev1api.Volume{ VolumeSource: corev1api.VolumeSource{ ConfigMap: &corev1api.ConfigMapVolumeSource{}, }, }, expected: ConfigMap, }, { name: "Test VsphereVolume", inputVol: &corev1api.Volume{ VolumeSource: corev1api.VolumeSource{ VsphereVolume: &corev1api.VsphereVirtualDiskVolumeSource{}, }, }, expected: VsphereVolume, }, { name: "Test Quobyte", inputVol: &corev1api.Volume{ VolumeSource: corev1api.VolumeSource{ Quobyte: &corev1api.QuobyteVolumeSource{}, }, }, expected: Quobyte, }, { name: "Test AzureDisk", inputVol: &corev1api.Volume{ VolumeSource: corev1api.VolumeSource{ AzureDisk: &corev1api.AzureDiskVolumeSource{}, }, }, expected: AzureDisk, }, { name: "Test PhotonPersistentDisk", inputVol: &corev1api.Volume{ VolumeSource: corev1api.VolumeSource{ PhotonPersistentDisk: &corev1api.PhotonPersistentDiskVolumeSource{}, }, }, expected: PhotonPersistentDisk, }, { name: "Test Projected", inputVol: &corev1api.Volume{ VolumeSource: corev1api.VolumeSource{ Projected: &corev1api.ProjectedVolumeSource{}, }, }, expected: Projected, }, { name: "Test PortworxVolume", inputVol: &corev1api.Volume{ VolumeSource: corev1api.VolumeSource{ PortworxVolume: &corev1api.PortworxVolumeSource{}, }, }, expected: PortworxVolume, }, { name: "Test ScaleIO", inputVol: &corev1api.Volume{ VolumeSource: corev1api.VolumeSource{ ScaleIO: &corev1api.ScaleIOVolumeSource{}, }, }, expected: ScaleIO, }, { name: "Test StorageOS", inputVol: &corev1api.Volume{ VolumeSource: corev1api.VolumeSource{ StorageOS: &corev1api.StorageOSVolumeSource{}, }, }, expected: StorageOS, }, { name: "Test CSI", inputVol: &corev1api.Volume{ VolumeSource: corev1api.VolumeSource{ CSI: &corev1api.CSIVolumeSource{}, }, }, expected: CSI, }, { name: "Test Ephemeral", inputVol: &corev1api.Volume{ VolumeSource: corev1api.VolumeSource{ Ephemeral: &corev1api.EphemeralVolumeSource{}, }, }, expected: Ephemeral, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { result := getVolumeTypeFromVolume(tc.inputVol) if result != tc.expected { t.Errorf("Expected %s, but got %s", tc.expected, result) } }) } } ================================================ FILE: internal/restartabletest/restartable_delegate.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package restartabletest import ( "reflect" "testing" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt/process" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" ) type MockRestartableProcess struct { mock.Mock } func (rp *MockRestartableProcess) AddReinitializer(key process.KindAndName, r process.Reinitializer) { rp.Called(key, r) } func (rp *MockRestartableProcess) Reset() error { args := rp.Called() return args.Error(0) } func (rp *MockRestartableProcess) ResetIfNeeded() error { args := rp.Called() return args.Error(0) } func (rp *MockRestartableProcess) GetByKindAndName(key process.KindAndName) (any, error) { args := rp.Called(key) return args.Get(0), args.Error(1) } func (rp *MockRestartableProcess) Stop() { rp.Called() } type RestartableDelegateTest struct { Function string Inputs []any ExpectedErrorOutputs []any ExpectedDelegateOutputs []any } type Mockable interface { Test(t mock.TestingT) On(method string, args ...any) *mock.Call AssertExpectations(t mock.TestingT) bool } func RunRestartableDelegateTests( t *testing.T, kind common.PluginKind, newRestartable func(key process.KindAndName, p process.RestartableProcess) any, newMock func() Mockable, tests ...RestartableDelegateTest, ) { t.Helper() for _, tc := range tests { t.Run(tc.Function, func(t *testing.T) { p := new(MockRestartableProcess) p.Test(t) defer p.AssertExpectations(t) // getDelegate error p.On("ResetIfNeeded").Return(errors.Errorf("reset error")).Once() name := "delegateName" key := process.KindAndName{Kind: kind, Name: name} r := newRestartable(key, p) // Get the method we're going to call using reflection method := reflect.ValueOf(r).MethodByName(tc.Function) require.NotEmpty(t, method) // Convert the test case inputs ([]any) to []reflect.Value var inputValues []reflect.Value for i := range tc.Inputs { inputValues = append(inputValues, reflect.ValueOf(tc.Inputs[i])) } // Invoke the method being tested actual := method.Call(inputValues) // This Function asserts that the actual outputs match the expected outputs checkOutputs := func(expected []any, actual []reflect.Value) { require.Len(t, actual, len(expected)) for i := range actual { // Get the underlying value from the reflect.Value a := actual[i].Interface() // Check if it's an error actualErr, actualErrOk := a.(error) // Check if the expected output element is an error expectedErr, expectedErrOk := expected[i].(error) // If both are errors, use EqualError if actualErrOk && expectedErrOk { require.EqualError(t, actualErr, expectedErr.Error()) continue } // If Function returns nil as struct return type, we cannot just // compare the interface to nil as its type will not be nil, // only the value will be if expected[i] == nil && reflect.ValueOf(a).Kind() == reflect.Ptr { assert.True(t, reflect.ValueOf(a).IsNil()) continue } // Otherwise, use plain Equal assert.Equal(t, expected[i], a) } } // Make sure we get what we expected when getDelegate returned an error checkOutputs(tc.ExpectedErrorOutputs, actual) // Invoke delegate, make sure all returned values are passed through p.On("ResetIfNeeded").Return(nil) delegate := newMock() delegate.Test(t) defer delegate.AssertExpectations(t) p.On("GetByKindAndName", key).Return(delegate, nil) // Set up the mocked method in the delegate delegate.On(tc.Function, tc.Inputs...).Return(tc.ExpectedDelegateOutputs...) // Invoke the method being tested actual = method.Call(inputValues) // Make sure we get what we expected when invoking the delegate checkOutputs(tc.ExpectedDelegateOutputs, actual) }) } } ================================================ FILE: internal/storage/storagelocation.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package storage import ( "context" "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) // DefaultBackupLocationInfo holds server default backup storage location information type DefaultBackupLocationInfo struct { // StorageLocation is the name of the backup storage location designated as the default on the server side. // Deprecated TODO(2.0) StorageLocation string // ServerValidationFrequency is the server default validation frequency for all backup storage locations ServerValidationFrequency time.Duration } // IsReadyToValidate calculates if a given backup storage location is ready to be validated. // // Rules: // Users can choose a validation frequency per location. This will override the server's default value // To skip/stop validation, set the frequency to zero // This will always return "true" for the first attempt at validating a location, regardless of its validation frequency setting // Otherwise, it returns "ready" only when NOW is equal to or after the next validation time // (next validation time: last validation time + validation frequency) func IsReadyToValidate(bslValidationFrequency *metav1.Duration, lastValidationTime *metav1.Time, serverValidationFrequency time.Duration, log logrus.FieldLogger) bool { validationFrequency := serverValidationFrequency // If the bsl validation frequency is not specifically set, skip this block and continue, using the server's default if bslValidationFrequency != nil { validationFrequency = bslValidationFrequency.Duration } if validationFrequency < 0 { log.Debugf("Validation period must be non-negative, changing from %d to %d", validationFrequency, serverValidationFrequency) validationFrequency = serverValidationFrequency } lastValidation := lastValidationTime if lastValidation == nil { // Regardless of validation frequency, we want to validate all BSLs at least once. return true } if validationFrequency == 0 { // Validation was disabled so return false. log.Debug("Validation period for this backup location is set to 0, skipping validation") return false } // We want to validate BSL only if the set validation frequency/ interval has elapsed. nextValidation := lastValidation.Add(validationFrequency) // next validation time: last validation time + validation frequency return !time.Now().UTC().Before(nextValidation) // ready only when NOW is equal to or after the next validation time } // ListBackupStorageLocations verifies if there are any backup storage locations. // For all purposes, if either there is an error while attempting to fetch items or // if there are no items an error would be returned since the functioning of the system // would be haulted. func ListBackupStorageLocations(ctx context.Context, kbClient client.Client, namespace string) (velerov1api.BackupStorageLocationList, error) { var locations velerov1api.BackupStorageLocationList if err := kbClient.List(ctx, &locations, &client.ListOptions{ Namespace: namespace, }); err != nil { return velerov1api.BackupStorageLocationList{}, err } if len(locations.Items) == 0 { return velerov1api.BackupStorageLocationList{}, errors.New("no backup storage locations found") } return locations, nil } func GetDefaultBackupStorageLocations(ctx context.Context, kbClient client.Client, namespace string) (*velerov1api.BackupStorageLocationList, error) { locations := new(velerov1api.BackupStorageLocationList) defaultLocations := new(velerov1api.BackupStorageLocationList) if err := kbClient.List(context.Background(), locations, &client.ListOptions{Namespace: namespace}); err != nil { return defaultLocations, errors.Wrapf(err, "failed to list backup storage locations in namespace %s", namespace) } for _, location := range locations.Items { if location.Spec.Default { defaultLocations.Items = append(defaultLocations.Items, location) } } return defaultLocations, nil } ================================================ FILE: internal/storage/storagelocation_test.go ================================================ /*Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package storage import ( "testing" "time" . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client/fake" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" velerotest "github.com/vmware-tanzu/velero/pkg/test" "github.com/vmware-tanzu/velero/pkg/util" ) func TestIsReadyToValidate(t *testing.T) { tests := []struct { name string bslValidationFrequency *metav1.Duration lastValidationTime *metav1.Time defaultLocationInfo DefaultBackupLocationInfo ready bool }{ { name: "validate when true when validation frequency is zero and lastValidationTime is nil", bslValidationFrequency: &metav1.Duration{Duration: 0}, defaultLocationInfo: DefaultBackupLocationInfo{ ServerValidationFrequency: 0, }, ready: true, }, { name: "don't validate when false when validation is disabled and lastValidationTime is not nil", bslValidationFrequency: &metav1.Duration{Duration: 0}, lastValidationTime: &metav1.Time{Time: time.Now()}, defaultLocationInfo: DefaultBackupLocationInfo{ ServerValidationFrequency: 0, }, ready: false, }, { name: "validate as per location setting, as that takes precedence, and always if it has never been validated before regardless of the frequency setting", bslValidationFrequency: &metav1.Duration{Duration: 1 * time.Hour}, defaultLocationInfo: DefaultBackupLocationInfo{ ServerValidationFrequency: 0, }, ready: true, }, { name: "don't validate as per location setting, as it is set to zero and that takes precedence", bslValidationFrequency: &metav1.Duration{Duration: 0}, defaultLocationInfo: DefaultBackupLocationInfo{ ServerValidationFrequency: 1, }, lastValidationTime: &metav1.Time{Time: time.Now()}, ready: false, }, { name: "validate as per default setting when location setting is not set", defaultLocationInfo: DefaultBackupLocationInfo{ ServerValidationFrequency: 1, }, ready: true, }, { name: "don't validate when default setting is set to zero and the location setting is not set", defaultLocationInfo: DefaultBackupLocationInfo{ ServerValidationFrequency: 0, }, lastValidationTime: &metav1.Time{Time: time.Now()}, ready: false, }, { name: "don't validate when now is before the NEXT validation time (validation frequency + last validation time)", bslValidationFrequency: &metav1.Duration{Duration: 1 * time.Second}, lastValidationTime: &metav1.Time{Time: time.Now()}, defaultLocationInfo: DefaultBackupLocationInfo{ ServerValidationFrequency: 0, }, ready: false, }, { name: "validate when now is equal to the NEXT validation time (validation frequency + last validation time)", bslValidationFrequency: &metav1.Duration{Duration: 1 * time.Second}, lastValidationTime: &metav1.Time{Time: time.Now().Add(-1 * time.Second)}, defaultLocationInfo: DefaultBackupLocationInfo{ ServerValidationFrequency: 0, }, ready: true, }, { name: "validate when now is after the NEXT validation time (validation frequency + last validation time)", bslValidationFrequency: &metav1.Duration{Duration: 1 * time.Second}, lastValidationTime: &metav1.Time{Time: time.Now().Add(-2 * time.Second)}, defaultLocationInfo: DefaultBackupLocationInfo{ ServerValidationFrequency: 0, }, ready: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { g := NewWithT(t) log := velerotest.NewLogger() actual := IsReadyToValidate(tt.bslValidationFrequency, tt.lastValidationTime, tt.defaultLocationInfo.ServerValidationFrequency, log) g.Expect(actual).To(BeIdenticalTo(tt.ready)) }) } } func TestListBackupStorageLocations(t *testing.T) { tests := []struct { name string backupLocations *velerov1api.BackupStorageLocationList expectError bool }{ { name: "1 existing location does not return an error", backupLocations: &velerov1api.BackupStorageLocationList{ Items: []velerov1api.BackupStorageLocation{ *builder.ForBackupStorageLocation("ns-1", "location-1").Result(), }, }, expectError: false, }, { name: "multiple existing location does not return an error", backupLocations: &velerov1api.BackupStorageLocationList{ Items: []velerov1api.BackupStorageLocation{ *builder.ForBackupStorageLocation("ns-1", "location-1").Result(), *builder.ForBackupStorageLocation("ns-1", "location-2").Result(), *builder.ForBackupStorageLocation("ns-1", "location-3").Result(), }, }, expectError: false, }, { name: "no existing locations returns an error", backupLocations: &velerov1api.BackupStorageLocationList{}, expectError: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { g := NewWithT(t) client := fake.NewClientBuilder().WithScheme(util.VeleroScheme).WithRuntimeObjects(tt.backupLocations).Build() if tt.expectError { _, err := ListBackupStorageLocations(t.Context(), client, "ns-1") g.Expect(err).To(HaveOccurred()) } else { _, err := ListBackupStorageLocations(t.Context(), client, "ns-1") g.Expect(err).ToNot(HaveOccurred()) } }) } } ================================================ FILE: internal/velero/images.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package velero import ( "fmt" "github.com/vmware-tanzu/velero/pkg/buildinfo" ) // Use Dockerhub as the default registry if the build process didn't supply a registry func imageRegistry() string { if buildinfo.ImageRegistry == "" { return "velero" } return buildinfo.ImageRegistry } // ImageTag returns the image tag that should be used by Velero images. // It uses the Version from the buildinfo or "latest" if the build process didn't supply a version. func ImageTag() string { if buildinfo.Version == "" { return "latest" } return buildinfo.Version } // DefaultVeleroImage returns the default container image to use for this version of Velero. func DefaultVeleroImage() string { return fmt.Sprintf("%s/%s:%s", imageRegistry(), "velero", ImageTag()) } ================================================ FILE: internal/velero/images_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package velero import ( "fmt" "testing" "github.com/stretchr/testify/assert" "github.com/vmware-tanzu/velero/pkg/buildinfo" ) func TestImageTag(t *testing.T) { testCases := []struct { name string buildInfoVersion string want string }{ { name: "tag is latest when buildinfo.Version is empty", want: "latest", }, { name: "tag is buildinfo.Version when not empty", buildInfoVersion: "custom-build-version", want: "custom-build-version", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { originalVersion := buildinfo.Version buildinfo.Version = tc.buildInfoVersion defer func() { buildinfo.Version = originalVersion }() assert.Equal(t, tc.want, ImageTag()) }) } } func TestImageRegistry(t *testing.T) { testCases := []struct { name string buildInfoRegistry string want string }{ { name: "registry is velero when buildinfo.ImageRegistry is empty", want: "velero", }, { name: "registry is buildinfo.ImageRegistry when not empty", buildInfoRegistry: "custom-build-image-registry", want: "custom-build-image-registry", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { originalImageRegistry := buildinfo.ImageRegistry buildinfo.ImageRegistry = tc.buildInfoRegistry defer func() { buildinfo.ImageRegistry = originalImageRegistry }() assert.Equal(t, tc.want, imageRegistry()) }) } } func testDefaultImage(t *testing.T, defaultImageFn func() string, imageName string) { t.Helper() testCases := []struct { name string buildInfoVersion string buildInfoRegistry string want string }{ { name: "image uses velero as registry and latest as tag when buildinfo.ImageRegistry and buildinfo.Version are empty", want: fmt.Sprintf("velero/%s:latest", imageName), }, { name: "image uses buildinfo.ImageRegistry as registry when not empty", buildInfoRegistry: "custom-build-image-registry", want: fmt.Sprintf("custom-build-image-registry/%s:latest", imageName), }, { name: "image uses buildinfo.Version as tag when not empty", buildInfoVersion: "custom-build-version", want: fmt.Sprintf("velero/%s:custom-build-version", imageName), }, { name: "image uses both buildinfo.ImageRegistry and buildinfo.Version when not empty", buildInfoRegistry: "custom-build-image-registry", buildInfoVersion: "custom-build-version", want: fmt.Sprintf("custom-build-image-registry/%s:custom-build-version", imageName), }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { originalImageRegistry := buildinfo.ImageRegistry originalVersion := buildinfo.Version buildinfo.ImageRegistry = tc.buildInfoRegistry buildinfo.Version = tc.buildInfoVersion defer func() { buildinfo.ImageRegistry = originalImageRegistry buildinfo.Version = originalVersion }() assert.Equal(t, tc.want, defaultImageFn()) }) } } func TestDefaultVeleroImage(t *testing.T) { testDefaultImage(t, DefaultVeleroImage, "velero") } ================================================ FILE: internal/velero/serverstatusrequest.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package velero import ( velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/plugin/framework" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" ) type PluginLister interface { // List returns all PluginIdentifiers for kind. List(kind common.PluginKind) []framework.PluginIdentifier } // GetInstalledPluginInfo returns a list of installed plugins func GetInstalledPluginInfo(pluginLister PluginLister) []velerov1api.PluginInfo { var plugins []velerov1api.PluginInfo for _, v := range common.AllPluginKinds() { list := pluginLister.List(v) for _, plugin := range list { pluginInfo := velerov1api.PluginInfo{ Name: plugin.Name, Kind: plugin.Kind.String(), } plugins = append(plugins, pluginInfo) } } return plugins } ================================================ FILE: internal/volume/native_snapshot.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package volume // Snapshot stores information about a persistent volume snapshot taken as // part of a Velero backup. type Snapshot struct { Spec SnapshotSpec `json:"spec"` Status SnapshotStatus `json:"status"` } type SnapshotSpec struct { // BackupName is the name of the Velero backup this snapshot // is associated with. BackupName string `json:"backupName"` // BackupUID is the UID of the Velero backup this snapshot // is associated with. BackupUID string `json:"backupUID"` // Location is the name of the VolumeSnapshotLocation where this snapshot is stored. Location string `json:"location"` // PersistentVolumeName is the Kubernetes name for the volume. PersistentVolumeName string `json:"persistentVolumeName"` // ProviderVolumeID is the provider's ID for the volume. ProviderVolumeID string `json:"providerVolumeID"` // VolumeType is the type of the disk/volume in the cloud provider // API. VolumeType string `json:"volumeType"` // VolumeAZ is the where the volume is provisioned // in the cloud provider. VolumeAZ string `json:"volumeAZ,omitempty"` // VolumeIOPS is the optional value of provisioned IOPS for the // disk/volume in the cloud provider API. VolumeIOPS *int64 `json:"volumeIOPS,omitempty"` } type SnapshotStatus struct { // ProviderSnapshotID is the ID of the snapshot taken in the cloud // provider API of this volume. ProviderSnapshotID string `json:"providerSnapshotID,omitempty"` // Phase is the current state of the VolumeSnapshot. Phase SnapshotPhase `json:"phase,omitempty"` } // SnapshotPhase is the lifecycle phase of a Velero volume snapshot. type SnapshotPhase string const ( // SnapshotPhaseNew means the volume snapshot has been created but not // yet processed by the VolumeSnapshotController. SnapshotPhaseNew SnapshotPhase = "New" // SnapshotPhaseCompleted means the volume snapshot was successfully created and can be restored from.. SnapshotPhaseCompleted SnapshotPhase = "Completed" // SnapshotPhaseFailed means the volume snapshot was unable to execute. SnapshotPhaseFailed SnapshotPhase = "Failed" ) ================================================ FILE: internal/volume/snapshotlocation.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package volume import ( "github.com/pkg/errors" "github.com/vmware-tanzu/velero/internal/credentials" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) // UpdateVolumeSnapshotLocationWithCredentialConfig adds the credentials file path to the config // if the VSL specifies a credential func UpdateVolumeSnapshotLocationWithCredentialConfig(location *velerov1api.VolumeSnapshotLocation, credentialStore credentials.FileStore) error { if location.Spec.Config == nil { location.Spec.Config = make(map[string]string) } // If the VSL specifies a credential, fetch its path on disk and pass to // plugin via the config. if location.Spec.Credential != nil && credentialStore != nil { credsFile, err := credentialStore.Path(location.Spec.Credential) if err != nil { return errors.Wrap(err, "unable to get credentials") } location.Spec.Config["credentialsFile"] = credsFile } return nil } ================================================ FILE: internal/volume/utils.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package volume import ( "regexp" ) // it has to have the same value as "github.com/vmware-tanzu/velero/pkg/restore".ItemRestoreResultCreated const itemRestoreResultCreated = "created" // RestoredPVCFromRestoredResourceList returns a set of PVCs that were restored from the given restoredResourceList. func RestoredPVCFromRestoredResourceList(restoredResourceList map[string][]string) map[string]struct{} { pvcKey := "v1/PersistentVolumeClaim" pvcList := make(map[string]struct{}) for _, pvc := range restoredResourceList[pvcKey] { // the format of pvc string in restoredResourceList is like: "namespace/pvcName(status)" // extract the substring before "(created)" if the status in rightmost Parenthesis is "created" r := regexp.MustCompile(`\(([^)]+)\)`) matches := r.FindAllStringSubmatch(pvc, -1) if len(matches) > 0 && matches[len(matches)-1][1] == itemRestoreResultCreated { pvcList[pvc[:len(pvc)-len("(created)")]] = struct{}{} } } return pvcList } ================================================ FILE: internal/volume/utils_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package volume import ( "testing" "github.com/stretchr/testify/assert" ) func TestGetRestoredPVCFromRestoredResourceList(t *testing.T) { // test empty list restoredResourceList := map[string][]string{} actual := RestoredPVCFromRestoredResourceList(restoredResourceList) assert.Empty(t, actual) // test no match restoredResourceList = map[string][]string{ "v1/PersistentVolumeClaim": { "namespace1/pvc1(updated)", }, "v1/PersistentVolume": { "namespace1/pv(created)", }, } actual = RestoredPVCFromRestoredResourceList(restoredResourceList) assert.Empty(t, actual) // test matches restoredResourceList = map[string][]string{ "v1/PersistentVolumeClaim": { "namespace1/pvc1(created)", "namespace2/pvc2(updated)", "namespace3/pvc(3)(created)", }, } expected := map[string]struct{}{ "namespace1/pvc1": {}, "namespace3/pvc(3)": {}, } actual = RestoredPVCFromRestoredResourceList(restoredResourceList) assert.Equal(t, expected, actual) } ================================================ FILE: internal/volume/volumes_information.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package volume import ( "context" "strconv" "strings" "sync" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kbclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/pkg/label" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" "github.com/vmware-tanzu/velero/pkg/features" "github.com/vmware-tanzu/velero/pkg/itemoperation" "github.com/vmware-tanzu/velero/pkg/kuberesource" ) type Method string const ( NativeSnapshot Method = "NativeSnapshot" PodVolumeBackup Method = "PodVolumeBackup" CSISnapshot Method = "CSISnapshot" PodVolumeRestore Method = "PodVolumeRestore" ) const ( FieldValueIsUnknown string = "unknown" veleroDatamover string = "velero" ) type BackupVolumeInfo struct { // The PVC's name. PVCName string `json:"pvcName,omitempty"` // The PVC's namespace PVCNamespace string `json:"pvcNamespace,omitempty"` // The PV name. PVName string `json:"pvName,omitempty"` // The way the volume data is backed up. The valid value includes `VeleroNativeSnapshot`, `PodVolumeBackup` and `CSISnapshot`. BackupMethod Method `json:"backupMethod,omitempty"` // Whether the volume's snapshot data is moved to specified storage. SnapshotDataMoved bool `json:"snapshotDataMoved"` // Whether the local snapshot is preserved after snapshot is moved. // The local snapshot may be a result of CSI snapshot backup(no data movement) // or a CSI snapshot data movement plus preserve local snapshot. PreserveLocalSnapshot bool `json:"preserveLocalSnapshot"` // Whether the Volume is skipped in this backup. Skipped bool `json:"skipped"` // The reason for the volume is skipped in the backup. SkippedReason string `json:"skippedReason,omitempty"` // Snapshot starts timestamp. StartTimestamp *metav1.Time `json:"startTimestamp,omitempty"` // Snapshot completes timestamp. CompletionTimestamp *metav1.Time `json:"completionTimestamp,omitempty"` // Whether the volume data is backed up successfully. Result VolumeResult `json:"result,omitempty"` CSISnapshotInfo *CSISnapshotInfo `json:"csiSnapshotInfo,omitempty"` SnapshotDataMovementInfo *SnapshotDataMovementInfo `json:"snapshotDataMovementInfo,omitempty"` NativeSnapshotInfo *NativeSnapshotInfo `json:"nativeSnapshotInfo,omitempty"` PVBInfo *PodVolumeInfo `json:"pvbInfo,omitempty"` PVInfo *PVInfo `json:"pvInfo,omitempty"` } type VolumeResult string const ( VolumeResultSucceeded VolumeResult = "succeeded" VolumeResultFailed VolumeResult = "failed" //VolumeResultCanceled VolumeResult = "canceled" ) type RestoreVolumeInfo struct { // The name of the restored PVC PVCName string `json:"pvcName,omitempty"` // The namespace of the restored PVC PVCNamespace string `json:"pvcNamespace,omitempty"` // The name of the restored PV, it is possible that in one item there is only PVC or PV info. // But if both PVC and PV exist in one item of volume info, they should matched, and if the PV is bound to a PVC, // they should coexist in one item. PVName string `json:"pvName,omitempty"` // The way the volume data is restored. RestoreMethod Method `json:"restoreMethod,omitempty"` // Whether the volume's data are restored via data movement SnapshotDataMoved bool `json:"snapshotDataMoved"` CSISnapshotInfo *CSISnapshotInfo `json:"csiSnapshotInfo,omitempty"` SnapshotDataMovementInfo *SnapshotDataMovementInfo `json:"snapshotDataMovementInfo,omitempty"` NativeSnapshotInfo *NativeSnapshotInfo `json:"nativeSnapshotInfo,omitempty"` PVRInfo *PodVolumeInfo `json:"pvrInfo,omitempty"` } // CSISnapshotInfo is used for displaying the CSI snapshot status type CSISnapshotInfo struct { // It's the storage provider's snapshot ID for CSI. SnapshotHandle string `json:"snapshotHandle"` // The snapshot corresponding volume size. Size int64 `json:"size"` // The name of the CSI driver. Driver string `json:"driver"` // The name of the VolumeSnapshotContent. VSCName string `json:"vscName"` // The Async Operation's ID. OperationID string `json:"operationID,omitempty"` // The VolumeSnapshot's Status.ReadyToUse value ReadyToUse *bool } // SnapshotDataMovementInfo is used for displaying the snapshot data mover status. type SnapshotDataMovementInfo struct { // The data mover used by the backup. The valid values are `velero` and ``(equals to `velero`). DataMover string `json:"dataMover"` // The type of the uploader that uploads the snapshot data. The valid values are `kopia` and `restic`. UploaderType string `json:"uploaderType"` // The name or ID of the snapshot associated object(SAO). // SAO is used to support local snapshots for the snapshot data mover, // e.g. it could be a VolumeSnapshot for CSI snapshot data movement. RetainedSnapshot string `json:"retainedSnapshot,omitempty"` // It's the filesystem repository's snapshot ID. SnapshotHandle string `json:"snapshotHandle"` // The Async Operation's ID. OperationID string `json:"operationID"` // Moved snapshot data size. Size int64 `json:"size"` // Moved snapshot incremental size. IncrementalSize int64 `json:"incrementalSize,omitempty"` // The DataUpload's Status.Phase value Phase velerov2alpha1.DataUploadPhase } // NativeSnapshotInfo is used for displaying the Velero native snapshot status. // A Velero Native Snapshot is a cloud storage snapshot taken by the Velero native // plugins, e.g. velero-plugin-for-aws, velero-plugin-for-gcp, and // velero-plugin-for-microsoft-azure. type NativeSnapshotInfo struct { // It's the storage provider's snapshot ID for the Velero-native snapshot. SnapshotHandle string `json:"snapshotHandle"` // The cloud provider snapshot volume type. VolumeType string `json:"volumeType"` // The cloud provider snapshot volume's availability zones. VolumeAZ string `json:"volumeAZ"` // The cloud provider snapshot volume's IOPS. IOPS string `json:"iops"` // The NativeSnapshot's Status.Phase value Phase SnapshotPhase } func newNativeSnapshotInfo(s *Snapshot) *NativeSnapshotInfo { var iops int64 if s.Spec.VolumeIOPS != nil { iops = *s.Spec.VolumeIOPS } return &NativeSnapshotInfo{ SnapshotHandle: s.Status.ProviderSnapshotID, VolumeType: s.Spec.VolumeType, VolumeAZ: s.Spec.VolumeAZ, IOPS: strconv.FormatInt(iops, 10), Phase: s.Status.Phase, } } // PodVolumeInfo is used for displaying the PodVolumeBackup/PodVolumeRestore snapshot status. type PodVolumeInfo struct { // It's the file-system uploader's snapshot ID for PodVolumeBackup/PodVolumeRestore. SnapshotHandle string `json:"snapshotHandle,omitempty"` // The snapshot corresponding volume size. Size int64 `json:"size,omitempty"` // The incremental snapshot size. IncrementalSize int64 `json:"incrementalSize,omitempty"` // The type of the uploader that uploads the data. The valid values are `kopia` and `restic`. UploaderType string `json:"uploaderType"` // The PVC's corresponding volume name used by Pod // https://github.com/kubernetes/kubernetes/blob/e4b74dd12fa8cb63c174091d5536a10b8ec19d34/pkg/apis/core/types.go#L48 VolumeName string `json:"volumeName"` // The Pod name mounting this PVC. PodName string `json:"podName"` // The Pod namespace PodNamespace string `json:"podNamespace"` // The PVB-taken k8s node's name. // This field will be empty when the struct is used to represent a podvolumerestore. NodeName string `json:"nodeName,omitempty"` // The PVB's Status.Phase value Phase velerov1api.PodVolumeBackupPhase } func newPodVolumeInfoFromPVB(pvb *velerov1api.PodVolumeBackup) *PodVolumeInfo { return &PodVolumeInfo{ SnapshotHandle: pvb.Status.SnapshotID, Size: pvb.Status.Progress.TotalBytes, IncrementalSize: pvb.Status.IncrementalBytes, UploaderType: pvb.Spec.UploaderType, VolumeName: pvb.Spec.Volume, PodName: pvb.Spec.Pod.Name, PodNamespace: pvb.Spec.Pod.Namespace, NodeName: pvb.Spec.Node, Phase: pvb.Status.Phase, } } func newPodVolumeInfoFromPVR(pvr *velerov1api.PodVolumeRestore) *PodVolumeInfo { return &PodVolumeInfo{ SnapshotHandle: pvr.Spec.SnapshotID, Size: pvr.Status.Progress.TotalBytes, UploaderType: pvr.Spec.UploaderType, VolumeName: pvr.Spec.Volume, PodName: pvr.Spec.Pod.Name, PodNamespace: pvr.Spec.Pod.Namespace, } } // PVInfo is used to store some PV information modified after creation. // Those information are lost after PV recreation. type PVInfo struct { // ReclaimPolicy of PV. It could be different from the referenced StorageClass. ReclaimPolicy string `json:"reclaimPolicy"` // The PV's labels should be kept after recreation. Labels map[string]string `json:"labels"` } // BackupVolumesInformation contains the information needs by generating // the backup BackupVolumeInfo array. type BackupVolumesInformation struct { // A map contains the backup-included PV detail content. The key is PV name. pvMap *pvcPvMap volumeInfos []*BackupVolumeInfo logger logrus.FieldLogger crClient kbclient.Client volumeSnapshots []snapshotv1api.VolumeSnapshot volumeSnapshotContents []snapshotv1api.VolumeSnapshotContent volumeSnapshotClasses []snapshotv1api.VolumeSnapshotClass SkippedPVs map[string]string NativeSnapshots []*Snapshot PodVolumeBackups []*velerov1api.PodVolumeBackup BackupOperations []*itemoperation.BackupOperation BackupName string } type pvcPvInfo struct { PVCName string PVCNamespace string PV corev1api.PersistentVolume } func (v *BackupVolumesInformation) Init() { v.pvMap = &pvcPvMap{ data: make(map[string]pvcPvInfo), } v.volumeInfos = make([]*BackupVolumeInfo, 0) } func (v *BackupVolumesInformation) InsertPVMap(pv corev1api.PersistentVolume, pvcName, pvcNamespace string) { if v.pvMap == nil { v.Init() } v.pvMap.insert(pv, pvcName, pvcNamespace) } func (v *BackupVolumesInformation) Result( csiVolumeSnapshots []snapshotv1api.VolumeSnapshot, csiVolumeSnapshotContents []snapshotv1api.VolumeSnapshotContent, csiVolumesnapshotClasses []snapshotv1api.VolumeSnapshotClass, crClient kbclient.Client, logger logrus.FieldLogger, ) []*BackupVolumeInfo { v.logger = logger v.crClient = crClient v.volumeSnapshots = csiVolumeSnapshots v.volumeSnapshotContents = csiVolumeSnapshotContents v.volumeSnapshotClasses = csiVolumesnapshotClasses v.generateVolumeInfoForSkippedPV() v.generateVolumeInfoForVeleroNativeSnapshot() v.generateVolumeInfoForCSIVolumeSnapshot() v.generateVolumeInfoFromPVB() v.generateVolumeInfoFromDataUpload() return v.volumeInfos } // generateVolumeInfoForSkippedPV generate VolumeInfos for SkippedPV. func (v *BackupVolumesInformation) generateVolumeInfoForSkippedPV() { tmpVolumeInfos := make([]*BackupVolumeInfo, 0) for pvName, skippedReason := range v.SkippedPVs { if pvcPVInfo := v.pvMap.retrieve(pvName, "", ""); pvcPVInfo != nil { volumeInfo := &BackupVolumeInfo{ PVCName: pvcPVInfo.PVCName, PVCNamespace: pvcPVInfo.PVCNamespace, PVName: pvName, SnapshotDataMoved: false, Skipped: true, SkippedReason: skippedReason, PVInfo: &PVInfo{ ReclaimPolicy: string(pvcPVInfo.PV.Spec.PersistentVolumeReclaimPolicy), Labels: pvcPVInfo.PV.Labels, }, } tmpVolumeInfos = append(tmpVolumeInfos, volumeInfo) } else { v.logger.Warnf("Cannot find info for PV %s", pvName) continue } } v.volumeInfos = append(v.volumeInfos, tmpVolumeInfos...) } // generateVolumeInfoForVeleroNativeSnapshot generate VolumeInfos for Velero native snapshot func (v *BackupVolumesInformation) generateVolumeInfoForVeleroNativeSnapshot() { tmpVolumeInfos := make([]*BackupVolumeInfo, 0) for _, nativeSnapshot := range v.NativeSnapshots { if pvcPVInfo := v.pvMap.retrieve(nativeSnapshot.Spec.PersistentVolumeName, "", ""); pvcPVInfo != nil { volumeResult := VolumeResultFailed if nativeSnapshot.Status.Phase == SnapshotPhaseCompleted { volumeResult = VolumeResultSucceeded } volumeInfo := &BackupVolumeInfo{ BackupMethod: NativeSnapshot, PVCName: pvcPVInfo.PVCName, PVCNamespace: pvcPVInfo.PVCNamespace, PVName: pvcPVInfo.PV.Name, SnapshotDataMoved: false, Skipped: false, // Only set Succeeded to true when the NativeSnapshot's phase is Completed, // although NativeSnapshot doesn't check whether the snapshot creation result. Result: volumeResult, NativeSnapshotInfo: newNativeSnapshotInfo(nativeSnapshot), PVInfo: &PVInfo{ ReclaimPolicy: string(pvcPVInfo.PV.Spec.PersistentVolumeReclaimPolicy), Labels: pvcPVInfo.PV.Labels, }, } tmpVolumeInfos = append(tmpVolumeInfos, volumeInfo) } else { v.logger.Warnf("cannot find info for PV %s", nativeSnapshot.Spec.PersistentVolumeName) continue } } v.volumeInfos = append(v.volumeInfos, tmpVolumeInfos...) } // generateVolumeInfoForCSIVolumeSnapshot generate VolumeInfos for CSI VolumeSnapshot func (v *BackupVolumesInformation) generateVolumeInfoForCSIVolumeSnapshot() { tmpVolumeInfos := make([]*BackupVolumeInfo, 0) for _, volumeSnapshot := range v.volumeSnapshots { var volumeSnapshotContent *snapshotv1api.VolumeSnapshotContent // This is protective logic. The passed-in VS should be all related // to this backup. if volumeSnapshot.Labels[velerov1api.BackupNameLabel] != v.BackupName { continue } if volumeSnapshot.Status == nil || volumeSnapshot.Status.BoundVolumeSnapshotContentName == nil { v.logger.Warnf("Cannot fine VolumeSnapshotContent for VolumeSnapshot %s/%s", volumeSnapshot.Namespace, volumeSnapshot.Name) continue } if volumeSnapshot.Spec.Source.PersistentVolumeClaimName == nil { v.logger.Warnf("VolumeSnapshot %s/%s doesn't have a source PVC", volumeSnapshot.Namespace, volumeSnapshot.Name) continue } for index := range v.volumeSnapshotContents { if *volumeSnapshot.Status.BoundVolumeSnapshotContentName == v.volumeSnapshotContents[index].Name { volumeSnapshotContent = &v.volumeSnapshotContents[index] } } if volumeSnapshotContent == nil { v.logger.Warnf("fail to get VolumeSnapshotContent for VolumeSnapshot: %s/%s", volumeSnapshot.Namespace, volumeSnapshot.Name) continue } var operation itemoperation.BackupOperation for _, op := range v.BackupOperations { if op.Spec.ResourceIdentifier.GroupResource.String() == kuberesource.VolumeSnapshots.String() && op.Spec.ResourceIdentifier.Name == volumeSnapshot.Name && op.Spec.ResourceIdentifier.Namespace == volumeSnapshot.Namespace { operation = *op } } var size int64 if volumeSnapshot.Status.RestoreSize != nil { size = volumeSnapshot.Status.RestoreSize.Value() } snapshotHandle := "" if volumeSnapshotContent.Status.SnapshotHandle != nil { snapshotHandle = *volumeSnapshotContent.Status.SnapshotHandle } if pvcPVInfo := v.pvMap.retrieve("", *volumeSnapshot.Spec.Source.PersistentVolumeClaimName, volumeSnapshot.Namespace); pvcPVInfo != nil { volumeInfo := &BackupVolumeInfo{ BackupMethod: CSISnapshot, PVCName: pvcPVInfo.PVCName, PVCNamespace: pvcPVInfo.PVCNamespace, PVName: pvcPVInfo.PV.Name, Skipped: false, SnapshotDataMoved: false, PreserveLocalSnapshot: true, CSISnapshotInfo: &CSISnapshotInfo{ VSCName: *volumeSnapshot.Status.BoundVolumeSnapshotContentName, Size: size, Driver: volumeSnapshotContent.Spec.Driver, SnapshotHandle: snapshotHandle, OperationID: operation.Spec.OperationID, ReadyToUse: volumeSnapshot.Status.ReadyToUse, }, PVInfo: &PVInfo{ ReclaimPolicy: string(pvcPVInfo.PV.Spec.PersistentVolumeReclaimPolicy), Labels: pvcPVInfo.PV.Labels, }, } if volumeSnapshot.Status.CreationTime != nil { volumeInfo.StartTimestamp = volumeSnapshot.Status.CreationTime } tmpVolumeInfos = append(tmpVolumeInfos, volumeInfo) } else { v.logger.Warnf("cannot find info for PVC %s/%s", volumeSnapshot.Namespace, volumeSnapshot.Spec.Source.PersistentVolumeClaimName) continue } } v.volumeInfos = append(v.volumeInfos, tmpVolumeInfos...) } // generateVolumeInfoFromPVB generate BackupVolumeInfo for PVB. func (v *BackupVolumesInformation) generateVolumeInfoFromPVB() { tmpVolumeInfos := make([]*BackupVolumeInfo, 0) for _, pvb := range v.PodVolumeBackups { volumeInfo := &BackupVolumeInfo{ BackupMethod: PodVolumeBackup, SnapshotDataMoved: false, Skipped: false, StartTimestamp: pvb.Status.StartTimestamp, CompletionTimestamp: pvb.Status.CompletionTimestamp, PVBInfo: newPodVolumeInfoFromPVB(pvb), } // Only set Succeeded to true when the PVB's phase is Completed. if pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseCompleted { volumeInfo.Result = VolumeResultSucceeded } else { volumeInfo.Result = VolumeResultFailed } pvcName, err := pvcByPodvolume(context.TODO(), v.crClient, pvb.Spec.Pod.Name, pvb.Spec.Pod.Namespace, pvb.Spec.Volume) if err != nil { v.logger.WithError(err).Warn("Fail to get PVC from PodVolumeBackup: ", pvb.Name) continue } if pvcName != "" { if pvcPVInfo := v.pvMap.retrieve("", pvcName, pvb.Spec.Pod.Namespace); pvcPVInfo != nil { volumeInfo.PVCName = pvcPVInfo.PVCName volumeInfo.PVCNamespace = pvcPVInfo.PVCNamespace volumeInfo.PVName = pvcPVInfo.PV.Name volumeInfo.PVInfo = &PVInfo{ ReclaimPolicy: string(pvcPVInfo.PV.Spec.PersistentVolumeReclaimPolicy), Labels: pvcPVInfo.PV.Labels, } } else { v.logger.Warnf("Cannot find info for PVC %s/%s", pvb.Spec.Pod.Namespace, pvcName) continue } } else { v.logger.Debug("The PVB %s doesn't have a corresponding PVC", pvb.Name) } tmpVolumeInfos = append(tmpVolumeInfos, volumeInfo) } v.volumeInfos = append(v.volumeInfos, tmpVolumeInfos...) } func (v *BackupVolumesInformation) getVolumeSnapshotClasses() ( []snapshotv1api.VolumeSnapshotClass, error, ) { vsClassList := new(snapshotv1api.VolumeSnapshotClassList) if err := v.crClient.List(context.TODO(), vsClassList); err != nil { v.logger.Warnf("Cannot list VolumeSnapshotClass with error %s.", err.Error()) return nil, err } return vsClassList.Items, nil } // generateVolumeInfoFromDataUpload generate BackupVolumeInfo for DataUpload. func (v *BackupVolumesInformation) generateVolumeInfoFromDataUpload() { if !features.IsEnabled(velerov1api.CSIFeatureFlag) { v.logger.Debug("Skip generating BackupVolumeInfo when the CSI feature is disabled.") return } // Retrieve the operations containing DataUpload. duOperationMap := make(map[kbclient.ObjectKey]*itemoperation.BackupOperation) for _, operation := range v.BackupOperations { if operation.Spec.ResourceIdentifier.GroupResource.String() == kuberesource.PersistentVolumeClaims.String() { for _, identifier := range operation.Spec.PostOperationItems { if identifier.GroupResource.String() == "datauploads.velero.io" { duOperationMap[kbclient.ObjectKey{ Namespace: identifier.Namespace, Name: identifier.Name, }] = operation break } } } } if len(duOperationMap) <= 0 { // No DataUpload is found. Return early. return } tmpVolumeInfos := make([]*BackupVolumeInfo, 0) for duObjectKey, operation := range duOperationMap { dataUpload := new(velerov2alpha1.DataUpload) err := v.crClient.Get( context.TODO(), duObjectKey, dataUpload, ) if err != nil { v.logger.Warnf("Fail to get DataUpload %s: %s", duObjectKey.Namespace+"/"+duObjectKey.Name, err.Error(), ) continue } if pvcPVInfo := v.pvMap.retrieve( "", operation.Spec.ResourceIdentifier.Name, operation.Spec.ResourceIdentifier.Namespace, ); pvcPVInfo != nil { dataMover := veleroDatamover if dataUpload.Spec.DataMover != "" { dataMover = dataUpload.Spec.DataMover } volumeInfo := &BackupVolumeInfo{ BackupMethod: CSISnapshot, PVCName: pvcPVInfo.PVCName, PVCNamespace: pvcPVInfo.PVCNamespace, PVName: pvcPVInfo.PV.Name, SnapshotDataMoved: true, Skipped: false, CSISnapshotInfo: &CSISnapshotInfo{ SnapshotHandle: FieldValueIsUnknown, VSCName: FieldValueIsUnknown, OperationID: FieldValueIsUnknown, Driver: dataUpload.Spec.CSISnapshot.Driver, }, SnapshotDataMovementInfo: &SnapshotDataMovementInfo{ DataMover: dataMover, UploaderType: velerov1api.BackupRepositoryTypeKopia, OperationID: operation.Spec.OperationID, Phase: dataUpload.Status.Phase, }, PVInfo: &PVInfo{ ReclaimPolicy: string(pvcPVInfo.PV.Spec.PersistentVolumeReclaimPolicy), Labels: pvcPVInfo.PV.Labels, }, } if dataUpload.Status.StartTimestamp != nil { volumeInfo.StartTimestamp = dataUpload.Status.StartTimestamp } tmpVolumeInfos = append(tmpVolumeInfos, volumeInfo) } else { v.logger.Warnf("Cannot find info for PVC %s/%s", operation.Spec.ResourceIdentifier.Namespace, operation.Spec.ResourceIdentifier.Name) continue } } v.volumeInfos = append(v.volumeInfos, tmpVolumeInfos...) } type pvcPvMap struct { data map[string]pvcPvInfo } func (m *pvcPvMap) insert(pv corev1api.PersistentVolume, pvcName, pvcNamespace string) { m.data[pv.Name] = pvcPvInfo{ PVCName: pvcName, PVCNamespace: pvcNamespace, PV: pv, } } func (m *pvcPvMap) retrieve(pvName, pvcName, pvcNS string) *pvcPvInfo { if pvName != "" { if info, ok := m.data[pvName]; ok { return &info } return nil } if pvcNS == "" || pvcName == "" { return nil } for _, info := range m.data { if pvcNS == info.PVCNamespace && pvcName == info.PVCName { return &info } } return nil } func pvcByPodvolume(ctx context.Context, crClient kbclient.Client, podName, podNamespace, volumeName string) (string, error) { pod := new(corev1api.Pod) err := crClient.Get(ctx, kbclient.ObjectKey{Namespace: podNamespace, Name: podName}, pod) if err != nil { return "", errors.Wrap(err, "failed to get pod") } for _, volume := range pod.Spec.Volumes { if volume.Name == volumeName && volume.PersistentVolumeClaim != nil { return volume.PersistentVolumeClaim.ClaimName, nil } } return "", nil } // RestoreVolumeInfoTracker is used to track the volume information during restore. // It is used to generate the RestoreVolumeInfo array. type RestoreVolumeInfoTracker struct { *sync.Mutex restore *velerov1api.Restore log logrus.FieldLogger client kbclient.Client pvPvc *pvcPvMap // map of PV name to the NativeSnapshotInfo from which the PV is restored pvNativeSnapshotMap map[string]*NativeSnapshotInfo // map of PVC object to the CSISnapshot object from which the PV is restored // the key is in the form of $pvc-ns/$pvc-name pvcCSISnapshotMap map[string]snapshotv1api.VolumeSnapshot datadownloadList *velerov2alpha1.DataDownloadList pvrs []*velerov1api.PodVolumeRestore } // Populate data objects in the tracker, which will be used to generate the RestoreVolumeInfo array in Result() // The input param resourceList should be the final result of the restore. func (t *RestoreVolumeInfoTracker) Populate(ctx context.Context, restoredResourceList map[string][]string) { pvcs := RestoredPVCFromRestoredResourceList(restoredResourceList) t.Lock() defer t.Unlock() for item := range pvcs { n := strings.Split(item, "/") pvcNS, pvcName := n[0], n[1] log := t.log.WithField("namespace", pvcNS).WithField("name", pvcName) pvc := &corev1api.PersistentVolumeClaim{} if err := t.client.Get(ctx, kbclient.ObjectKey{Namespace: pvcNS, Name: pvcName}, pvc); err != nil { log.WithError(err).Error("Failed to get PVC") continue } // Collect the CSI VolumeSnapshot objects referenced by the restored PVCs, if pvc.Spec.DataSource != nil && pvc.Spec.DataSource.Kind == "VolumeSnapshot" { vs := &snapshotv1api.VolumeSnapshot{} if err := t.client.Get(ctx, kbclient.ObjectKey{Namespace: pvcNS, Name: pvc.Spec.DataSource.Name}, vs); err != nil { log.WithError(err).Error("Failed to get VolumeSnapshot") } else { t.pvcCSISnapshotMap[pvc.Namespace+"/"+pvcName] = *vs } } if pvc.Status.Phase == corev1api.ClaimBound && pvc.Spec.VolumeName != "" { pv := &corev1api.PersistentVolume{} if err := t.client.Get(ctx, kbclient.ObjectKey{Name: pvc.Spec.VolumeName}, pv); err != nil { log.WithError(err).Error("Failed to get PV") } else { t.pvPvc.insert(*pv, pvcName, pvcNS) } } else { log.Warn("PVC is not bound or has no volume name") continue } } if err := t.client.List(ctx, t.datadownloadList, &kbclient.ListOptions{ Namespace: t.restore.Namespace, LabelSelector: label.NewSelectorForRestore(t.restore.Name), }); err != nil { t.log.WithError(err).Error("Failed to List DataDownloads") } } // Result generates the RestoreVolumeInfo array, the data should come from the Tracker itself and it should not connect tokkkk API // server again. func (t *RestoreVolumeInfoTracker) Result() []*RestoreVolumeInfo { volumeInfos := make([]*RestoreVolumeInfo, 0) // Generate RestoreVolumeInfo for PVRs for _, pvr := range t.pvrs { volumeInfo := &RestoreVolumeInfo{ SnapshotDataMoved: false, PVRInfo: newPodVolumeInfoFromPVR(pvr), RestoreMethod: PodVolumeRestore, } pvcName, err := pvcByPodvolume(context.TODO(), t.client, pvr.Spec.Pod.Name, pvr.Spec.Pod.Namespace, pvr.Spec.Volume) if err != nil { t.log.WithError(err).Warn("Fail to get PVC from PodVolumeRestore: ", pvr.Name) continue } if pvcName != "" { volumeInfo.PVCName = pvcName volumeInfo.PVCNamespace = pvr.Spec.Pod.Namespace if pvcPVInfo := t.pvPvc.retrieve("", pvcName, pvr.Spec.Pod.Namespace); pvcPVInfo != nil { volumeInfo.PVName = pvcPVInfo.PV.Name } } else { // In this case, the volume is not bound to a PVC and // the PVR will not be able to populate into the volume, so we'll skip it t.log.Warnf("unable to get PVC for PodVolumeRestore %s/%s, pod: %s/%s, volume: %s", pvr.Namespace, pvr.Name, pvr.Spec.Pod.Namespace, pvr.Spec.Pod.Name, pvr.Spec.Volume) continue } volumeInfos = append(volumeInfos, volumeInfo) } // Generate RestoreVolumeInfo for PVs restored from NativeSnapshots for pvName, snapshotInfo := range t.pvNativeSnapshotMap { volumeInfo := &RestoreVolumeInfo{ PVName: pvName, SnapshotDataMoved: false, NativeSnapshotInfo: snapshotInfo, RestoreMethod: NativeSnapshot, } if pvcPVInfo := t.pvPvc.retrieve(pvName, "", ""); pvcPVInfo != nil { volumeInfo.PVCName = pvcPVInfo.PVCName volumeInfo.PVCNamespace = pvcPVInfo.PVCNamespace } volumeInfos = append(volumeInfos, volumeInfo) } // Generate RestoreVolumeInfo for PVs restored from CSISnapshots for pvc, csiSnapshot := range t.pvcCSISnapshotMap { n := strings.Split(pvc, "/") if len(n) != 2 { t.log.Warnf("Invalid PVC key '%s' in the pvc-CSISnapshot map, skip populating it to volume info", pvc) continue } pvcNS, pvcName := n[0], n[1] var restoreSize int64 if csiSnapshot.Status != nil && csiSnapshot.Status.RestoreSize != nil { restoreSize = csiSnapshot.Status.RestoreSize.Value() } vscName := "" if csiSnapshot.Spec.Source.VolumeSnapshotContentName != nil { vscName = *csiSnapshot.Spec.Source.VolumeSnapshotContentName } volumeInfo := &RestoreVolumeInfo{ PVCNamespace: pvcNS, PVCName: pvcName, SnapshotDataMoved: false, RestoreMethod: CSISnapshot, CSISnapshotInfo: &CSISnapshotInfo{ SnapshotHandle: csiSnapshot.Annotations[velerov1api.VolumeSnapshotHandleAnnotation], Size: restoreSize, Driver: csiSnapshot.Annotations[velerov1api.DriverNameAnnotation], VSCName: vscName, }, } if pvcPVInfo := t.pvPvc.retrieve("", pvcName, pvcNS); pvcPVInfo != nil { volumeInfo.PVName = pvcPVInfo.PV.Name } volumeInfos = append(volumeInfos, volumeInfo) } for _, dd := range t.datadownloadList.Items { var pvcName, pvcNS, pvName string if pvcPVInfo := t.pvPvc.retrieve(dd.Spec.TargetVolume.PV, dd.Spec.TargetVolume.PVC, dd.Spec.TargetVolume.Namespace); pvcPVInfo != nil { pvcName = pvcPVInfo.PVCName pvcNS = pvcPVInfo.PVCNamespace pvName = pvcPVInfo.PV.Name } else { pvcName = dd.Spec.TargetVolume.PVC pvName = dd.Spec.TargetVolume.PV pvcNS = dd.Spec.TargetVolume.Namespace } operationID := dd.Labels[velerov1api.AsyncOperationIDLabel] dataMover := veleroDatamover if dd.Spec.DataMover != "" { dataMover = dd.Spec.DataMover } volumeInfo := &RestoreVolumeInfo{ PVName: pvName, PVCNamespace: pvcNS, PVCName: pvcName, SnapshotDataMoved: true, // The method will be CSI always no CSI related CRs are created during restore, because // the datadownload was initiated in CSI plugin // For the same reason, no CSI snapshot info will be populated into volumeInfo RestoreMethod: CSISnapshot, SnapshotDataMovementInfo: &SnapshotDataMovementInfo{ DataMover: dataMover, UploaderType: velerov1api.BackupRepositoryTypeKopia, SnapshotHandle: dd.Spec.SnapshotID, OperationID: operationID, }, } volumeInfos = append(volumeInfos, volumeInfo) } return volumeInfos } func NewRestoreVolInfoTracker(restore *velerov1api.Restore, logger logrus.FieldLogger, client kbclient.Client) *RestoreVolumeInfoTracker { return &RestoreVolumeInfoTracker{ Mutex: &sync.Mutex{}, client: client, log: logger, restore: restore, pvPvc: &pvcPvMap{ data: make(map[string]pvcPvInfo), }, pvNativeSnapshotMap: make(map[string]*NativeSnapshotInfo), pvcCSISnapshotMap: make(map[string]snapshotv1api.VolumeSnapshot), datadownloadList: &velerov2alpha1.DataDownloadList{}, } } func (t *RestoreVolumeInfoTracker) TrackNativeSnapshot(pvName string, snapshotHandle, volumeType, volumeAZ string, iops int64) { t.Lock() defer t.Unlock() t.pvNativeSnapshotMap[pvName] = &NativeSnapshotInfo{ SnapshotHandle: snapshotHandle, VolumeType: volumeType, VolumeAZ: volumeAZ, IOPS: strconv.FormatInt(iops, 10), } } func (t *RestoreVolumeInfoTracker) RenamePVForNativeSnapshot(oldName, newName string) { t.Lock() defer t.Unlock() if snapshotInfo, ok := t.pvNativeSnapshotMap[oldName]; ok { t.pvNativeSnapshotMap[newName] = snapshotInfo delete(t.pvNativeSnapshotMap, oldName) } } func (t *RestoreVolumeInfoTracker) TrackPodVolume(pvr *velerov1api.PodVolumeRestore) { t.Lock() defer t.Unlock() t.pvrs = append(t.pvrs, pvr) } ================================================ FILE: internal/volume/volumes_information_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package volume import ( "sync" "testing" "github.com/stretchr/testify/assert" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/features" "github.com/vmware-tanzu/velero/pkg/itemoperation" "github.com/vmware-tanzu/velero/pkg/plugin/velero" velerotest "github.com/vmware-tanzu/velero/pkg/test" "github.com/vmware-tanzu/velero/pkg/util/logging" ) func TestGenerateVolumeInfoForSkippedPV(t *testing.T) { tests := []struct { name string skippedPVName string pvMap map[string]pvcPvInfo expectedVolumeInfos []*BackupVolumeInfo }{ { name: "Cannot find info for PV", skippedPVName: "testPV", pvMap: map[string]pvcPvInfo{ "velero/testPVC": { PVCName: "testPVC", PVCNamespace: "velero", PV: corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "testPV", Labels: map[string]string{"a": "b"}, }, Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeReclaimPolicy: corev1api.PersistentVolumeReclaimDelete, }, }, }, }, expectedVolumeInfos: []*BackupVolumeInfo{}, }, { name: "Normal Skipped PV info", skippedPVName: "testPV", pvMap: map[string]pvcPvInfo{ "velero/testPVC": { PVCName: "testPVC", PVCNamespace: "velero", PV: corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "testPV", Labels: map[string]string{"a": "b"}, }, Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeReclaimPolicy: corev1api.PersistentVolumeReclaimDelete, }, }, }, "testPV": { PVCName: "testPVC", PVCNamespace: "velero", PV: corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "testPV", Labels: map[string]string{"a": "b"}, }, Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeReclaimPolicy: corev1api.PersistentVolumeReclaimDelete, }, }, }, }, expectedVolumeInfos: []*BackupVolumeInfo{ { PVCName: "testPVC", PVCNamespace: "velero", PVName: "testPV", Skipped: true, SkippedReason: "CSI: skipped for PodVolumeBackup", PVInfo: &PVInfo{ ReclaimPolicy: "Delete", Labels: map[string]string{ "a": "b", }, }, }, }, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { volumesInfo := BackupVolumesInformation{} volumesInfo.Init() if tc.skippedPVName != "" { volumesInfo.SkippedPVs = map[string]string{ tc.skippedPVName: "CSI: skipped for PodVolumeBackup", } } if tc.pvMap != nil { for k, v := range tc.pvMap { if k == v.PV.Name { volumesInfo.pvMap.insert(v.PV, v.PVCName, v.PVCNamespace) } } } volumesInfo.logger = logging.DefaultLogger(logrus.DebugLevel, logging.FormatJSON) volumesInfo.generateVolumeInfoForSkippedPV() require.Equal(t, tc.expectedVolumeInfos, volumesInfo.volumeInfos) }) } } func TestGenerateVolumeInfoForVeleroNativeSnapshot(t *testing.T) { tests := []struct { name string nativeSnapshot Snapshot pvMap map[string]pvcPvInfo expectedVolumeInfos []*BackupVolumeInfo }{ { name: "Native snapshot's IPOS pointer is nil", nativeSnapshot: Snapshot{ Spec: SnapshotSpec{ PersistentVolumeName: "testPV", VolumeIOPS: nil, }, }, expectedVolumeInfos: []*BackupVolumeInfo{}, }, { name: "Cannot find info for the PV", nativeSnapshot: Snapshot{ Spec: SnapshotSpec{ PersistentVolumeName: "testPV", VolumeIOPS: int64Ptr(100), }, }, expectedVolumeInfos: []*BackupVolumeInfo{}, }, { name: "Cannot find PV info in pvMap", pvMap: map[string]pvcPvInfo{ "velero/testPVC": { PVCName: "testPVC", PVCNamespace: "velero", PV: corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "testPV", Labels: map[string]string{"a": "b"}, }, Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeReclaimPolicy: corev1api.PersistentVolumeReclaimDelete, }, }, }, }, nativeSnapshot: Snapshot{ Spec: SnapshotSpec{ PersistentVolumeName: "testPV", VolumeIOPS: int64Ptr(100), VolumeType: "ssd", VolumeAZ: "us-central1-a", }, Status: SnapshotStatus{ ProviderSnapshotID: "pvc-b31e3386-4bbb-4937-95d-7934cd62-b0a1-494b-95d7-0687440e8d0c", }, }, expectedVolumeInfos: []*BackupVolumeInfo{}, }, { name: "Normal native snapshot with failed phase", pvMap: map[string]pvcPvInfo{ "testPV": { PVCName: "testPVC", PVCNamespace: "velero", PV: corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "testPV", Labels: map[string]string{"a": "b"}, }, Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeReclaimPolicy: corev1api.PersistentVolumeReclaimDelete, }, }, }, }, nativeSnapshot: Snapshot{ Spec: SnapshotSpec{ PersistentVolumeName: "testPV", VolumeIOPS: int64Ptr(100), VolumeType: "ssd", VolumeAZ: "us-central1-a", }, Status: SnapshotStatus{ ProviderSnapshotID: "pvc-b31e3386-4bbb-4937-95d-7934cd62-b0a1-494b-95d7-0687440e8d0c", Phase: SnapshotPhaseFailed, }, }, expectedVolumeInfos: []*BackupVolumeInfo{ { PVCName: "testPVC", PVCNamespace: "velero", PVName: "testPV", BackupMethod: NativeSnapshot, Result: VolumeResultFailed, PVInfo: &PVInfo{ ReclaimPolicy: "Delete", Labels: map[string]string{ "a": "b", }, }, NativeSnapshotInfo: &NativeSnapshotInfo{ SnapshotHandle: "pvc-b31e3386-4bbb-4937-95d-7934cd62-b0a1-494b-95d7-0687440e8d0c", VolumeType: "ssd", VolumeAZ: "us-central1-a", IOPS: "100", Phase: SnapshotPhaseFailed, }, }, }, }, { name: "Normal native snapshot", pvMap: map[string]pvcPvInfo{ "testPV": { PVCName: "testPVC", PVCNamespace: "velero", PV: corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "testPV", Labels: map[string]string{"a": "b"}, }, Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeReclaimPolicy: corev1api.PersistentVolumeReclaimDelete, }, }, }, }, nativeSnapshot: Snapshot{ Spec: SnapshotSpec{ PersistentVolumeName: "testPV", VolumeIOPS: int64Ptr(100), VolumeType: "ssd", VolumeAZ: "us-central1-a", }, Status: SnapshotStatus{ ProviderSnapshotID: "pvc-b31e3386-4bbb-4937-95d-7934cd62-b0a1-494b-95d7-0687440e8d0c", Phase: SnapshotPhaseCompleted, }, }, expectedVolumeInfos: []*BackupVolumeInfo{ { PVCName: "testPVC", PVCNamespace: "velero", PVName: "testPV", BackupMethod: NativeSnapshot, Result: VolumeResultSucceeded, PVInfo: &PVInfo{ ReclaimPolicy: "Delete", Labels: map[string]string{ "a": "b", }, }, NativeSnapshotInfo: &NativeSnapshotInfo{ SnapshotHandle: "pvc-b31e3386-4bbb-4937-95d-7934cd62-b0a1-494b-95d7-0687440e8d0c", VolumeType: "ssd", VolumeAZ: "us-central1-a", IOPS: "100", Phase: SnapshotPhaseCompleted, }, }, }, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { volumesInfo := BackupVolumesInformation{} volumesInfo.Init() volumesInfo.NativeSnapshots = append(volumesInfo.NativeSnapshots, &tc.nativeSnapshot) if tc.pvMap != nil { for k, v := range tc.pvMap { if k == v.PV.Name { volumesInfo.pvMap.insert(v.PV, v.PVCName, v.PVCNamespace) } } } volumesInfo.logger = logging.DefaultLogger(logrus.DebugLevel, logging.FormatJSON) volumesInfo.generateVolumeInfoForVeleroNativeSnapshot() require.Equal(t, tc.expectedVolumeInfos, volumesInfo.volumeInfos) }) } } func TestGenerateVolumeInfoForCSIVolumeSnapshot(t *testing.T) { resourceQuantity := resource.MustParse("100Gi") now := metav1.Now() readyToUse := true tests := []struct { name string volumeSnapshot snapshotv1api.VolumeSnapshot volumeSnapshotContent snapshotv1api.VolumeSnapshotContent volumeSnapshotClass snapshotv1api.VolumeSnapshotClass pvMap map[string]pvcPvInfo operation *itemoperation.BackupOperation expectedVolumeInfos []*BackupVolumeInfo }{ { name: "VS doesn't have VolumeSnapshotClass name", volumeSnapshot: snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: "testVS", Namespace: "velero", }, Spec: snapshotv1api.VolumeSnapshotSpec{}, }, expectedVolumeInfos: []*BackupVolumeInfo{}, }, { name: "VS doesn't have status", volumeSnapshot: snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: "testVS", Namespace: "velero", }, Spec: snapshotv1api.VolumeSnapshotSpec{ VolumeSnapshotClassName: stringPtr("testClass"), }, }, expectedVolumeInfos: []*BackupVolumeInfo{}, }, { name: "VS doesn't have PVC", volumeSnapshot: snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: "testVS", Namespace: "velero", }, Spec: snapshotv1api.VolumeSnapshotSpec{ VolumeSnapshotClassName: stringPtr("testClass"), }, Status: &snapshotv1api.VolumeSnapshotStatus{ BoundVolumeSnapshotContentName: stringPtr("testContent"), }, }, expectedVolumeInfos: []*BackupVolumeInfo{}, }, { name: "Cannot find VSC for VS", volumeSnapshot: snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: "testVS", Namespace: "velero", }, Spec: snapshotv1api.VolumeSnapshotSpec{ VolumeSnapshotClassName: stringPtr("testClass"), Source: snapshotv1api.VolumeSnapshotSource{ PersistentVolumeClaimName: stringPtr("testPVC"), }, }, Status: &snapshotv1api.VolumeSnapshotStatus{ BoundVolumeSnapshotContentName: stringPtr("testContent"), }, }, expectedVolumeInfos: []*BackupVolumeInfo{}, }, { name: "Cannot find BackupVolumeInfo for PVC", volumeSnapshot: snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: "testVS", Namespace: "velero", }, Spec: snapshotv1api.VolumeSnapshotSpec{ VolumeSnapshotClassName: stringPtr("testClass"), Source: snapshotv1api.VolumeSnapshotSource{ PersistentVolumeClaimName: stringPtr("testPVC"), }, }, Status: &snapshotv1api.VolumeSnapshotStatus{ BoundVolumeSnapshotContentName: stringPtr("testContent"), }, }, volumeSnapshotContent: *builder.ForVolumeSnapshotContent("testContent").Status(&snapshotv1api.VolumeSnapshotContentStatus{SnapshotHandle: stringPtr("testSnapshotHandle")}).Result(), expectedVolumeInfos: []*BackupVolumeInfo{}, }, { name: "Normal VolumeSnapshot case", volumeSnapshot: snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: "testVS", Namespace: "velero", CreationTimestamp: now, }, Spec: snapshotv1api.VolumeSnapshotSpec{ VolumeSnapshotClassName: stringPtr("testClass"), Source: snapshotv1api.VolumeSnapshotSource{ PersistentVolumeClaimName: stringPtr("testPVC"), }, }, Status: &snapshotv1api.VolumeSnapshotStatus{ BoundVolumeSnapshotContentName: stringPtr("testContent"), CreationTime: &now, RestoreSize: &resourceQuantity, ReadyToUse: &readyToUse, }, }, volumeSnapshotContent: *builder.ForVolumeSnapshotContent("testContent").Driver("pd.csi.storage.gke.io").Status(&snapshotv1api.VolumeSnapshotContentStatus{SnapshotHandle: stringPtr("testSnapshotHandle")}).Result(), pvMap: map[string]pvcPvInfo{ "testPV": { PVCName: "testPVC", PVCNamespace: "velero", PV: corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "testPV", Labels: map[string]string{"a": "b"}, }, Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeReclaimPolicy: corev1api.PersistentVolumeReclaimDelete, }, }, }, }, operation: &itemoperation.BackupOperation{ Spec: itemoperation.BackupOperationSpec{ OperationID: "testID", ResourceIdentifier: velero.ResourceIdentifier{ GroupResource: schema.GroupResource{ Group: "snapshot.storage.k8s.io", Resource: "volumesnapshots", }, Namespace: "velero", Name: "testVS", }, }, }, expectedVolumeInfos: []*BackupVolumeInfo{ { PVCName: "testPVC", PVCNamespace: "velero", PVName: "testPV", BackupMethod: CSISnapshot, StartTimestamp: &now, PreserveLocalSnapshot: true, CSISnapshotInfo: &CSISnapshotInfo{ Driver: "pd.csi.storage.gke.io", SnapshotHandle: "testSnapshotHandle", Size: 107374182400, VSCName: "testContent", OperationID: "testID", ReadyToUse: &readyToUse, }, PVInfo: &PVInfo{ ReclaimPolicy: "Delete", Labels: map[string]string{ "a": "b", }, }, }, }, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { volumesInfo := BackupVolumesInformation{} volumesInfo.Init() if tc.pvMap != nil { for k, v := range tc.pvMap { if k == v.PV.Name { volumesInfo.pvMap.insert(v.PV, v.PVCName, v.PVCNamespace) } } } if tc.operation != nil { volumesInfo.BackupOperations = append(volumesInfo.BackupOperations, tc.operation) } volumesInfo.volumeSnapshots = []snapshotv1api.VolumeSnapshot{tc.volumeSnapshot} volumesInfo.volumeSnapshotContents = []snapshotv1api.VolumeSnapshotContent{tc.volumeSnapshotContent} volumesInfo.volumeSnapshotClasses = []snapshotv1api.VolumeSnapshotClass{tc.volumeSnapshotClass} volumesInfo.logger = logging.DefaultLogger(logrus.DebugLevel, logging.FormatJSON) volumesInfo.generateVolumeInfoForCSIVolumeSnapshot() require.Equal(t, tc.expectedVolumeInfos, volumesInfo.volumeInfos) }) } } func TestGenerateVolumeInfoFromPVB(t *testing.T) { now := metav1.Now() tests := []struct { name string pvb *velerov1api.PodVolumeBackup pod *corev1api.Pod pvMap map[string]pvcPvInfo expectedVolumeInfos []*BackupVolumeInfo }{ { name: "cannot find PVB's pod, should fail", pvb: builder.ForPodVolumeBackup("velero", "testPVB").PodName("testPod").PodNamespace("velero").Result(), expectedVolumeInfos: []*BackupVolumeInfo{}, }, { name: "PVB doesn't have a related PVC", pvb: builder.ForPodVolumeBackup("velero", "testPVB").PodName("testPod").PodNamespace("velero").Result(), pod: builder.ForPod("velero", "testPod").Containers(&corev1api.Container{ Name: "test", VolumeMounts: []corev1api.VolumeMount{ { Name: "testVolume", MountPath: "/data", }, }, }).Volumes( &corev1api.Volume{ Name: "", VolumeSource: corev1api.VolumeSource{ HostPath: &corev1api.HostPathVolumeSource{}, }, }, ).Result(), expectedVolumeInfos: []*BackupVolumeInfo{ { PVCName: "", PVCNamespace: "", PVName: "", BackupMethod: PodVolumeBackup, Result: VolumeResultFailed, PVBInfo: &PodVolumeInfo{ PodName: "testPod", PodNamespace: "velero", }, }, }, }, { name: "Backup doesn't have information for PVC", pvb: builder.ForPodVolumeBackup("velero", "testPVB").PodName("testPod").PodNamespace("velero").Result(), pod: builder.ForPod("velero", "testPod").Containers(&corev1api.Container{ Name: "test", VolumeMounts: []corev1api.VolumeMount{ { Name: "testVolume", MountPath: "/data", }, }, }).Volumes( &corev1api.Volume{ Name: "", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "testPVC", }, }, }, ).Result(), expectedVolumeInfos: []*BackupVolumeInfo{}, }, { name: "PVB's volume has a PVC with failed phase", pvMap: map[string]pvcPvInfo{ "testPV": { PVCName: "testPVC", PVCNamespace: "velero", PV: corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "testPV", Labels: map[string]string{"a": "b"}, }, Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeReclaimPolicy: corev1api.PersistentVolumeReclaimDelete, }, }, }, }, pvb: builder.ForPodVolumeBackup("velero", "testPVB"). PodName("testPod"). PodNamespace("velero"). StartTimestamp(&now). CompletionTimestamp(&now). Phase(velerov1api.PodVolumeBackupPhaseFailed). Result(), pod: builder.ForPod("velero", "testPod").Containers(&corev1api.Container{ Name: "test", VolumeMounts: []corev1api.VolumeMount{ { Name: "testVolume", MountPath: "/data", }, }, }).Volumes( &corev1api.Volume{ Name: "", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "testPVC", }, }, }, ).Result(), expectedVolumeInfos: []*BackupVolumeInfo{ { PVCName: "testPVC", PVCNamespace: "velero", PVName: "testPV", BackupMethod: PodVolumeBackup, StartTimestamp: &now, CompletionTimestamp: &now, Result: VolumeResultFailed, PVBInfo: &PodVolumeInfo{ PodName: "testPod", PodNamespace: "velero", Phase: velerov1api.PodVolumeBackupPhaseFailed, }, PVInfo: &PVInfo{ ReclaimPolicy: string(corev1api.PersistentVolumeReclaimDelete), Labels: map[string]string{"a": "b"}, }, }, }, }, { name: "PVB's volume has a PVC", pvMap: map[string]pvcPvInfo{ "testPV": { PVCName: "testPVC", PVCNamespace: "velero", PV: corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "testPV", Labels: map[string]string{"a": "b"}, }, Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeReclaimPolicy: corev1api.PersistentVolumeReclaimDelete, }, }, }, }, pvb: builder.ForPodVolumeBackup("velero", "testPVB"). PodName("testPod"). PodNamespace("velero"). StartTimestamp(&now). CompletionTimestamp(&now). Phase(velerov1api.PodVolumeBackupPhaseCompleted). Result(), pod: builder.ForPod("velero", "testPod").Containers(&corev1api.Container{ Name: "test", VolumeMounts: []corev1api.VolumeMount{ { Name: "testVolume", MountPath: "/data", }, }, }).Volumes( &corev1api.Volume{ Name: "", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "testPVC", }, }, }, ).Result(), expectedVolumeInfos: []*BackupVolumeInfo{ { PVCName: "testPVC", PVCNamespace: "velero", PVName: "testPV", BackupMethod: PodVolumeBackup, StartTimestamp: &now, CompletionTimestamp: &now, Result: VolumeResultSucceeded, PVBInfo: &PodVolumeInfo{ PodName: "testPod", PodNamespace: "velero", Phase: velerov1api.PodVolumeBackupPhaseCompleted, }, PVInfo: &PVInfo{ ReclaimPolicy: string(corev1api.PersistentVolumeReclaimDelete), Labels: map[string]string{"a": "b"}, }, }, }, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { volumesInfo := BackupVolumesInformation{} volumesInfo.Init() volumesInfo.crClient = velerotest.NewFakeControllerRuntimeClient(t) volumesInfo.PodVolumeBackups = append(volumesInfo.PodVolumeBackups, tc.pvb) if tc.pvMap != nil { for k, v := range tc.pvMap { if k == v.PV.Name { volumesInfo.pvMap.insert(v.PV, v.PVCName, v.PVCNamespace) } } } if tc.pod != nil { require.NoError(t, volumesInfo.crClient.Create(t.Context(), tc.pod)) } volumesInfo.logger = logging.DefaultLogger(logrus.DebugLevel, logging.FormatJSON) volumesInfo.generateVolumeInfoFromPVB() require.Equal(t, tc.expectedVolumeInfos, volumesInfo.volumeInfos) }) } } func TestGenerateVolumeInfoFromDataUpload(t *testing.T) { // The unstructured conversion will loose the time precision to second // level. To make test pass. Set the now precision at second at the // beginning. now := metav1.Now().Rfc3339Copy() features.Enable(velerov1api.CSIFeatureFlag) defer features.Disable(velerov1api.CSIFeatureFlag) tests := []struct { name string vs *snapshotv1api.VolumeSnapshot vsc *snapshotv1api.VolumeSnapshotContent dataUpload *velerov2alpha1.DataUpload operation *itemoperation.BackupOperation pvMap map[string]pvcPvInfo expectedVolumeInfos []*BackupVolumeInfo }{ { name: "Operation is not for PVC", operation: &itemoperation.BackupOperation{ Spec: itemoperation.BackupOperationSpec{ ResourceIdentifier: velero.ResourceIdentifier{ GroupResource: schema.GroupResource{ Group: "", Resource: "configmaps", }, }, }, }, expectedVolumeInfos: []*BackupVolumeInfo{}, }, { name: "Operation doesn't have DataUpload PostItemOperation", operation: &itemoperation.BackupOperation{ Spec: itemoperation.BackupOperationSpec{ ResourceIdentifier: velero.ResourceIdentifier{ GroupResource: schema.GroupResource{ Group: "", Resource: "persistentvolumeclaims", }, Namespace: "velero", Name: "testPVC", }, PostOperationItems: []velero.ResourceIdentifier{ { GroupResource: schema.GroupResource{ Group: "", Resource: "configmaps", }, }, }, }, }, expectedVolumeInfos: []*BackupVolumeInfo{}, }, { name: "DataUpload cannot be found for operation", operation: &itemoperation.BackupOperation{ Spec: itemoperation.BackupOperationSpec{ OperationID: "testOperation", ResourceIdentifier: velero.ResourceIdentifier{ GroupResource: schema.GroupResource{ Group: "", Resource: "persistentvolumeclaims", }, Namespace: "velero", Name: "testPVC", }, PostOperationItems: []velero.ResourceIdentifier{ { GroupResource: schema.GroupResource{ Group: "velero.io", Resource: "datauploads", }, Namespace: "velero", Name: "testDU", }, }, }, }, expectedVolumeInfos: []*BackupVolumeInfo{}, }, { name: "Normal DataUpload case", dataUpload: builder.ForDataUpload("velero", "testDU"). DataMover("velero"). CSISnapshot(&velerov2alpha1.CSISnapshotSpec{ VolumeSnapshot: "vs-01", SnapshotClass: "testClass", Driver: "pd.csi.storage.gke.io", }).SnapshotID("testSnapshotHandle"). StartTimestamp(&now). Phase(velerov2alpha1.DataUploadPhaseCompleted). Result(), vs: builder.ForVolumeSnapshot(velerov1api.DefaultNamespace, "vs-01").Status().BoundVolumeSnapshotContentName("vsc-01").Result(), vsc: builder.ForVolumeSnapshotContent("vsc-01").Driver("pd.csi.storage.gke.io").Result(), operation: &itemoperation.BackupOperation{ Spec: itemoperation.BackupOperationSpec{ OperationID: "testOperation", ResourceIdentifier: velero.ResourceIdentifier{ GroupResource: schema.GroupResource{ Group: "", Resource: "persistentvolumeclaims", }, Namespace: "velero", Name: "testPVC", }, PostOperationItems: []velero.ResourceIdentifier{ { GroupResource: schema.GroupResource{ Group: "velero.io", Resource: "datauploads", }, Namespace: "velero", Name: "testDU", }, }, }, }, pvMap: map[string]pvcPvInfo{ "testPV": { PVCName: "testPVC", PVCNamespace: "velero", PV: corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "testPV", Labels: map[string]string{"a": "b"}, }, Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeReclaimPolicy: corev1api.PersistentVolumeReclaimDelete, }, }, }, }, expectedVolumeInfos: []*BackupVolumeInfo{ { PVCName: "testPVC", PVCNamespace: "velero", PVName: "testPV", BackupMethod: CSISnapshot, SnapshotDataMoved: true, StartTimestamp: &now, CSISnapshotInfo: &CSISnapshotInfo{ VSCName: FieldValueIsUnknown, SnapshotHandle: FieldValueIsUnknown, OperationID: FieldValueIsUnknown, Size: 0, Driver: "pd.csi.storage.gke.io", }, SnapshotDataMovementInfo: &SnapshotDataMovementInfo{ DataMover: "velero", UploaderType: "kopia", OperationID: "testOperation", Phase: velerov2alpha1.DataUploadPhaseCompleted, }, PVInfo: &PVInfo{ ReclaimPolicy: string(corev1api.PersistentVolumeReclaimDelete), Labels: map[string]string{"a": "b"}, }, }, }, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { volumesInfo := BackupVolumesInformation{} volumesInfo.Init() if tc.operation != nil { volumesInfo.BackupOperations = append(volumesInfo.BackupOperations, tc.operation) } if tc.pvMap != nil { for k, v := range tc.pvMap { if k == v.PV.Name { volumesInfo.pvMap.insert(v.PV, v.PVCName, v.PVCNamespace) } } } objects := make([]runtime.Object, 0) if tc.dataUpload != nil { objects = append(objects, tc.dataUpload) } if tc.vs != nil { objects = append(objects, tc.vs) } if tc.vsc != nil { objects = append(objects, tc.vsc) } volumesInfo.crClient = velerotest.NewFakeControllerRuntimeClient(t, objects...) volumesInfo.logger = logging.DefaultLogger(logrus.DebugLevel, logging.FormatJSON) volumesInfo.generateVolumeInfoFromDataUpload() if len(tc.expectedVolumeInfos) > 0 { require.Equal(t, tc.expectedVolumeInfos[0].PVInfo, volumesInfo.volumeInfos[0].PVInfo) require.Equal(t, tc.expectedVolumeInfos[0].SnapshotDataMovementInfo, volumesInfo.volumeInfos[0].SnapshotDataMovementInfo) require.Equal(t, tc.expectedVolumeInfos[0].CSISnapshotInfo, volumesInfo.volumeInfos[0].CSISnapshotInfo) } }) } } func TestRestoreVolumeInfoTrackNativeSnapshot(t *testing.T) { fakeCilent := velerotest.NewFakeControllerRuntimeClient(t) restore := builder.ForRestore("velero", "testRestore").Result() tracker := NewRestoreVolInfoTracker(restore, logrus.New(), fakeCilent) tracker.TrackNativeSnapshot("testPV", "snap-001", "ebs", "us-west-1", 10000) assert.Equal(t, NativeSnapshotInfo{ SnapshotHandle: "snap-001", VolumeType: "ebs", VolumeAZ: "us-west-1", IOPS: "10000", }, *tracker.pvNativeSnapshotMap["testPV"]) tracker.TrackNativeSnapshot("testPV", "snap-002", "ebs", "us-west-2", 15000) assert.Equal(t, NativeSnapshotInfo{ SnapshotHandle: "snap-002", VolumeType: "ebs", VolumeAZ: "us-west-2", IOPS: "15000", }, *tracker.pvNativeSnapshotMap["testPV"]) tracker.RenamePVForNativeSnapshot("testPV", "newPV") _, ok := tracker.pvNativeSnapshotMap["testPV"] assert.False(t, ok) assert.Equal(t, NativeSnapshotInfo{ SnapshotHandle: "snap-002", VolumeType: "ebs", VolumeAZ: "us-west-2", IOPS: "15000", }, *tracker.pvNativeSnapshotMap["newPV"]) } func TestRestoreVolumeInfoResult(t *testing.T) { fakeClient := velerotest.NewFakeControllerRuntimeClient(t, builder.ForPod("testNS", "testPod"). Volumes(builder.ForVolume("data-volume-1").PersistentVolumeClaimSource("testPVC2").Result()). Result()) testRestore := builder.ForRestore("velero", "testRestore").Result() tests := []struct { name string tracker *RestoreVolumeInfoTracker expectResultValues []RestoreVolumeInfo }{ { name: "empty", tracker: &RestoreVolumeInfoTracker{ Mutex: &sync.Mutex{}, client: fakeClient, log: logrus.New(), restore: testRestore, pvPvc: &pvcPvMap{ data: make(map[string]pvcPvInfo), }, pvNativeSnapshotMap: map[string]*NativeSnapshotInfo{}, pvcCSISnapshotMap: map[string]snapshotv1api.VolumeSnapshot{}, datadownloadList: &velerov2alpha1.DataDownloadList{}, pvrs: []*velerov1api.PodVolumeRestore{}, }, expectResultValues: []RestoreVolumeInfo{}, }, { name: "native snapshot and podvolumes", tracker: &RestoreVolumeInfoTracker{ Mutex: &sync.Mutex{}, client: fakeClient, log: logrus.New(), restore: testRestore, pvPvc: &pvcPvMap{ data: map[string]pvcPvInfo{ "testPV": { PVCName: "testPVC", PVCNamespace: "testNS", PV: *builder.ForPersistentVolume("testPV").Result(), }, "testPV2": { PVCName: "testPVC2", PVCNamespace: "testNS", PV: *builder.ForPersistentVolume("testPV2").Result(), }, }, }, pvNativeSnapshotMap: map[string]*NativeSnapshotInfo{ "testPV": { SnapshotHandle: "snap-001", VolumeType: "ebs", VolumeAZ: "us-west-1", IOPS: "10000", }, }, pvcCSISnapshotMap: map[string]snapshotv1api.VolumeSnapshot{}, datadownloadList: &velerov2alpha1.DataDownloadList{}, pvrs: []*velerov1api.PodVolumeRestore{ builder.ForPodVolumeRestore("velero", "testRestore-1234"). PodNamespace("testNS"). PodName("testPod"). Volume("data-volume-1"). UploaderType("kopia"). SnapshotID("pvr-snap-001").Result(), }, }, expectResultValues: []RestoreVolumeInfo{ { PVCName: "testPVC2", PVCNamespace: "testNS", PVName: "testPV2", RestoreMethod: PodVolumeRestore, SnapshotDataMoved: false, PVRInfo: &PodVolumeInfo{ SnapshotHandle: "pvr-snap-001", PodName: "testPod", PodNamespace: "testNS", UploaderType: "kopia", VolumeName: "data-volume-1", }, }, { PVCName: "testPVC", PVCNamespace: "testNS", PVName: "testPV", RestoreMethod: NativeSnapshot, SnapshotDataMoved: false, NativeSnapshotInfo: &NativeSnapshotInfo{ SnapshotHandle: "snap-001", VolumeType: "ebs", VolumeAZ: "us-west-1", IOPS: "10000", }, }, }, }, { name: "CSI snapshot without datamovement and podvolumes", tracker: &RestoreVolumeInfoTracker{ Mutex: &sync.Mutex{}, client: fakeClient, log: logrus.New(), restore: testRestore, pvPvc: &pvcPvMap{ data: map[string]pvcPvInfo{ "testPV": { PVCName: "testPVC", PVCNamespace: "testNS", PV: *builder.ForPersistentVolume("testPV").Result(), }, "testPV2": { PVCName: "testPVC2", PVCNamespace: "testNS", PV: *builder.ForPersistentVolume("testPV2").Result(), }, }, }, pvNativeSnapshotMap: map[string]*NativeSnapshotInfo{}, pvcCSISnapshotMap: map[string]snapshotv1api.VolumeSnapshot{ "testNS/testPVC": *builder.ForVolumeSnapshot("sourceNS", "testCSISnapshot"). ObjectMeta( builder.WithAnnotations(velerov1api.VolumeSnapshotHandleAnnotation, "csi-snap-001", velerov1api.DriverNameAnnotation, "test-csi-driver"), ).SourceVolumeSnapshotContentName("test-vsc-001"). Status().RestoreSize("1Gi").Result(), }, datadownloadList: &velerov2alpha1.DataDownloadList{}, pvrs: []*velerov1api.PodVolumeRestore{ builder.ForPodVolumeRestore("velero", "testRestore-1234"). PodNamespace("testNS"). PodName("testPod"). Volume("data-volume-1"). UploaderType("kopia"). SnapshotID("pvr-snap-001").Result(), }, }, expectResultValues: []RestoreVolumeInfo{ { PVCName: "testPVC2", PVCNamespace: "testNS", PVName: "testPV2", RestoreMethod: PodVolumeRestore, SnapshotDataMoved: false, PVRInfo: &PodVolumeInfo{ SnapshotHandle: "pvr-snap-001", PodName: "testPod", PodNamespace: "testNS", UploaderType: "kopia", VolumeName: "data-volume-1", }, }, { PVCName: "testPVC", PVCNamespace: "testNS", PVName: "testPV", RestoreMethod: CSISnapshot, SnapshotDataMoved: false, CSISnapshotInfo: &CSISnapshotInfo{ SnapshotHandle: "csi-snap-001", VSCName: "test-vsc-001", Size: 1073741824, Driver: "test-csi-driver", }, }, }, }, { name: "CSI snapshot with datamovement", tracker: &RestoreVolumeInfoTracker{ Mutex: &sync.Mutex{}, client: fakeClient, log: logrus.New(), restore: testRestore, pvPvc: &pvcPvMap{ data: map[string]pvcPvInfo{ "testPV": { PVCName: "testPVC", PVCNamespace: "testNS", PV: *builder.ForPersistentVolume("testPV").Result(), }, "testPV2": { PVCName: "testPVC2", PVCNamespace: "testNS", PV: *builder.ForPersistentVolume("testPV2").Result(), }, }, }, pvNativeSnapshotMap: map[string]*NativeSnapshotInfo{}, pvcCSISnapshotMap: map[string]snapshotv1api.VolumeSnapshot{}, datadownloadList: &velerov2alpha1.DataDownloadList{ Items: []velerov2alpha1.DataDownload{ *builder.ForDataDownload("velero", "testDataDownload-1"). ObjectMeta(builder.WithLabels(velerov1api.AsyncOperationIDLabel, "dd-operation-001")). SnapshotID("dd-snap-001"). TargetVolume(velerov2alpha1.TargetVolumeSpec{ PVC: "testPVC", Namespace: "testNS", }). Result(), *builder.ForDataDownload("velero", "testDataDownload-2"). ObjectMeta(builder.WithLabels(velerov1api.AsyncOperationIDLabel, "dd-operation-002")). SnapshotID("dd-snap-002"). TargetVolume(velerov2alpha1.TargetVolumeSpec{ PVC: "testPVC2", Namespace: "testNS", }). Result(), }, }, pvrs: []*velerov1api.PodVolumeRestore{}, }, expectResultValues: []RestoreVolumeInfo{ { PVCName: "testPVC", PVCNamespace: "testNS", PVName: "testPV", RestoreMethod: CSISnapshot, SnapshotDataMoved: true, SnapshotDataMovementInfo: &SnapshotDataMovementInfo{ DataMover: "velero", UploaderType: velerov1api.BackupRepositoryTypeKopia, SnapshotHandle: "dd-snap-001", OperationID: "dd-operation-001", }, }, { PVCName: "testPVC2", PVCNamespace: "testNS", PVName: "testPV2", RestoreMethod: CSISnapshot, SnapshotDataMoved: true, SnapshotDataMovementInfo: &SnapshotDataMovementInfo{ DataMover: "velero", UploaderType: velerov1api.BackupRepositoryTypeKopia, SnapshotHandle: "dd-snap-002", OperationID: "dd-operation-002", }, }, }, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { result := tc.tracker.Result() valuesList := []RestoreVolumeInfo{} for _, item := range result { valuesList = append(valuesList, *item) } assert.Equal(t, tc.expectResultValues, valuesList) }) } } func stringPtr(str string) *string { return &str } func int64Ptr(val int) *int64 { i := int64(val) return &i } func TestGetVolumeSnapshotClasses(t *testing.T) { class := &snapshotv1api.VolumeSnapshotClass{ ObjectMeta: metav1.ObjectMeta{ Name: "class", ResourceVersion: "999", }, } volumesInfo := BackupVolumesInformation{ logger: logging.DefaultLogger(logrus.DebugLevel, logging.FormatJSON), crClient: velerotest.NewFakeControllerRuntimeClient(t, class), } result, err := volumesInfo.getVolumeSnapshotClasses() require.NoError(t, err) require.Equal(t, []snapshotv1api.VolumeSnapshotClass{*class}, result) } ================================================ FILE: internal/volumehelper/volume_policy_helper.go ================================================ package volumehelper import ( "context" "fmt" "strings" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" crclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/internal/resourcepolicies" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/util/boolptr" kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube" podvolumeutil "github.com/vmware-tanzu/velero/pkg/util/podvolume" ) type VolumeHelper interface { ShouldPerformSnapshot(obj runtime.Unstructured, groupResource schema.GroupResource) (bool, error) ShouldPerformFSBackup(volume corev1api.Volume, pod corev1api.Pod) (bool, error) } type volumeHelperImpl struct { volumePolicy *resourcepolicies.Policies snapshotVolumes *bool logger logrus.FieldLogger client crclient.Client defaultVolumesToFSBackup bool // This parameter is used to align the fs-backup with snapshot action, // because PVC is already filtered by the resource filter before getting // to the volume policy check, but fs-backup is based on the pod resource, // the resource filter on PVC and PV doesn't work on this scenario. backupExcludePVC bool // pvcPodCache provides cached PVC to Pod mappings for improved performance. // When there are many PVCs and pods, using this cache avoids O(N*M) lookups. pvcPodCache *podvolumeutil.PVCPodCache } // NewVolumeHelperImpl creates a VolumeHelper without PVC-to-Pod caching. // // Deprecated: Use NewVolumeHelperImplWithNamespaces or NewVolumeHelperImplWithCache instead // for better performance. These functions provide PVC-to-Pod caching which avoids O(N*M) // complexity when there are many PVCs and pods. See issue #9179 for details. func NewVolumeHelperImpl( volumePolicy *resourcepolicies.Policies, snapshotVolumes *bool, logger logrus.FieldLogger, client crclient.Client, defaultVolumesToFSBackup bool, backupExcludePVC bool, ) VolumeHelper { // Pass nil namespaces - no cache will be built, so this never fails. // This is used by plugins that don't need the cache optimization. vh, _ := NewVolumeHelperImplWithNamespaces( volumePolicy, snapshotVolumes, logger, client, defaultVolumesToFSBackup, backupExcludePVC, nil, ) return vh } // NewVolumeHelperImplWithNamespaces creates a VolumeHelper with a PVC-to-Pod cache for improved performance. // The cache is built internally from the provided namespaces list. // This avoids O(N*M) complexity when there are many PVCs and pods. // See issue #9179 for details. // Returns an error if cache building fails - callers should not proceed with backup in this case. func NewVolumeHelperImplWithNamespaces( volumePolicy *resourcepolicies.Policies, snapshotVolumes *bool, logger logrus.FieldLogger, client crclient.Client, defaultVolumesToFSBackup bool, backupExcludePVC bool, namespaces []string, ) (VolumeHelper, error) { var pvcPodCache *podvolumeutil.PVCPodCache if len(namespaces) > 0 { pvcPodCache = podvolumeutil.NewPVCPodCache() if err := pvcPodCache.BuildCacheForNamespaces(context.Background(), namespaces, client); err != nil { return nil, err } logger.Infof("Built PVC-to-Pod cache for %d namespaces", len(namespaces)) } return &volumeHelperImpl{ volumePolicy: volumePolicy, snapshotVolumes: snapshotVolumes, logger: logger, client: client, defaultVolumesToFSBackup: defaultVolumesToFSBackup, backupExcludePVC: backupExcludePVC, pvcPodCache: pvcPodCache, }, nil } // NewVolumeHelperImplWithCache creates a VolumeHelper using an externally managed PVC-to-Pod cache. // This is used by plugins that build the cache lazily per-namespace (following the pattern from PR #9226). // The cache can be nil, in which case PVC-to-Pod lookups will fall back to direct API calls. func NewVolumeHelperImplWithCache( backup velerov1api.Backup, client crclient.Client, logger logrus.FieldLogger, pvcPodCache *podvolumeutil.PVCPodCache, ) (VolumeHelper, error) { resourcePolicies, err := resourcepolicies.GetResourcePoliciesFromBackup(backup, client, logger) if err != nil { return nil, errors.Wrap(err, "failed to get volume policies from backup") } return &volumeHelperImpl{ volumePolicy: resourcePolicies, snapshotVolumes: backup.Spec.SnapshotVolumes, logger: logger, client: client, defaultVolumesToFSBackup: boolptr.IsSetToTrue(backup.Spec.DefaultVolumesToFsBackup), backupExcludePVC: boolptr.IsSetToTrue(backup.Spec.SnapshotMoveData), pvcPodCache: pvcPodCache, }, nil } func (v *volumeHelperImpl) ShouldPerformSnapshot(obj runtime.Unstructured, groupResource schema.GroupResource) (bool, error) { // check if volume policy exists and also check if the object(pv/pvc) fits a volume policy criteria and see if the associated action is snapshot // if it is not snapshot then skip the code path for snapshotting the PV/PVC pvc := new(corev1api.PersistentVolumeClaim) pv := new(corev1api.PersistentVolume) var err error var pvNotFoundErr error if groupResource == kuberesource.PersistentVolumeClaims { if err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), &pvc); err != nil { v.logger.WithError(err).Error("fail to convert unstructured into PVC") return false, err } pv, err = kubeutil.GetPVForPVC(pvc, v.client) if err != nil { // Any error means PV not available - save to return later if no policy matches v.logger.Debugf("PV not found for PVC %s: %v", pvc.Namespace+"/"+pvc.Name, err) pvNotFoundErr = err pv = nil } } if groupResource == kuberesource.PersistentVolumes { if err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), &pv); err != nil { v.logger.WithError(err).Error("fail to convert unstructured into PV") return false, err } } if v.volumePolicy != nil { vfd := resourcepolicies.NewVolumeFilterData(pv, nil, pvc) action, err := v.volumePolicy.GetMatchAction(vfd) if err != nil { v.logger.WithError(err).Errorf("fail to get VolumePolicy match action for %+v", vfd) return false, err } // If there is a match action, and the action type is snapshot, return true, // or the action type is not snapshot, then return false. // If there is no match action, go on to the next check. if action != nil { if action.Type == resourcepolicies.Snapshot { v.logger.Infof("performing snapshot action for %+v", vfd) return true, nil } else { v.logger.Infof("Skip snapshot action for %+v as the action type is %s", vfd, action.Type) return false, nil } } } // If resource is PVC, and PV is nil (e.g., Pending/Lost PVC with no matching policy), return the original error if groupResource == kuberesource.PersistentVolumeClaims && pv == nil && pvNotFoundErr != nil { v.logger.WithError(pvNotFoundErr).Errorf("fail to get PV for PVC %s", pvc.Namespace+"/"+pvc.Name) return false, pvNotFoundErr } // If this PV is claimed, see if we've already taken a (pod volume backup) // snapshot of the contents of this PV. If so, don't take a snapshot. if pv.Spec.ClaimRef != nil { // Use cached lookup if available for better performance with many PVCs/pods pods, err := podvolumeutil.GetPodsUsingPVCWithCache( pv.Spec.ClaimRef.Namespace, pv.Spec.ClaimRef.Name, v.client, v.pvcPodCache, ) if err != nil { v.logger.WithError(err).Errorf("fail to get pod for PV %s", pv.Name) return false, err } for _, pod := range pods { for _, vol := range pod.Spec.Volumes { if vol.PersistentVolumeClaim != nil && vol.PersistentVolumeClaim.ClaimName == pv.Spec.ClaimRef.Name && v.shouldPerformFSBackupLegacy(vol, pod) { v.logger.Infof("Skipping snapshot of pv %s because it is backed up with PodVolumeBackup.", pv.Name) return false, nil } } } } if !boolptr.IsSetToFalse(v.snapshotVolumes) { // If the backup.Spec.SnapshotVolumes is not set, or set to true, then should take the snapshot. v.logger.Infof("performing snapshot action for pv %s as the snapshotVolumes is not set to false", pv.Name) return true, nil } v.logger.Infof("skipping snapshot action for pv %s possibly due to no volume policy setting or snapshotVolumes is false", pv.Name) return false, nil } func (v volumeHelperImpl) ShouldPerformFSBackup(volume corev1api.Volume, pod corev1api.Pod) (bool, error) { if !v.shouldIncludeVolumeInBackup(volume) { v.logger.Debugf("skip fs-backup action for pod %s's volume %s, due to not pass volume check.", pod.Namespace+"/"+pod.Name, volume.Name) return false, nil } var pvNotFoundErr error if v.volumePolicy != nil { var resource any var err error resource = &volume var pvc = &corev1api.PersistentVolumeClaim{} if volume.VolumeSource.PersistentVolumeClaim != nil { pvc, err = kubeutil.GetPVCForPodVolume(&volume, &pod, v.client) if err != nil { v.logger.WithError(err).Errorf("fail to get PVC for pod %s", pod.Namespace+"/"+pod.Name) return false, err } pvResource, err := kubeutil.GetPVForPVC(pvc, v.client) if err != nil { // Any error means PV not available - save to return later if no policy matches v.logger.Debugf("PV not found for PVC %s: %v", pvc.Namespace+"/"+pvc.Name, err) pvNotFoundErr = err } else { resource = pvResource } } pv, podVolume, err := v.getVolumeFromResource(resource) if err != nil { return false, err } vfd := resourcepolicies.NewVolumeFilterData(pv, podVolume, pvc) action, err := v.volumePolicy.GetMatchAction(vfd) if err != nil { v.logger.WithError(err).Error("fail to get VolumePolicy match action for volume") return false, err } if action != nil { if action.Type == resourcepolicies.FSBackup { v.logger.Infof("Perform fs-backup action for volume %s of pod %s due to volume policy match", volume.Name, pod.Namespace+"/"+pod.Name) return true, nil } else { v.logger.Infof("Skip fs-backup action for volume %s for pod %s because the action type is %s", volume.Name, pod.Namespace+"/"+pod.Name, action.Type) return false, nil } } // If no policy matched and PV was not found, return the original error if pvNotFoundErr != nil { v.logger.WithError(pvNotFoundErr).Errorf("fail to get PV for PVC %s", pvc.Namespace+"/"+pvc.Name) return false, pvNotFoundErr } } if v.shouldPerformFSBackupLegacy(volume, pod) { v.logger.Infof("Perform fs-backup action for volume %s of pod %s due to opt-in/out way", volume.Name, pod.Namespace+"/"+pod.Name) return true, nil } else { v.logger.Infof("Skip fs-backup action for volume %s of pod %s due to opt-in/out way", volume.Name, pod.Namespace+"/"+pod.Name) return false, nil } } func (v volumeHelperImpl) shouldPerformFSBackupLegacy( volume corev1api.Volume, pod corev1api.Pod, ) bool { // Check volume in opt-in way if !v.defaultVolumesToFSBackup { optInVolumeNames := podvolumeutil.GetVolumesToBackup(&pod) for _, volumeName := range optInVolumeNames { if volume.Name == volumeName { return true } } return false } else { // Check volume in opt-out way optOutVolumeNames := podvolumeutil.GetVolumesToExclude(&pod) for _, volumeName := range optOutVolumeNames { if volume.Name == volumeName { return false } } return true } } func (v *volumeHelperImpl) shouldIncludeVolumeInBackup(vol corev1api.Volume) bool { includeVolumeInBackup := true // cannot backup hostpath volumes as they are not mounted into /var/lib/kubelet/pods // and therefore not accessible to the node agent daemon set. if vol.HostPath != nil { includeVolumeInBackup = false } // don't backup volumes mounting secrets. Secrets will be backed up separately. if vol.Secret != nil { includeVolumeInBackup = false } // don't backup volumes mounting ConfigMaps. ConfigMaps will be backed up separately. if vol.ConfigMap != nil { includeVolumeInBackup = false } // don't backup volumes mounted as projected volumes, all data in those come from kube state. if vol.Projected != nil { includeVolumeInBackup = false } // don't backup DownwardAPI volumes, all data in those come from kube state. if vol.DownwardAPI != nil { includeVolumeInBackup = false } if vol.PersistentVolumeClaim != nil && v.backupExcludePVC { includeVolumeInBackup = false } // don't include volumes that mount the default service account token. if strings.HasPrefix(vol.Name, "default-token") { includeVolumeInBackup = false } return includeVolumeInBackup } func (v *volumeHelperImpl) getVolumeFromResource(resource any) (*corev1api.PersistentVolume, *corev1api.Volume, error) { if pv, ok := resource.(*corev1api.PersistentVolume); ok { return pv, nil, nil } else if podVol, ok := resource.(*corev1api.Volume); ok { return nil, podVol, nil } return nil, nil, fmt.Errorf("resource is not a PersistentVolume or Volume") } ================================================ FILE: internal/volumehelper/volume_policy_helper_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package volumehelper import ( "testing" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/utils/ptr" "github.com/vmware-tanzu/velero/internal/resourcepolicies" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/kuberesource" velerotest "github.com/vmware-tanzu/velero/pkg/test" podvolumeutil "github.com/vmware-tanzu/velero/pkg/util/podvolume" ) func TestVolumeHelperImpl_ShouldPerformSnapshot(t *testing.T) { testCases := []struct { name string inputObj runtime.Object groupResource schema.GroupResource pod *corev1api.Pod resourcePolicies *resourcepolicies.ResourcePolicies snapshotVolumesFlag *bool defaultVolumesToFSBackup bool shouldSnapshot bool expectedErr bool }{ { name: "VolumePolicy match, returns true and no error", inputObj: builder.ForPersistentVolume("example-pv").StorageClass("gp2-csi").ClaimRef("ns", "pvc-1").Result(), groupResource: kuberesource.PersistentVolumes, resourcePolicies: &resourcepolicies.ResourcePolicies{ Version: "v1", VolumePolicies: []resourcepolicies.VolumePolicy{ { Conditions: map[string]any{ "storageClass": []string{"gp2-csi"}, }, Action: resourcepolicies.Action{ Type: resourcepolicies.Snapshot, }, }, }, }, snapshotVolumesFlag: ptr.To(true), shouldSnapshot: true, expectedErr: false, }, { name: "VolumePolicy match, snapshotVolumes is false, return true and no error", inputObj: builder.ForPersistentVolume("example-pv").StorageClass("gp2-csi").ClaimRef("ns", "pvc-1").Result(), groupResource: kuberesource.PersistentVolumes, resourcePolicies: &resourcepolicies.ResourcePolicies{ Version: "v1", VolumePolicies: []resourcepolicies.VolumePolicy{ { Conditions: map[string]any{ "storageClass": []string{"gp2-csi"}, }, Action: resourcepolicies.Action{ Type: resourcepolicies.Snapshot, }, }, }, }, snapshotVolumesFlag: ptr.To(false), shouldSnapshot: true, expectedErr: false, }, { name: "VolumePolicy match but action is unexpected, return false and no error", inputObj: builder.ForPersistentVolume("example-pv").StorageClass("gp2-csi").ClaimRef("ns", "pvc-1").Result(), groupResource: kuberesource.PersistentVolumes, resourcePolicies: &resourcepolicies.ResourcePolicies{ Version: "v1", VolumePolicies: []resourcepolicies.VolumePolicy{ { Conditions: map[string]any{ "storageClass": []string{"gp2-csi"}, }, Action: resourcepolicies.Action{ Type: resourcepolicies.FSBackup, }, }, }, }, snapshotVolumesFlag: ptr.To(true), shouldSnapshot: false, expectedErr: false, }, { name: "VolumePolicy not match, not selected by fs-backup as opt-out way, snapshotVolumes is true, returns true and no error", inputObj: builder.ForPersistentVolume("example-pv").StorageClass("gp3-csi").ClaimRef("ns", "pvc-1").Result(), groupResource: kuberesource.PersistentVolumes, pod: builder.ForPod("ns", "pod-1").Result(), resourcePolicies: &resourcepolicies.ResourcePolicies{ Version: "v1", VolumePolicies: []resourcepolicies.VolumePolicy{ { Conditions: map[string]any{ "storageClass": []string{"gp2-csi"}, }, Action: resourcepolicies.Action{ Type: resourcepolicies.Snapshot, }, }, }, }, snapshotVolumesFlag: ptr.To(true), shouldSnapshot: true, expectedErr: false, }, { name: "VolumePolicy not match, selected by fs-backup as opt-out way, snapshotVolumes is true, returns false and no error", inputObj: builder.ForPersistentVolume("example-pv").StorageClass("gp3-csi").ClaimRef("ns", "pvc-1").Result(), groupResource: kuberesource.PersistentVolumes, pod: builder.ForPod("ns", "pod-1").Volumes( &corev1api.Volume{ Name: "volume", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "pvc-1", }, }, }, ).Result(), resourcePolicies: &resourcepolicies.ResourcePolicies{ Version: "v1", VolumePolicies: []resourcepolicies.VolumePolicy{ { Conditions: map[string]any{ "storageClass": []string{"gp2-csi"}, }, Action: resourcepolicies.Action{ Type: resourcepolicies.Snapshot, }, }, }, }, snapshotVolumesFlag: ptr.To(true), defaultVolumesToFSBackup: true, shouldSnapshot: false, expectedErr: false, }, { name: "VolumePolicy not match, selected by fs-backup as opt-out way, snapshotVolumes is true, returns false and no error", inputObj: builder.ForPersistentVolume("example-pv").StorageClass("gp3-csi").ClaimRef("ns", "pvc-1").Result(), groupResource: kuberesource.PersistentVolumes, pod: builder.ForPod("ns", "pod-1"). ObjectMeta(builder.WithAnnotations(velerov1api.VolumesToExcludeAnnotation, "volume")). Volumes( &corev1api.Volume{ Name: "volume", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "pvc-1", }, }, }, ).Result(), resourcePolicies: &resourcepolicies.ResourcePolicies{ Version: "v1", VolumePolicies: []resourcepolicies.VolumePolicy{ { Conditions: map[string]any{ "storageClass": []string{"gp2-csi"}, }, Action: resourcepolicies.Action{ Type: resourcepolicies.Snapshot, }, }, }, }, snapshotVolumesFlag: ptr.To(true), defaultVolumesToFSBackup: true, shouldSnapshot: true, expectedErr: false, }, { name: "VolumePolicy not match, not selected by fs-backup as opt-in way, snapshotVolumes is true, returns false and no error", inputObj: builder.ForPersistentVolume("example-pv").StorageClass("gp3-csi").ClaimRef("ns", "pvc-1").Result(), groupResource: kuberesource.PersistentVolumes, pod: builder.ForPod("ns", "pod-1"). ObjectMeta(builder.WithAnnotations(velerov1api.VolumesToBackupAnnotation, "volume")). Volumes( &corev1api.Volume{ Name: "volume", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "pvc-1", }, }, }, ).Result(), resourcePolicies: &resourcepolicies.ResourcePolicies{ Version: "v1", VolumePolicies: []resourcepolicies.VolumePolicy{ { Conditions: map[string]any{ "storageClass": []string{"gp2-csi"}, }, Action: resourcepolicies.Action{ Type: resourcepolicies.Snapshot, }, }, }, }, snapshotVolumesFlag: ptr.To(true), defaultVolumesToFSBackup: false, shouldSnapshot: false, expectedErr: false, }, { name: "VolumePolicy not match, not selected by fs-backup as opt-in way, snapshotVolumes is true, returns true and no error", inputObj: builder.ForPersistentVolume("example-pv").StorageClass("gp3-csi").ClaimRef("ns", "pvc-1").Result(), groupResource: kuberesource.PersistentVolumes, pod: builder.ForPod("ns", "pod-1"). Volumes( &corev1api.Volume{ Name: "volume", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "pvc-1", }, }, }, ).Result(), resourcePolicies: &resourcepolicies.ResourcePolicies{ Version: "v1", VolumePolicies: []resourcepolicies.VolumePolicy{ { Conditions: map[string]any{ "storageClass": []string{"gp2-csi"}, }, Action: resourcepolicies.Action{ Type: resourcepolicies.Snapshot, }, }, }, }, snapshotVolumesFlag: ptr.To(true), defaultVolumesToFSBackup: false, shouldSnapshot: true, expectedErr: false, }, { name: "No VolumePolicy, not selected by fs-backup, snapshotVolumes is true, returns true and no error", inputObj: builder.ForPersistentVolume("example-pv").StorageClass("gp3-csi").ClaimRef("ns", "pvc-1").Result(), groupResource: kuberesource.PersistentVolumes, resourcePolicies: nil, snapshotVolumesFlag: ptr.To(true), shouldSnapshot: true, expectedErr: false, }, { name: "No VolumePolicy, not selected by fs-backup, snapshotVolumes is false, returns false and no error", inputObj: builder.ForPersistentVolume("example-pv").StorageClass("gp3-csi").ClaimRef("ns", "pvc-1").Result(), groupResource: kuberesource.PersistentVolumes, resourcePolicies: nil, snapshotVolumesFlag: ptr.To(false), shouldSnapshot: false, expectedErr: false, }, { name: "PVC not having PV, return false and error when no matching policy", inputObj: builder.ForPersistentVolumeClaim("default", "example-pvc").StorageClass("gp2-csi").Result(), groupResource: kuberesource.PersistentVolumeClaims, resourcePolicies: &resourcepolicies.ResourcePolicies{ Version: "v1", }, snapshotVolumesFlag: ptr.To(true), shouldSnapshot: false, expectedErr: true, }, } objs := []runtime.Object{ &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: "ns", Name: "pvc-1", }, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { fakeClient := velerotest.NewFakeControllerRuntimeClient(t, objs...) if tc.pod != nil { fakeClient.Create(t.Context(), tc.pod) } var p *resourcepolicies.Policies if tc.resourcePolicies != nil { p = &resourcepolicies.Policies{} err := p.BuildPolicy(tc.resourcePolicies) if err != nil { t.Fatalf("failed to build policy with error %v", err) } } vh := NewVolumeHelperImpl( p, tc.snapshotVolumesFlag, logrus.StandardLogger(), fakeClient, tc.defaultVolumesToFSBackup, false, ) obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.inputObj) require.NoError(t, err) actualShouldSnapshot, actualError := vh.ShouldPerformSnapshot(&unstructured.Unstructured{Object: obj}, tc.groupResource) if tc.expectedErr { require.Error(t, actualError, "Want error; Got nil error") return } require.Equalf(t, tc.shouldSnapshot, actualShouldSnapshot, "Want shouldSnapshot as %t; Got shouldSnapshot as %t", tc.shouldSnapshot, actualShouldSnapshot) }) } } func TestVolumeHelperImpl_ShouldIncludeVolumeInBackup(t *testing.T) { testCases := []struct { name string vol corev1api.Volume backupExcludePVC bool shouldInclude bool }{ { name: "volume has host path so do not include", vol: corev1api.Volume{ Name: "sample-volume", VolumeSource: corev1api.VolumeSource{ HostPath: &corev1api.HostPathVolumeSource{ Path: "some-path", }, }, }, backupExcludePVC: false, shouldInclude: false, }, { name: "volume has secret mounted so do not include", vol: corev1api.Volume{ Name: "sample-volume", VolumeSource: corev1api.VolumeSource{ Secret: &corev1api.SecretVolumeSource{ SecretName: "sample-secret", Items: []corev1api.KeyToPath{ { Key: "username", Path: "my-username", }, }, }, }, }, backupExcludePVC: false, shouldInclude: false, }, { name: "volume has configmap so do not include", vol: corev1api.Volume{ Name: "sample-volume", VolumeSource: corev1api.VolumeSource{ ConfigMap: &corev1api.ConfigMapVolumeSource{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "sample-cm", }, }, }, }, backupExcludePVC: false, shouldInclude: false, }, { name: "volume is mounted as project volume so do not include", vol: corev1api.Volume{ Name: "sample-volume", VolumeSource: corev1api.VolumeSource{ Projected: &corev1api.ProjectedVolumeSource{ Sources: []corev1api.VolumeProjection{}, }, }, }, backupExcludePVC: false, shouldInclude: false, }, { name: "volume has downwardAPI so do not include", vol: corev1api.Volume{ Name: "sample-volume", VolumeSource: corev1api.VolumeSource{ DownwardAPI: &corev1api.DownwardAPIVolumeSource{ Items: []corev1api.DownwardAPIVolumeFile{ { Path: "labels", FieldRef: &corev1api.ObjectFieldSelector{ FieldPath: "metadata.labels", }, }, }, }, }, }, backupExcludePVC: false, shouldInclude: false, }, { name: "volume has pvc and backupExcludePVC is true so do not include", vol: corev1api.Volume{ Name: "sample-volume", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "sample-pvc", }, }, }, backupExcludePVC: true, shouldInclude: false, }, { name: "volume name has prefix default-token so do not include", vol: corev1api.Volume{ Name: "default-token-vol-name", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "sample-pvc", }, }, }, backupExcludePVC: false, shouldInclude: false, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { resourcePolicies := resourcepolicies.ResourcePolicies{ Version: "v1", VolumePolicies: []resourcepolicies.VolumePolicy{ { Conditions: map[string]any{ "storageClass": []string{"gp2-csi"}, }, Action: resourcepolicies.Action{ Type: resourcepolicies.Snapshot, }, }, }, } policies := resourcePolicies p := &resourcepolicies.Policies{} err := p.BuildPolicy(&policies) if err != nil { t.Fatalf("failed to build policy with error %v", err) } vh := &volumeHelperImpl{ volumePolicy: p, snapshotVolumes: ptr.To(true), logger: velerotest.NewLogger(), backupExcludePVC: tc.backupExcludePVC, } actualShouldInclude := vh.shouldIncludeVolumeInBackup(tc.vol) assert.Equalf(t, actualShouldInclude, tc.shouldInclude, "Want shouldInclude as %v; Got actualShouldInclude as %v", tc.shouldInclude, actualShouldInclude) }) } } func TestVolumeHelperImpl_ShouldPerformFSBackup(t *testing.T) { testCases := []struct { name string pod *corev1api.Pod resources []runtime.Object resourcePolicies *resourcepolicies.ResourcePolicies snapshotVolumesFlag *bool defaultVolumesToFSBackup bool shouldFSBackup bool expectedErr bool }{ { name: "HostPath volume should be skipped.", pod: builder.ForPod("ns", "pod-1"). Volumes( &corev1api.Volume{ Name: "", VolumeSource: corev1api.VolumeSource{ HostPath: &corev1api.HostPathVolumeSource{ Path: "/mnt/test", }, }, }).Result(), shouldFSBackup: false, expectedErr: false, }, { name: "VolumePolicy match, return true and no error", pod: builder.ForPod("ns", "pod-1"). Volumes( &corev1api.Volume{ Name: "", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "pvc-1", }, }, }).Result(), resources: []runtime.Object{ builder.ForPersistentVolumeClaim("ns", "pvc-1"). VolumeName("pv-1"). StorageClass("gp2-csi").Phase(corev1api.ClaimBound).Result(), builder.ForPersistentVolume("pv-1").StorageClass("gp2-csi").Result(), }, resourcePolicies: &resourcepolicies.ResourcePolicies{ Version: "v1", VolumePolicies: []resourcepolicies.VolumePolicy{ { Conditions: map[string]any{ "storageClass": []string{"gp2-csi"}, }, Action: resourcepolicies.Action{ Type: resourcepolicies.FSBackup, }, }, }, }, shouldFSBackup: true, expectedErr: false, }, { name: "Volume source is emptyDir, VolumePolicy match, return true and no error", pod: builder.ForPod("ns", "pod-1"). Volumes( &corev1api.Volume{ Name: "", VolumeSource: corev1api.VolumeSource{ EmptyDir: &corev1api.EmptyDirVolumeSource{}, }, }).Result(), resourcePolicies: &resourcepolicies.ResourcePolicies{ Version: "v1", VolumePolicies: []resourcepolicies.VolumePolicy{ { Conditions: map[string]any{ "volumeTypes": []string{"emptyDir"}, }, Action: resourcepolicies.Action{ Type: resourcepolicies.FSBackup, }, }, }, }, shouldFSBackup: true, expectedErr: false, }, { name: "VolumePolicy match, action type is not fs-backup, return false and no error", pod: builder.ForPod("ns", "pod-1"). Volumes( &corev1api.Volume{ Name: "", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "pvc-1", }, }, }).Result(), resources: []runtime.Object{ builder.ForPersistentVolumeClaim("ns", "pvc-1"). VolumeName("pv-1"). StorageClass("gp2-csi").Phase(corev1api.ClaimBound).Result(), builder.ForPersistentVolume("pv-1").StorageClass("gp2-csi").Result(), }, resourcePolicies: &resourcepolicies.ResourcePolicies{ Version: "v1", VolumePolicies: []resourcepolicies.VolumePolicy{ { Conditions: map[string]any{ "storageClass": []string{"gp2-csi"}, }, Action: resourcepolicies.Action{ Type: resourcepolicies.Snapshot, }, }, }, }, shouldFSBackup: false, expectedErr: false, }, { name: "VolumePolicy not match, selected by opt-in way, return true and no error", pod: builder.ForPod("ns", "pod-1"). ObjectMeta(builder.WithAnnotations(velerov1api.VolumesToBackupAnnotation, "pvc-1")). Volumes( &corev1api.Volume{ Name: "pvc-1", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "pvc-1", }, }, }).Result(), resources: []runtime.Object{ builder.ForPersistentVolumeClaim("ns", "pvc-1"). VolumeName("pv-1"). StorageClass("gp2-csi").Phase(corev1api.ClaimBound).Result(), builder.ForPersistentVolume("pv-1").StorageClass("gp2-csi").Result(), }, resourcePolicies: &resourcepolicies.ResourcePolicies{ Version: "v1", VolumePolicies: []resourcepolicies.VolumePolicy{ { Conditions: map[string]any{ "storageClass": []string{"gp3-csi"}, }, Action: resourcepolicies.Action{ Type: resourcepolicies.FSBackup, }, }, }, }, shouldFSBackup: true, expectedErr: false, }, { name: "No VolumePolicy, not selected by opt-out way, return false and no error", pod: builder.ForPod("ns", "pod-1"). ObjectMeta(builder.WithAnnotations(velerov1api.VolumesToExcludeAnnotation, "pvc-1")). Volumes( &corev1api.Volume{ Name: "pvc-1", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "pvc-1", }, }, }).Result(), resources: []runtime.Object{ builder.ForPersistentVolumeClaim("ns", "pvc-1"). VolumeName("pv-1"). StorageClass("gp2-csi").Phase(corev1api.ClaimBound).Result(), builder.ForPersistentVolume("pv-1").StorageClass("gp2-csi").Result(), }, defaultVolumesToFSBackup: true, shouldFSBackup: false, expectedErr: false, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { fakeClient := velerotest.NewFakeControllerRuntimeClient(t, tc.resources...) if tc.pod != nil { fakeClient.Create(t.Context(), tc.pod) } var p *resourcepolicies.Policies if tc.resourcePolicies != nil { p = &resourcepolicies.Policies{} err := p.BuildPolicy(tc.resourcePolicies) if err != nil { t.Fatalf("failed to build policy with error %v", err) } } vh := NewVolumeHelperImpl( p, tc.snapshotVolumesFlag, logrus.StandardLogger(), fakeClient, tc.defaultVolumesToFSBackup, false, ) actualShouldFSBackup, actualError := vh.ShouldPerformFSBackup(tc.pod.Spec.Volumes[0], *tc.pod) if tc.expectedErr { require.Error(t, actualError, "Want error; Got nil error") return } require.Equalf(t, tc.shouldFSBackup, actualShouldFSBackup, "Want shouldFSBackup as %t; Got shouldFSBackup as %t", tc.shouldFSBackup, actualShouldFSBackup) }) } } func TestGetVolumeFromResource(t *testing.T) { helper := &volumeHelperImpl{} t.Run("PersistentVolume input", func(t *testing.T) { pv := &corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "test-pv", }, } outPV, outPod, err := helper.getVolumeFromResource(pv) require.NoError(t, err) assert.NotNil(t, outPV) assert.Nil(t, outPod) assert.Equal(t, "test-pv", outPV.Name) }) t.Run("Volume input", func(t *testing.T) { vol := &corev1api.Volume{ Name: "test-volume", } outPV, outPod, err := helper.getVolumeFromResource(vol) require.NoError(t, err) assert.Nil(t, outPV) assert.NotNil(t, outPod) assert.Equal(t, "test-volume", outPod.Name) }) t.Run("Invalid input", func(t *testing.T) { _, _, err := helper.getVolumeFromResource("invalid") assert.ErrorContains(t, err, "resource is not a PersistentVolume or Volume") }) } func TestVolumeHelperImplWithCache_ShouldPerformSnapshot(t *testing.T) { testCases := []struct { name string inputObj runtime.Object groupResource schema.GroupResource pod *corev1api.Pod resourcePolicies *resourcepolicies.ResourcePolicies snapshotVolumesFlag *bool defaultVolumesToFSBackup bool buildCache bool shouldSnapshot bool expectedErr bool }{ { name: "VolumePolicy match with cache, returns true", inputObj: builder.ForPersistentVolume("example-pv").StorageClass("gp2-csi").ClaimRef("ns", "pvc-1").Result(), groupResource: kuberesource.PersistentVolumes, resourcePolicies: &resourcepolicies.ResourcePolicies{ Version: "v1", VolumePolicies: []resourcepolicies.VolumePolicy{ { Conditions: map[string]any{ "storageClass": []string{"gp2-csi"}, }, Action: resourcepolicies.Action{ Type: resourcepolicies.Snapshot, }, }, }, }, snapshotVolumesFlag: ptr.To(true), buildCache: true, shouldSnapshot: true, expectedErr: false, }, { name: "VolumePolicy not match, fs-backup via opt-out with cache, skips snapshot", inputObj: builder.ForPersistentVolume("example-pv").StorageClass("gp3-csi").ClaimRef("ns", "pvc-1").Result(), groupResource: kuberesource.PersistentVolumes, pod: builder.ForPod("ns", "pod-1").Volumes( &corev1api.Volume{ Name: "volume", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "pvc-1", }, }, }, ).Result(), resourcePolicies: &resourcepolicies.ResourcePolicies{ Version: "v1", VolumePolicies: []resourcepolicies.VolumePolicy{ { Conditions: map[string]any{ "storageClass": []string{"gp2-csi"}, }, Action: resourcepolicies.Action{ Type: resourcepolicies.Snapshot, }, }, }, }, snapshotVolumesFlag: ptr.To(true), defaultVolumesToFSBackup: true, buildCache: true, shouldSnapshot: false, expectedErr: false, }, { name: "Cache not built, falls back to direct lookup", inputObj: builder.ForPersistentVolume("example-pv").StorageClass("gp2-csi").ClaimRef("ns", "pvc-1").Result(), groupResource: kuberesource.PersistentVolumes, resourcePolicies: &resourcepolicies.ResourcePolicies{ Version: "v1", VolumePolicies: []resourcepolicies.VolumePolicy{ { Conditions: map[string]any{ "storageClass": []string{"gp2-csi"}, }, Action: resourcepolicies.Action{ Type: resourcepolicies.Snapshot, }, }, }, }, snapshotVolumesFlag: ptr.To(true), buildCache: false, shouldSnapshot: true, expectedErr: false, }, { name: "No volume policy, defaultVolumesToFSBackup with cache, skips snapshot", inputObj: builder.ForPersistentVolume("example-pv").StorageClass("gp2-csi").ClaimRef("ns", "pvc-1").Result(), groupResource: kuberesource.PersistentVolumes, pod: builder.ForPod("ns", "pod-1").Volumes( &corev1api.Volume{ Name: "volume", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "pvc-1", }, }, }, ).Result(), resourcePolicies: nil, snapshotVolumesFlag: ptr.To(true), defaultVolumesToFSBackup: true, buildCache: true, shouldSnapshot: false, expectedErr: false, }, } objs := []runtime.Object{ &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: "ns", Name: "pvc-1", }, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { fakeClient := velerotest.NewFakeControllerRuntimeClient(t, objs...) if tc.pod != nil { require.NoError(t, fakeClient.Create(t.Context(), tc.pod)) } var p *resourcepolicies.Policies if tc.resourcePolicies != nil { p = &resourcepolicies.Policies{} err := p.BuildPolicy(tc.resourcePolicies) require.NoError(t, err) } var namespaces []string if tc.buildCache { namespaces = []string{"ns"} } vh, err := NewVolumeHelperImplWithNamespaces( p, tc.snapshotVolumesFlag, logrus.StandardLogger(), fakeClient, tc.defaultVolumesToFSBackup, false, namespaces, ) require.NoError(t, err) obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.inputObj) require.NoError(t, err) actualShouldSnapshot, actualError := vh.ShouldPerformSnapshot(&unstructured.Unstructured{Object: obj}, tc.groupResource) if tc.expectedErr { require.Error(t, actualError) return } require.NoError(t, actualError) require.Equalf(t, tc.shouldSnapshot, actualShouldSnapshot, "Want shouldSnapshot as %t; Got shouldSnapshot as %t", tc.shouldSnapshot, actualShouldSnapshot) }) } } func TestVolumeHelperImplWithCache_ShouldPerformFSBackup(t *testing.T) { testCases := []struct { name string pod *corev1api.Pod resources []runtime.Object resourcePolicies *resourcepolicies.ResourcePolicies snapshotVolumesFlag *bool defaultVolumesToFSBackup bool buildCache bool shouldFSBackup bool expectedErr bool }{ { name: "VolumePolicy match with cache, return true", pod: builder.ForPod("ns", "pod-1"). Volumes( &corev1api.Volume{ Name: "vol-1", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "pvc-1", }, }, }).Result(), resources: []runtime.Object{ builder.ForPersistentVolumeClaim("ns", "pvc-1"). VolumeName("pv-1"). StorageClass("gp2-csi").Phase(corev1api.ClaimBound).Result(), builder.ForPersistentVolume("pv-1").StorageClass("gp2-csi").Result(), }, resourcePolicies: &resourcepolicies.ResourcePolicies{ Version: "v1", VolumePolicies: []resourcepolicies.VolumePolicy{ { Conditions: map[string]any{ "storageClass": []string{"gp2-csi"}, }, Action: resourcepolicies.Action{ Type: resourcepolicies.FSBackup, }, }, }, }, buildCache: true, shouldFSBackup: true, expectedErr: false, }, { name: "VolumePolicy match with cache, action is snapshot, return false", pod: builder.ForPod("ns", "pod-1"). Volumes( &corev1api.Volume{ Name: "vol-1", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "pvc-1", }, }, }).Result(), resources: []runtime.Object{ builder.ForPersistentVolumeClaim("ns", "pvc-1"). VolumeName("pv-1"). StorageClass("gp2-csi").Phase(corev1api.ClaimBound).Result(), builder.ForPersistentVolume("pv-1").StorageClass("gp2-csi").Result(), }, resourcePolicies: &resourcepolicies.ResourcePolicies{ Version: "v1", VolumePolicies: []resourcepolicies.VolumePolicy{ { Conditions: map[string]any{ "storageClass": []string{"gp2-csi"}, }, Action: resourcepolicies.Action{ Type: resourcepolicies.Snapshot, }, }, }, }, buildCache: true, shouldFSBackup: false, expectedErr: false, }, { name: "Cache not built, falls back to direct lookup, opt-in annotation", pod: builder.ForPod("ns", "pod-1"). ObjectMeta(builder.WithAnnotations(velerov1api.VolumesToBackupAnnotation, "vol-1")). Volumes( &corev1api.Volume{ Name: "vol-1", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "pvc-1", }, }, }).Result(), resources: []runtime.Object{ builder.ForPersistentVolumeClaim("ns", "pvc-1"). VolumeName("pv-1"). StorageClass("gp2-csi").Phase(corev1api.ClaimBound).Result(), builder.ForPersistentVolume("pv-1").StorageClass("gp2-csi").Result(), }, buildCache: false, defaultVolumesToFSBackup: false, shouldFSBackup: true, expectedErr: false, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { fakeClient := velerotest.NewFakeControllerRuntimeClient(t, tc.resources...) if tc.pod != nil { require.NoError(t, fakeClient.Create(t.Context(), tc.pod)) } var p *resourcepolicies.Policies if tc.resourcePolicies != nil { p = &resourcepolicies.Policies{} err := p.BuildPolicy(tc.resourcePolicies) require.NoError(t, err) } var namespaces []string if tc.buildCache { namespaces = []string{"ns"} } vh, err := NewVolumeHelperImplWithNamespaces( p, tc.snapshotVolumesFlag, logrus.StandardLogger(), fakeClient, tc.defaultVolumesToFSBackup, false, namespaces, ) require.NoError(t, err) actualShouldFSBackup, actualError := vh.ShouldPerformFSBackup(tc.pod.Spec.Volumes[0], *tc.pod) if tc.expectedErr { require.Error(t, actualError) return } require.NoError(t, actualError) require.Equalf(t, tc.shouldFSBackup, actualShouldFSBackup, "Want shouldFSBackup as %t; Got shouldFSBackup as %t", tc.shouldFSBackup, actualShouldFSBackup) }) } } // TestNewVolumeHelperImplWithCache tests the NewVolumeHelperImplWithCache constructor // which is used by plugins that build the cache lazily per-namespace. func TestNewVolumeHelperImplWithCache(t *testing.T) { testCases := []struct { name string backup velerov1api.Backup resourcePolicyConfigMap *corev1api.ConfigMap pvcPodCache bool // whether to pass a cache expectError bool }{ { name: "creates VolumeHelper with nil cache", backup: velerov1api.Backup{ ObjectMeta: metav1.ObjectMeta{ Name: "test-backup", Namespace: "velero", }, Spec: velerov1api.BackupSpec{ SnapshotVolumes: ptr.To(true), DefaultVolumesToFsBackup: ptr.To(false), }, }, pvcPodCache: false, expectError: false, }, { name: "creates VolumeHelper with non-nil cache", backup: velerov1api.Backup{ ObjectMeta: metav1.ObjectMeta{ Name: "test-backup", Namespace: "velero", }, Spec: velerov1api.BackupSpec{ SnapshotVolumes: ptr.To(true), DefaultVolumesToFsBackup: ptr.To(true), SnapshotMoveData: ptr.To(true), }, }, pvcPodCache: true, expectError: false, }, { name: "creates VolumeHelper with resource policies", backup: velerov1api.Backup{ ObjectMeta: metav1.ObjectMeta{ Name: "test-backup", Namespace: "velero", }, Spec: velerov1api.BackupSpec{ SnapshotVolumes: ptr.To(true), ResourcePolicy: &corev1api.TypedLocalObjectReference{ Kind: "ConfigMap", Name: "resource-policy", }, }, }, resourcePolicyConfigMap: &corev1api.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "resource-policy", Namespace: "velero", }, Data: map[string]string{ "policy": `version: v1 volumePolicies: - conditions: storageClass: - gp2-csi action: type: snapshot`, }, }, pvcPodCache: true, expectError: false, }, { name: "fails when resource policy ConfigMap not found", backup: velerov1api.Backup{ ObjectMeta: metav1.ObjectMeta{ Name: "test-backup", Namespace: "velero", }, Spec: velerov1api.BackupSpec{ ResourcePolicy: &corev1api.TypedLocalObjectReference{ Kind: "ConfigMap", Name: "non-existent-policy", }, }, }, pvcPodCache: false, expectError: true, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { var objs []runtime.Object if tc.resourcePolicyConfigMap != nil { objs = append(objs, tc.resourcePolicyConfigMap) } fakeClient := velerotest.NewFakeControllerRuntimeClient(t, objs...) var cache *podvolumeutil.PVCPodCache if tc.pvcPodCache { cache = podvolumeutil.NewPVCPodCache() } vh, err := NewVolumeHelperImplWithCache( tc.backup, fakeClient, logrus.StandardLogger(), cache, ) if tc.expectError { require.Error(t, err) require.Nil(t, vh) } else { require.NoError(t, err) require.NotNil(t, vh) } }) } } // TestNewVolumeHelperImplWithCache_UsesCache verifies that the VolumeHelper created // via NewVolumeHelperImplWithCache actually uses the provided cache for lookups. func TestNewVolumeHelperImplWithCache_UsesCache(t *testing.T) { // Create a pod that uses a PVC via opt-out (defaultVolumesToFsBackup=true) pod := builder.ForPod("ns", "pod-1").Volumes( &corev1api.Volume{ Name: "volume", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "pvc-1", }, }, }, ).Result() pvc := &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: "ns", Name: "pvc-1", }, } pv := builder.ForPersistentVolume("example-pv").StorageClass("gp2-csi").ClaimRef("ns", "pvc-1").Result() fakeClient := velerotest.NewFakeControllerRuntimeClient(t, pvc, pv, pod) // Build cache for the namespace cache := podvolumeutil.NewPVCPodCache() err := cache.BuildCacheForNamespace(t.Context(), "ns", fakeClient) require.NoError(t, err) backup := velerov1api.Backup{ ObjectMeta: metav1.ObjectMeta{ Name: "test-backup", Namespace: "velero", }, Spec: velerov1api.BackupSpec{ SnapshotVolumes: ptr.To(true), DefaultVolumesToFsBackup: ptr.To(true), // opt-out mode }, } vh, err := NewVolumeHelperImplWithCache(backup, fakeClient, logrus.StandardLogger(), cache) require.NoError(t, err) // Convert PV to unstructured obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(pv) require.NoError(t, err) // ShouldPerformSnapshot should return false because the volume is selected for fs-backup // This relies on the cache to find the pod using the PVC shouldSnapshot, err := vh.ShouldPerformSnapshot(&unstructured.Unstructured{Object: obj}, kuberesource.PersistentVolumes) require.NoError(t, err) require.False(t, shouldSnapshot, "Expected snapshot to be skipped due to fs-backup selection via cache") } // TestVolumeHelperImpl_ShouldPerformSnapshot_UnboundPVC tests that Pending and Lost PVCs with // phase-based skip policies don't cause errors when GetPVForPVC would fail. func TestVolumeHelperImpl_ShouldPerformSnapshot_UnboundPVC(t *testing.T) { testCases := []struct { name string inputPVC *corev1api.PersistentVolumeClaim resourcePolicies *resourcepolicies.ResourcePolicies shouldSnapshot bool expectedErr bool }{ { name: "Pending PVC with phase-based skip policy should not error and return false", inputPVC: builder.ForPersistentVolumeClaim("ns", "pvc-pending"). StorageClass("non-existent-class"). Phase(corev1api.ClaimPending). Result(), resourcePolicies: &resourcepolicies.ResourcePolicies{ Version: "v1", VolumePolicies: []resourcepolicies.VolumePolicy{ { Conditions: map[string]any{ "pvcPhase": []string{"Pending"}, }, Action: resourcepolicies.Action{ Type: resourcepolicies.Skip, }, }, }, }, shouldSnapshot: false, expectedErr: false, }, { name: "Pending PVC without matching skip policy should error (no PV)", inputPVC: builder.ForPersistentVolumeClaim("ns", "pvc-pending-no-policy"). StorageClass("non-existent-class"). Phase(corev1api.ClaimPending). Result(), resourcePolicies: &resourcepolicies.ResourcePolicies{ Version: "v1", VolumePolicies: []resourcepolicies.VolumePolicy{ { Conditions: map[string]any{ "storageClass": []string{"gp2-csi"}, }, Action: resourcepolicies.Action{ Type: resourcepolicies.Skip, }, }, }, }, shouldSnapshot: false, expectedErr: true, }, { name: "Lost PVC with phase-based skip policy should not error and return false", inputPVC: builder.ForPersistentVolumeClaim("ns", "pvc-lost"). StorageClass("some-class"). Phase(corev1api.ClaimLost). Result(), resourcePolicies: &resourcepolicies.ResourcePolicies{ Version: "v1", VolumePolicies: []resourcepolicies.VolumePolicy{ { Conditions: map[string]any{ "pvcPhase": []string{"Lost"}, }, Action: resourcepolicies.Action{ Type: resourcepolicies.Skip, }, }, }, }, shouldSnapshot: false, expectedErr: false, }, { name: "Lost PVC with policy for Pending and Lost should not error and return false", inputPVC: builder.ForPersistentVolumeClaim("ns", "pvc-lost"). StorageClass("some-class"). Phase(corev1api.ClaimLost). Result(), resourcePolicies: &resourcepolicies.ResourcePolicies{ Version: "v1", VolumePolicies: []resourcepolicies.VolumePolicy{ { Conditions: map[string]any{ "pvcPhase": []string{"Pending", "Lost"}, }, Action: resourcepolicies.Action{ Type: resourcepolicies.Skip, }, }, }, }, shouldSnapshot: false, expectedErr: false, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { fakeClient := velerotest.NewFakeControllerRuntimeClient(t) var p *resourcepolicies.Policies if tc.resourcePolicies != nil { p = &resourcepolicies.Policies{} err := p.BuildPolicy(tc.resourcePolicies) require.NoError(t, err) } vh := NewVolumeHelperImpl( p, ptr.To(true), logrus.StandardLogger(), fakeClient, false, false, ) obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.inputPVC) require.NoError(t, err) actualShouldSnapshot, actualError := vh.ShouldPerformSnapshot(&unstructured.Unstructured{Object: obj}, kuberesource.PersistentVolumeClaims) if tc.expectedErr { require.Error(t, actualError, "Want error; Got nil error") return } require.NoError(t, actualError) require.Equalf(t, tc.shouldSnapshot, actualShouldSnapshot, "Want shouldSnapshot as %t; Got shouldSnapshot as %t", tc.shouldSnapshot, actualShouldSnapshot) }) } } // TestVolumeHelperImpl_ShouldPerformFSBackup_UnboundPVC tests that Pending and Lost PVCs with // phase-based skip policies don't cause errors when GetPVForPVC would fail. func TestVolumeHelperImpl_ShouldPerformFSBackup_UnboundPVC(t *testing.T) { testCases := []struct { name string pod *corev1api.Pod pvc *corev1api.PersistentVolumeClaim resourcePolicies *resourcepolicies.ResourcePolicies shouldFSBackup bool expectedErr bool }{ { name: "Pending PVC with phase-based skip policy should not error and return false", pod: builder.ForPod("ns", "pod-1"). Volumes( &corev1api.Volume{ Name: "vol-pending", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "pvc-pending", }, }, }).Result(), pvc: builder.ForPersistentVolumeClaim("ns", "pvc-pending"). StorageClass("non-existent-class"). Phase(corev1api.ClaimPending). Result(), resourcePolicies: &resourcepolicies.ResourcePolicies{ Version: "v1", VolumePolicies: []resourcepolicies.VolumePolicy{ { Conditions: map[string]any{ "pvcPhase": []string{"Pending"}, }, Action: resourcepolicies.Action{ Type: resourcepolicies.Skip, }, }, }, }, shouldFSBackup: false, expectedErr: false, }, { name: "Pending PVC without matching skip policy should error (no PV)", pod: builder.ForPod("ns", "pod-1"). Volumes( &corev1api.Volume{ Name: "vol-pending", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "pvc-pending-no-policy", }, }, }).Result(), pvc: builder.ForPersistentVolumeClaim("ns", "pvc-pending-no-policy"). StorageClass("non-existent-class"). Phase(corev1api.ClaimPending). Result(), resourcePolicies: &resourcepolicies.ResourcePolicies{ Version: "v1", VolumePolicies: []resourcepolicies.VolumePolicy{ { Conditions: map[string]any{ "storageClass": []string{"gp2-csi"}, }, Action: resourcepolicies.Action{ Type: resourcepolicies.Skip, }, }, }, }, shouldFSBackup: false, expectedErr: true, }, { name: "Lost PVC with phase-based skip policy should not error and return false", pod: builder.ForPod("ns", "pod-1"). Volumes( &corev1api.Volume{ Name: "vol-lost", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "pvc-lost", }, }, }).Result(), pvc: builder.ForPersistentVolumeClaim("ns", "pvc-lost"). StorageClass("some-class"). Phase(corev1api.ClaimLost). Result(), resourcePolicies: &resourcepolicies.ResourcePolicies{ Version: "v1", VolumePolicies: []resourcepolicies.VolumePolicy{ { Conditions: map[string]any{ "pvcPhase": []string{"Lost"}, }, Action: resourcepolicies.Action{ Type: resourcepolicies.Skip, }, }, }, }, shouldFSBackup: false, expectedErr: false, }, { name: "Lost PVC with policy for Pending and Lost should not error and return false", pod: builder.ForPod("ns", "pod-1"). Volumes( &corev1api.Volume{ Name: "vol-lost", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "pvc-lost", }, }, }).Result(), pvc: builder.ForPersistentVolumeClaim("ns", "pvc-lost"). StorageClass("some-class"). Phase(corev1api.ClaimLost). Result(), resourcePolicies: &resourcepolicies.ResourcePolicies{ Version: "v1", VolumePolicies: []resourcepolicies.VolumePolicy{ { Conditions: map[string]any{ "pvcPhase": []string{"Pending", "Lost"}, }, Action: resourcepolicies.Action{ Type: resourcepolicies.Skip, }, }, }, }, shouldFSBackup: false, expectedErr: false, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { fakeClient := velerotest.NewFakeControllerRuntimeClient(t, tc.pvc) require.NoError(t, fakeClient.Create(t.Context(), tc.pod)) var p *resourcepolicies.Policies if tc.resourcePolicies != nil { p = &resourcepolicies.Policies{} err := p.BuildPolicy(tc.resourcePolicies) require.NoError(t, err) } vh := NewVolumeHelperImpl( p, ptr.To(true), logrus.StandardLogger(), fakeClient, false, false, ) actualShouldFSBackup, actualError := vh.ShouldPerformFSBackup(tc.pod.Spec.Volumes[0], *tc.pod) if tc.expectedErr { require.Error(t, actualError, "Want error; Got nil error") return } require.NoError(t, actualError) require.Equalf(t, tc.shouldFSBackup, actualShouldFSBackup, "Want shouldFSBackup as %t; Got shouldFSBackup as %t", tc.shouldFSBackup, actualShouldFSBackup) }) } } ================================================ FILE: netlify.toml ================================================ [build] base = "site/" command = "hugo --gc --minify" publish = "site/public" [context.production.environment] HUGO_VERSION = "0.73.0" [context.deploy-preview.environment] HUGO_VERSION = "0.73.0" ================================================ FILE: pkg/apis/velero/shared/data_move_operation_progress.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package shared // DataMoveOperationProgress represents the progress of a // data movement operation // +k8s:deepcopy-gen=true type DataMoveOperationProgress struct { // +optional TotalBytes int64 `json:"totalBytes,omitempty"` // +optional BytesDone int64 `json:"bytesDone,omitempty"` } ================================================ FILE: pkg/apis/velero/v1/backup_repository_types.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v1 import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // BackupRepositorySpec is the specification for a BackupRepository. type BackupRepositorySpec struct { // VolumeNamespace is the namespace this backup repository contains // pod volume backups for. VolumeNamespace string `json:"volumeNamespace"` // BackupStorageLocation is the name of the BackupStorageLocation // that should contain this repository. BackupStorageLocation string `json:"backupStorageLocation"` // RepositoryType indicates the type of the backend repository // +kubebuilder:validation:Enum=kopia;restic;"" // +optional RepositoryType string `json:"repositoryType"` // ResticIdentifier is the full restic-compatible string for identifying // this repository. This field is only used when RepositoryType is "restic". // +optional ResticIdentifier string `json:"resticIdentifier,omitempty"` // MaintenanceFrequency is how often maintenance should be run. MaintenanceFrequency metav1.Duration `json:"maintenanceFrequency"` // RepositoryConfig is for repository-specific configuration fields. // +optional // +nullable RepositoryConfig map[string]string `json:"repositoryConfig,omitempty"` } // BackupRepositoryPhase represents the lifecycle phase of a BackupRepository. // +kubebuilder:validation:Enum=New;Ready;NotReady type BackupRepositoryPhase string const ( BackupRepositoryPhaseNew BackupRepositoryPhase = "New" BackupRepositoryPhaseReady BackupRepositoryPhase = "Ready" BackupRepositoryPhaseNotReady BackupRepositoryPhase = "NotReady" BackupRepositoryTypeRestic string = "restic" BackupRepositoryTypeKopia string = "kopia" ) // BackupRepositoryStatus is the current status of a BackupRepository. type BackupRepositoryStatus struct { // Phase is the current state of the BackupRepository. // +optional Phase BackupRepositoryPhase `json:"phase,omitempty"` // Message is a message about the current status of the BackupRepository. // +optional Message string `json:"message,omitempty"` // LastMaintenanceTime is the last time repo maintenance succeeded. // +optional // +nullable LastMaintenanceTime *metav1.Time `json:"lastMaintenanceTime,omitempty"` // RecentMaintenance is status of the recent repo maintenance. // +optional RecentMaintenance []BackupRepositoryMaintenanceStatus `json:"recentMaintenance,omitempty"` } // BackupRepositoryMaintenanceResult represents the result of a repo maintenance. // +kubebuilder:validation:Enum=Succeeded;Failed type BackupRepositoryMaintenanceResult string const ( BackupRepositoryMaintenanceSucceeded BackupRepositoryMaintenanceResult = "Succeeded" BackupRepositoryMaintenanceFailed BackupRepositoryMaintenanceResult = "Failed" ) type BackupRepositoryMaintenanceStatus struct { // Result is the result of the repo maintenance. // +optional Result BackupRepositoryMaintenanceResult `json:"result,omitempty"` // StartTimestamp is the start time of the repo maintenance. // +optional // +nullable StartTimestamp *metav1.Time `json:"startTimestamp,omitempty"` // CompleteTimestamp is the completion time of the repo maintenance. // +optional // +nullable CompleteTimestamp *metav1.Time `json:"completeTimestamp,omitempty"` // Message is a message about the current status of the repo maintenance. // +optional Message string `json:"message,omitempty"` } // TODO(2.0) After converting all resources to use the runtime-controller client, // the genclient and k8s:deepcopy markers will no longer be needed and should be removed. // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // +kubebuilder:object:root=true // +kubebuilder:object:generate=true // +kubebuilder:storageversion // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" // +kubebuilder:printcolumn:name="Repository Type",type="string",JSONPath=".spec.repositoryType" // type BackupRepository struct { metav1.TypeMeta `json:",inline"` // +optional metav1.ObjectMeta `json:"metadata,omitempty"` // +optional Spec BackupRepositorySpec `json:"spec,omitempty"` // +optional Status BackupRepositoryStatus `json:"status,omitempty"` } // TODO(2.0) After converting all resources to use the runtime-controller client, // the k8s:deepcopy marker will no longer be needed and should be removed. // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // +kubebuilder:object:root=true // +kubebuilder:rbac:groups=velero.io,resources=backuprepositories,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=velero.io,resources=backuprepositories/status,verbs=get;update;patch // BackupRepositoryList is a list of BackupRepositories. type BackupRepositoryList struct { metav1.TypeMeta `json:",inline"` // +optional metav1.ListMeta `json:"metadata,omitempty"` Items []BackupRepository `json:"items"` } ================================================ FILE: pkg/apis/velero/v1/backup_types.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v1 import ( corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) type Metadata struct { Labels map[string]string `json:"labels,omitempty"` } // BackupSpec defines the specification for a Velero backup. type BackupSpec struct { // +optional Metadata `json:"metadata,omitempty"` // IncludedNamespaces is a slice of namespace names to include objects // from. If empty, all namespaces are included. // +optional // +nullable IncludedNamespaces []string `json:"includedNamespaces,omitempty"` // ExcludedNamespaces contains a list of namespaces that are not // included in the backup. // +optional // +nullable ExcludedNamespaces []string `json:"excludedNamespaces,omitempty"` // IncludedResources is a slice of resource names to include // in the backup. If empty, all resources are included. // +optional // +nullable IncludedResources []string `json:"includedResources,omitempty"` // ExcludedResources is a slice of resource names that are not // included in the backup. // +optional // +nullable ExcludedResources []string `json:"excludedResources,omitempty"` // IncludedClusterScopedResources is a slice of cluster-scoped // resource type names to include in the backup. // If set to "*", all cluster-scoped resource types are included. // The default value is empty, which means only related // cluster-scoped resources are included. // +optional // +nullable IncludedClusterScopedResources []string `json:"includedClusterScopedResources,omitempty"` // ExcludedClusterScopedResources is a slice of cluster-scoped // resource type names to exclude from the backup. // If set to "*", all cluster-scoped resource types are excluded. // The default value is empty. // +optional // +nullable ExcludedClusterScopedResources []string `json:"excludedClusterScopedResources,omitempty"` // IncludedNamespaceScopedResources is a slice of namespace-scoped // resource type names to include in the backup. // The default value is "*". // +optional // +nullable IncludedNamespaceScopedResources []string `json:"includedNamespaceScopedResources,omitempty"` // ExcludedNamespaceScopedResources is a slice of namespace-scoped // resource type names to exclude from the backup. // If set to "*", all namespace-scoped resource types are excluded. // The default value is empty. // +optional // +nullable ExcludedNamespaceScopedResources []string `json:"excludedNamespaceScopedResources,omitempty"` // LabelSelector is a metav1.LabelSelector to filter with // when adding individual objects to the backup. If empty // or nil, all objects are included. Optional. // +optional // +nullable LabelSelector *metav1.LabelSelector `json:"labelSelector,omitempty"` // OrLabelSelectors is list of metav1.LabelSelector to filter with // when adding individual objects to the backup. If multiple provided // they will be joined by the OR operator. LabelSelector as well as // OrLabelSelectors cannot co-exist in backup request, only one of them // can be used. // +optional // +nullable OrLabelSelectors []*metav1.LabelSelector `json:"orLabelSelectors,omitempty"` // SnapshotVolumes specifies whether to take snapshots // of any PV's referenced in the set of objects included // in the Backup. // +optional // +nullable SnapshotVolumes *bool `json:"snapshotVolumes,omitempty"` // TTL is a time.Duration-parseable string describing how long // the Backup should be retained for. // +optional TTL metav1.Duration `json:"ttl,omitempty"` // VolumeGroupSnapshotLabelKey specifies the label key to group PVCs under a VGS. // +optional VolumeGroupSnapshotLabelKey string `json:"volumeGroupSnapshotLabelKey,omitempty"` // IncludeClusterResources specifies whether cluster-scoped resources // should be included for consideration in the backup. // +optional // +nullable IncludeClusterResources *bool `json:"includeClusterResources,omitempty"` // Hooks represent custom behaviors that should be executed at different phases of the backup. // +optional Hooks BackupHooks `json:"hooks,omitempty"` // StorageLocation is a string containing the name of a BackupStorageLocation where the backup should be stored. // +optional StorageLocation string `json:"storageLocation,omitempty"` // VolumeSnapshotLocations is a list containing names of VolumeSnapshotLocations associated with this backup. // +optional VolumeSnapshotLocations []string `json:"volumeSnapshotLocations,omitempty"` // DefaultVolumesToRestic specifies whether restic should be used to take a // backup of all pod volumes by default. // // Deprecated: this field is no longer used and will be removed entirely in future. Use DefaultVolumesToFsBackup instead. // +optional // +nullable DefaultVolumesToRestic *bool `json:"defaultVolumesToRestic,omitempty"` // DefaultVolumesToFsBackup specifies whether pod volume file system backup should be used // for all volumes by default. // +optional // +nullable DefaultVolumesToFsBackup *bool `json:"defaultVolumesToFsBackup,omitempty"` // OrderedResources specifies the backup order of resources of specific Kind. // The map key is the resource name and value is a list of object names separated by commas. // Each resource name has format "namespace/objectname". For cluster resources, simply use "objectname". // +optional // +nullable OrderedResources map[string]string `json:"orderedResources,omitempty"` // CSISnapshotTimeout specifies the time used to wait for CSI VolumeSnapshot status turns to // ReadyToUse during creation, before returning error as timeout. // The default value is 10 minute. // +optional CSISnapshotTimeout metav1.Duration `json:"csiSnapshotTimeout,omitempty"` // ItemOperationTimeout specifies the time used to wait for asynchronous BackupItemAction operations // The default value is 4 hour. // +optional ItemOperationTimeout metav1.Duration `json:"itemOperationTimeout,omitempty"` // ResourcePolicy specifies the referenced resource policies that backup should follow // +optional ResourcePolicy *corev1api.TypedLocalObjectReference `json:"resourcePolicy,omitempty"` // SnapshotMoveData specifies whether snapshot data should be moved // +optional // +nullable SnapshotMoveData *bool `json:"snapshotMoveData,omitempty"` // DataMover specifies the data mover to be used by the backup. // If DataMover is "" or "velero", the built-in data mover will be used. // +optional DataMover string `json:"datamover,omitempty"` // UploaderConfig specifies the configuration for the uploader. // +optional // +nullable UploaderConfig *UploaderConfigForBackup `json:"uploaderConfig,omitempty"` } // UploaderConfigForBackup defines the configuration for the uploader when doing backup. type UploaderConfigForBackup struct { // ParallelFilesUpload is the number of files parallel uploads to perform when using the uploader. // +optional ParallelFilesUpload int `json:"parallelFilesUpload,omitempty"` } // BackupHooks contains custom behaviors that should be executed at different phases of the backup. type BackupHooks struct { // Resources are hooks that should be executed when backing up individual instances of a resource. // +optional // +nullable Resources []BackupResourceHookSpec `json:"resources,omitempty"` } // BackupResourceHookSpec defines one or more BackupResourceHooks that should be executed based on // the rules defined for namespaces, resources, and label selector. type BackupResourceHookSpec struct { // Name is the name of this hook. Name string `json:"name"` // IncludedNamespaces specifies the namespaces to which this hook spec applies. If empty, it applies // to all namespaces. // +optional // +nullable IncludedNamespaces []string `json:"includedNamespaces,omitempty"` // ExcludedNamespaces specifies the namespaces to which this hook spec does not apply. // +optional // +nullable ExcludedNamespaces []string `json:"excludedNamespaces,omitempty"` // IncludedResources specifies the resources to which this hook spec applies. If empty, it applies // to all resources. // +optional // +nullable IncludedResources []string `json:"includedResources,omitempty"` // ExcludedResources specifies the resources to which this hook spec does not apply. // +optional // +nullable ExcludedResources []string `json:"excludedResources,omitempty"` // LabelSelector, if specified, filters the resources to which this hook spec applies. // +optional // +nullable LabelSelector *metav1.LabelSelector `json:"labelSelector,omitempty"` // PreHooks is a list of BackupResourceHooks to execute prior to storing the item in the backup. // These are executed before any "additional items" from item actions are processed. // +optional PreHooks []BackupResourceHook `json:"pre,omitempty"` // PostHooks is a list of BackupResourceHooks to execute after storing the item in the backup. // These are executed after all "additional items" from item actions are processed. // +optional PostHooks []BackupResourceHook `json:"post,omitempty"` } // BackupResourceHook defines a hook for a resource. type BackupResourceHook struct { // Exec defines an exec hook. Exec *ExecHook `json:"exec"` } // ExecHook is a hook that uses the pod exec API to execute a command in a container in a pod. type ExecHook struct { // Container is the container in the pod where the command should be executed. If not specified, // the pod's first container is used. // +optional Container string `json:"container,omitempty"` // Command is the command and arguments to execute. // +kubebuilder:validation:MinItems=1 Command []string `json:"command"` // OnError specifies how Velero should behave if it encounters an error executing this hook. // +optional OnError HookErrorMode `json:"onError,omitempty"` // Timeout defines the maximum amount of time Velero should wait for the hook to complete before // considering the execution a failure. // +optional Timeout metav1.Duration `json:"timeout,omitempty"` } // HookErrorMode defines how Velero should treat an error from a hook. // +kubebuilder:validation:Enum=Continue;Fail type HookErrorMode string const ( // HookErrorModeContinue means that an error from a hook is acceptable and the backup/restore can // proceed with the rest of hooks' execution. This backup/restore should be in `PartiallyFailed` status. HookErrorModeContinue HookErrorMode = "Continue" // HookErrorModeFail means that an error from a hook is problematic and Velero should stop executing following hooks. // This backup/restore should be in `PartiallyFailed` status. HookErrorModeFail HookErrorMode = "Fail" ) // BackupPhase is a string representation of the lifecycle phase // of a Velero backup. // +kubebuilder:validation:Enum=New;Queued;ReadyToStart;FailedValidation;InProgress;WaitingForPluginOperations;WaitingForPluginOperationsPartiallyFailed;Finalizing;FinalizingPartiallyFailed;Completed;PartiallyFailed;Failed;Deleting type BackupPhase string const ( // BackupPhaseNew means the backup has been created but not // yet processed by the BackupController. BackupPhaseNew BackupPhase = "New" // BackupPhaseQueued means the backup has been added to the queue and is waiting for the Queue to move it out of the queue. BackupPhaseQueued BackupPhase = "Queued" // BackupPhaseReadyToStart means the backup has been pulled from the queue and is ready to start. BackupPhaseReadyToStart BackupPhase = "ReadyToStart" // BackupPhaseFailedValidation means the backup has failed // the controller's validations and therefore will not run. BackupPhaseFailedValidation BackupPhase = "FailedValidation" // BackupPhaseInProgress means the backup is currently executing. BackupPhaseInProgress BackupPhase = "InProgress" // BackupPhaseWaitingForPluginOperations means the backup of // Kubernetes resources, creation of snapshots, and other // async plugin operations was successful and snapshot data is // currently uploading or other plugin operations are still // ongoing. The backup is not usable yet. BackupPhaseWaitingForPluginOperations BackupPhase = "WaitingForPluginOperations" // BackupPhaseWaitingForPluginOperationsPartiallyFailed means // the backup of Kubernetes resources, creation of snapshots, // and other async plugin operations partially failed (final // phase will be PartiallyFailed) and snapshot data is // currently uploading or other plugin operations are still // ongoing. The backup is not usable yet. BackupPhaseWaitingForPluginOperationsPartiallyFailed BackupPhase = "WaitingForPluginOperationsPartiallyFailed" // BackupPhaseFinalizing means the backup of // Kubernetes resources, creation of snapshots, and other // async plugin operations were successful and snapshot upload and // other plugin operations are now complete, but the Backup is awaiting // final update of resources modified during async operations. // The backup is not usable yet. BackupPhaseFinalizing BackupPhase = "Finalizing" // BackupPhaseFinalizingPartiallyFailed means the backup of // Kubernetes resources, creation of snapshots, and other // async plugin operations were successful and snapshot upload and // other plugin operations are now complete, but one or more errors // occurred during backup or async operation processing, and the // Backup is awaiting final update of resources modified during async // operations. The backup is not usable yet. BackupPhaseFinalizingPartiallyFailed BackupPhase = "FinalizingPartiallyFailed" // BackupPhaseCompleted means the backup has run successfully without // errors. BackupPhaseCompleted BackupPhase = "Completed" // BackupPhasePartiallyFailed means the backup has run to completion // but encountered 1+ errors backing up individual items. BackupPhasePartiallyFailed BackupPhase = "PartiallyFailed" // BackupPhaseFailed means the backup ran but encountered an error that // prevented it from completing successfully. BackupPhaseFailed BackupPhase = "Failed" // BackupPhaseDeleting means the backup and all its associated data are being deleted. BackupPhaseDeleting BackupPhase = "Deleting" ) // BackupStatus captures the current status of a Velero backup. type BackupStatus struct { // Version is the backup format major version. // Deprecated: Please see FormatVersion // +optional Version int `json:"version,omitempty"` // FormatVersion is the backup format version, including major, minor, and patch version. // +optional FormatVersion string `json:"formatVersion,omitempty"` // Expiration is when this Backup is eligible for garbage-collection. // +optional // +nullable Expiration *metav1.Time `json:"expiration,omitempty"` // Phase is the current state of the Backup. // +optional Phase BackupPhase `json:"phase,omitempty"` // QueuePosition is the position of the backup in the queue. // Only relevant when Phase is "Queued" // +optional QueuePosition int `json:"queuePosition,omitempty"` // ValidationErrors is a slice of all validation errors (if // applicable). // +optional // +nullable ValidationErrors []string `json:"validationErrors,omitempty"` // StartTimestamp records the time a backup was started. // Separate from CreationTimestamp, since that value changes // on restores. // The server's time is used for StartTimestamps // +optional // +nullable StartTimestamp *metav1.Time `json:"startTimestamp,omitempty"` // CompletionTimestamp records the time a backup was completed. // Completion time is recorded even on failed backups. // Completion time is recorded before uploading the backup object. // The server's time is used for CompletionTimestamps // +optional // +nullable CompletionTimestamp *metav1.Time `json:"completionTimestamp,omitempty"` // VolumeSnapshotsAttempted is the total number of attempted // volume snapshots for this backup. // +optional VolumeSnapshotsAttempted int `json:"volumeSnapshotsAttempted,omitempty"` // VolumeSnapshotsCompleted is the total number of successfully // completed volume snapshots for this backup. // +optional VolumeSnapshotsCompleted int `json:"volumeSnapshotsCompleted,omitempty"` // FailureReason is an error that caused the entire backup to fail. // +optional FailureReason string `json:"failureReason,omitempty"` // Warnings is a count of all warning messages that were generated during // execution of the backup. The actual warnings are in the backup's log // file in object storage. // +optional Warnings int `json:"warnings,omitempty"` // Errors is a count of all error messages that were generated during // execution of the backup. The actual errors are in the backup's log // file in object storage. // +optional Errors int `json:"errors,omitempty"` // Progress contains information about the backup's execution progress. Note // that this information is best-effort only -- if Velero fails to update it // during a backup for any reason, it may be inaccurate/stale. // +optional // +nullable Progress *BackupProgress `json:"progress,omitempty"` // CSIVolumeSnapshotsAttempted is the total number of attempted // CSI VolumeSnapshots for this backup. // +optional CSIVolumeSnapshotsAttempted int `json:"csiVolumeSnapshotsAttempted,omitempty"` // CSIVolumeSnapshotsCompleted is the total number of successfully // completed CSI VolumeSnapshots for this backup. // +optional CSIVolumeSnapshotsCompleted int `json:"csiVolumeSnapshotsCompleted,omitempty"` // BackupItemOperationsAttempted is the total number of attempted // async BackupItemAction operations for this backup. // +optional BackupItemOperationsAttempted int `json:"backupItemOperationsAttempted,omitempty"` // BackupItemOperationsCompleted is the total number of successfully completed // async BackupItemAction operations for this backup. // +optional BackupItemOperationsCompleted int `json:"backupItemOperationsCompleted,omitempty"` // BackupItemOperationsFailed is the total number of async // BackupItemAction operations for this backup which ended with an error. // +optional BackupItemOperationsFailed int `json:"backupItemOperationsFailed,omitempty"` // HookStatus contains information about the status of the hooks. // +optional // +nullable HookStatus *HookStatus `json:"hookStatus,omitempty"` } // BackupProgress stores information about the progress of a Backup's execution. type BackupProgress struct { // TotalItems is the total number of items to be backed up. This number may change // throughout the execution of the backup due to plugins that return additional related // items to back up, the velero.io/exclude-from-backup label, and various other // filters that happen as items are processed. // +optional TotalItems int `json:"totalItems,omitempty"` // ItemsBackedUp is the number of items that have actually been written to the // backup tarball so far. // +optional ItemsBackedUp int `json:"itemsBackedUp,omitempty"` } // HookStatus stores information about the status of the hooks. type HookStatus struct { // HooksAttempted is the total number of attempted hooks // Specifically, HooksAttempted represents the number of hooks that failed to execute // and the number of hooks that executed successfully. // +optional HooksAttempted int `json:"hooksAttempted,omitempty"` // HooksFailed is the total number of hooks which ended with an error // +optional HooksFailed int `json:"hooksFailed,omitempty"` } // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // +kubebuilder:object:root=true // +kubebuilder:object:generate=true // +kubebuilder:storageversion // +kubebuilder:rbac:groups=velero.io,resources=backups,verbs=create;delete;get;list;patch;update;watch // +kubebuilder:rbac:groups=velero.io,resources=backups/status,verbs=get;update;patch // Backup is a Velero resource that represents the capture of Kubernetes // cluster state at a point in time (API objects and associated volume state). type Backup struct { metav1.TypeMeta `json:",inline"` // +optional metav1.ObjectMeta `json:"metadata,omitempty"` // +optional Spec BackupSpec `json:"spec,omitempty"` // +optional Status BackupStatus `json:"status,omitempty"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // BackupList is a list of Backups. type BackupList struct { metav1.TypeMeta `json:",inline"` // +optional metav1.ListMeta `json:"metadata,omitempty"` Items []Backup `json:"items"` } ================================================ FILE: pkg/apis/velero/v1/backupstoragelocation_types.go ================================================ /* Copyright 2017, 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v1 import ( "errors" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" ) // BackupStorageLocationSpec defines the desired state of a Velero BackupStorageLocation type BackupStorageLocationSpec struct { // Provider is the provider of the backup storage. Provider string `json:"provider"` // Config is for provider-specific configuration fields. // +optional Config map[string]string `json:"config,omitempty"` // Credential contains the credential information intended to be used with this location // +optional Credential *corev1api.SecretKeySelector `json:"credential,omitempty"` StorageType `json:",inline"` // Default indicates this location is the default backup storage location. // +optional Default bool `json:"default,omitempty"` // AccessMode defines the permissions for the backup storage location. // +optional AccessMode BackupStorageLocationAccessMode `json:"accessMode,omitempty"` // BackupSyncPeriod defines how frequently to sync backup API objects from object storage. A value of 0 disables sync. // +optional // +nullable BackupSyncPeriod *metav1.Duration `json:"backupSyncPeriod,omitempty"` // ValidationFrequency defines how frequently to validate the corresponding object storage. A value of 0 disables validation. // +optional // +nullable ValidationFrequency *metav1.Duration `json:"validationFrequency,omitempty"` } // BackupStorageLocationStatus defines the observed state of BackupStorageLocation type BackupStorageLocationStatus struct { // Phase is the current state of the BackupStorageLocation. // +optional Phase BackupStorageLocationPhase `json:"phase,omitempty"` // LastSyncedTime is the last time the contents of the location were synced into // the cluster. // +optional // +nullable LastSyncedTime *metav1.Time `json:"lastSyncedTime,omitempty"` // LastValidationTime is the last time the backup store location was validated // the cluster. // +optional // +nullable LastValidationTime *metav1.Time `json:"lastValidationTime,omitempty"` // Message is a message about the backup storage location's status. // +optional Message string `json:"message,omitempty"` // LastSyncedRevision is the value of the `metadata/revision` file in the backup // storage location the last time the BSL's contents were synced into the cluster. // // Deprecated: this field is no longer updated or used for detecting changes to // the location's contents and will be removed entirely in v2.0. // +optional LastSyncedRevision types.UID `json:"lastSyncedRevision,omitempty"` // AccessMode is an unused field. // // Deprecated: there is now an AccessMode field on the Spec and this field // will be removed entirely as of v2.0. // +optional AccessMode BackupStorageLocationAccessMode `json:"accessMode,omitempty"` } // TODO(2.0) After converting all resources to use the runtime-controller client, // the genclient and k8s:deepcopy markers will no longer be needed and should be removed. // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // +kubebuilder:object:root=true // +kubebuilder:resource:shortName=bsl // +kubebuilder:object:generate=true // +kubebuilder:storageversion // +kubebuilder:printcolumn:name="Phase",type="string",JSONPath=".status.phase",description="Backup Storage Location status such as Available/Unavailable" // +kubebuilder:printcolumn:name="Last Validated",type="date",JSONPath=".status.lastValidationTime",description="LastValidationTime is the last time the backup store location was validated" // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" // +kubebuilder:printcolumn:name="Default",type="boolean",JSONPath=".spec.default",description="Default backup storage location" // BackupStorageLocation is a location where Velero stores backup objects type BackupStorageLocation struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` Spec BackupStorageLocationSpec `json:"spec,omitempty"` Status BackupStorageLocationStatus `json:"status,omitempty"` } // TODO(2.0) After converting all resources to use the runtime-controller client, // the k8s:deepcopy marker will no longer be needed and should be removed. // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // +kubebuilder:object:root=true // +kubebuilder:rbac:groups=velero.io,resources=backupstoragelocations,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=velero.io,resources=backupstoragelocations/status,verbs=get;update;patch // BackupStorageLocationList contains a list of BackupStorageLocation type BackupStorageLocationList struct { metav1.TypeMeta `json:",inline"` metav1.ListMeta `json:"metadata,omitempty"` Items []BackupStorageLocation `json:"items"` } // StorageType represents the type of storage that a backup location uses. // ObjectStorage must be non-nil, since it is currently the only supported StorageType. type StorageType struct { ObjectStorage *ObjectStorageLocation `json:"objectStorage"` } // ObjectStorageLocation specifies the settings necessary to connect to a provider's object storage. type ObjectStorageLocation struct { // Bucket is the bucket to use for object storage. Bucket string `json:"bucket"` // Prefix is the path inside a bucket to use for Velero storage. Optional. // +optional Prefix string `json:"prefix,omitempty"` // CACert defines a CA bundle to use when verifying TLS connections to the provider. // Deprecated: Use CACertRef instead. // +optional CACert []byte `json:"caCert,omitempty"` // CACertRef is a reference to a Secret containing the CA certificate bundle to use // when verifying TLS connections to the provider. The Secret must be in the same // namespace as the BackupStorageLocation. // +optional CACertRef *corev1api.SecretKeySelector `json:"caCertRef,omitempty"` } // BackupStorageLocationPhase is the lifecycle phase of a Velero BackupStorageLocation. // +kubebuilder:validation:Enum=Available;Unavailable // +kubebuilder:default=Unavailable type BackupStorageLocationPhase string const ( // BackupStorageLocationPhaseAvailable means the location is available to read and write from. BackupStorageLocationPhaseAvailable BackupStorageLocationPhase = "Available" // BackupStorageLocationPhaseUnavailable means the location is unavailable to read and write from. BackupStorageLocationPhaseUnavailable BackupStorageLocationPhase = "Unavailable" ) // BackupStorageLocationAccessMode represents the permissions for a BackupStorageLocation. // +kubebuilder:validation:Enum=ReadOnly;ReadWrite type BackupStorageLocationAccessMode string const ( // BackupStorageLocationAccessModeReadOnly represents read-only access to a BackupStorageLocation. BackupStorageLocationAccessModeReadOnly BackupStorageLocationAccessMode = "ReadOnly" // BackupStorageLocationAccessModeReadWrite represents read and write access to a BackupStorageLocation. BackupStorageLocationAccessModeReadWrite BackupStorageLocationAccessMode = "ReadWrite" ) // TODO(2.0): remove the AccessMode field from BackupStorageLocationStatus. // TODO(2.0): remove the LastSyncedRevision field from BackupStorageLocationStatus. // Validate validates the BackupStorageLocation to ensure that only one of CACert or CACertRef is set. func (bsl *BackupStorageLocation) Validate() error { if bsl.Spec.ObjectStorage != nil && bsl.Spec.ObjectStorage.CACert != nil && bsl.Spec.ObjectStorage.CACertRef != nil { return errors.New("cannot specify both caCert and caCertRef in objectStorage") } return nil } ================================================ FILE: pkg/apis/velero/v1/backupstoragelocation_types_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v1 import ( "testing" corev1api "k8s.io/api/core/v1" ) func TestBackupStorageLocationValidate(t *testing.T) { tests := []struct { name string bsl *BackupStorageLocation expectError bool }{ { name: "valid - neither CACert nor CACertRef set", bsl: &BackupStorageLocation{ Spec: BackupStorageLocationSpec{ StorageType: StorageType{ ObjectStorage: &ObjectStorageLocation{ Bucket: "test-bucket", }, }, }, }, expectError: false, }, { name: "valid - only CACert set", bsl: &BackupStorageLocation{ Spec: BackupStorageLocationSpec{ StorageType: StorageType{ ObjectStorage: &ObjectStorageLocation{ Bucket: "test-bucket", CACert: []byte("test-cert"), }, }, }, }, expectError: false, }, { name: "valid - only CACertRef set", bsl: &BackupStorageLocation{ Spec: BackupStorageLocationSpec{ StorageType: StorageType{ ObjectStorage: &ObjectStorageLocation{ Bucket: "test-bucket", CACertRef: &corev1api.SecretKeySelector{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "ca-cert-secret", }, Key: "ca.crt", }, }, }, }, }, expectError: false, }, { name: "invalid - both CACert and CACertRef set", bsl: &BackupStorageLocation{ Spec: BackupStorageLocationSpec{ StorageType: StorageType{ ObjectStorage: &ObjectStorageLocation{ Bucket: "test-bucket", CACert: []byte("test-cert"), CACertRef: &corev1api.SecretKeySelector{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "ca-cert-secret", }, Key: "ca.crt", }, }, }, }, }, expectError: true, }, { name: "valid - no ObjectStorage", bsl: &BackupStorageLocation{ Spec: BackupStorageLocationSpec{ StorageType: StorageType{ ObjectStorage: nil, }, }, }, expectError: false, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { err := test.bsl.Validate() if test.expectError && err == nil { t.Errorf("expected error but got none") } if !test.expectError && err != nil { t.Errorf("expected no error but got: %v", err) } }) } } ================================================ FILE: pkg/apis/velero/v1/constants.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v1 const ( // DefaultNamespace is the Kubernetes namespace that is used by default for // the Velero server and API objects. DefaultNamespace = "velero" // ResourcesDir is a top-level directory expected in backups which contains sub-directories // for each resource type in the backup. ResourcesDir = "resources" // MetadataDir is a top-level directory expected in backups which contains // files that store metadata about the backup, such as the backup version. MetadataDir = "metadata" // ClusterScopedDir is the name of the directory containing cluster-scoped // resources within a Velero backup. ClusterScopedDir = "cluster" // NamespaceScopedDir is the name of the directory containing namespace-scoped // resource within a Velero backup. NamespaceScopedDir = "namespaces" // CSIFeatureFlag is the feature flag string that defines whether or not CSI features are being used. CSIFeatureFlag = "EnableCSI" // PreferredVersionDir is the suffix name of the directory containing the preferred version of the API group // resource within a Velero backup. PreferredVersionDir = "-preferredversion" // APIGroupVersionsFeatureFlag is the feature flag string that defines whether or not to handle multiple API Group Versions APIGroupVersionsFeatureFlag = "EnableAPIGroupVersions" ) ================================================ FILE: pkg/apis/velero/v1/delete_backup_request_types.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v1 import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" // DeleteBackupRequestSpec is the specification for which backups to delete. type DeleteBackupRequestSpec struct { BackupName string `json:"backupName"` } // DeleteBackupRequestPhase represents the lifecycle phase of a DeleteBackupRequest. // +kubebuilder:validation:Enum=New;InProgress;Processed type DeleteBackupRequestPhase string const ( // DeleteBackupRequestPhaseNew means the DeleteBackupRequest has not been processed yet. DeleteBackupRequestPhaseNew DeleteBackupRequestPhase = "New" // DeleteBackupRequestPhaseInProgress means the DeleteBackupRequest is being processed. DeleteBackupRequestPhaseInProgress DeleteBackupRequestPhase = "InProgress" // DeleteBackupRequestPhaseProcessed means the DeleteBackupRequest has been processed. DeleteBackupRequestPhaseProcessed DeleteBackupRequestPhase = "Processed" ) // DeleteBackupRequestStatus is the current status of a DeleteBackupRequest. type DeleteBackupRequestStatus struct { // Phase is the current state of the DeleteBackupRequest. // +optional Phase DeleteBackupRequestPhase `json:"phase,omitempty"` // Errors contains any errors that were encountered during the deletion process. // +optional // +nullable Errors []string `json:"errors,omitempty"` } // TODO(2.0) After converting all resources to use the runtime-controller client, the genclient and k8s:deepcopy markers will no longer be needed and should be removed. // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // +kubebuilder:object:root=true // +kubebuilder:object:generate=true // +kubebuilder:storageversion // +kubebuilder:printcolumn:name="BackupName",type="string",JSONPath=".spec.backupName",description="The name of the backup to be deleted" // +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.phase",description="The status of the deletion request" // DeleteBackupRequest is a request to delete one or more backups. type DeleteBackupRequest struct { metav1.TypeMeta `json:",inline"` // +optional metav1.ObjectMeta `json:"metadata,omitempty"` // +optional Spec DeleteBackupRequestSpec `json:"spec,omitempty"` // +optional Status DeleteBackupRequestStatus `json:"status,omitempty"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // +kubebuilder:object:root=true // DeleteBackupRequestList is a list of DeleteBackupRequests. type DeleteBackupRequestList struct { metav1.TypeMeta `json:",inline"` // +optional metav1.ListMeta `json:"metadata,omitempty"` Items []DeleteBackupRequest `json:"items"` } ================================================ FILE: pkg/apis/velero/v1/doc.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ // +k8s:deepcopy-gen=package // Package v1 is the v1 version of the API. // +groupName=velero.io package v1 ================================================ FILE: pkg/apis/velero/v1/download_request_types.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v1 import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" // DownloadRequestSpec is the specification for a download request. type DownloadRequestSpec struct { // Target is what to download (e.g. logs for a backup). Target DownloadTarget `json:"target"` } // DownloadTargetKind represents what type of file to download. // +kubebuilder:validation:Enum=BackupLog;BackupContents;BackupVolumeSnapshots;BackupItemOperations;BackupResourceList;BackupResults;RestoreLog;RestoreResults;RestoreResourceList;RestoreItemOperations;CSIBackupVolumeSnapshots;CSIBackupVolumeSnapshotContents;BackupVolumeInfos;RestoreVolumeInfo type DownloadTargetKind string const ( DownloadTargetKindBackupLog DownloadTargetKind = "BackupLog" DownloadTargetKindBackupContents DownloadTargetKind = "BackupContents" DownloadTargetKindBackupVolumeSnapshots DownloadTargetKind = "BackupVolumeSnapshots" DownloadTargetKindBackupItemOperations DownloadTargetKind = "BackupItemOperations" DownloadTargetKindBackupResourceList DownloadTargetKind = "BackupResourceList" DownloadTargetKindBackupResults DownloadTargetKind = "BackupResults" DownloadTargetKindRestoreLog DownloadTargetKind = "RestoreLog" DownloadTargetKindRestoreResults DownloadTargetKind = "RestoreResults" DownloadTargetKindRestoreResourceList DownloadTargetKind = "RestoreResourceList" DownloadTargetKindRestoreItemOperations DownloadTargetKind = "RestoreItemOperations" DownloadTargetKindCSIBackupVolumeSnapshots DownloadTargetKind = "CSIBackupVolumeSnapshots" DownloadTargetKindCSIBackupVolumeSnapshotContents DownloadTargetKind = "CSIBackupVolumeSnapshotContents" DownloadTargetKindBackupVolumeInfos DownloadTargetKind = "BackupVolumeInfos" DownloadTargetKindRestoreVolumeInfo DownloadTargetKind = "RestoreVolumeInfo" ) // DownloadTarget is the specification for what kind of file to download, and the name of the // resource with which it's associated. type DownloadTarget struct { // Kind is the type of file to download. Kind DownloadTargetKind `json:"kind"` // Name is the name of the Kubernetes resource with which the file is associated. Name string `json:"name"` } // DownloadRequestPhase represents the lifecycle phase of a DownloadRequest. // +kubebuilder:validation:Enum=New;Processed type DownloadRequestPhase string const ( // DownloadRequestPhaseNew means the DownloadRequest has not been processed by the // DownloadRequestController yet. DownloadRequestPhaseNew DownloadRequestPhase = "New" // DownloadRequestPhaseProcessed means the DownloadRequest has been processed by the // DownloadRequestController. DownloadRequestPhaseProcessed DownloadRequestPhase = "Processed" ) // DownloadRequestStatus is the current status of a DownloadRequest. type DownloadRequestStatus struct { // Phase is the current state of the DownloadRequest. // +optional Phase DownloadRequestPhase `json:"phase,omitempty"` // DownloadURL contains the pre-signed URL for the target file. // +optional DownloadURL string `json:"downloadURL,omitempty"` // Expiration is when this DownloadRequest expires and can be deleted by the system. // +optional // +nullable Expiration *metav1.Time `json:"expiration,omitempty"` } // TODO(2.0) After converting all resources to use the runtime-controller client, // the k8s:deepcopy marker will no longer be needed and should be removed. // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // +kubebuilder:object:root=true // +kubebuilder:object:generate=true // +kubebuilder:storageversion // DownloadRequest is a request to download an artifact from backup object storage, such as a backup // log file. type DownloadRequest struct { metav1.TypeMeta `json:",inline"` // +optional metav1.ObjectMeta `json:"metadata,omitempty"` // +optional Spec DownloadRequestSpec `json:"spec,omitempty"` // +optional Status DownloadRequestStatus `json:"status,omitempty"` } // TODO(2.0) After converting all resources to use the runtime-controller client, // the k8s:deepcopy marker will no longer be needed and should be removed. // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // +kubebuilder:object:root=true // +kubebuilder:rbac:groups=velero.io,resources=downloadrequests,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=velero.io,resources=downloadrequests/status,verbs=get;update;patch // DownloadRequestList is a list of DownloadRequests. type DownloadRequestList struct { metav1.TypeMeta `json:",inline"` // +optional metav1.ListMeta `json:"metadata,omitempty"` Items []DownloadRequest `json:"items"` } ================================================ FILE: pkg/apis/velero/v1/groupversion_info.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ // Package v1 contains API Schema definitions for the velero v1 API group // +kubebuilder:object:generate=true // +groupName=velero.io package v1 import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" ) var ( // SchemeGroupVersion is group version used to register these objects SchemeGroupVersion = schema.GroupVersion{Group: "velero.io", Version: "v1"} // SchemeBuilder is used to add go types to the GroupVersionKind scheme SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) // AddToScheme adds the types in this group-version to the given scheme. AddToScheme = SchemeBuilder.AddToScheme ) ================================================ FILE: pkg/apis/velero/v1/labels_annotations.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v1 const ( // BackupNameLabel is the label key used to identify a backup by name. BackupNameLabel = "velero.io/backup-name" // BackupUIDLabel is the label key used to identify a backup by uid. BackupUIDLabel = "velero.io/backup-uid" // RestoreNameLabel is the label key used to identify a restore by name. RestoreNameLabel = "velero.io/restore-name" // ScheduleNameLabel is the label key used to identify a schedule by name. ScheduleNameLabel = "velero.io/schedule-name" // RestoreUIDLabel is the label key used to identify a restore by uid. RestoreUIDLabel = "velero.io/restore-uid" // PodUIDLabel is the label key used to identify a pod by uid. PodUIDLabel = "velero.io/pod-uid" // PVCUIDLabel is the label key used to identify a PVC by uid. PVCUIDLabel = "velero.io/pvc-uid" // PodVolumeOperationTimeoutAnnotation is the annotation key used to apply // a backup/restore-specific timeout value for pod volume operations (i.e. // pod volume backups/restores). PodVolumeOperationTimeoutAnnotation = "velero.io/pod-volume-timeout" // StorageLocationLabel is the label key used to identify the storage // location of a backup. StorageLocationLabel = "velero.io/storage-location" // VolumeNamespaceLabel is the label key used to identify which // namespace a repository stores backups for. VolumeNamespaceLabel = "velero.io/volume-namespace" // RepositoryTypeLabel is the label key used to identify the type of a repository RepositoryTypeLabel = "velero.io/repository-type" // DataUploadLabel is the label key used to identify the dataupload for snapshot backup pod DataUploadLabel = "velero.io/data-upload" // DataUploadSnapshotInfoLabel is used to identify the configmap that contains the snapshot info of a data upload // normally the value of the label should the "true" or "false" DataUploadSnapshotInfoLabel = "velero.io/data-upload-snapshot-info" // DataDownloadLabel is the label key used to identify the datadownload for snapshot restore pod DataDownloadLabel = "velero.io/data-download" // SourceClusterK8sVersionAnnotation is the label key used to identify the k8s // git version of the backup , i.e. v1.16.4 SourceClusterK8sGitVersionAnnotation = "velero.io/source-cluster-k8s-gitversion" // SourceClusterK8sMajorVersionAnnotation is the label key used to identify the k8s // major version of the backup , i.e. 1 SourceClusterK8sMajorVersionAnnotation = "velero.io/source-cluster-k8s-major-version" // SourceClusterK8sMajorVersionAnnotation is the label key used to identify the k8s // minor version of the backup , i.e. 16 SourceClusterK8sMinorVersionAnnotation = "velero.io/source-cluster-k8s-minor-version" // ResourceTimeoutAnnotation is the annotation key used to carry the global resource // timeout value for backup to plugins. ResourceTimeoutAnnotation = "velero.io/resource-timeout" // AsyncOperationIDLabel is the label key used to identify the async operation ID AsyncOperationIDLabel = "velero.io/async-operation-id" // PVCNameLabel is the label key used to identify the PVC's namespace and name. // The format is /. PVCNamespaceNameLabel = "velero.io/pvc-namespace-name" // ResourceUsageLabel is the label key to explain the Velero resource usage. ResourceUsageLabel = "velero.io/resource-usage" // VolumesToBackupAnnotation is the annotation on a pod whose mounted volumes // need to be backed up using pod volume backup. VolumesToBackupAnnotation = "backup.velero.io/backup-volumes" // VolumesToExcludeAnnotation is the annotation on a pod whose mounted volumes // should be excluded from pod volume backup. VolumesToExcludeAnnotation = "backup.velero.io/backup-volumes-excludes" // ExcludeFromBackupLabel is the label to exclude k8s resource from backup, // even if the resource contains a matching selector label. ExcludeFromBackupLabel = "velero.io/exclude-from-backup" // SkipFromBackupAnnotation is the annotation used by internal BackupItemActions // to indicate that a resource should be skipped from backup, // even if it doesn't have the ExcludeFromBackupLabel. // This is used in cases where we want to skip backup of a resource based on some logic in a plugin. // // Notice: SkipFromBackupAnnotation's priority is higher than MustIncludeAdditionalItemAnnotation. // If SkipFromBackupAnnotation is set, the resource will be skipped even if MustIncludeAdditionalItemAnnotation is set. SkipFromBackupAnnotation = "velero.io/skip-from-backup" // defaultVGSLabelKey is the default label key used to group PVCs under a VolumeGroupSnapshot DefaultVGSLabelKey = "velero.io/volume-group" // PVBLabel is the label key used to identify the pvb for pvb pod PVBLabel = "velero.io/pod-volume-backup" // PVRLabel is the label key used to identify the pvb for pvr pod PVRLabel = "velero.io/pod-volume-restore" ) type AsyncOperationIDPrefix string const ( AsyncOperationIDPrefixDataDownload AsyncOperationIDPrefix = "dd-" AsyncOperationIDPrefixDataUpload AsyncOperationIDPrefix = "du-" ) type VeleroResourceUsage string const ( VeleroResourceUsageDataUploadResult VeleroResourceUsage = "DataUpload" ) // CSI related plugin actions' constant variable const ( VolumeSnapshotLabel = "velero.io/volume-snapshot-name" VolumeSnapshotHandleAnnotation = "velero.io/csi-volumesnapshot-handle" VolumeSnapshotRestoreSize = "velero.io/csi-volumesnapshot-restore-size" DriverNameAnnotation = "velero.io/csi-driver-name" VSCDeletionPolicyAnnotation = "velero.io/csi-vsc-deletion-policy" VolumeSnapshotClassSelectorLabel = "velero.io/csi-volumesnapshot-class" VolumeSnapshotClassDriverBackupAnnotationPrefix = "velero.io/csi-volumesnapshot-class" VolumeSnapshotClassDriverPVCAnnotation = "velero.io/csi-volumesnapshot-class" // https://kubernetes.io/zh-cn/docs/concepts/storage/volume-snapshot-classes/ VolumeSnapshotClassKubernetesAnnotation = "snapshot.storage.kubernetes.io/is-default-class" // There is no release w/ these constants exported. Using the strings for now. // CSI Annotation volumesnapshotclass // https://github.com/kubernetes-csi/external-snapshotter/blob/master/pkg/utils/util.go#L59-L60 PrefixedListSecretNameAnnotation = "csi.storage.k8s.io/snapshotter-list-secret-name" // #nosec G101 PrefixedListSecretNamespaceAnnotation = "csi.storage.k8s.io/snapshotter-list-secret-namespace" // #nosec G101 // CSI Annotation volumesnapshotcontents PrefixedSecretNameAnnotation = "csi.storage.k8s.io/snapshotter-secret-name" // #nosec G101 PrefixedSecretNamespaceAnnotation = "csi.storage.k8s.io/snapshotter-secret-namespace" // #nosec G101 // Velero checks this annotation to determine whether to skip resource excluding check. MustIncludeAdditionalItemAnnotation = "backup.velero.io/must-include-additional-items" // SkippedNoCSIPVAnnotation - Velero checks this annotation on processed PVC to // find out if the snapshot was skipped b/c the PV is not provisioned via CSI SkippedNoCSIPVAnnotation = "backup.velero.io/skipped-no-csi-pv" // DynamicPVRestoreLabel is the label key for dynamic PV restore DynamicPVRestoreLabel = "velero.io/dynamic-pv-restore" // DataUploadNameAnnotation is the label key for the DataUpload name DataUploadNameAnnotation = "velero.io/data-upload-name" // Label used on VolumeGroupSnapshotClass to mark it as Velero's default for a CSI driver VolumeGroupSnapshotClassDefaultLabel = "velero.io/csi-volumegroupsnapshot-class" // Annotation on PVC to override the VGS class to use VolumeGroupSnapshotClassAnnotationPVC = "velero.io/csi-volume-group-snapshot-class" // Annotation prefix on Backup to override VGS class per CSI driver VolumeGroupSnapshotClassAnnotationBackupPrefix = "velero.io/csi-volumegroupsnapshot-class_" ) ================================================ FILE: pkg/apis/velero/v1/pod_volume_backup_types.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v1 import ( corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/vmware-tanzu/velero/pkg/apis/velero/shared" ) // PodVolumeBackupSpec is the specification for a PodVolumeBackup. type PodVolumeBackupSpec struct { // Node is the name of the node that the Pod is running on. Node string `json:"node"` // Pod is a reference to the pod containing the volume to be backed up. Pod corev1api.ObjectReference `json:"pod"` // Volume is the name of the volume within the Pod to be backed // up. Volume string `json:"volume"` // BackupStorageLocation is the name of the backup storage location // where the backup repository is stored. BackupStorageLocation string `json:"backupStorageLocation"` // RepoIdentifier is the backup repository identifier. RepoIdentifier string `json:"repoIdentifier"` // UploaderType is the type of the uploader to handle the data transfer. // +kubebuilder:validation:Enum=kopia;restic;"" // +optional UploaderType string `json:"uploaderType"` // Tags are a map of key-value pairs that should be applied to the // volume backup as tags. // +optional Tags map[string]string `json:"tags,omitempty"` // UploaderSettings are a map of key-value pairs that should be applied to the // uploader configuration. // +optional // +nullable UploaderSettings map[string]string `json:"uploaderSettings,omitempty"` // Cancel indicates request to cancel the ongoing PodVolumeBackup. It can be set // when the PodVolumeBackup is in InProgress phase Cancel bool `json:"cancel,omitempty"` } // PodVolumeBackupPhase represents the lifecycle phase of a PodVolumeBackup. // +kubebuilder:validation:Enum=New;Accepted;Prepared;InProgress;Canceling;Canceled;Completed;Failed type PodVolumeBackupPhase string const ( PodVolumeBackupPhaseNew PodVolumeBackupPhase = "New" PodVolumeBackupPhaseAccepted PodVolumeBackupPhase = "Accepted" PodVolumeBackupPhasePrepared PodVolumeBackupPhase = "Prepared" PodVolumeBackupPhaseInProgress PodVolumeBackupPhase = "InProgress" PodVolumeBackupPhaseCanceling PodVolumeBackupPhase = "Canceling" PodVolumeBackupPhaseCanceled PodVolumeBackupPhase = "Canceled" PodVolumeBackupPhaseCompleted PodVolumeBackupPhase = "Completed" PodVolumeBackupPhaseFailed PodVolumeBackupPhase = "Failed" ) // PodVolumeBackupStatus is the current status of a PodVolumeBackup. type PodVolumeBackupStatus struct { // Phase is the current state of the PodVolumeBackup. // +optional Phase PodVolumeBackupPhase `json:"phase,omitempty"` // Path is the full path within the controller pod being backed up. // +optional Path string `json:"path,omitempty"` // SnapshotID is the identifier for the snapshot of the pod volume. // +optional SnapshotID string `json:"snapshotID,omitempty"` // Message is a message about the pod volume backup's status. // +optional Message string `json:"message,omitempty"` // StartTimestamp records the time a backup was started. // Separate from CreationTimestamp, since that value changes // on restores. // The server's time is used for StartTimestamps // +optional // +nullable StartTimestamp *metav1.Time `json:"startTimestamp,omitempty"` // CompletionTimestamp records the time a backup was completed. // Completion time is recorded even on failed backups. // Completion time is recorded before uploading the backup object. // The server's time is used for CompletionTimestamps // +optional // +nullable CompletionTimestamp *metav1.Time `json:"completionTimestamp,omitempty"` // Progress holds the total number of bytes of the volume and the current // number of backed up bytes. This can be used to display progress information // about the backup operation. // +optional Progress shared.DataMoveOperationProgress `json:"progress,omitempty"` // IncrementalBytes holds the number of bytes new or changed since the last backup // +optional IncrementalBytes int64 `json:"incrementalBytes,omitempty"` // AcceptedTimestamp records the time the pod volume backup is to be prepared. // The server's time is used for AcceptedTimestamp // +optional // +nullable AcceptedTimestamp *metav1.Time `json:"acceptedTimestamp,omitempty"` } // TODO(2.0) After converting all resources to use the runttime-controller client, // the genclient and k8s:deepcopy markers will no longer be needed and should be removed. // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // +kubebuilder:storageversion // +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.phase",description="PodVolumeBackup status such as New/InProgress" // +kubebuilder:printcolumn:name="Started",type="date",JSONPath=".status.startTimestamp",description="Time duration since this PodVolumeBackup was started" // +kubebuilder:printcolumn:name="Bytes Done",type="integer",format="int64",JSONPath=".status.progress.bytesDone",description="Completed bytes" // +kubebuilder:printcolumn:name="Total Bytes",type="integer",format="int64",JSONPath=".status.progress.totalBytes",description="Total bytes" // +kubebuilder:printcolumn:name="Incremental Bytes",type="integer",format="int64",JSONPath=".status.incrementalBytes",description="Incremental bytes",priority=10 // +kubebuilder:printcolumn:name="Storage Location",type="string",JSONPath=".spec.backupStorageLocation",description="Name of the Backup Storage Location where this backup should be stored" // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Time duration since this PodVolumeBackup was created" // +kubebuilder:printcolumn:name="Node",type="string",JSONPath=".status.node",description="Name of the node where the PodVolumeBackup is processed" // +kubebuilder:printcolumn:name="Uploader",type="string",JSONPath=".spec.uploaderType",description="The type of the uploader to handle data transfer" // +kubebuilder:object:root=true // +kubebuilder:object:generate=true type PodVolumeBackup struct { metav1.TypeMeta `json:",inline"` // +optional metav1.ObjectMeta `json:"metadata,omitempty"` // +optional Spec PodVolumeBackupSpec `json:"spec,omitempty"` // +optional Status PodVolumeBackupStatus `json:"status,omitempty"` } // TODO(2.0) After converting all resources to use the runtime-controller client, // the k8s:deepcopy marker will no longer be needed and should be removed. // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // +kubebuilder:object:root=true // +kubebuilder:rbac:groups=velero.io,resources=podvolumebackups,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=velero.io,resources=podvolumebackups/status,verbs=get;update;patch // PodVolumeBackupList is a list of PodVolumeBackups. type PodVolumeBackupList struct { metav1.TypeMeta `json:",inline"` // +optional metav1.ListMeta `json:"metadata,omitempty"` Items []PodVolumeBackup `json:"items"` } ================================================ FILE: pkg/apis/velero/v1/pod_volume_restore_type.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v1 import ( corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/vmware-tanzu/velero/pkg/apis/velero/shared" ) // PodVolumeRestoreSpec is the specification for a PodVolumeRestore. type PodVolumeRestoreSpec struct { // Pod is a reference to the pod containing the volume to be restored. Pod corev1api.ObjectReference `json:"pod"` // Volume is the name of the volume within the Pod to be restored. Volume string `json:"volume"` // BackupStorageLocation is the name of the backup storage location // where the backup repository is stored. BackupStorageLocation string `json:"backupStorageLocation"` // RepoIdentifier is the backup repository identifier. RepoIdentifier string `json:"repoIdentifier"` // UploaderType is the type of the uploader to handle the data transfer. // +kubebuilder:validation:Enum=kopia;restic;"" // +optional UploaderType string `json:"uploaderType"` // SnapshotID is the ID of the volume snapshot to be restored. SnapshotID string `json:"snapshotID"` // SourceNamespace is the original namespace for namaspace mapping. SourceNamespace string `json:"sourceNamespace"` // UploaderSettings are a map of key-value pairs that should be applied to the // uploader configuration. // +optional // +nullable UploaderSettings map[string]string `json:"uploaderSettings,omitempty"` // Cancel indicates request to cancel the ongoing PodVolumeRestore. It can be set // when the PodVolumeRestore is in InProgress phase Cancel bool `json:"cancel,omitempty"` // SnapshotSize is the logical size in Bytes of the snapshot. // +optional SnapshotSize int64 `json:"snapshotSize,omitempty"` } // PodVolumeRestorePhase represents the lifecycle phase of a PodVolumeRestore. // +kubebuilder:validation:Enum=New;Accepted;Prepared;InProgress;Canceling;Canceled;Completed;Failed type PodVolumeRestorePhase string const ( PodVolumeRestorePhaseNew PodVolumeRestorePhase = "New" PodVolumeRestorePhaseAccepted PodVolumeRestorePhase = "Accepted" PodVolumeRestorePhasePrepared PodVolumeRestorePhase = "Prepared" PodVolumeRestorePhaseInProgress PodVolumeRestorePhase = "InProgress" PodVolumeRestorePhaseCanceling PodVolumeRestorePhase = "Canceling" PodVolumeRestorePhaseCanceled PodVolumeRestorePhase = "Canceled" PodVolumeRestorePhaseCompleted PodVolumeRestorePhase = "Completed" PodVolumeRestorePhaseFailed PodVolumeRestorePhase = "Failed" ) // PodVolumeRestoreStatus is the current status of a PodVolumeRestore. type PodVolumeRestoreStatus struct { // Phase is the current state of the PodVolumeRestore. // +optional Phase PodVolumeRestorePhase `json:"phase,omitempty"` // Message is a message about the pod volume restore's status. // +optional Message string `json:"message,omitempty"` // StartTimestamp records the time a restore was started. // The server's time is used for StartTimestamps // +optional // +nullable StartTimestamp *metav1.Time `json:"startTimestamp,omitempty"` // CompletionTimestamp records the time a restore was completed. // Completion time is recorded even on failed restores. // The server's time is used for CompletionTimestamps // +optional // +nullable CompletionTimestamp *metav1.Time `json:"completionTimestamp,omitempty"` // Progress holds the total number of bytes of the snapshot and the current // number of restored bytes. This can be used to display progress information // about the restore operation. // +optional Progress shared.DataMoveOperationProgress `json:"progress,omitempty"` // AcceptedTimestamp records the time the pod volume restore is to be prepared. // The server's time is used for AcceptedTimestamp // +optional // +nullable AcceptedTimestamp *metav1.Time `json:"acceptedTimestamp,omitempty"` // Node is name of the node where the pod volume restore is processed. // +optional Node string `json:"node,omitempty"` } // TODO(2.0) After converting all resources to use the runtime-controller client, the genclient and k8s:deepcopy markers will no longer be needed and should be removed. // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // +kubebuilder:object:generate=true // +kubebuilder:object:root=true // +kubebuilder:storageversion // +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.phase",description="PodVolumeRestore status such as New/InProgress" // +kubebuilder:printcolumn:name="Started",type="date",JSONPath=".status.startTimestamp",description="Time duration since this PodVolumeRestore was started" // +kubebuilder:printcolumn:name="Bytes Done",type="integer",format="int64",JSONPath=".status.progress.bytesDone",description="Completed bytes" // +kubebuilder:printcolumn:name="Total Bytes",type="integer",format="int64",JSONPath=".status.progress.totalBytes",description="Total bytes" // +kubebuilder:printcolumn:name="Storage Location",type="string",JSONPath=".spec.backupStorageLocation",description="Name of the Backup Storage Location where the backup data is stored" // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Time duration since this PodVolumeRestore was created" // +kubebuilder:printcolumn:name="Node",type="string",JSONPath=".status.node",description="Name of the node where the PodVolumeRestore is processed" // +kubebuilder:printcolumn:name="Uploader Type",type="string",JSONPath=".spec.uploaderType",description="The type of the uploader to handle data transfer" type PodVolumeRestore struct { metav1.TypeMeta `json:",inline"` // +optional metav1.ObjectMeta `json:"metadata,omitempty"` // +optional Spec PodVolumeRestoreSpec `json:"spec,omitempty"` // +optional Status PodVolumeRestoreStatus `json:"status,omitempty"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // +kubebuilder:object:generate=true // +kubebuilder:object:root=true // PodVolumeRestoreList is a list of PodVolumeRestores. type PodVolumeRestoreList struct { metav1.TypeMeta `json:",inline"` // +optional metav1.ListMeta `json:"metadata,omitempty"` Items []PodVolumeRestore `json:"items"` } ================================================ FILE: pkg/apis/velero/v1/register.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v1 import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/sets" ) // Resource gets a Velero GroupResource for a specified resource func Resource(resource string) schema.GroupResource { return SchemeGroupVersion.WithResource(resource).GroupResource() } type typeInfo struct { PluralName string ItemType runtime.Object ItemListType runtime.Object } func newTypeInfo(pluralName string, itemType, itemListType runtime.Object) typeInfo { return typeInfo{ PluralName: pluralName, ItemType: itemType, ItemListType: itemListType, } } // CustomResources returns a map of all custom resources within the Velero // API group, keyed on Kind. func CustomResources() map[string]typeInfo { return map[string]typeInfo{ "Backup": newTypeInfo("backups", &Backup{}, &BackupList{}), "Restore": newTypeInfo("restores", &Restore{}, &RestoreList{}), "Schedule": newTypeInfo("schedules", &Schedule{}, &ScheduleList{}), "DownloadRequest": newTypeInfo("downloadrequests", &DownloadRequest{}, &DownloadRequestList{}), "DeleteBackupRequest": newTypeInfo("deletebackuprequests", &DeleteBackupRequest{}, &DeleteBackupRequestList{}), "PodVolumeBackup": newTypeInfo("podvolumebackups", &PodVolumeBackup{}, &PodVolumeBackupList{}), "PodVolumeRestore": newTypeInfo("podvolumerestores", &PodVolumeRestore{}, &PodVolumeRestoreList{}), "BackupRepository": newTypeInfo("backuprepositories", &BackupRepository{}, &BackupRepositoryList{}), "BackupStorageLocation": newTypeInfo("backupstoragelocations", &BackupStorageLocation{}, &BackupStorageLocationList{}), "VolumeSnapshotLocation": newTypeInfo("volumesnapshotlocations", &VolumeSnapshotLocation{}, &VolumeSnapshotLocationList{}), "ServerStatusRequest": newTypeInfo("serverstatusrequests", &ServerStatusRequest{}, &ServerStatusRequestList{}), } } // CustomResourceKinds returns a list of all custom resources kinds within the Velero func CustomResourceKinds() sets.Set[string] { kinds := sets.New[string]() resources := CustomResources() for kind := range resources { kinds.Insert(kind) } return kinds } func addKnownTypes(scheme *runtime.Scheme) error { for _, typeInfo := range CustomResources() { scheme.AddKnownTypes(SchemeGroupVersion, typeInfo.ItemType, typeInfo.ItemListType) } metav1.AddToGroupVersion(scheme, SchemeGroupVersion) return nil } ================================================ FILE: pkg/apis/velero/v1/restore_types.go ================================================ /* Copyright 2017, 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v1 import ( corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ) // RestoreSpec defines the specification for a Velero restore. type RestoreSpec struct { // BackupName is the unique name of the Velero backup to restore // from. // +optional BackupName string `json:"backupName,omitempty"` // ScheduleName is the unique name of the Velero schedule to restore // from. If specified, and BackupName is empty, Velero will restore // from the most recent successful backup created from this schedule. // +optional ScheduleName string `json:"scheduleName,omitempty"` // IncludedNamespaces is a slice of namespace names to include objects // from. If empty, all namespaces are included. // +optional // +nullable IncludedNamespaces []string `json:"includedNamespaces,omitempty"` // ExcludedNamespaces contains a list of namespaces that are not // included in the restore. // +optional // +nullable ExcludedNamespaces []string `json:"excludedNamespaces,omitempty"` // IncludedResources is a slice of resource names to include // in the restore. If empty, all resources in the backup are included. // +optional // +nullable IncludedResources []string `json:"includedResources,omitempty"` // ExcludedResources is a slice of resource names that are not // included in the restore. // +optional // +nullable ExcludedResources []string `json:"excludedResources,omitempty"` // NamespaceMapping is a map of source namespace names // to target namespace names to restore into. Any source // namespaces not included in the map will be restored into // namespaces of the same name. // +optional NamespaceMapping map[string]string `json:"namespaceMapping,omitempty"` // LabelSelector is a metav1.LabelSelector to filter with // when restoring individual objects from the backup. If empty // or nil, all objects are included. Optional. // +optional // +nullable LabelSelector *metav1.LabelSelector `json:"labelSelector,omitempty"` // OrLabelSelectors is list of metav1.LabelSelector to filter with // when restoring individual objects from the backup. If multiple provided // they will be joined by the OR operator. LabelSelector as well as // OrLabelSelectors cannot co-exist in restore request, only one of them // can be used // +optional // +nullable OrLabelSelectors []*metav1.LabelSelector `json:"orLabelSelectors,omitempty"` // RestorePVs specifies whether to restore all included // PVs from snapshot // +optional // +nullable RestorePVs *bool `json:"restorePVs,omitempty"` // RestoreStatus specifies which resources we should restore the status // field. If nil, no objects are included. Optional. // +optional // +nullable RestoreStatus *RestoreStatusSpec `json:"restoreStatus,omitempty"` // PreserveNodePorts specifies whether to restore old nodePorts from backup. // +optional // +nullable PreserveNodePorts *bool `json:"preserveNodePorts,omitempty"` // IncludeClusterResources specifies whether cluster-scoped resources // should be included for consideration in the restore. If null, defaults // to true. // +optional // +nullable IncludeClusterResources *bool `json:"includeClusterResources,omitempty"` // Hooks represent custom behaviors that should be executed during or post restore. // +optional Hooks RestoreHooks `json:"hooks,omitempty"` // ExistingResourcePolicy specifies the restore behavior for the Kubernetes resource to be restored // +optional // +nullable ExistingResourcePolicy PolicyType `json:"existingResourcePolicy,omitempty"` // ItemOperationTimeout specifies the time used to wait for RestoreItemAction operations // The default value is 4 hour. // +optional ItemOperationTimeout metav1.Duration `json:"itemOperationTimeout,omitempty"` // ResourceModifier specifies the reference to JSON resource patches that should be applied to resources before restoration. // +optional // +nullable ResourceModifier *corev1api.TypedLocalObjectReference `json:"resourceModifier,omitempty"` // UploaderConfig specifies the configuration for the restore. // +optional // +nullable UploaderConfig *UploaderConfigForRestore `json:"uploaderConfig,omitempty"` } // UploaderConfigForRestore defines the configuration for the restore. type UploaderConfigForRestore struct { // WriteSparseFiles is a flag to indicate whether write files sparsely or not. // +optional // +nullable WriteSparseFiles *bool `json:"writeSparseFiles,omitempty"` // ParallelFilesDownload is the concurrency number setting for restore. // +optional ParallelFilesDownload int `json:"parallelFilesDownload,omitempty"` } // RestoreHooks contains custom behaviors that should be executed during or post restore. type RestoreHooks struct { Resources []RestoreResourceHookSpec `json:"resources,omitempty"` } type RestoreStatusSpec struct { // IncludedResources specifies the resources to which will restore the status. // If empty, it applies to all resources. // +optional // +nullable IncludedResources []string `json:"includedResources,omitempty"` // ExcludedResources specifies the resources to which will not restore the status. // +optional // +nullable ExcludedResources []string `json:"excludedResources,omitempty"` } // RestoreResourceHookSpec defines one or more RestoreResrouceHooks that should be executed based on // the rules defined for namespaces, resources, and label selector. type RestoreResourceHookSpec struct { // Name is the name of this hook. Name string `json:"name"` // IncludedNamespaces specifies the namespaces to which this hook spec applies. If empty, it applies // to all namespaces. // +optional // +nullable IncludedNamespaces []string `json:"includedNamespaces,omitempty"` // ExcludedNamespaces specifies the namespaces to which this hook spec does not apply. // +optional // +nullable ExcludedNamespaces []string `json:"excludedNamespaces,omitempty"` // IncludedResources specifies the resources to which this hook spec applies. If empty, it applies // to all resources. // +optional // +nullable IncludedResources []string `json:"includedResources,omitempty"` // ExcludedResources specifies the resources to which this hook spec does not apply. // +optional // +nullable ExcludedResources []string `json:"excludedResources,omitempty"` // LabelSelector, if specified, filters the resources to which this hook spec applies. // +optional // +nullable LabelSelector *metav1.LabelSelector `json:"labelSelector,omitempty"` // PostHooks is a list of RestoreResourceHooks to execute during and after restoring a resource. // +optional PostHooks []RestoreResourceHook `json:"postHooks,omitempty"` } // RestoreResourceHook defines a restore hook for a resource. type RestoreResourceHook struct { // Exec defines an exec restore hook. Exec *ExecRestoreHook `json:"exec,omitempty"` // Init defines an init restore hook. Init *InitRestoreHook `json:"init,omitempty"` } // ExecRestoreHook is a hook that uses pod exec API to execute a command inside a container in a pod type ExecRestoreHook struct { // Container is the container in the pod where the command should be executed. If not specified, // the pod's first container is used. // +optional Container string `json:"container,omitempty"` // Command is the command and arguments to execute from within a container after a pod has been restored. // +kubebuilder:validation:MinItems=1 Command []string `json:"command"` // OnError specifies how Velero should behave if it encounters an error executing this hook. // +optional OnError HookErrorMode `json:"onError,omitempty"` // ExecTimeout defines the maximum amount of time Velero should wait for the hook to complete before // considering the execution a failure. // +optional ExecTimeout metav1.Duration `json:"execTimeout,omitempty"` // WaitTimeout defines the maximum amount of time Velero should wait for the container to be Ready // before attempting to run the command. // +optional WaitTimeout metav1.Duration `json:"waitTimeout,omitempty"` // WaitForReady ensures command will be launched when container is Ready instead of Running. // +optional // +nullable WaitForReady *bool `json:"waitForReady,omitempty"` } // InitRestoreHook is a hook that adds an init container to a PodSpec to run commands before the // workload pod is able to start. type InitRestoreHook struct { // +kubebuilder:pruning:PreserveUnknownFields // InitContainers is list of init containers to be added to a pod during its restore. // +optional InitContainers []runtime.RawExtension `json:"initContainers"` // Timeout defines the maximum amount of time Velero should wait for the initContainers to complete. // +optional Timeout metav1.Duration `json:"timeout,omitempty"` } // RestorePhase is a string representation of the lifecycle phase // of a Velero restore // +kubebuilder:validation:Enum=New;FailedValidation;InProgress;WaitingForPluginOperations;WaitingForPluginOperationsPartiallyFailed;Completed;PartiallyFailed;Failed;Finalizing;FinalizingPartiallyFailed type RestorePhase string const ( // RestorePhaseNew means the restore has been created but not // yet processed by the RestoreController RestorePhaseNew RestorePhase = "New" // RestorePhaseFailedValidation means the restore has failed // the controller's validations and therefore will not run. RestorePhaseFailedValidation RestorePhase = "FailedValidation" // RestorePhaseInProgress means the restore is currently executing. RestorePhaseInProgress RestorePhase = "InProgress" // RestorePhaseWaitingForPluginOperations means the restore of // Kubernetes resources and other async plugin operations was // successful and plugin operations are still ongoing. The // restore is not complete yet. RestorePhaseWaitingForPluginOperations RestorePhase = "WaitingForPluginOperations" // RestorePhaseWaitingForPluginOperationsPartiallyFailed means // the restore of Kubernetes resources and other async plugin // operations partially failed (final phase will be // PartiallyFailed) and other plugin operations are still // ongoing. The restore is not complete yet. RestorePhaseWaitingForPluginOperationsPartiallyFailed RestorePhase = "WaitingForPluginOperationsPartiallyFailed" // RestorePhaseFinalizing means the restore of // Kubernetes resources and other async plugin operations were successful and // other plugin operations are now complete, but the restore is awaiting // the completion of wrap-up tasks before the restore process enters terminal phase. RestorePhaseFinalizing RestorePhase = "Finalizing" // RestorePhaseFinalizingPartiallyFailed means the restore of // Kubernetes resources and other async plugin operations were successful and // other plugin operations are now complete, but one or more errors // occurred during restore or async operation processing. The restore is awaiting // the completion of wrap-up tasks before the restore process enters terminal phase. RestorePhaseFinalizingPartiallyFailed RestorePhase = "FinalizingPartiallyFailed" // RestorePhaseCompleted means the restore has run successfully // without errors. RestorePhaseCompleted RestorePhase = "Completed" // RestorePhasePartiallyFailed means the restore has run to completion // but encountered 1+ errors restoring individual items. RestorePhasePartiallyFailed RestorePhase = "PartiallyFailed" // RestorePhaseFailed means the restore was unable to execute. // The failing error is recorded in status.FailureReason. RestorePhaseFailed RestorePhase = "Failed" // PolicyTypeNone means velero will not overwrite the resource // in cluster with the one in backup whether changed/unchanged. PolicyTypeNone PolicyType = "none" // PolicyTypeUpdate means velero will try to attempt a patch on // the changed resources. PolicyTypeUpdate PolicyType = "update" ) // RestoreStatus captures the current status of a Velero restore type RestoreStatus struct { // Phase is the current state of the Restore // +optional Phase RestorePhase `json:"phase,omitempty"` // ValidationErrors is a slice of all validation errors (if // applicable) // +optional // +nullable ValidationErrors []string `json:"validationErrors,omitempty"` // Warnings is a count of all warning messages that were generated during // execution of the restore. The actual warnings are stored in object storage. // +optional Warnings int `json:"warnings,omitempty"` // Errors is a count of all error messages that were generated during // execution of the restore. The actual errors are stored in object storage. // +optional Errors int `json:"errors,omitempty"` // FailureReason is an error that caused the entire restore to fail. // +optional FailureReason string `json:"failureReason,omitempty"` // StartTimestamp records the time the restore operation was started. // The server's time is used for StartTimestamps // +optional // +nullable StartTimestamp *metav1.Time `json:"startTimestamp,omitempty"` // CompletionTimestamp records the time the restore operation was completed. // Completion time is recorded even on failed restore. // The server's time is used for StartTimestamps // +optional // +nullable CompletionTimestamp *metav1.Time `json:"completionTimestamp,omitempty"` // Progress contains information about the restore's execution progress. Note // that this information is best-effort only -- if Velero fails to update it // during a restore for any reason, it may be inaccurate/stale. // +optional // +nullable Progress *RestoreProgress `json:"progress,omitempty"` // RestoreItemOperationsAttempted is the total number of attempted // async RestoreItemAction operations for this restore. // +optional RestoreItemOperationsAttempted int `json:"restoreItemOperationsAttempted,omitempty"` // RestoreItemOperationsCompleted is the total number of successfully completed // async RestoreItemAction operations for this restore. // +optional RestoreItemOperationsCompleted int `json:"restoreItemOperationsCompleted,omitempty"` // RestoreItemOperationsFailed is the total number of async // RestoreItemAction operations for this restore which ended with an error. // +optional RestoreItemOperationsFailed int `json:"restoreItemOperationsFailed,omitempty"` // HookStatus contains information about the status of the hooks. // +optional // +nullable HookStatus *HookStatus `json:"hookStatus,omitempty"` } // RestoreProgress stores information about the restore's execution progress type RestoreProgress struct { // TotalItems is the total number of items to be restored. This number may change // throughout the execution of the restore due to plugins that return additional related // items to restore // +optional TotalItems int `json:"totalItems,omitempty"` // ItemsRestored is the number of items that have actually been restored so far // +optional ItemsRestored int `json:"itemsRestored,omitempty"` } // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // +kubebuilder:object:root=true // +kubebuilder:object:generate=true // +kubebuilder:storageversion // +kubebuilder:rbac:groups=velero.io,resources=restores,verbs=create;delete;get;list;patch;update;watch // +kubebuilder:rbac:groups=velero.io,resources=restores/status,verbs=get;update;patch // Restore is a Velero resource that represents the application of // resources from a Velero backup to a target Kubernetes cluster. type Restore struct { metav1.TypeMeta `json:",inline"` // +optional metav1.ObjectMeta `json:"metadata,omitempty"` // +optional Spec RestoreSpec `json:"spec,omitempty"` // +optional Status RestoreStatus `json:"status,omitempty"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // RestoreList is a list of Restores. type RestoreList struct { metav1.TypeMeta `json:",inline"` // +optional metav1.ListMeta `json:"metadata"` Items []Restore `json:"items"` } // PolicyType helps specify the ExistingResourcePolicy type PolicyType string ================================================ FILE: pkg/apis/velero/v1/schedule_types.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v1 import ( "fmt" "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // ScheduleSpec defines the specification for a Velero schedule type ScheduleSpec struct { // Template is the definition of the Backup to be run // on the provided schedule Template BackupSpec `json:"template"` // Schedule is a Cron expression defining when to run // the Backup. Schedule string `json:"schedule"` // UseOwnerReferencesBackup specifies whether to use // OwnerReferences on backups created by this Schedule. // +optional // +nullable UseOwnerReferencesInBackup *bool `json:"useOwnerReferencesInBackup,omitempty"` // Paused specifies whether the schedule is paused or not // +optional Paused bool `json:"paused,omitempty"` // SkipImmediately specifies whether to skip backup if schedule is due immediately from `schedule.status.lastBackup` timestamp when schedule is unpaused or if schedule is new. // If true, backup will be skipped immediately when schedule is unpaused if it is due based on .Status.LastBackupTimestamp or schedule is new, and will run at next schedule time. // If false, backup will not be skipped immediately when schedule is unpaused, but will run at next schedule time. // If empty, will follow server configuration (default: false). // +optional SkipImmediately *bool `json:"skipImmediately,omitempty"` } // SchedulePhase is a string representation of the lifecycle phase // of a Velero schedule // +kubebuilder:validation:Enum=New;Enabled;FailedValidation type SchedulePhase string const ( // SchedulePhaseNew means the schedule has been created but not // yet processed by the ScheduleController SchedulePhaseNew SchedulePhase = "New" // SchedulePhaseEnabled means the schedule has been validated and // will now be triggering backups according to the schedule spec. SchedulePhaseEnabled SchedulePhase = "Enabled" // SchedulePhaseFailedValidation means the schedule has failed // the controller's validations and therefore will not trigger backups. SchedulePhaseFailedValidation SchedulePhase = "FailedValidation" ) // ScheduleStatus captures the current state of a Velero schedule type ScheduleStatus struct { // Phase is the current phase of the Schedule // +optional Phase SchedulePhase `json:"phase,omitempty"` // LastBackup is the last time a Backup was run for this // Schedule schedule // +optional // +nullable LastBackup *metav1.Time `json:"lastBackup,omitempty"` // LastSkipped is the last time a Schedule was skipped // +optional // +nullable LastSkipped *metav1.Time `json:"lastSkipped,omitempty"` // ValidationErrors is a slice of all validation errors (if // applicable) // +optional ValidationErrors []string `json:"validationErrors,omitempty"` } // TODO(2.0) After converting all resources to use the runtime-controller client, the genclient and k8s:deepcopy markers will no longer be needed and should be removed. // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // +kubebuilder:object:generate=true // +kubebuilder:object:root=true // +kubebuilder:storageversion // +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.phase",description="Status of the schedule" // +kubebuilder:printcolumn:name="Schedule",type="string",JSONPath=".spec.schedule",description="A Cron expression defining when to run the Backup" // +kubebuilder:printcolumn:name="LastBackup",type="date",JSONPath=".status.lastBackup",description="The last time a Backup was run for this schedule" // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" // +kubebuilder:printcolumn:name="Paused",type="boolean",JSONPath=".spec.paused" // Schedule is a Velero resource that represents a pre-scheduled or // periodic Backup that should be run. type Schedule struct { metav1.TypeMeta `json:",inline"` // +optional metav1.ObjectMeta `json:"metadata"` // +optional Spec ScheduleSpec `json:"spec,omitempty"` // +optional Status ScheduleStatus `json:"status,omitempty"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // +kubebuilder:object:generate=true // +kubebuilder:object:root=true // ScheduleList is a list of Schedules. type ScheduleList struct { metav1.TypeMeta `json:",inline"` // +optional metav1.ListMeta `json:"metadata,omitempty"` Items []Schedule `json:"items"` } // TimestampedName returns the default backup name format based on the schedule func (s *Schedule) TimestampedName(timestamp time.Time) string { return fmt.Sprintf("%s-%s", s.Name, timestamp.Format("20060102150405")) } ================================================ FILE: pkg/apis/velero/v1/server_status_request_types.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v1 import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // TODO(2.0) After converting all resources to use the runtime-controller client, // the genclient and k8s:deepcopy markers will no longer be needed and should be removed. // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // +kubebuilder:object:root=true // +kubebuilder:resource:shortName=ssr // +kubebuilder:object:generate=true // +kubebuilder:storageversion // ServerStatusRequest is a request to access current status information about // the Velero server. type ServerStatusRequest struct { metav1.TypeMeta `json:",inline"` // +optional metav1.ObjectMeta `json:"metadata,omitempty"` // +optional Spec ServerStatusRequestSpec `json:"spec,omitempty"` // +optional Status ServerStatusRequestStatus `json:"status,omitempty"` } // ServerStatusRequestSpec is the specification for a ServerStatusRequest. type ServerStatusRequestSpec struct { } // ServerStatusRequestPhase represents the lifecycle phase of a ServerStatusRequest. // +kubebuilder:validation:Enum=New;Processed type ServerStatusRequestPhase string const ( // ServerStatusRequestPhaseNew means the ServerStatusRequest has not been processed yet. ServerStatusRequestPhaseNew ServerStatusRequestPhase = "New" // ServerStatusRequestPhaseProcessed means the ServerStatusRequest has been processed. ServerStatusRequestPhaseProcessed ServerStatusRequestPhase = "Processed" ) // PluginInfo contains attributes of a Velero plugin type PluginInfo struct { Name string `json:"name"` Kind string `json:"kind"` } // ServerStatusRequestStatus is the current status of a ServerStatusRequest. type ServerStatusRequestStatus struct { // Phase is the current lifecycle phase of the ServerStatusRequest. // +optional Phase ServerStatusRequestPhase `json:"phase,omitempty"` // ProcessedTimestamp is when the ServerStatusRequest was processed // by the ServerStatusRequestController. // +optional // +nullable ProcessedTimestamp *metav1.Time `json:"processedTimestamp,omitempty"` // ServerVersion is the Velero server version. // +optional ServerVersion string `json:"serverVersion,omitempty"` // Plugins list information about the plugins running on the Velero server // +optional // +nullable Plugins []PluginInfo `json:"plugins,omitempty"` } // TODO(2.0) After converting all resources to use the runtime-controller client, // the k8s:deepcopy marker will no longer be needed and should be removed. // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // +kubebuilder:object:root=true // +kubebuilder:rbac:groups=velero.io,resources=serverstatusrequests,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=velero.io,resources=serverstatusrequests/status,verbs=get;update;patch // ServerStatusRequestList is a list of ServerStatusRequests. type ServerStatusRequestList struct { metav1.TypeMeta `json:",inline"` // +optional metav1.ListMeta `json:"metadata,omitempty"` Items []ServerStatusRequest `json:"items"` } ================================================ FILE: pkg/apis/velero/v1/volume_snapshot_location_type.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v1 import ( corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // +kubebuilder:object:root=true // +kubebuilder:resource:shortName=vsl // +kubebuilder:object:generate=true // +kubebuilder:storageversion // VolumeSnapshotLocation is a location where Velero stores volume snapshots. type VolumeSnapshotLocation struct { metav1.TypeMeta `json:",inline"` // +optional metav1.ObjectMeta `json:"metadata,omitempty"` // +optional Spec VolumeSnapshotLocationSpec `json:"spec,omitempty"` // +optional Status VolumeSnapshotLocationStatus `json:"status,omitempty"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // +kubebuilder:object:root=true // +kubebuilder:rbac:groups=velero.io,resources=volumesnapshotlocations,verbs=get;list;watch;create;update;patch;delete // VolumeSnapshotLocationList is a list of VolumeSnapshotLocations. type VolumeSnapshotLocationList struct { metav1.TypeMeta `json:",inline"` // +optional metav1.ListMeta `json:"metadata,omitempty"` Items []VolumeSnapshotLocation `json:"items"` } // VolumeSnapshotLocationSpec defines the specification for a Velero VolumeSnapshotLocation. type VolumeSnapshotLocationSpec struct { // Provider is the provider of the volume storage. Provider string `json:"provider"` // Config is for provider-specific configuration fields. // +optional Config map[string]string `json:"config,omitempty"` // Credential contains the credential information intended to be used with this location // +optional Credential *corev1api.SecretKeySelector `json:"credential,omitempty"` } // VolumeSnapshotLocationPhase is the lifecycle phase of a Velero VolumeSnapshotLocation. // +kubebuilder:validation:Enum=Available;Unavailable type VolumeSnapshotLocationPhase string const ( // VolumeSnapshotLocationPhaseAvailable means the location is available to read and write from. VolumeSnapshotLocationPhaseAvailable VolumeSnapshotLocationPhase = "Available" // VolumeSnapshotLocationPhaseUnavailable means the location is unavailable to read and write from. VolumeSnapshotLocationPhaseUnavailable VolumeSnapshotLocationPhase = "Unavailable" ) // VolumeSnapshotLocationStatus describes the current status of a Velero VolumeSnapshotLocation. type VolumeSnapshotLocationStatus struct { // +optional Phase VolumeSnapshotLocationPhase `json:"phase,omitempty"` } ================================================ FILE: pkg/apis/velero/v1/zz_generated.deepcopy.go ================================================ //go:build !ignore_autogenerated // Code generated by controller-gen. DO NOT EDIT. package v1 import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Backup) DeepCopyInto(out *Backup) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Spec.DeepCopyInto(&out.Spec) in.Status.DeepCopyInto(&out.Status) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Backup. func (in *Backup) DeepCopy() *Backup { if in == nil { return nil } out := new(Backup) in.DeepCopyInto(out) return out } // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. func (in *Backup) DeepCopyObject() runtime.Object { if c := in.DeepCopy(); c != nil { return c } return nil } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BackupHooks) DeepCopyInto(out *BackupHooks) { *out = *in if in.Resources != nil { in, out := &in.Resources, &out.Resources *out = make([]BackupResourceHookSpec, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupHooks. func (in *BackupHooks) DeepCopy() *BackupHooks { if in == nil { return nil } out := new(BackupHooks) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BackupList) DeepCopyInto(out *BackupList) { *out = *in out.TypeMeta = in.TypeMeta in.ListMeta.DeepCopyInto(&out.ListMeta) if in.Items != nil { in, out := &in.Items, &out.Items *out = make([]Backup, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupList. func (in *BackupList) DeepCopy() *BackupList { if in == nil { return nil } out := new(BackupList) in.DeepCopyInto(out) return out } // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. func (in *BackupList) DeepCopyObject() runtime.Object { if c := in.DeepCopy(); c != nil { return c } return nil } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BackupProgress) DeepCopyInto(out *BackupProgress) { *out = *in } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupProgress. func (in *BackupProgress) DeepCopy() *BackupProgress { if in == nil { return nil } out := new(BackupProgress) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BackupRepository) DeepCopyInto(out *BackupRepository) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Spec.DeepCopyInto(&out.Spec) in.Status.DeepCopyInto(&out.Status) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupRepository. func (in *BackupRepository) DeepCopy() *BackupRepository { if in == nil { return nil } out := new(BackupRepository) in.DeepCopyInto(out) return out } // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. func (in *BackupRepository) DeepCopyObject() runtime.Object { if c := in.DeepCopy(); c != nil { return c } return nil } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BackupRepositoryList) DeepCopyInto(out *BackupRepositoryList) { *out = *in out.TypeMeta = in.TypeMeta in.ListMeta.DeepCopyInto(&out.ListMeta) if in.Items != nil { in, out := &in.Items, &out.Items *out = make([]BackupRepository, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupRepositoryList. func (in *BackupRepositoryList) DeepCopy() *BackupRepositoryList { if in == nil { return nil } out := new(BackupRepositoryList) in.DeepCopyInto(out) return out } // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. func (in *BackupRepositoryList) DeepCopyObject() runtime.Object { if c := in.DeepCopy(); c != nil { return c } return nil } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BackupRepositoryMaintenanceStatus) DeepCopyInto(out *BackupRepositoryMaintenanceStatus) { *out = *in if in.StartTimestamp != nil { in, out := &in.StartTimestamp, &out.StartTimestamp *out = (*in).DeepCopy() } if in.CompleteTimestamp != nil { in, out := &in.CompleteTimestamp, &out.CompleteTimestamp *out = (*in).DeepCopy() } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupRepositoryMaintenanceStatus. func (in *BackupRepositoryMaintenanceStatus) DeepCopy() *BackupRepositoryMaintenanceStatus { if in == nil { return nil } out := new(BackupRepositoryMaintenanceStatus) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BackupRepositorySpec) DeepCopyInto(out *BackupRepositorySpec) { *out = *in out.MaintenanceFrequency = in.MaintenanceFrequency if in.RepositoryConfig != nil { in, out := &in.RepositoryConfig, &out.RepositoryConfig *out = make(map[string]string, len(*in)) for key, val := range *in { (*out)[key] = val } } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupRepositorySpec. func (in *BackupRepositorySpec) DeepCopy() *BackupRepositorySpec { if in == nil { return nil } out := new(BackupRepositorySpec) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BackupRepositoryStatus) DeepCopyInto(out *BackupRepositoryStatus) { *out = *in if in.LastMaintenanceTime != nil { in, out := &in.LastMaintenanceTime, &out.LastMaintenanceTime *out = (*in).DeepCopy() } if in.RecentMaintenance != nil { in, out := &in.RecentMaintenance, &out.RecentMaintenance *out = make([]BackupRepositoryMaintenanceStatus, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupRepositoryStatus. func (in *BackupRepositoryStatus) DeepCopy() *BackupRepositoryStatus { if in == nil { return nil } out := new(BackupRepositoryStatus) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BackupResourceHook) DeepCopyInto(out *BackupResourceHook) { *out = *in if in.Exec != nil { in, out := &in.Exec, &out.Exec *out = new(ExecHook) (*in).DeepCopyInto(*out) } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupResourceHook. func (in *BackupResourceHook) DeepCopy() *BackupResourceHook { if in == nil { return nil } out := new(BackupResourceHook) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BackupResourceHookSpec) DeepCopyInto(out *BackupResourceHookSpec) { *out = *in if in.IncludedNamespaces != nil { in, out := &in.IncludedNamespaces, &out.IncludedNamespaces *out = make([]string, len(*in)) copy(*out, *in) } if in.ExcludedNamespaces != nil { in, out := &in.ExcludedNamespaces, &out.ExcludedNamespaces *out = make([]string, len(*in)) copy(*out, *in) } if in.IncludedResources != nil { in, out := &in.IncludedResources, &out.IncludedResources *out = make([]string, len(*in)) copy(*out, *in) } if in.ExcludedResources != nil { in, out := &in.ExcludedResources, &out.ExcludedResources *out = make([]string, len(*in)) copy(*out, *in) } if in.LabelSelector != nil { in, out := &in.LabelSelector, &out.LabelSelector *out = new(metav1.LabelSelector) (*in).DeepCopyInto(*out) } if in.PreHooks != nil { in, out := &in.PreHooks, &out.PreHooks *out = make([]BackupResourceHook, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } if in.PostHooks != nil { in, out := &in.PostHooks, &out.PostHooks *out = make([]BackupResourceHook, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupResourceHookSpec. func (in *BackupResourceHookSpec) DeepCopy() *BackupResourceHookSpec { if in == nil { return nil } out := new(BackupResourceHookSpec) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BackupSpec) DeepCopyInto(out *BackupSpec) { *out = *in in.Metadata.DeepCopyInto(&out.Metadata) if in.IncludedNamespaces != nil { in, out := &in.IncludedNamespaces, &out.IncludedNamespaces *out = make([]string, len(*in)) copy(*out, *in) } if in.ExcludedNamespaces != nil { in, out := &in.ExcludedNamespaces, &out.ExcludedNamespaces *out = make([]string, len(*in)) copy(*out, *in) } if in.IncludedResources != nil { in, out := &in.IncludedResources, &out.IncludedResources *out = make([]string, len(*in)) copy(*out, *in) } if in.ExcludedResources != nil { in, out := &in.ExcludedResources, &out.ExcludedResources *out = make([]string, len(*in)) copy(*out, *in) } if in.IncludedClusterScopedResources != nil { in, out := &in.IncludedClusterScopedResources, &out.IncludedClusterScopedResources *out = make([]string, len(*in)) copy(*out, *in) } if in.ExcludedClusterScopedResources != nil { in, out := &in.ExcludedClusterScopedResources, &out.ExcludedClusterScopedResources *out = make([]string, len(*in)) copy(*out, *in) } if in.IncludedNamespaceScopedResources != nil { in, out := &in.IncludedNamespaceScopedResources, &out.IncludedNamespaceScopedResources *out = make([]string, len(*in)) copy(*out, *in) } if in.ExcludedNamespaceScopedResources != nil { in, out := &in.ExcludedNamespaceScopedResources, &out.ExcludedNamespaceScopedResources *out = make([]string, len(*in)) copy(*out, *in) } if in.LabelSelector != nil { in, out := &in.LabelSelector, &out.LabelSelector *out = new(metav1.LabelSelector) (*in).DeepCopyInto(*out) } if in.OrLabelSelectors != nil { in, out := &in.OrLabelSelectors, &out.OrLabelSelectors *out = make([]*metav1.LabelSelector, len(*in)) for i := range *in { if (*in)[i] != nil { in, out := &(*in)[i], &(*out)[i] *out = new(metav1.LabelSelector) (*in).DeepCopyInto(*out) } } } if in.SnapshotVolumes != nil { in, out := &in.SnapshotVolumes, &out.SnapshotVolumes *out = new(bool) **out = **in } out.TTL = in.TTL if in.IncludeClusterResources != nil { in, out := &in.IncludeClusterResources, &out.IncludeClusterResources *out = new(bool) **out = **in } in.Hooks.DeepCopyInto(&out.Hooks) if in.VolumeSnapshotLocations != nil { in, out := &in.VolumeSnapshotLocations, &out.VolumeSnapshotLocations *out = make([]string, len(*in)) copy(*out, *in) } if in.DefaultVolumesToRestic != nil { in, out := &in.DefaultVolumesToRestic, &out.DefaultVolumesToRestic *out = new(bool) **out = **in } if in.DefaultVolumesToFsBackup != nil { in, out := &in.DefaultVolumesToFsBackup, &out.DefaultVolumesToFsBackup *out = new(bool) **out = **in } if in.OrderedResources != nil { in, out := &in.OrderedResources, &out.OrderedResources *out = make(map[string]string, len(*in)) for key, val := range *in { (*out)[key] = val } } out.CSISnapshotTimeout = in.CSISnapshotTimeout out.ItemOperationTimeout = in.ItemOperationTimeout if in.ResourcePolicy != nil { in, out := &in.ResourcePolicy, &out.ResourcePolicy *out = new(corev1.TypedLocalObjectReference) (*in).DeepCopyInto(*out) } if in.SnapshotMoveData != nil { in, out := &in.SnapshotMoveData, &out.SnapshotMoveData *out = new(bool) **out = **in } if in.UploaderConfig != nil { in, out := &in.UploaderConfig, &out.UploaderConfig *out = new(UploaderConfigForBackup) **out = **in } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupSpec. func (in *BackupSpec) DeepCopy() *BackupSpec { if in == nil { return nil } out := new(BackupSpec) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BackupStatus) DeepCopyInto(out *BackupStatus) { *out = *in if in.Expiration != nil { in, out := &in.Expiration, &out.Expiration *out = (*in).DeepCopy() } if in.ValidationErrors != nil { in, out := &in.ValidationErrors, &out.ValidationErrors *out = make([]string, len(*in)) copy(*out, *in) } if in.StartTimestamp != nil { in, out := &in.StartTimestamp, &out.StartTimestamp *out = (*in).DeepCopy() } if in.CompletionTimestamp != nil { in, out := &in.CompletionTimestamp, &out.CompletionTimestamp *out = (*in).DeepCopy() } if in.Progress != nil { in, out := &in.Progress, &out.Progress *out = new(BackupProgress) **out = **in } if in.HookStatus != nil { in, out := &in.HookStatus, &out.HookStatus *out = new(HookStatus) **out = **in } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupStatus. func (in *BackupStatus) DeepCopy() *BackupStatus { if in == nil { return nil } out := new(BackupStatus) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BackupStorageLocation) DeepCopyInto(out *BackupStorageLocation) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Spec.DeepCopyInto(&out.Spec) in.Status.DeepCopyInto(&out.Status) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupStorageLocation. func (in *BackupStorageLocation) DeepCopy() *BackupStorageLocation { if in == nil { return nil } out := new(BackupStorageLocation) in.DeepCopyInto(out) return out } // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. func (in *BackupStorageLocation) DeepCopyObject() runtime.Object { if c := in.DeepCopy(); c != nil { return c } return nil } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BackupStorageLocationList) DeepCopyInto(out *BackupStorageLocationList) { *out = *in out.TypeMeta = in.TypeMeta in.ListMeta.DeepCopyInto(&out.ListMeta) if in.Items != nil { in, out := &in.Items, &out.Items *out = make([]BackupStorageLocation, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupStorageLocationList. func (in *BackupStorageLocationList) DeepCopy() *BackupStorageLocationList { if in == nil { return nil } out := new(BackupStorageLocationList) in.DeepCopyInto(out) return out } // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. func (in *BackupStorageLocationList) DeepCopyObject() runtime.Object { if c := in.DeepCopy(); c != nil { return c } return nil } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BackupStorageLocationSpec) DeepCopyInto(out *BackupStorageLocationSpec) { *out = *in if in.Config != nil { in, out := &in.Config, &out.Config *out = make(map[string]string, len(*in)) for key, val := range *in { (*out)[key] = val } } if in.Credential != nil { in, out := &in.Credential, &out.Credential *out = new(corev1.SecretKeySelector) (*in).DeepCopyInto(*out) } in.StorageType.DeepCopyInto(&out.StorageType) if in.BackupSyncPeriod != nil { in, out := &in.BackupSyncPeriod, &out.BackupSyncPeriod *out = new(metav1.Duration) **out = **in } if in.ValidationFrequency != nil { in, out := &in.ValidationFrequency, &out.ValidationFrequency *out = new(metav1.Duration) **out = **in } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupStorageLocationSpec. func (in *BackupStorageLocationSpec) DeepCopy() *BackupStorageLocationSpec { if in == nil { return nil } out := new(BackupStorageLocationSpec) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BackupStorageLocationStatus) DeepCopyInto(out *BackupStorageLocationStatus) { *out = *in if in.LastSyncedTime != nil { in, out := &in.LastSyncedTime, &out.LastSyncedTime *out = (*in).DeepCopy() } if in.LastValidationTime != nil { in, out := &in.LastValidationTime, &out.LastValidationTime *out = (*in).DeepCopy() } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupStorageLocationStatus. func (in *BackupStorageLocationStatus) DeepCopy() *BackupStorageLocationStatus { if in == nil { return nil } out := new(BackupStorageLocationStatus) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DeleteBackupRequest) DeepCopyInto(out *DeleteBackupRequest) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) out.Spec = in.Spec in.Status.DeepCopyInto(&out.Status) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeleteBackupRequest. func (in *DeleteBackupRequest) DeepCopy() *DeleteBackupRequest { if in == nil { return nil } out := new(DeleteBackupRequest) in.DeepCopyInto(out) return out } // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. func (in *DeleteBackupRequest) DeepCopyObject() runtime.Object { if c := in.DeepCopy(); c != nil { return c } return nil } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DeleteBackupRequestList) DeepCopyInto(out *DeleteBackupRequestList) { *out = *in out.TypeMeta = in.TypeMeta in.ListMeta.DeepCopyInto(&out.ListMeta) if in.Items != nil { in, out := &in.Items, &out.Items *out = make([]DeleteBackupRequest, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeleteBackupRequestList. func (in *DeleteBackupRequestList) DeepCopy() *DeleteBackupRequestList { if in == nil { return nil } out := new(DeleteBackupRequestList) in.DeepCopyInto(out) return out } // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. func (in *DeleteBackupRequestList) DeepCopyObject() runtime.Object { if c := in.DeepCopy(); c != nil { return c } return nil } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DeleteBackupRequestSpec) DeepCopyInto(out *DeleteBackupRequestSpec) { *out = *in } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeleteBackupRequestSpec. func (in *DeleteBackupRequestSpec) DeepCopy() *DeleteBackupRequestSpec { if in == nil { return nil } out := new(DeleteBackupRequestSpec) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DeleteBackupRequestStatus) DeepCopyInto(out *DeleteBackupRequestStatus) { *out = *in if in.Errors != nil { in, out := &in.Errors, &out.Errors *out = make([]string, len(*in)) copy(*out, *in) } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeleteBackupRequestStatus. func (in *DeleteBackupRequestStatus) DeepCopy() *DeleteBackupRequestStatus { if in == nil { return nil } out := new(DeleteBackupRequestStatus) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DownloadRequest) DeepCopyInto(out *DownloadRequest) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) out.Spec = in.Spec in.Status.DeepCopyInto(&out.Status) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DownloadRequest. func (in *DownloadRequest) DeepCopy() *DownloadRequest { if in == nil { return nil } out := new(DownloadRequest) in.DeepCopyInto(out) return out } // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. func (in *DownloadRequest) DeepCopyObject() runtime.Object { if c := in.DeepCopy(); c != nil { return c } return nil } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DownloadRequestList) DeepCopyInto(out *DownloadRequestList) { *out = *in out.TypeMeta = in.TypeMeta in.ListMeta.DeepCopyInto(&out.ListMeta) if in.Items != nil { in, out := &in.Items, &out.Items *out = make([]DownloadRequest, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DownloadRequestList. func (in *DownloadRequestList) DeepCopy() *DownloadRequestList { if in == nil { return nil } out := new(DownloadRequestList) in.DeepCopyInto(out) return out } // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. func (in *DownloadRequestList) DeepCopyObject() runtime.Object { if c := in.DeepCopy(); c != nil { return c } return nil } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DownloadRequestSpec) DeepCopyInto(out *DownloadRequestSpec) { *out = *in out.Target = in.Target } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DownloadRequestSpec. func (in *DownloadRequestSpec) DeepCopy() *DownloadRequestSpec { if in == nil { return nil } out := new(DownloadRequestSpec) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DownloadRequestStatus) DeepCopyInto(out *DownloadRequestStatus) { *out = *in if in.Expiration != nil { in, out := &in.Expiration, &out.Expiration *out = (*in).DeepCopy() } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DownloadRequestStatus. func (in *DownloadRequestStatus) DeepCopy() *DownloadRequestStatus { if in == nil { return nil } out := new(DownloadRequestStatus) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DownloadTarget) DeepCopyInto(out *DownloadTarget) { *out = *in } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DownloadTarget. func (in *DownloadTarget) DeepCopy() *DownloadTarget { if in == nil { return nil } out := new(DownloadTarget) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ExecHook) DeepCopyInto(out *ExecHook) { *out = *in if in.Command != nil { in, out := &in.Command, &out.Command *out = make([]string, len(*in)) copy(*out, *in) } out.Timeout = in.Timeout } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExecHook. func (in *ExecHook) DeepCopy() *ExecHook { if in == nil { return nil } out := new(ExecHook) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ExecRestoreHook) DeepCopyInto(out *ExecRestoreHook) { *out = *in if in.Command != nil { in, out := &in.Command, &out.Command *out = make([]string, len(*in)) copy(*out, *in) } out.ExecTimeout = in.ExecTimeout out.WaitTimeout = in.WaitTimeout if in.WaitForReady != nil { in, out := &in.WaitForReady, &out.WaitForReady *out = new(bool) **out = **in } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExecRestoreHook. func (in *ExecRestoreHook) DeepCopy() *ExecRestoreHook { if in == nil { return nil } out := new(ExecRestoreHook) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HookStatus) DeepCopyInto(out *HookStatus) { *out = *in } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HookStatus. func (in *HookStatus) DeepCopy() *HookStatus { if in == nil { return nil } out := new(HookStatus) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *InitRestoreHook) DeepCopyInto(out *InitRestoreHook) { *out = *in if in.InitContainers != nil { in, out := &in.InitContainers, &out.InitContainers *out = make([]runtime.RawExtension, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } out.Timeout = in.Timeout } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InitRestoreHook. func (in *InitRestoreHook) DeepCopy() *InitRestoreHook { if in == nil { return nil } out := new(InitRestoreHook) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Metadata) DeepCopyInto(out *Metadata) { *out = *in if in.Labels != nil { in, out := &in.Labels, &out.Labels *out = make(map[string]string, len(*in)) for key, val := range *in { (*out)[key] = val } } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Metadata. func (in *Metadata) DeepCopy() *Metadata { if in == nil { return nil } out := new(Metadata) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ObjectStorageLocation) DeepCopyInto(out *ObjectStorageLocation) { *out = *in if in.CACert != nil { in, out := &in.CACert, &out.CACert *out = make([]byte, len(*in)) copy(*out, *in) } if in.CACertRef != nil { in, out := &in.CACertRef, &out.CACertRef *out = new(corev1.SecretKeySelector) (*in).DeepCopyInto(*out) } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectStorageLocation. func (in *ObjectStorageLocation) DeepCopy() *ObjectStorageLocation { if in == nil { return nil } out := new(ObjectStorageLocation) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PluginInfo) DeepCopyInto(out *PluginInfo) { *out = *in } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PluginInfo. func (in *PluginInfo) DeepCopy() *PluginInfo { if in == nil { return nil } out := new(PluginInfo) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PodVolumeBackup) DeepCopyInto(out *PodVolumeBackup) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Spec.DeepCopyInto(&out.Spec) in.Status.DeepCopyInto(&out.Status) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodVolumeBackup. func (in *PodVolumeBackup) DeepCopy() *PodVolumeBackup { if in == nil { return nil } out := new(PodVolumeBackup) in.DeepCopyInto(out) return out } // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. func (in *PodVolumeBackup) DeepCopyObject() runtime.Object { if c := in.DeepCopy(); c != nil { return c } return nil } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PodVolumeBackupList) DeepCopyInto(out *PodVolumeBackupList) { *out = *in out.TypeMeta = in.TypeMeta in.ListMeta.DeepCopyInto(&out.ListMeta) if in.Items != nil { in, out := &in.Items, &out.Items *out = make([]PodVolumeBackup, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodVolumeBackupList. func (in *PodVolumeBackupList) DeepCopy() *PodVolumeBackupList { if in == nil { return nil } out := new(PodVolumeBackupList) in.DeepCopyInto(out) return out } // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. func (in *PodVolumeBackupList) DeepCopyObject() runtime.Object { if c := in.DeepCopy(); c != nil { return c } return nil } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PodVolumeBackupSpec) DeepCopyInto(out *PodVolumeBackupSpec) { *out = *in out.Pod = in.Pod if in.Tags != nil { in, out := &in.Tags, &out.Tags *out = make(map[string]string, len(*in)) for key, val := range *in { (*out)[key] = val } } if in.UploaderSettings != nil { in, out := &in.UploaderSettings, &out.UploaderSettings *out = make(map[string]string, len(*in)) for key, val := range *in { (*out)[key] = val } } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodVolumeBackupSpec. func (in *PodVolumeBackupSpec) DeepCopy() *PodVolumeBackupSpec { if in == nil { return nil } out := new(PodVolumeBackupSpec) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PodVolumeBackupStatus) DeepCopyInto(out *PodVolumeBackupStatus) { *out = *in if in.StartTimestamp != nil { in, out := &in.StartTimestamp, &out.StartTimestamp *out = (*in).DeepCopy() } if in.CompletionTimestamp != nil { in, out := &in.CompletionTimestamp, &out.CompletionTimestamp *out = (*in).DeepCopy() } out.Progress = in.Progress if in.AcceptedTimestamp != nil { in, out := &in.AcceptedTimestamp, &out.AcceptedTimestamp *out = (*in).DeepCopy() } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodVolumeBackupStatus. func (in *PodVolumeBackupStatus) DeepCopy() *PodVolumeBackupStatus { if in == nil { return nil } out := new(PodVolumeBackupStatus) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PodVolumeRestore) DeepCopyInto(out *PodVolumeRestore) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Spec.DeepCopyInto(&out.Spec) in.Status.DeepCopyInto(&out.Status) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodVolumeRestore. func (in *PodVolumeRestore) DeepCopy() *PodVolumeRestore { if in == nil { return nil } out := new(PodVolumeRestore) in.DeepCopyInto(out) return out } // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. func (in *PodVolumeRestore) DeepCopyObject() runtime.Object { if c := in.DeepCopy(); c != nil { return c } return nil } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PodVolumeRestoreList) DeepCopyInto(out *PodVolumeRestoreList) { *out = *in out.TypeMeta = in.TypeMeta in.ListMeta.DeepCopyInto(&out.ListMeta) if in.Items != nil { in, out := &in.Items, &out.Items *out = make([]PodVolumeRestore, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodVolumeRestoreList. func (in *PodVolumeRestoreList) DeepCopy() *PodVolumeRestoreList { if in == nil { return nil } out := new(PodVolumeRestoreList) in.DeepCopyInto(out) return out } // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. func (in *PodVolumeRestoreList) DeepCopyObject() runtime.Object { if c := in.DeepCopy(); c != nil { return c } return nil } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PodVolumeRestoreSpec) DeepCopyInto(out *PodVolumeRestoreSpec) { *out = *in out.Pod = in.Pod if in.UploaderSettings != nil { in, out := &in.UploaderSettings, &out.UploaderSettings *out = make(map[string]string, len(*in)) for key, val := range *in { (*out)[key] = val } } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodVolumeRestoreSpec. func (in *PodVolumeRestoreSpec) DeepCopy() *PodVolumeRestoreSpec { if in == nil { return nil } out := new(PodVolumeRestoreSpec) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PodVolumeRestoreStatus) DeepCopyInto(out *PodVolumeRestoreStatus) { *out = *in if in.StartTimestamp != nil { in, out := &in.StartTimestamp, &out.StartTimestamp *out = (*in).DeepCopy() } if in.CompletionTimestamp != nil { in, out := &in.CompletionTimestamp, &out.CompletionTimestamp *out = (*in).DeepCopy() } out.Progress = in.Progress if in.AcceptedTimestamp != nil { in, out := &in.AcceptedTimestamp, &out.AcceptedTimestamp *out = (*in).DeepCopy() } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodVolumeRestoreStatus. func (in *PodVolumeRestoreStatus) DeepCopy() *PodVolumeRestoreStatus { if in == nil { return nil } out := new(PodVolumeRestoreStatus) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Restore) DeepCopyInto(out *Restore) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Spec.DeepCopyInto(&out.Spec) in.Status.DeepCopyInto(&out.Status) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Restore. func (in *Restore) DeepCopy() *Restore { if in == nil { return nil } out := new(Restore) in.DeepCopyInto(out) return out } // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. func (in *Restore) DeepCopyObject() runtime.Object { if c := in.DeepCopy(); c != nil { return c } return nil } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RestoreHooks) DeepCopyInto(out *RestoreHooks) { *out = *in if in.Resources != nil { in, out := &in.Resources, &out.Resources *out = make([]RestoreResourceHookSpec, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RestoreHooks. func (in *RestoreHooks) DeepCopy() *RestoreHooks { if in == nil { return nil } out := new(RestoreHooks) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RestoreList) DeepCopyInto(out *RestoreList) { *out = *in out.TypeMeta = in.TypeMeta in.ListMeta.DeepCopyInto(&out.ListMeta) if in.Items != nil { in, out := &in.Items, &out.Items *out = make([]Restore, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RestoreList. func (in *RestoreList) DeepCopy() *RestoreList { if in == nil { return nil } out := new(RestoreList) in.DeepCopyInto(out) return out } // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. func (in *RestoreList) DeepCopyObject() runtime.Object { if c := in.DeepCopy(); c != nil { return c } return nil } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RestoreProgress) DeepCopyInto(out *RestoreProgress) { *out = *in } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RestoreProgress. func (in *RestoreProgress) DeepCopy() *RestoreProgress { if in == nil { return nil } out := new(RestoreProgress) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RestoreResourceHook) DeepCopyInto(out *RestoreResourceHook) { *out = *in if in.Exec != nil { in, out := &in.Exec, &out.Exec *out = new(ExecRestoreHook) (*in).DeepCopyInto(*out) } if in.Init != nil { in, out := &in.Init, &out.Init *out = new(InitRestoreHook) (*in).DeepCopyInto(*out) } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RestoreResourceHook. func (in *RestoreResourceHook) DeepCopy() *RestoreResourceHook { if in == nil { return nil } out := new(RestoreResourceHook) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RestoreResourceHookSpec) DeepCopyInto(out *RestoreResourceHookSpec) { *out = *in if in.IncludedNamespaces != nil { in, out := &in.IncludedNamespaces, &out.IncludedNamespaces *out = make([]string, len(*in)) copy(*out, *in) } if in.ExcludedNamespaces != nil { in, out := &in.ExcludedNamespaces, &out.ExcludedNamespaces *out = make([]string, len(*in)) copy(*out, *in) } if in.IncludedResources != nil { in, out := &in.IncludedResources, &out.IncludedResources *out = make([]string, len(*in)) copy(*out, *in) } if in.ExcludedResources != nil { in, out := &in.ExcludedResources, &out.ExcludedResources *out = make([]string, len(*in)) copy(*out, *in) } if in.LabelSelector != nil { in, out := &in.LabelSelector, &out.LabelSelector *out = new(metav1.LabelSelector) (*in).DeepCopyInto(*out) } if in.PostHooks != nil { in, out := &in.PostHooks, &out.PostHooks *out = make([]RestoreResourceHook, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RestoreResourceHookSpec. func (in *RestoreResourceHookSpec) DeepCopy() *RestoreResourceHookSpec { if in == nil { return nil } out := new(RestoreResourceHookSpec) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RestoreSpec) DeepCopyInto(out *RestoreSpec) { *out = *in if in.IncludedNamespaces != nil { in, out := &in.IncludedNamespaces, &out.IncludedNamespaces *out = make([]string, len(*in)) copy(*out, *in) } if in.ExcludedNamespaces != nil { in, out := &in.ExcludedNamespaces, &out.ExcludedNamespaces *out = make([]string, len(*in)) copy(*out, *in) } if in.IncludedResources != nil { in, out := &in.IncludedResources, &out.IncludedResources *out = make([]string, len(*in)) copy(*out, *in) } if in.ExcludedResources != nil { in, out := &in.ExcludedResources, &out.ExcludedResources *out = make([]string, len(*in)) copy(*out, *in) } if in.NamespaceMapping != nil { in, out := &in.NamespaceMapping, &out.NamespaceMapping *out = make(map[string]string, len(*in)) for key, val := range *in { (*out)[key] = val } } if in.LabelSelector != nil { in, out := &in.LabelSelector, &out.LabelSelector *out = new(metav1.LabelSelector) (*in).DeepCopyInto(*out) } if in.OrLabelSelectors != nil { in, out := &in.OrLabelSelectors, &out.OrLabelSelectors *out = make([]*metav1.LabelSelector, len(*in)) for i := range *in { if (*in)[i] != nil { in, out := &(*in)[i], &(*out)[i] *out = new(metav1.LabelSelector) (*in).DeepCopyInto(*out) } } } if in.RestorePVs != nil { in, out := &in.RestorePVs, &out.RestorePVs *out = new(bool) **out = **in } if in.RestoreStatus != nil { in, out := &in.RestoreStatus, &out.RestoreStatus *out = new(RestoreStatusSpec) (*in).DeepCopyInto(*out) } if in.PreserveNodePorts != nil { in, out := &in.PreserveNodePorts, &out.PreserveNodePorts *out = new(bool) **out = **in } if in.IncludeClusterResources != nil { in, out := &in.IncludeClusterResources, &out.IncludeClusterResources *out = new(bool) **out = **in } in.Hooks.DeepCopyInto(&out.Hooks) out.ItemOperationTimeout = in.ItemOperationTimeout if in.ResourceModifier != nil { in, out := &in.ResourceModifier, &out.ResourceModifier *out = new(corev1.TypedLocalObjectReference) (*in).DeepCopyInto(*out) } if in.UploaderConfig != nil { in, out := &in.UploaderConfig, &out.UploaderConfig *out = new(UploaderConfigForRestore) (*in).DeepCopyInto(*out) } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RestoreSpec. func (in *RestoreSpec) DeepCopy() *RestoreSpec { if in == nil { return nil } out := new(RestoreSpec) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RestoreStatus) DeepCopyInto(out *RestoreStatus) { *out = *in if in.ValidationErrors != nil { in, out := &in.ValidationErrors, &out.ValidationErrors *out = make([]string, len(*in)) copy(*out, *in) } if in.StartTimestamp != nil { in, out := &in.StartTimestamp, &out.StartTimestamp *out = (*in).DeepCopy() } if in.CompletionTimestamp != nil { in, out := &in.CompletionTimestamp, &out.CompletionTimestamp *out = (*in).DeepCopy() } if in.Progress != nil { in, out := &in.Progress, &out.Progress *out = new(RestoreProgress) **out = **in } if in.HookStatus != nil { in, out := &in.HookStatus, &out.HookStatus *out = new(HookStatus) **out = **in } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RestoreStatus. func (in *RestoreStatus) DeepCopy() *RestoreStatus { if in == nil { return nil } out := new(RestoreStatus) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RestoreStatusSpec) DeepCopyInto(out *RestoreStatusSpec) { *out = *in if in.IncludedResources != nil { in, out := &in.IncludedResources, &out.IncludedResources *out = make([]string, len(*in)) copy(*out, *in) } if in.ExcludedResources != nil { in, out := &in.ExcludedResources, &out.ExcludedResources *out = make([]string, len(*in)) copy(*out, *in) } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RestoreStatusSpec. func (in *RestoreStatusSpec) DeepCopy() *RestoreStatusSpec { if in == nil { return nil } out := new(RestoreStatusSpec) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Schedule) DeepCopyInto(out *Schedule) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Spec.DeepCopyInto(&out.Spec) in.Status.DeepCopyInto(&out.Status) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Schedule. func (in *Schedule) DeepCopy() *Schedule { if in == nil { return nil } out := new(Schedule) in.DeepCopyInto(out) return out } // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. func (in *Schedule) DeepCopyObject() runtime.Object { if c := in.DeepCopy(); c != nil { return c } return nil } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ScheduleList) DeepCopyInto(out *ScheduleList) { *out = *in out.TypeMeta = in.TypeMeta in.ListMeta.DeepCopyInto(&out.ListMeta) if in.Items != nil { in, out := &in.Items, &out.Items *out = make([]Schedule, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScheduleList. func (in *ScheduleList) DeepCopy() *ScheduleList { if in == nil { return nil } out := new(ScheduleList) in.DeepCopyInto(out) return out } // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. func (in *ScheduleList) DeepCopyObject() runtime.Object { if c := in.DeepCopy(); c != nil { return c } return nil } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ScheduleSpec) DeepCopyInto(out *ScheduleSpec) { *out = *in in.Template.DeepCopyInto(&out.Template) if in.UseOwnerReferencesInBackup != nil { in, out := &in.UseOwnerReferencesInBackup, &out.UseOwnerReferencesInBackup *out = new(bool) **out = **in } if in.SkipImmediately != nil { in, out := &in.SkipImmediately, &out.SkipImmediately *out = new(bool) **out = **in } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScheduleSpec. func (in *ScheduleSpec) DeepCopy() *ScheduleSpec { if in == nil { return nil } out := new(ScheduleSpec) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ScheduleStatus) DeepCopyInto(out *ScheduleStatus) { *out = *in if in.LastBackup != nil { in, out := &in.LastBackup, &out.LastBackup *out = (*in).DeepCopy() } if in.LastSkipped != nil { in, out := &in.LastSkipped, &out.LastSkipped *out = (*in).DeepCopy() } if in.ValidationErrors != nil { in, out := &in.ValidationErrors, &out.ValidationErrors *out = make([]string, len(*in)) copy(*out, *in) } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScheduleStatus. func (in *ScheduleStatus) DeepCopy() *ScheduleStatus { if in == nil { return nil } out := new(ScheduleStatus) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ServerStatusRequest) DeepCopyInto(out *ServerStatusRequest) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) out.Spec = in.Spec in.Status.DeepCopyInto(&out.Status) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServerStatusRequest. func (in *ServerStatusRequest) DeepCopy() *ServerStatusRequest { if in == nil { return nil } out := new(ServerStatusRequest) in.DeepCopyInto(out) return out } // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. func (in *ServerStatusRequest) DeepCopyObject() runtime.Object { if c := in.DeepCopy(); c != nil { return c } return nil } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ServerStatusRequestList) DeepCopyInto(out *ServerStatusRequestList) { *out = *in out.TypeMeta = in.TypeMeta in.ListMeta.DeepCopyInto(&out.ListMeta) if in.Items != nil { in, out := &in.Items, &out.Items *out = make([]ServerStatusRequest, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServerStatusRequestList. func (in *ServerStatusRequestList) DeepCopy() *ServerStatusRequestList { if in == nil { return nil } out := new(ServerStatusRequestList) in.DeepCopyInto(out) return out } // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. func (in *ServerStatusRequestList) DeepCopyObject() runtime.Object { if c := in.DeepCopy(); c != nil { return c } return nil } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ServerStatusRequestSpec) DeepCopyInto(out *ServerStatusRequestSpec) { *out = *in } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServerStatusRequestSpec. func (in *ServerStatusRequestSpec) DeepCopy() *ServerStatusRequestSpec { if in == nil { return nil } out := new(ServerStatusRequestSpec) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ServerStatusRequestStatus) DeepCopyInto(out *ServerStatusRequestStatus) { *out = *in if in.ProcessedTimestamp != nil { in, out := &in.ProcessedTimestamp, &out.ProcessedTimestamp *out = (*in).DeepCopy() } if in.Plugins != nil { in, out := &in.Plugins, &out.Plugins *out = make([]PluginInfo, len(*in)) copy(*out, *in) } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServerStatusRequestStatus. func (in *ServerStatusRequestStatus) DeepCopy() *ServerStatusRequestStatus { if in == nil { return nil } out := new(ServerStatusRequestStatus) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *StorageType) DeepCopyInto(out *StorageType) { *out = *in if in.ObjectStorage != nil { in, out := &in.ObjectStorage, &out.ObjectStorage *out = new(ObjectStorageLocation) (*in).DeepCopyInto(*out) } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StorageType. func (in *StorageType) DeepCopy() *StorageType { if in == nil { return nil } out := new(StorageType) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *UploaderConfigForBackup) DeepCopyInto(out *UploaderConfigForBackup) { *out = *in } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UploaderConfigForBackup. func (in *UploaderConfigForBackup) DeepCopy() *UploaderConfigForBackup { if in == nil { return nil } out := new(UploaderConfigForBackup) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *UploaderConfigForRestore) DeepCopyInto(out *UploaderConfigForRestore) { *out = *in if in.WriteSparseFiles != nil { in, out := &in.WriteSparseFiles, &out.WriteSparseFiles *out = new(bool) **out = **in } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UploaderConfigForRestore. func (in *UploaderConfigForRestore) DeepCopy() *UploaderConfigForRestore { if in == nil { return nil } out := new(UploaderConfigForRestore) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VolumeSnapshotLocation) DeepCopyInto(out *VolumeSnapshotLocation) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Spec.DeepCopyInto(&out.Spec) out.Status = in.Status } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeSnapshotLocation. func (in *VolumeSnapshotLocation) DeepCopy() *VolumeSnapshotLocation { if in == nil { return nil } out := new(VolumeSnapshotLocation) in.DeepCopyInto(out) return out } // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. func (in *VolumeSnapshotLocation) DeepCopyObject() runtime.Object { if c := in.DeepCopy(); c != nil { return c } return nil } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VolumeSnapshotLocationList) DeepCopyInto(out *VolumeSnapshotLocationList) { *out = *in out.TypeMeta = in.TypeMeta in.ListMeta.DeepCopyInto(&out.ListMeta) if in.Items != nil { in, out := &in.Items, &out.Items *out = make([]VolumeSnapshotLocation, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeSnapshotLocationList. func (in *VolumeSnapshotLocationList) DeepCopy() *VolumeSnapshotLocationList { if in == nil { return nil } out := new(VolumeSnapshotLocationList) in.DeepCopyInto(out) return out } // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. func (in *VolumeSnapshotLocationList) DeepCopyObject() runtime.Object { if c := in.DeepCopy(); c != nil { return c } return nil } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VolumeSnapshotLocationSpec) DeepCopyInto(out *VolumeSnapshotLocationSpec) { *out = *in if in.Config != nil { in, out := &in.Config, &out.Config *out = make(map[string]string, len(*in)) for key, val := range *in { (*out)[key] = val } } if in.Credential != nil { in, out := &in.Credential, &out.Credential *out = new(corev1.SecretKeySelector) (*in).DeepCopyInto(*out) } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeSnapshotLocationSpec. func (in *VolumeSnapshotLocationSpec) DeepCopy() *VolumeSnapshotLocationSpec { if in == nil { return nil } out := new(VolumeSnapshotLocationSpec) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VolumeSnapshotLocationStatus) DeepCopyInto(out *VolumeSnapshotLocationStatus) { *out = *in } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeSnapshotLocationStatus. func (in *VolumeSnapshotLocationStatus) DeepCopy() *VolumeSnapshotLocationStatus { if in == nil { return nil } out := new(VolumeSnapshotLocationStatus) in.DeepCopyInto(out) return out } ================================================ FILE: pkg/apis/velero/v2alpha1/data_download_types.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v2alpha1 import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/vmware-tanzu/velero/pkg/apis/velero/shared" ) // DataDownloadSpec is the specification for a DataDownload. type DataDownloadSpec struct { // TargetVolume is the information of the target PVC and PV. TargetVolume TargetVolumeSpec `json:"targetVolume"` // BackupStorageLocation is the name of the backup storage location // where the backup repository is stored. BackupStorageLocation string `json:"backupStorageLocation"` // DataMover specifies the data mover to be used by the backup. // If DataMover is "" or "velero", the built-in data mover will be used. // +optional DataMover string `json:"datamover,omitempty"` // SnapshotID is the ID of the Velero backup snapshot to be restored from. SnapshotID string `json:"snapshotID"` // SourceNamespace is the original namespace where the volume is backed up from. // It may be different from SourcePVC's namespace if namespace is remapped during restore. SourceNamespace string `json:"sourceNamespace"` // DataMoverConfig is for data-mover-specific configuration fields. // +optional DataMoverConfig map[string]string `json:"dataMoverConfig,omitempty"` // Cancel indicates request to cancel the ongoing DataDownload. It can be set // when the DataDownload is in InProgress phase Cancel bool `json:"cancel,omitempty"` // OperationTimeout specifies the time used to wait internal operations, // before returning error as timeout. OperationTimeout metav1.Duration `json:"operationTimeout"` // NodeOS is OS of the node where the DataDownload is processed. // +optional NodeOS NodeOS `json:"nodeOS,omitempty"` // SnapshotSize is the logical size in Bytes of the snapshot. // +optional SnapshotSize int64 `json:"snapshotSize,omitempty"` } // TargetVolumeSpec is the specification for a target PVC. type TargetVolumeSpec struct { // PVC is the name of the target PVC that is created by Velero restore PVC string `json:"pvc"` // PV is the name of the target PV that is created by Velero restore PV string `json:"pv"` // Namespace is the target namespace Namespace string `json:"namespace"` } // DataDownloadPhase represents the lifecycle phase of a DataDownload. // +kubebuilder:validation:Enum=New;Accepted;Prepared;InProgress;Canceling;Canceled;Completed;Failed type DataDownloadPhase string const ( DataDownloadPhaseNew DataDownloadPhase = "New" DataDownloadPhaseAccepted DataDownloadPhase = "Accepted" DataDownloadPhasePrepared DataDownloadPhase = "Prepared" DataDownloadPhaseInProgress DataDownloadPhase = "InProgress" DataDownloadPhaseCanceling DataDownloadPhase = "Canceling" DataDownloadPhaseCanceled DataDownloadPhase = "Canceled" DataDownloadPhaseCompleted DataDownloadPhase = "Completed" DataDownloadPhaseFailed DataDownloadPhase = "Failed" ) // DataDownloadStatus is the current status of a DataDownload. type DataDownloadStatus struct { // Phase is the current state of the DataDownload. // +optional Phase DataDownloadPhase `json:"phase,omitempty"` // Message is a message about the DataDownload's status. // +optional Message string `json:"message,omitempty"` // StartTimestamp records the time a restore was started. // The server's time is used for StartTimestamps // +optional // +nullable StartTimestamp *metav1.Time `json:"startTimestamp,omitempty"` // CompletionTimestamp records the time a restore was completed. // Completion time is recorded even on failed restores. // The server's time is used for CompletionTimestamps // +optional // +nullable CompletionTimestamp *metav1.Time `json:"completionTimestamp,omitempty"` // Progress holds the total number of bytes of the snapshot and the current // number of restored bytes. This can be used to display progress information // about the restore operation. // +optional Progress shared.DataMoveOperationProgress `json:"progress,omitempty"` // Node is name of the node where the DataDownload is processed. // +optional Node string `json:"node,omitempty"` // Node is name of the node where the DataUpload is prepared. // +optional AcceptedByNode string `json:"acceptedByNode,omitempty"` // AcceptedTimestamp records the time the DataUpload is to be prepared. // The server's time is used for AcceptedTimestamp // +optional // +nullable AcceptedTimestamp *metav1.Time `json:"acceptedTimestamp,omitempty"` } // TODO(2.0) After converting all resources to use the runtime-controller client, the genclient and k8s:deepcopy markers will no longer be needed and should be removed. // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // +kubebuilder:object:generate=true // +kubebuilder:object:root=true // +kubebuilder:storageversion // +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.phase",description="DataDownload status such as New/InProgress" // +kubebuilder:printcolumn:name="Started",type="date",JSONPath=".status.startTimestamp",description="Time duration since this DataDownload was started" // +kubebuilder:printcolumn:name="Bytes Done",type="integer",format="int64",JSONPath=".status.progress.bytesDone",description="Completed bytes" // +kubebuilder:printcolumn:name="Total Bytes",type="integer",format="int64",JSONPath=".status.progress.totalBytes",description="Total bytes" // +kubebuilder:printcolumn:name="Storage Location",type="string",JSONPath=".spec.backupStorageLocation",description="Name of the Backup Storage Location where the backup data is stored" // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Time duration since this DataDownload was created" // +kubebuilder:printcolumn:name="Node",type="string",JSONPath=".status.node",description="Name of the node where the DataDownload is processed" // DataDownload acts as the protocol between data mover plugins and data mover controller for the datamover restore operation type DataDownload struct { metav1.TypeMeta `json:",inline"` // +optional metav1.ObjectMeta `json:"metadata,omitempty"` // +optional Spec DataDownloadSpec `json:"spec,omitempty"` // +optional Status DataDownloadStatus `json:"status,omitempty"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // +kubebuilder:object:generate=true // +kubebuilder:object:root=true // +kubebuilder:rbac:groups=velero.io,resources=datadownloads,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=velero.io,resources=datadownloads/status,verbs=get;update;patch // DataDownloadList is a list of DataDownloads. type DataDownloadList struct { metav1.TypeMeta `json:",inline"` // +optional metav1.ListMeta `json:"metadata,omitempty"` Items []DataDownload `json:"items"` } ================================================ FILE: pkg/apis/velero/v2alpha1/data_upload_types.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v2alpha1 import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/vmware-tanzu/velero/pkg/apis/velero/shared" ) // DataUploadSpec is the specification for a DataUpload. type DataUploadSpec struct { // SnapshotType is the type of the snapshot to be backed up. SnapshotType SnapshotType `json:"snapshotType"` // If SnapshotType is CSI, CSISnapshot provides the information of the CSI snapshot. // +optional // +nullable CSISnapshot *CSISnapshotSpec `json:"csiSnapshot"` // SourcePVC is the name of the PVC which the snapshot is taken for. SourcePVC string `json:"sourcePVC"` // DataMover specifies the data mover to be used by the backup. // If DataMover is "" or "velero", the built-in data mover will be used. // +optional DataMover string `json:"datamover,omitempty"` // BackupStorageLocation is the name of the backup storage location // where the backup repository is stored. BackupStorageLocation string `json:"backupStorageLocation"` // SourceNamespace is the original namespace where the volume is backed up from. // It is the same namespace for SourcePVC and CSI namespaced objects. SourceNamespace string `json:"sourceNamespace"` // DataMoverConfig is for data-mover-specific configuration fields. // +optional // +nullable DataMoverConfig map[string]string `json:"dataMoverConfig,omitempty"` // Cancel indicates request to cancel the ongoing DataUpload. It can be set // when the DataUpload is in InProgress phase Cancel bool `json:"cancel,omitempty"` // OperationTimeout specifies the time used to wait internal operations, // before returning error as timeout. OperationTimeout metav1.Duration `json:"operationTimeout"` } type SnapshotType string const ( SnapshotTypeCSI SnapshotType = "CSI" ) // CSISnapshotSpec is the specification for a CSI snapshot. type CSISnapshotSpec struct { // VolumeSnapshot is the name of the volume snapshot to be backed up VolumeSnapshot string `json:"volumeSnapshot"` // StorageClass is the name of the storage class of the PVC that the volume snapshot is created from StorageClass string `json:"storageClass"` // SnapshotClass is the name of the snapshot class that the volume snapshot is created with // +optional SnapshotClass string `json:"snapshotClass"` // Driver is the driver used by the VolumeSnapshotContent // +optional Driver string `json:"driver,omitempty"` } // DataUploadPhase represents the lifecycle phase of a DataUpload. // +kubebuilder:validation:Enum=New;Accepted;Prepared;InProgress;Canceling;Canceled;Completed;Failed type DataUploadPhase string const ( DataUploadPhaseNew DataUploadPhase = "New" DataUploadPhaseAccepted DataUploadPhase = "Accepted" DataUploadPhasePrepared DataUploadPhase = "Prepared" DataUploadPhaseInProgress DataUploadPhase = "InProgress" DataUploadPhaseCanceling DataUploadPhase = "Canceling" DataUploadPhaseCanceled DataUploadPhase = "Canceled" DataUploadPhaseCompleted DataUploadPhase = "Completed" DataUploadPhaseFailed DataUploadPhase = "Failed" ) // NodeOS represents OS of a node. // +kubebuilder:validation:Enum=auto;linux;windows type NodeOS string const ( NodeOSLinux NodeOS = "linux" NodeOSWindows NodeOS = "windows" NodeOSAuto NodeOS = "auto" ) // DataUploadStatus is the current status of a DataUpload. type DataUploadStatus struct { // Phase is the current state of the DataUpload. // +optional Phase DataUploadPhase `json:"phase,omitempty"` // Path is the full path of the snapshot volume being backed up. // +optional Path string `json:"path,omitempty"` // SnapshotID is the identifier for the snapshot in the backup repository. // +optional SnapshotID string `json:"snapshotID,omitempty"` // DataMoverResult stores data-mover-specific information as a result of the DataUpload. // +optional // +nullable DataMoverResult *map[string]string `json:"dataMoverResult,omitempty"` // Message is a message about the DataUpload's status. // +optional Message string `json:"message,omitempty"` // StartTimestamp records the time a backup was started. // Separate from CreationTimestamp, since that value changes // on restores. // The server's time is used for StartTimestamps // +optional // +nullable StartTimestamp *metav1.Time `json:"startTimestamp,omitempty"` // CompletionTimestamp records the time a backup was completed. // Completion time is recorded even on failed backups. // Completion time is recorded before uploading the backup object. // The server's time is used for CompletionTimestamps // +optional // +nullable CompletionTimestamp *metav1.Time `json:"completionTimestamp,omitempty"` // Progress holds the total number of bytes of the volume and the current // number of backed up bytes. This can be used to display progress information // about the backup operation. // +optional Progress shared.DataMoveOperationProgress `json:"progress,omitempty"` // IncrementalBytes holds the number of bytes new or changed since the last backup // +optional IncrementalBytes int64 `json:"incrementalBytes,omitempty"` // Node is name of the node where the DataUpload is processed. // +optional Node string `json:"node,omitempty"` // NodeOS is OS of the node where the DataUpload is processed. // +optional NodeOS NodeOS `json:"nodeOS,omitempty"` // AcceptedByNode is name of the node where the DataUpload is prepared. // +optional AcceptedByNode string `json:"acceptedByNode,omitempty"` // AcceptedTimestamp records the time the DataUpload is to be prepared. // The server's time is used for AcceptedTimestamp // +optional // +nullable AcceptedTimestamp *metav1.Time `json:"acceptedTimestamp,omitempty"` } // TODO(2.0) After converting all resources to use the runttime-controller client, // the genclient and k8s:deepcopy markers will no longer be needed and should be removed. // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // +kubebuilder:object:root=true // +kubebuilder:object:generate=true // +kubebuilder:storageversion // +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.phase",description="DataUpload status such as New/InProgress" // +kubebuilder:printcolumn:name="Started",type="date",JSONPath=".status.startTimestamp",description="Time duration since this DataUpload was started" // +kubebuilder:printcolumn:name="Bytes Done",type="integer",format="int64",JSONPath=".status.progress.bytesDone",description="Completed bytes" // +kubebuilder:printcolumn:name="Total Bytes",type="integer",format="int64",JSONPath=".status.progress.totalBytes",description="Total bytes" // +kubebuilder:printcolumn:name="Incremental Bytes",type="integer",format="int64",JSONPath=".status.incrementalBytes",description="Incremental bytes",priority=10 // +kubebuilder:printcolumn:name="Storage Location",type="string",JSONPath=".spec.backupStorageLocation",description="Name of the Backup Storage Location where this backup should be stored" // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Time duration since this DataUpload was created" // +kubebuilder:printcolumn:name="Node",type="string",JSONPath=".status.node",description="Name of the node where the DataUpload is processed" // DataUpload acts as the protocol between data mover plugins and data mover controller for the datamover backup operation type DataUpload struct { metav1.TypeMeta `json:",inline"` // +optional metav1.ObjectMeta `json:"metadata,omitempty"` // +optional Spec DataUploadSpec `json:"spec,omitempty"` // +optional Status DataUploadStatus `json:"status,omitempty"` } // TODO(2.0) After converting all resources to use the runtime-controller client, // the k8s:deepcopy marker will no longer be needed and should be removed. // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // +kubebuilder:object:root=true // +kubebuilder:rbac:groups=velero.io,resources=datauploads,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=velero.io,resources=datauploads/status,verbs=get;update;patch // DataUploadList is a list of DataUploads. type DataUploadList struct { metav1.TypeMeta `json:",inline"` // +optional metav1.ListMeta `json:"metadata,omitempty"` Items []DataUpload `json:"items"` } // DataUploadResult represents the SnasphotBackup result to be used by DataDownload. type DataUploadResult struct { // BackupStorageLocation is the name of the backup storage location // where the backup repository is stored. BackupStorageLocation string `json:"backupStorageLocation"` // DataMover specifies the data mover used by the DataUpload // +optional DataMover string `json:"datamover,omitempty"` // SnapshotID is the identifier for the snapshot in the backup repository. SnapshotID string `json:"snapshotID,omitempty"` // SourceNamespace is the original namespace where the volume is backed up from. SourceNamespace string `json:"sourceNamespace"` // DataMoverResult stores data-mover-specific information as a result of the DataUpload. // +optional // +nullable DataMoverResult *map[string]string `json:"dataMoverResult,omitempty"` // NodeOS is OS of the node where the DataUpload is processed. // +optional NodeOS NodeOS `json:"nodeOS,omitempty"` // SnapshotSize is the logical size in Bytes of the snapshot. // +optional SnapshotSize int64 `json:"snapshotSize,omitempty"` } ================================================ FILE: pkg/apis/velero/v2alpha1/doc.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ // +k8s:deepcopy-gen=package // Package v2alpha1 is the v2alpha1 version of the API. // +groupName=velero.io package v2alpha1 ================================================ FILE: pkg/apis/velero/v2alpha1/groupversion_info.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ // Package v2alpha1 contains API Schema definitions for the velero v2alpha1 API group // +kubebuilder:object:generate=true // +groupName=velero.io package v2alpha1 import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" ) var ( // SchemeGroupVersion is group version used to register these objects SchemeGroupVersion = schema.GroupVersion{Group: "velero.io", Version: "v2alpha1"} // SchemeBuilder is used to add go types to the GroupVersionKind scheme SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) // AddToScheme adds the types in this group-version to the given scheme. AddToScheme = SchemeBuilder.AddToScheme ) ================================================ FILE: pkg/apis/velero/v2alpha1/register.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v2alpha1 import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/sets" ) // Resource gets a Velero GroupResource for a specified resource func Resource(resource string) schema.GroupResource { return SchemeGroupVersion.WithResource(resource).GroupResource() } type typeInfo struct { PluralName string ItemType runtime.Object ItemListType runtime.Object } func newTypeInfo(pluralName string, itemType, itemListType runtime.Object) typeInfo { return typeInfo{ PluralName: pluralName, ItemType: itemType, ItemListType: itemListType, } } // CustomResources returns a map of all custom resources within the Velero // API group, keyed on Kind. func CustomResources() map[string]typeInfo { return map[string]typeInfo{ "DataUpload": newTypeInfo("datauploads", &DataUpload{}, &DataUploadList{}), "DataDownload": newTypeInfo("datadownloads", &DataDownload{}, &DataDownloadList{}), } } // CustomResourceKinds returns a list of all custom resources kinds within the Velero func CustomResourceKinds() sets.Set[string] { kinds := sets.New[string]() resources := CustomResources() for kind := range resources { kinds.Insert(kind) } return kinds } func addKnownTypes(scheme *runtime.Scheme) error { for _, typeInfo := range CustomResources() { scheme.AddKnownTypes(SchemeGroupVersion, typeInfo.ItemType, typeInfo.ItemListType) } metav1.AddToGroupVersion(scheme, SchemeGroupVersion) return nil } ================================================ FILE: pkg/apis/velero/v2alpha1/zz_generated.deepcopy.go ================================================ //go:build !ignore_autogenerated // Code generated by controller-gen. DO NOT EDIT. package v2alpha1 import ( "k8s.io/apimachinery/pkg/runtime" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CSISnapshotSpec) DeepCopyInto(out *CSISnapshotSpec) { *out = *in } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CSISnapshotSpec. func (in *CSISnapshotSpec) DeepCopy() *CSISnapshotSpec { if in == nil { return nil } out := new(CSISnapshotSpec) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DataDownload) DeepCopyInto(out *DataDownload) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Spec.DeepCopyInto(&out.Spec) in.Status.DeepCopyInto(&out.Status) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataDownload. func (in *DataDownload) DeepCopy() *DataDownload { if in == nil { return nil } out := new(DataDownload) in.DeepCopyInto(out) return out } // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. func (in *DataDownload) DeepCopyObject() runtime.Object { if c := in.DeepCopy(); c != nil { return c } return nil } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DataDownloadList) DeepCopyInto(out *DataDownloadList) { *out = *in out.TypeMeta = in.TypeMeta in.ListMeta.DeepCopyInto(&out.ListMeta) if in.Items != nil { in, out := &in.Items, &out.Items *out = make([]DataDownload, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataDownloadList. func (in *DataDownloadList) DeepCopy() *DataDownloadList { if in == nil { return nil } out := new(DataDownloadList) in.DeepCopyInto(out) return out } // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. func (in *DataDownloadList) DeepCopyObject() runtime.Object { if c := in.DeepCopy(); c != nil { return c } return nil } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DataDownloadSpec) DeepCopyInto(out *DataDownloadSpec) { *out = *in out.TargetVolume = in.TargetVolume if in.DataMoverConfig != nil { in, out := &in.DataMoverConfig, &out.DataMoverConfig *out = make(map[string]string, len(*in)) for key, val := range *in { (*out)[key] = val } } out.OperationTimeout = in.OperationTimeout } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataDownloadSpec. func (in *DataDownloadSpec) DeepCopy() *DataDownloadSpec { if in == nil { return nil } out := new(DataDownloadSpec) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DataDownloadStatus) DeepCopyInto(out *DataDownloadStatus) { *out = *in if in.StartTimestamp != nil { in, out := &in.StartTimestamp, &out.StartTimestamp *out = (*in).DeepCopy() } if in.CompletionTimestamp != nil { in, out := &in.CompletionTimestamp, &out.CompletionTimestamp *out = (*in).DeepCopy() } out.Progress = in.Progress if in.AcceptedTimestamp != nil { in, out := &in.AcceptedTimestamp, &out.AcceptedTimestamp *out = (*in).DeepCopy() } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataDownloadStatus. func (in *DataDownloadStatus) DeepCopy() *DataDownloadStatus { if in == nil { return nil } out := new(DataDownloadStatus) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DataUpload) DeepCopyInto(out *DataUpload) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Spec.DeepCopyInto(&out.Spec) in.Status.DeepCopyInto(&out.Status) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataUpload. func (in *DataUpload) DeepCopy() *DataUpload { if in == nil { return nil } out := new(DataUpload) in.DeepCopyInto(out) return out } // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. func (in *DataUpload) DeepCopyObject() runtime.Object { if c := in.DeepCopy(); c != nil { return c } return nil } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DataUploadList) DeepCopyInto(out *DataUploadList) { *out = *in out.TypeMeta = in.TypeMeta in.ListMeta.DeepCopyInto(&out.ListMeta) if in.Items != nil { in, out := &in.Items, &out.Items *out = make([]DataUpload, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataUploadList. func (in *DataUploadList) DeepCopy() *DataUploadList { if in == nil { return nil } out := new(DataUploadList) in.DeepCopyInto(out) return out } // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. func (in *DataUploadList) DeepCopyObject() runtime.Object { if c := in.DeepCopy(); c != nil { return c } return nil } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DataUploadResult) DeepCopyInto(out *DataUploadResult) { *out = *in if in.DataMoverResult != nil { in, out := &in.DataMoverResult, &out.DataMoverResult *out = new(map[string]string) if **in != nil { in, out := *in, *out *out = make(map[string]string, len(*in)) for key, val := range *in { (*out)[key] = val } } } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataUploadResult. func (in *DataUploadResult) DeepCopy() *DataUploadResult { if in == nil { return nil } out := new(DataUploadResult) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DataUploadSpec) DeepCopyInto(out *DataUploadSpec) { *out = *in if in.CSISnapshot != nil { in, out := &in.CSISnapshot, &out.CSISnapshot *out = new(CSISnapshotSpec) **out = **in } if in.DataMoverConfig != nil { in, out := &in.DataMoverConfig, &out.DataMoverConfig *out = make(map[string]string, len(*in)) for key, val := range *in { (*out)[key] = val } } out.OperationTimeout = in.OperationTimeout } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataUploadSpec. func (in *DataUploadSpec) DeepCopy() *DataUploadSpec { if in == nil { return nil } out := new(DataUploadSpec) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DataUploadStatus) DeepCopyInto(out *DataUploadStatus) { *out = *in if in.DataMoverResult != nil { in, out := &in.DataMoverResult, &out.DataMoverResult *out = new(map[string]string) if **in != nil { in, out := *in, *out *out = make(map[string]string, len(*in)) for key, val := range *in { (*out)[key] = val } } } if in.StartTimestamp != nil { in, out := &in.StartTimestamp, &out.StartTimestamp *out = (*in).DeepCopy() } if in.CompletionTimestamp != nil { in, out := &in.CompletionTimestamp, &out.CompletionTimestamp *out = (*in).DeepCopy() } out.Progress = in.Progress if in.AcceptedTimestamp != nil { in, out := &in.AcceptedTimestamp, &out.AcceptedTimestamp *out = (*in).DeepCopy() } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataUploadStatus. func (in *DataUploadStatus) DeepCopy() *DataUploadStatus { if in == nil { return nil } out := new(DataUploadStatus) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TargetVolumeSpec) DeepCopyInto(out *TargetVolumeSpec) { *out = *in } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TargetVolumeSpec. func (in *TargetVolumeSpec) DeepCopy() *TargetVolumeSpec { if in == nil { return nil } out := new(TargetVolumeSpec) in.DeepCopyInto(out) return out } ================================================ FILE: pkg/archive/extractor.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package archive import ( "archive/tar" "compress/gzip" "io" "path/filepath" "github.com/sirupsen/logrus" "github.com/vmware-tanzu/velero/pkg/util/filesystem" ) // Extractor unzips/extracts a backup tarball to a local // temp directory. type Extractor struct { log logrus.FieldLogger fs filesystem.Interface } func NewExtractor(log logrus.FieldLogger, fs filesystem.Interface) *Extractor { return &Extractor{ log: log, fs: fs, } } // UnzipAndExtractBackup extracts a reader on a gzipped tarball to a local temp directory func (e *Extractor) UnzipAndExtractBackup(src io.Reader) (string, error) { gzr, err := gzip.NewReader(src) if err != nil { e.log.Infof("error creating gzip reader: %v", err) return "", err } defer gzr.Close() return e.readBackup(tar.NewReader(gzr)) } func (e *Extractor) writeFile(target string, tarRdr *tar.Reader) error { file, err := e.fs.Create(target) if err != nil { return err } defer file.Close() if _, err := io.Copy(file, tarRdr); err != nil { return err } return nil } func (e *Extractor) readBackup(tarRdr *tar.Reader) (string, error) { dir, err := e.fs.TempDir("", "") if err != nil { e.log.Infof("error creating temp dir: %v", err) return "", err } for { header, err := tarRdr.Next() if err == io.EOF { break } if err != nil { e.log.Infof("error reading tar: %v", err) return "", err } target := filepath.Join(dir, header.Name) //nolint:gosec // Internal usage. No need to check. switch header.Typeflag { case tar.TypeDir: err := e.fs.MkdirAll(target, header.FileInfo().Mode()) if err != nil { e.log.Infof("mkdirall error: %v", err) return "", err } case tar.TypeReg: // make sure we have the directory created err := e.fs.MkdirAll(filepath.Dir(target), header.FileInfo().Mode()) if err != nil { e.log.Infof("mkdirall error: %v", err) return "", err } // create the file if err := e.writeFile(target, tarRdr); err != nil { e.log.Infof("error copying: %v", err) return "", err } } } return dir, nil } ================================================ FILE: pkg/archive/extractor_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package archive import ( "archive/tar" "compress/gzip" "io" "os" "testing" "github.com/stretchr/testify/require" "github.com/vmware-tanzu/velero/pkg/test" "github.com/vmware-tanzu/velero/pkg/util/filesystem" ) func TestUnzipAndExtractBackup(t *testing.T) { tests := []struct { name string files []string IsTarball bool wantErr bool }{ { name: "when the format of backup file is invalid, an error is returned", files: []string{}, IsTarball: false, wantErr: true, }, { name: "when the backup tarball is empty, the function should work correctly and returns no error", files: []string{}, IsTarball: true, wantErr: false, }, { name: "when the backup tarball includes a mix of items, the function should work correctly and returns no error", files: []string{ "root-dir/resources/namespace/cluster/example.json", "root-dir/resources/pods/namespaces/example.json", "root-dir/metadata/version", }, IsTarball: true, wantErr: false, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { ext := NewExtractor(test.NewLogger(), test.NewFakeFileSystem()) var fileName string var err error if tc.IsTarball { fileName, err = createArchive(tc.files, ext.fs) } else { fileName, err = createRegular(ext.fs) } require.NoError(t, err) file, err := ext.fs.OpenFile(fileName, os.O_RDWR|os.O_CREATE, 0644) require.NoError(t, err) _, err = ext.UnzipAndExtractBackup(file.(io.Reader)) if tc.wantErr && (err == nil) { t.Errorf("%s: wanted error but got nil", tc.name) } if !tc.wantErr && (err != nil) { t.Errorf("%s: wanted no error but got err: %v", tc.name, err) } }) } } func createArchive(files []string, fs filesystem.Interface) (string, error) { outName := "output.tar.gz" out, err := fs.Create(outName) if err != nil { return outName, err } defer out.Close() gw := gzip.NewWriter(out) defer gw.Close() tw := tar.NewWriter(gw) defer tw.Close() // Iterate over files and add them to the tar archive for _, file := range files { err := addToArchive(tw, file, fs) if err != nil { return outName, err } } return outName, nil } func addToArchive(tw *tar.Writer, filename string, fs filesystem.Interface) error { // Create the file file, err := fs.Create(filename) if err != nil { return err } defer file.Close() // Get FileInfo about size, mode, etc. info, err := fs.Stat(filename) if err != nil { return err } // Create a tar Header from the FileInfo data header, err := tar.FileInfoHeader(info, info.Name()) if err != nil { return err } header.Name = filename err = tw.WriteHeader(header) if err != nil { return err } return nil } func createRegular(fs filesystem.Interface) (string, error) { outName := "output" out, err := fs.Create(outName) if err != nil { return outName, err } defer out.Close() return outName, nil } ================================================ FILE: pkg/archive/filesystem.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package archive import ( "encoding/json" "path/filepath" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/util/filesystem" ) // GetItemFilePath returns an item's file path once extracted from a Velero backup archive. func GetItemFilePath(rootDir, groupResource, namespace, name string) string { return GetVersionedItemFilePath(rootDir, groupResource, namespace, name, "") } // GetVersionedItemFilePath returns an item's file path once extracted from a Velero backup archive, with version included. func GetVersionedItemFilePath(rootDir, groupResource, namespace, name, versionPath string) string { return filepath.Join(rootDir, velerov1api.ResourcesDir, groupResource, versionPath, GetScopeDir(namespace), namespace, name+".json") } // GetScopeDir returns NamespaceScopedDir if namespace is present, or ClusterScopedDir if empty func GetScopeDir(namespace string) string { if namespace == "" { return velerov1api.ClusterScopedDir } return velerov1api.NamespaceScopedDir } // Unmarshal reads the specified file, unmarshals the JSON contained within it // and returns an Unstructured object. func Unmarshal(fs filesystem.Interface, filePath string) (*unstructured.Unstructured, error) { var obj unstructured.Unstructured bytes, err := fs.ReadFile(filePath) if err != nil { return nil, err } err = json.Unmarshal(bytes, &obj) if err != nil { return nil, err } return &obj, nil } ================================================ FILE: pkg/archive/filesystem_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package archive import ( "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/test" ) func TestGetItemFilePath(t *testing.T) { res := GetItemFilePath("root", "resource", "", "item") assert.Equal(t, "root/resources/resource/cluster/item.json", res) res = GetItemFilePath("root", "resource", "namespace", "item") assert.Equal(t, "root/resources/resource/namespaces/namespace/item.json", res) res = GetItemFilePath("", "resource", "", "item") assert.Equal(t, "resources/resource/cluster/item.json", res) res = GetVersionedItemFilePath("root", "resource", "", "item", "") assert.Equal(t, "root/resources/resource/cluster/item.json", res) res = GetVersionedItemFilePath("root", "resource", "namespace", "item", "") assert.Equal(t, "root/resources/resource/namespaces/namespace/item.json", res) res = GetVersionedItemFilePath("root", "resource", "namespace", "item", "v1") assert.Equal(t, "root/resources/resource/v1/namespaces/namespace/item.json", res) res = GetVersionedItemFilePath("root", "resource", "", "item", "v1") assert.Equal(t, "root/resources/resource/v1/cluster/item.json", res) res = GetVersionedItemFilePath("", "resource", "", "item", "") assert.Equal(t, "resources/resource/cluster/item.json", res) } func TestGetScopeDir(t *testing.T) { res := GetScopeDir("") assert.Equal(t, velerov1api.ClusterScopedDir, res) res = GetScopeDir("test-namespace") assert.Equal(t, velerov1api.NamespaceScopedDir, res) } func TestUnmarshal(t *testing.T) { fs := test.NewFakeFileSystem() filePath := "pod.json" fileContent := `{ "apiVersion": "v1", "kind": "Pod", "metadata": { "name": "example-pod" }, "spec": { "containers": [{ "name": "example-container", "image": "example-image" }] } }` out, err := fs.Create(filePath) require.NoError(t, err) _, err = out.Write([]byte(fileContent)) require.NoError(t, err) _, err = Unmarshal(fs, filePath) require.NoError(t, err) } ================================================ FILE: pkg/archive/parser.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package archive import ( "os" "path/filepath" "strings" "github.com/pkg/errors" "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/util/filesystem" ) var ErrNotExist = errors.New("does not exist") // Parser traverses an extracted archive on disk to validate // it and provide a helpful representation of it to consumers. type Parser struct { log logrus.FieldLogger fs filesystem.Interface } // ResourceItems contains the collection of items of a given resource type // within a backup, grouped by namespace (or empty string for cluster-scoped // resources). type ResourceItems struct { // GroupResource is API group and resource name, // formatted as "resource.group". For the "core" // API group, the ".group" suffix is omitted. GroupResource string // ItemsByNamespace is a map from namespace (or empty string // for cluster-scoped resources) to a list of individual item // names contained in the archive. Item names **do not** include // the file extension. ItemsByNamespace map[string][]string } // NewParser constructs a Parser. func NewParser(log logrus.FieldLogger, fs filesystem.Interface) *Parser { return &Parser{ log: log, fs: fs, } } // Parse reads an extracted backup on the file system and returns // a structured catalog of the resources and items contained within it. func (p *Parser) Parse(dir string) (map[string]*ResourceItems, error) { // ensure top-level "resources" directory exists, and read subdirectories // of it, where each one is expected to correspond to a resource. resourcesDir := filepath.Join(dir, velerov1api.ResourcesDir) resourceDirs, err := p.checkAndReadDir(resourcesDir) if err != nil { return nil, err } // loop through each subdirectory (one per resource) and assemble // catalog of items within it. resources := map[string]*ResourceItems{} for _, resourceDir := range resourceDirs { if !resourceDir.IsDir() { p.log.Warnf("Ignoring unexpected file %q in directory %q", resourceDir.Name(), strings.TrimPrefix(resourcesDir, dir+"/")) continue } resourceItems := &ResourceItems{ GroupResource: resourceDir.Name(), ItemsByNamespace: map[string][]string{}, } // check for existence of a "cluster" subdirectory containing cluster-scoped // instances of this resource, and read its contents if it exists. clusterScopedDir := filepath.Join(resourcesDir, resourceDir.Name(), velerov1api.ClusterScopedDir) exists, err := p.fs.DirExists(clusterScopedDir) if err != nil { return nil, errors.Wrapf(err, "error checking for existence of directory %q", strings.TrimPrefix(clusterScopedDir, dir+"/")) } if exists { items, err := p.getResourceItemsForScope(clusterScopedDir, dir) if err != nil { return nil, err } if len(items) > 0 { resourceItems.ItemsByNamespace[""] = items } } // check for existence of a "namespaces" subdirectory containing further subdirectories, // one per namespace, and read its contents if it exists. namespaceScopedDir := filepath.Join(resourcesDir, resourceDir.Name(), velerov1api.NamespaceScopedDir) exists, err = p.fs.DirExists(namespaceScopedDir) if err != nil { return nil, errors.Wrapf(err, "error checking for existence of directory %q", strings.TrimPrefix(namespaceScopedDir, dir+"/")) } if exists { namespaceDirs, err := p.fs.ReadDir(namespaceScopedDir) if err != nil { return nil, errors.Wrapf(err, "error reading contents of directory %q", strings.TrimPrefix(namespaceScopedDir, dir+"/")) } for _, namespaceDir := range namespaceDirs { if !namespaceDir.IsDir() { p.log.Warnf("Ignoring unexpected file %q in directory %q", namespaceDir.Name(), strings.TrimPrefix(namespaceScopedDir, dir+"/")) continue } items, err := p.getResourceItemsForScope(filepath.Join(namespaceScopedDir, namespaceDir.Name()), dir) if err != nil { return nil, err } if len(items) > 0 { resourceItems.ItemsByNamespace[namespaceDir.Name()] = items } } } resources[resourceDir.Name()] = resourceItems } return resources, nil } // getResourceItemsForScope returns the list of items with a namespace or // cluster-scoped subdirectory for a specific resource. func (p *Parser) getResourceItemsForScope(dir, archiveRootDir string) ([]string, error) { files, err := p.fs.ReadDir(dir) if err != nil { return nil, errors.Wrapf(err, "error reading contents of directory %q", strings.TrimPrefix(dir, archiveRootDir+"/")) } var items []string for _, file := range files { if file.IsDir() { p.log.Warnf("Ignoring unexpected subdirectory %q in directory %q", file.Name(), strings.TrimPrefix(dir, archiveRootDir+"/")) continue } items = append(items, strings.TrimSuffix(file.Name(), ".json")) } return items, nil } // checkAndReadDir is a wrapper around fs.DirExists and fs.ReadDir that does checks // and returns errors if directory cannot be read. func (p *Parser) checkAndReadDir(dir string) ([]os.FileInfo, error) { exists, err := p.fs.DirExists(dir) if err != nil { return nil, errors.Wrapf(err, "error checking for existence of directory %q", filepath.ToSlash(dir)) } if !exists { return nil, errors.Wrapf(ErrNotExist, "directory %q", filepath.ToSlash(dir)) } contents, err := p.fs.ReadDir(dir) if err != nil { return nil, errors.Wrapf(err, "reading contents of %q", filepath.ToSlash(dir)) } return contents, nil } // ParseGroupVersions extracts the versions for each API Group from the backup // directory names and stores them in a metav1 APIGroup object. func (p *Parser) ParseGroupVersions(dir string) (map[string]metav1.APIGroup, error) { resourcesDir := filepath.Join(dir, velerov1api.ResourcesDir) // Get the subdirectories inside the "resources" directory. The subdirectories // will have resource.group names like "horizontalpodautoscalers.autoscaling". rgDirs, err := p.checkAndReadDir(resourcesDir) if err != nil { return nil, err } resourceAGs := make(map[string]metav1.APIGroup) // Loop through the resource.group directory names. for _, rgd := range rgDirs { group := metav1.APIGroup{ Name: extractGroupName(rgd.Name()), } rgdPath := filepath.Join(resourcesDir, rgd.Name()) // Inside each of the resource.group directories are directories whose // names are API Group versions like "v1" or "v1-preferredversion" gvDirs, err := p.checkAndReadDir(rgdPath) if err != nil { return nil, err } var supportedVersions []metav1.GroupVersionForDiscovery for _, gvd := range gvDirs { gvdName := gvd.Name() // Don't save the namespaces or clusters directories in list of // supported API Group Versions. if gvdName == "namespaces" || gvdName == "cluster" { continue } version := metav1.GroupVersionForDiscovery{ GroupVersion: strings.TrimPrefix(group.Name+"/"+gvdName, "/"), Version: gvdName, } if strings.Contains(gvdName, velerov1api.PreferredVersionDir) { gvdName = strings.TrimSuffix(gvdName, velerov1api.PreferredVersionDir) // Update version and group version to be without suffix. version.Version = gvdName version.GroupVersion = strings.TrimPrefix(group.Name+"/"+gvdName, "/") group.PreferredVersion = version } supportedVersions = append(supportedVersions, version) } group.Versions = supportedVersions resourceAGs[rgd.Name()] = group } return resourceAGs, nil } // extractGroupName will take a concatenated resource.group and extract the group, // if there is one. Resources like "pods" which has no group and will return an // empty string. func extractGroupName(resourceGroupDir string) string { parts := strings.SplitN(resourceGroupDir, ".", 2) var group string if len(parts) == 2 { group = parts[1] } return group } ================================================ FILE: pkg/archive/parser_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package archive import ( "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/vmware-tanzu/velero/pkg/test" ) func TestParse(t *testing.T) { tests := []struct { name string files []string dir string wantErrMsg error want map[string]*ResourceItems }{ { name: "when there is no top-level resources directory, an error is returned", dir: "root-dir", wantErrMsg: ErrNotExist, }, { name: "when there are no directories under the resources directory, an empty map is returned", dir: "root-dir", files: []string{"root-dir/resources/"}, want: map[string]*ResourceItems{}, }, { name: "a mix of cluster-scoped and namespaced items across multiple resources are correctly returned", dir: "root-dir", files: []string{ "root-dir/resources/widgets.foo/cluster/item-1.json", "root-dir/resources/widgets.foo/cluster/item-2.json", "root-dir/resources/widgets.foo/namespaces/ns-1/item-1.json", "root-dir/resources/widgets.foo/namespaces/ns-1/item-2.json", "root-dir/resources/widgets.foo/namespaces/ns-2/item-1.json", "root-dir/resources/widgets.foo/namespaces/ns-2/item-2.json", "root-dir/resources/dongles.foo/cluster/item-3.json", "root-dir/resources/dongles.foo/cluster/item-4.json", "root-dir/resources/dongles.bar/namespaces/ns-3/item-3.json", "root-dir/resources/dongles.bar/namespaces/ns-3/item-4.json", "root-dir/resources/dongles.bar/namespaces/ns-4/item-5.json", "root-dir/resources/dongles.bar/namespaces/ns-4/item-6.json", }, want: map[string]*ResourceItems{ "widgets.foo": { GroupResource: "widgets.foo", ItemsByNamespace: map[string][]string{ "": {"item-1", "item-2"}, "ns-1": {"item-1", "item-2"}, "ns-2": {"item-1", "item-2"}, }, }, "dongles.foo": { GroupResource: "dongles.foo", ItemsByNamespace: map[string][]string{ "": {"item-3", "item-4"}, }, }, "dongles.bar": { GroupResource: "dongles.bar", ItemsByNamespace: map[string][]string{ "ns-3": {"item-3", "item-4"}, "ns-4": {"item-5", "item-6"}, }, }, }, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { p := &Parser{ log: test.NewLogger(), fs: test.NewFakeFileSystem(), } for _, file := range tc.files { require.NoError(t, p.fs.MkdirAll(file, 0755)) if !strings.HasSuffix(file, "/") { res, err := p.fs.Create(file) require.NoError(t, err) require.NoError(t, res.Close()) } } res, err := p.Parse(tc.dir) if tc.wantErrMsg != nil { assert.ErrorIs(t, err, tc.wantErrMsg, "Error should be: %v, got: %v", tc.wantErrMsg, err) } else { require.NoError(t, err) assert.Equal(t, tc.want, res) } }) } } func TestParseGroupVersions(t *testing.T) { tests := []struct { name string files []string backupDir string wantErrMsg error want map[string]metav1.APIGroup }{ { name: "when there is no top-level resources directory, an error is returned", backupDir: "/var/folders", wantErrMsg: ErrNotExist, }, { name: "when there are no directories under the resources directory, an empty map is returned", backupDir: "/var/folders", files: []string{"/var/folders/resources/"}, want: map[string]metav1.APIGroup{}, }, { name: "when there is a mix of cluster-scoped and namespaced items for resources with preferred or multiple API groups, all group versions are correctly returned", backupDir: "/var/folders", files: []string{ "/var/folders/resources/clusterroles.rbac.authorization.k8s.io/v1-preferredversion/cluster/system/controller/attachdetach-controller.json", "/var/folders/resources/clusterroles.rbac.authorization.k8s.io/cluster/system/controller/attachdetach-controller.json", "/var/folders/resources/horizontalpodautoscalers.autoscaling/namespaces/myexample/php-apache-autoscaler.json", "/var/folders/resources/horizontalpodautoscalers.autoscaling/v1-preferredversion/namespaces/myexample/php-apache-autoscaler.json", "/var/folders/resources/horizontalpodautoscalers.autoscaling/v2beta1/namespaces/myexample/php-apache-autoscaler.json", "/var/folders/resources/horizontalpodautoscalers.autoscaling/v2beta2/namespaces/myexample/php-apache-autoscaler.json", "/var/folders/resources/pods/namespaces/nginx-example/nginx-deployment-57d5dcb68-wrqsc.json", "/var/folders/resources/pods/v1-preferredversion/namespaces/nginx-example/nginx-deployment-57d5dcb68-wrqsc.json", }, want: map[string]metav1.APIGroup{ "clusterroles.rbac.authorization.k8s.io": { Name: "rbac.authorization.k8s.io", Versions: []metav1.GroupVersionForDiscovery{ { GroupVersion: "rbac.authorization.k8s.io/v1", Version: "v1", }, }, PreferredVersion: metav1.GroupVersionForDiscovery{ GroupVersion: "rbac.authorization.k8s.io/v1", Version: "v1", }, }, "horizontalpodautoscalers.autoscaling": { Name: "autoscaling", Versions: []metav1.GroupVersionForDiscovery{ { GroupVersion: "autoscaling/v1", Version: "v1", }, { GroupVersion: "autoscaling/v2beta1", Version: "v2beta1", }, { GroupVersion: "autoscaling/v2beta2", Version: "v2beta2", }, }, PreferredVersion: metav1.GroupVersionForDiscovery{ GroupVersion: "autoscaling/v1", Version: "v1", }, }, "pods": { Name: "", Versions: []metav1.GroupVersionForDiscovery{ { GroupVersion: "v1", Version: "v1", }, }, PreferredVersion: metav1.GroupVersionForDiscovery{ GroupVersion: "v1", Version: "v1", }, }, }, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { p := &Parser{ log: test.NewLogger(), fs: test.NewFakeFileSystem(), } for _, file := range tc.files { require.NoError(t, p.fs.MkdirAll(file, 0755)) if !strings.HasSuffix(file, "/") { res, err := p.fs.Create(file) require.NoError(t, err) require.NoError(t, res.Close()) } } res, err := p.ParseGroupVersions(tc.backupDir) if tc.wantErrMsg != nil { assert.ErrorIs(t, err, tc.wantErrMsg, "Error should be: %v, got: %v", tc.wantErrMsg, err) } else { require.NoError(t, err) assert.Equal(t, tc.want, res) } }) } } func TestExtractGroupName(t *testing.T) { tests := []struct { name string rgDir string want string }{ { name: "Directory has no dots (only a group name)", rgDir: "pods", want: "", }, { name: "Directory has one concatenation dot (has both resource and group name which have 0 dots", rgDir: "cronjobs.batch", want: "batch", }, { name: "Directory has 3 dots in name (group has 2 dot)", rgDir: "leases.coordination.k8s.io", want: "coordination.k8s.io", }, { name: "Directory has 4 dots in name (group has 3 dots)", rgDir: "roles.rbac.authorization.k8s.io", want: "rbac.authorization.k8s.io", }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { grp := extractGroupName(tc.rgDir) assert.Equal(t, tc.want, grp) }) } } ================================================ FILE: pkg/backup/actions/backup_pv_action.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actions import ( "strings" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/plugin/velero" "github.com/vmware-tanzu/velero/pkg/util/actionhelpers" ) // PVCAction inspects a PersistentVolumeClaim for the PersistentVolume // that it references and backs it up type PVCAction struct { log logrus.FieldLogger } func NewPVCAction(logger logrus.FieldLogger) *PVCAction { return &PVCAction{log: logger} } func (a *PVCAction) AppliesTo() (velero.ResourceSelector, error) { return velero.ResourceSelector{ IncludedResources: []string{"persistentvolumeclaims"}, }, nil } // Execute finds the PersistentVolume bound by the provided // PersistentVolumeClaim, if any, and backs it up func (a *PVCAction) Execute(item runtime.Unstructured, backup *v1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, error) { a.log.Info("Executing PVCAction") pvc := new(corev1api.PersistentVolumeClaim) if err := runtime.DefaultUnstructuredConverter.FromUnstructured(item.UnstructuredContent(), &pvc); err != nil { return nil, nil, errors.Wrap(err, "unable to convert unstructured item to persistent volume claim") } if pvc.Status.Phase != corev1api.ClaimBound || pvc.Spec.VolumeName == "" { return item, nil, nil } // remove dataSource if exists from prior restored CSI volumes if pvc.Spec.DataSource != nil { pvc.Spec.DataSource = nil } if pvc.Spec.DataSourceRef != nil { pvc.Spec.DataSourceRef = nil } // When StorageClassName is set to "", it means no StorageClass is specified, // even the default StorageClass is not used. Only keep the Selector for this case. // https://kubernetes.io/docs/concepts/storage/persistent-volumes/#reserving-a-persistentvolume if pvc.Spec.StorageClassName == nil || *pvc.Spec.StorageClassName != "" { // Clean the selector to make the PVC to dynamically allocate PV. pvc.Spec.Selector = nil } // Clean stale Velero labels from PVC metadata and selector a.cleanupStaleVeleroLabels(pvc, backup) pvcMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&pvc) if err != nil { return nil, nil, errors.Wrap(err, "unable to convert pvc to unstructured item") } return &unstructured.Unstructured{Object: pvcMap}, actionhelpers.RelatedItemsForPVC(pvc, a.log), nil } // cleanupStaleVeleroLabels removes stale Velero labels from both the PVC metadata // and the selector's match labels to ensure clean backups func (a *PVCAction) cleanupStaleVeleroLabels(pvc *corev1api.PersistentVolumeClaim, backup *v1.Backup) { // Clean stale Velero labels from selector match labels if pvc.Spec.Selector != nil && pvc.Spec.Selector.MatchLabels != nil { for k := range pvc.Spec.Selector.MatchLabels { if strings.HasPrefix(k, "velero.io/") { a.log.Infof("Deleting stale Velero label %s from PVC %s selector", k, pvc.Name) delete(pvc.Spec.Selector.MatchLabels, k) } } } // Clean stale Velero labels from main metadata if pvc.Labels != nil { for k, v := range pvc.Labels { // Only remove labels that are clearly stale from previous operations shouldRemove := false // Always remove restore-name labels as these are from previous restores if k == v1.RestoreNameLabel { shouldRemove = true } if k == v1.MustIncludeAdditionalItemAnnotation { shouldRemove = true } // Remove backup-name labels that don't match current backup if k == v1.BackupNameLabel && v != backup.Name { shouldRemove = true } // Remove volume-snapshot-name labels from previous CSI backups // Note: If this backup creates new CSI snapshots, the CSI action will add them back if k == v1.VolumeSnapshotLabel { shouldRemove = true } if shouldRemove { a.log.Infof("Deleting stale Velero label %s=%s from PVC %s", k, v, pvc.Name) delete(pvc.Labels, k) } } } } ================================================ FILE: pkg/backup/actions/backup_pv_action_test.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actions import ( "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/plugin/velero" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func TestBackupPVAction(t *testing.T) { pvc := &unstructured.Unstructured{ Object: map[string]any{ "spec": map[string]any{}, "status": map[string]any{}, }, } backup := &v1.Backup{} a := NewPVCAction(velerotest.NewLogger()) // no spec.volumeName should result in no error // and no additional items _, additional, err := a.Execute(pvc, backup) require.NoError(t, err) assert.Empty(t, additional) // empty spec.volumeName should result in no error // and no additional items pvc.Object["spec"].(map[string]any)["volumeName"] = "" _, additional, err = a.Execute(pvc, backup) require.NoError(t, err) assert.Empty(t, additional) // Action should clean the spec.Selector when the StorageClassName is not set. input := builder.ForPersistentVolumeClaim("abc", "abc").VolumeName("pv").Selector(&metav1.LabelSelector{MatchLabels: map[string]string{"abc": "abc"}}).Phase(corev1api.ClaimBound).Result() inputUnstructured, err := runtime.DefaultUnstructuredConverter.ToUnstructured(input) require.NoError(t, err) item, additional, err := a.Execute(&unstructured.Unstructured{Object: inputUnstructured}, backup) require.NoError(t, err) require.Len(t, additional, 1) modifiedPVC := new(corev1api.PersistentVolumeClaim) require.NoError(t, runtime.DefaultUnstructuredConverter.FromUnstructured(item.UnstructuredContent(), modifiedPVC)) require.Nil(t, modifiedPVC.Spec.Selector) // Action should clean the spec.Selector when the StorageClassName is set to specific StorageClass input2 := builder.ForPersistentVolumeClaim("abc", "abc").VolumeName("pv").StorageClass("sc1").Selector(&metav1.LabelSelector{MatchLabels: map[string]string{"abc": "abc"}}).Phase(corev1api.ClaimBound).Result() inputUnstructured2, err2 := runtime.DefaultUnstructuredConverter.ToUnstructured(input2) require.NoError(t, err2) item2, additional2, err2 := a.Execute(&unstructured.Unstructured{Object: inputUnstructured2}, backup) require.NoError(t, err2) require.Len(t, additional2, 1) modifiedPVC2 := new(corev1api.PersistentVolumeClaim) require.NoError(t, runtime.DefaultUnstructuredConverter.FromUnstructured(item2.UnstructuredContent(), modifiedPVC2)) require.Nil(t, modifiedPVC2.Spec.Selector) // Action should keep the spec.Selector when the StorageClassName is set to "" input3 := builder.ForPersistentVolumeClaim("abc", "abc").StorageClass("").Selector(&metav1.LabelSelector{MatchLabels: map[string]string{"abc": "abc"}}).VolumeName("pv").Phase(corev1api.ClaimBound).Result() inputUnstructured3, err3 := runtime.DefaultUnstructuredConverter.ToUnstructured(input3) require.NoError(t, err3) item3, additional3, err3 := a.Execute(&unstructured.Unstructured{Object: inputUnstructured3}, backup) require.NoError(t, err3) require.Len(t, additional3, 1) modifiedPVC3 := new(corev1api.PersistentVolumeClaim) require.NoError(t, runtime.DefaultUnstructuredConverter.FromUnstructured(item3.UnstructuredContent(), modifiedPVC3)) require.Equal(t, input3.Spec.Selector, modifiedPVC3.Spec.Selector) // Action should delete label started with"velero.io/" from the spec.Selector when the StorageClassName is set to "" input4 := builder.ForPersistentVolumeClaim("abc", "abc").StorageClass("").Selector(&metav1.LabelSelector{MatchLabels: map[string]string{"velero.io/abc": "abc", "abc": "abc"}}).VolumeName("pv").Phase(corev1api.ClaimBound).Result() inputUnstructured4, err4 := runtime.DefaultUnstructuredConverter.ToUnstructured(input4) require.NoError(t, err4) item4, additional4, err4 := a.Execute(&unstructured.Unstructured{Object: inputUnstructured4}, backup) require.NoError(t, err4) require.Len(t, additional4, 1) modifiedPVC4 := new(corev1api.PersistentVolumeClaim) require.NoError(t, runtime.DefaultUnstructuredConverter.FromUnstructured(item4.UnstructuredContent(), modifiedPVC4)) require.Equal(t, &metav1.LabelSelector{MatchLabels: map[string]string{"abc": "abc"}}, modifiedPVC4.Spec.Selector) // Action should clean the spec.Selector when the StorageClassName has value input5 := builder.ForPersistentVolumeClaim("abc", "abc").StorageClass("sc1").Selector(&metav1.LabelSelector{MatchLabels: map[string]string{"velero.io/abc": "abc", "abc": "abc"}}).VolumeName("pv").Phase(corev1api.ClaimBound).Result() inputUnstructured5, err5 := runtime.DefaultUnstructuredConverter.ToUnstructured(input5) require.NoError(t, err5) item5, additional5, err5 := a.Execute(&unstructured.Unstructured{Object: inputUnstructured5}, backup) require.NoError(t, err5) require.Len(t, additional5, 1) modifiedPVC5 := new(corev1api.PersistentVolumeClaim) require.NoError(t, runtime.DefaultUnstructuredConverter.FromUnstructured(item5.UnstructuredContent(), modifiedPVC5)) require.Nil(t, modifiedPVC5.Spec.Selector) // non-empty spec.volumeName when status.phase is empty // should result in no error and no additional items pvc.Object["spec"].(map[string]any)["volumeName"] = "myVolume" _, additional, err = a.Execute(pvc, backup) require.NoError(t, err) require.Empty(t, additional) // non-empty spec.volumeName when status.phase is 'Pending' // should result in no error and no additional items pvc.Object["status"].(map[string]any)["phase"] = corev1api.ClaimPending _, additional, err = a.Execute(pvc, backup) require.NoError(t, err) require.Empty(t, additional) // non-empty spec.volumeName when status.phase is 'Lost' // should result in no error and no additional items pvc.Object["status"].(map[string]any)["phase"] = corev1api.ClaimLost _, additional, err = a.Execute(pvc, backup) require.NoError(t, err) require.Empty(t, additional) // non-empty spec.volumeName when status.phase is 'Bound' // should result in no error and one additional item for the PV pvc.Object["status"].(map[string]any)["phase"] = corev1api.ClaimBound _, additional, err = a.Execute(pvc, backup) require.NoError(t, err) require.Len(t, additional, 1) assert.Equal(t, velero.ResourceIdentifier{GroupResource: kuberesource.PersistentVolumes, Name: "myVolume"}, additional[0]) // empty spec.volumeName when status.phase is 'Bound' should // result in no error and no additional items pvc.Object["spec"].(map[string]any)["volumeName"] = "" _, additional, err = a.Execute(pvc, backup) require.NoError(t, err) assert.Empty(t, additional) } func TestCleanupStaleVeleroLabels(t *testing.T) { tests := []struct { name string inputPVC *corev1api.PersistentVolumeClaim backup *v1.Backup expectedLabels map[string]string expectedSelector *metav1.LabelSelector }{ { name: "removes restore-name labels", inputPVC: &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: "test-pvc", Labels: map[string]string{ "velero.io/restore-name": "old-restore", "app": "myapp", }, }, }, backup: &v1.Backup{ObjectMeta: metav1.ObjectMeta{Name: "current-backup"}}, expectedLabels: map[string]string{ "app": "myapp", }, }, { name: "removes backup-name labels that don't match current backup", inputPVC: &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: "test-pvc", Labels: map[string]string{ "velero.io/backup-name": "old-backup", "app": "myapp", }, }, }, backup: &v1.Backup{ObjectMeta: metav1.ObjectMeta{Name: "current-backup"}}, expectedLabels: map[string]string{ "app": "myapp", }, }, { name: "keeps backup-name labels that match current backup", inputPVC: &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: "test-pvc", Labels: map[string]string{ "velero.io/backup-name": "current-backup", "app": "myapp", }, }, }, backup: &v1.Backup{ObjectMeta: metav1.ObjectMeta{Name: "current-backup"}}, expectedLabels: map[string]string{ "velero.io/backup-name": "current-backup", "app": "myapp", }, }, { name: "removes volume-snapshot-name labels", inputPVC: &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: "test-pvc", Labels: map[string]string{ "velero.io/volume-snapshot-name": "old-snapshot", "app": "myapp", }, }, }, backup: &v1.Backup{ObjectMeta: metav1.ObjectMeta{Name: "current-backup"}}, expectedLabels: map[string]string{ "app": "myapp", }, }, { name: "removes velero labels from selector match labels", inputPVC: &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: "test-pvc", }, Spec: corev1api.PersistentVolumeClaimSpec{ Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "velero.io/restore-name": "old-restore", "velero.io/backup-name": "old-backup", "app": "myapp", }, }, }, }, backup: &v1.Backup{ObjectMeta: metav1.ObjectMeta{Name: "current-backup"}}, expectedLabels: nil, expectedSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "app": "myapp", }, }, }, { name: "handles PVC with no labels", inputPVC: &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: "test-pvc", }, }, backup: &v1.Backup{ObjectMeta: metav1.ObjectMeta{Name: "current-backup"}}, expectedLabels: nil, }, { name: "handles PVC with no selector", inputPVC: &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: "test-pvc", Labels: map[string]string{ "app": "myapp", }, }, }, backup: &v1.Backup{ObjectMeta: metav1.ObjectMeta{Name: "current-backup"}}, expectedLabels: map[string]string{ "app": "myapp", }, expectedSelector: nil, }, { name: "removes multiple stale velero labels", inputPVC: &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: "test-pvc", Labels: map[string]string{ "velero.io/restore-name": "old-restore", "velero.io/backup-name": "old-backup", "velero.io/volume-snapshot-name": "old-snapshot", "app": "myapp", "env": "prod", }, }, Spec: corev1api.PersistentVolumeClaimSpec{ Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "velero.io/restore-name": "old-restore", "app": "myapp", }, }, }, }, backup: &v1.Backup{ObjectMeta: metav1.ObjectMeta{Name: "current-backup"}}, expectedLabels: map[string]string{ "app": "myapp", "env": "prod", }, expectedSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "app": "myapp", }, }, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { action := NewPVCAction(velerotest.NewLogger()) // Create a copy of the input PVC to avoid modifying the test case pvcCopy := tc.inputPVC.DeepCopy() action.cleanupStaleVeleroLabels(pvcCopy, tc.backup) assert.Equal(t, tc.expectedLabels, pvcCopy.Labels, "Labels should match expected values") assert.Equal(t, tc.expectedSelector, pvcCopy.Spec.Selector, "Selector should match expected values") }) } } ================================================ FILE: pkg/backup/actions/csi/pvc_action.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package csi import ( "context" "fmt" "strconv" "time" "k8s.io/client-go/util/retry" volumegroupsnapshotv1beta1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1beta1" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" storagev1api "k8s.io/api/storage/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" crclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "k8s.io/apimachinery/pkg/api/resource" internalvolumehelper "github.com/vmware-tanzu/velero/internal/volumehelper" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" veleroclient "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/label" plugincommon "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" "github.com/vmware-tanzu/velero/pkg/plugin/utils/volumehelper" "github.com/vmware-tanzu/velero/pkg/plugin/velero" biav2 "github.com/vmware-tanzu/velero/pkg/plugin/velero/backupitemaction/v2" uploaderUtil "github.com/vmware-tanzu/velero/pkg/uploader/util" "github.com/vmware-tanzu/velero/pkg/util/boolptr" "github.com/vmware-tanzu/velero/pkg/util/csi" kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube" podvolumeutil "github.com/vmware-tanzu/velero/pkg/util/podvolume" ) // TODO: Replace hardcoded VolumeSnapshot finalizer strings with constants from // "github.com/kubernetes-csi/external-snapshotter/v8/pkg/utils" // once module/toolchain upgrades are done. // Finalizer constants const ( VolumeSnapshotFinalizerGroupProtection = "snapshot.storage.kubernetes.io/volumesnapshot-in-group-protection" VolumeSnapshotFinalizerSourceProtection = "snapshot.storage.kubernetes.io/volumesnapshot-as-source-protection" ) // pvcBackupItemAction is a backup item action plugin for Velero. type pvcBackupItemAction struct { log logrus.FieldLogger crClient crclient.Client // pvcPodCache provides lazy per-namespace caching of PVC-to-Pod mappings. // Since plugin instances are unique per backup (created via newPluginManager and // cleaned up via CleanupClients at backup completion), we can safely cache this // without mutex or backup UID tracking. // This avoids the O(N*M) performance issue when there are many PVCs and pods. // See issue #9179 and PR #9226 for details. pvcPodCache *podvolumeutil.PVCPodCache } // AppliesTo returns information indicating that the PVCBackupItemAction // should be invoked to backup PVCs. func (p *pvcBackupItemAction) AppliesTo() (velero.ResourceSelector, error) { return velero.ResourceSelector{ IncludedResources: []string{"persistentvolumeclaims"}, }, nil } func (p *pvcBackupItemAction) validateBackup(backup velerov1api.Backup) (valid bool) { if backup.Status.Phase == velerov1api.BackupPhaseFinalizing || backup.Status.Phase == velerov1api.BackupPhaseFinalizingPartiallyFailed { p.log.WithFields( logrus.Fields{ "Backup": fmt.Sprintf("%s/%s", backup.Namespace, backup.Name), "Phase": backup.Status.Phase, }, ).Debug("Backup is in finalizing phase. Skip this PVC.") return false } return true } // ensurePVCPodCacheForNamespace ensures the PVC-to-Pod cache is built for the given namespace. // This uses lazy per-namespace caching following the pattern from PR #9226. // Since plugin instances are unique per backup, we can safely cache without mutex or backup UID tracking. func (p *pvcBackupItemAction) ensurePVCPodCacheForNamespace(ctx context.Context, namespace string) error { // Initialize cache if needed if p.pvcPodCache == nil { p.pvcPodCache = podvolumeutil.NewPVCPodCache() } // Build cache for namespace if not already done if !p.pvcPodCache.IsNamespaceBuilt(namespace) { p.log.Debugf("Building PVC-to-Pod cache for namespace %s", namespace) if err := p.pvcPodCache.BuildCacheForNamespace(ctx, namespace, p.crClient); err != nil { return errors.Wrapf(err, "failed to build PVC-to-Pod cache for namespace %s", namespace) } } return nil } // getVolumeHelperWithCache creates a VolumeHelper using the pre-built PVC-to-Pod cache. // The cache should be ensured for the relevant namespace(s) before calling this. func (p *pvcBackupItemAction) getVolumeHelperWithCache(backup *velerov1api.Backup) (internalvolumehelper.VolumeHelper, error) { // Create VolumeHelper with our lazy-built cache vh, err := internalvolumehelper.NewVolumeHelperImplWithCache( *backup, p.crClient, p.log, p.pvcPodCache, ) if err != nil { return nil, errors.Wrap(err, "failed to create VolumeHelper") } return vh, nil } // getOrCreateVolumeHelper returns a VolumeHelper with lazy per-namespace caching. // The VolumeHelper uses the pvcPodCache which is populated lazily as namespaces are encountered. // Callers should use ensurePVCPodCacheForNamespace before calling methods that need // PVC-to-Pod lookups for a specific namespace. // Since plugin instances are unique per backup (created via newPluginManager and // cleaned up via CleanupClients at backup completion), we can safely cache this. // See issue #9179 and PR #9226 for details. func (p *pvcBackupItemAction) getOrCreateVolumeHelper(backup *velerov1api.Backup) (internalvolumehelper.VolumeHelper, error) { // Initialize the PVC-to-Pod cache if needed if p.pvcPodCache == nil { p.pvcPodCache = podvolumeutil.NewPVCPodCache() } // Return the VolumeHelper with our lazily-built cache // The cache will be populated incrementally as namespaces are encountered return p.getVolumeHelperWithCache(backup) } func (p *pvcBackupItemAction) validatePVCandPV( pvc corev1api.PersistentVolumeClaim, item runtime.Unstructured, ) ( valid bool, updateItem runtime.Unstructured, err error, ) { updateItem = item // no storage class: we don't know how to map to a VolumeSnapshotClass if pvc.Spec.StorageClassName == nil { return false, updateItem, errors.Errorf( "Cannot snapshot PVC %s/%s, PVC has no storage class.", pvc.Namespace, pvc.Name) } p.log.Debugf( "Fetching underlying PV for PVC %s", fmt.Sprintf("%s/%s", pvc.Namespace, pvc.Name), ) // Do nothing if this is not a CSI provisioned volume pv, err := kubeutil.GetPVForPVC(&pvc, p.crClient) if err != nil { return false, updateItem, errors.WithStack(err) } if pv.Spec.PersistentVolumeSource.CSI == nil { p.log.Infof( "Skipping PVC %s/%s, associated PV %s is not a CSI volume", pvc.Namespace, pvc.Name, pv.Name) kubeutil.AddAnnotations( &pvc.ObjectMeta, map[string]string{ velerov1api.SkippedNoCSIPVAnnotation: "true", }) data, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&pvc) updateItem = &unstructured.Unstructured{Object: data} return false, updateItem, err } return true, updateItem, nil } func (p *pvcBackupItemAction) createVolumeSnapshot( pvc corev1api.PersistentVolumeClaim, backup *velerov1api.Backup, ) ( vs *snapshotv1api.VolumeSnapshot, err error, ) { p.log.Debugf("Fetching storage class for PV %s", *pvc.Spec.StorageClassName) storageClass := new(storagev1api.StorageClass) if err := p.crClient.Get( context.TODO(), crclient.ObjectKey{Name: *pvc.Spec.StorageClassName}, storageClass, ); err != nil { return nil, errors.Wrap(err, "error getting storage class") } p.log.Debugf("Fetching VolumeSnapshotClass for %s", storageClass.Provisioner) vsClass, err := csi.GetVolumeSnapshotClass( storageClass.Provisioner, backup, &pvc, p.log, p.crClient, ) if err != nil { return nil, errors.Wrapf( err, "failed to get VolumeSnapshotClass for StorageClass %s", storageClass.Name, ) } p.log.Infof("VolumeSnapshotClass=%s", vsClass.Name) vsLabels := map[string]string{} for k, v := range pvc.ObjectMeta.Labels { vsLabels[k] = v } vsLabels[velerov1api.BackupNameLabel] = label.GetValidName(backup.Name) // Craft the vs object to be created vs = &snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ GenerateName: "velero-" + pvc.Name + "-", Namespace: pvc.Namespace, Labels: vsLabels, }, Spec: snapshotv1api.VolumeSnapshotSpec{ Source: snapshotv1api.VolumeSnapshotSource{ PersistentVolumeClaimName: &pvc.Name, }, VolumeSnapshotClassName: &vsClass.Name, }, } if err := p.crClient.Create(context.TODO(), vs); err != nil { return nil, errors.Wrapf( err, "error creating volume snapshot", ) } p.log.Infof( "Created VolumeSnapshot %s", fmt.Sprintf("%s/%s", vs.Namespace, vs.Name), ) return vs, nil } // Execute recognizes PVCs backed by volumes provisioned by CSI drivers // with VolumeSnapshotting capability and creates snapshots of the // underlying PVs by creating VolumeSnapshot CSI API objects that will // trigger the CSI driver to perform the snapshot operation on the volume. func (p *pvcBackupItemAction) Execute( item runtime.Unstructured, backup *velerov1api.Backup, ) ( runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error, ) { p.log.Info("Starting PVCBackupItemAction") if valid := p.validateBackup(*backup); !valid { return item, nil, "", nil, nil } var pvc corev1api.PersistentVolumeClaim if err := runtime.DefaultUnstructuredConverter.FromUnstructured( item.UnstructuredContent(), &pvc, ); err != nil { return nil, nil, "", nil, errors.WithStack(err) } if valid, item, err := p.validatePVCandPV( pvc, item, ); !valid { if err != nil { return nil, nil, "", nil, err } return item, nil, "", nil, nil } // Ensure PVC-to-Pod cache is built for this namespace (lazy per-namespace caching) if err := p.ensurePVCPodCacheForNamespace(context.TODO(), pvc.Namespace); err != nil { return nil, nil, "", nil, err } // Get or create the cached VolumeHelper for this backup vh, err := p.getOrCreateVolumeHelper(backup) if err != nil { return nil, nil, "", nil, err } shouldSnapshot, err := volumehelper.ShouldPerformSnapshotWithVolumeHelper( item, kuberesource.PersistentVolumeClaims, *backup, p.crClient, p.log, vh, ) if err != nil { return nil, nil, "", nil, err } if !shouldSnapshot { p.log.Debugf("CSI plugin skip snapshot for PVC %s according to the VolumeHelper setting.", pvc.Namespace+"/"+pvc.Name) return nil, nil, "", nil, err } vs, err := p.getVolumeSnapshotReference(context.TODO(), pvc, backup) if err != nil { return nil, nil, "", nil, err } // Wait until VS associated VSC snapshot handle created before // continue.we later require the vsc restore size vsc, err := csi.WaitUntilVSCHandleIsReady( vs, p.crClient, p.log, backup.Spec.CSISnapshotTimeout.Duration, ) if err != nil { p.log.Errorf("Failed to wait for VolumeSnapshot %s/%s to become ReadyToUse within timeout %v: %s", vs.Namespace, vs.Name, backup.Spec.CSISnapshotTimeout.Duration, err.Error()) csi.CleanupVolumeSnapshot(vs, p.crClient, p.log) return nil, nil, "", nil, errors.WithStack(err) } labels := map[string]string{ velerov1api.VolumeSnapshotLabel: vs.Name, velerov1api.BackupNameLabel: backup.Name, } annotations := map[string]string{ velerov1api.VolumeSnapshotLabel: vs.Name, velerov1api.MustIncludeAdditionalItemAnnotation: "true", } var additionalItems []velero.ResourceIdentifier operationID := "" var itemToUpdate []velero.ResourceIdentifier if boolptr.IsSetToTrue(backup.Spec.SnapshotMoveData) { operationID = label.GetValidName( string( velerov1api.AsyncOperationIDPrefixDataUpload, ) + string(backup.UID) + "." + string(pvc.UID), ) dataUploadLog := p.log.WithFields(logrus.Fields{ "Source PVC": fmt.Sprintf("%s/%s", pvc.Namespace, pvc.Name), "VolumeSnapshot": fmt.Sprintf("%s/%s", vs.Namespace, vs.Name), "Operation ID": operationID, "Backup": backup.Name, }) dataUploadLog.Info("Starting data upload of backup") dataUpload, err := createDataUpload( context.Background(), backup, p.crClient, vs, &pvc, operationID, vsc, ) if err != nil { dataUploadLog.WithError(err).Error("failed to submit DataUpload") // TODO: need to use DeleteVolumeSnapshotIfAny, after data mover // adopting the controller-runtime client. if deleteErr := p.crClient.Delete(context.TODO(), vs); deleteErr != nil { if !apierrors.IsNotFound(deleteErr) { dataUploadLog.WithError(deleteErr).Error("fail to delete VolumeSnapshot") } } // Return without modification to not fail the backup, // and the above error log makes the backup partially fail. return item, nil, "", nil, nil } else { itemToUpdate = []velero.ResourceIdentifier{ { GroupResource: schema.GroupResource{ Group: "velero.io", Resource: "datauploads", }, Namespace: dataUpload.Namespace, Name: dataUpload.Name, }, } // Set the DataUploadNameLabel, which is used for restore to // let CSI plugin check whether it should handle the volume. // If volume is CSI migration, PVC doesn't have the annotation. annotations[velerov1api.DataUploadNameAnnotation] = dataUpload.Namespace + "/" + dataUpload.Name dataUploadLog.Info("DataUpload is submitted successfully.") } } else { setPVCRequestSizeToVSRestoreSize(&pvc, vsc, p.log) additionalItems = []velero.ResourceIdentifier{ { GroupResource: kuberesource.VolumeSnapshots, Namespace: vs.Namespace, Name: vs.Name, }, } } kubeutil.AddAnnotations(&pvc.ObjectMeta, annotations) kubeutil.AddLabels(&pvc.ObjectMeta, labels) p.log.Infof("Returning from PVCBackupItemAction with %d additionalItems to backup", len(additionalItems)) for _, ai := range additionalItems { p.log.Debugf("%s: %s", ai.GroupResource.String(), ai.Name) } pvcMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&pvc) if err != nil { return nil, nil, "", nil, errors.WithStack(err) } return &unstructured.Unstructured{Object: pvcMap}, additionalItems, operationID, itemToUpdate, nil } func (p *pvcBackupItemAction) Name() string { return "PVCBackupItemAction" } func (p *pvcBackupItemAction) Progress( operationID string, backup *velerov1api.Backup, ) (velero.OperationProgress, error) { progress := velero.OperationProgress{} if operationID == "" { return progress, biav2.InvalidOperationIDError(operationID) } dataUpload, err := getDataUpload(context.Background(), p.crClient, operationID) if err != nil { p.log.Errorf( "fail to get DataUpload for backup %s/%s by operation ID %s: %s", backup.Namespace, backup.Name, operationID, err.Error(), ) return progress, err } if dataUpload.Status.Phase == velerov2alpha1.DataUploadPhaseNew || dataUpload.Status.Phase == "" { p.log.Debugf("DataUpload is still not processed yet. Skip progress update.") return progress, nil } progress.Description = string(dataUpload.Status.Phase) progress.OperationUnits = "Bytes" progress.NCompleted = dataUpload.Status.Progress.BytesDone progress.NTotal = dataUpload.Status.Progress.TotalBytes if dataUpload.Status.StartTimestamp != nil { progress.Started = dataUpload.Status.StartTimestamp.Time } if dataUpload.Status.CompletionTimestamp != nil { progress.Updated = dataUpload.Status.CompletionTimestamp.Time } if dataUpload.Status.Phase == velerov2alpha1.DataUploadPhaseCompleted { progress.Completed = true } else if dataUpload.Status.Phase == velerov2alpha1.DataUploadPhaseFailed { progress.Completed = true progress.Err = dataUpload.Status.Message } else if dataUpload.Status.Phase == velerov2alpha1.DataUploadPhaseCanceled { progress.Completed = true progress.Err = "DataUpload is canceled" } return progress, nil } func (p *pvcBackupItemAction) Cancel(operationID string, backup *velerov1api.Backup) error { if operationID == "" { return biav2.InvalidOperationIDError(operationID) } dataUpload, err := getDataUpload(context.Background(), p.crClient, operationID) if err != nil { p.log.Errorf( "fail to get DataUpload for backup %s/%s: %s", backup.Namespace, backup.Name, err.Error(), ) return err } return cancelDataUpload(context.Background(), p.crClient, dataUpload) } func newDataUpload( backup *velerov1api.Backup, vs *snapshotv1api.VolumeSnapshot, pvc *corev1api.PersistentVolumeClaim, operationID string, vsc *snapshotv1api.VolumeSnapshotContent, ) *velerov2alpha1.DataUpload { dataUpload := &velerov2alpha1.DataUpload{ TypeMeta: metav1.TypeMeta{ APIVersion: velerov2alpha1.SchemeGroupVersion.String(), Kind: "DataUpload", }, ObjectMeta: metav1.ObjectMeta{ Namespace: backup.Namespace, GenerateName: backup.Name + "-", OwnerReferences: []metav1.OwnerReference{ { APIVersion: velerov1api.SchemeGroupVersion.String(), Kind: "Backup", Name: backup.Name, UID: backup.UID, Controller: boolptr.True(), }, }, Labels: map[string]string{ velerov1api.BackupNameLabel: label.GetValidName(backup.Name), velerov1api.BackupUIDLabel: string(backup.UID), velerov1api.PVCUIDLabel: string(pvc.UID), velerov1api.AsyncOperationIDLabel: operationID, }, }, Spec: velerov2alpha1.DataUploadSpec{ SnapshotType: velerov2alpha1.SnapshotTypeCSI, CSISnapshot: &velerov2alpha1.CSISnapshotSpec{ VolumeSnapshot: vs.Name, StorageClass: *pvc.Spec.StorageClassName, Driver: vsc.Spec.Driver, }, SourcePVC: pvc.Name, DataMover: backup.Spec.DataMover, BackupStorageLocation: backup.Spec.StorageLocation, SourceNamespace: pvc.Namespace, OperationTimeout: backup.Spec.CSISnapshotTimeout, }, } if vs.Spec.VolumeSnapshotClassName != nil { dataUpload.Spec.CSISnapshot.SnapshotClass = *vs.Spec.VolumeSnapshotClassName } if backup.Spec.UploaderConfig != nil && backup.Spec.UploaderConfig.ParallelFilesUpload > 0 { dataUpload.Spec.DataMoverConfig = make(map[string]string) dataUpload.Spec.DataMoverConfig[uploaderUtil.ParallelFilesUpload] = strconv.Itoa(backup.Spec.UploaderConfig.ParallelFilesUpload) } return dataUpload } func createDataUpload( ctx context.Context, backup *velerov1api.Backup, crClient crclient.Client, vs *snapshotv1api.VolumeSnapshot, pvc *corev1api.PersistentVolumeClaim, operationID string, vsc *snapshotv1api.VolumeSnapshotContent, ) (*velerov2alpha1.DataUpload, error) { dataUpload := newDataUpload(backup, vs, pvc, operationID, vsc) err := crClient.Create(ctx, dataUpload) if err != nil { return nil, errors.Wrap(err, "fail to create DataUpload CR") } return dataUpload, err } func getDataUpload( ctx context.Context, crClient crclient.Client, operationID string, ) (*velerov2alpha1.DataUpload, error) { dataUploadList := new(velerov2alpha1.DataUploadList) err := crClient.List(ctx, dataUploadList, &crclient.ListOptions{ LabelSelector: labels.SelectorFromSet( map[string]string{velerov1api.AsyncOperationIDLabel: operationID}, ), }) if err != nil { return nil, errors.Wrapf(err, "error to list DataUpload") } if len(dataUploadList.Items) == 0 { return nil, errors.Errorf("not found DataUpload for operationID %s", operationID) } if len(dataUploadList.Items) > 1 { return nil, errors.Errorf("more than one DataUpload found operationID %s", operationID) } return &dataUploadList.Items[0], nil } func cancelDataUpload( ctx context.Context, crClient crclient.Client, dataUpload *velerov2alpha1.DataUpload, ) error { updatedDataUpload := dataUpload.DeepCopy() updatedDataUpload.Spec.Cancel = true err := crClient.Patch(ctx, updatedDataUpload, crclient.MergeFrom(dataUpload)) if err != nil { return errors.Wrap(err, "error patch DataUpload") } return nil } func NewPvcBackupItemAction(f veleroclient.Factory) plugincommon.HandlerInitializer { return func(logger logrus.FieldLogger) (any, error) { crClient, err := f.KubebuilderClient() if err != nil { return nil, errors.WithStack(err) } return &pvcBackupItemAction{ log: logger, crClient: crClient, }, nil } } func (p *pvcBackupItemAction) getVolumeSnapshotReference( ctx context.Context, pvc corev1api.PersistentVolumeClaim, backup *velerov1api.Backup, ) (*snapshotv1api.VolumeSnapshot, error) { vgsLabelKey := backup.Spec.VolumeGroupSnapshotLabelKey group, hasLabel := pvc.Labels[vgsLabelKey] if vgsLabelKey != "" && hasLabel && group != "" { p.log.Infof("PVC %s/%s is part of VolumeGroupSnapshot group %q via label %q", pvc.Namespace, pvc.Name, group, vgsLabelKey) // Try to find an existing VS created via a previous VGS in the current backup existingVS, err := p.findExistingVSForBackup(ctx, backup.UID, backup.Name, pvc.Name, pvc.Namespace) if err != nil { return nil, errors.Wrapf(err, "failed to find existing VolumeSnapshot for PVC %s/%s", pvc.Namespace, pvc.Name) } if existingVS != nil { if existingVS.Status != nil && existingVS.Status.VolumeGroupSnapshotName != nil { p.log.Infof("Reusing existing VolumeSnapshot %s for PVC %s", existingVS.Name, pvc.Name) return existingVS, nil } else { return nil, errors.Errorf("found VolumeSnapshot %s for PVC %s, but it was not created via VolumeGroupSnapshot (missing volumeGroupSnapshotName)", existingVS.Name, pvc.Name) } } p.log.Infof("No existing VS found for PVC %s; creating new VGS", pvc.Name) // List all PVCs in the VGS group groupedPVCs, err := p.listGroupedPVCs(ctx, pvc.Namespace, vgsLabelKey, group) if err != nil { return nil, errors.Wrapf(err, "failed to list PVCs in VolumeGroupSnapshot group %q in namespace %q", group, pvc.Namespace) } // Ensure PVC-to-Pod cache is built for this namespace (lazy per-namespace caching) if err := p.ensurePVCPodCacheForNamespace(ctx, pvc.Namespace); err != nil { return nil, errors.Wrapf(err, "failed to build PVC-to-Pod cache for namespace %s", pvc.Namespace) } // Get the cached VolumeHelper for filtering PVCs by volume policy vh, err := p.getOrCreateVolumeHelper(backup) if err != nil { return nil, errors.Wrapf(err, "failed to get VolumeHelper for filtering PVCs in group %q", group) } // Filter PVCs by volume policy filteredPVCs, err := p.filterPVCsByVolumePolicy(groupedPVCs, backup, vh) if err != nil { return nil, errors.Wrapf(err, "failed to filter PVCs by volume policy for VolumeGroupSnapshot group %q", group) } // Warn if any PVCs were filtered out if len(filteredPVCs) < len(groupedPVCs) { for _, originalPVC := range groupedPVCs { found := false for _, filteredPVC := range filteredPVCs { if originalPVC.Name == filteredPVC.Name { found = true break } } if !found { p.log.Warnf("PVC %s/%s has VolumeGroupSnapshot label %s=%s but is excluded by volume policy", originalPVC.Namespace, originalPVC.Name, vgsLabelKey, group) } } } // Determine the CSI driver for the grouped PVCs driver, err := p.determineCSIDriver(filteredPVCs) if err != nil { return nil, errors.Wrapf(err, "failed to determine CSI driver for PVCs in VolumeGroupSnapshot group %q", group) } if driver == "" { return nil, errors.New("no csi driver found, failing the backup") } // Determine the VGSClass to be used for the VGS object to be created vgsClass, err := p.determineVGSClass(ctx, driver, backup, &pvc) if err != nil { return nil, errors.Wrapf(err, "failed to determine VolumeGroupSnapshotClass for CSI driver %q", driver) } // Create the VGS object newVGS, err := p.createVolumeGroupSnapshot(ctx, backup, pvc, vgsLabelKey, group, vgsClass) if err != nil { return nil, errors.Wrapf(err, "failed to create VolumeGroupSnapshot for PVC %s/%s", pvc.Namespace, pvc.Name) } // Wait for all the VS objects associated with the VGS to have status and VGS Name (VS readiness is checked in legacy flow) and get the PVC-to-VS map vsMap, err := p.waitForVGSAssociatedVS(ctx, filteredPVCs, newVGS, backup.Spec.CSISnapshotTimeout.Duration) if err != nil { return nil, errors.Wrapf(err, "timeout waiting for VolumeSnapshots to have status created via VolumeGroupSnapshot %s", newVGS.Name) } // Update the VS objects: remove VGS owner references and finalizers; add backup metadata labels. err = p.updateVGSCreatedVS(ctx, vsMap, newVGS, backup) if err != nil { return nil, errors.Wrapf(err, "failed to update VolumeSnapshots created by VolumeGroupSnapshot %s", newVGS.Name) } // Wait for VGSC binding in the VGS status err = p.waitForVGSCBinding(ctx, newVGS, backup.Spec.CSISnapshotTimeout.Duration) if err != nil { return nil, errors.Wrapf(err, "timeout waiting for VolumeGroupSnapshotContent binding for VolumeGroupSnapshot %s", newVGS.Name) } // Re-fetch latest VGS to ensure status is populated after VGSC binding latestVGS := &volumegroupsnapshotv1beta1.VolumeGroupSnapshot{} if err := p.crClient.Get(ctx, crclient.ObjectKeyFromObject(newVGS), latestVGS); err != nil { return nil, errors.Wrapf(err, "failed to re-fetch VolumeGroupSnapshot %s after VGSC binding wait", newVGS.Name) } // Patch the VGSC deletionPolicy to Retain. err = p.patchVGSCDeletionPolicy(ctx, latestVGS) if err != nil { return nil, errors.Wrapf(err, "failed to patch VolumeGroupSnapshotContent Deletion Policy for VolumeGroupSnapshot %s", newVGS.Name) } // Delete the VGS and VGSC err = p.deleteVGSAndVGSC(ctx, latestVGS) if err != nil { return nil, errors.Wrapf(err, "failed to get VolumeSnapshot for PVC %s/%s created by VolumeGroupSnapshot %s", pvc.Namespace, pvc.Name, newVGS.Name) } // Use the VS that was created for this PVC via VGS. vs, found := vsMap[pvc.Name] if !found { return nil, errors.Wrapf(err, "failed to get VolumeSnapshot for PVC %s/%s created by VolumeGroupSnapshot %s", pvc.Namespace, pvc.Name, newVGS.Name) } return vs, nil } // Legacy fallback: create individual VS return p.createVolumeSnapshot(pvc, backup) } func (p *pvcBackupItemAction) findExistingVSForBackup( ctx context.Context, backupUID types.UID, backupName, pvcName, namespace string, ) (*snapshotv1api.VolumeSnapshot, error) { vsList := &snapshotv1api.VolumeSnapshotList{} labelSelector := labels.SelectorFromSet(map[string]string{ velerov1api.BackupNameLabel: label.GetValidName(backupName), velerov1api.BackupUIDLabel: string(backupUID), }) if err := p.crClient.List(ctx, vsList, crclient.InNamespace(namespace), crclient.MatchingLabelsSelector{Selector: labelSelector}, ); err != nil { return nil, errors.Wrap(err, "failed to list VolumeSnapshots with backup labels") } for _, vs := range vsList.Items { if vs.Spec.Source.PersistentVolumeClaimName != nil && *vs.Spec.Source.PersistentVolumeClaimName == pvcName { return &vs, nil } } return nil, nil } func (p *pvcBackupItemAction) listGroupedPVCs(ctx context.Context, namespace, labelKey, groupValue string) ([]corev1api.PersistentVolumeClaim, error) { pvcList := new(corev1api.PersistentVolumeClaimList) if err := p.crClient.List( ctx, pvcList, crclient.InNamespace(namespace), crclient.MatchingLabels{labelKey: groupValue}, ); err != nil { return nil, errors.Wrap(err, "failed to list grouped PVCs") } return pvcList.Items, nil } func (p *pvcBackupItemAction) filterPVCsByVolumePolicy( pvcs []corev1api.PersistentVolumeClaim, backup *velerov1api.Backup, vh internalvolumehelper.VolumeHelper, ) ([]corev1api.PersistentVolumeClaim, error) { var filteredPVCs []corev1api.PersistentVolumeClaim for _, pvc := range pvcs { // Convert PVC to unstructured for ShouldPerformSnapshotWithVolumeHelper pvcMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&pvc) if err != nil { return nil, errors.Wrapf(err, "failed to convert PVC %s/%s to unstructured", pvc.Namespace, pvc.Name) } unstructuredPVC := &unstructured.Unstructured{Object: pvcMap} // Check if this PVC should be snapshotted according to volume policies // Uses the cached VolumeHelper for better performance with many PVCs/pods shouldSnapshot, err := volumehelper.ShouldPerformSnapshotWithVolumeHelper( unstructuredPVC, kuberesource.PersistentVolumeClaims, *backup, p.crClient, p.log, vh, ) if err != nil { return nil, errors.Wrapf(err, "failed to check volume policy for PVC %s/%s", pvc.Namespace, pvc.Name) } if shouldSnapshot { filteredPVCs = append(filteredPVCs, pvc) } } return filteredPVCs, nil } func (p *pvcBackupItemAction) determineCSIDriver( pvcs []corev1api.PersistentVolumeClaim, ) (string, error) { var driver string for _, pvc := range pvcs { pv, err := kubeutil.GetPVForPVC(&pvc, p.crClient) if err != nil { return "", err } if pv.Spec.CSI == nil { return "", errors.Errorf("PV %s for PVC %s is not CSI provisioned", pv.Name, pvc.Name) } current := pv.Spec.CSI.Driver if driver == "" { driver = current } else if driver != current { return "", errors.Errorf("found multiple CSI drivers: %s and %s", driver, current) } } return driver, nil } func (p *pvcBackupItemAction) determineVGSClass( ctx context.Context, driver string, backup *velerov1api.Backup, pvc *corev1api.PersistentVolumeClaim, ) (string, error) { // 1. PVC-level override if pvc != nil { if val, ok := pvc.Annotations[velerov1api.VolumeGroupSnapshotClassAnnotationPVC]; ok && val != "" { return val, nil } } // 2. Backup-level override key := fmt.Sprintf(velerov1api.VolumeGroupSnapshotClassAnnotationBackupPrefix+"%s", driver) if val, ok := backup.Annotations[key]; ok && val != "" { return val, nil } // 3. Fallback to label-based default vgsClassList := &volumegroupsnapshotv1beta1.VolumeGroupSnapshotClassList{} if err := p.crClient.List(ctx, vgsClassList); err != nil { return "", errors.Wrap(err, "failed to list VolumeGroupSnapshotClasses") } var matched []string for _, class := range vgsClassList.Items { if class.Driver != driver { continue } if val, ok := class.Labels[velerov1api.VolumeGroupSnapshotClassDefaultLabel]; ok && val == "true" { matched = append(matched, class.Name) } } if len(matched) == 1 { return matched[0], nil } else if len(matched) == 0 { return "", errors.Errorf("no VolumeGroupSnapshotClass found for driver %q for PVC %s", driver, pvc.Name) } else { return "", errors.Errorf("multiple VolumeGroupSnapshotClasses found for driver %q with label velero.io/csi-volumegroupsnapshot-class=true", driver) } } func (p *pvcBackupItemAction) createVolumeGroupSnapshot( ctx context.Context, backup *velerov1api.Backup, pvc corev1api.PersistentVolumeClaim, vgsLabelKey, vgsLabelValue, vgsClassName string, ) (*volumegroupsnapshotv1beta1.VolumeGroupSnapshot, error) { vgsLabels := map[string]string{ velerov1api.BackupNameLabel: label.GetValidName(backup.Name), velerov1api.BackupUIDLabel: string(backup.UID), vgsLabelKey: vgsLabelValue, } vgs := &volumegroupsnapshotv1beta1.VolumeGroupSnapshot{ ObjectMeta: metav1.ObjectMeta{ GenerateName: fmt.Sprintf("velero-%s-", vgsLabelValue), Namespace: pvc.Namespace, Labels: vgsLabels, }, Spec: volumegroupsnapshotv1beta1.VolumeGroupSnapshotSpec{ VolumeGroupSnapshotClassName: &vgsClassName, Source: volumegroupsnapshotv1beta1.VolumeGroupSnapshotSource{ Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{ vgsLabelKey: vgsLabelValue, }, }, }, }, } if err := p.crClient.Create(ctx, vgs); err != nil { return nil, errors.Wrap(err, "failed to create VolumeGroupSnapshot") } refetchedVGS, err := p.getVGSByLabels(ctx, pvc.Namespace, vgsLabels) if err != nil { return nil, errors.Wrap(err, "failed to re-fetch VGS after creation") } p.log.Infof("Re-fetched Created VolumeGroupSnapshot %s/%s for PVC group label %s=%s", refetchedVGS.Namespace, refetchedVGS.Name, vgsLabelKey, vgsLabelValue) return refetchedVGS, nil } func (p *pvcBackupItemAction) waitForVGSAssociatedVS( ctx context.Context, groupedPVCs []corev1api.PersistentVolumeClaim, vgs *volumegroupsnapshotv1beta1.VolumeGroupSnapshot, timeout time.Duration, ) (map[string]*snapshotv1api.VolumeSnapshot, error) { expected := len(groupedPVCs) vsMap := make(map[string]*snapshotv1api.VolumeSnapshot) err := wait.PollUntilContextTimeout(ctx, time.Second, timeout, true, func(ctx context.Context) (done bool, err error) { vsList := &snapshotv1api.VolumeSnapshotList{} if err := p.crClient.List(ctx, vsList, crclient.InNamespace(vgs.Namespace)); err != nil { return false, err } vsMap = make(map[string]*snapshotv1api.VolumeSnapshot) for _, vs := range vsList.Items { if !hasOwnerReference(&vs, vgs) { continue } if vs.Status != nil && vs.Status.VolumeGroupSnapshotName != nil && *vs.Status.VolumeGroupSnapshotName == vgs.Name { if vs.Spec.Source.PersistentVolumeClaimName != nil { vsMap[*vs.Spec.Source.PersistentVolumeClaimName] = vs.DeepCopy() } } } if expected == 0 { return false, nil } if len(vsMap) == expected { return true, nil } return false, nil }) if err != nil { return nil, errors.Wrapf(err, "timeout waiting for VolumeSnapshots associated with VGS %s", vgs.Name) } return vsMap, nil } func hasOwnerReference(obj metav1.Object, vgs *volumegroupsnapshotv1beta1.VolumeGroupSnapshot) bool { for _, ref := range obj.GetOwnerReferences() { if ref.Kind == kuberesource.VGSKind && ref.APIVersion == volumegroupsnapshotv1beta1.GroupName+"/"+volumegroupsnapshotv1beta1.SchemeGroupVersion.Version && ref.UID == vgs.UID { return true } } return false } func (p *pvcBackupItemAction) updateVGSCreatedVS( ctx context.Context, vsMap map[string]*snapshotv1api.VolumeSnapshot, vgs *volumegroupsnapshotv1beta1.VolumeGroupSnapshot, backup *velerov1api.Backup, ) error { for pvcName, vs := range vsMap { if vs == nil || vs.Status == nil || vs.Status.VolumeGroupSnapshotName == nil || *vs.Status.VolumeGroupSnapshotName != vgs.Name { continue } err := retry.RetryOnConflict(retry.DefaultRetry, func() error { // Re-fetch the latest VS to avoid conflict latestVS := &snapshotv1api.VolumeSnapshot{} if err := p.crClient.Get(ctx, crclient.ObjectKeyFromObject(vs), latestVS); err != nil { return errors.Wrapf(err, "failed to get latest VolumeSnapshot %s (PVC %s)", vs.Name, pvcName) } // Remove VGS owner ref if err := controllerutil.RemoveOwnerReference(vgs, latestVS, p.crClient.Scheme()); err != nil { return errors.Wrapf(err, "failed to remove VGS owner reference from VS %s", vs.Name) } // Remove known finalizers controllerutil.RemoveFinalizer(latestVS, VolumeSnapshotFinalizerGroupProtection) controllerutil.RemoveFinalizer(latestVS, VolumeSnapshotFinalizerSourceProtection) // Add Velero labels if latestVS.Labels == nil { latestVS.Labels = make(map[string]string) } latestVS.Labels[velerov1api.BackupNameLabel] = backup.Name latestVS.Labels[velerov1api.BackupUIDLabel] = string(backup.UID) // Attempt to update return p.crClient.Update(ctx, latestVS) }) if err != nil { return errors.Wrapf(err, "failed to update VS %s (PVC %s) after retrying on conflict", vs.Name, pvcName) } } return nil } func (p *pvcBackupItemAction) patchVGSCDeletionPolicy(ctx context.Context, vgs *volumegroupsnapshotv1beta1.VolumeGroupSnapshot) error { if vgs == nil || vgs.Status == nil || vgs.Status.BoundVolumeGroupSnapshotContentName == nil { return errors.New("VolumeGroupSnapshotContent name not found in VGS status") } vgscName := vgs.Status.BoundVolumeGroupSnapshotContentName return retry.RetryOnConflict(retry.DefaultBackoff, func() error { vgsc := &volumegroupsnapshotv1beta1.VolumeGroupSnapshotContent{} if err := p.crClient.Get(ctx, crclient.ObjectKey{Name: *vgscName}, vgsc); err != nil { return errors.Wrapf(err, "failed to get VolumeGroupSnapshotContent %s for VolumeGroupSnapshot %s/%s", *vgscName, vgs.Namespace, vgs.Name) } if vgsc.Spec.DeletionPolicy == snapshotv1api.VolumeSnapshotContentDelete { p.log.Infof("Patching VGSC %s to Retain deletionPolicy", *vgscName) vgsc.Spec.DeletionPolicy = snapshotv1api.VolumeSnapshotContentRetain if err := p.crClient.Update(ctx, vgsc); err != nil { return errors.Wrapf(err, "failed to update VGSC %s deletionPolicy", *vgscName) } } else { p.log.Infof("VGSC %s already set to deletionPolicy=%s", *vgscName, vgsc.Spec.DeletionPolicy) } return nil }) } func (p *pvcBackupItemAction) deleteVGSAndVGSC(ctx context.Context, vgs *volumegroupsnapshotv1beta1.VolumeGroupSnapshot) error { if vgs.Status != nil && vgs.Status.BoundVolumeGroupSnapshotContentName != nil { vgsc := &volumegroupsnapshotv1beta1.VolumeGroupSnapshotContent{ ObjectMeta: metav1.ObjectMeta{ Name: *vgs.Status.BoundVolumeGroupSnapshotContentName, }, } p.log.Infof("Deleting VolumeGroupSnapshotContent %s", vgsc.Name) if err := p.crClient.Delete(ctx, vgsc); err != nil && !apierrors.IsNotFound(err) { p.log.Warnf("Failed to delete VolumeGroupSnapshotContent %s: %v", vgsc.Name, err) return errors.Wrapf(err, "failed to delete VolumeGroupSnapshotContent %s", vgsc.Name) } } else { p.log.Infof("No BoundVolumeGroupSnapshotContentName set in VolumeGroupSnapshot %s/%s", vgs.Namespace, vgs.Name) } p.log.Infof("Deleting VolumeGroupSnapshot %s/%s", vgs.Namespace, vgs.Name) if err := p.crClient.Delete(ctx, vgs); err != nil && !apierrors.IsNotFound(err) { p.log.Warnf("Failed to delete VolumeGroupSnapshot %s/%s: %v", vgs.Namespace, vgs.Name, err) return errors.Wrapf(err, "failed to delete VolumeGroupSnapshot %s/%s", vgs.Namespace, vgs.Name) } return nil } func (p *pvcBackupItemAction) waitForVGSCBinding( ctx context.Context, vgs *volumegroupsnapshotv1beta1.VolumeGroupSnapshot, timeout time.Duration, ) error { return wait.PollUntilContextTimeout(ctx, time.Second, timeout, true, func(ctx context.Context) (bool, error) { vgsRef := &volumegroupsnapshotv1beta1.VolumeGroupSnapshot{} if err := p.crClient.Get(ctx, crclient.ObjectKeyFromObject(vgs), vgsRef); err != nil { return false, err } if vgsRef.Status != nil && vgsRef.Status.BoundVolumeGroupSnapshotContentName != nil { return true, nil } return false, nil }) } func (p *pvcBackupItemAction) getVGSByLabels(ctx context.Context, namespace string, labels map[string]string) (*volumegroupsnapshotv1beta1.VolumeGroupSnapshot, error) { vgsList := &volumegroupsnapshotv1beta1.VolumeGroupSnapshotList{} if err := p.crClient.List(ctx, vgsList, crclient.InNamespace(namespace), crclient.MatchingLabels(labels), ); err != nil { return nil, errors.Wrap(err, "failed to list VolumeGroupSnapshots by labels") } if len(vgsList.Items) == 0 { return nil, errors.New("no VolumeGroupSnapshot found matching labels") } if len(vgsList.Items) > 1 { return nil, errors.New("multiple VolumeGroupSnapshots found matching labels") } return &vgsList.Items[0], nil } func setPVCRequestSizeToVSRestoreSize( pvc *corev1api.PersistentVolumeClaim, vsc *snapshotv1api.VolumeSnapshotContent, logger logrus.FieldLogger, ) { if vsc.Status.RestoreSize != nil { logger.Debugf("Patching PVC request size to fit the volumesnapshot restore size %d", vsc.Status.RestoreSize) restoreSize := *resource.NewQuantity(*vsc.Status.RestoreSize, resource.BinarySI) // It is possible that the volume provider allocated a larger // capacity volume than what was requested in the backed up PVC. // In this scenario the volumesnapshot of the PVC will end being // larger than its requested storage size. Such a PVC, on restore // as-is, will be stuck attempting to use a VolumeSnapshot as a // data source for a PVC that is not large enough. // To counter that, here we set the storage request on the PVC // to the larger of the PVC's storage request and the size of the // VolumeSnapshot setPVCStorageResourceRequest(pvc, restoreSize, logger) } } func setPVCStorageResourceRequest( pvc *corev1api.PersistentVolumeClaim, restoreSize resource.Quantity, log logrus.FieldLogger, ) { { if pvc.Spec.Resources.Requests == nil { pvc.Spec.Resources.Requests = corev1api.ResourceList{} } storageReq, exists := pvc.Spec.Resources.Requests[corev1api.ResourceStorage] if !exists || storageReq.Cmp(restoreSize) < 0 { pvc.Spec.Resources.Requests[corev1api.ResourceStorage] = restoreSize rs := pvc.Spec.Resources.Requests[corev1api.ResourceStorage] log.Infof("Resetting storage requests for PVC %s/%s to %s", pvc.Namespace, pvc.Name, rs.String()) } } } ================================================ FILE: pkg/backup/actions/csi/pvc_action_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package csi import ( "context" "fmt" "strings" "testing" "time" "github.com/vmware-tanzu/velero/pkg/kuberesource" volumegroupsnapshotv1beta1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1beta1" "github.com/stretchr/testify/assert" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/pointer" "github.com/vmware-tanzu/velero/pkg/label" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" storagev1api "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/wait" crclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/pkg/apis/velero/shared" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" "github.com/vmware-tanzu/velero/pkg/builder" factorymocks "github.com/vmware-tanzu/velero/pkg/client/mocks" "github.com/vmware-tanzu/velero/pkg/plugin/velero" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) const testDriver = "csi.example.com" // errorInjectingClient is a wrapper around a normal client that injects an error // when a specific resource type (VolumeSnapshot) is created. type errorInjectingClient struct { crclient.Client } // Create overrides the embedded client's Create method. func (c *errorInjectingClient) Create(ctx context.Context, obj crclient.Object, opts ...crclient.CreateOption) error { // Check if the object being created is a VolumeSnapshot. if _, ok := obj.(*snapshotv1api.VolumeSnapshot); ok { // If it is, return our injected error instead of proceeding. return errors.New("injected error on create") } // For all other object types, call the original, embedded Create method. return c.Client.Create(ctx, obj, opts...) } func TestExecute(t *testing.T) { boolTrue := true tests := []struct { name string backup *velerov1api.Backup pvc *corev1api.PersistentVolumeClaim pv *corev1api.PersistentVolume sc *storagev1api.StorageClass vsClass *snapshotv1api.VolumeSnapshotClass operationID string expectedErr error expectErr bool // Use bool for cases where we just need to check for any error expectedBackup *velerov1api.Backup expectedDataUpload *velerov2alpha1.DataUpload expectedPVC *corev1api.PersistentVolumeClaim resourcePolicy *corev1api.ConfigMap failVSCreate bool skipVSReadyUpdate bool // New flag to control VS readiness }{ { name: "Skip PVC BIA when backup is in finalizing phase", backup: builder.ForBackup("velero", "test").Phase(velerov1api.BackupPhaseFinalizing).Result(), }, { name: "Fail when creating volumesnapshot returns error", backup: builder.ForBackup("velero", "test").CSISnapshotTimeout(1 * time.Minute).Result(), pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").VolumeName("testPV").StorageClass("testSC").Phase(corev1api.ClaimBound).Result(), pv: builder.ForPersistentVolume("testPV").CSI("hostpath", "testVolume").Result(), sc: builder.ForStorageClass("testSC").Provisioner("hostpath").Result(), vsClass: builder.ForVolumeSnapshotClass("testVSClass").Driver("hostpath").ObjectMeta(builder.WithLabels(velerov1api.VolumeSnapshotClassSelectorLabel, "")).Result(), failVSCreate: true, expectedErr: errors.New("error creating volume snapshot: injected error on create"), }, { name: "Fail when waiting for VolumeSnapshot to be ready times out", backup: builder.ForBackup("velero", "test").CSISnapshotTimeout(20 * time.Millisecond).Result(), // Short timeout pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").VolumeName("testPV").StorageClass("testSC").Phase(corev1api.ClaimBound).Result(), pv: builder.ForPersistentVolume("testPV").CSI("hostpath", "testVolume").Result(), sc: builder.ForStorageClass("testSC").Provisioner("hostpath").Result(), vsClass: builder.ForVolumeSnapshotClass("testVSClass").Driver("hostpath").ObjectMeta(builder.WithLabels(velerov1api.VolumeSnapshotClassSelectorLabel, "")).Result(), skipVSReadyUpdate: true, // This will cause the timeout expectErr: true, // Expect an error, but the exact message can vary }, { name: "Test SnapshotMoveData", backup: builder.ForBackup("velero", "test").SnapshotMoveData(true).CSISnapshotTimeout(1 * time.Minute).Result(), pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").VolumeName("testPV").StorageClass("testSC").Phase(corev1api.ClaimBound).Result(), pv: builder.ForPersistentVolume("testPV").CSI("hostpath", "testVolume").Result(), sc: builder.ForStorageClass("testSC").Provisioner("hostpath").Result(), vsClass: builder.ForVolumeSnapshotClass("testVSClass").Driver("hostpath").ObjectMeta(builder.WithLabels(velerov1api.VolumeSnapshotClassSelectorLabel, "")).Result(), operationID: ".", expectedDataUpload: &velerov2alpha1.DataUpload{ TypeMeta: metav1.TypeMeta{ Kind: "DataUpload", APIVersion: velerov2alpha1.SchemeGroupVersion.String(), }, ObjectMeta: metav1.ObjectMeta{ GenerateName: "test-", Namespace: "velero", Labels: map[string]string{ velerov1api.BackupNameLabel: "test", velerov1api.BackupUIDLabel: "", velerov1api.PVCUIDLabel: "", velerov1api.AsyncOperationIDLabel: "du-.", }, OwnerReferences: []metav1.OwnerReference{ { APIVersion: "velero.io/v1", Kind: "Backup", Name: "test", UID: "", Controller: &boolTrue, }, }, }, Spec: velerov2alpha1.DataUploadSpec{ SnapshotType: velerov2alpha1.SnapshotTypeCSI, CSISnapshot: &velerov2alpha1.CSISnapshotSpec{ VolumeSnapshot: "", StorageClass: "testSC", SnapshotClass: "testVSClass", }, SourcePVC: "testPVC", SourceNamespace: "velero", OperationTimeout: metav1.Duration{Duration: 1 * time.Minute}, }, }, }, { name: "Verify PVC is modified as expected", backup: builder.ForBackup("velero", "test").SnapshotMoveData(true).CSISnapshotTimeout(1 * time.Minute).Result(), pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").VolumeName("testPV").StorageClass("testSC").Phase(corev1api.ClaimBound).Result(), pv: builder.ForPersistentVolume("testPV").CSI("hostpath", "testVolume").Result(), sc: builder.ForStorageClass("testSC").Provisioner("hostpath").Result(), vsClass: builder.ForVolumeSnapshotClass("tescVSClass").Driver("hostpath").ObjectMeta(builder.WithLabels(velerov1api.VolumeSnapshotClassSelectorLabel, "")).Result(), operationID: ".", expectedPVC: builder.ForPersistentVolumeClaim("velero", "testPVC"). ObjectMeta(builder.WithAnnotations(velerov1api.MustIncludeAdditionalItemAnnotation, "true", velerov1api.DataUploadNameAnnotation, "velero/"), builder.WithLabels(velerov1api.BackupNameLabel, "test")). VolumeName("testPV").StorageClass("testSC").Phase(corev1api.ClaimBound).Result(), }, { name: "Test ResourcePolicy", backup: builder.ForBackup("velero", "test").ResourcePolicies("resourcePolicy").SnapshotVolumes(false).CSISnapshotTimeout(time.Duration(3600) * time.Second).Result(), resourcePolicy: builder.ForConfigMap("velero", "resourcePolicy").Data("policy", "{\"version\":\"v1\", \"volumePolicies\":[{\"conditions\":{\"csi\": {}},\"action\":{\"type\":\"snapshot\"}}]}").Result(), pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").VolumeName("testPV").StorageClass("testSC").Phase(corev1api.ClaimBound).Result(), pv: builder.ForPersistentVolume("testPV").CSI("hostpath", "testVolume").Result(), sc: builder.ForStorageClass("testSC").Provisioner("hostpath").Result(), vsClass: builder.ForVolumeSnapshotClass("tescVSClass").Driver("hostpath").ObjectMeta(builder.WithLabels(velerov1api.VolumeSnapshotClassSelectorLabel, "")).Result(), }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { logger := logrus.New() logger.Level = logrus.DebugLevel objects := make([]runtime.Object, 0) if tc.pvc != nil { objects = append(objects, tc.pvc) } if tc.pv != nil { objects = append(objects, tc.pv) } if tc.sc != nil { objects = append(objects, tc.sc) } if tc.vsClass != nil { objects = append(objects, tc.vsClass) } if tc.resourcePolicy != nil { objects = append(objects, tc.resourcePolicy) } var crClient crclient.Client if tc.failVSCreate { realFakeClient := velerotest.NewFakeControllerRuntimeClient(t, objects...) crClient = &errorInjectingClient{Client: realFakeClient} } else { crClient = velerotest.NewFakeControllerRuntimeClient(t, objects...) } pvcBIA := pvcBackupItemAction{ log: logger, crClient: crClient, } pvcMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&tc.pvc) require.NoError(t, err) if tc.pvc != nil && !tc.failVSCreate && !tc.skipVSReadyUpdate { go func() { var vsList snapshotv1api.VolumeSnapshotList err := wait.PollUntilContextTimeout(t.Context(), 1*time.Second, 10*time.Second, true, func(ctx context.Context) (bool, error) { err = pvcBIA.crClient.List(ctx, &vsList, &crclient.ListOptions{Namespace: tc.pvc.Namespace}) require.NoError(t, err) if err != nil || len(vsList.Items) == 0 { return false, err } return true, nil }) require.NoError(t, err) vscName := "testVSC" readyToUse := true vsList.Items[0].Status = &snapshotv1api.VolumeSnapshotStatus{ BoundVolumeSnapshotContentName: &vscName, ReadyToUse: &readyToUse, } err = pvcBIA.crClient.Update(t.Context(), &vsList.Items[0]) require.NoError(t, err) handleName := "testHandle" vsc := builder.ForVolumeSnapshotContent("testVSC").Status(&snapshotv1api.VolumeSnapshotContentStatus{SnapshotHandle: &handleName}).Result() err = pvcBIA.crClient.Create(t.Context(), vsc) require.NoError(t, err) }() } resultUnstructed, _, _, _, err := pvcBIA.Execute(&unstructured.Unstructured{Object: pvcMap}, tc.backup) if tc.expectedErr != nil { require.EqualError(t, err, tc.expectedErr.Error()) } else if tc.expectErr { require.Error(t, err) // On timeout failure, check that the cleanup logic was called if tc.skipVSReadyUpdate { vsList := new(snapshotv1api.VolumeSnapshotList) errList := crClient.List(t.Context(), vsList, &crclient.ListOptions{Namespace: tc.pvc.Namespace}) require.NoError(t, errList) require.Empty(t, vsList.Items, "VolumeSnapshot should have been cleaned up after readiness check failed") } } else { require.NoError(t, err) } if tc.expectedDataUpload != nil { dataUploadList := new(velerov2alpha1.DataUploadList) err := crClient.List(t.Context(), dataUploadList, &crclient.ListOptions{LabelSelector: labels.SelectorFromSet(map[string]string{velerov1api.BackupNameLabel: tc.backup.Name})}) require.NoError(t, err) require.Len(t, dataUploadList.Items, 1) require.True(t, cmp.Equal(tc.expectedDataUpload, &dataUploadList.Items[0], cmpopts.IgnoreFields(velerov2alpha1.DataUpload{}, "ResourceVersion", "Name", "Spec.CSISnapshot.VolumeSnapshot"))) } if tc.expectedPVC != nil { resultPVC := new(corev1api.PersistentVolumeClaim) runtime.DefaultUnstructuredConverter.FromUnstructured(resultUnstructed.UnstructuredContent(), resultPVC) require.True(t, cmp.Equal(tc.expectedPVC, resultPVC, cmpopts.IgnoreFields(corev1api.PersistentVolumeClaim{}, "ResourceVersion", "Annotations", "Labels"))) } }) } } func TestProgress(t *testing.T) { currentTime := time.Now() tests := []struct { name string backup *velerov1api.Backup dataUpload *velerov2alpha1.DataUpload operationID string expectedErr string expectedProgress velero.OperationProgress }{ { name: "DataUpload cannot be found", backup: builder.ForBackup("velero", "test").Result(), operationID: "testing", expectedErr: "not found DataUpload for operationID testing", }, { name: "DataUpload is found", backup: builder.ForBackup("velero", "test").Result(), dataUpload: &velerov2alpha1.DataUpload{ TypeMeta: metav1.TypeMeta{ Kind: "DataUpload", APIVersion: "v2alpha1", }, ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", Name: "testing", Labels: map[string]string{ velerov1api.AsyncOperationIDLabel: "testing", }, }, Status: velerov2alpha1.DataUploadStatus{ Phase: velerov2alpha1.DataUploadPhaseFailed, Progress: shared.DataMoveOperationProgress{ BytesDone: 1000, TotalBytes: 1000, }, StartTimestamp: &metav1.Time{Time: currentTime}, CompletionTimestamp: &metav1.Time{Time: currentTime}, Message: "Testing error", }, }, operationID: "testing", expectedProgress: velero.OperationProgress{ Completed: true, Err: "Testing error", NCompleted: 1000, NTotal: 1000, OperationUnits: "Bytes", Description: "Failed", Started: currentTime, Updated: currentTime, }, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { crClient := velerotest.NewFakeControllerRuntimeClient(t) logger := logrus.New() pvcBIA := pvcBackupItemAction{ log: logger, crClient: crClient, } if tc.dataUpload != nil { err := crClient.Create(t.Context(), tc.dataUpload) require.NoError(t, err) } progress, err := pvcBIA.Progress(tc.operationID, tc.backup) if tc.expectedErr != "" { require.Equal(t, tc.expectedErr, err.Error()) } require.True(t, cmp.Equal(tc.expectedProgress, progress, cmpopts.IgnoreFields(velero.OperationProgress{}, "Started", "Updated"))) }) } } func TestCancel(t *testing.T) { tests := []struct { name string backup *velerov1api.Backup dataUpload velerov2alpha1.DataUpload operationID string expectedErr error expectedDataUpload velerov2alpha1.DataUpload }{ { name: "Cancel DataUpload", backup: builder.ForBackup("velero", "test").Result(), dataUpload: velerov2alpha1.DataUpload{ TypeMeta: metav1.TypeMeta{ Kind: "DataUpload", APIVersion: velerov2alpha1.SchemeGroupVersion.String(), }, ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", Name: "testing", Labels: map[string]string{ velerov1api.AsyncOperationIDLabel: "testing", }, }, }, operationID: "testing", expectedDataUpload: velerov2alpha1.DataUpload{ TypeMeta: metav1.TypeMeta{ Kind: "DataUpload", APIVersion: velerov2alpha1.SchemeGroupVersion.String(), }, ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", Name: "testing", Labels: map[string]string{ velerov1api.AsyncOperationIDLabel: "testing", }, }, Spec: velerov2alpha1.DataUploadSpec{ Cancel: true, }, }, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { crClient := velerotest.NewFakeControllerRuntimeClient(t) logger := logrus.New() pvcBIA := pvcBackupItemAction{ log: logger, crClient: crClient, } err := crClient.Create(t.Context(), &tc.dataUpload) require.NoError(t, err) err = pvcBIA.Cancel(tc.operationID, tc.backup) require.NoError(t, err) du := new(velerov2alpha1.DataUpload) err = crClient.Get(t.Context(), crclient.ObjectKey{Namespace: tc.dataUpload.Namespace, Name: tc.dataUpload.Name}, du) require.NoError(t, err) require.True(t, cmp.Equal(tc.expectedDataUpload, *du, cmpopts.IgnoreFields(velerov2alpha1.DataUpload{}, "ResourceVersion"))) }) } } func TestPVCAppliesTo(t *testing.T) { p := pvcBackupItemAction{ log: logrus.StandardLogger(), } selector, err := p.AppliesTo() require.NoError(t, err) require.Equal( t, velero.ResourceSelector{ IncludedResources: []string{"persistentvolumeclaims"}, }, selector, ) } func TestNewPVCBackupItemAction(t *testing.T) { logger := logrus.StandardLogger() crClient := velerotest.NewFakeControllerRuntimeClient(t) f := &factorymocks.Factory{} f.On("KubebuilderClient").Return(nil, fmt.Errorf("")) plugin := NewPvcBackupItemAction(f) _, err := plugin(logger) require.Error(t, err) f1 := &factorymocks.Factory{} f1.On("KubebuilderClient").Return(crClient, nil) plugin1 := NewPvcBackupItemAction(f1) _, err1 := plugin1(logger) require.NoError(t, err1) } func TestListGroupedPVCs(t *testing.T) { tests := []struct { name string namespace string labelKey string groupValue string pvcs []corev1api.PersistentVolumeClaim expectCount int expectError bool }{ { name: "Match single PVC with label", namespace: "ns1", labelKey: "vgs-key", groupValue: "group-a", pvcs: []corev1api.PersistentVolumeClaim{ { ObjectMeta: metav1.ObjectMeta{ Name: "pvc1", Namespace: "ns1", Labels: map[string]string{ "vgs-key": "group-a", }, }, }, }, expectCount: 1, }, { name: "No matching PVCs", namespace: "ns1", labelKey: "vgs-key", groupValue: "group-b", pvcs: []corev1api.PersistentVolumeClaim{ { ObjectMeta: metav1.ObjectMeta{ Name: "pvc1", Namespace: "ns1", Labels: map[string]string{ "vgs-key": "group-a", }, }, }, }, expectCount: 0, }, { name: "Match multiple PVCs", namespace: "ns1", labelKey: "vgs-key", groupValue: "group-a", pvcs: []corev1api.PersistentVolumeClaim{ { ObjectMeta: metav1.ObjectMeta{ Name: "pvc1", Namespace: "ns1", Labels: map[string]string{"vgs-key": "group-a"}, }, }, { ObjectMeta: metav1.ObjectMeta{ Name: "pvc2", Namespace: "ns1", Labels: map[string]string{"vgs-key": "group-a"}, }, }, }, expectCount: 2, }, { name: "Different namespace", namespace: "ns2", labelKey: "vgs-key", groupValue: "group-a", pvcs: []corev1api.PersistentVolumeClaim{ { ObjectMeta: metav1.ObjectMeta{ Name: "pvc1", Namespace: "ns1", Labels: map[string]string{"vgs-key": "group-a"}, }, }, }, expectCount: 0, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { var objs []runtime.Object for i := range tt.pvcs { objs = append(objs, &tt.pvcs[i]) } client := velerotest.NewFakeControllerRuntimeClient(t, objs...) action := &pvcBackupItemAction{ log: logrus.New(), crClient: client, } result, err := action.listGroupedPVCs(t.Context(), tt.namespace, tt.labelKey, tt.groupValue) if tt.expectError { require.Error(t, err) } else { require.NoError(t, err) require.Len(t, result, tt.expectCount) } }) } } func TestFilterPVCsByVolumePolicy(t *testing.T) { tests := []struct { name string pvcs []corev1api.PersistentVolumeClaim pvs []corev1api.PersistentVolume volumePolicyStr string expectCount int expectError bool }{ { name: "All PVCs should be included when no volume policy", pvcs: []corev1api.PersistentVolumeClaim{ { ObjectMeta: metav1.ObjectMeta{Name: "pvc-1", Namespace: "ns-1"}, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "pv-1", StorageClassName: pointer.String("sc-1"), }, Status: corev1api.PersistentVolumeClaimStatus{Phase: corev1api.ClaimBound}, }, { ObjectMeta: metav1.ObjectMeta{Name: "pvc-2", Namespace: "ns-1"}, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "pv-2", StorageClassName: pointer.String("sc-1"), }, Status: corev1api.PersistentVolumeClaimStatus{Phase: corev1api.ClaimBound}, }, }, pvs: []corev1api.PersistentVolume{ { ObjectMeta: metav1.ObjectMeta{Name: "pv-1"}, Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ CSI: &corev1api.CSIPersistentVolumeSource{Driver: "csi-driver-1"}, }, }, }, { ObjectMeta: metav1.ObjectMeta{Name: "pv-2"}, Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ CSI: &corev1api.CSIPersistentVolumeSource{Driver: "csi-driver-1"}, }, }, }, }, expectCount: 2, }, { name: "Filter out NFS PVC by volume policy", pvcs: []corev1api.PersistentVolumeClaim{ { ObjectMeta: metav1.ObjectMeta{Name: "pvc-csi", Namespace: "ns-1"}, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "pv-csi", StorageClassName: pointer.String("sc-1"), }, Status: corev1api.PersistentVolumeClaimStatus{Phase: corev1api.ClaimBound}, }, { ObjectMeta: metav1.ObjectMeta{Name: "pvc-nfs", Namespace: "ns-1"}, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "pv-nfs", StorageClassName: pointer.String("sc-nfs"), }, Status: corev1api.PersistentVolumeClaimStatus{Phase: corev1api.ClaimBound}, }, }, pvs: []corev1api.PersistentVolume{ { ObjectMeta: metav1.ObjectMeta{Name: "pv-csi"}, Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ CSI: &corev1api.CSIPersistentVolumeSource{Driver: "csi-driver"}, }, }, }, { ObjectMeta: metav1.ObjectMeta{Name: "pv-nfs"}, Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ NFS: &corev1api.NFSVolumeSource{ Server: "nfs-server", Path: "/export", }, }, }, }, }, volumePolicyStr: ` version: v1 volumePolicies: - conditions: nfs: {} action: type: skip `, expectCount: 1, }, { name: "All PVCs filtered out by volume policy", pvcs: []corev1api.PersistentVolumeClaim{ { ObjectMeta: metav1.ObjectMeta{Name: "pvc-nfs-1", Namespace: "ns-1"}, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "pv-nfs-1", StorageClassName: pointer.String("sc-nfs"), }, Status: corev1api.PersistentVolumeClaimStatus{Phase: corev1api.ClaimBound}, }, { ObjectMeta: metav1.ObjectMeta{Name: "pvc-nfs-2", Namespace: "ns-1"}, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "pv-nfs-2", StorageClassName: pointer.String("sc-nfs"), }, Status: corev1api.PersistentVolumeClaimStatus{Phase: corev1api.ClaimBound}, }, }, pvs: []corev1api.PersistentVolume{ { ObjectMeta: metav1.ObjectMeta{Name: "pv-nfs-1"}, Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ NFS: &corev1api.NFSVolumeSource{ Server: "nfs-server", Path: "/export/1", }, }, }, }, { ObjectMeta: metav1.ObjectMeta{Name: "pv-nfs-2"}, Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ NFS: &corev1api.NFSVolumeSource{ Server: "nfs-server", Path: "/export/2", }, }, }, }, }, volumePolicyStr: ` version: v1 volumePolicies: - conditions: nfs: {} action: type: skip `, expectCount: 0, }, { name: "Filter out non-CSI PVCs from mixed driver group", pvcs: []corev1api.PersistentVolumeClaim{ { ObjectMeta: metav1.ObjectMeta{ Name: "pvc-linstor", Namespace: "ns-1", Labels: map[string]string{"app.kubernetes.io/instance": "myapp"}, }, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "pv-linstor", StorageClassName: pointer.String("sc-linstor"), }, Status: corev1api.PersistentVolumeClaimStatus{Phase: corev1api.ClaimBound}, }, { ObjectMeta: metav1.ObjectMeta{ Name: "pvc-nfs", Namespace: "ns-1", Labels: map[string]string{"app.kubernetes.io/instance": "myapp"}, }, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "pv-nfs", StorageClassName: pointer.String("sc-nfs"), }, Status: corev1api.PersistentVolumeClaimStatus{Phase: corev1api.ClaimBound}, }, }, pvs: []corev1api.PersistentVolume{ { ObjectMeta: metav1.ObjectMeta{Name: "pv-linstor"}, Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ CSI: &corev1api.CSIPersistentVolumeSource{Driver: "linstor.csi.linbit.com"}, }, }, }, { ObjectMeta: metav1.ObjectMeta{Name: "pv-nfs"}, Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ NFS: &corev1api.NFSVolumeSource{ Server: "nfs-server", Path: "/export", }, }, }, }, }, volumePolicyStr: ` version: v1 volumePolicies: - conditions: nfs: {} action: type: skip `, expectCount: 1, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { objs := []runtime.Object{} for i := range tt.pvs { objs = append(objs, &tt.pvs[i]) } client := velerotest.NewFakeControllerRuntimeClient(t, objs...) backup := &velerov1api.Backup{ ObjectMeta: metav1.ObjectMeta{ Name: "test-backup", Namespace: "velero", }, Spec: velerov1api.BackupSpec{}, } // Add volume policy ConfigMap if specified if tt.volumePolicyStr != "" { cm := &corev1api.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "volume-policy", Namespace: "velero", }, Data: map[string]string{ "volume-policy": tt.volumePolicyStr, }, } require.NoError(t, client.Create(t.Context(), cm)) backup.Spec.ResourcePolicy = &corev1api.TypedLocalObjectReference{ Kind: "ConfigMap", Name: "volume-policy", } } action := &pvcBackupItemAction{ log: velerotest.NewLogger(), crClient: client, } // Pass nil for VolumeHelper in tests - it will fall back to creating a new one per call // This is the expected behavior for testing and third-party plugins result, err := action.filterPVCsByVolumePolicy(tt.pvcs, backup, nil) if tt.expectError { require.Error(t, err) } else { require.NoError(t, err) require.Len(t, result, tt.expectCount) // For mixed driver scenarios, verify filtered result can determine single CSI driver if tt.name == "Filter out non-CSI PVCs from mixed driver group" && len(result) > 0 { driver, err := action.determineCSIDriver(result) require.NoError(t, err, "After filtering, determineCSIDriver should not fail with multiple drivers error") require.Equal(t, "linstor.csi.linbit.com", driver, "Should have the Linstor driver after filtering out NFS") } } }) } } // TestFilterPVCsByVolumePolicyWithVolumeHelper tests filterPVCsByVolumePolicy when a // pre-created VolumeHelper is passed (non-nil). This exercises the cached path used // by the CSI PVC BIA plugin for better performance. func TestFilterPVCsByVolumePolicyWithVolumeHelper(t *testing.T) { // Create test PVCs and PVs pvcs := []corev1api.PersistentVolumeClaim{ { ObjectMeta: metav1.ObjectMeta{Name: "pvc-csi", Namespace: "ns-1"}, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "pv-csi", StorageClassName: pointer.String("sc-csi"), }, Status: corev1api.PersistentVolumeClaimStatus{Phase: corev1api.ClaimBound}, }, { ObjectMeta: metav1.ObjectMeta{Name: "pvc-nfs", Namespace: "ns-1"}, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "pv-nfs", StorageClassName: pointer.String("sc-nfs"), }, Status: corev1api.PersistentVolumeClaimStatus{Phase: corev1api.ClaimBound}, }, } pvs := []corev1api.PersistentVolume{ { ObjectMeta: metav1.ObjectMeta{Name: "pv-csi"}, Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ CSI: &corev1api.CSIPersistentVolumeSource{Driver: "csi-driver"}, }, }, }, { ObjectMeta: metav1.ObjectMeta{Name: "pv-nfs"}, Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ NFS: &corev1api.NFSVolumeSource{ Server: "nfs-server", Path: "/export", }, }, }, }, } // Create fake client with PVs objs := []runtime.Object{} for i := range pvs { objs = append(objs, &pvs[i]) } client := velerotest.NewFakeControllerRuntimeClient(t, objs...) // Create backup with volume policy that skips NFS volumes volumePolicyStr := ` version: v1 volumePolicies: - conditions: nfs: {} action: type: skip ` cm := &corev1api.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "volume-policy", Namespace: "velero", }, Data: map[string]string{ "volume-policy": volumePolicyStr, }, } require.NoError(t, client.Create(t.Context(), cm)) backup := &velerov1api.Backup{ ObjectMeta: metav1.ObjectMeta{ Name: "test-backup", Namespace: "velero", }, Spec: velerov1api.BackupSpec{ ResourcePolicy: &corev1api.TypedLocalObjectReference{ Kind: "ConfigMap", Name: "volume-policy", }, }, } action := &pvcBackupItemAction{ log: velerotest.NewLogger(), crClient: client, } // Create a VolumeHelper using the same method the plugin would use vh, err := action.getOrCreateVolumeHelper(backup) require.NoError(t, err) require.NotNil(t, vh) // Test with the pre-created VolumeHelper (non-nil path) result, err := action.filterPVCsByVolumePolicy(pvcs, backup, vh) require.NoError(t, err) // Should filter out the NFS PVC, leaving only the CSI PVC require.Len(t, result, 1) require.Equal(t, "pvc-csi", result[0].Name) } func TestDetermineCSIDriver(t *testing.T) { tests := []struct { name string pvcs []corev1api.PersistentVolumeClaim pvs []corev1api.PersistentVolume expectError bool expectedDriver string }{ { name: "Single PVC with CSI PV", pvcs: []corev1api.PersistentVolumeClaim{ { ObjectMeta: metav1.ObjectMeta{Name: "pvc-1", Namespace: "ns-1"}, Spec: corev1api.PersistentVolumeClaimSpec{VolumeName: "pv-1"}, Status: corev1api.PersistentVolumeClaimStatus{Phase: corev1api.ClaimBound}, }, }, pvs: []corev1api.PersistentVolume{ { ObjectMeta: metav1.ObjectMeta{Name: "pv-1"}, Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ CSI: &corev1api.CSIPersistentVolumeSource{Driver: "csi-driver"}, }, }, }, }, expectedDriver: "csi-driver", }, { name: "Multiple PVCs with same CSI driver", pvcs: []corev1api.PersistentVolumeClaim{ { ObjectMeta: metav1.ObjectMeta{Name: "pvc-1", Namespace: "ns-1"}, Spec: corev1api.PersistentVolumeClaimSpec{VolumeName: "pv-1"}, Status: corev1api.PersistentVolumeClaimStatus{Phase: corev1api.ClaimBound}, }, { ObjectMeta: metav1.ObjectMeta{Name: "pvc-2", Namespace: "ns-1"}, Spec: corev1api.PersistentVolumeClaimSpec{VolumeName: "pv-2"}, Status: corev1api.PersistentVolumeClaimStatus{Phase: corev1api.ClaimBound}, }, }, pvs: []corev1api.PersistentVolume{ { ObjectMeta: metav1.ObjectMeta{Name: "pv-1"}, Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ CSI: &corev1api.CSIPersistentVolumeSource{Driver: "csi-driver"}, }, }, }, { ObjectMeta: metav1.ObjectMeta{Name: "pv-2"}, Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ CSI: &corev1api.CSIPersistentVolumeSource{Driver: "csi-driver"}, }, }, }, }, expectedDriver: "csi-driver", }, { name: "PV not CSI provisioned", pvcs: []corev1api.PersistentVolumeClaim{ { ObjectMeta: metav1.ObjectMeta{Name: "pvc-1", Namespace: "ns-1"}, Spec: corev1api.PersistentVolumeClaimSpec{VolumeName: "pv-1"}, Status: corev1api.PersistentVolumeClaimStatus{Phase: corev1api.ClaimBound}, }, }, pvs: []corev1api.PersistentVolume{ { ObjectMeta: metav1.ObjectMeta{Name: "pv-1"}, Spec: corev1api.PersistentVolumeSpec{}, }, }, expectError: true, }, { name: "Multiple PVCs with different CSI drivers", pvcs: []corev1api.PersistentVolumeClaim{ { ObjectMeta: metav1.ObjectMeta{Name: "pvc-1", Namespace: "ns-1"}, Spec: corev1api.PersistentVolumeClaimSpec{VolumeName: "pv-1"}, Status: corev1api.PersistentVolumeClaimStatus{Phase: corev1api.ClaimBound}, }, { ObjectMeta: metav1.ObjectMeta{Name: "pvc-2", Namespace: "ns-1"}, Spec: corev1api.PersistentVolumeClaimSpec{VolumeName: "pv-2"}, Status: corev1api.PersistentVolumeClaimStatus{Phase: corev1api.ClaimBound}, }, }, pvs: []corev1api.PersistentVolume{ { ObjectMeta: metav1.ObjectMeta{Name: "pv-1"}, Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ CSI: &corev1api.CSIPersistentVolumeSource{Driver: "csi-driver-1"}, }, }, }, { ObjectMeta: metav1.ObjectMeta{Name: "pv-2"}, Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ CSI: &corev1api.CSIPersistentVolumeSource{Driver: "csi-driver-2"}, }, }, }, }, expectError: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { var initObjs []runtime.Object for i := range tt.pvcs { pvc := tt.pvcs[i] initObjs = append(initObjs, &pvc) } for i := range tt.pvs { pv := tt.pvs[i] initObjs = append(initObjs, &pv) } client := velerotest.NewFakeControllerRuntimeClient(t, initObjs...) action := &pvcBackupItemAction{ log: logrus.New(), crClient: client, } driver, err := action.determineCSIDriver(tt.pvcs) if tt.expectError { require.Error(t, err) } else { require.NoError(t, err) require.Equal(t, tt.expectedDriver, driver) } }) } } func TestDetermineVGSClass(t *testing.T) { tests := []struct { name string backup *velerov1api.Backup pvc *corev1api.PersistentVolumeClaim existingVGSClass []volumegroupsnapshotv1beta1.VolumeGroupSnapshotClass expectError bool expectResult string }{ { name: "PVC annotation override", pvc: &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ velerov1api.VolumeGroupSnapshotClassAnnotationPVC: "pvc-class", }, }, }, backup: &velerov1api.Backup{}, expectResult: "pvc-class", }, { name: "Backup annotation override", pvc: &corev1api.PersistentVolumeClaim{}, backup: &velerov1api.Backup{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ fmt.Sprintf("%s%s", velerov1api.VolumeGroupSnapshotClassAnnotationBackupPrefix, testDriver): "backup-class", }, }, }, expectResult: "backup-class", }, { name: "Default label-based match", pvc: &corev1api.PersistentVolumeClaim{}, backup: &velerov1api.Backup{}, existingVGSClass: []volumegroupsnapshotv1beta1.VolumeGroupSnapshotClass{ { ObjectMeta: metav1.ObjectMeta{ Name: "default-class", Labels: map[string]string{velerov1api.VolumeGroupSnapshotClassDefaultLabel: "true"}, }, Driver: testDriver, }, }, expectResult: "default-class", }, { name: "No matching VGS class", pvc: &corev1api.PersistentVolumeClaim{}, backup: &velerov1api.Backup{}, expectError: true, }, { name: "Multiple matching VGS classes", pvc: &corev1api.PersistentVolumeClaim{}, backup: &velerov1api.Backup{}, existingVGSClass: []volumegroupsnapshotv1beta1.VolumeGroupSnapshotClass{ { ObjectMeta: metav1.ObjectMeta{ Name: "class1", Labels: map[string]string{velerov1api.VolumeGroupSnapshotClassDefaultLabel: "true"}, }, Driver: testDriver, }, { ObjectMeta: metav1.ObjectMeta{ Name: "class2", Labels: map[string]string{velerov1api.VolumeGroupSnapshotClassDefaultLabel: "true"}, }, Driver: testDriver, }, }, expectError: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { var initObjs []runtime.Object for _, vgsClass := range tt.existingVGSClass { vgsClassCopy := vgsClass initObjs = append(initObjs, &vgsClassCopy) } client := velerotest.NewFakeControllerRuntimeClient(t, initObjs...) logger := logrus.New() require.NoError(t, volumegroupsnapshotv1beta1.AddToScheme(client.Scheme())) action := &pvcBackupItemAction{crClient: client, log: logger} result, err := action.determineVGSClass(t.Context(), testDriver, tt.backup, tt.pvc) if tt.expectError { require.Error(t, err) } else { require.NoError(t, err) require.Equal(t, tt.expectResult, result) } }) } } func TestCreateVolumeGroupSnapshot(t *testing.T) { testNamespace := "test-ns" testLabelKey := "velero.io/test-vgs-label" testLabelValue := "group-1" testVGSClass := "test-class" testBackup := &velerov1api.Backup{ ObjectMeta: metav1.ObjectMeta{ Name: "test-backup", UID: "test-uid", }, } testPVC := corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: "test-pvc", Namespace: testNamespace, Labels: map[string]string{ testLabelKey: testLabelValue, }, }, } crClient := velerotest.NewFakeControllerRuntimeClient(t) log := logrus.New() action := &pvcBackupItemAction{ log: log, crClient: crClient, } vgs, err := action.createVolumeGroupSnapshot(t.Context(), testBackup, testPVC, testLabelKey, testLabelValue, testVGSClass) require.NoError(t, err) require.NotNil(t, vgs) // Verify VGS fields assert.Equal(t, testNamespace, vgs.Namespace) assert.NotEmpty(t, vgs.GenerateName) assert.Equal(t, testVGSClass, *vgs.Spec.VolumeGroupSnapshotClassName) assert.NotNil(t, vgs.Spec.Source.Selector) assert.Equal(t, testLabelValue, vgs.Spec.Source.Selector.MatchLabels[testLabelKey]) assert.Equal(t, testLabelValue, vgs.Labels[testLabelKey]) assert.Equal(t, label.GetValidName(testBackup.Name), vgs.Labels[velerov1api.BackupNameLabel]) assert.Equal(t, string(testBackup.UID), vgs.Labels[velerov1api.BackupUIDLabel]) // Check that it exists in fake client retrieved := &volumegroupsnapshotv1beta1.VolumeGroupSnapshot{} err = crClient.Get(t.Context(), crclient.ObjectKey{Name: vgs.Name, Namespace: vgs.Namespace}, retrieved) require.NoError(t, err) } func TestWaitForVGSAssociatedVS(t *testing.T) { vgs := &volumegroupsnapshotv1beta1.VolumeGroupSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: "test-vgs", Namespace: "test-ns", UID: types.UID("1234-5678-uuid"), }, } makeVS := func(name string, hasStatus bool, hasVGSName bool, owned bool, pvcName string) *snapshotv1api.VolumeSnapshot { var refs []metav1.OwnerReference if owned { refs = []metav1.OwnerReference{ { APIVersion: "groupsnapshot.storage.k8s.io/v1beta1", Kind: "VolumeGroupSnapshot", Name: vgs.Name, UID: vgs.UID, }, } } vs := &snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: vgs.Namespace, OwnerReferences: refs, }, Spec: snapshotv1api.VolumeSnapshotSpec{ Source: snapshotv1api.VolumeSnapshotSource{ PersistentVolumeClaimName: pointer.String(pvcName), }, }, } if hasStatus { vs.Status = &snapshotv1api.VolumeSnapshotStatus{} if hasVGSName { vs.Status.VolumeGroupSnapshotName = pointer.String(vgs.Name) } } return vs } makePVC := func(name string) corev1api.PersistentVolumeClaim { return corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: vgs.Namespace, }, } } tests := []struct { name string vsList []*snapshotv1api.VolumeSnapshot groupedPVCs []corev1api.PersistentVolumeClaim expectErr bool expectVSMap int }{ { name: "all owned VS have VGS name", vsList: []*snapshotv1api.VolumeSnapshot{ makeVS("vs1", true, true, true, "pvc1"), makeVS("vs2", true, true, true, "pvc2"), }, groupedPVCs: []corev1api.PersistentVolumeClaim{ makePVC("pvc1"), makePVC("pvc2"), }, expectErr: false, expectVSMap: 2, }, { name: "one owned VS missing VGS name", vsList: []*snapshotv1api.VolumeSnapshot{ makeVS("vs1", true, true, true, "pvc1"), makeVS("vs2", true, false, true, "pvc2"), }, groupedPVCs: []corev1api.PersistentVolumeClaim{ makePVC("pvc1"), makePVC("pvc2"), }, expectErr: true, }, { name: "owned VS has no status", vsList: []*snapshotv1api.VolumeSnapshot{ makeVS("vs1", false, false, true, "pvc1"), }, groupedPVCs: []corev1api.PersistentVolumeClaim{ makePVC("pvc1"), }, expectErr: true, }, { name: "unrelated VS ignored", vsList: []*snapshotv1api.VolumeSnapshot{ makeVS("vs1", true, true, false, "pvc1"), }, groupedPVCs: []corev1api.PersistentVolumeClaim{ makePVC("pvc1"), }, expectErr: true, }, { name: "no owned VS present", vsList: []*snapshotv1api.VolumeSnapshot{ makeVS("vs1", true, true, false, "pvc1"), }, groupedPVCs: []corev1api.PersistentVolumeClaim{ makePVC("pvc1"), }, expectErr: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { var objs []runtime.Object objs = append(objs, vgs) for _, vs := range tt.vsList { objs = append(objs, vs) } for _, pvc := range tt.groupedPVCs { objs = append(objs, &pvc) } client := velerotest.NewFakeControllerRuntimeClient(t, objs...) action := &pvcBackupItemAction{ log: velerotest.NewLogger(), crClient: client, } vsMap, err := action.waitForVGSAssociatedVS(t.Context(), tt.groupedPVCs, vgs, 2*time.Second) if tt.expectErr { if err == nil { t.Errorf("expected error but got nil") } } else { if err != nil { t.Errorf("unexpected error: %v", err) } if len(vsMap) != tt.expectVSMap { t.Errorf("expected vsMap length %d, got %d", tt.expectVSMap, len(vsMap)) } } }) } } func TestUpdateVGSCreatedVS(t *testing.T) { backup := &velerov1api.Backup{ ObjectMeta: metav1.ObjectMeta{ Name: "backup-1", UID: "backup-uid-123", }, } vgs := &volumegroupsnapshotv1beta1.VolumeGroupSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: "test-vgs", Namespace: "ns", UID: "vgs-uid-123", }, } makeVS := func(name string, withVGSOwner bool, vgsNamePtr *string, pvcName string) *snapshotv1api.VolumeSnapshot { var refs []metav1.OwnerReference if withVGSOwner { refs = []metav1.OwnerReference{ { APIVersion: "groupsnapshot.storage.k8s.io/v1beta1", Kind: "VolumeGroupSnapshot", Name: vgs.Name, UID: vgs.UID, }, } } return &snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: vgs.Namespace, OwnerReferences: refs, Finalizers: []string{ VolumeSnapshotFinalizerGroupProtection, VolumeSnapshotFinalizerSourceProtection, }, }, Status: &snapshotv1api.VolumeSnapshotStatus{ ReadyToUse: pointer.Bool(true), VolumeGroupSnapshotName: vgsNamePtr, }, Spec: snapshotv1api.VolumeSnapshotSpec{ Source: snapshotv1api.VolumeSnapshotSource{ PersistentVolumeClaimName: pointer.String(pvcName), }, }, } } tests := []struct { name string vs *snapshotv1api.VolumeSnapshot expectOwnerCleared bool expectFinalizersCleared bool expectLabelPatched bool }{ { name: "should update owned VS", vs: makeVS("vs-owned", true, pointer.String(vgs.Name), "pvc-1"), expectOwnerCleared: true, expectFinalizersCleared: true, expectLabelPatched: true, }, { name: "should skip VS not owned by VGS", vs: makeVS("vs-unowned", false, nil, "pvc-1"), expectOwnerCleared: false, expectFinalizersCleared: false, expectLabelPatched: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { client := velerotest.NewFakeControllerRuntimeClient(t, vgs, tt.vs) action := &pvcBackupItemAction{ log: velerotest.NewLogger(), crClient: client, } // Build vsMap using the PVC name from the VS vsMap := map[string]*snapshotv1api.VolumeSnapshot{ *tt.vs.Spec.Source.PersistentVolumeClaimName: tt.vs, } err := action.updateVGSCreatedVS(t.Context(), vsMap, vgs, backup) require.NoError(t, err) // Fetch updated VS updated := &snapshotv1api.VolumeSnapshot{} err = client.Get(t.Context(), crclient.ObjectKey{Name: tt.vs.Name, Namespace: tt.vs.Namespace}, updated) require.NoError(t, err) if tt.expectOwnerCleared { assert.Empty(t, updated.OwnerReferences, "expected ownerReferences to be cleared") } else { assert.Equal(t, tt.vs.OwnerReferences, updated.OwnerReferences, "expected ownerReferences to remain unchanged") } if tt.expectFinalizersCleared { assert.Empty(t, updated.Finalizers, "expected finalizers to be cleared") } else { assert.Equal(t, tt.vs.Finalizers, updated.Finalizers, "expected finalizers to remain unchanged") } if tt.expectLabelPatched { assert.Equal(t, "backup-1", updated.Labels[velerov1api.BackupNameLabel]) assert.Equal(t, "backup-uid-123", updated.Labels[velerov1api.BackupUIDLabel]) } else { assert.Nil(t, updated.Labels, "expected no labels to be patched") } }) } } func TestPatchVGSCDeletionPolicy(t *testing.T) { tests := []struct { name string initialPolicy snapshotv1api.DeletionPolicy expectedPolicy snapshotv1api.DeletionPolicy expectPatch bool expectErr bool }{ { name: "patches Delete to Retain", initialPolicy: snapshotv1api.VolumeSnapshotContentDelete, expectedPolicy: snapshotv1api.VolumeSnapshotContentRetain, expectPatch: true, }, { name: "no patch if already Retain", initialPolicy: snapshotv1api.VolumeSnapshotContentRetain, expectedPolicy: snapshotv1api.VolumeSnapshotContentRetain, expectPatch: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { vgsc := &volumegroupsnapshotv1beta1.VolumeGroupSnapshotContent{ ObjectMeta: metav1.ObjectMeta{Name: "test-vgsc"}, Spec: volumegroupsnapshotv1beta1.VolumeGroupSnapshotContentSpec{ DeletionPolicy: tt.initialPolicy, }, } vgs := &volumegroupsnapshotv1beta1.VolumeGroupSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: "test-vgs", Namespace: "ns", }, Status: &volumegroupsnapshotv1beta1.VolumeGroupSnapshotStatus{ BoundVolumeGroupSnapshotContentName: pointer.String("test-vgsc"), }, } client := velerotest.NewFakeControllerRuntimeClient(t, vgs, vgsc) action := &pvcBackupItemAction{ log: velerotest.NewLogger(), crClient: client, } err := action.patchVGSCDeletionPolicy(t.Context(), vgs) if tt.expectErr { require.Error(t, err) return } require.NoError(t, err) updated := &volumegroupsnapshotv1beta1.VolumeGroupSnapshotContent{} err = client.Get(t.Context(), crclient.ObjectKey{Name: "test-vgsc"}, updated) require.NoError(t, err) require.Equal(t, tt.expectedPolicy, updated.Spec.DeletionPolicy) }) } } func TestDeleteVGSAndVGSC(t *testing.T) { makeVGS := func(name, namespace string, boundVGSCName *string) *volumegroupsnapshotv1beta1.VolumeGroupSnapshot { return &volumegroupsnapshotv1beta1.VolumeGroupSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: namespace, }, Status: &volumegroupsnapshotv1beta1.VolumeGroupSnapshotStatus{ BoundVolumeGroupSnapshotContentName: boundVGSCName, }, } } makeVGSC := func(name string) *volumegroupsnapshotv1beta1.VolumeGroupSnapshotContent { return &volumegroupsnapshotv1beta1.VolumeGroupSnapshotContent{ ObjectMeta: metav1.ObjectMeta{ Name: name, }, } } tests := []struct { name string vgs *volumegroupsnapshotv1beta1.VolumeGroupSnapshot existingVGSC *volumegroupsnapshotv1beta1.VolumeGroupSnapshotContent expectVGSCDelete bool expectVGSDelete bool }{ { name: "deletes both VGSC and VGS", vgs: makeVGS("test-vgs", "ns", pointer.String("test-vgsc")), existingVGSC: makeVGSC("test-vgsc"), expectVGSCDelete: true, expectVGSDelete: true, }, { name: "VGSC not found, still deletes VGS", vgs: makeVGS("test-vgs", "ns", pointer.String("missing-vgsc")), existingVGSC: nil, expectVGSCDelete: false, expectVGSDelete: true, }, { name: "no BoundVGSCName set, only deletes VGS", vgs: makeVGS("test-vgs", "ns", nil), existingVGSC: nil, expectVGSCDelete: false, expectVGSDelete: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { var objs []runtime.Object objs = append(objs, tt.vgs) if tt.existingVGSC != nil { objs = append(objs, tt.existingVGSC) } client := velerotest.NewFakeControllerRuntimeClient(t, objs...) action := &pvcBackupItemAction{ log: velerotest.NewLogger(), crClient: client, } err := action.deleteVGSAndVGSC(t.Context(), tt.vgs) require.NoError(t, err) // Check VGSC is deleted if tt.expectVGSCDelete { got := &volumegroupsnapshotv1beta1.VolumeGroupSnapshotContent{} err = client.Get(t.Context(), crclient.ObjectKey{Name: "test-vgsc"}, got) assert.True(t, apierrors.IsNotFound(err), "expected VGSC to be deleted") } // Check VGS is deleted gotVGS := &volumegroupsnapshotv1beta1.VolumeGroupSnapshot{} err = client.Get(t.Context(), crclient.ObjectKey{Name: "test-vgs", Namespace: "ns"}, gotVGS) assert.True(t, apierrors.IsNotFound(err), "expected VGS to be deleted") }) } } func TestFindExistingVSForBackup(t *testing.T) { backupUID := types.UID("backup-uid-123") backupName := "backup-1" pvcName := "pvc-1" namespace := "ns" makeVS := func(name, pvc string, match bool) *snapshotv1api.VolumeSnapshot { labels := map[string]string{} if match { labels[velerov1api.BackupNameLabel] = label.GetValidName(backupName) labels[velerov1api.BackupUIDLabel] = string(backupUID) } return &snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: namespace, Labels: labels, }, Spec: snapshotv1api.VolumeSnapshotSpec{ Source: snapshotv1api.VolumeSnapshotSource{ PersistentVolumeClaimName: pointer.String(pvc), }, }, } } tests := []struct { name string vsList []*snapshotv1api.VolumeSnapshot expectName string expectNil bool }{ { name: "should find matching VS", vsList: []*snapshotv1api.VolumeSnapshot{ makeVS("vs-match", pvcName, true), }, expectName: "vs-match", expectNil: false, }, { name: "should skip VS with non-matching labels", vsList: []*snapshotv1api.VolumeSnapshot{ makeVS("vs-nolabel", pvcName, false), }, expectNil: true, }, { name: "should skip VS with different PVC name", vsList: []*snapshotv1api.VolumeSnapshot{ makeVS("vs-other-pvc", "other-pvc", true), }, expectNil: true, }, { name: "should return nil if VS list is empty", vsList: []*snapshotv1api.VolumeSnapshot{}, expectNil: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { var objs []runtime.Object for _, vs := range tt.vsList { objs = append(objs, vs) } client := velerotest.NewFakeControllerRuntimeClient(t, objs...) action := &pvcBackupItemAction{ log: velerotest.NewLogger(), crClient: client, } vs, err := action.findExistingVSForBackup(t.Context(), backupUID, backupName, pvcName, namespace) require.NoError(t, err) if tt.expectNil { assert.Nil(t, vs) } else { require.NotNil(t, vs) assert.Equal(t, tt.expectName, vs.Name) } }) } } func TestWaitForVGSCBinding(t *testing.T) { makeVGS := func(name string, withStatus bool) *volumegroupsnapshotv1beta1.VolumeGroupSnapshot { vgs := &volumegroupsnapshotv1beta1.VolumeGroupSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: "ns", }, } if withStatus { contentName := "vgsc-123" vgs.Status = &volumegroupsnapshotv1beta1.VolumeGroupSnapshotStatus{ BoundVolumeGroupSnapshotContentName: &contentName, } } return vgs } tests := []struct { name string vgs *volumegroupsnapshotv1beta1.VolumeGroupSnapshot expectErr bool }{ { name: "status is already bound", vgs: makeVGS("vgs1", true), expectErr: false, }, { name: "status is nil", vgs: makeVGS("vgs2", false), expectErr: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { client := velerotest.NewFakeControllerRuntimeClient(t, tt.vgs.DeepCopy()) action := &pvcBackupItemAction{ log: velerotest.NewLogger(), crClient: client, } err := action.waitForVGSCBinding(t.Context(), tt.vgs, 1*time.Second) if tt.expectErr { require.Error(t, err) } else { require.NoError(t, err) require.NotNil(t, tt.vgs.Status) require.NotNil(t, tt.vgs.Status.BoundVolumeGroupSnapshotContentName) require.Equal(t, "vgsc-123", *tt.vgs.Status.BoundVolumeGroupSnapshotContentName) } }) } } func TestGetVGSByLabels(t *testing.T) { labelKey := "velero.io/backup-name" labelVal := "backup-123" testLabels := map[string]string{labelKey: labelVal} makeVGS := func(name string, labels map[string]string) *volumegroupsnapshotv1beta1.VolumeGroupSnapshot { return &volumegroupsnapshotv1beta1.VolumeGroupSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: "test-ns", Labels: labels, }, } } tests := []struct { name string vgsObjects []runtime.Object expectError string expectName string }{ { name: "exactly one matching VGS", vgsObjects: []runtime.Object{ makeVGS("vgs1", testLabels), }, expectName: "vgs1", }, { name: "no matching VGS", vgsObjects: []runtime.Object{}, expectError: "no VolumeGroupSnapshot found matching labels", }, { name: "multiple matching VGS", vgsObjects: []runtime.Object{ makeVGS("vgs1", testLabels), makeVGS("vgs2", testLabels), }, expectError: "multiple VolumeGroupSnapshots found matching labels", }, { name: "client list error", vgsObjects: []runtime.Object{}, expectError: "failed to list VolumeGroupSnapshots by labels", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { var client crclient.Client if tt.name == "client list error" { // Inject a client that always errors on List client = &failingClient{} } else { client = velerotest.NewFakeControllerRuntimeClient(t, tt.vgsObjects...) } action := &pvcBackupItemAction{ log: velerotest.NewLogger(), crClient: client, } vgs, err := action.getVGSByLabels(t.Context(), "test-ns", testLabels) if tt.expectError != "" { if err == nil || !strings.Contains(err.Error(), tt.expectError) { t.Errorf("expected error containing '%s', got: %v", tt.expectError, err) } } else { if err != nil { t.Errorf("unexpected error: %v", err) } if vgs == nil || vgs.Name != tt.expectName { t.Errorf("expected VGS name %s, got %v", tt.expectName, vgs) } } }) } } // failingClient is a dummy client that fails on List type failingClient struct { crclient.Client } func (f *failingClient) List(ctx context.Context, list crclient.ObjectList, opts ...crclient.ListOption) error { return fmt.Errorf("simulated list error") } func TestHasOwnerReference(t *testing.T) { vgs := &volumegroupsnapshotv1beta1.VolumeGroupSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: "test-vgs", Namespace: "test-ns", UID: types.UID("1234-uid"), }, } tests := []struct { name string ownerRef metav1.OwnerReference expect bool }{ { name: "match kind, apiversion, uid", ownerRef: metav1.OwnerReference{ Kind: kuberesource.VGSKind, APIVersion: volumegroupsnapshotv1beta1.GroupName + "/" + volumegroupsnapshotv1beta1.SchemeGroupVersion.Version, UID: vgs.UID, }, expect: true, }, { name: "mismatch kind", ownerRef: metav1.OwnerReference{ Kind: "other-kind", APIVersion: volumegroupsnapshotv1beta1.GroupName + "/" + volumegroupsnapshotv1beta1.SchemeGroupVersion.Version, UID: vgs.UID, }, expect: false, }, { name: "mismatch apiversion", ownerRef: metav1.OwnerReference{ Kind: kuberesource.VGSKind, APIVersion: "wrong.group/v1", UID: vgs.UID, }, expect: false, }, { name: "mismatch uid", ownerRef: metav1.OwnerReference{ Kind: kuberesource.VGSKind, APIVersion: volumegroupsnapshotv1beta1.GroupName + "/" + volumegroupsnapshotv1beta1.SchemeGroupVersion.Version, UID: "wrong-uid", }, expect: false, }, { name: "no owner references", ownerRef: metav1.OwnerReference{}, expect: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { obj := &metav1.ObjectMeta{ Name: "dummy", Namespace: "test-ns", } if tt.name != "no owner references" { obj.OwnerReferences = []metav1.OwnerReference{tt.ownerRef} } found := hasOwnerReference(obj, vgs) assert.Equal(t, tt.expect, found) }) } } func TestPVCRequestSize(t *testing.T) { logger := logrus.New() tests := []struct { name string pvcInitial *corev1api.PersistentVolumeClaim // Use full PVC to allow for nil Requests restoreSize string expectedSize string }{ { name: "UpdateRequired: PVC request is lower than restore size", pvcInitial: func() *corev1api.PersistentVolumeClaim { pvc := builder.ForPersistentVolumeClaim("velero", "testPVC").Result() pvc.Spec.Resources.Requests = corev1api.ResourceList{ corev1api.ResourceStorage: resource.MustParse("1Gi"), } return pvc }(), restoreSize: "2Gi", expectedSize: "2Gi", }, { name: "NoUpdateRequired: PVC request is larger than restore size", pvcInitial: func() *corev1api.PersistentVolumeClaim { pvc := builder.ForPersistentVolumeClaim("velero", "testPVC").Result() pvc.Spec.Resources.Requests = corev1api.ResourceList{ corev1api.ResourceStorage: resource.MustParse("3Gi"), } return pvc }(), restoreSize: "2Gi", expectedSize: "3Gi", }, { name: "PVC has no initial storage request", pvcInitial: func() *corev1api.PersistentVolumeClaim { pvc := builder.ForPersistentVolumeClaim("velero", "testPVC").Result() pvc.Spec.Resources.Requests = corev1api.ResourceList{} // Empty request list return pvc }(), restoreSize: "2Gi", expectedSize: "2Gi", }, { name: "PVC has no initial Resources.Requests map", pvcInitial: func() *corev1api.PersistentVolumeClaim { pvc := builder.ForPersistentVolumeClaim("velero", "testPVC").Result() pvc.Spec.Resources.Requests = nil // This will trigger the line to be covered return pvc }(), restoreSize: "2Gi", expectedSize: "2Gi", }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { // Create a VolumeSnapshotContent with restore size rsQty := resource.MustParse(tc.restoreSize) vsc := &snapshotv1api.VolumeSnapshotContent{ ObjectMeta: metav1.ObjectMeta{ Name: "testVSC", }, Status: &snapshotv1api.VolumeSnapshotContentStatus{ RestoreSize: pointer.Int64(rsQty.Value()), }, } // Call the function under test pvc := tc.pvcInitial setPVCRequestSizeToVSRestoreSize(pvc, vsc, logger) // Verify that the PVC storage request is updated as expected. updatedSize := pvc.Spec.Resources.Requests[corev1api.ResourceStorage] expected := resource.MustParse(tc.expectedSize) // Corrected line below: require.Equal(t, 0, expected.Cmp(updatedSize), "Expected size %s, but got %s", expected.String(), updatedSize.String()) }) } } // TestGetOrCreateVolumeHelper tests the VolumeHelper and PVC-to-Pod cache behavior. // Since plugin instances are unique per backup (created via newPluginManager and // cleaned up via CleanupClients at backup completion), we verify that the pvcPodCache // is properly initialized and reused across calls. func TestGetOrCreateVolumeHelper(t *testing.T) { client := velerotest.NewFakeControllerRuntimeClient(t) action := &pvcBackupItemAction{ log: velerotest.NewLogger(), crClient: client, } backup := &velerov1api.Backup{ ObjectMeta: metav1.ObjectMeta{ Name: "test-backup", Namespace: "velero", UID: types.UID("test-uid-1"), }, } // Initially, pvcPodCache should be nil require.Nil(t, action.pvcPodCache, "pvcPodCache should be nil initially") // Get VolumeHelper first time - should create new cache and VolumeHelper vh1, err := action.getOrCreateVolumeHelper(backup) require.NoError(t, err) require.NotNil(t, vh1) // pvcPodCache should now be initialized require.NotNil(t, action.pvcPodCache, "pvcPodCache should be initialized after first call") cache1 := action.pvcPodCache // Get VolumeHelper second time - should reuse the same cache vh2, err := action.getOrCreateVolumeHelper(backup) require.NoError(t, err) require.NotNil(t, vh2) // The pvcPodCache should be the same instance require.Same(t, cache1, action.pvcPodCache, "Expected same pvcPodCache instance on repeated calls") } ================================================ FILE: pkg/backup/actions/csi/volumesnapshot_action.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package csi import ( "context" "fmt" "strings" "time" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" crclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/label" plugincommon "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" "github.com/vmware-tanzu/velero/pkg/plugin/velero" biav2 "github.com/vmware-tanzu/velero/pkg/plugin/velero/backupitemaction/v2" "github.com/vmware-tanzu/velero/pkg/util/boolptr" "github.com/vmware-tanzu/velero/pkg/util/csi" kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube" ) // volumeSnapshotBackupItemAction is a backup item action plugin to backup // CSI VolumeSnapshot objects using Velero type volumeSnapshotBackupItemAction struct { log logrus.FieldLogger crClient crclient.Client } // AppliesTo returns information indicating that the // VolumeSnapshotBackupItemAction should be invoked to // backup VolumeSnapshots. func (p *volumeSnapshotBackupItemAction) AppliesTo() ( velero.ResourceSelector, error, ) { return velero.ResourceSelector{ IncludedResources: []string{"volumesnapshots.snapshot.storage.k8s.io"}, }, nil } // Execute backs up a CSI VolumeSnapshot object and captures, as labels and annotations, // information from its associated VolumeSnapshotContents such as CSI driver name, // storage snapshot handle and namespace and name of the snapshot delete secret, if any. // It returns the VolumeSnapshotClass and the VolumeSnapshotContents as additional items // to be backed up. func (p *volumeSnapshotBackupItemAction) Execute( item runtime.Unstructured, backup *velerov1api.Backup, ) ( runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error, ) { p.log.Infof("Executing VolumeSnapshotBackupItemAction") vs := new(snapshotv1api.VolumeSnapshot) if err := runtime.DefaultUnstructuredConverter.FromUnstructured( item.UnstructuredContent(), vs); err != nil { return nil, nil, "", nil, errors.WithStack(err) } if backup.Status.Phase == velerov1api.BackupPhaseFinalizing || backup.Status.Phase == velerov1api.BackupPhaseFinalizingPartiallyFailed { p.log. WithField("Backup", fmt.Sprintf("%s/%s", backup.Namespace, backup.Name)). WithField("BackupPhase", backup.Status.Phase).Debugf("Cleaning VolumeSnapshots.") csi.DeleteReadyVolumeSnapshot(*vs, p.crClient, p.log) return item, nil, "", nil, nil } additionalItems := make([]velero.ResourceIdentifier, 0) if vs.Spec.VolumeSnapshotClassName != nil { // This is still needed to add the VolumeSnapshotClass to the backup. // The secret with VolumeSnapshotClass is still relevant to backup. additionalItems = append( additionalItems, velero.ResourceIdentifier{ GroupResource: kuberesource.VolumeSnapshotClasses, Name: *vs.Spec.VolumeSnapshotClassName, }, ) // Because async operation will update VolumeSnapshot during finalizing phase. // No matter what we do, VolumeSnapshotClass cannot be deleted. So skip it. // Just deleting VolumeSnapshotClass during restore and delete is enough. } p.log.Infof("Getting VolumesnapshotContent for Volumesnapshot %s/%s", vs.Namespace, vs.Name) ctx := context.TODO() vsc, err := csi.GetVSCForVS(ctx, vs, p.crClient) if err != nil { csi.CleanupVolumeSnapshot(vs, p.crClient, p.log) return nil, nil, "", nil, errors.WithStack(err) } annotations := make(map[string]string) if vsc != nil { // when we are backing up VolumeSnapshots created outside of velero, we // will not await VolumeSnapshot reconciliation and in this case // GetVolumeSnapshotContentForVolumeSnapshot may not find the associated // VolumeSnapshotContents to add to the backup. This is not an error // encountered in the backup process. So we add the VolumeSnapshotContent // to the backup only if one is found. additionalItems = append(additionalItems, velero.ResourceIdentifier{ GroupResource: kuberesource.VolumeSnapshotContents, Name: vsc.Name, }) annotations[velerov1api.VSCDeletionPolicyAnnotation] = string(vsc.Spec.DeletionPolicy) if vsc.Status != nil { if vsc.Status.SnapshotHandle != nil { // Capture storage provider snapshot handle and CSI driver name // to be used on restore to create a static VolumeSnapshotContent // that will be the source of the VolumeSnapshot. annotations[velerov1api.VolumeSnapshotHandleAnnotation] = *vsc.Status.SnapshotHandle annotations[velerov1api.DriverNameAnnotation] = vsc.Spec.Driver } if vsc.Status.RestoreSize != nil { annotations[velerov1api.VolumeSnapshotRestoreSize] = resource.NewQuantity( *vsc.Status.RestoreSize, resource.BinarySI).String() } } p.log.Infof("Patching VolumeSnapshotContent %s with velero BackupNameLabel", vsc.Name) // If we created the VolumeSnapshotContent object during this ongoing backup, // we would have created it with a DeletionPolicy of Retain. // But, we want to retain these VolumeSnapshotContent ONLY for the lifetime // of the backup. To that effect, during velero backup // deletion, we will update the DeletionPolicy of the VolumeSnapshotContent // and then delete the VolumeSnapshot object which will cascade delete the // VolumeSnapshotContent and the associated snapshot in the storage // provider (handled by the CSI driver and the CSI common controller). // However, in the event that the VolumeSnapshot object is deleted outside // of the backup deletion process, it is possible that the dynamically created // VolumeSnapshotContent object will be left as an orphaned and non-discoverable // resource in the cluster as well as in the storage provider. To avoid piling // up of such orphaned resources, we will want to discover and delete the // dynamically created VolumeSnapshotContent. We do that by adding // the "velero.io/backup-name" label on the VolumeSnapshotContent. // Further, we want to add this label only on VolumeSnapshotContents that // were created during an ongoing velero backup. originVSC := vsc.DeepCopy() kubeutil.AddLabels( &vsc.ObjectMeta, map[string]string{ velerov1api.BackupNameLabel: label.GetValidName(backup.Name), }, ) if vscPatchError := p.crClient.Patch( context.TODO(), vsc, crclient.MergeFrom(originVSC), ); vscPatchError != nil { p.log.Warnf("Failed to patch VolumeSnapshotContent %s: %v", vsc.Name, vscPatchError) } } // Before applying the BIA v2, the in-cluster VS state is not persisted into backup. // After the change, because the final state of VS will be stored in backup as the // result of async operation result, need to patch the annotations into VS to work, // because restore will check the annotations information. originVS := vs.DeepCopy() kubeutil.AddAnnotations(&vs.ObjectMeta, annotations) if err := p.crClient.Patch( context.TODO(), vs, crclient.MergeFrom(originVS), ); err != nil { p.log.Errorf("Fail to patch VolumeSnapshot: %s.", err.Error()) return nil, nil, "", nil, errors.WithStack(err) } annotations[velerov1api.MustIncludeAdditionalItemAnnotation] = "true" // save newly applied annotations into the backed-up VolumeSnapshot item kubeutil.AddAnnotations(&vs.ObjectMeta, annotations) vsMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(vs) if err != nil { return nil, nil, "", nil, errors.WithStack(err) } p.log.Infof("Returning from VolumeSnapshotBackupItemAction with %d additionalItems to backup", len(additionalItems)) for _, ai := range additionalItems { p.log.Debugf("%s: %s", ai.GroupResource.String(), ai.Name) } operationID := "" var itemToUpdate []velero.ResourceIdentifier // Only return Async operation for VSC created for this backup. // The operationID is of the form // operationID = vs.Namespace + "/" + vs.Name + "/" + time.Now().Format(time.RFC3339) itemToUpdate = []velero.ResourceIdentifier{ { GroupResource: kuberesource.VolumeSnapshots, Namespace: vs.Namespace, Name: vs.Name, }, { GroupResource: kuberesource.VolumeSnapshotContents, Name: vsc.Name, }, } return &unstructured.Unstructured{Object: vsMap}, additionalItems, operationID, itemToUpdate, nil } // Name returns the plugin's name. func (p *volumeSnapshotBackupItemAction) Name() string { return "VolumeSnapshotBackupItemAction" } func (p *volumeSnapshotBackupItemAction) Progress( operationID string, backup *velerov1api.Backup, ) (velero.OperationProgress, error) { progress := velero.OperationProgress{} if operationID == "" { return progress, biav2.InvalidOperationIDError(operationID) } // The operationID is of the form // operationIDParts := strings.Split(operationID, "/") if len(operationIDParts) != 3 { p.log.Errorf("invalid operation ID %s", operationID) return progress, biav2.InvalidOperationIDError(operationID) } var err error if progress.Started, err = time.Parse(time.RFC3339, operationIDParts[2]); err != nil { p.log.Errorf("error parsing operation ID's StartedTime", "part into time %s: %s", operationID, err.Error()) return progress, errors.WithStack(err) } vs := new(snapshotv1api.VolumeSnapshot) if err := p.crClient.Get( context.Background(), crclient.ObjectKey{ Namespace: operationIDParts[0], Name: operationIDParts[1], }, vs); err != nil { p.log.Errorf("error getting volumesnapshot %s/%s: %s", operationIDParts[0], operationIDParts[1], err.Error()) return progress, errors.WithStack(err) } if vs.Status == nil { p.log.Debugf("VolumeSnapshot %s/%s has an empty status. Skip progress update.", vs.Namespace, vs.Name) return progress, nil } if boolptr.IsSetToTrue(vs.Status.ReadyToUse) { p.log.Debugf("VolumeSnapshot %s/%s is ReadyToUse. Continue on querying corresponding VolumeSnapshotContent.", vs.Namespace, vs.Name) } else if vs.Status.Error != nil { errorMessage := "" if vs.Status.Error.Message != nil { errorMessage = *vs.Status.Error.Message } p.log.Warnf("VolumeSnapshot has a temporary error %s. Snapshot controller will retry later.", errorMessage) return progress, nil } if vs.Status != nil && vs.Status.BoundVolumeSnapshotContentName != nil { vsc := new(snapshotv1api.VolumeSnapshotContent) err := p.crClient.Get( context.Background(), crclient.ObjectKey{Name: *vs.Status.BoundVolumeSnapshotContentName}, vsc, ) if err != nil { p.log.Errorf("error getting VolumeSnapshotContent %s: %s", *vs.Status.BoundVolumeSnapshotContentName, err.Error()) return progress, errors.WithStack(err) } if vsc.Status == nil { p.log.Debugf("VolumeSnapshotContent %s has an empty Status. Skip progress update.", vsc.Name) return progress, nil } now := time.Now() if boolptr.IsSetToTrue(vsc.Status.ReadyToUse) { progress.Completed = true progress.Updated = now } else if vsc.Status.Error != nil { progress.Completed = true progress.Updated = now if vsc.Status.Error.Message != nil { progress.Err = *vsc.Status.Error.Message } p.log.Warnf("VolumeSnapshotContent meets an error %s.", progress.Err) } } return progress, nil } // Cancel is not implemented for VolumeSnapshotBackupItemAction func (p *volumeSnapshotBackupItemAction) Cancel( operationID string, backup *velerov1api.Backup, ) error { // CSI Specification doesn't support canceling a snapshot creation. return nil } // NewVolumeSnapshotBackupItemAction returns // VolumeSnapshotBackupItemAction instance. func NewVolumeSnapshotBackupItemAction( f client.Factory, ) plugincommon.HandlerInitializer { return func(logger logrus.FieldLogger) (any, error) { crClient, err := f.KubebuilderClient() if err != nil { return nil, errors.WithStack(err) } return &volumeSnapshotBackupItemAction{ log: logger, crClient: crClient, }, nil } } ================================================ FILE: pkg/backup/actions/csi/volumesnapshot_action_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package csi import ( "fmt" "testing" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" factorymocks "github.com/vmware-tanzu/velero/pkg/client/mocks" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/plugin/velero" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func TestVSExecute(t *testing.T) { snapshotHandle := "handle" tests := []struct { name string backup *velerov1api.Backup vs *snapshotv1api.VolumeSnapshot vsc *snapshotv1api.VolumeSnapshotContent expectedErr string expectedAdditionalItems []velero.ResourceIdentifier expectedItemToUpdate []velero.ResourceIdentifier }{ { name: "Normal case", backup: builder.ForBackup("velero", "backup"). Phase(velerov1api.BackupPhaseInProgress).Result(), vs: builder.ForVolumeSnapshot("velero", "vs"). ObjectMeta(builder.WithLabels( velerov1api.BackupNameLabel, "backup")). VolumeSnapshotClass("class").Status(). BoundVolumeSnapshotContentName("vsc").Result(), vsc: builder.ForVolumeSnapshotContent("vsc").Status( &snapshotv1api.VolumeSnapshotContentStatus{ SnapshotHandle: &snapshotHandle, }, ).Result(), expectedErr: "", expectedAdditionalItems: []velero.ResourceIdentifier{ { GroupResource: kuberesource.VolumeSnapshotClasses, Name: "class", }, { GroupResource: kuberesource.VolumeSnapshotContents, Name: "vsc", }, }, expectedItemToUpdate: []velero.ResourceIdentifier{ { GroupResource: kuberesource.VolumeSnapshots, Namespace: "velero", Name: "vs", }, { GroupResource: kuberesource.VolumeSnapshotContents, Name: "vsc", }, }, }, { name: "VS not have VSClass", backup: builder.ForBackup("velero", "backup"). Phase(velerov1api.BackupPhaseInProgress).Result(), vs: builder.ForVolumeSnapshot("velero", "vs"). ObjectMeta(builder.WithLabels( velerov1api.BackupNameLabel, "backup")). Status(). BoundVolumeSnapshotContentName("vsc").Result(), vsc: builder.ForVolumeSnapshotContent("vsc").Status( &snapshotv1api.VolumeSnapshotContentStatus{ SnapshotHandle: &snapshotHandle, }, ).Result(), expectedErr: "", expectedAdditionalItems: []velero.ResourceIdentifier{ { GroupResource: kuberesource.VolumeSnapshotContents, Name: "vsc", }, }, expectedItemToUpdate: []velero.ResourceIdentifier{ { GroupResource: kuberesource.VolumeSnapshots, Namespace: "velero", Name: "vs", }, { GroupResource: kuberesource.VolumeSnapshotContents, Name: "vsc", }, }, }, { name: "Backup in finalizing phase - skip VSC lookup", backup: builder.ForBackup("velero", "backup"). Phase(velerov1api.BackupPhaseFinalizing).Result(), vs: builder.ForVolumeSnapshot("velero", "vs"). ObjectMeta(builder.WithLabels( velerov1api.BackupNameLabel, "backup")). Status(). BoundVolumeSnapshotContentName("vsc").Result(), vsc: nil, // VSC won't be created/fetched expectedErr: "", expectedAdditionalItems: nil, expectedItemToUpdate: nil, }, { name: "Backup in finalizing partially failed phase - skip VSC lookup", backup: builder.ForBackup("velero", "backup"). Phase(velerov1api.BackupPhaseFinalizingPartiallyFailed).Result(), vs: builder.ForVolumeSnapshot("velero", "vs"). ObjectMeta(builder.WithLabels( velerov1api.BackupNameLabel, "backup")). Status(). BoundVolumeSnapshotContentName("vsc").Result(), vsc: nil, // VSC won't be created/fetched expectedErr: "", expectedAdditionalItems: nil, expectedItemToUpdate: nil, }, } for _, tc := range tests { t.Run(tc.name, func(*testing.T) { vsBIA := volumeSnapshotBackupItemAction{ log: logrus.New(), crClient: velerotest.NewFakeControllerRuntimeClient(t, tc.vs), } item, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.vs) require.NoError(t, err) if tc.vsc != nil { require.NoError(t, vsBIA.crClient.Create(t.Context(), tc.vsc)) } _, additionalItems, _, itemToUpdate, err := vsBIA.Execute(&unstructured.UnstructuredList{Object: item}, tc.backup) if tc.expectedErr == "" { require.NoError(t, err) } else { require.Equal(t, tc.expectedErr, err.Error()) } require.ElementsMatch(t, tc.expectedAdditionalItems, additionalItems) require.ElementsMatch(t, tc.expectedItemToUpdate, itemToUpdate) }) } } func TestVSProgress(t *testing.T) { errorStr := "error" readyToUse := true tests := []struct { name string backup *velerov1api.Backup vs *snapshotv1api.VolumeSnapshot vsc *snapshotv1api.VolumeSnapshotContent operationID string expectedErr bool expectedProgress *velero.OperationProgress }{ { name: "Empty OperationID", operationID: "", backup: builder.ForBackup("velero", "backup").Result(), expectedErr: true, }, { name: "OperationID doesn't have slash", operationID: "invalid", backup: builder.ForBackup("velero", "backup").Result(), expectedErr: true, }, { name: "OperationID doesn't have valid timestamp", operationID: "ns/name/invalid", backup: builder.ForBackup("velero", "backup").Result(), expectedErr: true, }, { name: "OperationID represents VS does not exist", operationID: "ns/name/2024-04-11T18:49:00+08:00", backup: builder.ForBackup("velero", "backup").Result(), expectedErr: true, }, { name: "VS status is nil", operationID: "ns/name/2024-04-11T18:49:00+08:00", vs: builder.ForVolumeSnapshot("ns", "name").Result(), backup: builder.ForBackup("velero", "backup").Result(), expectedErr: false, }, { name: "VS status has error", operationID: "ns/name/2024-04-11T18:49:00+08:00", vs: builder.ForVolumeSnapshot("ns", "name").Status(). StatusError(snapshotv1api.VolumeSnapshotError{ Message: &errorStr, }).Result(), backup: builder.ForBackup("velero", "backup").Result(), expectedErr: false, }, { name: "Fail to get VSC", operationID: "ns/name/2024-04-11T18:49:00+08:00", vs: builder.ForVolumeSnapshot("ns", "name").Status(). ReadyToUse(true).BoundVolumeSnapshotContentName("vsc").Result(), backup: builder.ForBackup("velero", "backup").Result(), expectedErr: true, }, { name: "VSC status is nil", operationID: "ns/name/2024-04-11T18:49:00+08:00", vs: builder.ForVolumeSnapshot("ns", "name").Status(). ReadyToUse(true).BoundVolumeSnapshotContentName("vsc").Result(), vsc: builder.ForVolumeSnapshotContent("vsc").Result(), backup: builder.ForBackup("velero", "backup").Result(), expectedErr: false, }, { name: "VSC is ReadyToUse", operationID: "ns/name/2024-04-11T18:49:00+08:00", vs: builder.ForVolumeSnapshot("ns", "name").Status(). ReadyToUse(true).BoundVolumeSnapshotContentName("vsc").Result(), vsc: builder.ForVolumeSnapshotContent("vsc"). Status(&snapshotv1api.VolumeSnapshotContentStatus{ ReadyToUse: &readyToUse, }).Result(), backup: builder.ForBackup("velero", "backup").Result(), expectedErr: false, expectedProgress: &velero.OperationProgress{Completed: true}, }, { name: "VSC status has error", operationID: "ns/name/2024-04-11T18:49:00+08:00", vs: builder.ForVolumeSnapshot("ns", "name").Status(). ReadyToUse(true).BoundVolumeSnapshotContentName("vsc").Result(), vsc: builder.ForVolumeSnapshotContent("vsc"). Status(&snapshotv1api.VolumeSnapshotContentStatus{ Error: &snapshotv1api.VolumeSnapshotError{ Message: &errorStr, }, }).Result(), backup: builder.ForBackup("velero", "backup").Result(), expectedErr: false, expectedProgress: &velero.OperationProgress{ Completed: true, Err: "error", }, }, } for _, tc := range tests { t.Run(tc.name, func(*testing.T) { crClient := velerotest.NewFakeControllerRuntimeClient(t) logger := logrus.New() vsBIA := volumeSnapshotBackupItemAction{ log: logger, crClient: crClient, } if tc.vs != nil { err := crClient.Create(t.Context(), tc.vs) require.NoError(t, err) } if tc.vsc != nil { require.NoError(t, crClient.Create(t.Context(), tc.vsc)) } progress, err := vsBIA.Progress(tc.operationID, tc.backup) if tc.expectedErr == false { require.NoError(t, err) } if tc.expectedProgress != nil { require.True( t, cmp.Equal( *tc.expectedProgress, progress, cmpopts.IgnoreFields( velero.OperationProgress{}, "Started", "Updated", ), ), ) } }) } } func TestVSAppliesTo(t *testing.T) { p := volumeSnapshotBackupItemAction{ log: logrus.StandardLogger(), } selector, err := p.AppliesTo() require.NoError(t, err) require.Equal( t, velero.ResourceSelector{ IncludedResources: []string{"volumesnapshots.snapshot.storage.k8s.io"}, }, selector, ) } func TestNewVolumeSnapshotBackupItemAction(t *testing.T) { logger := logrus.StandardLogger() crClient := velerotest.NewFakeControllerRuntimeClient(t) f := &factorymocks.Factory{} f.On("KubebuilderClient").Return(nil, fmt.Errorf("")) plugin := NewVolumeSnapshotBackupItemAction(f) _, err := plugin(logger) require.Error(t, err) f1 := &factorymocks.Factory{} f1.On("KubebuilderClient").Return(crClient, nil) plugin1 := NewVolumeSnapshotBackupItemAction(f1) _, err1 := plugin1(logger) require.NoError(t, err1) } ================================================ FILE: pkg/backup/actions/csi/volumesnapshotclass_action.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package csi import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/plugin/velero" csiutil "github.com/vmware-tanzu/velero/pkg/util/csi" kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube" ) // volumeSnapshotClassBackupItemAction is a backup item action plugin to // backup CSI VolumeSnapshotClass objects using Velero type volumeSnapshotClassBackupItemAction struct { log logrus.FieldLogger } // AppliesTo returns information indicating that the // VolumeSnapshotClassBackupItemAction action should be invoked // to backup VolumeSnapshotClass. func (p *volumeSnapshotClassBackupItemAction) AppliesTo() ( velero.ResourceSelector, error, ) { return velero.ResourceSelector{ IncludedResources: []string{"volumesnapshotclasses.snapshot.storage.k8s.io"}, }, nil } // Execute backs up a VolumeSnapshotClass object and returns as additional // items any snapshot lister secret that may be referenced in its annotations. func (p *volumeSnapshotClassBackupItemAction) Execute( item runtime.Unstructured, backup *velerov1api.Backup, ) ( runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error, ) { p.log.Infof("Executing VolumeSnapshotClassBackupItemAction") var snapClass snapshotv1api.VolumeSnapshotClass if err := runtime.DefaultUnstructuredConverter.FromUnstructured( item.UnstructuredContent(), &snapClass, ); err != nil { return nil, nil, "", nil, errors.WithStack(err) } additionalItems := []velero.ResourceIdentifier{} if csiutil.IsVolumeSnapshotClassHasListerSecret(&snapClass) { additionalItems = append(additionalItems, velero.ResourceIdentifier{ GroupResource: kuberesource.Secrets, Name: snapClass.Annotations[velerov1api.PrefixedListSecretNameAnnotation], Namespace: snapClass.Annotations[velerov1api.PrefixedListSecretNamespaceAnnotation], }) kubeutil.AddAnnotations(&snapClass.ObjectMeta, map[string]string{ velerov1api.MustIncludeAdditionalItemAnnotation: "true", }) } snapClassMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&snapClass) if err != nil { return nil, nil, "", nil, errors.WithStack(err) } p.log.Infof( "Returning from VolumeSnapshotClassBackupItemAction with %d additionalItems to backup", len(additionalItems), ) return &unstructured.Unstructured{Object: snapClassMap}, additionalItems, "", nil, nil } // Name returns the plugin's name. func (p *volumeSnapshotClassBackupItemAction) Name() string { return "VolumeSnapshotClassBackupItemAction" } // Progress is not implemented for VolumeSnapshotClassBackupItemAction func (p *volumeSnapshotClassBackupItemAction) Progress( operationID string, backup *velerov1api.Backup, ) (velero.OperationProgress, error) { return velero.OperationProgress{}, nil } // Cancel is not implemented for VolumeSnapshotClassBackupItemAction func (p *volumeSnapshotClassBackupItemAction) Cancel( operationID string, backup *velerov1api.Backup, ) error { return nil } // NewVolumeSnapshotClassBackupItemAction returns a // VolumeSnapshotClassBackupItemAction instance. func NewVolumeSnapshotClassBackupItemAction(logger logrus.FieldLogger) (any, error) { return &volumeSnapshotClassBackupItemAction{log: logger}, nil } ================================================ FILE: pkg/backup/actions/csi/volumesnapshotclass_action_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package csi import ( "testing" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/plugin/velero" ) func TestVSClassExecute(t *testing.T) { tests := []struct { name string item runtime.Unstructured vsClass *snapshotv1api.VolumeSnapshotClass backup *velerov1api.Backup expectErr bool expectedItems []velero.ResourceIdentifier }{ { name: "No Secret in the VS Class, no return additional items", vsClass: builder.ForVolumeSnapshotClass("test").Result(), backup: builder.ForBackup("velero", "backup").Result(), expectErr: false, }, { name: "Normal case, additional items should return", vsClass: builder.ForVolumeSnapshotClass("test").ObjectMeta(builder.WithAnnotationsMap( map[string]string{ velerov1api.PrefixedListSecretNameAnnotation: "name", velerov1api.PrefixedListSecretNamespaceAnnotation: "namespace", }, )).Result(), backup: builder.ForBackup("velero", "backup").Result(), expectErr: false, expectedItems: []velero.ResourceIdentifier{ { GroupResource: kuberesource.Secrets, Namespace: "namespace", Name: "name", }, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { p, err := NewVolumeSnapshotClassBackupItemAction(logrus.StandardLogger()) require.NoError(t, err) action := p.(*volumeSnapshotClassBackupItemAction) if test.vsClass != nil { vsMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(test.vsClass) require.NoError(t, err) test.item = &unstructured.Unstructured{Object: vsMap} } _, additionalItems, _, _, err := action.Execute( test.item, test.backup, ) if test.expectErr == false { require.NoError(t, err) } if len(test.expectedItems) > 0 { require.Equal(t, test.expectedItems, additionalItems) } }) } } func TestVSClassAppliesTo(t *testing.T) { p := volumeSnapshotClassBackupItemAction{ log: logrus.StandardLogger(), } selector, err := p.AppliesTo() require.NoError(t, err) require.Equal( t, velero.ResourceSelector{ IncludedResources: []string{"volumesnapshotclasses.snapshot.storage.k8s.io"}, }, selector, ) } ================================================ FILE: pkg/backup/actions/csi/volumesnapshotcontent_action.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package csi import ( "fmt" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/plugin/velero" csiutil "github.com/vmware-tanzu/velero/pkg/util/csi" kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube" ) // volumeSnapshotContentBackupItemAction is a backup item action plugin to backup // CSI VolumeSnapshotContent objects using Velero type volumeSnapshotContentBackupItemAction struct { log logrus.FieldLogger } // AppliesTo returns information indicating that the // VolumeSnapshotContentBackupItemAction action should be invoked to // backup VolumeSnapshotContents. func (p *volumeSnapshotContentBackupItemAction) AppliesTo() (velero.ResourceSelector, error) { return velero.ResourceSelector{ IncludedResources: []string{"volumesnapshotcontents.snapshot.storage.k8s.io"}, }, nil } // Execute returns the unmodified VolumeSnapshotContent object along // with the snapshot deletion secret, if any, from its annotation // as additional items to backup. func (p *volumeSnapshotContentBackupItemAction) Execute( item runtime.Unstructured, backup *velerov1api.Backup, ) ( runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error, ) { p.log.Infof("Executing VolumeSnapshotContentBackupItemAction") if backup.Status.Phase == velerov1api.BackupPhaseFinalizing || backup.Status.Phase == velerov1api.BackupPhaseFinalizingPartiallyFailed { p.log.WithField("Backup", fmt.Sprintf("%s/%s", backup.Namespace, backup.Name)). WithField("BackupPhase", backup.Status.Phase). Debug("Skipping VolumeSnapshotContentBackupItemAction", "as backup is in finalizing phase.") return item, nil, "", nil, nil } var snapCont snapshotv1api.VolumeSnapshotContent if err := runtime.DefaultUnstructuredConverter.FromUnstructured( item.UnstructuredContent(), &snapCont, ); err != nil { return nil, nil, "", nil, errors.WithStack(err) } additionalItems := []velero.ResourceIdentifier{} // we should backup the snapshot deletion secrets that may be referenced // in the VolumeSnapshotContent's annotation if csiutil.IsVolumeSnapshotContentHasDeleteSecret(&snapCont) { additionalItems = append( additionalItems, velero.ResourceIdentifier{ GroupResource: kuberesource.Secrets, Name: snapCont.Annotations[velerov1api.PrefixedSecretNameAnnotation], Namespace: snapCont.Annotations[velerov1api.PrefixedSecretNamespaceAnnotation], }) kubeutil.AddAnnotations(&snapCont.ObjectMeta, map[string]string{ velerov1api.MustIncludeAdditionalItemAnnotation: "true", }) } // Because async operation will update VolumeSnapshotContent during finalizing phase. // No matter what we do, VolumeSnapshotClass cannot be deleted. So skip it. // Just deleting VolumeSnapshotClass during restore and delete is enough. snapContMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&snapCont) if err != nil { return nil, nil, "", nil, errors.WithStack(err) } p.log.Infof( "Returning from VolumeSnapshotContentBackupItemAction", "with %d additionalItems to backup", len(additionalItems), ) return &unstructured.Unstructured{Object: snapContMap}, additionalItems, "", nil, nil } // Name returns the plugin's name. func (p *volumeSnapshotContentBackupItemAction) Name() string { return "VolumeSnapshotContentBackupItemAction" } // Progress is not implemented for VolumeSnapshotContentBackupItemAction. func (p *volumeSnapshotContentBackupItemAction) Progress( operationID string, backup *velerov1api.Backup, ) (velero.OperationProgress, error) { return velero.OperationProgress{}, nil } // Cancel is not implemented for VolumeSnapshotContentBackupItemAction. func (p *volumeSnapshotContentBackupItemAction) Cancel( operationID string, backup *velerov1api.Backup, ) error { // CSI Specification doesn't support canceling a snapshot creation. return nil } // NewVolumeSnapshotContentBackupItemAction returns a // VolumeSnapshotContentBackupItemAction instance. func NewVolumeSnapshotContentBackupItemAction( logger logrus.FieldLogger, ) (any, error) { return &volumeSnapshotContentBackupItemAction{log: logger}, nil } ================================================ FILE: pkg/backup/actions/csi/volumesnapshotcontent_action_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package csi import ( "testing" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/plugin/velero" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func TestVSCExecute(t *testing.T) { tests := []struct { name string item runtime.Unstructured vsc *snapshotv1api.VolumeSnapshotContent backup *velerov1api.Backup expectErr bool expectedItems []velero.ResourceIdentifier }{ { name: "Invalid VolumeSnapshotContent", item: velerotest.UnstructuredOrDie( ` { "apiVersion": "v1", "kind": "Pod", "metadata": { "namespace": "ns", "name": "foo" } } `, ), backup: builder.ForBackup("velero", "backup").Result(), expectErr: true, }, { name: "Normal case, additional items should return", vsc: builder.ForVolumeSnapshotContent("test").ObjectMeta(builder.WithAnnotationsMap( map[string]string{ velerov1api.PrefixedSecretNameAnnotation: "name", velerov1api.PrefixedSecretNamespaceAnnotation: "namespace", }, )).Result(), backup: builder.ForBackup("velero", "backup").Result(), expectErr: false, expectedItems: []velero.ResourceIdentifier{ { GroupResource: kuberesource.Secrets, Namespace: "namespace", Name: "name", }, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { p, err := NewVolumeSnapshotContentBackupItemAction(logrus.StandardLogger()) require.NoError(t, err) action := p.(*volumeSnapshotContentBackupItemAction) if test.vsc != nil { vsMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(test.vsc) require.NoError(t, err) test.item = &unstructured.Unstructured{Object: vsMap} } _, additionalItems, _, _, err := action.Execute( test.item, test.backup, ) if test.expectErr == false { require.NoError(t, err) } if len(test.expectedItems) > 0 { require.Equal(t, test.expectedItems, additionalItems) } }) } } func TestVSCAppliesTo(t *testing.T) { p := volumeSnapshotContentBackupItemAction{ log: logrus.StandardLogger(), } selector, err := p.AppliesTo() require.NoError(t, err) require.Equal( t, velero.ResourceSelector{ IncludedResources: []string{"volumesnapshotcontents.snapshot.storage.k8s.io"}, }, selector, ) } ================================================ FILE: pkg/backup/actions/pod_action.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actions import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/plugin/velero" "github.com/vmware-tanzu/velero/pkg/util/actionhelpers" ) // PodAction implements ItemAction. type PodAction struct { log logrus.FieldLogger } // NewPodAction creates a new ItemAction for pods. func NewPodAction(logger logrus.FieldLogger) *PodAction { return &PodAction{log: logger} } // AppliesTo returns a ResourceSelector that applies only to pods. func (a *PodAction) AppliesTo() (velero.ResourceSelector, error) { return velero.ResourceSelector{ IncludedResources: []string{"pods"}, }, nil } // Execute scans the pod's spec.volumes for persistentVolumeClaim volumes and returns a // ResourceIdentifier list containing references to all of the persistentVolumeClaim volumes used by // the pod. This ensures that when a pod is backed up, all referenced PVCs are backed up too. func (a *PodAction) Execute(item runtime.Unstructured, backup *v1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, error) { a.log.Info("Executing podAction") defer a.log.Info("Done executing podAction") pod := new(corev1api.Pod) if err := runtime.DefaultUnstructuredConverter.FromUnstructured(item.UnstructuredContent(), pod); err != nil { return nil, nil, errors.WithStack(err) } return item, actionhelpers.RelatedItemsForPod(pod, a.log), nil } ================================================ FILE: pkg/backup/actions/pod_action_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actions import ( "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/runtime" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/plugin/velero" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func TestPodActionAppliesTo(t *testing.T) { a := NewPodAction(velerotest.NewLogger()) actual, err := a.AppliesTo() require.NoError(t, err) expected := velero.ResourceSelector{ IncludedResources: []string{"pods"}, } assert.Equal(t, expected, actual) } func TestPodActionExecute(t *testing.T) { tests := []struct { name string pod runtime.Unstructured expected []velero.ResourceIdentifier }{ { name: "no spec.volumes", pod: velerotest.UnstructuredOrDie(` { "apiVersion": "v1", "kind": "Pod", "metadata": { "namespace": "foo", "name": "bar" } } `), }, { name: "persistentVolumeClaim without claimName", pod: velerotest.UnstructuredOrDie(` { "apiVersion": "v1", "kind": "Pod", "metadata": { "namespace": "foo", "name": "bar" }, "spec": { "volumes": [ { "persistentVolumeClaim": {} } ] } } `), }, { name: "full test, mix of volume types", pod: velerotest.UnstructuredOrDie(` { "apiVersion": "v1", "kind": "Pod", "metadata": { "namespace": "foo", "name": "bar" }, "spec": { "volumes": [ { "persistentVolumeClaim": {} }, { "emptyDir": {} }, { "persistentVolumeClaim": {"claimName": "claim1"} }, { "emptyDir": {} }, { "persistentVolumeClaim": {"claimName": "claim2"} } ] } } `), expected: []velero.ResourceIdentifier{ {GroupResource: kuberesource.PersistentVolumeClaims, Namespace: "foo", Name: "claim1"}, {GroupResource: kuberesource.PersistentVolumeClaims, Namespace: "foo", Name: "claim2"}, }, }, { name: "test priority class", pod: velerotest.UnstructuredOrDie(` { "apiVersion": "v1", "kind": "Pod", "metadata": { "namespace": "foo", "name": "bar" }, "spec": { "priorityClassName": "testPriorityClass" } } `), expected: []velero.ResourceIdentifier{ {GroupResource: kuberesource.PriorityClasses, Name: "testPriorityClass"}, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { a := NewPodAction(velerotest.NewLogger()) updated, additionalItems, err := a.Execute(test.pod, nil) require.NoError(t, err) assert.Equal(t, test.pod, updated) assert.Equal(t, test.expected, additionalItems) }) } } ================================================ FILE: pkg/backup/actions/remap_crd_version_action.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actions import ( "context" "encoding/json" "github.com/pkg/errors" "github.com/sirupsen/logrus" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" apiextv1beta1client "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" velerodiscovery "github.com/vmware-tanzu/velero/pkg/discovery" v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/plugin/velero" ) // RemapCRDVersionAction inspects CustomResourceDefinition and decides if it is a v1 // CRD that needs to be backed up as v1beta1. type RemapCRDVersionAction struct { logger logrus.FieldLogger betaCRDClient apiextv1beta1client.CustomResourceDefinitionInterface discoveryHelper velerodiscovery.Helper } // NewRemapCRDVersionAction instantiates a new RemapCRDVersionAction plugin. func NewRemapCRDVersionAction(logger logrus.FieldLogger, betaCRDClient apiextv1beta1client.CustomResourceDefinitionInterface, discoveryHelper velerodiscovery.Helper) *RemapCRDVersionAction { return &RemapCRDVersionAction{logger: logger, betaCRDClient: betaCRDClient, discoveryHelper: discoveryHelper} } // AppliesTo selects the resources the plugin should run against. In this case, CustomResourceDefinitions. func (a *RemapCRDVersionAction) AppliesTo() (velero.ResourceSelector, error) { return velero.ResourceSelector{ IncludedResources: []string{"customresourcedefinition.apiextensions.k8s.io"}, }, nil } // Execute executes logic necessary to check a CustomResourceDefinition and inspect it for characteristics that necessitate saving it as v1beta1 instead of v1. func (a *RemapCRDVersionAction) Execute(item runtime.Unstructured, backup *v1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, error) { a.logger.Info("Executing RemapCRDVersionAction") // This plugin is only relevant for CRDs retrieved from the v1 endpoint that were installed via the v1beta1 // endpoint, so we can exit immediately if the resource in question isn't v1. apiVersion, ok, err := unstructured.NestedString(item.UnstructuredContent(), "apiVersion") if err != nil { return nil, nil, errors.Wrap(err, "unable to read apiVersion from CRD") } if ok && apiVersion != "apiextensions.k8s.io/v1" { a.logger.Info("Exiting RemapCRDVersionAction, CRD is not v1") return item, nil, nil } // This plugin will exit if the CRD was installed via v1beta1 but the cluster does not support v1beta1 CRD supportv1b1 := false CheckVersion: for _, g := range a.discoveryHelper.APIGroups() { if g.Name == apiextv1.GroupName { for _, v := range g.Versions { if v.Version == apiextv1beta1.SchemeGroupVersion.Version { supportv1b1 = true break CheckVersion } } } } if !supportv1b1 { a.logger.Info("Exiting RemapCRDVersionAction, the cluster does not support v1beta1 CRD") return item, nil, nil } // We've got a v1 CRD and the cluster supports v1beta1 CRD, so proceed. var crd apiextv1.CustomResourceDefinition // Do not use runtime.DefaultUnstructuredConverter.FromUnstructured here because it has a bug when converting integers/whole // numbers in float fields (https://github.com/kubernetes/kubernetes/issues/87675). // Using JSON as a go-between avoids this issue, without adding a bunch of type conversion by using unstructured helper functions // to inspect the fields we want to look at. js, err := json.Marshal(item.UnstructuredContent()) if err != nil { return nil, nil, errors.Wrap(err, "unable to convert unstructured item to JSON") } if err = json.Unmarshal(js, &crd); err != nil { return nil, nil, errors.Wrap(err, "unable to convert JSON to CRD Go type") } log := a.logger.WithField("plugin", "RemapCRDVersionAction").WithField("CRD", crd.Name) switch { case hasSingleVersion(crd), hasNonStructuralSchema(crd), hasPreserveUnknownFields(crd): log.Infof("CustomResourceDefinition %s appears to be v1beta1, fetching the v1beta version", crd.Name) item, err = fetchV1beta1CRD(crd.Name, a.betaCRDClient) if err != nil { return nil, nil, err } default: log.Infof("CustomResourceDefinition %s does not appear to be v1beta1, backing up as v1", crd.Name) } return item, nil, nil } func fetchV1beta1CRD(name string, betaCRDClient apiextv1beta1client.CustomResourceDefinitionInterface) (*unstructured.Unstructured, error) { betaCRD, err := betaCRDClient.Get(context.TODO(), name, metav1.GetOptions{}) if err != nil { return nil, errors.Wrapf(err, "error fetching v1beta1 version of %s", name) } // Individual items fetched from the API don't always have the kind/API version set // See https://github.com/kubernetes/kubernetes/issues/3030. Unsure why this is happening here and not in main Velero; // probably has to do with List calls and Dynamic client vs typed client // Set these all the time, since they shouldn't ever be different, anyway betaCRD.Kind = "CustomResourceDefinition" betaCRD.APIVersion = apiextv1beta1.SchemeGroupVersion.String() m, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&betaCRD) if err != nil { return nil, errors.Wrapf(err, "error converting v1beta1 version of %s to unstructured", name) } item := &unstructured.Unstructured{Object: m} return item, nil } // hasPreserveUnknownFields determines whether or not a CRD is set to preserve unknown fields or not. func hasPreserveUnknownFields(crd apiextv1.CustomResourceDefinition) bool { return crd.Spec.PreserveUnknownFields } // hasNonStructuralSchema determines whether or not a CRD has had a nonstructural schema condition applied. func hasNonStructuralSchema(crd apiextv1.CustomResourceDefinition) bool { var ret bool for _, c := range crd.Status.Conditions { if c.Type == apiextv1.NonStructuralSchema { ret = true break } } return ret } // hasSingleVersion checks a CRD to see if it has a single version with no schema information. func hasSingleVersion(crd apiextv1.CustomResourceDefinition) bool { // Looking for 1 version should be enough to tell if it's a v1beta1 CRD, as all v1beta1 CRD versions share the same schema. // v1 CRDs can have different schemas per version // The silently upgraded versions will often have a `versions` entry that looks like this: // versions: // - name: v1 // served: true // storage: true // This is acceptable when re-submitted to a v1beta1 endpoint on restore. var ret bool if len(crd.Spec.Versions) > 0 { if crd.Spec.Versions[0].Schema == nil || crd.Spec.Versions[0].Schema.OpenAPIV3Schema == nil { ret = true } } return ret } ================================================ FILE: pkg/backup/actions/remap_crd_version_action_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actions import ( "encoding/json" "fmt" "os" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" apiextfakes "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/fake" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" velerodiscovery "github.com/vmware-tanzu/velero/pkg/discovery" v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func TestRemapCRDVersionAction(t *testing.T) { backup := &v1.Backup{} clientset := apiextfakes.NewSimpleClientset() betaClient := clientset.ApiextensionsV1beta1().CustomResourceDefinitions() // build a v1beta1 CRD with the same name and add it to the fake client that the plugin is going to call. // keep the same one for all 3 tests, since there's little value in recreating it b := builder.ForCustomResourceDefinitionV1Beta1("test.velero.io") c := b.Result() _, err := betaClient.Create(t.Context(), c, metav1.CreateOptions{}) require.NoError(t, err) a := NewRemapCRDVersionAction(velerotest.NewLogger(), betaClient, fakeDiscoveryHelper()) t.Run("Test a v1 CRD without any Schema information", func(t *testing.T) { b := builder.ForV1CustomResourceDefinition("test.velero.io") // Set a version that does not include and schema information. b.Version(builder.ForV1CustomResourceDefinitionVersion("v1").Served(true).Storage(true).Result()) c := b.Result() obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&c) require.NoError(t, err) // Execute the plugin, which will call the fake client item, _, err := a.Execute(&unstructured.Unstructured{Object: obj}, backup) require.NoError(t, err) assert.Equal(t, "apiextensions.k8s.io/v1beta1", item.UnstructuredContent()["apiVersion"]) }) t.Run("Test a v1 CRD with a NonStructuralSchema Condition", func(t *testing.T) { b := builder.ForV1CustomResourceDefinition("test.velero.io") b.Condition(builder.ForV1CustomResourceDefinitionCondition().Type(apiextv1.NonStructuralSchema).Result()) c := b.Result() obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&c) require.NoError(t, err) item, _, err := a.Execute(&unstructured.Unstructured{Object: obj}, backup) require.NoError(t, err) assert.Equal(t, "apiextensions.k8s.io/v1beta1", item.UnstructuredContent()["apiVersion"]) }) t.Run("Having an integer on a float64 field should work (issue 2319)", func(t *testing.T) { b := builder.ForV1CustomResourceDefinition("test.velero.io") // 5 here is just an int value, it could be any other whole number. schema := builder.ForJSONSchemaPropsBuilder().Maximum(5).Result() b.Version(builder.ForV1CustomResourceDefinitionVersion("v1").Served(true).Storage(true).Schema(schema).Result()) c := b.Result() // Marshall in and out of JSON because the problem doesn't manifest when we use ToUnstructured directly // This should simulate the JSON passing over the wire in an HTTP request/response with a dynamic client js, err := json.Marshal(c) require.NoError(t, err) var u unstructured.Unstructured err = json.Unmarshal(js, &u) require.NoError(t, err) _, _, err = a.Execute(&u, backup) require.NoError(t, err) }) t.Run("Having Spec.PreserveUnknownFields set to true will return a v1beta1 version of the CRD", func(t *testing.T) { b := builder.ForV1CustomResourceDefinition("test.velero.io") b.PreserveUnknownFields(true) c := b.Result() obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&c) require.NoError(t, err) item, _, err := a.Execute(&unstructured.Unstructured{Object: obj}, backup) require.NoError(t, err) assert.Equal(t, "apiextensions.k8s.io/v1beta1", item.UnstructuredContent()["apiVersion"]) }) t.Run("When the cluster only supports v1 CRD, v1 CRD will be returned even the input has Spec.PreserveUnknownFields set to true (issue 4080)", func(t *testing.T) { a.discoveryHelper = &velerotest.FakeDiscoveryHelper{ APIGroupsList: []metav1.APIGroup{ { Name: apiextv1.GroupName, Versions: []metav1.GroupVersionForDiscovery{ { Version: apiextv1.SchemeGroupVersion.Version, }, }, }, }, } b := builder.ForV1CustomResourceDefinition("test.velero.io") b.PreserveUnknownFields(true) c := b.Result() obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&c) require.NoError(t, err) item, _, err := a.Execute(&unstructured.Unstructured{Object: obj}, backup) require.NoError(t, err) assert.Equal(t, "apiextensions.k8s.io/v1", item.UnstructuredContent()["apiVersion"]) // set it back to the default one a.discoveryHelper = fakeDiscoveryHelper() }) } // TestRemapCRDVersionActionData tests the RemapCRDVersionAction plugin against actual CRD to confirm that the v1beta1 version is returned when the v1 version is passed in to the plugin. func TestRemapCRDVersionActionData(t *testing.T) { backup := &v1.Backup{} clientset := apiextfakes.NewSimpleClientset() betaClient := clientset.ApiextensionsV1beta1().CustomResourceDefinitions() a := NewRemapCRDVersionAction(velerotest.NewLogger(), betaClient, fakeDiscoveryHelper()) tests := []struct { crd string expectAdditionalColumns bool }{ { crd: "elasticsearches.elasticsearch.k8s.elastic.co", expectAdditionalColumns: true, }, { crd: "kibanas.kibana.k8s.elastic.co", expectAdditionalColumns: true, }, { crd: "gcpsamples.gcp.stacks.crossplane.io", }, { crd: "alertmanagers.monitoring.coreos.com", }, { crd: "prometheuses.monitoring.coreos.com", }, } for _, test := range tests { tName := fmt.Sprintf("%s CRD passed in as v1 should be returned as v1beta1", test.crd) t.Run(tName, func(t *testing.T) { // We don't need a Go struct of the v1 data, just an unstructured to pass into the plugin. v1File := fmt.Sprintf("testdata/v1/%s.json", test.crd) f, err := os.ReadFile(v1File) require.NoError(t, err) var obj unstructured.Unstructured err = json.Unmarshal(f, &obj) require.NoError(t, err) // Load a v1beta1 struct into the beta client to be returned v1beta1File := fmt.Sprintf("testdata/v1beta1/%s.json", test.crd) f, err = os.ReadFile(v1beta1File) require.NoError(t, err) var crd apiextv1beta1.CustomResourceDefinition err = json.Unmarshal(f, &crd) require.NoError(t, err) _, err = betaClient.Create(t.Context(), &crd, metav1.CreateOptions{}) require.NoError(t, err) // Run method under test item, _, err := a.Execute(&obj, backup) require.NoError(t, err) assert.Equal(t, "apiextensions.k8s.io/v1beta1", item.UnstructuredContent()["apiVersion"]) assert.Equal(t, crd.Kind, item.GetObjectKind().GroupVersionKind().GroupKind().Kind) name, _, err := unstructured.NestedString(item.UnstructuredContent(), "metadata", "name") require.NoError(t, err) assert.Equal(t, crd.Name, name) uid, _, err := unstructured.NestedString(item.UnstructuredContent(), "metadata", "uid") require.NoError(t, err) assert.Equal(t, string(crd.UID), uid) // For ElasticSearch and Kibana, problems manifested when additionalPrinterColumns was moved from the top-level spec down to the // versions slice. if test.expectAdditionalColumns { _, ok := item.UnstructuredContent()["spec"].(map[string]any)["additionalPrinterColumns"] assert.True(t, ok) } // Clean up the item created in the test. betaClient.Delete(t.Context(), crd.Name, metav1.DeleteOptions{}) }) } } func fakeDiscoveryHelper() velerodiscovery.Helper { return &velerotest.FakeDiscoveryHelper{ APIGroupsList: []metav1.APIGroup{ { Name: apiextv1.GroupName, Versions: []metav1.GroupVersionForDiscovery{ { Version: apiextv1beta1.SchemeGroupVersion.Version, }, { Version: apiextv1.SchemeGroupVersion.Version, }, }, }, }, } } ================================================ FILE: pkg/backup/actions/service_account_action.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actions import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerodiscovery "github.com/vmware-tanzu/velero/pkg/discovery" "github.com/vmware-tanzu/velero/pkg/plugin/velero" "github.com/vmware-tanzu/velero/pkg/util/actionhelpers" ) // ServiceAccountAction implements ItemAction. type ServiceAccountAction struct { log logrus.FieldLogger clusterRoleBindings []actionhelpers.ClusterRoleBinding } // NewServiceAccountAction creates a new ItemAction for service accounts. func NewServiceAccountAction(logger logrus.FieldLogger, clusterRoleBindingListers map[string]actionhelpers.ClusterRoleBindingLister, discoveryHelper velerodiscovery.Helper) (*ServiceAccountAction, error) { crbs, err := actionhelpers.ClusterRoleBindingsForAction(clusterRoleBindingListers, discoveryHelper) if err != nil { return nil, err } return &ServiceAccountAction{ log: logger, clusterRoleBindings: crbs, }, nil } // AppliesTo returns a ResourceSelector that applies only to service accounts. func (a *ServiceAccountAction) AppliesTo() (velero.ResourceSelector, error) { return velero.ResourceSelector{ IncludedResources: []string{"serviceaccounts"}, }, nil } // Execute checks for any ClusterRoleBindings that have this service account as a subject, and // adds the ClusterRoleBinding and associated ClusterRole to the list of additional items to // be backed up. func (a *ServiceAccountAction) Execute(item runtime.Unstructured, backup *v1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, error) { a.log.Info("Running ServiceAccountAction") defer a.log.Info("Done running ServiceAccountAction") objectMeta, err := meta.Accessor(item) if err != nil { return nil, nil, errors.WithStack(err) } return item, actionhelpers.RelatedItemsForServiceAccount(objectMeta, a.clusterRoleBindings, a.log), nil } ================================================ FILE: pkg/backup/actions/service_account_action_test.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actions import ( "fmt" "sort" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" rbacv1 "k8s.io/api/rbac/v1" rbacbeta "k8s.io/api/rbac/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/plugin/velero" velerotest "github.com/vmware-tanzu/velero/pkg/test" "github.com/vmware-tanzu/velero/pkg/util/actionhelpers" ) func newV1ClusterRoleBindingList(rbacCRBList []rbacv1.ClusterRoleBinding) []actionhelpers.ClusterRoleBinding { var crbs []actionhelpers.ClusterRoleBinding for _, c := range rbacCRBList { crbs = append(crbs, actionhelpers.V1ClusterRoleBinding{Crb: c}) } return crbs } func newV1beta1ClusterRoleBindingList(rbacCRBList []rbacbeta.ClusterRoleBinding) []actionhelpers.ClusterRoleBinding { var crbs []actionhelpers.ClusterRoleBinding for _, c := range rbacCRBList { crbs = append(crbs, actionhelpers.V1beta1ClusterRoleBinding{Crb: c}) } return crbs } type FakeV1ClusterRoleBindingLister struct { v1crbs []rbacv1.ClusterRoleBinding } func (f FakeV1ClusterRoleBindingLister) List() ([]actionhelpers.ClusterRoleBinding, error) { var crbs []actionhelpers.ClusterRoleBinding for _, c := range f.v1crbs { crbs = append(crbs, actionhelpers.V1ClusterRoleBinding{Crb: c}) } return crbs, nil } type FakeV1beta1ClusterRoleBindingLister struct { v1beta1crbs []rbacbeta.ClusterRoleBinding } func (f FakeV1beta1ClusterRoleBindingLister) List() ([]actionhelpers.ClusterRoleBinding, error) { var crbs []actionhelpers.ClusterRoleBinding for _, c := range f.v1beta1crbs { crbs = append(crbs, actionhelpers.V1beta1ClusterRoleBinding{Crb: c}) } return crbs, nil } func TestServiceAccountActionAppliesTo(t *testing.T) { // Instantiating the struct directly since using // NewServiceAccountAction requires a full Kubernetes clientset a := &ServiceAccountAction{} actual, err := a.AppliesTo() require.NoError(t, err) expected := velero.ResourceSelector{ IncludedResources: []string{"serviceaccounts"}, } assert.Equal(t, expected, actual) } func TestNewServiceAccountAction(t *testing.T) { tests := []struct { name string version string expectedCRBs []actionhelpers.ClusterRoleBinding }{ { name: "rbac v1 API instantiates an saAction", version: rbacv1.SchemeGroupVersion.Version, expectedCRBs: []actionhelpers.ClusterRoleBinding{ actionhelpers.V1ClusterRoleBinding{ Crb: rbacv1.ClusterRoleBinding{ ObjectMeta: metav1.ObjectMeta{ Name: "v1crb-1", }, }, }, actionhelpers.V1ClusterRoleBinding{ Crb: rbacv1.ClusterRoleBinding{ ObjectMeta: metav1.ObjectMeta{ Name: "v1crb-2", }, }, }, }, }, { name: "rbac v1beta1 API instantiates an saAction", version: rbacbeta.SchemeGroupVersion.Version, expectedCRBs: []actionhelpers.ClusterRoleBinding{ actionhelpers.V1beta1ClusterRoleBinding{ Crb: rbacbeta.ClusterRoleBinding{ ObjectMeta: metav1.ObjectMeta{ Name: "v1beta1crb-1", }, }, }, actionhelpers.V1beta1ClusterRoleBinding{ Crb: rbacbeta.ClusterRoleBinding{ ObjectMeta: metav1.ObjectMeta{ Name: "v1beta1crb-2", }, }, }, }, }, { name: "no RBAC API instantiates an saAction with empty slice", version: "", expectedCRBs: []actionhelpers.ClusterRoleBinding{}, }, } // Set up all of our fakes outside the test loop discoveryHelper := velerotest.FakeDiscoveryHelper{} logger := velerotest.NewLogger() v1crbs := []rbacv1.ClusterRoleBinding{ { ObjectMeta: metav1.ObjectMeta{ Name: "v1crb-1", }, }, { ObjectMeta: metav1.ObjectMeta{ Name: "v1crb-2", }, }, } v1beta1crbs := []rbacbeta.ClusterRoleBinding{ { ObjectMeta: metav1.ObjectMeta{ Name: "v1beta1crb-1", }, }, { ObjectMeta: metav1.ObjectMeta{ Name: "v1beta1crb-2", }, }, } clusterRoleBindingListers := map[string]actionhelpers.ClusterRoleBindingLister{ rbacv1.SchemeGroupVersion.Version: FakeV1ClusterRoleBindingLister{v1crbs: v1crbs}, rbacbeta.SchemeGroupVersion.Version: FakeV1beta1ClusterRoleBindingLister{v1beta1crbs: v1beta1crbs}, "": actionhelpers.NoopClusterRoleBindingLister{}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { // We only care about the preferred version, nothing else in the list discoveryHelper.APIGroupsList = []metav1.APIGroup{ { Name: rbacv1.GroupName, PreferredVersion: metav1.GroupVersionForDiscovery{ Version: test.version, }, }, } action, err := NewServiceAccountAction(logger, clusterRoleBindingListers, &discoveryHelper) require.NoError(t, err) assert.Equal(t, test.expectedCRBs, action.clusterRoleBindings) }) } } func TestServiceAccountActionExecute(t *testing.T) { tests := []struct { name string serviceAccount runtime.Unstructured crbs []rbacv1.ClusterRoleBinding expectedAdditionalItems []velero.ResourceIdentifier }{ { name: "no crbs", serviceAccount: velerotest.UnstructuredOrDie(` { "apiVersion": "v1", "kind": "ServiceAccount", "metadata": { "namespace": "velero", "name": "velero" } } `), crbs: nil, expectedAdditionalItems: nil, }, { name: "no matching crbs", serviceAccount: velerotest.UnstructuredOrDie(` { "apiVersion": "v1", "kind": "ServiceAccount", "metadata": { "namespace": "velero", "name": "velero" } } `), crbs: []rbacv1.ClusterRoleBinding{ { Subjects: []rbacv1.Subject{ { Kind: "non-matching-kind", Namespace: "non-matching-ns", Name: "non-matching-name", }, { Kind: "non-matching-kind", Namespace: "velero", Name: "velero", }, { Kind: rbacv1.ServiceAccountKind, Namespace: "non-matching-ns", Name: "velero", }, { Kind: rbacv1.ServiceAccountKind, Namespace: "velero", Name: "non-matching-name", }, }, RoleRef: rbacv1.RoleRef{ Name: "role", }, }, }, expectedAdditionalItems: nil, }, { name: "some matching crbs", serviceAccount: velerotest.UnstructuredOrDie(` { "apiVersion": "v1", "kind": "ServiceAccount", "metadata": { "namespace": "velero", "name": "velero" } } `), crbs: []rbacv1.ClusterRoleBinding{ { ObjectMeta: metav1.ObjectMeta{ Name: "crb-1", }, Subjects: []rbacv1.Subject{ { Kind: "non-matching-kind", Namespace: "non-matching-ns", Name: "non-matching-name", }, }, RoleRef: rbacv1.RoleRef{ Name: "role-1", }, }, { ObjectMeta: metav1.ObjectMeta{ Name: "crb-2", }, Subjects: []rbacv1.Subject{ { Kind: "non-matching-kind", Namespace: "non-matching-ns", Name: "non-matching-name", }, { Kind: rbacv1.ServiceAccountKind, Namespace: "velero", Name: "velero", }, }, RoleRef: rbacv1.RoleRef{ Name: "role-2", }, }, { ObjectMeta: metav1.ObjectMeta{ Name: "crb-3", }, Subjects: []rbacv1.Subject{ { Kind: rbacv1.ServiceAccountKind, Namespace: "velero", Name: "velero", }, }, RoleRef: rbacv1.RoleRef{ Name: "role-3", }, }, { ObjectMeta: metav1.ObjectMeta{ Name: "crb-4", }, Subjects: []rbacv1.Subject{ { Kind: rbacv1.ServiceAccountKind, Namespace: "velero", Name: "velero", }, { Kind: "non-matching-kind", Namespace: "non-matching-ns", Name: "non-matching-name", }, }, RoleRef: rbacv1.RoleRef{ Name: "role-4", }, }, }, expectedAdditionalItems: []velero.ResourceIdentifier{ { GroupResource: kuberesource.ClusterRoleBindings, Name: "crb-2", }, { GroupResource: kuberesource.ClusterRoleBindings, Name: "crb-3", }, { GroupResource: kuberesource.ClusterRoleBindings, Name: "crb-4", }, { GroupResource: kuberesource.ClusterRoles, Name: "role-2", }, { GroupResource: kuberesource.ClusterRoles, Name: "role-3", }, { GroupResource: kuberesource.ClusterRoles, Name: "role-4", }, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { // Create the action struct directly so we don't need to mock a clientset action := &ServiceAccountAction{ log: velerotest.NewLogger(), clusterRoleBindings: newV1ClusterRoleBindingList(test.crbs), } res, additional, err := action.Execute(test.serviceAccount, nil) assert.Equal(t, test.serviceAccount, res) require.NoError(t, err) // ensure slices are ordered for valid comparison sort.Slice(test.expectedAdditionalItems, func(i, j int) bool { return fmt.Sprintf("%s.%s", test.expectedAdditionalItems[i].GroupResource.String(), test.expectedAdditionalItems[i].Name) < fmt.Sprintf("%s.%s", test.expectedAdditionalItems[j].GroupResource.String(), test.expectedAdditionalItems[j].Name) }) sort.Slice(additional, func(i, j int) bool { return fmt.Sprintf("%s.%s", additional[i].GroupResource.String(), additional[i].Name) < fmt.Sprintf("%s.%s", additional[j].GroupResource.String(), additional[j].Name) }) assert.Equal(t, test.expectedAdditionalItems, additional) }) } } func TestServiceAccountActionExecuteOnBeta1(t *testing.T) { tests := []struct { name string serviceAccount runtime.Unstructured crbs []rbacbeta.ClusterRoleBinding expectedAdditionalItems []velero.ResourceIdentifier }{ { name: "no crbs", serviceAccount: velerotest.UnstructuredOrDie(` { "apiVersion": "v1", "kind": "ServiceAccount", "metadata": { "namespace": "velero", "name": "velero" } } `), crbs: nil, expectedAdditionalItems: nil, }, { name: "no matching crbs", serviceAccount: velerotest.UnstructuredOrDie(` { "apiVersion": "v1", "kind": "ServiceAccount", "metadata": { "namespace": "velero", "name": "velero" } } `), crbs: []rbacbeta.ClusterRoleBinding{ { Subjects: []rbacbeta.Subject{ { Kind: "non-matching-kind", Namespace: "non-matching-ns", Name: "non-matching-name", }, { Kind: "non-matching-kind", Namespace: "velero", Name: "velero", }, { Kind: rbacbeta.ServiceAccountKind, Namespace: "non-matching-ns", Name: "velero", }, { Kind: rbacbeta.ServiceAccountKind, Namespace: "velero", Name: "non-matching-name", }, }, RoleRef: rbacbeta.RoleRef{ Name: "role", }, }, }, expectedAdditionalItems: nil, }, { name: "some matching crbs", serviceAccount: velerotest.UnstructuredOrDie(` { "apiVersion": "v1", "kind": "ServiceAccount", "metadata": { "namespace": "velero", "name": "velero" } } `), crbs: []rbacbeta.ClusterRoleBinding{ { ObjectMeta: metav1.ObjectMeta{ Name: "crb-1", }, Subjects: []rbacbeta.Subject{ { Kind: "non-matching-kind", Namespace: "non-matching-ns", Name: "non-matching-name", }, }, RoleRef: rbacbeta.RoleRef{ Name: "role-1", }, }, { ObjectMeta: metav1.ObjectMeta{ Name: "crb-2", }, Subjects: []rbacbeta.Subject{ { Kind: "non-matching-kind", Namespace: "non-matching-ns", Name: "non-matching-name", }, { Kind: rbacbeta.ServiceAccountKind, Namespace: "velero", Name: "velero", }, }, RoleRef: rbacbeta.RoleRef{ Name: "role-2", }, }, { ObjectMeta: metav1.ObjectMeta{ Name: "crb-3", }, Subjects: []rbacbeta.Subject{ { Kind: rbacbeta.ServiceAccountKind, Namespace: "velero", Name: "velero", }, }, RoleRef: rbacbeta.RoleRef{ Name: "role-3", }, }, { ObjectMeta: metav1.ObjectMeta{ Name: "crb-4", }, Subjects: []rbacbeta.Subject{ { Kind: rbacbeta.ServiceAccountKind, Namespace: "velero", Name: "velero", }, { Kind: "non-matching-kind", Namespace: "non-matching-ns", Name: "non-matching-name", }, }, RoleRef: rbacbeta.RoleRef{ Name: "role-4", }, }, }, expectedAdditionalItems: []velero.ResourceIdentifier{ { GroupResource: kuberesource.ClusterRoleBindings, Name: "crb-2", }, { GroupResource: kuberesource.ClusterRoleBindings, Name: "crb-3", }, { GroupResource: kuberesource.ClusterRoleBindings, Name: "crb-4", }, { GroupResource: kuberesource.ClusterRoles, Name: "role-2", }, { GroupResource: kuberesource.ClusterRoles, Name: "role-3", }, { GroupResource: kuberesource.ClusterRoles, Name: "role-4", }, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { // Create the action struct directly so we don't need to mock a clientset action := &ServiceAccountAction{ log: velerotest.NewLogger(), clusterRoleBindings: newV1beta1ClusterRoleBindingList(test.crbs), } res, additional, err := action.Execute(test.serviceAccount, nil) assert.Equal(t, test.serviceAccount, res) require.NoError(t, err) // ensure slices are ordered for valid comparison sort.Slice(test.expectedAdditionalItems, func(i, j int) bool { return fmt.Sprintf("%s.%s", test.expectedAdditionalItems[i].GroupResource.String(), test.expectedAdditionalItems[i].Name) < fmt.Sprintf("%s.%s", test.expectedAdditionalItems[j].GroupResource.String(), test.expectedAdditionalItems[j].Name) }) sort.Slice(additional, func(i, j int) bool { return fmt.Sprintf("%s.%s", additional[i].GroupResource.String(), additional[i].Name) < fmt.Sprintf("%s.%s", additional[j].GroupResource.String(), additional[j].Name) }) assert.Equal(t, test.expectedAdditionalItems, additional) }) } } ================================================ FILE: pkg/backup/actions/testdata/v1/alertmanagers.monitoring.coreos.com.json ================================================ { "apiVersion": "apiextensions.k8s.io/v1", "kind": "CustomResourceDefinition", "metadata": { "annotations": { "helm.sh/hook": "crd-install", "kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"apiextensions.k8s.io/v1beta1\",\"kind\":\"CustomResourceDefinition\",\"metadata\":{\"annotations\":{\"helm.sh/hook\":\"crd-install\"},\"creationTimestamp\":null,\"labels\":{\"app\":\"prometheus-operator\"},\"name\":\"alertmanagers.monitoring.coreos.com\"},\"spec\":{\"group\":\"monitoring.coreos.com\",\"names\":{\"kind\":\"Alertmanager\",\"plural\":\"alertmanagers\"},\"scope\":\"Namespaced\",\"validation\":{\"openAPIV3Schema\":{\"properties\":{\"apiVersion\":{\"description\":\"APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources\",\"type\":\"string\"},\"kind\":{\"description\":\"Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds\",\"type\":\"string\"},\"spec\":{\"description\":\"AlertmanagerSpec is a specification of the desired behavior of the Alertmanager cluster. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/api-conventions.md#spec-and-status\",\"properties\":{\"additionalPeers\":{\"description\":\"AdditionalPeers allows injecting a set of additional Alertmanagers to peer with to form a highly available cluster.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"affinity\":{\"description\":\"Affinity is a group of affinity scheduling rules.\",\"properties\":{\"nodeAffinity\":{\"description\":\"Node affinity is a group of node affinity scheduling rules.\",\"properties\":{\"preferredDuringSchedulingIgnoredDuringExecution\":{\"description\":\"The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding \\\"weight\\\" to the sum if the node matches the corresponding matchExpressions; the node(s) with the highest sum are the most preferred.\",\"items\":{\"description\":\"An empty preferred scheduling term matches all objects with implicit weight 0 (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op).\",\"properties\":{\"preference\":{\"description\":\"A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm.\",\"properties\":{\"matchExpressions\":{\"description\":\"A list of node selector requirements by node's labels.\",\"items\":{\"description\":\"A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"The label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.\",\"type\":\"string\"},\"values\":{\"description\":\"An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"]},\"type\":\"array\"},\"matchFields\":{\"description\":\"A list of node selector requirements by node's fields.\",\"items\":{\"description\":\"A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"The label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.\",\"type\":\"string\"},\"values\":{\"description\":\"An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"]},\"type\":\"array\"}}},\"weight\":{\"description\":\"Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100.\",\"format\":\"int32\",\"type\":\"integer\"}},\"required\":[\"weight\",\"preference\"]},\"type\":\"array\"},\"requiredDuringSchedulingIgnoredDuringExecution\":{\"description\":\"A node selector represents the union of the results of one or more label queries over a set of nodes; that is, it represents the OR of the selectors represented by the node selector terms.\",\"properties\":{\"nodeSelectorTerms\":{\"description\":\"Required. A list of node selector terms. The terms are ORed.\",\"items\":{\"description\":\"A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm.\",\"properties\":{\"matchExpressions\":{\"description\":\"A list of node selector requirements by node's labels.\",\"items\":{\"description\":\"A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"The label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.\",\"type\":\"string\"},\"values\":{\"description\":\"An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"]},\"type\":\"array\"},\"matchFields\":{\"description\":\"A list of node selector requirements by node's fields.\",\"items\":{\"description\":\"A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"The label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.\",\"type\":\"string\"},\"values\":{\"description\":\"An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"]},\"type\":\"array\"}}},\"type\":\"array\"}},\"required\":[\"nodeSelectorTerms\"]}}},\"podAffinity\":{\"description\":\"Pod affinity is a group of inter pod affinity scheduling rules.\",\"properties\":{\"preferredDuringSchedulingIgnoredDuringExecution\":{\"description\":\"The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding \\\"weight\\\" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred.\",\"items\":{\"description\":\"The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s)\",\"properties\":{\"podAffinityTerm\":{\"description\":\"Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key \\u003ctopologyKey\\u003e matches that of any node on which a pod of the set of pods is running\",\"properties\":{\"labelSelector\":{\"description\":\"A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.\",\"properties\":{\"matchExpressions\":{\"description\":\"matchExpressions is a list of label selector requirements. The requirements are ANDed.\",\"items\":{\"description\":\"A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"key is the label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\",\"type\":\"string\"},\"values\":{\"description\":\"values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"]},\"type\":\"array\"},\"matchLabels\":{\"description\":\"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \\\"key\\\", the operator is \\\"In\\\", and the values array contains only \\\"value\\\". The requirements are ANDed.\",\"type\":\"object\"}}},\"namespaces\":{\"description\":\"namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \\\"this pod's namespace\\\"\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"topologyKey\":{\"description\":\"This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.\",\"type\":\"string\"}},\"required\":[\"topologyKey\"]},\"weight\":{\"description\":\"weight associated with matching the corresponding podAffinityTerm, in the range 1-100.\",\"format\":\"int32\",\"type\":\"integer\"}},\"required\":[\"weight\",\"podAffinityTerm\"]},\"type\":\"array\"},\"requiredDuringSchedulingIgnoredDuringExecution\":{\"description\":\"If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied.\",\"items\":{\"description\":\"Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key \\u003ctopologyKey\\u003e matches that of any node on which a pod of the set of pods is running\",\"properties\":{\"labelSelector\":{\"description\":\"A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.\",\"properties\":{\"matchExpressions\":{\"description\":\"matchExpressions is a list of label selector requirements. The requirements are ANDed.\",\"items\":{\"description\":\"A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"key is the label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\",\"type\":\"string\"},\"values\":{\"description\":\"values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"]},\"type\":\"array\"},\"matchLabels\":{\"description\":\"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \\\"key\\\", the operator is \\\"In\\\", and the values array contains only \\\"value\\\". The requirements are ANDed.\",\"type\":\"object\"}}},\"namespaces\":{\"description\":\"namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \\\"this pod's namespace\\\"\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"topologyKey\":{\"description\":\"This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.\",\"type\":\"string\"}},\"required\":[\"topologyKey\"]},\"type\":\"array\"}}},\"podAntiAffinity\":{\"description\":\"Pod anti affinity is a group of inter pod anti affinity scheduling rules.\",\"properties\":{\"preferredDuringSchedulingIgnoredDuringExecution\":{\"description\":\"The scheduler will prefer to schedule pods to nodes that satisfy the anti-affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding \\\"weight\\\" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred.\",\"items\":{\"description\":\"The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s)\",\"properties\":{\"podAffinityTerm\":{\"description\":\"Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key \\u003ctopologyKey\\u003e matches that of any node on which a pod of the set of pods is running\",\"properties\":{\"labelSelector\":{\"description\":\"A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.\",\"properties\":{\"matchExpressions\":{\"description\":\"matchExpressions is a list of label selector requirements. The requirements are ANDed.\",\"items\":{\"description\":\"A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"key is the label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\",\"type\":\"string\"},\"values\":{\"description\":\"values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"]},\"type\":\"array\"},\"matchLabels\":{\"description\":\"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \\\"key\\\", the operator is \\\"In\\\", and the values array contains only \\\"value\\\". The requirements are ANDed.\",\"type\":\"object\"}}},\"namespaces\":{\"description\":\"namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \\\"this pod's namespace\\\"\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"topologyKey\":{\"description\":\"This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.\",\"type\":\"string\"}},\"required\":[\"topologyKey\"]},\"weight\":{\"description\":\"weight associated with matching the corresponding podAffinityTerm, in the range 1-100.\",\"format\":\"int32\",\"type\":\"integer\"}},\"required\":[\"weight\",\"podAffinityTerm\"]},\"type\":\"array\"},\"requiredDuringSchedulingIgnoredDuringExecution\":{\"description\":\"If the anti-affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the anti-affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied.\",\"items\":{\"description\":\"Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key \\u003ctopologyKey\\u003e matches that of any node on which a pod of the set of pods is running\",\"properties\":{\"labelSelector\":{\"description\":\"A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.\",\"properties\":{\"matchExpressions\":{\"description\":\"matchExpressions is a list of label selector requirements. The requirements are ANDed.\",\"items\":{\"description\":\"A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"key is the label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\",\"type\":\"string\"},\"values\":{\"description\":\"values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"]},\"type\":\"array\"},\"matchLabels\":{\"description\":\"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \\\"key\\\", the operator is \\\"In\\\", and the values array contains only \\\"value\\\". The requirements are ANDed.\",\"type\":\"object\"}}},\"namespaces\":{\"description\":\"namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \\\"this pod's namespace\\\"\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"topologyKey\":{\"description\":\"This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.\",\"type\":\"string\"}},\"required\":[\"topologyKey\"]},\"type\":\"array\"}}}}},\"baseImage\":{\"description\":\"Base image that is used to deploy pods, without tag.\",\"type\":\"string\"},\"configMaps\":{\"description\":\"ConfigMaps is a list of ConfigMaps in the same namespace as the Alertmanager object, which shall be mounted into the Alertmanager Pods. The ConfigMaps are mounted into /etc/alertmanager/configmaps/\\u003cconfigmap-name\\u003e.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"configSecret\":{\"description\":\"ConfigSecret is the name of a Kubernetes Secret in the same namespace as the Alertmanager object, which contains configuration for this Alertmanager instance. Defaults to 'alertmanager-' The secret is mounted into /etc/alertmanager/config.\",\"type\":\"string\"},\"containers\":{\"description\":\"Containers allows injecting additional containers. This is meant to allow adding an authentication proxy to an Alertmanager pod.\",\"items\":{\"description\":\"A single application container that you want to run within a pod.\",\"properties\":{\"args\":{\"description\":\"Arguments to the entrypoint. The docker image's CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"command\":{\"description\":\"Entrypoint array. Not executed within a shell. The docker image's ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"env\":{\"description\":\"List of environment variables to set in the container. Cannot be updated.\",\"items\":{\"description\":\"EnvVar represents an environment variable present in a Container.\",\"properties\":{\"name\":{\"description\":\"Name of the environment variable. Must be a C_IDENTIFIER.\",\"type\":\"string\"},\"value\":{\"description\":\"Variable references $(VAR_NAME) are expanded using the previous defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Defaults to \\\"\\\".\",\"type\":\"string\"},\"valueFrom\":{\"description\":\"EnvVarSource represents a source for the value of an EnvVar.\",\"properties\":{\"configMapKeyRef\":{\"description\":\"Selects a key from a ConfigMap.\",\"properties\":{\"key\":{\"description\":\"The key to select.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the ConfigMap or it's key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"]},\"fieldRef\":{\"description\":\"ObjectFieldSelector selects an APIVersioned field of an object.\",\"properties\":{\"apiVersion\":{\"description\":\"Version of the schema the FieldPath is written in terms of, defaults to \\\"v1\\\".\",\"type\":\"string\"},\"fieldPath\":{\"description\":\"Path of the field to select in the specified API version.\",\"type\":\"string\"}},\"required\":[\"fieldPath\"]},\"resourceFieldRef\":{\"description\":\"ResourceFieldSelector represents container resources (cpu, memory) and their output format\",\"properties\":{\"containerName\":{\"description\":\"Container name: required for volumes, optional for env vars\",\"type\":\"string\"},\"divisor\":{},\"resource\":{\"description\":\"Required: resource to select\",\"type\":\"string\"}},\"required\":[\"resource\"]},\"secretKeyRef\":{\"description\":\"SecretKeySelector selects a key of a Secret.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or it's key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"]}}}},\"required\":[\"name\"]},\"type\":\"array\"},\"envFrom\":{\"description\":\"List of sources to populate environment variables in the container. The keys defined within a source must be a C_IDENTIFIER. All invalid keys will be reported as an event when the container is starting. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated.\",\"items\":{\"description\":\"EnvFromSource represents the source of a set of ConfigMaps\",\"properties\":{\"configMapRef\":{\"description\":\"ConfigMapEnvSource selects a ConfigMap to populate the environment variables with.\\nThe contents of the target ConfigMap's Data field will represent the key-value pairs as environment variables.\",\"properties\":{\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the ConfigMap must be defined\",\"type\":\"boolean\"}}},\"prefix\":{\"description\":\"An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER.\",\"type\":\"string\"},\"secretRef\":{\"description\":\"SecretEnvSource selects a Secret to populate the environment variables with.\\nThe contents of the target Secret's Data field will represent the key-value pairs as environment variables.\",\"properties\":{\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret must be defined\",\"type\":\"boolean\"}}}}},\"type\":\"array\"},\"image\":{\"description\":\"Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images This field is optional to allow higher level config management to default or override container images in workload controllers like Deployments and StatefulSets.\",\"type\":\"string\"},\"imagePullPolicy\":{\"description\":\"Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images\",\"type\":\"string\"},\"lifecycle\":{\"description\":\"Lifecycle describes actions that the management system should take in response to container lifecycle events. For the PostStart and PreStop lifecycle handlers, management of the container blocks until the action is complete, unless the container process fails, in which case the handler is aborted.\",\"properties\":{\"postStart\":{\"description\":\"Handler defines a specific action that should be taken\",\"properties\":{\"exec\":{\"description\":\"ExecAction describes a \\\"run in container\\\" action.\",\"properties\":{\"command\":{\"description\":\"Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}}},\"httpGet\":{\"description\":\"HTTPGetAction describes an action based on HTTP Get requests.\",\"properties\":{\"host\":{\"description\":\"Host name to connect to, defaults to the pod IP. You probably want to set \\\"Host\\\" in httpHeaders instead.\",\"type\":\"string\"},\"httpHeaders\":{\"description\":\"Custom headers to set in the request. HTTP allows repeated headers.\",\"items\":{\"description\":\"HTTPHeader describes a custom header to be used in HTTP probes\",\"properties\":{\"name\":{\"description\":\"The header field name\",\"type\":\"string\"},\"value\":{\"description\":\"The header field value\",\"type\":\"string\"}},\"required\":[\"name\",\"value\"]},\"type\":\"array\"},\"path\":{\"description\":\"Path to access on the HTTP server.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"string\"},{\"type\":\"integer\"}]},\"scheme\":{\"description\":\"Scheme to use for connecting to the host. Defaults to HTTP.\",\"type\":\"string\"}},\"required\":[\"port\"]},\"tcpSocket\":{\"description\":\"TCPSocketAction describes an action based on opening a socket\",\"properties\":{\"host\":{\"description\":\"Optional: Host name to connect to, defaults to the pod IP.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"string\"},{\"type\":\"integer\"}]}},\"required\":[\"port\"]}}},\"preStop\":{\"description\":\"Handler defines a specific action that should be taken\",\"properties\":{\"exec\":{\"description\":\"ExecAction describes a \\\"run in container\\\" action.\",\"properties\":{\"command\":{\"description\":\"Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}}},\"httpGet\":{\"description\":\"HTTPGetAction describes an action based on HTTP Get requests.\",\"properties\":{\"host\":{\"description\":\"Host name to connect to, defaults to the pod IP. You probably want to set \\\"Host\\\" in httpHeaders instead.\",\"type\":\"string\"},\"httpHeaders\":{\"description\":\"Custom headers to set in the request. HTTP allows repeated headers.\",\"items\":{\"description\":\"HTTPHeader describes a custom header to be used in HTTP probes\",\"properties\":{\"name\":{\"description\":\"The header field name\",\"type\":\"string\"},\"value\":{\"description\":\"The header field value\",\"type\":\"string\"}},\"required\":[\"name\",\"value\"]},\"type\":\"array\"},\"path\":{\"description\":\"Path to access on the HTTP server.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"string\"},{\"type\":\"integer\"}]},\"scheme\":{\"description\":\"Scheme to use for connecting to the host. Defaults to HTTP.\",\"type\":\"string\"}},\"required\":[\"port\"]},\"tcpSocket\":{\"description\":\"TCPSocketAction describes an action based on opening a socket\",\"properties\":{\"host\":{\"description\":\"Optional: Host name to connect to, defaults to the pod IP.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"string\"},{\"type\":\"integer\"}]}},\"required\":[\"port\"]}}}}},\"livenessProbe\":{\"description\":\"Probe describes a health check to be performed against a container to determine whether it is alive or ready to receive traffic.\",\"properties\":{\"exec\":{\"description\":\"ExecAction describes a \\\"run in container\\\" action.\",\"properties\":{\"command\":{\"description\":\"Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}}},\"failureThreshold\":{\"description\":\"Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.\",\"format\":\"int32\",\"type\":\"integer\"},\"httpGet\":{\"description\":\"HTTPGetAction describes an action based on HTTP Get requests.\",\"properties\":{\"host\":{\"description\":\"Host name to connect to, defaults to the pod IP. You probably want to set \\\"Host\\\" in httpHeaders instead.\",\"type\":\"string\"},\"httpHeaders\":{\"description\":\"Custom headers to set in the request. HTTP allows repeated headers.\",\"items\":{\"description\":\"HTTPHeader describes a custom header to be used in HTTP probes\",\"properties\":{\"name\":{\"description\":\"The header field name\",\"type\":\"string\"},\"value\":{\"description\":\"The header field value\",\"type\":\"string\"}},\"required\":[\"name\",\"value\"]},\"type\":\"array\"},\"path\":{\"description\":\"Path to access on the HTTP server.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"string\"},{\"type\":\"integer\"}]},\"scheme\":{\"description\":\"Scheme to use for connecting to the host. Defaults to HTTP.\",\"type\":\"string\"}},\"required\":[\"port\"]},\"initialDelaySeconds\":{\"description\":\"Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\",\"format\":\"int32\",\"type\":\"integer\"},\"periodSeconds\":{\"description\":\"How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.\",\"format\":\"int32\",\"type\":\"integer\"},\"successThreshold\":{\"description\":\"Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness. Minimum value is 1.\",\"format\":\"int32\",\"type\":\"integer\"},\"tcpSocket\":{\"description\":\"TCPSocketAction describes an action based on opening a socket\",\"properties\":{\"host\":{\"description\":\"Optional: Host name to connect to, defaults to the pod IP.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"string\"},{\"type\":\"integer\"}]}},\"required\":[\"port\"]},\"timeoutSeconds\":{\"description\":\"Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\",\"format\":\"int32\",\"type\":\"integer\"}}},\"name\":{\"description\":\"Name of the container specified as a DNS_LABEL. Each container in a pod must have a unique name (DNS_LABEL). Cannot be updated.\",\"type\":\"string\"},\"ports\":{\"description\":\"List of ports to expose from the container. Exposing a port here gives the system additional information about the network connections a container uses, but is primarily informational. Not specifying a port here DOES NOT prevent that port from being exposed. Any port which is listening on the default \\\"0.0.0.0\\\" address inside a container will be accessible from the network. Cannot be updated.\",\"items\":{\"description\":\"ContainerPort represents a network port in a single container.\",\"properties\":{\"containerPort\":{\"description\":\"Number of port to expose on the pod's IP address. This must be a valid port number, 0 \\u003c x \\u003c 65536.\",\"format\":\"int32\",\"type\":\"integer\"},\"hostIP\":{\"description\":\"What host IP to bind the external port to.\",\"type\":\"string\"},\"hostPort\":{\"description\":\"Number of port to expose on the host. If specified, this must be a valid port number, 0 \\u003c x \\u003c 65536. If HostNetwork is specified, this must match ContainerPort. Most containers do not need this.\",\"format\":\"int32\",\"type\":\"integer\"},\"name\":{\"description\":\"If specified, this must be an IANA_SVC_NAME and unique within the pod. Each named port in a pod must have a unique name. Name for the port that can be referred to by services.\",\"type\":\"string\"},\"protocol\":{\"description\":\"Protocol for port. Must be UDP, TCP, or SCTP. Defaults to \\\"TCP\\\".\",\"type\":\"string\"}},\"required\":[\"containerPort\"]},\"type\":\"array\"},\"readinessProbe\":{\"description\":\"Probe describes a health check to be performed against a container to determine whether it is alive or ready to receive traffic.\",\"properties\":{\"exec\":{\"description\":\"ExecAction describes a \\\"run in container\\\" action.\",\"properties\":{\"command\":{\"description\":\"Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}}},\"failureThreshold\":{\"description\":\"Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.\",\"format\":\"int32\",\"type\":\"integer\"},\"httpGet\":{\"description\":\"HTTPGetAction describes an action based on HTTP Get requests.\",\"properties\":{\"host\":{\"description\":\"Host name to connect to, defaults to the pod IP. You probably want to set \\\"Host\\\" in httpHeaders instead.\",\"type\":\"string\"},\"httpHeaders\":{\"description\":\"Custom headers to set in the request. HTTP allows repeated headers.\",\"items\":{\"description\":\"HTTPHeader describes a custom header to be used in HTTP probes\",\"properties\":{\"name\":{\"description\":\"The header field name\",\"type\":\"string\"},\"value\":{\"description\":\"The header field value\",\"type\":\"string\"}},\"required\":[\"name\",\"value\"]},\"type\":\"array\"},\"path\":{\"description\":\"Path to access on the HTTP server.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"string\"},{\"type\":\"integer\"}]},\"scheme\":{\"description\":\"Scheme to use for connecting to the host. Defaults to HTTP.\",\"type\":\"string\"}},\"required\":[\"port\"]},\"initialDelaySeconds\":{\"description\":\"Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\",\"format\":\"int32\",\"type\":\"integer\"},\"periodSeconds\":{\"description\":\"How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.\",\"format\":\"int32\",\"type\":\"integer\"},\"successThreshold\":{\"description\":\"Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness. Minimum value is 1.\",\"format\":\"int32\",\"type\":\"integer\"},\"tcpSocket\":{\"description\":\"TCPSocketAction describes an action based on opening a socket\",\"properties\":{\"host\":{\"description\":\"Optional: Host name to connect to, defaults to the pod IP.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"string\"},{\"type\":\"integer\"}]}},\"required\":[\"port\"]},\"timeoutSeconds\":{\"description\":\"Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\",\"format\":\"int32\",\"type\":\"integer\"}}},\"resources\":{\"description\":\"ResourceRequirements describes the compute resource requirements.\",\"properties\":{\"limits\":{\"description\":\"Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/\",\"type\":\"object\"},\"requests\":{\"description\":\"Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/\",\"type\":\"object\"}}},\"securityContext\":{\"description\":\"SecurityContext holds security configuration that will be applied to a container. Some fields are present in both SecurityContext and PodSecurityContext. When both are set, the values in SecurityContext take precedence.\",\"properties\":{\"allowPrivilegeEscalation\":{\"description\":\"AllowPrivilegeEscalation controls whether a process can gain more privileges than its parent process. This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged 2) has CAP_SYS_ADMIN\",\"type\":\"boolean\"},\"capabilities\":{\"description\":\"Adds and removes POSIX capabilities from running containers.\",\"properties\":{\"add\":{\"description\":\"Added capabilities\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"drop\":{\"description\":\"Removed capabilities\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}}},\"privileged\":{\"description\":\"Run container in privileged mode. Processes in privileged containers are essentially equivalent to root on the host. Defaults to false.\",\"type\":\"boolean\"},\"procMount\":{\"description\":\"procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled.\",\"type\":\"string\"},\"readOnlyRootFilesystem\":{\"description\":\"Whether this container has a read-only root filesystem. Default is false.\",\"type\":\"boolean\"},\"runAsGroup\":{\"description\":\"The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\",\"format\":\"int64\",\"type\":\"integer\"},\"runAsNonRoot\":{\"description\":\"Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\",\"type\":\"boolean\"},\"runAsUser\":{\"description\":\"The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\",\"format\":\"int64\",\"type\":\"integer\"},\"seLinuxOptions\":{\"description\":\"SELinuxOptions are the labels to be applied to the container\",\"properties\":{\"level\":{\"description\":\"Level is SELinux level label that applies to the container.\",\"type\":\"string\"},\"role\":{\"description\":\"Role is a SELinux role label that applies to the container.\",\"type\":\"string\"},\"type\":{\"description\":\"Type is a SELinux type label that applies to the container.\",\"type\":\"string\"},\"user\":{\"description\":\"User is a SELinux user label that applies to the container.\",\"type\":\"string\"}}}}},\"stdin\":{\"description\":\"Whether this container should allocate a buffer for stdin in the container runtime. If this is not set, reads from stdin in the container will always result in EOF. Default is false.\",\"type\":\"boolean\"},\"stdinOnce\":{\"description\":\"Whether the container runtime should close the stdin channel after it has been opened by a single attach. When stdin is true the stdin stream will remain open across multiple attach sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the first client attaches to stdin, and then remains open and accepts data until the client disconnects, at which time stdin is closed and remains closed until the container is restarted. If this flag is false, a container processes that reads from stdin will never receive an EOF. Default is false\",\"type\":\"boolean\"},\"terminationMessagePath\":{\"description\":\"Optional: Path at which the file to which the container's termination message will be written is mounted into the container's filesystem. Message written is intended to be brief final status, such as an assertion failure message. Will be truncated by the node if greater than 4096 bytes. The total message length across all containers will be limited to 12kb. Defaults to /dev/termination-log. Cannot be updated.\",\"type\":\"string\"},\"terminationMessagePolicy\":{\"description\":\"Indicate how the termination message should be populated. File will use the contents of terminationMessagePath to populate the container status message on both success and failure. FallbackToLogsOnError will use the last chunk of container log output if the termination message file is empty and the container exited with an error. The log output is limited to 2048 bytes or 80 lines, whichever is smaller. Defaults to File. Cannot be updated.\",\"type\":\"string\"},\"tty\":{\"description\":\"Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. Default is false.\",\"type\":\"boolean\"},\"volumeDevices\":{\"description\":\"volumeDevices is the list of block devices to be used by the container. This is a beta feature.\",\"items\":{\"description\":\"volumeDevice describes a mapping of a raw block device within a container.\",\"properties\":{\"devicePath\":{\"description\":\"devicePath is the path inside of the container that the device will be mapped to.\",\"type\":\"string\"},\"name\":{\"description\":\"name must match the name of a persistentVolumeClaim in the pod\",\"type\":\"string\"}},\"required\":[\"name\",\"devicePath\"]},\"type\":\"array\"},\"volumeMounts\":{\"description\":\"Pod volumes to mount into the container's filesystem. Cannot be updated.\",\"items\":{\"description\":\"VolumeMount describes a mounting of a Volume within a container.\",\"properties\":{\"mountPath\":{\"description\":\"Path within the container at which the volume should be mounted. Must not contain ':'.\",\"type\":\"string\"},\"mountPropagation\":{\"description\":\"mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10.\",\"type\":\"string\"},\"name\":{\"description\":\"This must match the Name of a Volume.\",\"type\":\"string\"},\"readOnly\":{\"description\":\"Mounted read-only if true, read-write otherwise (false or unspecified). Defaults to false.\",\"type\":\"boolean\"},\"subPath\":{\"description\":\"Path within the volume from which the container's volume should be mounted. Defaults to \\\"\\\" (volume's root).\",\"type\":\"string\"}},\"required\":[\"name\",\"mountPath\"]},\"type\":\"array\"},\"workingDir\":{\"description\":\"Container's working directory. If not specified, the container runtime's default will be used, which might be configured in the container image. Cannot be updated.\",\"type\":\"string\"}},\"required\":[\"name\"]},\"type\":\"array\"},\"externalUrl\":{\"description\":\"The external URL the Alertmanager instances will be available under. This is necessary to generate correct URLs. This is necessary if Alertmanager is not served from root of a DNS name.\",\"type\":\"string\"},\"image\":{\"description\":\"Image if specified has precedence over baseImage, tag and sha combinations. Specifying the version is still necessary to ensure the Prometheus Operator knows what version of Alertmanager is being configured.\",\"type\":\"string\"},\"imagePullSecrets\":{\"description\":\"An optional list of references to secrets in the same namespace to use for pulling prometheus and alertmanager images from registries see http://kubernetes.io/docs/user-guide/images#specifying-imagepullsecrets-on-a-pod\",\"items\":{\"description\":\"LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace.\",\"properties\":{\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\",\"type\":\"string\"}}},\"type\":\"array\"},\"listenLocal\":{\"description\":\"ListenLocal makes the Alertmanager server listen on loopback, so that it does not bind against the Pod IP. Note this is only for the Alertmanager UI, not the gossip communication.\",\"type\":\"boolean\"},\"logLevel\":{\"description\":\"Log level for Alertmanager to be configured with.\",\"type\":\"string\"},\"nodeSelector\":{\"description\":\"Define which Nodes the Pods are scheduled on.\",\"type\":\"object\"},\"paused\":{\"description\":\"If set to true all actions on the underlying managed objects are not goint to be performed, except for delete actions.\",\"type\":\"boolean\"},\"podMetadata\":{\"description\":\"ObjectMeta is metadata that all persisted resources must have, which includes all objects users must create.\",\"properties\":{\"annotations\":{\"description\":\"Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations\",\"type\":\"object\"},\"clusterName\":{\"description\":\"The name of the cluster which the object belongs to. This is used to distinguish resources with same name and namespace in different clusters. This field is not set anywhere right now and apiserver is going to ignore it if set in create or update request.\",\"type\":\"string\"},\"creationTimestamp\":{\"description\":\"Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.\",\"format\":\"date-time\",\"type\":\"string\"},\"deletionGracePeriodSeconds\":{\"description\":\"Number of seconds allowed for this object to gracefully terminate before it will be removed from the system. Only set when deletionTimestamp is also set. May only be shortened. Read-only.\",\"format\":\"int64\",\"type\":\"integer\"},\"deletionTimestamp\":{\"description\":\"Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.\",\"format\":\"date-time\",\"type\":\"string\"},\"finalizers\":{\"description\":\"Must be empty before the object is deleted from the registry. Each entry is an identifier for the responsible component that will remove the entry from the list. If the deletionTimestamp of the object is non-nil, entries in this list can only be removed.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"generateName\":{\"description\":\"GenerateName is an optional prefix, used by the server, to generate a unique name ONLY IF the Name field has not been provided. If this field is used, the name returned to the client will be different than the name passed. This value will also be combined with a unique suffix. The provided value has the same validation rules as the Name field, and may be truncated by the length of the suffix required to make the value unique on the server.\\nIf this field is specified and the generated name exists, the server will NOT return a 409 - instead, it will either return 201 Created or 500 with Reason ServerTimeout indicating a unique name could not be found in the time allotted, and the client should retry (optionally after the time indicated in the Retry-After header).\\nApplied only if Name is not specified. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#idempotency\",\"type\":\"string\"},\"generation\":{\"description\":\"A sequence number representing a specific generation of the desired state. Populated by the system. Read-only.\",\"format\":\"int64\",\"type\":\"integer\"},\"initializers\":{\"description\":\"Initializers tracks the progress of initialization.\",\"properties\":{\"pending\":{\"description\":\"Pending is a list of initializers that must execute in order before this object is visible. When the last pending initializer is removed, and no failing result is set, the initializers struct will be set to nil and the object is considered as initialized and visible to all clients.\",\"items\":{\"description\":\"Initializer is information about an initializer that has not yet completed.\",\"properties\":{\"name\":{\"description\":\"name of the process that is responsible for initializing this object.\",\"type\":\"string\"}},\"required\":[\"name\"]},\"type\":\"array\"},\"result\":{\"description\":\"Status is a return value for calls that don't return other objects.\",\"properties\":{\"apiVersion\":{\"description\":\"APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources\",\"type\":\"string\"},\"code\":{\"description\":\"Suggested HTTP return code for this status, 0 if not set.\",\"format\":\"int32\",\"type\":\"integer\"},\"details\":{\"description\":\"StatusDetails is a set of additional properties that MAY be set by the server to provide additional information about a response. The Reason field of a Status object defines what attributes will be set. Clients must ignore fields that do not match the defined type of each attribute, and should assume that any attribute may be empty, invalid, or under defined.\",\"properties\":{\"causes\":{\"description\":\"The Causes array includes more details associated with the StatusReason failure. Not all StatusReasons may provide detailed causes.\",\"items\":{\"description\":\"StatusCause provides more information about an api.Status failure, including cases when multiple errors are encountered.\",\"properties\":{\"field\":{\"description\":\"The field of the resource that has caused this error, as named by its JSON serialization. May include dot and postfix notation for nested attributes. Arrays are zero-indexed. Fields may appear more than once in an array of causes due to fields having multiple errors. Optional.\\nExamples:\\n \\\"name\\\" - the field \\\"name\\\" on the current resource\\n \\\"items[0].name\\\" - the field \\\"name\\\" on the first array entry in \\\"items\\\"\",\"type\":\"string\"},\"message\":{\"description\":\"A human-readable description of the cause of the error. This field may be presented as-is to a reader.\",\"type\":\"string\"},\"reason\":{\"description\":\"A machine-readable description of the cause of the error. If this value is empty there is no information available.\",\"type\":\"string\"}}},\"type\":\"array\"},\"group\":{\"description\":\"The group attribute of the resource associated with the status StatusReason.\",\"type\":\"string\"},\"kind\":{\"description\":\"The kind attribute of the resource associated with the status StatusReason. On some operations may differ from the requested resource Kind. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds\",\"type\":\"string\"},\"name\":{\"description\":\"The name attribute of the resource associated with the status StatusReason (when there is a single name which can be described).\",\"type\":\"string\"},\"retryAfterSeconds\":{\"description\":\"If specified, the time in seconds before the operation should be retried. Some errors may indicate the client must take an alternate action - for those errors this field may indicate how long to wait before taking the alternate action.\",\"format\":\"int32\",\"type\":\"integer\"},\"uid\":{\"description\":\"UID of the resource. (when there is a single resource which can be described). More info: http://kubernetes.io/docs/user-guide/identifiers#uids\",\"type\":\"string\"}}},\"kind\":{\"description\":\"Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds\",\"type\":\"string\"},\"message\":{\"description\":\"A human-readable description of the status of this operation.\",\"type\":\"string\"},\"metadata\":{\"description\":\"ListMeta describes metadata that synthetic resources must have, including lists and various status objects. A resource may have only one of {ObjectMeta, ListMeta}.\",\"properties\":{\"continue\":{\"description\":\"continue may be set if the user set a limit on the number of items returned, and indicates that the server has more data available. The value is opaque and may be used to issue another request to the endpoint that served this list to retrieve the next set of available objects. Continuing a consistent list may not be possible if the server configuration has changed or more than a few minutes have passed. The resourceVersion field returned when using this continue value will be identical to the value in the first response, unless you have received this token from an error message.\",\"type\":\"string\"},\"resourceVersion\":{\"description\":\"String that identifies the server's internal version of this object that can be used by clients to determine when objects have changed. Value must be treated as opaque by clients and passed unmodified back to the server. Populated by the system. Read-only. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency\",\"type\":\"string\"},\"selfLink\":{\"description\":\"selfLink is a URL representing this object. Populated by the system. Read-only.\",\"type\":\"string\"}}},\"reason\":{\"description\":\"A machine-readable description of why this operation is in the \\\"Failure\\\" status. If this value is empty there is no information available. A Reason clarifies an HTTP status code but does not override it.\",\"type\":\"string\"},\"status\":{\"description\":\"Status of the operation. One of: \\\"Success\\\" or \\\"Failure\\\". More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status\",\"type\":\"string\"}}}},\"required\":[\"pending\"]},\"labels\":{\"description\":\"Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels\",\"type\":\"object\"},\"name\":{\"description\":\"Name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names\",\"type\":\"string\"},\"namespace\":{\"description\":\"Namespace defines the space within each name must be unique. An empty namespace is equivalent to the \\\"default\\\" namespace, but \\\"default\\\" is the canonical representation. Not all objects are required to be scoped to a namespace - the value of this field for those objects will be empty.\\nMust be a DNS_LABEL. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/namespaces\",\"type\":\"string\"},\"ownerReferences\":{\"description\":\"List of objects depended by this object. If ALL objects in the list have been deleted, this object will be garbage collected. If this object is managed by a controller, then an entry in this list will point to this controller, with the controller field set to true. There cannot be more than one managing controller.\",\"items\":{\"description\":\"OwnerReference contains enough information to let you identify an owning object. An owning object must be in the same namespace as the dependent, or be cluster-scoped, so there is no namespace field.\",\"properties\":{\"apiVersion\":{\"description\":\"API version of the referent.\",\"type\":\"string\"},\"blockOwnerDeletion\":{\"description\":\"If true, AND if the owner has the \\\"foregroundDeletion\\\" finalizer, then the owner cannot be deleted from the key-value store until this reference is removed. Defaults to false. To set this field, a user needs \\\"delete\\\" permission of the owner, otherwise 422 (Unprocessable Entity) will be returned.\",\"type\":\"boolean\"},\"controller\":{\"description\":\"If true, this reference points to the managing controller.\",\"type\":\"boolean\"},\"kind\":{\"description\":\"Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#names\",\"type\":\"string\"},\"uid\":{\"description\":\"UID of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#uids\",\"type\":\"string\"}},\"required\":[\"apiVersion\",\"kind\",\"name\",\"uid\"]},\"type\":\"array\"},\"resourceVersion\":{\"description\":\"An opaque value that represents the internal version of this object that can be used by clients to determine when objects have changed. May be used for optimistic concurrency, change detection, and the watch operation on a resource or set of resources. Clients must treat these values as opaque and passed unmodified back to the server. They may only be valid for a particular resource or set of resources.\\nPopulated by the system. Read-only. Value must be treated as opaque by clients and . More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency\",\"type\":\"string\"},\"selfLink\":{\"description\":\"SelfLink is a URL representing this object. Populated by the system. Read-only.\",\"type\":\"string\"},\"uid\":{\"description\":\"UID is the unique in time and space value for this object. It is typically generated by the server on successful creation of a resource and is not allowed to change on PUT operations.\\nPopulated by the system. Read-only. More info: http://kubernetes.io/docs/user-guide/identifiers#uids\",\"type\":\"string\"}}},\"priorityClassName\":{\"description\":\"Priority class assigned to the Pods\",\"type\":\"string\"},\"replicas\":{\"description\":\"Size is the expected size of the alertmanager cluster. The controller will eventually make the size of the running cluster equal to the expected size.\",\"format\":\"int32\",\"type\":\"integer\"},\"resources\":{\"description\":\"ResourceRequirements describes the compute resource requirements.\",\"properties\":{\"limits\":{\"description\":\"Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/\",\"type\":\"object\"},\"requests\":{\"description\":\"Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/\",\"type\":\"object\"}}},\"retention\":{\"description\":\"Time duration Alertmanager shall retain data for. Default is '120h', and must match the regular expression `[0-9]+(ms|s|m|h)` (milliseconds seconds minutes hours).\",\"type\":\"string\"},\"routePrefix\":{\"description\":\"The route prefix Alertmanager registers HTTP handlers for. This is useful, if using ExternalURL and a proxy is rewriting HTTP routes of a request, and the actual ExternalURL is still true, but the server serves requests under a different route prefix. For example for use with `kubectl proxy`.\",\"type\":\"string\"},\"secrets\":{\"description\":\"Secrets is a list of Secrets in the same namespace as the Alertmanager object, which shall be mounted into the Alertmanager Pods. The Secrets are mounted into /etc/alertmanager/secrets/\\u003csecret-name\\u003e.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"securityContext\":{\"description\":\"PodSecurityContext holds pod-level security attributes and common container settings. Some fields are also present in container.securityContext. Field values of container.securityContext take precedence over field values of PodSecurityContext.\",\"properties\":{\"fsGroup\":{\"description\":\"A special supplemental group that applies to all containers in a pod. Some volume types allow the Kubelet to change the ownership of that volume to be owned by the pod:\\n1. The owning GID will be the FSGroup 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) 3. The permission bits are OR'd with rw-rw----\\nIf unset, the Kubelet will not modify the ownership and permissions of any volume.\",\"format\":\"int64\",\"type\":\"integer\"},\"runAsGroup\":{\"description\":\"The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container.\",\"format\":\"int64\",\"type\":\"integer\"},\"runAsNonRoot\":{\"description\":\"Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\",\"type\":\"boolean\"},\"runAsUser\":{\"description\":\"The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container.\",\"format\":\"int64\",\"type\":\"integer\"},\"seLinuxOptions\":{\"description\":\"SELinuxOptions are the labels to be applied to the container\",\"properties\":{\"level\":{\"description\":\"Level is SELinux level label that applies to the container.\",\"type\":\"string\"},\"role\":{\"description\":\"Role is a SELinux role label that applies to the container.\",\"type\":\"string\"},\"type\":{\"description\":\"Type is a SELinux type label that applies to the container.\",\"type\":\"string\"},\"user\":{\"description\":\"User is a SELinux user label that applies to the container.\",\"type\":\"string\"}}},\"supplementalGroups\":{\"description\":\"A list of groups applied to the first process run in each container, in addition to the container's primary GID. If unspecified, no groups will be added to any container.\",\"items\":{\"format\":\"int64\",\"type\":\"integer\"},\"type\":\"array\"},\"sysctls\":{\"description\":\"Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported sysctls (by the container runtime) might fail to launch.\",\"items\":{\"description\":\"Sysctl defines a kernel parameter to be set\",\"properties\":{\"name\":{\"description\":\"Name of a property to set\",\"type\":\"string\"},\"value\":{\"description\":\"Value of a property to set\",\"type\":\"string\"}},\"required\":[\"name\",\"value\"]},\"type\":\"array\"}}},\"serviceAccountName\":{\"description\":\"ServiceAccountName is the name of the ServiceAccount to use to run the Prometheus Pods.\",\"type\":\"string\"},\"sha\":{\"description\":\"SHA of Alertmanager container image to be deployed. Defaults to the value of `version`. Similar to a tag, but the SHA explicitly deploys an immutable container image. Version and Tag are ignored if SHA is set.\",\"type\":\"string\"},\"storage\":{\"description\":\"StorageSpec defines the configured storage for a group Prometheus servers. If neither `emptyDir` nor `volumeClaimTemplate` is specified, then by default an [EmptyDir](https://kubernetes.io/docs/concepts/storage/volumes/#emptydir) will be used.\",\"properties\":{\"emptyDir\":{\"description\":\"Represents an empty directory for a pod. Empty directory volumes support ownership management and SELinux relabeling.\",\"properties\":{\"medium\":{\"description\":\"What type of storage medium should back this directory. The default is \\\"\\\" which means to use the node's default medium. Must be an empty string (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir\",\"type\":\"string\"},\"sizeLimit\":{}}},\"volumeClaimTemplate\":{\"description\":\"PersistentVolumeClaim is a user's request for and claim to a persistent volume\",\"properties\":{\"apiVersion\":{\"description\":\"APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources\",\"type\":\"string\"},\"kind\":{\"description\":\"Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds\",\"type\":\"string\"},\"metadata\":{\"description\":\"ObjectMeta is metadata that all persisted resources must have, which includes all objects users must create.\",\"properties\":{\"annotations\":{\"description\":\"Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations\",\"type\":\"object\"},\"clusterName\":{\"description\":\"The name of the cluster which the object belongs to. This is used to distinguish resources with same name and namespace in different clusters. This field is not set anywhere right now and apiserver is going to ignore it if set in create or update request.\",\"type\":\"string\"},\"creationTimestamp\":{\"description\":\"Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.\",\"format\":\"date-time\",\"type\":\"string\"},\"deletionGracePeriodSeconds\":{\"description\":\"Number of seconds allowed for this object to gracefully terminate before it will be removed from the system. Only set when deletionTimestamp is also set. May only be shortened. Read-only.\",\"format\":\"int64\",\"type\":\"integer\"},\"deletionTimestamp\":{\"description\":\"Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.\",\"format\":\"date-time\",\"type\":\"string\"},\"finalizers\":{\"description\":\"Must be empty before the object is deleted from the registry. Each entry is an identifier for the responsible component that will remove the entry from the list. If the deletionTimestamp of the object is non-nil, entries in this list can only be removed.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"generateName\":{\"description\":\"GenerateName is an optional prefix, used by the server, to generate a unique name ONLY IF the Name field has not been provided. If this field is used, the name returned to the client will be different than the name passed. This value will also be combined with a unique suffix. The provided value has the same validation rules as the Name field, and may be truncated by the length of the suffix required to make the value unique on the server.\\nIf this field is specified and the generated name exists, the server will NOT return a 409 - instead, it will either return 201 Created or 500 with Reason ServerTimeout indicating a unique name could not be found in the time allotted, and the client should retry (optionally after the time indicated in the Retry-After header).\\nApplied only if Name is not specified. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#idempotency\",\"type\":\"string\"},\"generation\":{\"description\":\"A sequence number representing a specific generation of the desired state. Populated by the system. Read-only.\",\"format\":\"int64\",\"type\":\"integer\"},\"initializers\":{\"description\":\"Initializers tracks the progress of initialization.\",\"properties\":{\"pending\":{\"description\":\"Pending is a list of initializers that must execute in order before this object is visible. When the last pending initializer is removed, and no failing result is set, the initializers struct will be set to nil and the object is considered as initialized and visible to all clients.\",\"items\":{\"description\":\"Initializer is information about an initializer that has not yet completed.\",\"properties\":{\"name\":{\"description\":\"name of the process that is responsible for initializing this object.\",\"type\":\"string\"}},\"required\":[\"name\"]},\"type\":\"array\"},\"result\":{\"description\":\"Status is a return value for calls that don't return other objects.\",\"properties\":{\"apiVersion\":{\"description\":\"APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources\",\"type\":\"string\"},\"code\":{\"description\":\"Suggested HTTP return code for this status, 0 if not set.\",\"format\":\"int32\",\"type\":\"integer\"},\"details\":{\"description\":\"StatusDetails is a set of additional properties that MAY be set by the server to provide additional information about a response. The Reason field of a Status object defines what attributes will be set. Clients must ignore fields that do not match the defined type of each attribute, and should assume that any attribute may be empty, invalid, or under defined.\",\"properties\":{\"causes\":{\"description\":\"The Causes array includes more details associated with the StatusReason failure. Not all StatusReasons may provide detailed causes.\",\"items\":{\"description\":\"StatusCause provides more information about an api.Status failure, including cases when multiple errors are encountered.\",\"properties\":{\"field\":{\"description\":\"The field of the resource that has caused this error, as named by its JSON serialization. May include dot and postfix notation for nested attributes. Arrays are zero-indexed. Fields may appear more than once in an array of causes due to fields having multiple errors. Optional.\\nExamples:\\n \\\"name\\\" - the field \\\"name\\\" on the current resource\\n \\\"items[0].name\\\" - the field \\\"name\\\" on the first array entry in \\\"items\\\"\",\"type\":\"string\"},\"message\":{\"description\":\"A human-readable description of the cause of the error. This field may be presented as-is to a reader.\",\"type\":\"string\"},\"reason\":{\"description\":\"A machine-readable description of the cause of the error. If this value is empty there is no information available.\",\"type\":\"string\"}}},\"type\":\"array\"},\"group\":{\"description\":\"The group attribute of the resource associated with the status StatusReason.\",\"type\":\"string\"},\"kind\":{\"description\":\"The kind attribute of the resource associated with the status StatusReason. On some operations may differ from the requested resource Kind. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds\",\"type\":\"string\"},\"name\":{\"description\":\"The name attribute of the resource associated with the status StatusReason (when there is a single name which can be described).\",\"type\":\"string\"},\"retryAfterSeconds\":{\"description\":\"If specified, the time in seconds before the operation should be retried. Some errors may indicate the client must take an alternate action - for those errors this field may indicate how long to wait before taking the alternate action.\",\"format\":\"int32\",\"type\":\"integer\"},\"uid\":{\"description\":\"UID of the resource. (when there is a single resource which can be described). More info: http://kubernetes.io/docs/user-guide/identifiers#uids\",\"type\":\"string\"}}},\"kind\":{\"description\":\"Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds\",\"type\":\"string\"},\"message\":{\"description\":\"A human-readable description of the status of this operation.\",\"type\":\"string\"},\"metadata\":{\"description\":\"ListMeta describes metadata that synthetic resources must have, including lists and various status objects. A resource may have only one of {ObjectMeta, ListMeta}.\",\"properties\":{\"continue\":{\"description\":\"continue may be set if the user set a limit on the number of items returned, and indicates that the server has more data available. The value is opaque and may be used to issue another request to the endpoint that served this list to retrieve the next set of available objects. Continuing a consistent list may not be possible if the server configuration has changed or more than a few minutes have passed. The resourceVersion field returned when using this continue value will be identical to the value in the first response, unless you have received this token from an error message.\",\"type\":\"string\"},\"resourceVersion\":{\"description\":\"String that identifies the server's internal version of this object that can be used by clients to determine when objects have changed. Value must be treated as opaque by clients and passed unmodified back to the server. Populated by the system. Read-only. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency\",\"type\":\"string\"},\"selfLink\":{\"description\":\"selfLink is a URL representing this object. Populated by the system. Read-only.\",\"type\":\"string\"}}},\"reason\":{\"description\":\"A machine-readable description of why this operation is in the \\\"Failure\\\" status. If this value is empty there is no information available. A Reason clarifies an HTTP status code but does not override it.\",\"type\":\"string\"},\"status\":{\"description\":\"Status of the operation. One of: \\\"Success\\\" or \\\"Failure\\\". More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status\",\"type\":\"string\"}}}},\"required\":[\"pending\"]},\"labels\":{\"description\":\"Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels\",\"type\":\"object\"},\"name\":{\"description\":\"Name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names\",\"type\":\"string\"},\"namespace\":{\"description\":\"Namespace defines the space within each name must be unique. An empty namespace is equivalent to the \\\"default\\\" namespace, but \\\"default\\\" is the canonical representation. Not all objects are required to be scoped to a namespace - the value of this field for those objects will be empty.\\nMust be a DNS_LABEL. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/namespaces\",\"type\":\"string\"},\"ownerReferences\":{\"description\":\"List of objects depended by this object. If ALL objects in the list have been deleted, this object will be garbage collected. If this object is managed by a controller, then an entry in this list will point to this controller, with the controller field set to true. There cannot be more than one managing controller.\",\"items\":{\"description\":\"OwnerReference contains enough information to let you identify an owning object. An owning object must be in the same namespace as the dependent, or be cluster-scoped, so there is no namespace field.\",\"properties\":{\"apiVersion\":{\"description\":\"API version of the referent.\",\"type\":\"string\"},\"blockOwnerDeletion\":{\"description\":\"If true, AND if the owner has the \\\"foregroundDeletion\\\" finalizer, then the owner cannot be deleted from the key-value store until this reference is removed. Defaults to false. To set this field, a user needs \\\"delete\\\" permission of the owner, otherwise 422 (Unprocessable Entity) will be returned.\",\"type\":\"boolean\"},\"controller\":{\"description\":\"If true, this reference points to the managing controller.\",\"type\":\"boolean\"},\"kind\":{\"description\":\"Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#names\",\"type\":\"string\"},\"uid\":{\"description\":\"UID of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#uids\",\"type\":\"string\"}},\"required\":[\"apiVersion\",\"kind\",\"name\",\"uid\"]},\"type\":\"array\"},\"resourceVersion\":{\"description\":\"An opaque value that represents the internal version of this object that can be used by clients to determine when objects have changed. May be used for optimistic concurrency, change detection, and the watch operation on a resource or set of resources. Clients must treat these values as opaque and passed unmodified back to the server. They may only be valid for a particular resource or set of resources.\\nPopulated by the system. Read-only. Value must be treated as opaque by clients and . More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency\",\"type\":\"string\"},\"selfLink\":{\"description\":\"SelfLink is a URL representing this object. Populated by the system. Read-only.\",\"type\":\"string\"},\"uid\":{\"description\":\"UID is the unique in time and space value for this object. It is typically generated by the server on successful creation of a resource and is not allowed to change on PUT operations.\\nPopulated by the system. Read-only. More info: http://kubernetes.io/docs/user-guide/identifiers#uids\",\"type\":\"string\"}}},\"spec\":{\"description\":\"PersistentVolumeClaimSpec describes the common attributes of storage devices and allows a Source for provider-specific attributes\",\"properties\":{\"accessModes\":{\"description\":\"AccessModes contains the desired access modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"dataSource\":{\"description\":\"TypedLocalObjectReference contains enough information to let you locate the typed referenced object inside the same namespace.\",\"properties\":{\"apiGroup\":{\"description\":\"APIGroup is the group for the resource being referenced. If APIGroup is not specified, the specified Kind must be in the core API group. For any other third-party types, APIGroup is required.\",\"type\":\"string\"},\"kind\":{\"description\":\"Kind is the type of resource being referenced\",\"type\":\"string\"},\"name\":{\"description\":\"Name is the name of resource being referenced\",\"type\":\"string\"}},\"required\":[\"kind\",\"name\"]},\"resources\":{\"description\":\"ResourceRequirements describes the compute resource requirements.\",\"properties\":{\"limits\":{\"description\":\"Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/\",\"type\":\"object\"},\"requests\":{\"description\":\"Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/\",\"type\":\"object\"}}},\"selector\":{\"description\":\"A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.\",\"properties\":{\"matchExpressions\":{\"description\":\"matchExpressions is a list of label selector requirements. The requirements are ANDed.\",\"items\":{\"description\":\"A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"key is the label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\",\"type\":\"string\"},\"values\":{\"description\":\"values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"]},\"type\":\"array\"},\"matchLabels\":{\"description\":\"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \\\"key\\\", the operator is \\\"In\\\", and the values array contains only \\\"value\\\". The requirements are ANDed.\",\"type\":\"object\"}}},\"storageClassName\":{\"description\":\"Name of the StorageClass required by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1\",\"type\":\"string\"},\"volumeMode\":{\"description\":\"volumeMode defines what type of volume is required by the claim. Value of Filesystem is implied when not included in claim spec. This is a beta feature.\",\"type\":\"string\"},\"volumeName\":{\"description\":\"VolumeName is the binding reference to the PersistentVolume backing this claim.\",\"type\":\"string\"}}},\"status\":{\"description\":\"PersistentVolumeClaimStatus is the current status of a persistent volume claim.\",\"properties\":{\"accessModes\":{\"description\":\"AccessModes contains the actual access modes the volume backing the PVC has. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"capacity\":{\"description\":\"Represents the actual resources of the underlying volume.\",\"type\":\"object\"},\"conditions\":{\"description\":\"Current Condition of persistent volume claim. If underlying persistent volume is being resized then the Condition will be set to 'ResizeStarted'.\",\"items\":{\"description\":\"PersistentVolumeClaimCondition contains details about state of pvc\",\"properties\":{\"lastProbeTime\":{\"description\":\"Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.\",\"format\":\"date-time\",\"type\":\"string\"},\"lastTransitionTime\":{\"description\":\"Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.\",\"format\":\"date-time\",\"type\":\"string\"},\"message\":{\"description\":\"Human-readable message indicating details about last transition.\",\"type\":\"string\"},\"reason\":{\"description\":\"Unique, this should be a short, machine understandable string that gives the reason for condition's last transition. If it reports \\\"ResizeStarted\\\" that means the underlying persistent volume is being resized.\",\"type\":\"string\"},\"status\":{\"type\":\"string\"},\"type\":{\"type\":\"string\"}},\"required\":[\"type\",\"status\"]},\"type\":\"array\"},\"phase\":{\"description\":\"Phase represents the current phase of PersistentVolumeClaim.\",\"type\":\"string\"}}}}}}},\"tag\":{\"description\":\"Tag of Alertmanager container image to be deployed. Defaults to the value of `version`. Version is ignored if Tag is set.\",\"type\":\"string\"},\"tolerations\":{\"description\":\"If specified, the pod's tolerations.\",\"items\":{\"description\":\"The pod this Toleration is attached to tolerates any taint that matches the triple \\u003ckey,value,effect\\u003e using the matching operator \\u003coperator\\u003e.\",\"properties\":{\"effect\":{\"description\":\"Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.\",\"type\":\"string\"},\"key\":{\"description\":\"Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys.\",\"type\":\"string\"},\"operator\":{\"description\":\"Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category.\",\"type\":\"string\"},\"tolerationSeconds\":{\"description\":\"TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system.\",\"format\":\"int64\",\"type\":\"integer\"},\"value\":{\"description\":\"Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string.\",\"type\":\"string\"}}},\"type\":\"array\"},\"version\":{\"description\":\"Version the cluster should be on.\",\"type\":\"string\"}}},\"status\":{\"description\":\"AlertmanagerStatus is the most recent observed status of the Alertmanager cluster. Read-only. Not included when requesting from the apiserver, only from the Prometheus Operator API itself. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/api-conventions.md#spec-and-status\",\"properties\":{\"availableReplicas\":{\"description\":\"Total number of available pods (ready for at least minReadySeconds) targeted by this Alertmanager cluster.\",\"format\":\"int32\",\"type\":\"integer\"},\"paused\":{\"description\":\"Represents whether any actions on the underlying managed objects are being performed. Only delete actions will be performed.\",\"type\":\"boolean\"},\"replicas\":{\"description\":\"Total number of non-terminated pods targeted by this Alertmanager cluster (their labels match the selector).\",\"format\":\"int32\",\"type\":\"integer\"},\"unavailableReplicas\":{\"description\":\"Total number of unavailable pods targeted by this Alertmanager cluster.\",\"format\":\"int32\",\"type\":\"integer\"},\"updatedReplicas\":{\"description\":\"Total number of non-terminated pods targeted by this Alertmanager cluster that have the desired version spec.\",\"format\":\"int32\",\"type\":\"integer\"}},\"required\":[\"paused\",\"replicas\",\"updatedReplicas\",\"availableReplicas\",\"unavailableReplicas\"]}}}},\"version\":\"v1\"}}\n" }, "creationTimestamp": "2020-05-05T16:51:39Z", "generation": 1, "labels": { "app": "prometheus-operator" }, "name": "alertmanagers.monitoring.coreos.com", "resourceVersion": "206434", "selfLink": "/apis/apiextensions.k8s.io/v1/customresourcedefinitions/alertmanagers.monitoring.coreos.com", "uid": "768a7255-d97a-4762-a400-f359fb24f4c8" }, "spec": { "conversion": { "strategy": "None" }, "group": "monitoring.coreos.com", "names": { "kind": "Alertmanager", "listKind": "AlertmanagerList", "plural": "alertmanagers", "singular": "alertmanager" }, "preserveUnknownFields": true, "scope": "Namespaced", "versions": [ { "name": "v1", "schema": { "openAPIV3Schema": { "properties": { "apiVersion": { "description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources", "type": "string" }, "kind": { "description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds", "type": "string" }, "spec": { "description": "AlertmanagerSpec is a specification of the desired behavior of the Alertmanager cluster. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/api-conventions.md#spec-and-status", "properties": { "additionalPeers": { "description": "AdditionalPeers allows injecting a set of additional Alertmanagers to peer with to form a highly available cluster.", "items": { "type": "string" }, "type": "array" }, "affinity": { "description": "Affinity is a group of affinity scheduling rules.", "properties": { "nodeAffinity": { "description": "Node affinity is a group of node affinity scheduling rules.", "properties": { "preferredDuringSchedulingIgnoredDuringExecution": { "description": "The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding \"weight\" to the sum if the node matches the corresponding matchExpressions; the node(s) with the highest sum are the most preferred.", "items": { "description": "An empty preferred scheduling term matches all objects with implicit weight 0 (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op).", "properties": { "preference": { "description": "A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm.", "properties": { "matchExpressions": { "description": "A list of node selector requirements by node's labels.", "items": { "description": "A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "properties": { "key": { "description": "The label key that the selector applies to.", "type": "string" }, "operator": { "description": "Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.", "type": "string" }, "values": { "description": "An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.", "items": { "type": "string" }, "type": "array" } }, "required": [ "key", "operator" ] }, "type": "array" }, "matchFields": { "description": "A list of node selector requirements by node's fields.", "items": { "description": "A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "properties": { "key": { "description": "The label key that the selector applies to.", "type": "string" }, "operator": { "description": "Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.", "type": "string" }, "values": { "description": "An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.", "items": { "type": "string" }, "type": "array" } }, "required": [ "key", "operator" ] }, "type": "array" } } }, "weight": { "description": "Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100.", "format": "int32", "type": "integer" } }, "required": [ "weight", "preference" ] }, "type": "array" }, "requiredDuringSchedulingIgnoredDuringExecution": { "description": "A node selector represents the union of the results of one or more label queries over a set of nodes; that is, it represents the OR of the selectors represented by the node selector terms.", "properties": { "nodeSelectorTerms": { "description": "Required. A list of node selector terms. The terms are ORed.", "items": { "description": "A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm.", "properties": { "matchExpressions": { "description": "A list of node selector requirements by node's labels.", "items": { "description": "A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "properties": { "key": { "description": "The label key that the selector applies to.", "type": "string" }, "operator": { "description": "Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.", "type": "string" }, "values": { "description": "An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.", "items": { "type": "string" }, "type": "array" } }, "required": [ "key", "operator" ] }, "type": "array" }, "matchFields": { "description": "A list of node selector requirements by node's fields.", "items": { "description": "A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "properties": { "key": { "description": "The label key that the selector applies to.", "type": "string" }, "operator": { "description": "Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.", "type": "string" }, "values": { "description": "An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.", "items": { "type": "string" }, "type": "array" } }, "required": [ "key", "operator" ] }, "type": "array" } } }, "type": "array" } }, "required": [ "nodeSelectorTerms" ] } } }, "podAffinity": { "description": "Pod affinity is a group of inter pod affinity scheduling rules.", "properties": { "preferredDuringSchedulingIgnoredDuringExecution": { "description": "The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding \"weight\" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred.", "items": { "description": "The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s)", "properties": { "podAffinityTerm": { "description": "Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key \u003ctopologyKey\u003e matches that of any node on which a pod of the set of pods is running", "properties": { "labelSelector": { "description": "A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "items": { "type": "string" }, "type": "array" } }, "required": [ "key", "operator" ] }, "type": "array" }, "matchLabels": { "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object" } } }, "namespaces": { "description": "namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \"this pod's namespace\"", "items": { "type": "string" }, "type": "array" }, "topologyKey": { "description": "This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.", "type": "string" } }, "required": [ "topologyKey" ] }, "weight": { "description": "weight associated with matching the corresponding podAffinityTerm, in the range 1-100.", "format": "int32", "type": "integer" } }, "required": [ "weight", "podAffinityTerm" ] }, "type": "array" }, "requiredDuringSchedulingIgnoredDuringExecution": { "description": "If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied.", "items": { "description": "Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key \u003ctopologyKey\u003e matches that of any node on which a pod of the set of pods is running", "properties": { "labelSelector": { "description": "A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "items": { "type": "string" }, "type": "array" } }, "required": [ "key", "operator" ] }, "type": "array" }, "matchLabels": { "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object" } } }, "namespaces": { "description": "namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \"this pod's namespace\"", "items": { "type": "string" }, "type": "array" }, "topologyKey": { "description": "This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.", "type": "string" } }, "required": [ "topologyKey" ] }, "type": "array" } } }, "podAntiAffinity": { "description": "Pod anti affinity is a group of inter pod anti affinity scheduling rules.", "properties": { "preferredDuringSchedulingIgnoredDuringExecution": { "description": "The scheduler will prefer to schedule pods to nodes that satisfy the anti-affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding \"weight\" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred.", "items": { "description": "The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s)", "properties": { "podAffinityTerm": { "description": "Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key \u003ctopologyKey\u003e matches that of any node on which a pod of the set of pods is running", "properties": { "labelSelector": { "description": "A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "items": { "type": "string" }, "type": "array" } }, "required": [ "key", "operator" ] }, "type": "array" }, "matchLabels": { "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object" } } }, "namespaces": { "description": "namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \"this pod's namespace\"", "items": { "type": "string" }, "type": "array" }, "topologyKey": { "description": "This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.", "type": "string" } }, "required": [ "topologyKey" ] }, "weight": { "description": "weight associated with matching the corresponding podAffinityTerm, in the range 1-100.", "format": "int32", "type": "integer" } }, "required": [ "weight", "podAffinityTerm" ] }, "type": "array" }, "requiredDuringSchedulingIgnoredDuringExecution": { "description": "If the anti-affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the anti-affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied.", "items": { "description": "Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key \u003ctopologyKey\u003e matches that of any node on which a pod of the set of pods is running", "properties": { "labelSelector": { "description": "A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "items": { "type": "string" }, "type": "array" } }, "required": [ "key", "operator" ] }, "type": "array" }, "matchLabels": { "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object" } } }, "namespaces": { "description": "namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \"this pod's namespace\"", "items": { "type": "string" }, "type": "array" }, "topologyKey": { "description": "This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.", "type": "string" } }, "required": [ "topologyKey" ] }, "type": "array" } } } } }, "baseImage": { "description": "Base image that is used to deploy pods, without tag.", "type": "string" }, "configMaps": { "description": "ConfigMaps is a list of ConfigMaps in the same namespace as the Alertmanager object, which shall be mounted into the Alertmanager Pods. The ConfigMaps are mounted into /etc/alertmanager/configmaps/\u003cconfigmap-name\u003e.", "items": { "type": "string" }, "type": "array" }, "configSecret": { "description": "ConfigSecret is the name of a Kubernetes Secret in the same namespace as the Alertmanager object, which contains configuration for this Alertmanager instance. Defaults to 'alertmanager-' The secret is mounted into /etc/alertmanager/config.", "type": "string" }, "containers": { "description": "Containers allows injecting additional containers. This is meant to allow adding an authentication proxy to an Alertmanager pod.", "items": { "description": "A single application container that you want to run within a pod.", "properties": { "args": { "description": "Arguments to the entrypoint. The docker image's CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell", "items": { "type": "string" }, "type": "array" }, "command": { "description": "Entrypoint array. Not executed within a shell. The docker image's ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell", "items": { "type": "string" }, "type": "array" }, "env": { "description": "List of environment variables to set in the container. Cannot be updated.", "items": { "description": "EnvVar represents an environment variable present in a Container.", "properties": { "name": { "description": "Name of the environment variable. Must be a C_IDENTIFIER.", "type": "string" }, "value": { "description": "Variable references $(VAR_NAME) are expanded using the previous defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Defaults to \"\".", "type": "string" }, "valueFrom": { "description": "EnvVarSource represents a source for the value of an EnvVar.", "properties": { "configMapKeyRef": { "description": "Selects a key from a ConfigMap.", "properties": { "key": { "description": "The key to select.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", "type": "string" }, "optional": { "description": "Specify whether the ConfigMap or it's key must be defined", "type": "boolean" } }, "required": [ "key" ] }, "fieldRef": { "description": "ObjectFieldSelector selects an APIVersioned field of an object.", "properties": { "apiVersion": { "description": "Version of the schema the FieldPath is written in terms of, defaults to \"v1\".", "type": "string" }, "fieldPath": { "description": "Path of the field to select in the specified API version.", "type": "string" } }, "required": [ "fieldPath" ] }, "resourceFieldRef": { "description": "ResourceFieldSelector represents container resources (cpu, memory) and their output format", "properties": { "containerName": { "description": "Container name: required for volumes, optional for env vars", "type": "string" }, "divisor": {}, "resource": { "description": "Required: resource to select", "type": "string" } }, "required": [ "resource" ] }, "secretKeyRef": { "description": "SecretKeySelector selects a key of a Secret.", "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", "type": "string" }, "optional": { "description": "Specify whether the Secret or it's key must be defined", "type": "boolean" } }, "required": [ "key" ] } } } }, "required": [ "name" ] }, "type": "array" }, "envFrom": { "description": "List of sources to populate environment variables in the container. The keys defined within a source must be a C_IDENTIFIER. All invalid keys will be reported as an event when the container is starting. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated.", "items": { "description": "EnvFromSource represents the source of a set of ConfigMaps", "properties": { "configMapRef": { "description": "ConfigMapEnvSource selects a ConfigMap to populate the environment variables with.\nThe contents of the target ConfigMap's Data field will represent the key-value pairs as environment variables.", "properties": { "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", "type": "string" }, "optional": { "description": "Specify whether the ConfigMap must be defined", "type": "boolean" } } }, "prefix": { "description": "An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER.", "type": "string" }, "secretRef": { "description": "SecretEnvSource selects a Secret to populate the environment variables with.\nThe contents of the target Secret's Data field will represent the key-value pairs as environment variables.", "properties": { "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", "type": "string" }, "optional": { "description": "Specify whether the Secret must be defined", "type": "boolean" } } } } }, "type": "array" }, "image": { "description": "Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images This field is optional to allow higher level config management to default or override container images in workload controllers like Deployments and StatefulSets.", "type": "string" }, "imagePullPolicy": { "description": "Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images", "type": "string" }, "lifecycle": { "description": "Lifecycle describes actions that the management system should take in response to container lifecycle events. For the PostStart and PreStop lifecycle handlers, management of the container blocks until the action is complete, unless the container process fails, in which case the handler is aborted.", "properties": { "postStart": { "description": "Handler defines a specific action that should be taken", "properties": { "exec": { "description": "ExecAction describes a \"run in container\" action.", "properties": { "command": { "description": "Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.", "items": { "type": "string" }, "type": "array" } } }, "httpGet": { "description": "HTTPGetAction describes an action based on HTTP Get requests.", "properties": { "host": { "description": "Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.", "type": "string" }, "httpHeaders": { "description": "Custom headers to set in the request. HTTP allows repeated headers.", "items": { "description": "HTTPHeader describes a custom header to be used in HTTP probes", "properties": { "name": { "description": "The header field name", "type": "string" }, "value": { "description": "The header field value", "type": "string" } }, "required": [ "name", "value" ] }, "type": "array" }, "path": { "description": "Path to access on the HTTP server.", "type": "string" }, "port": { "anyOf": [ { "type": "string" }, { "type": "integer" } ] }, "scheme": { "description": "Scheme to use for connecting to the host. Defaults to HTTP.", "type": "string" } }, "required": [ "port" ] }, "tcpSocket": { "description": "TCPSocketAction describes an action based on opening a socket", "properties": { "host": { "description": "Optional: Host name to connect to, defaults to the pod IP.", "type": "string" }, "port": { "anyOf": [ { "type": "string" }, { "type": "integer" } ] } }, "required": [ "port" ] } } }, "preStop": { "description": "Handler defines a specific action that should be taken", "properties": { "exec": { "description": "ExecAction describes a \"run in container\" action.", "properties": { "command": { "description": "Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.", "items": { "type": "string" }, "type": "array" } } }, "httpGet": { "description": "HTTPGetAction describes an action based on HTTP Get requests.", "properties": { "host": { "description": "Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.", "type": "string" }, "httpHeaders": { "description": "Custom headers to set in the request. HTTP allows repeated headers.", "items": { "description": "HTTPHeader describes a custom header to be used in HTTP probes", "properties": { "name": { "description": "The header field name", "type": "string" }, "value": { "description": "The header field value", "type": "string" } }, "required": [ "name", "value" ] }, "type": "array" }, "path": { "description": "Path to access on the HTTP server.", "type": "string" }, "port": { "anyOf": [ { "type": "string" }, { "type": "integer" } ] }, "scheme": { "description": "Scheme to use for connecting to the host. Defaults to HTTP.", "type": "string" } }, "required": [ "port" ] }, "tcpSocket": { "description": "TCPSocketAction describes an action based on opening a socket", "properties": { "host": { "description": "Optional: Host name to connect to, defaults to the pod IP.", "type": "string" }, "port": { "anyOf": [ { "type": "string" }, { "type": "integer" } ] } }, "required": [ "port" ] } } } } }, "livenessProbe": { "description": "Probe describes a health check to be performed against a container to determine whether it is alive or ready to receive traffic.", "properties": { "exec": { "description": "ExecAction describes a \"run in container\" action.", "properties": { "command": { "description": "Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.", "items": { "type": "string" }, "type": "array" } } }, "failureThreshold": { "description": "Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.", "format": "int32", "type": "integer" }, "httpGet": { "description": "HTTPGetAction describes an action based on HTTP Get requests.", "properties": { "host": { "description": "Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.", "type": "string" }, "httpHeaders": { "description": "Custom headers to set in the request. HTTP allows repeated headers.", "items": { "description": "HTTPHeader describes a custom header to be used in HTTP probes", "properties": { "name": { "description": "The header field name", "type": "string" }, "value": { "description": "The header field value", "type": "string" } }, "required": [ "name", "value" ] }, "type": "array" }, "path": { "description": "Path to access on the HTTP server.", "type": "string" }, "port": { "anyOf": [ { "type": "string" }, { "type": "integer" } ] }, "scheme": { "description": "Scheme to use for connecting to the host. Defaults to HTTP.", "type": "string" } }, "required": [ "port" ] }, "initialDelaySeconds": { "description": "Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes", "format": "int32", "type": "integer" }, "periodSeconds": { "description": "How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.", "format": "int32", "type": "integer" }, "successThreshold": { "description": "Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness. Minimum value is 1.", "format": "int32", "type": "integer" }, "tcpSocket": { "description": "TCPSocketAction describes an action based on opening a socket", "properties": { "host": { "description": "Optional: Host name to connect to, defaults to the pod IP.", "type": "string" }, "port": { "anyOf": [ { "type": "string" }, { "type": "integer" } ] } }, "required": [ "port" ] }, "timeoutSeconds": { "description": "Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes", "format": "int32", "type": "integer" } } }, "name": { "description": "Name of the container specified as a DNS_LABEL. Each container in a pod must have a unique name (DNS_LABEL). Cannot be updated.", "type": "string" }, "ports": { "description": "List of ports to expose from the container. Exposing a port here gives the system additional information about the network connections a container uses, but is primarily informational. Not specifying a port here DOES NOT prevent that port from being exposed. Any port which is listening on the default \"0.0.0.0\" address inside a container will be accessible from the network. Cannot be updated.", "items": { "description": "ContainerPort represents a network port in a single container.", "properties": { "containerPort": { "description": "Number of port to expose on the pod's IP address. This must be a valid port number, 0 \u003c x \u003c 65536.", "format": "int32", "type": "integer" }, "hostIP": { "description": "What host IP to bind the external port to.", "type": "string" }, "hostPort": { "description": "Number of port to expose on the host. If specified, this must be a valid port number, 0 \u003c x \u003c 65536. If HostNetwork is specified, this must match ContainerPort. Most containers do not need this.", "format": "int32", "type": "integer" }, "name": { "description": "If specified, this must be an IANA_SVC_NAME and unique within the pod. Each named port in a pod must have a unique name. Name for the port that can be referred to by services.", "type": "string" }, "protocol": { "description": "Protocol for port. Must be UDP, TCP, or SCTP. Defaults to \"TCP\".", "type": "string" } }, "required": [ "containerPort" ] }, "type": "array" }, "readinessProbe": { "description": "Probe describes a health check to be performed against a container to determine whether it is alive or ready to receive traffic.", "properties": { "exec": { "description": "ExecAction describes a \"run in container\" action.", "properties": { "command": { "description": "Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.", "items": { "type": "string" }, "type": "array" } } }, "failureThreshold": { "description": "Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.", "format": "int32", "type": "integer" }, "httpGet": { "description": "HTTPGetAction describes an action based on HTTP Get requests.", "properties": { "host": { "description": "Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.", "type": "string" }, "httpHeaders": { "description": "Custom headers to set in the request. HTTP allows repeated headers.", "items": { "description": "HTTPHeader describes a custom header to be used in HTTP probes", "properties": { "name": { "description": "The header field name", "type": "string" }, "value": { "description": "The header field value", "type": "string" } }, "required": [ "name", "value" ] }, "type": "array" }, "path": { "description": "Path to access on the HTTP server.", "type": "string" }, "port": { "anyOf": [ { "type": "string" }, { "type": "integer" } ] }, "scheme": { "description": "Scheme to use for connecting to the host. Defaults to HTTP.", "type": "string" } }, "required": [ "port" ] }, "initialDelaySeconds": { "description": "Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes", "format": "int32", "type": "integer" }, "periodSeconds": { "description": "How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.", "format": "int32", "type": "integer" }, "successThreshold": { "description": "Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness. Minimum value is 1.", "format": "int32", "type": "integer" }, "tcpSocket": { "description": "TCPSocketAction describes an action based on opening a socket", "properties": { "host": { "description": "Optional: Host name to connect to, defaults to the pod IP.", "type": "string" }, "port": { "anyOf": [ { "type": "string" }, { "type": "integer" } ] } }, "required": [ "port" ] }, "timeoutSeconds": { "description": "Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes", "format": "int32", "type": "integer" } } }, "resources": { "description": "ResourceRequirements describes the compute resource requirements.", "properties": { "limits": { "description": "Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", "type": "object" }, "requests": { "description": "Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", "type": "object" } } }, "securityContext": { "description": "SecurityContext holds security configuration that will be applied to a container. Some fields are present in both SecurityContext and PodSecurityContext. When both are set, the values in SecurityContext take precedence.", "properties": { "allowPrivilegeEscalation": { "description": "AllowPrivilegeEscalation controls whether a process can gain more privileges than its parent process. This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged 2) has CAP_SYS_ADMIN", "type": "boolean" }, "capabilities": { "description": "Adds and removes POSIX capabilities from running containers.", "properties": { "add": { "description": "Added capabilities", "items": { "type": "string" }, "type": "array" }, "drop": { "description": "Removed capabilities", "items": { "type": "string" }, "type": "array" } } }, "privileged": { "description": "Run container in privileged mode. Processes in privileged containers are essentially equivalent to root on the host. Defaults to false.", "type": "boolean" }, "procMount": { "description": "procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled.", "type": "string" }, "readOnlyRootFilesystem": { "description": "Whether this container has a read-only root filesystem. Default is false.", "type": "boolean" }, "runAsGroup": { "description": "The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.", "format": "int64", "type": "integer" }, "runAsNonRoot": { "description": "Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.", "type": "boolean" }, "runAsUser": { "description": "The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.", "format": "int64", "type": "integer" }, "seLinuxOptions": { "description": "SELinuxOptions are the labels to be applied to the container", "properties": { "level": { "description": "Level is SELinux level label that applies to the container.", "type": "string" }, "role": { "description": "Role is a SELinux role label that applies to the container.", "type": "string" }, "type": { "description": "Type is a SELinux type label that applies to the container.", "type": "string" }, "user": { "description": "User is a SELinux user label that applies to the container.", "type": "string" } } } } }, "stdin": { "description": "Whether this container should allocate a buffer for stdin in the container runtime. If this is not set, reads from stdin in the container will always result in EOF. Default is false.", "type": "boolean" }, "stdinOnce": { "description": "Whether the container runtime should close the stdin channel after it has been opened by a single attach. When stdin is true the stdin stream will remain open across multiple attach sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the first client attaches to stdin, and then remains open and accepts data until the client disconnects, at which time stdin is closed and remains closed until the container is restarted. If this flag is false, a container processes that reads from stdin will never receive an EOF. Default is false", "type": "boolean" }, "terminationMessagePath": { "description": "Optional: Path at which the file to which the container's termination message will be written is mounted into the container's filesystem. Message written is intended to be brief final status, such as an assertion failure message. Will be truncated by the node if greater than 4096 bytes. The total message length across all containers will be limited to 12kb. Defaults to /dev/termination-log. Cannot be updated.", "type": "string" }, "terminationMessagePolicy": { "description": "Indicate how the termination message should be populated. File will use the contents of terminationMessagePath to populate the container status message on both success and failure. FallbackToLogsOnError will use the last chunk of container log output if the termination message file is empty and the container exited with an error. The log output is limited to 2048 bytes or 80 lines, whichever is smaller. Defaults to File. Cannot be updated.", "type": "string" }, "tty": { "description": "Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. Default is false.", "type": "boolean" }, "volumeDevices": { "description": "volumeDevices is the list of block devices to be used by the container. This is a beta feature.", "items": { "description": "volumeDevice describes a mapping of a raw block device within a container.", "properties": { "devicePath": { "description": "devicePath is the path inside of the container that the device will be mapped to.", "type": "string" }, "name": { "description": "name must match the name of a persistentVolumeClaim in the pod", "type": "string" } }, "required": [ "name", "devicePath" ] }, "type": "array" }, "volumeMounts": { "description": "Pod volumes to mount into the container's filesystem. Cannot be updated.", "items": { "description": "VolumeMount describes a mounting of a Volume within a container.", "properties": { "mountPath": { "description": "Path within the container at which the volume should be mounted. Must not contain ':'.", "type": "string" }, "mountPropagation": { "description": "mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10.", "type": "string" }, "name": { "description": "This must match the Name of a Volume.", "type": "string" }, "readOnly": { "description": "Mounted read-only if true, read-write otherwise (false or unspecified). Defaults to false.", "type": "boolean" }, "subPath": { "description": "Path within the volume from which the container's volume should be mounted. Defaults to \"\" (volume's root).", "type": "string" } }, "required": [ "name", "mountPath" ] }, "type": "array" }, "workingDir": { "description": "Container's working directory. If not specified, the container runtime's default will be used, which might be configured in the container image. Cannot be updated.", "type": "string" } }, "required": [ "name" ] }, "type": "array" }, "externalUrl": { "description": "The external URL the Alertmanager instances will be available under. This is necessary to generate correct URLs. This is necessary if Alertmanager is not served from root of a DNS name.", "type": "string" }, "image": { "description": "Image if specified has precedence over baseImage, tag and sha combinations. Specifying the version is still necessary to ensure the Prometheus Operator knows what version of Alertmanager is being configured.", "type": "string" }, "imagePullSecrets": { "description": "An optional list of references to secrets in the same namespace to use for pulling prometheus and alertmanager images from registries see http://kubernetes.io/docs/user-guide/images#specifying-imagepullsecrets-on-a-pod", "items": { "description": "LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace.", "properties": { "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", "type": "string" } } }, "type": "array" }, "listenLocal": { "description": "ListenLocal makes the Alertmanager server listen on loopback, so that it does not bind against the Pod IP. Note this is only for the Alertmanager UI, not the gossip communication.", "type": "boolean" }, "logLevel": { "description": "Log level for Alertmanager to be configured with.", "type": "string" }, "nodeSelector": { "description": "Define which Nodes the Pods are scheduled on.", "type": "object" }, "paused": { "description": "If set to true all actions on the underlying managed objects are not goint to be performed, except for delete actions.", "type": "boolean" }, "podMetadata": { "description": "ObjectMeta is metadata that all persisted resources must have, which includes all objects users must create.", "properties": { "annotations": { "description": "Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations", "type": "object" }, "clusterName": { "description": "The name of the cluster which the object belongs to. This is used to distinguish resources with same name and namespace in different clusters. This field is not set anywhere right now and apiserver is going to ignore it if set in create or update request.", "type": "string" }, "creationTimestamp": { "description": "Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.", "format": "date-time", "type": "string" }, "deletionGracePeriodSeconds": { "description": "Number of seconds allowed for this object to gracefully terminate before it will be removed from the system. Only set when deletionTimestamp is also set. May only be shortened. Read-only.", "format": "int64", "type": "integer" }, "deletionTimestamp": { "description": "Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.", "format": "date-time", "type": "string" }, "finalizers": { "description": "Must be empty before the object is deleted from the registry. Each entry is an identifier for the responsible component that will remove the entry from the list. If the deletionTimestamp of the object is non-nil, entries in this list can only be removed.", "items": { "type": "string" }, "type": "array" }, "generateName": { "description": "GenerateName is an optional prefix, used by the server, to generate a unique name ONLY IF the Name field has not been provided. If this field is used, the name returned to the client will be different than the name passed. This value will also be combined with a unique suffix. The provided value has the same validation rules as the Name field, and may be truncated by the length of the suffix required to make the value unique on the server.\nIf this field is specified and the generated name exists, the server will NOT return a 409 - instead, it will either return 201 Created or 500 with Reason ServerTimeout indicating a unique name could not be found in the time allotted, and the client should retry (optionally after the time indicated in the Retry-After header).\nApplied only if Name is not specified. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#idempotency", "type": "string" }, "generation": { "description": "A sequence number representing a specific generation of the desired state. Populated by the system. Read-only.", "format": "int64", "type": "integer" }, "initializers": { "description": "Initializers tracks the progress of initialization.", "properties": { "pending": { "description": "Pending is a list of initializers that must execute in order before this object is visible. When the last pending initializer is removed, and no failing result is set, the initializers struct will be set to nil and the object is considered as initialized and visible to all clients.", "items": { "description": "Initializer is information about an initializer that has not yet completed.", "properties": { "name": { "description": "name of the process that is responsible for initializing this object.", "type": "string" } }, "required": [ "name" ] }, "type": "array" }, "result": { "description": "Status is a return value for calls that don't return other objects.", "properties": { "apiVersion": { "description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources", "type": "string" }, "code": { "description": "Suggested HTTP return code for this status, 0 if not set.", "format": "int32", "type": "integer" }, "details": { "description": "StatusDetails is a set of additional properties that MAY be set by the server to provide additional information about a response. The Reason field of a Status object defines what attributes will be set. Clients must ignore fields that do not match the defined type of each attribute, and should assume that any attribute may be empty, invalid, or under defined.", "properties": { "causes": { "description": "The Causes array includes more details associated with the StatusReason failure. Not all StatusReasons may provide detailed causes.", "items": { "description": "StatusCause provides more information about an api.Status failure, including cases when multiple errors are encountered.", "properties": { "field": { "description": "The field of the resource that has caused this error, as named by its JSON serialization. May include dot and postfix notation for nested attributes. Arrays are zero-indexed. Fields may appear more than once in an array of causes due to fields having multiple errors. Optional.\nExamples:\n \"name\" - the field \"name\" on the current resource\n \"items[0].name\" - the field \"name\" on the first array entry in \"items\"", "type": "string" }, "message": { "description": "A human-readable description of the cause of the error. This field may be presented as-is to a reader.", "type": "string" }, "reason": { "description": "A machine-readable description of the cause of the error. If this value is empty there is no information available.", "type": "string" } } }, "type": "array" }, "group": { "description": "The group attribute of the resource associated with the status StatusReason.", "type": "string" }, "kind": { "description": "The kind attribute of the resource associated with the status StatusReason. On some operations may differ from the requested resource Kind. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds", "type": "string" }, "name": { "description": "The name attribute of the resource associated with the status StatusReason (when there is a single name which can be described).", "type": "string" }, "retryAfterSeconds": { "description": "If specified, the time in seconds before the operation should be retried. Some errors may indicate the client must take an alternate action - for those errors this field may indicate how long to wait before taking the alternate action.", "format": "int32", "type": "integer" }, "uid": { "description": "UID of the resource. (when there is a single resource which can be described). More info: http://kubernetes.io/docs/user-guide/identifiers#uids", "type": "string" } } }, "kind": { "description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds", "type": "string" }, "message": { "description": "A human-readable description of the status of this operation.", "type": "string" }, "metadata": { "description": "ListMeta describes metadata that synthetic resources must have, including lists and various status objects. A resource may have only one of {ObjectMeta, ListMeta}.", "properties": { "continue": { "description": "continue may be set if the user set a limit on the number of items returned, and indicates that the server has more data available. The value is opaque and may be used to issue another request to the endpoint that served this list to retrieve the next set of available objects. Continuing a consistent list may not be possible if the server configuration has changed or more than a few minutes have passed. The resourceVersion field returned when using this continue value will be identical to the value in the first response, unless you have received this token from an error message.", "type": "string" }, "resourceVersion": { "description": "String that identifies the server's internal version of this object that can be used by clients to determine when objects have changed. Value must be treated as opaque by clients and passed unmodified back to the server. Populated by the system. Read-only. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency", "type": "string" }, "selfLink": { "description": "selfLink is a URL representing this object. Populated by the system. Read-only.", "type": "string" } } }, "reason": { "description": "A machine-readable description of why this operation is in the \"Failure\" status. If this value is empty there is no information available. A Reason clarifies an HTTP status code but does not override it.", "type": "string" }, "status": { "description": "Status of the operation. One of: \"Success\" or \"Failure\". More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status", "type": "string" } } } }, "required": [ "pending" ] }, "labels": { "description": "Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels", "type": "object" }, "name": { "description": "Name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names", "type": "string" }, "namespace": { "description": "Namespace defines the space within each name must be unique. An empty namespace is equivalent to the \"default\" namespace, but \"default\" is the canonical representation. Not all objects are required to be scoped to a namespace - the value of this field for those objects will be empty.\nMust be a DNS_LABEL. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/namespaces", "type": "string" }, "ownerReferences": { "description": "List of objects depended by this object. If ALL objects in the list have been deleted, this object will be garbage collected. If this object is managed by a controller, then an entry in this list will point to this controller, with the controller field set to true. There cannot be more than one managing controller.", "items": { "description": "OwnerReference contains enough information to let you identify an owning object. An owning object must be in the same namespace as the dependent, or be cluster-scoped, so there is no namespace field.", "properties": { "apiVersion": { "description": "API version of the referent.", "type": "string" }, "blockOwnerDeletion": { "description": "If true, AND if the owner has the \"foregroundDeletion\" finalizer, then the owner cannot be deleted from the key-value store until this reference is removed. Defaults to false. To set this field, a user needs \"delete\" permission of the owner, otherwise 422 (Unprocessable Entity) will be returned.", "type": "boolean" }, "controller": { "description": "If true, this reference points to the managing controller.", "type": "boolean" }, "kind": { "description": "Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds", "type": "string" }, "name": { "description": "Name of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#names", "type": "string" }, "uid": { "description": "UID of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#uids", "type": "string" } }, "required": [ "apiVersion", "kind", "name", "uid" ] }, "type": "array" }, "resourceVersion": { "description": "An opaque value that represents the internal version of this object that can be used by clients to determine when objects have changed. May be used for optimistic concurrency, change detection, and the watch operation on a resource or set of resources. Clients must treat these values as opaque and passed unmodified back to the server. They may only be valid for a particular resource or set of resources.\nPopulated by the system. Read-only. Value must be treated as opaque by clients and . More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency", "type": "string" }, "selfLink": { "description": "SelfLink is a URL representing this object. Populated by the system. Read-only.", "type": "string" }, "uid": { "description": "UID is the unique in time and space value for this object. It is typically generated by the server on successful creation of a resource and is not allowed to change on PUT operations.\nPopulated by the system. Read-only. More info: http://kubernetes.io/docs/user-guide/identifiers#uids", "type": "string" } } }, "priorityClassName": { "description": "Priority class assigned to the Pods", "type": "string" }, "replicas": { "description": "Size is the expected size of the alertmanager cluster. The controller will eventually make the size of the running cluster equal to the expected size.", "format": "int32", "type": "integer" }, "resources": { "description": "ResourceRequirements describes the compute resource requirements.", "properties": { "limits": { "description": "Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", "type": "object" }, "requests": { "description": "Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", "type": "object" } } }, "retention": { "description": "Time duration Alertmanager shall retain data for. Default is '120h', and must match the regular expression `[0-9]+(ms|s|m|h)` (milliseconds seconds minutes hours).", "type": "string" }, "routePrefix": { "description": "The route prefix Alertmanager registers HTTP handlers for. This is useful, if using ExternalURL and a proxy is rewriting HTTP routes of a request, and the actual ExternalURL is still true, but the server serves requests under a different route prefix. For example for use with `kubectl proxy`.", "type": "string" }, "secrets": { "description": "Secrets is a list of Secrets in the same namespace as the Alertmanager object, which shall be mounted into the Alertmanager Pods. The Secrets are mounted into /etc/alertmanager/secrets/\u003csecret-name\u003e.", "items": { "type": "string" }, "type": "array" }, "securityContext": { "description": "PodSecurityContext holds pod-level security attributes and common container settings. Some fields are also present in container.securityContext. Field values of container.securityContext take precedence over field values of PodSecurityContext.", "properties": { "fsGroup": { "description": "A special supplemental group that applies to all containers in a pod. Some volume types allow the Kubelet to change the ownership of that volume to be owned by the pod:\n1. The owning GID will be the FSGroup 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) 3. The permission bits are OR'd with rw-rw----\nIf unset, the Kubelet will not modify the ownership and permissions of any volume.", "format": "int64", "type": "integer" }, "runAsGroup": { "description": "The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container.", "format": "int64", "type": "integer" }, "runAsNonRoot": { "description": "Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.", "type": "boolean" }, "runAsUser": { "description": "The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container.", "format": "int64", "type": "integer" }, "seLinuxOptions": { "description": "SELinuxOptions are the labels to be applied to the container", "properties": { "level": { "description": "Level is SELinux level label that applies to the container.", "type": "string" }, "role": { "description": "Role is a SELinux role label that applies to the container.", "type": "string" }, "type": { "description": "Type is a SELinux type label that applies to the container.", "type": "string" }, "user": { "description": "User is a SELinux user label that applies to the container.", "type": "string" } } }, "supplementalGroups": { "description": "A list of groups applied to the first process run in each container, in addition to the container's primary GID. If unspecified, no groups will be added to any container.", "items": { "format": "int64", "type": "integer" }, "type": "array" }, "sysctls": { "description": "Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported sysctls (by the container runtime) might fail to launch.", "items": { "description": "Sysctl defines a kernel parameter to be set", "properties": { "name": { "description": "Name of a property to set", "type": "string" }, "value": { "description": "Value of a property to set", "type": "string" } }, "required": [ "name", "value" ] }, "type": "array" } } }, "serviceAccountName": { "description": "ServiceAccountName is the name of the ServiceAccount to use to run the Prometheus Pods.", "type": "string" }, "sha": { "description": "SHA of Alertmanager container image to be deployed. Defaults to the value of `version`. Similar to a tag, but the SHA explicitly deploys an immutable container image. Version and Tag are ignored if SHA is set.", "type": "string" }, "storage": { "description": "StorageSpec defines the configured storage for a group Prometheus servers. If neither `emptyDir` nor `volumeClaimTemplate` is specified, then by default an [EmptyDir](https://kubernetes.io/docs/concepts/storage/volumes/#emptydir) will be used.", "properties": { "emptyDir": { "description": "Represents an empty directory for a pod. Empty directory volumes support ownership management and SELinux relabeling.", "properties": { "medium": { "description": "What type of storage medium should back this directory. The default is \"\" which means to use the node's default medium. Must be an empty string (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir", "type": "string" }, "sizeLimit": {} } }, "volumeClaimTemplate": { "description": "PersistentVolumeClaim is a user's request for and claim to a persistent volume", "properties": { "apiVersion": { "description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources", "type": "string" }, "kind": { "description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds", "type": "string" }, "metadata": { "description": "ObjectMeta is metadata that all persisted resources must have, which includes all objects users must create.", "properties": { "annotations": { "description": "Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations", "type": "object" }, "clusterName": { "description": "The name of the cluster which the object belongs to. This is used to distinguish resources with same name and namespace in different clusters. This field is not set anywhere right now and apiserver is going to ignore it if set in create or update request.", "type": "string" }, "creationTimestamp": { "description": "Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.", "format": "date-time", "type": "string" }, "deletionGracePeriodSeconds": { "description": "Number of seconds allowed for this object to gracefully terminate before it will be removed from the system. Only set when deletionTimestamp is also set. May only be shortened. Read-only.", "format": "int64", "type": "integer" }, "deletionTimestamp": { "description": "Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.", "format": "date-time", "type": "string" }, "finalizers": { "description": "Must be empty before the object is deleted from the registry. Each entry is an identifier for the responsible component that will remove the entry from the list. If the deletionTimestamp of the object is non-nil, entries in this list can only be removed.", "items": { "type": "string" }, "type": "array" }, "generateName": { "description": "GenerateName is an optional prefix, used by the server, to generate a unique name ONLY IF the Name field has not been provided. If this field is used, the name returned to the client will be different than the name passed. This value will also be combined with a unique suffix. The provided value has the same validation rules as the Name field, and may be truncated by the length of the suffix required to make the value unique on the server.\nIf this field is specified and the generated name exists, the server will NOT return a 409 - instead, it will either return 201 Created or 500 with Reason ServerTimeout indicating a unique name could not be found in the time allotted, and the client should retry (optionally after the time indicated in the Retry-After header).\nApplied only if Name is not specified. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#idempotency", "type": "string" }, "generation": { "description": "A sequence number representing a specific generation of the desired state. Populated by the system. Read-only.", "format": "int64", "type": "integer" }, "initializers": { "description": "Initializers tracks the progress of initialization.", "properties": { "pending": { "description": "Pending is a list of initializers that must execute in order before this object is visible. When the last pending initializer is removed, and no failing result is set, the initializers struct will be set to nil and the object is considered as initialized and visible to all clients.", "items": { "description": "Initializer is information about an initializer that has not yet completed.", "properties": { "name": { "description": "name of the process that is responsible for initializing this object.", "type": "string" } }, "required": [ "name" ] }, "type": "array" }, "result": { "description": "Status is a return value for calls that don't return other objects.", "properties": { "apiVersion": { "description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources", "type": "string" }, "code": { "description": "Suggested HTTP return code for this status, 0 if not set.", "format": "int32", "type": "integer" }, "details": { "description": "StatusDetails is a set of additional properties that MAY be set by the server to provide additional information about a response. The Reason field of a Status object defines what attributes will be set. Clients must ignore fields that do not match the defined type of each attribute, and should assume that any attribute may be empty, invalid, or under defined.", "properties": { "causes": { "description": "The Causes array includes more details associated with the StatusReason failure. Not all StatusReasons may provide detailed causes.", "items": { "description": "StatusCause provides more information about an api.Status failure, including cases when multiple errors are encountered.", "properties": { "field": { "description": "The field of the resource that has caused this error, as named by its JSON serialization. May include dot and postfix notation for nested attributes. Arrays are zero-indexed. Fields may appear more than once in an array of causes due to fields having multiple errors. Optional.\nExamples:\n \"name\" - the field \"name\" on the current resource\n \"items[0].name\" - the field \"name\" on the first array entry in \"items\"", "type": "string" }, "message": { "description": "A human-readable description of the cause of the error. This field may be presented as-is to a reader.", "type": "string" }, "reason": { "description": "A machine-readable description of the cause of the error. If this value is empty there is no information available.", "type": "string" } } }, "type": "array" }, "group": { "description": "The group attribute of the resource associated with the status StatusReason.", "type": "string" }, "kind": { "description": "The kind attribute of the resource associated with the status StatusReason. On some operations may differ from the requested resource Kind. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds", "type": "string" }, "name": { "description": "The name attribute of the resource associated with the status StatusReason (when there is a single name which can be described).", "type": "string" }, "retryAfterSeconds": { "description": "If specified, the time in seconds before the operation should be retried. Some errors may indicate the client must take an alternate action - for those errors this field may indicate how long to wait before taking the alternate action.", "format": "int32", "type": "integer" }, "uid": { "description": "UID of the resource. (when there is a single resource which can be described). More info: http://kubernetes.io/docs/user-guide/identifiers#uids", "type": "string" } } }, "kind": { "description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds", "type": "string" }, "message": { "description": "A human-readable description of the status of this operation.", "type": "string" }, "metadata": { "description": "ListMeta describes metadata that synthetic resources must have, including lists and various status objects. A resource may have only one of {ObjectMeta, ListMeta}.", "properties": { "continue": { "description": "continue may be set if the user set a limit on the number of items returned, and indicates that the server has more data available. The value is opaque and may be used to issue another request to the endpoint that served this list to retrieve the next set of available objects. Continuing a consistent list may not be possible if the server configuration has changed or more than a few minutes have passed. The resourceVersion field returned when using this continue value will be identical to the value in the first response, unless you have received this token from an error message.", "type": "string" }, "resourceVersion": { "description": "String that identifies the server's internal version of this object that can be used by clients to determine when objects have changed. Value must be treated as opaque by clients and passed unmodified back to the server. Populated by the system. Read-only. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency", "type": "string" }, "selfLink": { "description": "selfLink is a URL representing this object. Populated by the system. Read-only.", "type": "string" } } }, "reason": { "description": "A machine-readable description of why this operation is in the \"Failure\" status. If this value is empty there is no information available. A Reason clarifies an HTTP status code but does not override it.", "type": "string" }, "status": { "description": "Status of the operation. One of: \"Success\" or \"Failure\". More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status", "type": "string" } } } }, "required": [ "pending" ] }, "labels": { "description": "Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels", "type": "object" }, "name": { "description": "Name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names", "type": "string" }, "namespace": { "description": "Namespace defines the space within each name must be unique. An empty namespace is equivalent to the \"default\" namespace, but \"default\" is the canonical representation. Not all objects are required to be scoped to a namespace - the value of this field for those objects will be empty.\nMust be a DNS_LABEL. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/namespaces", "type": "string" }, "ownerReferences": { "description": "List of objects depended by this object. If ALL objects in the list have been deleted, this object will be garbage collected. If this object is managed by a controller, then an entry in this list will point to this controller, with the controller field set to true. There cannot be more than one managing controller.", "items": { "description": "OwnerReference contains enough information to let you identify an owning object. An owning object must be in the same namespace as the dependent, or be cluster-scoped, so there is no namespace field.", "properties": { "apiVersion": { "description": "API version of the referent.", "type": "string" }, "blockOwnerDeletion": { "description": "If true, AND if the owner has the \"foregroundDeletion\" finalizer, then the owner cannot be deleted from the key-value store until this reference is removed. Defaults to false. To set this field, a user needs \"delete\" permission of the owner, otherwise 422 (Unprocessable Entity) will be returned.", "type": "boolean" }, "controller": { "description": "If true, this reference points to the managing controller.", "type": "boolean" }, "kind": { "description": "Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds", "type": "string" }, "name": { "description": "Name of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#names", "type": "string" }, "uid": { "description": "UID of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#uids", "type": "string" } }, "required": [ "apiVersion", "kind", "name", "uid" ] }, "type": "array" }, "resourceVersion": { "description": "An opaque value that represents the internal version of this object that can be used by clients to determine when objects have changed. May be used for optimistic concurrency, change detection, and the watch operation on a resource or set of resources. Clients must treat these values as opaque and passed unmodified back to the server. They may only be valid for a particular resource or set of resources.\nPopulated by the system. Read-only. Value must be treated as opaque by clients and . More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency", "type": "string" }, "selfLink": { "description": "SelfLink is a URL representing this object. Populated by the system. Read-only.", "type": "string" }, "uid": { "description": "UID is the unique in time and space value for this object. It is typically generated by the server on successful creation of a resource and is not allowed to change on PUT operations.\nPopulated by the system. Read-only. More info: http://kubernetes.io/docs/user-guide/identifiers#uids", "type": "string" } } }, "spec": { "description": "PersistentVolumeClaimSpec describes the common attributes of storage devices and allows a Source for provider-specific attributes", "properties": { "accessModes": { "description": "AccessModes contains the desired access modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1", "items": { "type": "string" }, "type": "array" }, "dataSource": { "description": "TypedLocalObjectReference contains enough information to let you locate the typed referenced object inside the same namespace.", "properties": { "apiGroup": { "description": "APIGroup is the group for the resource being referenced. If APIGroup is not specified, the specified Kind must be in the core API group. For any other third-party types, APIGroup is required.", "type": "string" }, "kind": { "description": "Kind is the type of resource being referenced", "type": "string" }, "name": { "description": "Name is the name of resource being referenced", "type": "string" } }, "required": [ "kind", "name" ] }, "resources": { "description": "ResourceRequirements describes the compute resource requirements.", "properties": { "limits": { "description": "Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", "type": "object" }, "requests": { "description": "Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", "type": "object" } } }, "selector": { "description": "A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "items": { "type": "string" }, "type": "array" } }, "required": [ "key", "operator" ] }, "type": "array" }, "matchLabels": { "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object" } } }, "storageClassName": { "description": "Name of the StorageClass required by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1", "type": "string" }, "volumeMode": { "description": "volumeMode defines what type of volume is required by the claim. Value of Filesystem is implied when not included in claim spec. This is a beta feature.", "type": "string" }, "volumeName": { "description": "VolumeName is the binding reference to the PersistentVolume backing this claim.", "type": "string" } } }, "status": { "description": "PersistentVolumeClaimStatus is the current status of a persistent volume claim.", "properties": { "accessModes": { "description": "AccessModes contains the actual access modes the volume backing the PVC has. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1", "items": { "type": "string" }, "type": "array" }, "capacity": { "description": "Represents the actual resources of the underlying volume.", "type": "object" }, "conditions": { "description": "Current Condition of persistent volume claim. If underlying persistent volume is being resized then the Condition will be set to 'ResizeStarted'.", "items": { "description": "PersistentVolumeClaimCondition contains details about state of pvc", "properties": { "lastProbeTime": { "description": "Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.", "format": "date-time", "type": "string" }, "lastTransitionTime": { "description": "Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.", "format": "date-time", "type": "string" }, "message": { "description": "Human-readable message indicating details about last transition.", "type": "string" }, "reason": { "description": "Unique, this should be a short, machine understandable string that gives the reason for condition's last transition. If it reports \"ResizeStarted\" that means the underlying persistent volume is being resized.", "type": "string" }, "status": { "type": "string" }, "type": { "type": "string" } }, "required": [ "type", "status" ] }, "type": "array" }, "phase": { "description": "Phase represents the current phase of PersistentVolumeClaim.", "type": "string" } } } } } } }, "tag": { "description": "Tag of Alertmanager container image to be deployed. Defaults to the value of `version`. Version is ignored if Tag is set.", "type": "string" }, "tolerations": { "description": "If specified, the pod's tolerations.", "items": { "description": "The pod this Toleration is attached to tolerates any taint that matches the triple \u003ckey,value,effect\u003e using the matching operator \u003coperator\u003e.", "properties": { "effect": { "description": "Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.", "type": "string" }, "key": { "description": "Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys.", "type": "string" }, "operator": { "description": "Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category.", "type": "string" }, "tolerationSeconds": { "description": "TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system.", "format": "int64", "type": "integer" }, "value": { "description": "Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string.", "type": "string" } } }, "type": "array" }, "version": { "description": "Version the cluster should be on.", "type": "string" } } }, "status": { "description": "AlertmanagerStatus is the most recent observed status of the Alertmanager cluster. Read-only. Not included when requesting from the apiserver, only from the Prometheus Operator API itself. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/api-conventions.md#spec-and-status", "properties": { "availableReplicas": { "description": "Total number of available pods (ready for at least minReadySeconds) targeted by this Alertmanager cluster.", "format": "int32", "type": "integer" }, "paused": { "description": "Represents whether any actions on the underlying managed objects are being performed. Only delete actions will be performed.", "type": "boolean" }, "replicas": { "description": "Total number of non-terminated pods targeted by this Alertmanager cluster (their labels match the selector).", "format": "int32", "type": "integer" }, "unavailableReplicas": { "description": "Total number of unavailable pods targeted by this Alertmanager cluster.", "format": "int32", "type": "integer" }, "updatedReplicas": { "description": "Total number of non-terminated pods targeted by this Alertmanager cluster that have the desired version spec.", "format": "int32", "type": "integer" } }, "required": [ "paused", "replicas", "updatedReplicas", "availableReplicas", "unavailableReplicas" ] } } } }, "served": true, "storage": true } ] }, "status": { "acceptedNames": { "kind": "Alertmanager", "listKind": "AlertmanagerList", "plural": "alertmanagers", "singular": "alertmanager" }, "conditions": [ { "lastTransitionTime": "2020-05-05T16:51:39Z", "message": "no conflicts found", "reason": "NoConflicts", "status": "True", "type": "NamesAccepted" }, { "lastTransitionTime": "2020-05-05T16:51:39Z", "message": "the initial names have been accepted", "reason": "InitialNamesAccepted", "status": "True", "type": "Established" }, { "lastTransitionTime": "2020-05-05T16:51:39Z", "message": "[spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[nodeAffinity].properties[preferredDuringSchedulingIgnoredDuringExecution].items.properties[preference].properties[matchExpressions].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[nodeAffinity].properties[preferredDuringSchedulingIgnoredDuringExecution].items.properties[preference].properties[matchFields].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[nodeAffinity].properties[preferredDuringSchedulingIgnoredDuringExecution].items.properties[preference].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[nodeAffinity].properties[preferredDuringSchedulingIgnoredDuringExecution].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[nodeAffinity].properties[requiredDuringSchedulingIgnoredDuringExecution].properties[nodeSelectorTerms].items.properties[matchExpressions].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[nodeAffinity].properties[requiredDuringSchedulingIgnoredDuringExecution].properties[nodeSelectorTerms].items.properties[matchFields].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[nodeAffinity].properties[requiredDuringSchedulingIgnoredDuringExecution].properties[nodeSelectorTerms].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[nodeAffinity].properties[requiredDuringSchedulingIgnoredDuringExecution].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[nodeAffinity].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[podAffinity].properties[preferredDuringSchedulingIgnoredDuringExecution].items.properties[podAffinityTerm].properties[labelSelector].properties[matchExpressions].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[podAffinity].properties[preferredDuringSchedulingIgnoredDuringExecution].items.properties[podAffinityTerm].properties[labelSelector].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[podAffinity].properties[preferredDuringSchedulingIgnoredDuringExecution].items.properties[podAffinityTerm].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[podAffinity].properties[preferredDuringSchedulingIgnoredDuringExecution].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[podAffinity].properties[requiredDuringSchedulingIgnoredDuringExecution].items.properties[labelSelector].properties[matchExpressions].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[podAffinity].properties[requiredDuringSchedulingIgnoredDuringExecution].items.properties[labelSelector].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[podAffinity].properties[requiredDuringSchedulingIgnoredDuringExecution].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[podAffinity].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[podAntiAffinity].properties[preferredDuringSchedulingIgnoredDuringExecution].items.properties[podAffinityTerm].properties[labelSelector].properties[matchExpressions].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[podAntiAffinity].properties[preferredDuringSchedulingIgnoredDuringExecution].items.properties[podAffinityTerm].properties[labelSelector].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[podAntiAffinity].properties[preferredDuringSchedulingIgnoredDuringExecution].items.properties[podAffinityTerm].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[podAntiAffinity].properties[preferredDuringSchedulingIgnoredDuringExecution].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[podAntiAffinity].properties[requiredDuringSchedulingIgnoredDuringExecution].items.properties[labelSelector].properties[matchExpressions].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[podAntiAffinity].properties[requiredDuringSchedulingIgnoredDuringExecution].items.properties[labelSelector].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[podAntiAffinity].properties[requiredDuringSchedulingIgnoredDuringExecution].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[podAntiAffinity].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[envFrom].items.properties[configMapRef].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[envFrom].items.properties[secretRef].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[envFrom].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[env].items.properties[valueFrom].properties[configMapKeyRef].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[env].items.properties[valueFrom].properties[fieldRef].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[env].items.properties[valueFrom].properties[resourceFieldRef].properties[divisor].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[env].items.properties[valueFrom].properties[resourceFieldRef].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[env].items.properties[valueFrom].properties[secretKeyRef].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[env].items.properties[valueFrom].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[env].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[postStart].properties[exec].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[postStart].properties[httpGet].properties[httpHeaders].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[postStart].properties[httpGet].properties[port].anyOf[0].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[postStart].properties[httpGet].properties[port].anyOf[1].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[postStart].properties[httpGet].properties[port].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[postStart].properties[httpGet].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[postStart].properties[tcpSocket].properties[port].anyOf[0].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[postStart].properties[tcpSocket].properties[port].anyOf[1].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[postStart].properties[tcpSocket].properties[port].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[postStart].properties[tcpSocket].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[postStart].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[preStop].properties[exec].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[preStop].properties[httpGet].properties[httpHeaders].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[preStop].properties[httpGet].properties[port].anyOf[0].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[preStop].properties[httpGet].properties[port].anyOf[1].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[preStop].properties[httpGet].properties[port].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[preStop].properties[httpGet].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[preStop].properties[tcpSocket].properties[port].anyOf[0].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[preStop].properties[tcpSocket].properties[port].anyOf[1].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[preStop].properties[tcpSocket].properties[port].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[preStop].properties[tcpSocket].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[preStop].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[livenessProbe].properties[exec].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[livenessProbe].properties[httpGet].properties[httpHeaders].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[livenessProbe].properties[httpGet].properties[port].anyOf[0].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[livenessProbe].properties[httpGet].properties[port].anyOf[1].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[livenessProbe].properties[httpGet].properties[port].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[livenessProbe].properties[httpGet].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[livenessProbe].properties[tcpSocket].properties[port].anyOf[0].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[livenessProbe].properties[tcpSocket].properties[port].anyOf[1].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[livenessProbe].properties[tcpSocket].properties[port].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[livenessProbe].properties[tcpSocket].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[livenessProbe].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[ports].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[readinessProbe].properties[exec].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[readinessProbe].properties[httpGet].properties[httpHeaders].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[readinessProbe].properties[httpGet].properties[port].anyOf[0].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[readinessProbe].properties[httpGet].properties[port].anyOf[1].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[readinessProbe].properties[httpGet].properties[port].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[readinessProbe].properties[httpGet].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[readinessProbe].properties[tcpSocket].properties[port].anyOf[0].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[readinessProbe].properties[tcpSocket].properties[port].anyOf[1].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[readinessProbe].properties[tcpSocket].properties[port].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[readinessProbe].properties[tcpSocket].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[readinessProbe].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[resources].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[securityContext].properties[capabilities].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[securityContext].properties[seLinuxOptions].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[securityContext].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[volumeDevices].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[volumeMounts].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[imagePullSecrets].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[podMetadata].properties[initializers].properties[pending].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[podMetadata].properties[initializers].properties[result].properties[details].properties[causes].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[podMetadata].properties[initializers].properties[result].properties[details].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[podMetadata].properties[initializers].properties[result].properties[metadata].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[podMetadata].properties[initializers].properties[result].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[podMetadata].properties[initializers].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[podMetadata].properties[ownerReferences].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[podMetadata].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[resources].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[securityContext].properties[seLinuxOptions].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[securityContext].properties[sysctls].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[securityContext].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[storage].properties[emptyDir].properties[sizeLimit].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[storage].properties[emptyDir].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[storage].properties[volumeClaimTemplate].properties[metadata].properties[initializers].properties[pending].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[storage].properties[volumeClaimTemplate].properties[metadata].properties[initializers].properties[result].properties[details].properties[causes].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[storage].properties[volumeClaimTemplate].properties[metadata].properties[initializers].properties[result].properties[details].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[storage].properties[volumeClaimTemplate].properties[metadata].properties[initializers].properties[result].properties[metadata].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[storage].properties[volumeClaimTemplate].properties[metadata].properties[initializers].properties[result].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[storage].properties[volumeClaimTemplate].properties[metadata].properties[initializers].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[storage].properties[volumeClaimTemplate].properties[metadata].properties[ownerReferences].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[storage].properties[volumeClaimTemplate].properties[metadata].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[storage].properties[volumeClaimTemplate].properties[spec].properties[dataSource].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[storage].properties[volumeClaimTemplate].properties[spec].properties[resources].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[storage].properties[volumeClaimTemplate].properties[spec].properties[selector].properties[matchExpressions].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[storage].properties[volumeClaimTemplate].properties[spec].properties[selector].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[storage].properties[volumeClaimTemplate].properties[spec].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[storage].properties[volumeClaimTemplate].properties[status].properties[conditions].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[storage].properties[volumeClaimTemplate].properties[status].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[storage].properties[volumeClaimTemplate].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[storage].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[tolerations].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[status].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.type: Required value: must not be empty at the root]", "reason": "Violations", "status": "True", "type": "NonStructuralSchema" } ], "storedVersions": [ "v1" ] } } ================================================ FILE: pkg/backup/actions/testdata/v1/elasticsearches.elasticsearch.k8s.elastic.co.json ================================================ { "apiVersion": "apiextensions.k8s.io/v1", "kind": "CustomResourceDefinition", "metadata": { "annotations": { "controller-gen.kubebuilder.io/version": "v0.2.5", "kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"apiextensions.k8s.io/v1beta1\",\"kind\":\"CustomResourceDefinition\",\"metadata\":{\"annotations\":{\"controller-gen.kubebuilder.io/version\":\"v0.2.5\"},\"creationTimestamp\":null,\"name\":\"elasticsearches.elasticsearch.k8s.elastic.co\"},\"spec\":{\"additionalPrinterColumns\":[{\"JSONPath\":\".status.health\",\"name\":\"health\",\"type\":\"string\"},{\"JSONPath\":\".status.availableNodes\",\"description\":\"Available nodes\",\"name\":\"nodes\",\"type\":\"integer\"},{\"JSONPath\":\".spec.version\",\"description\":\"Elasticsearch version\",\"name\":\"version\",\"type\":\"string\"},{\"JSONPath\":\".status.phase\",\"name\":\"phase\",\"type\":\"string\"},{\"JSONPath\":\".metadata.creationTimestamp\",\"name\":\"age\",\"type\":\"date\"}],\"group\":\"elasticsearch.k8s.elastic.co\",\"names\":{\"categories\":[\"elastic\"],\"kind\":\"Elasticsearch\",\"listKind\":\"ElasticsearchList\",\"plural\":\"elasticsearches\",\"shortNames\":[\"es\"],\"singular\":\"elasticsearch\"},\"scope\":\"Namespaced\",\"subresources\":{\"status\":{}},\"validation\":{\"openAPIV3Schema\":{\"description\":\"Elasticsearch represents an Elasticsearch resource in a Kubernetes cluster.\",\"properties\":{\"apiVersion\":{\"description\":\"APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\",\"type\":\"string\"},\"kind\":{\"description\":\"Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\",\"type\":\"string\"},\"metadata\":{\"type\":\"object\"},\"spec\":{\"description\":\"ElasticsearchSpec holds the specification of an Elasticsearch cluster.\",\"properties\":{\"auth\":{\"description\":\"Auth contains user authentication and authorization security settings for Elasticsearch.\",\"properties\":{\"fileRealm\":{\"description\":\"FileRealm to propagate to the Elasticsearch cluster.\",\"items\":{\"description\":\"FileRealmSource references users to create in the Elasticsearch cluster.\",\"properties\":{\"secretName\":{\"description\":\"SecretName is the name of the secret.\",\"type\":\"string\"}},\"type\":\"object\"},\"type\":\"array\"},\"roles\":{\"description\":\"Roles to propagate to the Elasticsearch cluster.\",\"items\":{\"description\":\"RoleSource references roles to create in the Elasticsearch cluster.\",\"properties\":{\"secretName\":{\"description\":\"SecretName is the name of the secret.\",\"type\":\"string\"}},\"type\":\"object\"},\"type\":\"array\"}},\"type\":\"object\"},\"http\":{\"description\":\"HTTP holds HTTP layer settings for Elasticsearch.\",\"properties\":{\"service\":{\"description\":\"Service defines the template for the associated Kubernetes Service object.\",\"properties\":{\"metadata\":{\"description\":\"ObjectMeta is the metadata of the service. The name and namespace provided here are managed by ECK and will be ignored.\",\"type\":\"object\"},\"spec\":{\"description\":\"Spec is the specification of the service.\",\"properties\":{\"clusterIP\":{\"description\":\"clusterIP is the IP address of the service and is usually assigned randomly by the master. If an address is specified manually and is not in use by others, it will be allocated to the service; otherwise, creation of the service will fail. This field can not be changed through updates. Valid values are \\\"None\\\", empty string (\\\"\\\"), or a valid IP address. \\\"None\\\" can be specified for headless services when proxying is not required. Only applies to types ClusterIP, NodePort, and LoadBalancer. Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies\",\"type\":\"string\"},\"externalIPs\":{\"description\":\"externalIPs is a list of IP addresses for which nodes in the cluster will also accept traffic for this service. These IPs are not managed by Kubernetes. The user is responsible for ensuring that traffic arrives at a node with this IP. A common example is external load-balancers that are not part of the Kubernetes system.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"externalName\":{\"description\":\"externalName is the external reference that kubedns or equivalent will return as a CNAME record for this service. No proxying will be involved. Must be a valid RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) and requires Type to be ExternalName.\",\"type\":\"string\"},\"externalTrafficPolicy\":{\"description\":\"externalTrafficPolicy denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints. \\\"Local\\\" preserves the client source IP and avoids a second hop for LoadBalancer and Nodeport type services, but risks potentially imbalanced traffic spreading. \\\"Cluster\\\" obscures the client source IP and may cause a second hop to another node, but should have good overall load-spreading.\",\"type\":\"string\"},\"healthCheckNodePort\":{\"description\":\"healthCheckNodePort specifies the healthcheck nodePort for the service. If not specified, HealthCheckNodePort is created by the service api backend with the allocated nodePort. Will use user-specified nodePort value if specified by the client. Only effects when Type is set to LoadBalancer and ExternalTrafficPolicy is set to Local.\",\"format\":\"int32\",\"type\":\"integer\"},\"ipFamily\":{\"description\":\"ipFamily specifies whether this Service has a preference for a particular IP family (e.g. IPv4 vs. IPv6). If a specific IP family is requested, the clusterIP field will be allocated from that family, if it is available in the cluster. If no IP family is requested, the cluster's primary IP family will be used. Other IP fields (loadBalancerIP, loadBalancerSourceRanges, externalIPs) and controllers which allocate external load-balancers should use the same IP family. Endpoints for this Service will be of this family. This field is immutable after creation. Assigning a ServiceIPFamily not available in the cluster (e.g. IPv6 in IPv4 only cluster) is an error condition and will fail during clusterIP assignment.\",\"type\":\"string\"},\"loadBalancerIP\":{\"description\":\"Only applies to Service Type: LoadBalancer LoadBalancer will get created with the IP specified in this field. This feature depends on whether the underlying cloud-provider supports specifying the loadBalancerIP when a load balancer is created. This field will be ignored if the cloud-provider does not support the feature.\",\"type\":\"string\"},\"loadBalancerSourceRanges\":{\"description\":\"If specified and supported by the platform, this will restrict traffic through the cloud-provider load-balancer will be restricted to the specified client IPs. This field will be ignored if the cloud-provider does not support the feature.\\\" More info: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"ports\":{\"description\":\"The list of ports that are exposed by this service. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies\",\"items\":{\"description\":\"ServicePort contains information on service's port.\",\"properties\":{\"name\":{\"description\":\"The name of this port within the service. This must be a DNS_LABEL. All ports within a ServiceSpec must have unique names. When considering the endpoints for a Service, this must match the 'name' field in the EndpointPort. Optional if only one ServicePort is defined on this service.\",\"type\":\"string\"},\"nodePort\":{\"description\":\"The port on each node on which this service is exposed when type=NodePort or LoadBalancer. Usually assigned by the system. If specified, it will be allocated to the service if unused or else creation of the service will fail. Default is to auto-allocate a port if the ServiceType of this Service requires one. More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport\",\"format\":\"int32\",\"type\":\"integer\"},\"port\":{\"description\":\"The port that will be exposed by this service.\",\"format\":\"int32\",\"type\":\"integer\"},\"protocol\":{\"description\":\"The IP protocol for this port. Supports \\\"TCP\\\", \\\"UDP\\\", and \\\"SCTP\\\". Default is TCP.\",\"type\":\"string\"},\"targetPort\":{\"anyOf\":[{\"type\":\"integer\"},{\"type\":\"string\"}],\"description\":\"Number or name of the port to access on the pods targeted by the service. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. If this is a string, it will be looked up as a named port in the target Pod's container ports. If this is not specified, the value of the 'port' field is used (an identity map). This field is ignored for services with clusterIP=None, and should be omitted or set equal to the 'port' field. More info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service\"}},\"required\":[\"port\"],\"type\":\"object\"},\"type\":\"array\"},\"publishNotReadyAddresses\":{\"description\":\"publishNotReadyAddresses, when set to true, indicates that DNS implementations must publish the notReadyAddresses of subsets for the Endpoints associated with the Service. The default value is false. The primary use case for setting this field is to use a StatefulSet's Headless Service to propagate SRV records for its Pods without respect to their readiness for purpose of peer discovery.\",\"type\":\"boolean\"},\"selector\":{\"additionalProperties\":{\"type\":\"string\"},\"description\":\"Route service traffic to pods with label keys and values matching this selector. If empty or not present, the service is assumed to have an external process managing its endpoints, which Kubernetes will not modify. Only applies to types ClusterIP, NodePort, and LoadBalancer. Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/\",\"type\":\"object\"},\"sessionAffinity\":{\"description\":\"Supports \\\"ClientIP\\\" and \\\"None\\\". Used to maintain session affinity. Enable client IP based session affinity. Must be ClientIP or None. Defaults to None. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies\",\"type\":\"string\"},\"sessionAffinityConfig\":{\"description\":\"sessionAffinityConfig contains the configurations of session affinity.\",\"properties\":{\"clientIP\":{\"description\":\"clientIP contains the configurations of Client IP based session affinity.\",\"properties\":{\"timeoutSeconds\":{\"description\":\"timeoutSeconds specifies the seconds of ClientIP type session sticky time. The value must be \\u003e0 \\u0026\\u0026 \\u003c=86400(for 1 day) if ServiceAffinity == \\\"ClientIP\\\". Default value is 10800(for 3 hours).\",\"format\":\"int32\",\"type\":\"integer\"}},\"type\":\"object\"}},\"type\":\"object\"},\"topologyKeys\":{\"description\":\"topologyKeys is a preference-order list of topology keys which implementations of services should use to preferentially sort endpoints when accessing this Service, it can not be used at the same time as externalTrafficPolicy=Local. Topology keys must be valid label keys and at most 16 keys may be specified. Endpoints are chosen based on the first topology key with available backends. If this field is specified and all entries have no backends that match the topology of the client, the service has no backends for that client and connections should fail. The special value \\\"*\\\" may be used to mean \\\"any topology\\\". This catch-all value, if used, only makes sense as the last value in the list. If this is not specified or empty, no topology constraints will be applied.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"type\":{\"description\":\"type determines how the Service is exposed. Defaults to ClusterIP. Valid options are ExternalName, ClusterIP, NodePort, and LoadBalancer. \\\"ExternalName\\\" maps to the specified externalName. \\\"ClusterIP\\\" allocates a cluster-internal IP address for load-balancing to endpoints. Endpoints are determined by the selector or if that is not specified, by manual construction of an Endpoints object. If clusterIP is \\\"None\\\", no virtual IP is allocated and the endpoints are published as a set of endpoints rather than a stable IP. \\\"NodePort\\\" builds on ClusterIP and allocates a port on every node which routes to the clusterIP. \\\"LoadBalancer\\\" builds on NodePort and creates an external load-balancer (if supported in the current cloud) which routes to the clusterIP. More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types\",\"type\":\"string\"}},\"type\":\"object\"}},\"type\":\"object\"},\"tls\":{\"description\":\"TLS defines options for configuring TLS for HTTP.\",\"properties\":{\"certificate\":{\"description\":\"Certificate is a reference to a Kubernetes secret that contains the certificate and private key for enabling TLS. The referenced secret should contain the following: \\n - `ca.crt`: The certificate authority (optional). - `tls.crt`: The certificate (or a chain). - `tls.key`: The private key to the first certificate in the certificate chain.\",\"properties\":{\"secretName\":{\"description\":\"SecretName is the name of the secret.\",\"type\":\"string\"}},\"type\":\"object\"},\"selfSignedCertificate\":{\"description\":\"SelfSignedCertificate allows configuring the self-signed certificate generated by the operator.\",\"properties\":{\"disabled\":{\"description\":\"Disabled indicates that the provisioning of the self-signed certificate should be disabled.\",\"type\":\"boolean\"},\"subjectAltNames\":{\"description\":\"SubjectAlternativeNames is a list of SANs to include in the generated HTTP TLS certificate.\",\"items\":{\"description\":\"SubjectAlternativeName represents a SAN entry in a x509 certificate.\",\"properties\":{\"dns\":{\"description\":\"DNS is the DNS name of the subject.\",\"type\":\"string\"},\"ip\":{\"description\":\"IP is the IP address of the subject.\",\"type\":\"string\"}},\"type\":\"object\"},\"type\":\"array\"}},\"type\":\"object\"}},\"type\":\"object\"}},\"type\":\"object\"},\"image\":{\"description\":\"Image is the Elasticsearch Docker image to deploy.\",\"type\":\"string\"},\"nodeSets\":{\"description\":\"NodeSets allow specifying groups of Elasticsearch nodes sharing the same configuration and Pod templates. See: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-orchestration.html\",\"items\":{\"description\":\"NodeSet is the specification for a group of Elasticsearch nodes sharing the same configuration and a Pod template.\",\"properties\":{\"config\":{\"description\":\"Config holds the Elasticsearch configuration.\",\"type\":\"object\"},\"count\":{\"description\":\"Count of Elasticsearch nodes to deploy.\",\"format\":\"int32\",\"minimum\":1,\"type\":\"integer\"},\"name\":{\"description\":\"Name of this set of nodes. Becomes a part of the Elasticsearch node.name setting.\",\"maxLength\":23,\"pattern\":\"[a-zA-Z0-9-]+\",\"type\":\"string\"},\"podTemplate\":{\"description\":\"PodTemplate provides customisation options (labels, annotations, affinity rules, resource requests, and so on) for the Pods belonging to this NodeSet.\",\"type\":\"object\"},\"volumeClaimTemplates\":{\"description\":\"VolumeClaimTemplates is a list of persistent volume claims to be used by each Pod in this NodeSet. Every claim in this list must have a matching volumeMount in one of the containers defined in the PodTemplate. Items defined here take precedence over any default claims added by the operator with the same name. See: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-volume-claim-templates.html\",\"items\":{\"description\":\"PersistentVolumeClaim is a user's request for and claim to a persistent volume\",\"properties\":{\"apiVersion\":{\"description\":\"APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\",\"type\":\"string\"},\"kind\":{\"description\":\"Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\",\"type\":\"string\"},\"metadata\":{\"description\":\"Standard object's metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata\",\"type\":\"object\"},\"spec\":{\"description\":\"Spec defines the desired characteristics of a volume requested by a pod author. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims\",\"properties\":{\"accessModes\":{\"description\":\"AccessModes contains the desired access modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"dataSource\":{\"description\":\"This field requires the VolumeSnapshotDataSource alpha feature gate to be enabled and currently VolumeSnapshot is the only supported data source. If the provisioner can support VolumeSnapshot data source, it will create a new volume and data will be restored to the volume at the same time. If the provisioner does not support VolumeSnapshot data source, volume will not be created and the failure will be reported as an event. In the future, we plan to support more data source types and the behavior of the provisioner may change.\",\"properties\":{\"apiGroup\":{\"description\":\"APIGroup is the group for the resource being referenced. If APIGroup is not specified, the specified Kind must be in the core API group. For any other third-party types, APIGroup is required.\",\"type\":\"string\"},\"kind\":{\"description\":\"Kind is the type of resource being referenced\",\"type\":\"string\"},\"name\":{\"description\":\"Name is the name of resource being referenced\",\"type\":\"string\"}},\"required\":[\"kind\",\"name\"],\"type\":\"object\"},\"resources\":{\"description\":\"Resources represents the minimum resources the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources\",\"properties\":{\"limits\":{\"additionalProperties\":{\"anyOf\":[{\"type\":\"integer\"},{\"type\":\"string\"}],\"pattern\":\"^(\\\\+|-)?(([0-9]+(\\\\.[0-9]*)?)|(\\\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\\\+|-)?(([0-9]+(\\\\.[0-9]*)?)|(\\\\.[0-9]+))))?$\"},\"description\":\"Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/\",\"type\":\"object\"},\"requests\":{\"additionalProperties\":{\"anyOf\":[{\"type\":\"integer\"},{\"type\":\"string\"}],\"pattern\":\"^(\\\\+|-)?(([0-9]+(\\\\.[0-9]*)?)|(\\\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\\\+|-)?(([0-9]+(\\\\.[0-9]*)?)|(\\\\.[0-9]+))))?$\"},\"description\":\"Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/\",\"type\":\"object\"}},\"type\":\"object\"},\"selector\":{\"description\":\"A label query over volumes to consider for binding.\",\"properties\":{\"matchExpressions\":{\"description\":\"matchExpressions is a list of label selector requirements. The requirements are ANDed.\",\"items\":{\"description\":\"A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"key is the label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\",\"type\":\"string\"},\"values\":{\"description\":\"values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"],\"type\":\"object\"},\"type\":\"array\"},\"matchLabels\":{\"additionalProperties\":{\"type\":\"string\"},\"description\":\"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \\\"key\\\", the operator is \\\"In\\\", and the values array contains only \\\"value\\\". The requirements are ANDed.\",\"type\":\"object\"}},\"type\":\"object\"},\"storageClassName\":{\"description\":\"Name of the StorageClass required by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1\",\"type\":\"string\"},\"volumeMode\":{\"description\":\"volumeMode defines what type of volume is required by the claim. Value of Filesystem is implied when not included in claim spec. This is a beta feature.\",\"type\":\"string\"},\"volumeName\":{\"description\":\"VolumeName is the binding reference to the PersistentVolume backing this claim.\",\"type\":\"string\"}},\"type\":\"object\"},\"status\":{\"description\":\"Status represents the current information/status of a persistent volume claim. Read-only. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims\",\"properties\":{\"accessModes\":{\"description\":\"AccessModes contains the actual access modes the volume backing the PVC has. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"capacity\":{\"additionalProperties\":{\"anyOf\":[{\"type\":\"integer\"},{\"type\":\"string\"}],\"pattern\":\"^(\\\\+|-)?(([0-9]+(\\\\.[0-9]*)?)|(\\\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\\\+|-)?(([0-9]+(\\\\.[0-9]*)?)|(\\\\.[0-9]+))))?$\"},\"description\":\"Represents the actual resources of the underlying volume.\",\"type\":\"object\"},\"conditions\":{\"description\":\"Current Condition of persistent volume claim. If underlying persistent volume is being resized then the Condition will be set to 'ResizeStarted'.\",\"items\":{\"description\":\"PersistentVolumeClaimCondition contains details about state of pvc\",\"properties\":{\"lastProbeTime\":{\"description\":\"Last time we probed the condition.\",\"format\":\"date-time\",\"type\":\"string\"},\"lastTransitionTime\":{\"description\":\"Last time the condition transitioned from one status to another.\",\"format\":\"date-time\",\"type\":\"string\"},\"message\":{\"description\":\"Human-readable message indicating details about last transition.\",\"type\":\"string\"},\"reason\":{\"description\":\"Unique, this should be a short, machine understandable string that gives the reason for condition's last transition. If it reports \\\"ResizeStarted\\\" that means the underlying persistent volume is being resized.\",\"type\":\"string\"},\"status\":{\"type\":\"string\"},\"type\":{\"description\":\"PersistentVolumeClaimConditionType is a valid value of PersistentVolumeClaimCondition.Type\",\"type\":\"string\"}},\"required\":[\"status\",\"type\"],\"type\":\"object\"},\"type\":\"array\"},\"phase\":{\"description\":\"Phase represents the current phase of PersistentVolumeClaim.\",\"type\":\"string\"}},\"type\":\"object\"}},\"type\":\"object\"},\"type\":\"array\"}},\"required\":[\"count\",\"name\"],\"type\":\"object\"},\"minItems\":1,\"type\":\"array\"},\"podDisruptionBudget\":{\"description\":\"PodDisruptionBudget provides access to the default pod disruption budget for the Elasticsearch cluster. The default budget selects all cluster pods and sets `maxUnavailable` to 1. To disable, set `PodDisruptionBudget` to the empty value (`{}` in YAML).\",\"properties\":{\"metadata\":{\"description\":\"ObjectMeta is the metadata of the PDB. The name and namespace provided here are managed by ECK and will be ignored.\",\"type\":\"object\"},\"spec\":{\"description\":\"Spec is the specification of the PDB.\",\"properties\":{\"maxUnavailable\":{\"anyOf\":[{\"type\":\"integer\"},{\"type\":\"string\"}],\"description\":\"An eviction is allowed if at most \\\"maxUnavailable\\\" pods selected by \\\"selector\\\" are unavailable after the eviction, i.e. even in absence of the evicted pod. For example, one can prevent all voluntary evictions by specifying 0. This is a mutually exclusive setting with \\\"minAvailable\\\".\"},\"minAvailable\":{\"anyOf\":[{\"type\":\"integer\"},{\"type\":\"string\"}],\"description\":\"An eviction is allowed if at least \\\"minAvailable\\\" pods selected by \\\"selector\\\" will still be available after the eviction, i.e. even in the absence of the evicted pod. So for example you can prevent all voluntary evictions by specifying \\\"100%\\\".\"},\"selector\":{\"description\":\"Label query over pods whose evictions are managed by the disruption budget.\",\"properties\":{\"matchExpressions\":{\"description\":\"matchExpressions is a list of label selector requirements. The requirements are ANDed.\",\"items\":{\"description\":\"A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"key is the label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\",\"type\":\"string\"},\"values\":{\"description\":\"values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"],\"type\":\"object\"},\"type\":\"array\"},\"matchLabels\":{\"additionalProperties\":{\"type\":\"string\"},\"description\":\"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \\\"key\\\", the operator is \\\"In\\\", and the values array contains only \\\"value\\\". The requirements are ANDed.\",\"type\":\"object\"}},\"type\":\"object\"}},\"type\":\"object\"}},\"type\":\"object\"},\"remoteClusters\":{\"description\":\"RemoteClusters enables you to establish uni-directional connections to a remote Elasticsearch cluster.\",\"items\":{\"description\":\"RemoteCluster declares a remote Elasticsearch cluster connection.\",\"properties\":{\"elasticsearchRef\":{\"description\":\"ElasticsearchRef is a reference to an Elasticsearch cluster running within the same k8s cluster.\",\"properties\":{\"name\":{\"description\":\"Name of the Kubernetes object.\",\"type\":\"string\"},\"namespace\":{\"description\":\"Namespace of the Kubernetes object. If empty, defaults to the current namespace.\",\"type\":\"string\"}},\"required\":[\"name\"],\"type\":\"object\"},\"name\":{\"description\":\"Name is the name of the remote cluster as it is set in the Elasticsearch settings. The name is expected to be unique for each remote clusters.\",\"minLength\":1,\"type\":\"string\"}},\"required\":[\"name\"],\"type\":\"object\"},\"type\":\"array\"},\"secureSettings\":{\"description\":\"SecureSettings is a list of references to Kubernetes secrets containing sensitive configuration options for Elasticsearch. See: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-es-secure-settings.html\",\"items\":{\"description\":\"SecretSource defines a data source based on a Kubernetes Secret.\",\"properties\":{\"entries\":{\"description\":\"Entries define how to project each key-value pair in the secret to filesystem paths. If not defined, all keys will be projected to similarly named paths in the filesystem. If defined, only the specified keys will be projected to the corresponding paths.\",\"items\":{\"description\":\"KeyToPath defines how to map a key in a Secret object to a filesystem path.\",\"properties\":{\"key\":{\"description\":\"Key is the key contained in the secret.\",\"type\":\"string\"},\"path\":{\"description\":\"Path is the relative file path to map the key to. Path must not be an absolute file path and must not contain any \\\"..\\\" components.\",\"type\":\"string\"}},\"required\":[\"key\"],\"type\":\"object\"},\"type\":\"array\"},\"secretName\":{\"description\":\"SecretName is the name of the secret.\",\"type\":\"string\"}},\"required\":[\"secretName\"],\"type\":\"object\"},\"type\":\"array\"},\"serviceAccountName\":{\"description\":\"ServiceAccountName is used to check access from the current resource to a resource (eg. a remote Elasticsearch cluster) in a different namespace. Can only be used if ECK is enforcing RBAC on references.\",\"type\":\"string\"},\"transport\":{\"description\":\"Transport holds transport layer settings for Elasticsearch.\",\"properties\":{\"service\":{\"description\":\"Service defines the template for the associated Kubernetes Service object.\",\"properties\":{\"metadata\":{\"description\":\"ObjectMeta is the metadata of the service. The name and namespace provided here are managed by ECK and will be ignored.\",\"type\":\"object\"},\"spec\":{\"description\":\"Spec is the specification of the service.\",\"properties\":{\"clusterIP\":{\"description\":\"clusterIP is the IP address of the service and is usually assigned randomly by the master. If an address is specified manually and is not in use by others, it will be allocated to the service; otherwise, creation of the service will fail. This field can not be changed through updates. Valid values are \\\"None\\\", empty string (\\\"\\\"), or a valid IP address. \\\"None\\\" can be specified for headless services when proxying is not required. Only applies to types ClusterIP, NodePort, and LoadBalancer. Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies\",\"type\":\"string\"},\"externalIPs\":{\"description\":\"externalIPs is a list of IP addresses for which nodes in the cluster will also accept traffic for this service. These IPs are not managed by Kubernetes. The user is responsible for ensuring that traffic arrives at a node with this IP. A common example is external load-balancers that are not part of the Kubernetes system.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"externalName\":{\"description\":\"externalName is the external reference that kubedns or equivalent will return as a CNAME record for this service. No proxying will be involved. Must be a valid RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) and requires Type to be ExternalName.\",\"type\":\"string\"},\"externalTrafficPolicy\":{\"description\":\"externalTrafficPolicy denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints. \\\"Local\\\" preserves the client source IP and avoids a second hop for LoadBalancer and Nodeport type services, but risks potentially imbalanced traffic spreading. \\\"Cluster\\\" obscures the client source IP and may cause a second hop to another node, but should have good overall load-spreading.\",\"type\":\"string\"},\"healthCheckNodePort\":{\"description\":\"healthCheckNodePort specifies the healthcheck nodePort for the service. If not specified, HealthCheckNodePort is created by the service api backend with the allocated nodePort. Will use user-specified nodePort value if specified by the client. Only effects when Type is set to LoadBalancer and ExternalTrafficPolicy is set to Local.\",\"format\":\"int32\",\"type\":\"integer\"},\"ipFamily\":{\"description\":\"ipFamily specifies whether this Service has a preference for a particular IP family (e.g. IPv4 vs. IPv6). If a specific IP family is requested, the clusterIP field will be allocated from that family, if it is available in the cluster. If no IP family is requested, the cluster's primary IP family will be used. Other IP fields (loadBalancerIP, loadBalancerSourceRanges, externalIPs) and controllers which allocate external load-balancers should use the same IP family. Endpoints for this Service will be of this family. This field is immutable after creation. Assigning a ServiceIPFamily not available in the cluster (e.g. IPv6 in IPv4 only cluster) is an error condition and will fail during clusterIP assignment.\",\"type\":\"string\"},\"loadBalancerIP\":{\"description\":\"Only applies to Service Type: LoadBalancer LoadBalancer will get created with the IP specified in this field. This feature depends on whether the underlying cloud-provider supports specifying the loadBalancerIP when a load balancer is created. This field will be ignored if the cloud-provider does not support the feature.\",\"type\":\"string\"},\"loadBalancerSourceRanges\":{\"description\":\"If specified and supported by the platform, this will restrict traffic through the cloud-provider load-balancer will be restricted to the specified client IPs. This field will be ignored if the cloud-provider does not support the feature.\\\" More info: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"ports\":{\"description\":\"The list of ports that are exposed by this service. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies\",\"items\":{\"description\":\"ServicePort contains information on service's port.\",\"properties\":{\"name\":{\"description\":\"The name of this port within the service. This must be a DNS_LABEL. All ports within a ServiceSpec must have unique names. When considering the endpoints for a Service, this must match the 'name' field in the EndpointPort. Optional if only one ServicePort is defined on this service.\",\"type\":\"string\"},\"nodePort\":{\"description\":\"The port on each node on which this service is exposed when type=NodePort or LoadBalancer. Usually assigned by the system. If specified, it will be allocated to the service if unused or else creation of the service will fail. Default is to auto-allocate a port if the ServiceType of this Service requires one. More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport\",\"format\":\"int32\",\"type\":\"integer\"},\"port\":{\"description\":\"The port that will be exposed by this service.\",\"format\":\"int32\",\"type\":\"integer\"},\"protocol\":{\"description\":\"The IP protocol for this port. Supports \\\"TCP\\\", \\\"UDP\\\", and \\\"SCTP\\\". Default is TCP.\",\"type\":\"string\"},\"targetPort\":{\"anyOf\":[{\"type\":\"integer\"},{\"type\":\"string\"}],\"description\":\"Number or name of the port to access on the pods targeted by the service. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. If this is a string, it will be looked up as a named port in the target Pod's container ports. If this is not specified, the value of the 'port' field is used (an identity map). This field is ignored for services with clusterIP=None, and should be omitted or set equal to the 'port' field. More info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service\"}},\"required\":[\"port\"],\"type\":\"object\"},\"type\":\"array\"},\"publishNotReadyAddresses\":{\"description\":\"publishNotReadyAddresses, when set to true, indicates that DNS implementations must publish the notReadyAddresses of subsets for the Endpoints associated with the Service. The default value is false. The primary use case for setting this field is to use a StatefulSet's Headless Service to propagate SRV records for its Pods without respect to their readiness for purpose of peer discovery.\",\"type\":\"boolean\"},\"selector\":{\"additionalProperties\":{\"type\":\"string\"},\"description\":\"Route service traffic to pods with label keys and values matching this selector. If empty or not present, the service is assumed to have an external process managing its endpoints, which Kubernetes will not modify. Only applies to types ClusterIP, NodePort, and LoadBalancer. Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/\",\"type\":\"object\"},\"sessionAffinity\":{\"description\":\"Supports \\\"ClientIP\\\" and \\\"None\\\". Used to maintain session affinity. Enable client IP based session affinity. Must be ClientIP or None. Defaults to None. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies\",\"type\":\"string\"},\"sessionAffinityConfig\":{\"description\":\"sessionAffinityConfig contains the configurations of session affinity.\",\"properties\":{\"clientIP\":{\"description\":\"clientIP contains the configurations of Client IP based session affinity.\",\"properties\":{\"timeoutSeconds\":{\"description\":\"timeoutSeconds specifies the seconds of ClientIP type session sticky time. The value must be \\u003e0 \\u0026\\u0026 \\u003c=86400(for 1 day) if ServiceAffinity == \\\"ClientIP\\\". Default value is 10800(for 3 hours).\",\"format\":\"int32\",\"type\":\"integer\"}},\"type\":\"object\"}},\"type\":\"object\"},\"topologyKeys\":{\"description\":\"topologyKeys is a preference-order list of topology keys which implementations of services should use to preferentially sort endpoints when accessing this Service, it can not be used at the same time as externalTrafficPolicy=Local. Topology keys must be valid label keys and at most 16 keys may be specified. Endpoints are chosen based on the first topology key with available backends. If this field is specified and all entries have no backends that match the topology of the client, the service has no backends for that client and connections should fail. The special value \\\"*\\\" may be used to mean \\\"any topology\\\". This catch-all value, if used, only makes sense as the last value in the list. If this is not specified or empty, no topology constraints will be applied.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"type\":{\"description\":\"type determines how the Service is exposed. Defaults to ClusterIP. Valid options are ExternalName, ClusterIP, NodePort, and LoadBalancer. \\\"ExternalName\\\" maps to the specified externalName. \\\"ClusterIP\\\" allocates a cluster-internal IP address for load-balancing to endpoints. Endpoints are determined by the selector or if that is not specified, by manual construction of an Endpoints object. If clusterIP is \\\"None\\\", no virtual IP is allocated and the endpoints are published as a set of endpoints rather than a stable IP. \\\"NodePort\\\" builds on ClusterIP and allocates a port on every node which routes to the clusterIP. \\\"LoadBalancer\\\" builds on NodePort and creates an external load-balancer (if supported in the current cloud) which routes to the clusterIP. More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types\",\"type\":\"string\"}},\"type\":\"object\"}},\"type\":\"object\"}},\"type\":\"object\"},\"updateStrategy\":{\"description\":\"UpdateStrategy specifies how updates to the cluster should be performed.\",\"properties\":{\"changeBudget\":{\"description\":\"ChangeBudget defines the constraints to consider when applying changes to the Elasticsearch cluster.\",\"properties\":{\"maxSurge\":{\"description\":\"MaxSurge is the maximum number of new pods that can be created exceeding the original number of pods defined in the specification. MaxSurge is only taken into consideration when scaling up. Setting a negative value will disable the restriction. Defaults to unbounded if not specified.\",\"format\":\"int32\",\"type\":\"integer\"},\"maxUnavailable\":{\"description\":\"MaxUnavailable is the maximum number of pods that can be unavailable (not ready) during the update due to circumstances under the control of the operator. Setting a negative value will disable this restriction. Defaults to 1 if not specified.\",\"format\":\"int32\",\"type\":\"integer\"}},\"type\":\"object\"}},\"type\":\"object\"},\"version\":{\"description\":\"Version of Elasticsearch.\",\"type\":\"string\"}},\"required\":[\"nodeSets\",\"version\"],\"type\":\"object\"},\"status\":{\"description\":\"ElasticsearchStatus defines the observed state of Elasticsearch\",\"properties\":{\"availableNodes\":{\"format\":\"int32\",\"type\":\"integer\"},\"health\":{\"description\":\"ElasticsearchHealth is the health of the cluster as returned by the health API.\",\"type\":\"string\"},\"phase\":{\"description\":\"ElasticsearchOrchestrationPhase is the phase Elasticsearch is in from the controller point of view.\",\"type\":\"string\"}},\"type\":\"object\"}}}},\"version\":\"v1\",\"versions\":[{\"name\":\"v1\",\"served\":true,\"storage\":true},{\"name\":\"v1beta1\",\"served\":true,\"storage\":false},{\"name\":\"v1alpha1\",\"served\":false,\"storage\":false}]},\"status\":{\"acceptedNames\":{\"kind\":\"\",\"plural\":\"\"},\"conditions\":[],\"storedVersions\":[]}}\n" }, "creationTimestamp": "2020-04-28T23:31:51Z", "generation": 1, "labels": { "velero.io/backup-name": "es", "velero.io/restore-name": "es-crds" }, "name": "elasticsearches.elasticsearch.k8s.elastic.co", "resourceVersion": "1703536", "selfLink": "/apis/apiextensions.k8s.io/v1/customresourcedefinitions/elasticsearches.elasticsearch.k8s.elastic.co", "uid": "e8596856-29ae-47e4-8b14-5f7f027adf4a" }, "spec": { "conversion": { "strategy": "None" }, "group": "elasticsearch.k8s.elastic.co", "names": { "categories": [ "elastic" ], "kind": "Elasticsearch", "listKind": "ElasticsearchList", "plural": "elasticsearches", "shortNames": [ "es" ], "singular": "elasticsearch" }, "preserveUnknownFields": true, "scope": "Namespaced", "versions": [ { "additionalPrinterColumns": [ { "jsonPath": ".status.health", "name": "health", "type": "string" }, { "description": "Available nodes", "jsonPath": ".status.availableNodes", "name": "nodes", "type": "integer" }, { "description": "Elasticsearch version", "jsonPath": ".spec.version", "name": "version", "type": "string" }, { "jsonPath": ".status.phase", "name": "phase", "type": "string" }, { "jsonPath": ".metadata.creationTimestamp", "name": "age", "type": "date" } ], "name": "v1", "schema": { "openAPIV3Schema": { "description": "Elasticsearch represents an Elasticsearch resource in a Kubernetes cluster.", "properties": { "apiVersion": { "description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", "type": "string" }, "kind": { "description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", "type": "string" }, "metadata": { "type": "object" }, "spec": { "description": "ElasticsearchSpec holds the specification of an Elasticsearch cluster.", "properties": { "auth": { "description": "Auth contains user authentication and authorization security settings for Elasticsearch.", "properties": { "fileRealm": { "description": "FileRealm to propagate to the Elasticsearch cluster.", "items": { "description": "FileRealmSource references users to create in the Elasticsearch cluster.", "properties": { "secretName": { "description": "SecretName is the name of the secret.", "type": "string" } }, "type": "object" }, "type": "array" }, "roles": { "description": "Roles to propagate to the Elasticsearch cluster.", "items": { "description": "RoleSource references roles to create in the Elasticsearch cluster.", "properties": { "secretName": { "description": "SecretName is the name of the secret.", "type": "string" } }, "type": "object" }, "type": "array" } }, "type": "object" }, "http": { "description": "HTTP holds HTTP layer settings for Elasticsearch.", "properties": { "service": { "description": "Service defines the template for the associated Kubernetes Service object.", "properties": { "metadata": { "description": "ObjectMeta is the metadata of the service. The name and namespace provided here are managed by ECK and will be ignored.", "type": "object" }, "spec": { "description": "Spec is the specification of the service.", "properties": { "clusterIP": { "description": "clusterIP is the IP address of the service and is usually assigned randomly by the master. If an address is specified manually and is not in use by others, it will be allocated to the service; otherwise, creation of the service will fail. This field can not be changed through updates. Valid values are \"None\", empty string (\"\"), or a valid IP address. \"None\" can be specified for headless services when proxying is not required. Only applies to types ClusterIP, NodePort, and LoadBalancer. Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", "type": "string" }, "externalIPs": { "description": "externalIPs is a list of IP addresses for which nodes in the cluster will also accept traffic for this service. These IPs are not managed by Kubernetes. The user is responsible for ensuring that traffic arrives at a node with this IP. A common example is external load-balancers that are not part of the Kubernetes system.", "items": { "type": "string" }, "type": "array" }, "externalName": { "description": "externalName is the external reference that kubedns or equivalent will return as a CNAME record for this service. No proxying will be involved. Must be a valid RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) and requires Type to be ExternalName.", "type": "string" }, "externalTrafficPolicy": { "description": "externalTrafficPolicy denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints. \"Local\" preserves the client source IP and avoids a second hop for LoadBalancer and Nodeport type services, but risks potentially imbalanced traffic spreading. \"Cluster\" obscures the client source IP and may cause a second hop to another node, but should have good overall load-spreading.", "type": "string" }, "healthCheckNodePort": { "description": "healthCheckNodePort specifies the healthcheck nodePort for the service. If not specified, HealthCheckNodePort is created by the service api backend with the allocated nodePort. Will use user-specified nodePort value if specified by the client. Only effects when Type is set to LoadBalancer and ExternalTrafficPolicy is set to Local.", "format": "int32", "type": "integer" }, "ipFamily": { "description": "ipFamily specifies whether this Service has a preference for a particular IP family (e.g. IPv4 vs. IPv6). If a specific IP family is requested, the clusterIP field will be allocated from that family, if it is available in the cluster. If no IP family is requested, the cluster's primary IP family will be used. Other IP fields (loadBalancerIP, loadBalancerSourceRanges, externalIPs) and controllers which allocate external load-balancers should use the same IP family. Endpoints for this Service will be of this family. This field is immutable after creation. Assigning a ServiceIPFamily not available in the cluster (e.g. IPv6 in IPv4 only cluster) is an error condition and will fail during clusterIP assignment.", "type": "string" }, "loadBalancerIP": { "description": "Only applies to Service Type: LoadBalancer LoadBalancer will get created with the IP specified in this field. This feature depends on whether the underlying cloud-provider supports specifying the loadBalancerIP when a load balancer is created. This field will be ignored if the cloud-provider does not support the feature.", "type": "string" }, "loadBalancerSourceRanges": { "description": "If specified and supported by the platform, this will restrict traffic through the cloud-provider load-balancer will be restricted to the specified client IPs. This field will be ignored if the cloud-provider does not support the feature.\" More info: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/", "items": { "type": "string" }, "type": "array" }, "ports": { "description": "The list of ports that are exposed by this service. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", "items": { "description": "ServicePort contains information on service's port.", "properties": { "name": { "description": "The name of this port within the service. This must be a DNS_LABEL. All ports within a ServiceSpec must have unique names. When considering the endpoints for a Service, this must match the 'name' field in the EndpointPort. Optional if only one ServicePort is defined on this service.", "type": "string" }, "nodePort": { "description": "The port on each node on which this service is exposed when type=NodePort or LoadBalancer. Usually assigned by the system. If specified, it will be allocated to the service if unused or else creation of the service will fail. Default is to auto-allocate a port if the ServiceType of this Service requires one. More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport", "format": "int32", "type": "integer" }, "port": { "description": "The port that will be exposed by this service.", "format": "int32", "type": "integer" }, "protocol": { "description": "The IP protocol for this port. Supports \"TCP\", \"UDP\", and \"SCTP\". Default is TCP.", "type": "string" }, "targetPort": { "anyOf": [ { "type": "integer" }, { "type": "string" } ], "description": "Number or name of the port to access on the pods targeted by the service. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. If this is a string, it will be looked up as a named port in the target Pod's container ports. If this is not specified, the value of the 'port' field is used (an identity map). This field is ignored for services with clusterIP=None, and should be omitted or set equal to the 'port' field. More info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service" } }, "required": [ "port" ], "type": "object" }, "type": "array" }, "publishNotReadyAddresses": { "description": "publishNotReadyAddresses, when set to true, indicates that DNS implementations must publish the notReadyAddresses of subsets for the Endpoints associated with the Service. The default value is false. The primary use case for setting this field is to use a StatefulSet's Headless Service to propagate SRV records for its Pods without respect to their readiness for purpose of peer discovery.", "type": "boolean" }, "selector": { "additionalProperties": { "type": "string" }, "description": "Route service traffic to pods with label keys and values matching this selector. If empty or not present, the service is assumed to have an external process managing its endpoints, which Kubernetes will not modify. Only applies to types ClusterIP, NodePort, and LoadBalancer. Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/", "type": "object" }, "sessionAffinity": { "description": "Supports \"ClientIP\" and \"None\". Used to maintain session affinity. Enable client IP based session affinity. Must be ClientIP or None. Defaults to None. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", "type": "string" }, "sessionAffinityConfig": { "description": "sessionAffinityConfig contains the configurations of session affinity.", "properties": { "clientIP": { "description": "clientIP contains the configurations of Client IP based session affinity.", "properties": { "timeoutSeconds": { "description": "timeoutSeconds specifies the seconds of ClientIP type session sticky time. The value must be \u003e0 \u0026\u0026 \u003c=86400(for 1 day) if ServiceAffinity == \"ClientIP\". Default value is 10800(for 3 hours).", "format": "int32", "type": "integer" } }, "type": "object" } }, "type": "object" }, "topologyKeys": { "description": "topologyKeys is a preference-order list of topology keys which implementations of services should use to preferentially sort endpoints when accessing this Service, it can not be used at the same time as externalTrafficPolicy=Local. Topology keys must be valid label keys and at most 16 keys may be specified. Endpoints are chosen based on the first topology key with available backends. If this field is specified and all entries have no backends that match the topology of the client, the service has no backends for that client and connections should fail. The special value \"*\" may be used to mean \"any topology\". This catch-all value, if used, only makes sense as the last value in the list. If this is not specified or empty, no topology constraints will be applied.", "items": { "type": "string" }, "type": "array" }, "type": { "description": "type determines how the Service is exposed. Defaults to ClusterIP. Valid options are ExternalName, ClusterIP, NodePort, and LoadBalancer. \"ExternalName\" maps to the specified externalName. \"ClusterIP\" allocates a cluster-internal IP address for load-balancing to endpoints. Endpoints are determined by the selector or if that is not specified, by manual construction of an Endpoints object. If clusterIP is \"None\", no virtual IP is allocated and the endpoints are published as a set of endpoints rather than a stable IP. \"NodePort\" builds on ClusterIP and allocates a port on every node which routes to the clusterIP. \"LoadBalancer\" builds on NodePort and creates an external load-balancer (if supported in the current cloud) which routes to the clusterIP. More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types", "type": "string" } }, "type": "object" } }, "type": "object" }, "tls": { "description": "TLS defines options for configuring TLS for HTTP.", "properties": { "certificate": { "description": "Certificate is a reference to a Kubernetes secret that contains the certificate and private key for enabling TLS. The referenced secret should contain the following: \n - `ca.crt`: The certificate authority (optional). - `tls.crt`: The certificate (or a chain). - `tls.key`: The private key to the first certificate in the certificate chain.", "properties": { "secretName": { "description": "SecretName is the name of the secret.", "type": "string" } }, "type": "object" }, "selfSignedCertificate": { "description": "SelfSignedCertificate allows configuring the self-signed certificate generated by the operator.", "properties": { "disabled": { "description": "Disabled indicates that the provisioning of the self-signed certificate should be disabled.", "type": "boolean" }, "subjectAltNames": { "description": "SubjectAlternativeNames is a list of SANs to include in the generated HTTP TLS certificate.", "items": { "description": "SubjectAlternativeName represents a SAN entry in a x509 certificate.", "properties": { "dns": { "description": "DNS is the DNS name of the subject.", "type": "string" }, "ip": { "description": "IP is the IP address of the subject.", "type": "string" } }, "type": "object" }, "type": "array" } }, "type": "object" } }, "type": "object" } }, "type": "object" }, "image": { "description": "Image is the Elasticsearch Docker image to deploy.", "type": "string" }, "nodeSets": { "description": "NodeSets allow specifying groups of Elasticsearch nodes sharing the same configuration and Pod templates. See: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-orchestration.html", "items": { "description": "NodeSet is the specification for a group of Elasticsearch nodes sharing the same configuration and a Pod template.", "properties": { "config": { "description": "Config holds the Elasticsearch configuration.", "type": "object" }, "count": { "description": "Count of Elasticsearch nodes to deploy.", "format": "int32", "minimum": 1, "type": "integer" }, "name": { "description": "Name of this set of nodes. Becomes a part of the Elasticsearch node.name setting.", "maxLength": 23, "pattern": "[a-zA-Z0-9-]+", "type": "string" }, "podTemplate": { "description": "PodTemplate provides customisation options (labels, annotations, affinity rules, resource requests, and so on) for the Pods belonging to this NodeSet.", "type": "object" }, "volumeClaimTemplates": { "description": "VolumeClaimTemplates is a list of persistent volume claims to be used by each Pod in this NodeSet. Every claim in this list must have a matching volumeMount in one of the containers defined in the PodTemplate. Items defined here take precedence over any default claims added by the operator with the same name. See: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-volume-claim-templates.html", "items": { "description": "PersistentVolumeClaim is a user's request for and claim to a persistent volume", "properties": { "apiVersion": { "description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", "type": "string" }, "kind": { "description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", "type": "string" }, "metadata": { "description": "Standard object's metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata", "type": "object" }, "spec": { "description": "Spec defines the desired characteristics of a volume requested by a pod author. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims", "properties": { "accessModes": { "description": "AccessModes contains the desired access modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1", "items": { "type": "string" }, "type": "array" }, "dataSource": { "description": "This field requires the VolumeSnapshotDataSource alpha feature gate to be enabled and currently VolumeSnapshot is the only supported data source. If the provisioner can support VolumeSnapshot data source, it will create a new volume and data will be restored to the volume at the same time. If the provisioner does not support VolumeSnapshot data source, volume will not be created and the failure will be reported as an event. In the future, we plan to support more data source types and the behavior of the provisioner may change.", "properties": { "apiGroup": { "description": "APIGroup is the group for the resource being referenced. If APIGroup is not specified, the specified Kind must be in the core API group. For any other third-party types, APIGroup is required.", "type": "string" }, "kind": { "description": "Kind is the type of resource being referenced", "type": "string" }, "name": { "description": "Name is the name of resource being referenced", "type": "string" } }, "required": [ "kind", "name" ], "type": "object" }, "resources": { "description": "Resources represents the minimum resources the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources", "properties": { "limits": { "additionalProperties": { "anyOf": [ { "type": "integer" }, { "type": "string" } ], "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$" }, "description": "Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", "type": "object" }, "requests": { "additionalProperties": { "anyOf": [ { "type": "integer" }, { "type": "string" } ], "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$" }, "description": "Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", "type": "object" } }, "type": "object" }, "selector": { "description": "A label query over volumes to consider for binding.", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "items": { "type": "string" }, "type": "array" } }, "required": [ "key", "operator" ], "type": "object" }, "type": "array" }, "matchLabels": { "additionalProperties": { "type": "string" }, "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object" } }, "type": "object" }, "storageClassName": { "description": "Name of the StorageClass required by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1", "type": "string" }, "volumeMode": { "description": "volumeMode defines what type of volume is required by the claim. Value of Filesystem is implied when not included in claim spec. This is a beta feature.", "type": "string" }, "volumeName": { "description": "VolumeName is the binding reference to the PersistentVolume backing this claim.", "type": "string" } }, "type": "object" }, "status": { "description": "Status represents the current information/status of a persistent volume claim. Read-only. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims", "properties": { "accessModes": { "description": "AccessModes contains the actual access modes the volume backing the PVC has. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1", "items": { "type": "string" }, "type": "array" }, "capacity": { "additionalProperties": { "anyOf": [ { "type": "integer" }, { "type": "string" } ], "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$" }, "description": "Represents the actual resources of the underlying volume.", "type": "object" }, "conditions": { "description": "Current Condition of persistent volume claim. If underlying persistent volume is being resized then the Condition will be set to 'ResizeStarted'.", "items": { "description": "PersistentVolumeClaimCondition contains details about state of pvc", "properties": { "lastProbeTime": { "description": "Last time we probed the condition.", "format": "date-time", "type": "string" }, "lastTransitionTime": { "description": "Last time the condition transitioned from one status to another.", "format": "date-time", "type": "string" }, "message": { "description": "Human-readable message indicating details about last transition.", "type": "string" }, "reason": { "description": "Unique, this should be a short, machine understandable string that gives the reason for condition's last transition. If it reports \"ResizeStarted\" that means the underlying persistent volume is being resized.", "type": "string" }, "status": { "type": "string" }, "type": { "description": "PersistentVolumeClaimConditionType is a valid value of PersistentVolumeClaimCondition.Type", "type": "string" } }, "required": [ "status", "type" ], "type": "object" }, "type": "array" }, "phase": { "description": "Phase represents the current phase of PersistentVolumeClaim.", "type": "string" } }, "type": "object" } }, "type": "object" }, "type": "array" } }, "required": [ "count", "name" ], "type": "object" }, "minItems": 1, "type": "array" }, "podDisruptionBudget": { "description": "PodDisruptionBudget provides access to the default pod disruption budget for the Elasticsearch cluster. The default budget selects all cluster pods and sets `maxUnavailable` to 1. To disable, set `PodDisruptionBudget` to the empty value (`{}` in YAML).", "properties": { "metadata": { "description": "ObjectMeta is the metadata of the PDB. The name and namespace provided here are managed by ECK and will be ignored.", "type": "object" }, "spec": { "description": "Spec is the specification of the PDB.", "properties": { "maxUnavailable": { "anyOf": [ { "type": "integer" }, { "type": "string" } ], "description": "An eviction is allowed if at most \"maxUnavailable\" pods selected by \"selector\" are unavailable after the eviction, i.e. even in absence of the evicted pod. For example, one can prevent all voluntary evictions by specifying 0. This is a mutually exclusive setting with \"minAvailable\"." }, "minAvailable": { "anyOf": [ { "type": "integer" }, { "type": "string" } ], "description": "An eviction is allowed if at least \"minAvailable\" pods selected by \"selector\" will still be available after the eviction, i.e. even in the absence of the evicted pod. So for example you can prevent all voluntary evictions by specifying \"100%\"." }, "selector": { "description": "Label query over pods whose evictions are managed by the disruption budget.", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "items": { "type": "string" }, "type": "array" } }, "required": [ "key", "operator" ], "type": "object" }, "type": "array" }, "matchLabels": { "additionalProperties": { "type": "string" }, "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object" } }, "type": "object" } }, "type": "object" } }, "type": "object" }, "remoteClusters": { "description": "RemoteClusters enables you to establish uni-directional connections to a remote Elasticsearch cluster.", "items": { "description": "RemoteCluster declares a remote Elasticsearch cluster connection.", "properties": { "elasticsearchRef": { "description": "ElasticsearchRef is a reference to an Elasticsearch cluster running within the same k8s cluster.", "properties": { "name": { "description": "Name of the Kubernetes object.", "type": "string" }, "namespace": { "description": "Namespace of the Kubernetes object. If empty, defaults to the current namespace.", "type": "string" } }, "required": [ "name" ], "type": "object" }, "name": { "description": "Name is the name of the remote cluster as it is set in the Elasticsearch settings. The name is expected to be unique for each remote clusters.", "minLength": 1, "type": "string" } }, "required": [ "name" ], "type": "object" }, "type": "array" }, "secureSettings": { "description": "SecureSettings is a list of references to Kubernetes secrets containing sensitive configuration options for Elasticsearch. See: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-es-secure-settings.html", "items": { "description": "SecretSource defines a data source based on a Kubernetes Secret.", "properties": { "entries": { "description": "Entries define how to project each key-value pair in the secret to filesystem paths. If not defined, all keys will be projected to similarly named paths in the filesystem. If defined, only the specified keys will be projected to the corresponding paths.", "items": { "description": "KeyToPath defines how to map a key in a Secret object to a filesystem path.", "properties": { "key": { "description": "Key is the key contained in the secret.", "type": "string" }, "path": { "description": "Path is the relative file path to map the key to. Path must not be an absolute file path and must not contain any \"..\" components.", "type": "string" } }, "required": [ "key" ], "type": "object" }, "type": "array" }, "secretName": { "description": "SecretName is the name of the secret.", "type": "string" } }, "required": [ "secretName" ], "type": "object" }, "type": "array" }, "serviceAccountName": { "description": "ServiceAccountName is used to check access from the current resource to a resource (eg. a remote Elasticsearch cluster) in a different namespace. Can only be used if ECK is enforcing RBAC on references.", "type": "string" }, "transport": { "description": "Transport holds transport layer settings for Elasticsearch.", "properties": { "service": { "description": "Service defines the template for the associated Kubernetes Service object.", "properties": { "metadata": { "description": "ObjectMeta is the metadata of the service. The name and namespace provided here are managed by ECK and will be ignored.", "type": "object" }, "spec": { "description": "Spec is the specification of the service.", "properties": { "clusterIP": { "description": "clusterIP is the IP address of the service and is usually assigned randomly by the master. If an address is specified manually and is not in use by others, it will be allocated to the service; otherwise, creation of the service will fail. This field can not be changed through updates. Valid values are \"None\", empty string (\"\"), or a valid IP address. \"None\" can be specified for headless services when proxying is not required. Only applies to types ClusterIP, NodePort, and LoadBalancer. Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", "type": "string" }, "externalIPs": { "description": "externalIPs is a list of IP addresses for which nodes in the cluster will also accept traffic for this service. These IPs are not managed by Kubernetes. The user is responsible for ensuring that traffic arrives at a node with this IP. A common example is external load-balancers that are not part of the Kubernetes system.", "items": { "type": "string" }, "type": "array" }, "externalName": { "description": "externalName is the external reference that kubedns or equivalent will return as a CNAME record for this service. No proxying will be involved. Must be a valid RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) and requires Type to be ExternalName.", "type": "string" }, "externalTrafficPolicy": { "description": "externalTrafficPolicy denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints. \"Local\" preserves the client source IP and avoids a second hop for LoadBalancer and Nodeport type services, but risks potentially imbalanced traffic spreading. \"Cluster\" obscures the client source IP and may cause a second hop to another node, but should have good overall load-spreading.", "type": "string" }, "healthCheckNodePort": { "description": "healthCheckNodePort specifies the healthcheck nodePort for the service. If not specified, HealthCheckNodePort is created by the service api backend with the allocated nodePort. Will use user-specified nodePort value if specified by the client. Only effects when Type is set to LoadBalancer and ExternalTrafficPolicy is set to Local.", "format": "int32", "type": "integer" }, "ipFamily": { "description": "ipFamily specifies whether this Service has a preference for a particular IP family (e.g. IPv4 vs. IPv6). If a specific IP family is requested, the clusterIP field will be allocated from that family, if it is available in the cluster. If no IP family is requested, the cluster's primary IP family will be used. Other IP fields (loadBalancerIP, loadBalancerSourceRanges, externalIPs) and controllers which allocate external load-balancers should use the same IP family. Endpoints for this Service will be of this family. This field is immutable after creation. Assigning a ServiceIPFamily not available in the cluster (e.g. IPv6 in IPv4 only cluster) is an error condition and will fail during clusterIP assignment.", "type": "string" }, "loadBalancerIP": { "description": "Only applies to Service Type: LoadBalancer LoadBalancer will get created with the IP specified in this field. This feature depends on whether the underlying cloud-provider supports specifying the loadBalancerIP when a load balancer is created. This field will be ignored if the cloud-provider does not support the feature.", "type": "string" }, "loadBalancerSourceRanges": { "description": "If specified and supported by the platform, this will restrict traffic through the cloud-provider load-balancer will be restricted to the specified client IPs. This field will be ignored if the cloud-provider does not support the feature.\" More info: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/", "items": { "type": "string" }, "type": "array" }, "ports": { "description": "The list of ports that are exposed by this service. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", "items": { "description": "ServicePort contains information on service's port.", "properties": { "name": { "description": "The name of this port within the service. This must be a DNS_LABEL. All ports within a ServiceSpec must have unique names. When considering the endpoints for a Service, this must match the 'name' field in the EndpointPort. Optional if only one ServicePort is defined on this service.", "type": "string" }, "nodePort": { "description": "The port on each node on which this service is exposed when type=NodePort or LoadBalancer. Usually assigned by the system. If specified, it will be allocated to the service if unused or else creation of the service will fail. Default is to auto-allocate a port if the ServiceType of this Service requires one. More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport", "format": "int32", "type": "integer" }, "port": { "description": "The port that will be exposed by this service.", "format": "int32", "type": "integer" }, "protocol": { "description": "The IP protocol for this port. Supports \"TCP\", \"UDP\", and \"SCTP\". Default is TCP.", "type": "string" }, "targetPort": { "anyOf": [ { "type": "integer" }, { "type": "string" } ], "description": "Number or name of the port to access on the pods targeted by the service. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. If this is a string, it will be looked up as a named port in the target Pod's container ports. If this is not specified, the value of the 'port' field is used (an identity map). This field is ignored for services with clusterIP=None, and should be omitted or set equal to the 'port' field. More info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service" } }, "required": [ "port" ], "type": "object" }, "type": "array" }, "publishNotReadyAddresses": { "description": "publishNotReadyAddresses, when set to true, indicates that DNS implementations must publish the notReadyAddresses of subsets for the Endpoints associated with the Service. The default value is false. The primary use case for setting this field is to use a StatefulSet's Headless Service to propagate SRV records for its Pods without respect to their readiness for purpose of peer discovery.", "type": "boolean" }, "selector": { "additionalProperties": { "type": "string" }, "description": "Route service traffic to pods with label keys and values matching this selector. If empty or not present, the service is assumed to have an external process managing its endpoints, which Kubernetes will not modify. Only applies to types ClusterIP, NodePort, and LoadBalancer. Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/", "type": "object" }, "sessionAffinity": { "description": "Supports \"ClientIP\" and \"None\". Used to maintain session affinity. Enable client IP based session affinity. Must be ClientIP or None. Defaults to None. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", "type": "string" }, "sessionAffinityConfig": { "description": "sessionAffinityConfig contains the configurations of session affinity.", "properties": { "clientIP": { "description": "clientIP contains the configurations of Client IP based session affinity.", "properties": { "timeoutSeconds": { "description": "timeoutSeconds specifies the seconds of ClientIP type session sticky time. The value must be \u003e0 \u0026\u0026 \u003c=86400(for 1 day) if ServiceAffinity == \"ClientIP\". Default value is 10800(for 3 hours).", "format": "int32", "type": "integer" } }, "type": "object" } }, "type": "object" }, "topologyKeys": { "description": "topologyKeys is a preference-order list of topology keys which implementations of services should use to preferentially sort endpoints when accessing this Service, it can not be used at the same time as externalTrafficPolicy=Local. Topology keys must be valid label keys and at most 16 keys may be specified. Endpoints are chosen based on the first topology key with available backends. If this field is specified and all entries have no backends that match the topology of the client, the service has no backends for that client and connections should fail. The special value \"*\" may be used to mean \"any topology\". This catch-all value, if used, only makes sense as the last value in the list. If this is not specified or empty, no topology constraints will be applied.", "items": { "type": "string" }, "type": "array" }, "type": { "description": "type determines how the Service is exposed. Defaults to ClusterIP. Valid options are ExternalName, ClusterIP, NodePort, and LoadBalancer. \"ExternalName\" maps to the specified externalName. \"ClusterIP\" allocates a cluster-internal IP address for load-balancing to endpoints. Endpoints are determined by the selector or if that is not specified, by manual construction of an Endpoints object. If clusterIP is \"None\", no virtual IP is allocated and the endpoints are published as a set of endpoints rather than a stable IP. \"NodePort\" builds on ClusterIP and allocates a port on every node which routes to the clusterIP. \"LoadBalancer\" builds on NodePort and creates an external load-balancer (if supported in the current cloud) which routes to the clusterIP. More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types", "type": "string" } }, "type": "object" } }, "type": "object" } }, "type": "object" }, "updateStrategy": { "description": "UpdateStrategy specifies how updates to the cluster should be performed.", "properties": { "changeBudget": { "description": "ChangeBudget defines the constraints to consider when applying changes to the Elasticsearch cluster.", "properties": { "maxSurge": { "description": "MaxSurge is the maximum number of new pods that can be created exceeding the original number of pods defined in the specification. MaxSurge is only taken into consideration when scaling up. Setting a negative value will disable the restriction. Defaults to unbounded if not specified.", "format": "int32", "type": "integer" }, "maxUnavailable": { "description": "MaxUnavailable is the maximum number of pods that can be unavailable (not ready) during the update due to circumstances under the control of the operator. Setting a negative value will disable this restriction. Defaults to 1 if not specified.", "format": "int32", "type": "integer" } }, "type": "object" } }, "type": "object" }, "version": { "description": "Version of Elasticsearch.", "type": "string" } }, "required": [ "nodeSets", "version" ], "type": "object" }, "status": { "description": "ElasticsearchStatus defines the observed state of Elasticsearch", "properties": { "availableNodes": { "format": "int32", "type": "integer" }, "health": { "description": "ElasticsearchHealth is the health of the cluster as returned by the health API.", "type": "string" }, "phase": { "description": "ElasticsearchOrchestrationPhase is the phase Elasticsearch is in from the controller point of view.", "type": "string" } }, "type": "object" } } } }, "served": true, "storage": true, "subresources": { "status": {} } }, { "additionalPrinterColumns": [ { "jsonPath": ".status.health", "name": "health", "type": "string" }, { "description": "Available nodes", "jsonPath": ".status.availableNodes", "name": "nodes", "type": "integer" }, { "description": "Elasticsearch version", "jsonPath": ".spec.version", "name": "version", "type": "string" }, { "jsonPath": ".status.phase", "name": "phase", "type": "string" }, { "jsonPath": ".metadata.creationTimestamp", "name": "age", "type": "date" } ], "name": "v1beta1", "schema": { "openAPIV3Schema": { "description": "Elasticsearch represents an Elasticsearch resource in a Kubernetes cluster.", "properties": { "apiVersion": { "description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", "type": "string" }, "kind": { "description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", "type": "string" }, "metadata": { "type": "object" }, "spec": { "description": "ElasticsearchSpec holds the specification of an Elasticsearch cluster.", "properties": { "auth": { "description": "Auth contains user authentication and authorization security settings for Elasticsearch.", "properties": { "fileRealm": { "description": "FileRealm to propagate to the Elasticsearch cluster.", "items": { "description": "FileRealmSource references users to create in the Elasticsearch cluster.", "properties": { "secretName": { "description": "SecretName is the name of the secret.", "type": "string" } }, "type": "object" }, "type": "array" }, "roles": { "description": "Roles to propagate to the Elasticsearch cluster.", "items": { "description": "RoleSource references roles to create in the Elasticsearch cluster.", "properties": { "secretName": { "description": "SecretName is the name of the secret.", "type": "string" } }, "type": "object" }, "type": "array" } }, "type": "object" }, "http": { "description": "HTTP holds HTTP layer settings for Elasticsearch.", "properties": { "service": { "description": "Service defines the template for the associated Kubernetes Service object.", "properties": { "metadata": { "description": "ObjectMeta is the metadata of the service. The name and namespace provided here are managed by ECK and will be ignored.", "type": "object" }, "spec": { "description": "Spec is the specification of the service.", "properties": { "clusterIP": { "description": "clusterIP is the IP address of the service and is usually assigned randomly by the master. If an address is specified manually and is not in use by others, it will be allocated to the service; otherwise, creation of the service will fail. This field can not be changed through updates. Valid values are \"None\", empty string (\"\"), or a valid IP address. \"None\" can be specified for headless services when proxying is not required. Only applies to types ClusterIP, NodePort, and LoadBalancer. Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", "type": "string" }, "externalIPs": { "description": "externalIPs is a list of IP addresses for which nodes in the cluster will also accept traffic for this service. These IPs are not managed by Kubernetes. The user is responsible for ensuring that traffic arrives at a node with this IP. A common example is external load-balancers that are not part of the Kubernetes system.", "items": { "type": "string" }, "type": "array" }, "externalName": { "description": "externalName is the external reference that kubedns or equivalent will return as a CNAME record for this service. No proxying will be involved. Must be a valid RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) and requires Type to be ExternalName.", "type": "string" }, "externalTrafficPolicy": { "description": "externalTrafficPolicy denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints. \"Local\" preserves the client source IP and avoids a second hop for LoadBalancer and Nodeport type services, but risks potentially imbalanced traffic spreading. \"Cluster\" obscures the client source IP and may cause a second hop to another node, but should have good overall load-spreading.", "type": "string" }, "healthCheckNodePort": { "description": "healthCheckNodePort specifies the healthcheck nodePort for the service. If not specified, HealthCheckNodePort is created by the service api backend with the allocated nodePort. Will use user-specified nodePort value if specified by the client. Only effects when Type is set to LoadBalancer and ExternalTrafficPolicy is set to Local.", "format": "int32", "type": "integer" }, "ipFamily": { "description": "ipFamily specifies whether this Service has a preference for a particular IP family (e.g. IPv4 vs. IPv6). If a specific IP family is requested, the clusterIP field will be allocated from that family, if it is available in the cluster. If no IP family is requested, the cluster's primary IP family will be used. Other IP fields (loadBalancerIP, loadBalancerSourceRanges, externalIPs) and controllers which allocate external load-balancers should use the same IP family. Endpoints for this Service will be of this family. This field is immutable after creation. Assigning a ServiceIPFamily not available in the cluster (e.g. IPv6 in IPv4 only cluster) is an error condition and will fail during clusterIP assignment.", "type": "string" }, "loadBalancerIP": { "description": "Only applies to Service Type: LoadBalancer LoadBalancer will get created with the IP specified in this field. This feature depends on whether the underlying cloud-provider supports specifying the loadBalancerIP when a load balancer is created. This field will be ignored if the cloud-provider does not support the feature.", "type": "string" }, "loadBalancerSourceRanges": { "description": "If specified and supported by the platform, this will restrict traffic through the cloud-provider load-balancer will be restricted to the specified client IPs. This field will be ignored if the cloud-provider does not support the feature.\" More info: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/", "items": { "type": "string" }, "type": "array" }, "ports": { "description": "The list of ports that are exposed by this service. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", "items": { "description": "ServicePort contains information on service's port.", "properties": { "name": { "description": "The name of this port within the service. This must be a DNS_LABEL. All ports within a ServiceSpec must have unique names. When considering the endpoints for a Service, this must match the 'name' field in the EndpointPort. Optional if only one ServicePort is defined on this service.", "type": "string" }, "nodePort": { "description": "The port on each node on which this service is exposed when type=NodePort or LoadBalancer. Usually assigned by the system. If specified, it will be allocated to the service if unused or else creation of the service will fail. Default is to auto-allocate a port if the ServiceType of this Service requires one. More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport", "format": "int32", "type": "integer" }, "port": { "description": "The port that will be exposed by this service.", "format": "int32", "type": "integer" }, "protocol": { "description": "The IP protocol for this port. Supports \"TCP\", \"UDP\", and \"SCTP\". Default is TCP.", "type": "string" }, "targetPort": { "anyOf": [ { "type": "integer" }, { "type": "string" } ], "description": "Number or name of the port to access on the pods targeted by the service. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. If this is a string, it will be looked up as a named port in the target Pod's container ports. If this is not specified, the value of the 'port' field is used (an identity map). This field is ignored for services with clusterIP=None, and should be omitted or set equal to the 'port' field. More info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service" } }, "required": [ "port" ], "type": "object" }, "type": "array" }, "publishNotReadyAddresses": { "description": "publishNotReadyAddresses, when set to true, indicates that DNS implementations must publish the notReadyAddresses of subsets for the Endpoints associated with the Service. The default value is false. The primary use case for setting this field is to use a StatefulSet's Headless Service to propagate SRV records for its Pods without respect to their readiness for purpose of peer discovery.", "type": "boolean" }, "selector": { "additionalProperties": { "type": "string" }, "description": "Route service traffic to pods with label keys and values matching this selector. If empty or not present, the service is assumed to have an external process managing its endpoints, which Kubernetes will not modify. Only applies to types ClusterIP, NodePort, and LoadBalancer. Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/", "type": "object" }, "sessionAffinity": { "description": "Supports \"ClientIP\" and \"None\". Used to maintain session affinity. Enable client IP based session affinity. Must be ClientIP or None. Defaults to None. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", "type": "string" }, "sessionAffinityConfig": { "description": "sessionAffinityConfig contains the configurations of session affinity.", "properties": { "clientIP": { "description": "clientIP contains the configurations of Client IP based session affinity.", "properties": { "timeoutSeconds": { "description": "timeoutSeconds specifies the seconds of ClientIP type session sticky time. The value must be \u003e0 \u0026\u0026 \u003c=86400(for 1 day) if ServiceAffinity == \"ClientIP\". Default value is 10800(for 3 hours).", "format": "int32", "type": "integer" } }, "type": "object" } }, "type": "object" }, "topologyKeys": { "description": "topologyKeys is a preference-order list of topology keys which implementations of services should use to preferentially sort endpoints when accessing this Service, it can not be used at the same time as externalTrafficPolicy=Local. Topology keys must be valid label keys and at most 16 keys may be specified. Endpoints are chosen based on the first topology key with available backends. If this field is specified and all entries have no backends that match the topology of the client, the service has no backends for that client and connections should fail. The special value \"*\" may be used to mean \"any topology\". This catch-all value, if used, only makes sense as the last value in the list. If this is not specified or empty, no topology constraints will be applied.", "items": { "type": "string" }, "type": "array" }, "type": { "description": "type determines how the Service is exposed. Defaults to ClusterIP. Valid options are ExternalName, ClusterIP, NodePort, and LoadBalancer. \"ExternalName\" maps to the specified externalName. \"ClusterIP\" allocates a cluster-internal IP address for load-balancing to endpoints. Endpoints are determined by the selector or if that is not specified, by manual construction of an Endpoints object. If clusterIP is \"None\", no virtual IP is allocated and the endpoints are published as a set of endpoints rather than a stable IP. \"NodePort\" builds on ClusterIP and allocates a port on every node which routes to the clusterIP. \"LoadBalancer\" builds on NodePort and creates an external load-balancer (if supported in the current cloud) which routes to the clusterIP. More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types", "type": "string" } }, "type": "object" } }, "type": "object" }, "tls": { "description": "TLS defines options for configuring TLS for HTTP.", "properties": { "certificate": { "description": "Certificate is a reference to a Kubernetes secret that contains the certificate and private key for enabling TLS. The referenced secret should contain the following: \n - `ca.crt`: The certificate authority (optional). - `tls.crt`: The certificate (or a chain). - `tls.key`: The private key to the first certificate in the certificate chain.", "properties": { "secretName": { "description": "SecretName is the name of the secret.", "type": "string" } }, "type": "object" }, "selfSignedCertificate": { "description": "SelfSignedCertificate allows configuring the self-signed certificate generated by the operator.", "properties": { "disabled": { "description": "Disabled indicates that the provisioning of the self-signed certificate should be disabled.", "type": "boolean" }, "subjectAltNames": { "description": "SubjectAlternativeNames is a list of SANs to include in the generated HTTP TLS certificate.", "items": { "description": "SubjectAlternativeName represents a SAN entry in a x509 certificate.", "properties": { "dns": { "description": "DNS is the DNS name of the subject.", "type": "string" }, "ip": { "description": "IP is the IP address of the subject.", "type": "string" } }, "type": "object" }, "type": "array" } }, "type": "object" } }, "type": "object" } }, "type": "object" }, "image": { "description": "Image is the Elasticsearch Docker image to deploy.", "type": "string" }, "nodeSets": { "description": "NodeSets allow specifying groups of Elasticsearch nodes sharing the same configuration and Pod templates. See: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-orchestration.html", "items": { "description": "NodeSet is the specification for a group of Elasticsearch nodes sharing the same configuration and a Pod template.", "properties": { "config": { "description": "Config holds the Elasticsearch configuration.", "type": "object" }, "count": { "description": "Count of Elasticsearch nodes to deploy.", "format": "int32", "minimum": 1, "type": "integer" }, "name": { "description": "Name of this set of nodes. Becomes a part of the Elasticsearch node.name setting.", "maxLength": 23, "pattern": "[a-zA-Z0-9-]+", "type": "string" }, "podTemplate": { "description": "PodTemplate provides customisation options (labels, annotations, affinity rules, resource requests, and so on) for the Pods belonging to this NodeSet.", "type": "object" }, "volumeClaimTemplates": { "description": "VolumeClaimTemplates is a list of persistent volume claims to be used by each Pod in this NodeSet. Every claim in this list must have a matching volumeMount in one of the containers defined in the PodTemplate. Items defined here take precedence over any default claims added by the operator with the same name. See: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-volume-claim-templates.html", "items": { "description": "PersistentVolumeClaim is a user's request for and claim to a persistent volume", "properties": { "apiVersion": { "description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", "type": "string" }, "kind": { "description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", "type": "string" }, "metadata": { "description": "Standard object's metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata", "type": "object" }, "spec": { "description": "Spec defines the desired characteristics of a volume requested by a pod author. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims", "properties": { "accessModes": { "description": "AccessModes contains the desired access modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1", "items": { "type": "string" }, "type": "array" }, "dataSource": { "description": "This field requires the VolumeSnapshotDataSource alpha feature gate to be enabled and currently VolumeSnapshot is the only supported data source. If the provisioner can support VolumeSnapshot data source, it will create a new volume and data will be restored to the volume at the same time. If the provisioner does not support VolumeSnapshot data source, volume will not be created and the failure will be reported as an event. In the future, we plan to support more data source types and the behavior of the provisioner may change.", "properties": { "apiGroup": { "description": "APIGroup is the group for the resource being referenced. If APIGroup is not specified, the specified Kind must be in the core API group. For any other third-party types, APIGroup is required.", "type": "string" }, "kind": { "description": "Kind is the type of resource being referenced", "type": "string" }, "name": { "description": "Name is the name of resource being referenced", "type": "string" } }, "required": [ "kind", "name" ], "type": "object" }, "resources": { "description": "Resources represents the minimum resources the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources", "properties": { "limits": { "additionalProperties": { "anyOf": [ { "type": "integer" }, { "type": "string" } ], "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$" }, "description": "Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", "type": "object" }, "requests": { "additionalProperties": { "anyOf": [ { "type": "integer" }, { "type": "string" } ], "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$" }, "description": "Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", "type": "object" } }, "type": "object" }, "selector": { "description": "A label query over volumes to consider for binding.", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "items": { "type": "string" }, "type": "array" } }, "required": [ "key", "operator" ], "type": "object" }, "type": "array" }, "matchLabels": { "additionalProperties": { "type": "string" }, "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object" } }, "type": "object" }, "storageClassName": { "description": "Name of the StorageClass required by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1", "type": "string" }, "volumeMode": { "description": "volumeMode defines what type of volume is required by the claim. Value of Filesystem is implied when not included in claim spec. This is a beta feature.", "type": "string" }, "volumeName": { "description": "VolumeName is the binding reference to the PersistentVolume backing this claim.", "type": "string" } }, "type": "object" }, "status": { "description": "Status represents the current information/status of a persistent volume claim. Read-only. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims", "properties": { "accessModes": { "description": "AccessModes contains the actual access modes the volume backing the PVC has. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1", "items": { "type": "string" }, "type": "array" }, "capacity": { "additionalProperties": { "anyOf": [ { "type": "integer" }, { "type": "string" } ], "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$" }, "description": "Represents the actual resources of the underlying volume.", "type": "object" }, "conditions": { "description": "Current Condition of persistent volume claim. If underlying persistent volume is being resized then the Condition will be set to 'ResizeStarted'.", "items": { "description": "PersistentVolumeClaimCondition contains details about state of pvc", "properties": { "lastProbeTime": { "description": "Last time we probed the condition.", "format": "date-time", "type": "string" }, "lastTransitionTime": { "description": "Last time the condition transitioned from one status to another.", "format": "date-time", "type": "string" }, "message": { "description": "Human-readable message indicating details about last transition.", "type": "string" }, "reason": { "description": "Unique, this should be a short, machine understandable string that gives the reason for condition's last transition. If it reports \"ResizeStarted\" that means the underlying persistent volume is being resized.", "type": "string" }, "status": { "type": "string" }, "type": { "description": "PersistentVolumeClaimConditionType is a valid value of PersistentVolumeClaimCondition.Type", "type": "string" } }, "required": [ "status", "type" ], "type": "object" }, "type": "array" }, "phase": { "description": "Phase represents the current phase of PersistentVolumeClaim.", "type": "string" } }, "type": "object" } }, "type": "object" }, "type": "array" } }, "required": [ "count", "name" ], "type": "object" }, "minItems": 1, "type": "array" }, "podDisruptionBudget": { "description": "PodDisruptionBudget provides access to the default pod disruption budget for the Elasticsearch cluster. The default budget selects all cluster pods and sets `maxUnavailable` to 1. To disable, set `PodDisruptionBudget` to the empty value (`{}` in YAML).", "properties": { "metadata": { "description": "ObjectMeta is the metadata of the PDB. The name and namespace provided here are managed by ECK and will be ignored.", "type": "object" }, "spec": { "description": "Spec is the specification of the PDB.", "properties": { "maxUnavailable": { "anyOf": [ { "type": "integer" }, { "type": "string" } ], "description": "An eviction is allowed if at most \"maxUnavailable\" pods selected by \"selector\" are unavailable after the eviction, i.e. even in absence of the evicted pod. For example, one can prevent all voluntary evictions by specifying 0. This is a mutually exclusive setting with \"minAvailable\"." }, "minAvailable": { "anyOf": [ { "type": "integer" }, { "type": "string" } ], "description": "An eviction is allowed if at least \"minAvailable\" pods selected by \"selector\" will still be available after the eviction, i.e. even in the absence of the evicted pod. So for example you can prevent all voluntary evictions by specifying \"100%\"." }, "selector": { "description": "Label query over pods whose evictions are managed by the disruption budget.", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "items": { "type": "string" }, "type": "array" } }, "required": [ "key", "operator" ], "type": "object" }, "type": "array" }, "matchLabels": { "additionalProperties": { "type": "string" }, "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object" } }, "type": "object" } }, "type": "object" } }, "type": "object" }, "remoteClusters": { "description": "RemoteClusters enables you to establish uni-directional connections to a remote Elasticsearch cluster.", "items": { "description": "RemoteCluster declares a remote Elasticsearch cluster connection.", "properties": { "elasticsearchRef": { "description": "ElasticsearchRef is a reference to an Elasticsearch cluster running within the same k8s cluster.", "properties": { "name": { "description": "Name of the Kubernetes object.", "type": "string" }, "namespace": { "description": "Namespace of the Kubernetes object. If empty, defaults to the current namespace.", "type": "string" } }, "required": [ "name" ], "type": "object" }, "name": { "description": "Name is the name of the remote cluster as it is set in the Elasticsearch settings. The name is expected to be unique for each remote clusters.", "minLength": 1, "type": "string" } }, "required": [ "name" ], "type": "object" }, "type": "array" }, "secureSettings": { "description": "SecureSettings is a list of references to Kubernetes secrets containing sensitive configuration options for Elasticsearch. See: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-es-secure-settings.html", "items": { "description": "SecretSource defines a data source based on a Kubernetes Secret.", "properties": { "entries": { "description": "Entries define how to project each key-value pair in the secret to filesystem paths. If not defined, all keys will be projected to similarly named paths in the filesystem. If defined, only the specified keys will be projected to the corresponding paths.", "items": { "description": "KeyToPath defines how to map a key in a Secret object to a filesystem path.", "properties": { "key": { "description": "Key is the key contained in the secret.", "type": "string" }, "path": { "description": "Path is the relative file path to map the key to. Path must not be an absolute file path and must not contain any \"..\" components.", "type": "string" } }, "required": [ "key" ], "type": "object" }, "type": "array" }, "secretName": { "description": "SecretName is the name of the secret.", "type": "string" } }, "required": [ "secretName" ], "type": "object" }, "type": "array" }, "serviceAccountName": { "description": "ServiceAccountName is used to check access from the current resource to a resource (eg. a remote Elasticsearch cluster) in a different namespace. Can only be used if ECK is enforcing RBAC on references.", "type": "string" }, "transport": { "description": "Transport holds transport layer settings for Elasticsearch.", "properties": { "service": { "description": "Service defines the template for the associated Kubernetes Service object.", "properties": { "metadata": { "description": "ObjectMeta is the metadata of the service. The name and namespace provided here are managed by ECK and will be ignored.", "type": "object" }, "spec": { "description": "Spec is the specification of the service.", "properties": { "clusterIP": { "description": "clusterIP is the IP address of the service and is usually assigned randomly by the master. If an address is specified manually and is not in use by others, it will be allocated to the service; otherwise, creation of the service will fail. This field can not be changed through updates. Valid values are \"None\", empty string (\"\"), or a valid IP address. \"None\" can be specified for headless services when proxying is not required. Only applies to types ClusterIP, NodePort, and LoadBalancer. Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", "type": "string" }, "externalIPs": { "description": "externalIPs is a list of IP addresses for which nodes in the cluster will also accept traffic for this service. These IPs are not managed by Kubernetes. The user is responsible for ensuring that traffic arrives at a node with this IP. A common example is external load-balancers that are not part of the Kubernetes system.", "items": { "type": "string" }, "type": "array" }, "externalName": { "description": "externalName is the external reference that kubedns or equivalent will return as a CNAME record for this service. No proxying will be involved. Must be a valid RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) and requires Type to be ExternalName.", "type": "string" }, "externalTrafficPolicy": { "description": "externalTrafficPolicy denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints. \"Local\" preserves the client source IP and avoids a second hop for LoadBalancer and Nodeport type services, but risks potentially imbalanced traffic spreading. \"Cluster\" obscures the client source IP and may cause a second hop to another node, but should have good overall load-spreading.", "type": "string" }, "healthCheckNodePort": { "description": "healthCheckNodePort specifies the healthcheck nodePort for the service. If not specified, HealthCheckNodePort is created by the service api backend with the allocated nodePort. Will use user-specified nodePort value if specified by the client. Only effects when Type is set to LoadBalancer and ExternalTrafficPolicy is set to Local.", "format": "int32", "type": "integer" }, "ipFamily": { "description": "ipFamily specifies whether this Service has a preference for a particular IP family (e.g. IPv4 vs. IPv6). If a specific IP family is requested, the clusterIP field will be allocated from that family, if it is available in the cluster. If no IP family is requested, the cluster's primary IP family will be used. Other IP fields (loadBalancerIP, loadBalancerSourceRanges, externalIPs) and controllers which allocate external load-balancers should use the same IP family. Endpoints for this Service will be of this family. This field is immutable after creation. Assigning a ServiceIPFamily not available in the cluster (e.g. IPv6 in IPv4 only cluster) is an error condition and will fail during clusterIP assignment.", "type": "string" }, "loadBalancerIP": { "description": "Only applies to Service Type: LoadBalancer LoadBalancer will get created with the IP specified in this field. This feature depends on whether the underlying cloud-provider supports specifying the loadBalancerIP when a load balancer is created. This field will be ignored if the cloud-provider does not support the feature.", "type": "string" }, "loadBalancerSourceRanges": { "description": "If specified and supported by the platform, this will restrict traffic through the cloud-provider load-balancer will be restricted to the specified client IPs. This field will be ignored if the cloud-provider does not support the feature.\" More info: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/", "items": { "type": "string" }, "type": "array" }, "ports": { "description": "The list of ports that are exposed by this service. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", "items": { "description": "ServicePort contains information on service's port.", "properties": { "name": { "description": "The name of this port within the service. This must be a DNS_LABEL. All ports within a ServiceSpec must have unique names. When considering the endpoints for a Service, this must match the 'name' field in the EndpointPort. Optional if only one ServicePort is defined on this service.", "type": "string" }, "nodePort": { "description": "The port on each node on which this service is exposed when type=NodePort or LoadBalancer. Usually assigned by the system. If specified, it will be allocated to the service if unused or else creation of the service will fail. Default is to auto-allocate a port if the ServiceType of this Service requires one. More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport", "format": "int32", "type": "integer" }, "port": { "description": "The port that will be exposed by this service.", "format": "int32", "type": "integer" }, "protocol": { "description": "The IP protocol for this port. Supports \"TCP\", \"UDP\", and \"SCTP\". Default is TCP.", "type": "string" }, "targetPort": { "anyOf": [ { "type": "integer" }, { "type": "string" } ], "description": "Number or name of the port to access on the pods targeted by the service. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. If this is a string, it will be looked up as a named port in the target Pod's container ports. If this is not specified, the value of the 'port' field is used (an identity map). This field is ignored for services with clusterIP=None, and should be omitted or set equal to the 'port' field. More info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service" } }, "required": [ "port" ], "type": "object" }, "type": "array" }, "publishNotReadyAddresses": { "description": "publishNotReadyAddresses, when set to true, indicates that DNS implementations must publish the notReadyAddresses of subsets for the Endpoints associated with the Service. The default value is false. The primary use case for setting this field is to use a StatefulSet's Headless Service to propagate SRV records for its Pods without respect to their readiness for purpose of peer discovery.", "type": "boolean" }, "selector": { "additionalProperties": { "type": "string" }, "description": "Route service traffic to pods with label keys and values matching this selector. If empty or not present, the service is assumed to have an external process managing its endpoints, which Kubernetes will not modify. Only applies to types ClusterIP, NodePort, and LoadBalancer. Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/", "type": "object" }, "sessionAffinity": { "description": "Supports \"ClientIP\" and \"None\". Used to maintain session affinity. Enable client IP based session affinity. Must be ClientIP or None. Defaults to None. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", "type": "string" }, "sessionAffinityConfig": { "description": "sessionAffinityConfig contains the configurations of session affinity.", "properties": { "clientIP": { "description": "clientIP contains the configurations of Client IP based session affinity.", "properties": { "timeoutSeconds": { "description": "timeoutSeconds specifies the seconds of ClientIP type session sticky time. The value must be \u003e0 \u0026\u0026 \u003c=86400(for 1 day) if ServiceAffinity == \"ClientIP\". Default value is 10800(for 3 hours).", "format": "int32", "type": "integer" } }, "type": "object" } }, "type": "object" }, "topologyKeys": { "description": "topologyKeys is a preference-order list of topology keys which implementations of services should use to preferentially sort endpoints when accessing this Service, it can not be used at the same time as externalTrafficPolicy=Local. Topology keys must be valid label keys and at most 16 keys may be specified. Endpoints are chosen based on the first topology key with available backends. If this field is specified and all entries have no backends that match the topology of the client, the service has no backends for that client and connections should fail. The special value \"*\" may be used to mean \"any topology\". This catch-all value, if used, only makes sense as the last value in the list. If this is not specified or empty, no topology constraints will be applied.", "items": { "type": "string" }, "type": "array" }, "type": { "description": "type determines how the Service is exposed. Defaults to ClusterIP. Valid options are ExternalName, ClusterIP, NodePort, and LoadBalancer. \"ExternalName\" maps to the specified externalName. \"ClusterIP\" allocates a cluster-internal IP address for load-balancing to endpoints. Endpoints are determined by the selector or if that is not specified, by manual construction of an Endpoints object. If clusterIP is \"None\", no virtual IP is allocated and the endpoints are published as a set of endpoints rather than a stable IP. \"NodePort\" builds on ClusterIP and allocates a port on every node which routes to the clusterIP. \"LoadBalancer\" builds on NodePort and creates an external load-balancer (if supported in the current cloud) which routes to the clusterIP. More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types", "type": "string" } }, "type": "object" } }, "type": "object" } }, "type": "object" }, "updateStrategy": { "description": "UpdateStrategy specifies how updates to the cluster should be performed.", "properties": { "changeBudget": { "description": "ChangeBudget defines the constraints to consider when applying changes to the Elasticsearch cluster.", "properties": { "maxSurge": { "description": "MaxSurge is the maximum number of new pods that can be created exceeding the original number of pods defined in the specification. MaxSurge is only taken into consideration when scaling up. Setting a negative value will disable the restriction. Defaults to unbounded if not specified.", "format": "int32", "type": "integer" }, "maxUnavailable": { "description": "MaxUnavailable is the maximum number of pods that can be unavailable (not ready) during the update due to circumstances under the control of the operator. Setting a negative value will disable this restriction. Defaults to 1 if not specified.", "format": "int32", "type": "integer" } }, "type": "object" } }, "type": "object" }, "version": { "description": "Version of Elasticsearch.", "type": "string" } }, "required": [ "nodeSets", "version" ], "type": "object" }, "status": { "description": "ElasticsearchStatus defines the observed state of Elasticsearch", "properties": { "availableNodes": { "format": "int32", "type": "integer" }, "health": { "description": "ElasticsearchHealth is the health of the cluster as returned by the health API.", "type": "string" }, "phase": { "description": "ElasticsearchOrchestrationPhase is the phase Elasticsearch is in from the controller point of view.", "type": "string" } }, "type": "object" } } } }, "served": true, "storage": false, "subresources": { "status": {} } }, { "additionalPrinterColumns": [ { "jsonPath": ".status.health", "name": "health", "type": "string" }, { "description": "Available nodes", "jsonPath": ".status.availableNodes", "name": "nodes", "type": "integer" }, { "description": "Elasticsearch version", "jsonPath": ".spec.version", "name": "version", "type": "string" }, { "jsonPath": ".status.phase", "name": "phase", "type": "string" }, { "jsonPath": ".metadata.creationTimestamp", "name": "age", "type": "date" } ], "name": "v1alpha1", "schema": { "openAPIV3Schema": { "description": "Elasticsearch represents an Elasticsearch resource in a Kubernetes cluster.", "properties": { "apiVersion": { "description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", "type": "string" }, "kind": { "description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", "type": "string" }, "metadata": { "type": "object" }, "spec": { "description": "ElasticsearchSpec holds the specification of an Elasticsearch cluster.", "properties": { "auth": { "description": "Auth contains user authentication and authorization security settings for Elasticsearch.", "properties": { "fileRealm": { "description": "FileRealm to propagate to the Elasticsearch cluster.", "items": { "description": "FileRealmSource references users to create in the Elasticsearch cluster.", "properties": { "secretName": { "description": "SecretName is the name of the secret.", "type": "string" } }, "type": "object" }, "type": "array" }, "roles": { "description": "Roles to propagate to the Elasticsearch cluster.", "items": { "description": "RoleSource references roles to create in the Elasticsearch cluster.", "properties": { "secretName": { "description": "SecretName is the name of the secret.", "type": "string" } }, "type": "object" }, "type": "array" } }, "type": "object" }, "http": { "description": "HTTP holds HTTP layer settings for Elasticsearch.", "properties": { "service": { "description": "Service defines the template for the associated Kubernetes Service object.", "properties": { "metadata": { "description": "ObjectMeta is the metadata of the service. The name and namespace provided here are managed by ECK and will be ignored.", "type": "object" }, "spec": { "description": "Spec is the specification of the service.", "properties": { "clusterIP": { "description": "clusterIP is the IP address of the service and is usually assigned randomly by the master. If an address is specified manually and is not in use by others, it will be allocated to the service; otherwise, creation of the service will fail. This field can not be changed through updates. Valid values are \"None\", empty string (\"\"), or a valid IP address. \"None\" can be specified for headless services when proxying is not required. Only applies to types ClusterIP, NodePort, and LoadBalancer. Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", "type": "string" }, "externalIPs": { "description": "externalIPs is a list of IP addresses for which nodes in the cluster will also accept traffic for this service. These IPs are not managed by Kubernetes. The user is responsible for ensuring that traffic arrives at a node with this IP. A common example is external load-balancers that are not part of the Kubernetes system.", "items": { "type": "string" }, "type": "array" }, "externalName": { "description": "externalName is the external reference that kubedns or equivalent will return as a CNAME record for this service. No proxying will be involved. Must be a valid RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) and requires Type to be ExternalName.", "type": "string" }, "externalTrafficPolicy": { "description": "externalTrafficPolicy denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints. \"Local\" preserves the client source IP and avoids a second hop for LoadBalancer and Nodeport type services, but risks potentially imbalanced traffic spreading. \"Cluster\" obscures the client source IP and may cause a second hop to another node, but should have good overall load-spreading.", "type": "string" }, "healthCheckNodePort": { "description": "healthCheckNodePort specifies the healthcheck nodePort for the service. If not specified, HealthCheckNodePort is created by the service api backend with the allocated nodePort. Will use user-specified nodePort value if specified by the client. Only effects when Type is set to LoadBalancer and ExternalTrafficPolicy is set to Local.", "format": "int32", "type": "integer" }, "ipFamily": { "description": "ipFamily specifies whether this Service has a preference for a particular IP family (e.g. IPv4 vs. IPv6). If a specific IP family is requested, the clusterIP field will be allocated from that family, if it is available in the cluster. If no IP family is requested, the cluster's primary IP family will be used. Other IP fields (loadBalancerIP, loadBalancerSourceRanges, externalIPs) and controllers which allocate external load-balancers should use the same IP family. Endpoints for this Service will be of this family. This field is immutable after creation. Assigning a ServiceIPFamily not available in the cluster (e.g. IPv6 in IPv4 only cluster) is an error condition and will fail during clusterIP assignment.", "type": "string" }, "loadBalancerIP": { "description": "Only applies to Service Type: LoadBalancer LoadBalancer will get created with the IP specified in this field. This feature depends on whether the underlying cloud-provider supports specifying the loadBalancerIP when a load balancer is created. This field will be ignored if the cloud-provider does not support the feature.", "type": "string" }, "loadBalancerSourceRanges": { "description": "If specified and supported by the platform, this will restrict traffic through the cloud-provider load-balancer will be restricted to the specified client IPs. This field will be ignored if the cloud-provider does not support the feature.\" More info: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/", "items": { "type": "string" }, "type": "array" }, "ports": { "description": "The list of ports that are exposed by this service. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", "items": { "description": "ServicePort contains information on service's port.", "properties": { "name": { "description": "The name of this port within the service. This must be a DNS_LABEL. All ports within a ServiceSpec must have unique names. When considering the endpoints for a Service, this must match the 'name' field in the EndpointPort. Optional if only one ServicePort is defined on this service.", "type": "string" }, "nodePort": { "description": "The port on each node on which this service is exposed when type=NodePort or LoadBalancer. Usually assigned by the system. If specified, it will be allocated to the service if unused or else creation of the service will fail. Default is to auto-allocate a port if the ServiceType of this Service requires one. More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport", "format": "int32", "type": "integer" }, "port": { "description": "The port that will be exposed by this service.", "format": "int32", "type": "integer" }, "protocol": { "description": "The IP protocol for this port. Supports \"TCP\", \"UDP\", and \"SCTP\". Default is TCP.", "type": "string" }, "targetPort": { "anyOf": [ { "type": "integer" }, { "type": "string" } ], "description": "Number or name of the port to access on the pods targeted by the service. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. If this is a string, it will be looked up as a named port in the target Pod's container ports. If this is not specified, the value of the 'port' field is used (an identity map). This field is ignored for services with clusterIP=None, and should be omitted or set equal to the 'port' field. More info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service" } }, "required": [ "port" ], "type": "object" }, "type": "array" }, "publishNotReadyAddresses": { "description": "publishNotReadyAddresses, when set to true, indicates that DNS implementations must publish the notReadyAddresses of subsets for the Endpoints associated with the Service. The default value is false. The primary use case for setting this field is to use a StatefulSet's Headless Service to propagate SRV records for its Pods without respect to their readiness for purpose of peer discovery.", "type": "boolean" }, "selector": { "additionalProperties": { "type": "string" }, "description": "Route service traffic to pods with label keys and values matching this selector. If empty or not present, the service is assumed to have an external process managing its endpoints, which Kubernetes will not modify. Only applies to types ClusterIP, NodePort, and LoadBalancer. Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/", "type": "object" }, "sessionAffinity": { "description": "Supports \"ClientIP\" and \"None\". Used to maintain session affinity. Enable client IP based session affinity. Must be ClientIP or None. Defaults to None. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", "type": "string" }, "sessionAffinityConfig": { "description": "sessionAffinityConfig contains the configurations of session affinity.", "properties": { "clientIP": { "description": "clientIP contains the configurations of Client IP based session affinity.", "properties": { "timeoutSeconds": { "description": "timeoutSeconds specifies the seconds of ClientIP type session sticky time. The value must be \u003e0 \u0026\u0026 \u003c=86400(for 1 day) if ServiceAffinity == \"ClientIP\". Default value is 10800(for 3 hours).", "format": "int32", "type": "integer" } }, "type": "object" } }, "type": "object" }, "topologyKeys": { "description": "topologyKeys is a preference-order list of topology keys which implementations of services should use to preferentially sort endpoints when accessing this Service, it can not be used at the same time as externalTrafficPolicy=Local. Topology keys must be valid label keys and at most 16 keys may be specified. Endpoints are chosen based on the first topology key with available backends. If this field is specified and all entries have no backends that match the topology of the client, the service has no backends for that client and connections should fail. The special value \"*\" may be used to mean \"any topology\". This catch-all value, if used, only makes sense as the last value in the list. If this is not specified or empty, no topology constraints will be applied.", "items": { "type": "string" }, "type": "array" }, "type": { "description": "type determines how the Service is exposed. Defaults to ClusterIP. Valid options are ExternalName, ClusterIP, NodePort, and LoadBalancer. \"ExternalName\" maps to the specified externalName. \"ClusterIP\" allocates a cluster-internal IP address for load-balancing to endpoints. Endpoints are determined by the selector or if that is not specified, by manual construction of an Endpoints object. If clusterIP is \"None\", no virtual IP is allocated and the endpoints are published as a set of endpoints rather than a stable IP. \"NodePort\" builds on ClusterIP and allocates a port on every node which routes to the clusterIP. \"LoadBalancer\" builds on NodePort and creates an external load-balancer (if supported in the current cloud) which routes to the clusterIP. More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types", "type": "string" } }, "type": "object" } }, "type": "object" }, "tls": { "description": "TLS defines options for configuring TLS for HTTP.", "properties": { "certificate": { "description": "Certificate is a reference to a Kubernetes secret that contains the certificate and private key for enabling TLS. The referenced secret should contain the following: \n - `ca.crt`: The certificate authority (optional). - `tls.crt`: The certificate (or a chain). - `tls.key`: The private key to the first certificate in the certificate chain.", "properties": { "secretName": { "description": "SecretName is the name of the secret.", "type": "string" } }, "type": "object" }, "selfSignedCertificate": { "description": "SelfSignedCertificate allows configuring the self-signed certificate generated by the operator.", "properties": { "disabled": { "description": "Disabled indicates that the provisioning of the self-signed certificate should be disabled.", "type": "boolean" }, "subjectAltNames": { "description": "SubjectAlternativeNames is a list of SANs to include in the generated HTTP TLS certificate.", "items": { "description": "SubjectAlternativeName represents a SAN entry in a x509 certificate.", "properties": { "dns": { "description": "DNS is the DNS name of the subject.", "type": "string" }, "ip": { "description": "IP is the IP address of the subject.", "type": "string" } }, "type": "object" }, "type": "array" } }, "type": "object" } }, "type": "object" } }, "type": "object" }, "image": { "description": "Image is the Elasticsearch Docker image to deploy.", "type": "string" }, "nodeSets": { "description": "NodeSets allow specifying groups of Elasticsearch nodes sharing the same configuration and Pod templates. See: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-orchestration.html", "items": { "description": "NodeSet is the specification for a group of Elasticsearch nodes sharing the same configuration and a Pod template.", "properties": { "config": { "description": "Config holds the Elasticsearch configuration.", "type": "object" }, "count": { "description": "Count of Elasticsearch nodes to deploy.", "format": "int32", "minimum": 1, "type": "integer" }, "name": { "description": "Name of this set of nodes. Becomes a part of the Elasticsearch node.name setting.", "maxLength": 23, "pattern": "[a-zA-Z0-9-]+", "type": "string" }, "podTemplate": { "description": "PodTemplate provides customisation options (labels, annotations, affinity rules, resource requests, and so on) for the Pods belonging to this NodeSet.", "type": "object" }, "volumeClaimTemplates": { "description": "VolumeClaimTemplates is a list of persistent volume claims to be used by each Pod in this NodeSet. Every claim in this list must have a matching volumeMount in one of the containers defined in the PodTemplate. Items defined here take precedence over any default claims added by the operator with the same name. See: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-volume-claim-templates.html", "items": { "description": "PersistentVolumeClaim is a user's request for and claim to a persistent volume", "properties": { "apiVersion": { "description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", "type": "string" }, "kind": { "description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", "type": "string" }, "metadata": { "description": "Standard object's metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata", "type": "object" }, "spec": { "description": "Spec defines the desired characteristics of a volume requested by a pod author. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims", "properties": { "accessModes": { "description": "AccessModes contains the desired access modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1", "items": { "type": "string" }, "type": "array" }, "dataSource": { "description": "This field requires the VolumeSnapshotDataSource alpha feature gate to be enabled and currently VolumeSnapshot is the only supported data source. If the provisioner can support VolumeSnapshot data source, it will create a new volume and data will be restored to the volume at the same time. If the provisioner does not support VolumeSnapshot data source, volume will not be created and the failure will be reported as an event. In the future, we plan to support more data source types and the behavior of the provisioner may change.", "properties": { "apiGroup": { "description": "APIGroup is the group for the resource being referenced. If APIGroup is not specified, the specified Kind must be in the core API group. For any other third-party types, APIGroup is required.", "type": "string" }, "kind": { "description": "Kind is the type of resource being referenced", "type": "string" }, "name": { "description": "Name is the name of resource being referenced", "type": "string" } }, "required": [ "kind", "name" ], "type": "object" }, "resources": { "description": "Resources represents the minimum resources the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources", "properties": { "limits": { "additionalProperties": { "anyOf": [ { "type": "integer" }, { "type": "string" } ], "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$" }, "description": "Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", "type": "object" }, "requests": { "additionalProperties": { "anyOf": [ { "type": "integer" }, { "type": "string" } ], "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$" }, "description": "Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", "type": "object" } }, "type": "object" }, "selector": { "description": "A label query over volumes to consider for binding.", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "items": { "type": "string" }, "type": "array" } }, "required": [ "key", "operator" ], "type": "object" }, "type": "array" }, "matchLabels": { "additionalProperties": { "type": "string" }, "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object" } }, "type": "object" }, "storageClassName": { "description": "Name of the StorageClass required by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1", "type": "string" }, "volumeMode": { "description": "volumeMode defines what type of volume is required by the claim. Value of Filesystem is implied when not included in claim spec. This is a beta feature.", "type": "string" }, "volumeName": { "description": "VolumeName is the binding reference to the PersistentVolume backing this claim.", "type": "string" } }, "type": "object" }, "status": { "description": "Status represents the current information/status of a persistent volume claim. Read-only. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims", "properties": { "accessModes": { "description": "AccessModes contains the actual access modes the volume backing the PVC has. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1", "items": { "type": "string" }, "type": "array" }, "capacity": { "additionalProperties": { "anyOf": [ { "type": "integer" }, { "type": "string" } ], "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$" }, "description": "Represents the actual resources of the underlying volume.", "type": "object" }, "conditions": { "description": "Current Condition of persistent volume claim. If underlying persistent volume is being resized then the Condition will be set to 'ResizeStarted'.", "items": { "description": "PersistentVolumeClaimCondition contains details about state of pvc", "properties": { "lastProbeTime": { "description": "Last time we probed the condition.", "format": "date-time", "type": "string" }, "lastTransitionTime": { "description": "Last time the condition transitioned from one status to another.", "format": "date-time", "type": "string" }, "message": { "description": "Human-readable message indicating details about last transition.", "type": "string" }, "reason": { "description": "Unique, this should be a short, machine understandable string that gives the reason for condition's last transition. If it reports \"ResizeStarted\" that means the underlying persistent volume is being resized.", "type": "string" }, "status": { "type": "string" }, "type": { "description": "PersistentVolumeClaimConditionType is a valid value of PersistentVolumeClaimCondition.Type", "type": "string" } }, "required": [ "status", "type" ], "type": "object" }, "type": "array" }, "phase": { "description": "Phase represents the current phase of PersistentVolumeClaim.", "type": "string" } }, "type": "object" } }, "type": "object" }, "type": "array" } }, "required": [ "count", "name" ], "type": "object" }, "minItems": 1, "type": "array" }, "podDisruptionBudget": { "description": "PodDisruptionBudget provides access to the default pod disruption budget for the Elasticsearch cluster. The default budget selects all cluster pods and sets `maxUnavailable` to 1. To disable, set `PodDisruptionBudget` to the empty value (`{}` in YAML).", "properties": { "metadata": { "description": "ObjectMeta is the metadata of the PDB. The name and namespace provided here are managed by ECK and will be ignored.", "type": "object" }, "spec": { "description": "Spec is the specification of the PDB.", "properties": { "maxUnavailable": { "anyOf": [ { "type": "integer" }, { "type": "string" } ], "description": "An eviction is allowed if at most \"maxUnavailable\" pods selected by \"selector\" are unavailable after the eviction, i.e. even in absence of the evicted pod. For example, one can prevent all voluntary evictions by specifying 0. This is a mutually exclusive setting with \"minAvailable\"." }, "minAvailable": { "anyOf": [ { "type": "integer" }, { "type": "string" } ], "description": "An eviction is allowed if at least \"minAvailable\" pods selected by \"selector\" will still be available after the eviction, i.e. even in the absence of the evicted pod. So for example you can prevent all voluntary evictions by specifying \"100%\"." }, "selector": { "description": "Label query over pods whose evictions are managed by the disruption budget.", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "items": { "type": "string" }, "type": "array" } }, "required": [ "key", "operator" ], "type": "object" }, "type": "array" }, "matchLabels": { "additionalProperties": { "type": "string" }, "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object" } }, "type": "object" } }, "type": "object" } }, "type": "object" }, "remoteClusters": { "description": "RemoteClusters enables you to establish uni-directional connections to a remote Elasticsearch cluster.", "items": { "description": "RemoteCluster declares a remote Elasticsearch cluster connection.", "properties": { "elasticsearchRef": { "description": "ElasticsearchRef is a reference to an Elasticsearch cluster running within the same k8s cluster.", "properties": { "name": { "description": "Name of the Kubernetes object.", "type": "string" }, "namespace": { "description": "Namespace of the Kubernetes object. If empty, defaults to the current namespace.", "type": "string" } }, "required": [ "name" ], "type": "object" }, "name": { "description": "Name is the name of the remote cluster as it is set in the Elasticsearch settings. The name is expected to be unique for each remote clusters.", "minLength": 1, "type": "string" } }, "required": [ "name" ], "type": "object" }, "type": "array" }, "secureSettings": { "description": "SecureSettings is a list of references to Kubernetes secrets containing sensitive configuration options for Elasticsearch. See: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-es-secure-settings.html", "items": { "description": "SecretSource defines a data source based on a Kubernetes Secret.", "properties": { "entries": { "description": "Entries define how to project each key-value pair in the secret to filesystem paths. If not defined, all keys will be projected to similarly named paths in the filesystem. If defined, only the specified keys will be projected to the corresponding paths.", "items": { "description": "KeyToPath defines how to map a key in a Secret object to a filesystem path.", "properties": { "key": { "description": "Key is the key contained in the secret.", "type": "string" }, "path": { "description": "Path is the relative file path to map the key to. Path must not be an absolute file path and must not contain any \"..\" components.", "type": "string" } }, "required": [ "key" ], "type": "object" }, "type": "array" }, "secretName": { "description": "SecretName is the name of the secret.", "type": "string" } }, "required": [ "secretName" ], "type": "object" }, "type": "array" }, "serviceAccountName": { "description": "ServiceAccountName is used to check access from the current resource to a resource (eg. a remote Elasticsearch cluster) in a different namespace. Can only be used if ECK is enforcing RBAC on references.", "type": "string" }, "transport": { "description": "Transport holds transport layer settings for Elasticsearch.", "properties": { "service": { "description": "Service defines the template for the associated Kubernetes Service object.", "properties": { "metadata": { "description": "ObjectMeta is the metadata of the service. The name and namespace provided here are managed by ECK and will be ignored.", "type": "object" }, "spec": { "description": "Spec is the specification of the service.", "properties": { "clusterIP": { "description": "clusterIP is the IP address of the service and is usually assigned randomly by the master. If an address is specified manually and is not in use by others, it will be allocated to the service; otherwise, creation of the service will fail. This field can not be changed through updates. Valid values are \"None\", empty string (\"\"), or a valid IP address. \"None\" can be specified for headless services when proxying is not required. Only applies to types ClusterIP, NodePort, and LoadBalancer. Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", "type": "string" }, "externalIPs": { "description": "externalIPs is a list of IP addresses for which nodes in the cluster will also accept traffic for this service. These IPs are not managed by Kubernetes. The user is responsible for ensuring that traffic arrives at a node with this IP. A common example is external load-balancers that are not part of the Kubernetes system.", "items": { "type": "string" }, "type": "array" }, "externalName": { "description": "externalName is the external reference that kubedns or equivalent will return as a CNAME record for this service. No proxying will be involved. Must be a valid RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) and requires Type to be ExternalName.", "type": "string" }, "externalTrafficPolicy": { "description": "externalTrafficPolicy denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints. \"Local\" preserves the client source IP and avoids a second hop for LoadBalancer and Nodeport type services, but risks potentially imbalanced traffic spreading. \"Cluster\" obscures the client source IP and may cause a second hop to another node, but should have good overall load-spreading.", "type": "string" }, "healthCheckNodePort": { "description": "healthCheckNodePort specifies the healthcheck nodePort for the service. If not specified, HealthCheckNodePort is created by the service api backend with the allocated nodePort. Will use user-specified nodePort value if specified by the client. Only effects when Type is set to LoadBalancer and ExternalTrafficPolicy is set to Local.", "format": "int32", "type": "integer" }, "ipFamily": { "description": "ipFamily specifies whether this Service has a preference for a particular IP family (e.g. IPv4 vs. IPv6). If a specific IP family is requested, the clusterIP field will be allocated from that family, if it is available in the cluster. If no IP family is requested, the cluster's primary IP family will be used. Other IP fields (loadBalancerIP, loadBalancerSourceRanges, externalIPs) and controllers which allocate external load-balancers should use the same IP family. Endpoints for this Service will be of this family. This field is immutable after creation. Assigning a ServiceIPFamily not available in the cluster (e.g. IPv6 in IPv4 only cluster) is an error condition and will fail during clusterIP assignment.", "type": "string" }, "loadBalancerIP": { "description": "Only applies to Service Type: LoadBalancer LoadBalancer will get created with the IP specified in this field. This feature depends on whether the underlying cloud-provider supports specifying the loadBalancerIP when a load balancer is created. This field will be ignored if the cloud-provider does not support the feature.", "type": "string" }, "loadBalancerSourceRanges": { "description": "If specified and supported by the platform, this will restrict traffic through the cloud-provider load-balancer will be restricted to the specified client IPs. This field will be ignored if the cloud-provider does not support the feature.\" More info: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/", "items": { "type": "string" }, "type": "array" }, "ports": { "description": "The list of ports that are exposed by this service. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", "items": { "description": "ServicePort contains information on service's port.", "properties": { "name": { "description": "The name of this port within the service. This must be a DNS_LABEL. All ports within a ServiceSpec must have unique names. When considering the endpoints for a Service, this must match the 'name' field in the EndpointPort. Optional if only one ServicePort is defined on this service.", "type": "string" }, "nodePort": { "description": "The port on each node on which this service is exposed when type=NodePort or LoadBalancer. Usually assigned by the system. If specified, it will be allocated to the service if unused or else creation of the service will fail. Default is to auto-allocate a port if the ServiceType of this Service requires one. More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport", "format": "int32", "type": "integer" }, "port": { "description": "The port that will be exposed by this service.", "format": "int32", "type": "integer" }, "protocol": { "description": "The IP protocol for this port. Supports \"TCP\", \"UDP\", and \"SCTP\". Default is TCP.", "type": "string" }, "targetPort": { "anyOf": [ { "type": "integer" }, { "type": "string" } ], "description": "Number or name of the port to access on the pods targeted by the service. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. If this is a string, it will be looked up as a named port in the target Pod's container ports. If this is not specified, the value of the 'port' field is used (an identity map). This field is ignored for services with clusterIP=None, and should be omitted or set equal to the 'port' field. More info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service" } }, "required": [ "port" ], "type": "object" }, "type": "array" }, "publishNotReadyAddresses": { "description": "publishNotReadyAddresses, when set to true, indicates that DNS implementations must publish the notReadyAddresses of subsets for the Endpoints associated with the Service. The default value is false. The primary use case for setting this field is to use a StatefulSet's Headless Service to propagate SRV records for its Pods without respect to their readiness for purpose of peer discovery.", "type": "boolean" }, "selector": { "additionalProperties": { "type": "string" }, "description": "Route service traffic to pods with label keys and values matching this selector. If empty or not present, the service is assumed to have an external process managing its endpoints, which Kubernetes will not modify. Only applies to types ClusterIP, NodePort, and LoadBalancer. Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/", "type": "object" }, "sessionAffinity": { "description": "Supports \"ClientIP\" and \"None\". Used to maintain session affinity. Enable client IP based session affinity. Must be ClientIP or None. Defaults to None. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", "type": "string" }, "sessionAffinityConfig": { "description": "sessionAffinityConfig contains the configurations of session affinity.", "properties": { "clientIP": { "description": "clientIP contains the configurations of Client IP based session affinity.", "properties": { "timeoutSeconds": { "description": "timeoutSeconds specifies the seconds of ClientIP type session sticky time. The value must be \u003e0 \u0026\u0026 \u003c=86400(for 1 day) if ServiceAffinity == \"ClientIP\". Default value is 10800(for 3 hours).", "format": "int32", "type": "integer" } }, "type": "object" } }, "type": "object" }, "topologyKeys": { "description": "topologyKeys is a preference-order list of topology keys which implementations of services should use to preferentially sort endpoints when accessing this Service, it can not be used at the same time as externalTrafficPolicy=Local. Topology keys must be valid label keys and at most 16 keys may be specified. Endpoints are chosen based on the first topology key with available backends. If this field is specified and all entries have no backends that match the topology of the client, the service has no backends for that client and connections should fail. The special value \"*\" may be used to mean \"any topology\". This catch-all value, if used, only makes sense as the last value in the list. If this is not specified or empty, no topology constraints will be applied.", "items": { "type": "string" }, "type": "array" }, "type": { "description": "type determines how the Service is exposed. Defaults to ClusterIP. Valid options are ExternalName, ClusterIP, NodePort, and LoadBalancer. \"ExternalName\" maps to the specified externalName. \"ClusterIP\" allocates a cluster-internal IP address for load-balancing to endpoints. Endpoints are determined by the selector or if that is not specified, by manual construction of an Endpoints object. If clusterIP is \"None\", no virtual IP is allocated and the endpoints are published as a set of endpoints rather than a stable IP. \"NodePort\" builds on ClusterIP and allocates a port on every node which routes to the clusterIP. \"LoadBalancer\" builds on NodePort and creates an external load-balancer (if supported in the current cloud) which routes to the clusterIP. More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types", "type": "string" } }, "type": "object" } }, "type": "object" } }, "type": "object" }, "updateStrategy": { "description": "UpdateStrategy specifies how updates to the cluster should be performed.", "properties": { "changeBudget": { "description": "ChangeBudget defines the constraints to consider when applying changes to the Elasticsearch cluster.", "properties": { "maxSurge": { "description": "MaxSurge is the maximum number of new pods that can be created exceeding the original number of pods defined in the specification. MaxSurge is only taken into consideration when scaling up. Setting a negative value will disable the restriction. Defaults to unbounded if not specified.", "format": "int32", "type": "integer" }, "maxUnavailable": { "description": "MaxUnavailable is the maximum number of pods that can be unavailable (not ready) during the update due to circumstances under the control of the operator. Setting a negative value will disable this restriction. Defaults to 1 if not specified.", "format": "int32", "type": "integer" } }, "type": "object" } }, "type": "object" }, "version": { "description": "Version of Elasticsearch.", "type": "string" } }, "required": [ "nodeSets", "version" ], "type": "object" }, "status": { "description": "ElasticsearchStatus defines the observed state of Elasticsearch", "properties": { "availableNodes": { "format": "int32", "type": "integer" }, "health": { "description": "ElasticsearchHealth is the health of the cluster as returned by the health API.", "type": "string" }, "phase": { "description": "ElasticsearchOrchestrationPhase is the phase Elasticsearch is in from the controller point of view.", "type": "string" } }, "type": "object" } } } }, "served": false, "storage": false, "subresources": { "status": {} } } ] }, "status": { "acceptedNames": { "categories": [ "elastic" ], "kind": "Elasticsearch", "listKind": "ElasticsearchList", "plural": "elasticsearches", "shortNames": [ "es" ], "singular": "elasticsearch" }, "conditions": [ { "lastTransitionTime": "2020-04-28T23:31:51Z", "message": "[spec.validation.openAPIV3Schema.properties[spec].properties[http].properties[service].properties[spec].properties[ports].items.properties[targetPort].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[nodeSets].items.properties[volumeClaimTemplates].items.properties[spec].properties[resources].properties[limits].additionalProperties.type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[nodeSets].items.properties[volumeClaimTemplates].items.properties[spec].properties[resources].properties[requests].additionalProperties.type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[nodeSets].items.properties[volumeClaimTemplates].items.properties[status].properties[capacity].additionalProperties.type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[podDisruptionBudget].properties[spec].properties[maxUnavailable].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[podDisruptionBudget].properties[spec].properties[minAvailable].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[transport].properties[service].properties[spec].properties[ports].items.properties[targetPort].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.type: Required value: must not be empty at the root]", "reason": "Violations", "status": "True", "type": "NonStructuralSchema" }, { "lastTransitionTime": "2020-04-28T23:31:51Z", "message": "no conflicts found", "reason": "NoConflicts", "status": "True", "type": "NamesAccepted" }, { "lastTransitionTime": "2020-04-28T23:31:51Z", "message": "the initial names have been accepted", "reason": "InitialNamesAccepted", "status": "True", "type": "Established" } ], "storedVersions": [ "v1" ] } } ================================================ FILE: pkg/backup/actions/testdata/v1/gcpsamples.gcp.stacks.crossplane.io.json ================================================ { "apiVersion": "apiextensions.k8s.io/v1", "kind": "CustomResourceDefinition", "metadata": { "annotations": { "kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"apiextensions.k8s.io/v1beta1\",\"kind\":\"CustomResourceDefinition\",\"metadata\":{\"annotations\":{},\"creationTimestamp\":\"2020-04-20T16:57:37Z\",\"generation\":1,\"name\":\"gcpsamples.gcp.stacks.crossplane.io\",\"resourceVersion\":\"549\",\"selfLink\":\"/apis/apiextensions.k8s.io/v1/customresourcedefinitions/gcpsamples.gcp.stacks.crossplane.io\",\"uid\":\"db5f4321-3226-44b0-8247-66fd7ef59dc8\"},\"spec\":{\"conversion\":{\"strategy\":\"None\"},\"group\":\"gcp.stacks.crossplane.io\",\"names\":{\"kind\":\"GCPSample\",\"listKind\":\"GCPSampleList\",\"plural\":\"gcpsamples\",\"singular\":\"gcpsample\"},\"preserveUnknownFields\":true,\"scope\":\"Cluster\",\"versions\":[{\"name\":\"v1alpha1\",\"served\":true,\"storage\":true}]},\"status\":{\"acceptedNames\":{\"kind\":\"GCPSample\",\"listKind\":\"GCPSampleList\",\"plural\":\"gcpsamples\",\"singular\":\"gcpsample\"},\"conditions\":[{\"lastTransitionTime\":\"2020-04-20T16:57:37Z\",\"message\":\"no conflicts found\",\"reason\":\"NoConflicts\",\"status\":\"True\",\"type\":\"NamesAccepted\"},{\"lastTransitionTime\":\"2020-04-20T16:57:37Z\",\"message\":\"the initial names have been accepted\",\"reason\":\"InitialNamesAccepted\",\"status\":\"True\",\"type\":\"Established\"}],\"storedVersions\":[\"v1alpha1\"]}}\n" }, "creationTimestamp": "2020-04-20T17:27:56Z", "generation": 1, "name": "gcpsamples.gcp.stacks.crossplane.io", "resourceVersion": "5567", "selfLink": "/apis/apiextensions.k8s.io/v1/customresourcedefinitions/gcpsamples.gcp.stacks.crossplane.io", "uid": "c0bbac74-acab-4620-b628-1d5f91b19040" }, "spec": { "conversion": { "strategy": "None" }, "group": "gcp.stacks.crossplane.io", "names": { "kind": "GCPSample", "listKind": "GCPSampleList", "plural": "gcpsamples", "singular": "gcpsample" }, "preserveUnknownFields": true, "scope": "Cluster", "versions": [ { "name": "v1alpha1", "served": true, "storage": true } ] }, "status": { "acceptedNames": { "kind": "GCPSample", "listKind": "GCPSampleList", "plural": "gcpsamples", "singular": "gcpsample" }, "conditions": [ { "lastTransitionTime": "2020-04-20T17:27:56Z", "message": "no conflicts found", "reason": "NoConflicts", "status": "True", "type": "NamesAccepted" }, { "lastTransitionTime": "2020-04-20T17:27:56Z", "message": "the initial names have been accepted", "reason": "InitialNamesAccepted", "status": "True", "type": "Established" } ], "storedVersions": [ "v1alpha1" ] } } ================================================ FILE: pkg/backup/actions/testdata/v1/kibanas.kibana.k8s.elastic.co.json ================================================ { "apiVersion": "apiextensions.k8s.io/v1", "kind": "CustomResourceDefinition", "metadata": { "annotations": { "controller-gen.kubebuilder.io/version": "v0.2.5", "kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"apiextensions.k8s.io/v1beta1\",\"kind\":\"CustomResourceDefinition\",\"metadata\":{\"annotations\":{\"controller-gen.kubebuilder.io/version\":\"v0.2.5\"},\"creationTimestamp\":null,\"name\":\"kibanas.kibana.k8s.elastic.co\"},\"spec\":{\"additionalPrinterColumns\":[{\"JSONPath\":\".status.health\",\"name\":\"health\",\"type\":\"string\"},{\"JSONPath\":\".status.availableNodes\",\"description\":\"Available nodes\",\"name\":\"nodes\",\"type\":\"integer\"},{\"JSONPath\":\".spec.version\",\"description\":\"Kibana version\",\"name\":\"version\",\"type\":\"string\"},{\"JSONPath\":\".metadata.creationTimestamp\",\"name\":\"age\",\"type\":\"date\"}],\"group\":\"kibana.k8s.elastic.co\",\"names\":{\"categories\":[\"elastic\"],\"kind\":\"Kibana\",\"listKind\":\"KibanaList\",\"plural\":\"kibanas\",\"shortNames\":[\"kb\"],\"singular\":\"kibana\"},\"scope\":\"Namespaced\",\"subresources\":{\"status\":{}},\"validation\":{\"openAPIV3Schema\":{\"description\":\"Kibana represents a Kibana resource in a Kubernetes cluster.\",\"properties\":{\"apiVersion\":{\"description\":\"APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\",\"type\":\"string\"},\"kind\":{\"description\":\"Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\",\"type\":\"string\"},\"metadata\":{\"type\":\"object\"},\"spec\":{\"description\":\"KibanaSpec holds the specification of a Kibana instance.\",\"properties\":{\"config\":{\"description\":\"Config holds the Kibana configuration. See: https://www.elastic.co/guide/en/kibana/current/settings.html\",\"type\":\"object\"},\"count\":{\"description\":\"Count of Kibana instances to deploy.\",\"format\":\"int32\",\"type\":\"integer\"},\"elasticsearchRef\":{\"description\":\"ElasticsearchRef is a reference to an Elasticsearch cluster running in the same Kubernetes cluster.\",\"properties\":{\"name\":{\"description\":\"Name of the Kubernetes object.\",\"type\":\"string\"},\"namespace\":{\"description\":\"Namespace of the Kubernetes object. If empty, defaults to the current namespace.\",\"type\":\"string\"}},\"required\":[\"name\"],\"type\":\"object\"},\"http\":{\"description\":\"HTTP holds the HTTP layer configuration for Kibana.\",\"properties\":{\"service\":{\"description\":\"Service defines the template for the associated Kubernetes Service object.\",\"properties\":{\"metadata\":{\"description\":\"ObjectMeta is the metadata of the service. The name and namespace provided here are managed by ECK and will be ignored.\",\"type\":\"object\"},\"spec\":{\"description\":\"Spec is the specification of the service.\",\"properties\":{\"clusterIP\":{\"description\":\"clusterIP is the IP address of the service and is usually assigned randomly by the master. If an address is specified manually and is not in use by others, it will be allocated to the service; otherwise, creation of the service will fail. This field can not be changed through updates. Valid values are \\\"None\\\", empty string (\\\"\\\"), or a valid IP address. \\\"None\\\" can be specified for headless services when proxying is not required. Only applies to types ClusterIP, NodePort, and LoadBalancer. Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies\",\"type\":\"string\"},\"externalIPs\":{\"description\":\"externalIPs is a list of IP addresses for which nodes in the cluster will also accept traffic for this service. These IPs are not managed by Kubernetes. The user is responsible for ensuring that traffic arrives at a node with this IP. A common example is external load-balancers that are not part of the Kubernetes system.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"externalName\":{\"description\":\"externalName is the external reference that kubedns or equivalent will return as a CNAME record for this service. No proxying will be involved. Must be a valid RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) and requires Type to be ExternalName.\",\"type\":\"string\"},\"externalTrafficPolicy\":{\"description\":\"externalTrafficPolicy denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints. \\\"Local\\\" preserves the client source IP and avoids a second hop for LoadBalancer and Nodeport type services, but risks potentially imbalanced traffic spreading. \\\"Cluster\\\" obscures the client source IP and may cause a second hop to another node, but should have good overall load-spreading.\",\"type\":\"string\"},\"healthCheckNodePort\":{\"description\":\"healthCheckNodePort specifies the healthcheck nodePort for the service. If not specified, HealthCheckNodePort is created by the service api backend with the allocated nodePort. Will use user-specified nodePort value if specified by the client. Only effects when Type is set to LoadBalancer and ExternalTrafficPolicy is set to Local.\",\"format\":\"int32\",\"type\":\"integer\"},\"ipFamily\":{\"description\":\"ipFamily specifies whether this Service has a preference for a particular IP family (e.g. IPv4 vs. IPv6). If a specific IP family is requested, the clusterIP field will be allocated from that family, if it is available in the cluster. If no IP family is requested, the cluster's primary IP family will be used. Other IP fields (loadBalancerIP, loadBalancerSourceRanges, externalIPs) and controllers which allocate external load-balancers should use the same IP family. Endpoints for this Service will be of this family. This field is immutable after creation. Assigning a ServiceIPFamily not available in the cluster (e.g. IPv6 in IPv4 only cluster) is an error condition and will fail during clusterIP assignment.\",\"type\":\"string\"},\"loadBalancerIP\":{\"description\":\"Only applies to Service Type: LoadBalancer LoadBalancer will get created with the IP specified in this field. This feature depends on whether the underlying cloud-provider supports specifying the loadBalancerIP when a load balancer is created. This field will be ignored if the cloud-provider does not support the feature.\",\"type\":\"string\"},\"loadBalancerSourceRanges\":{\"description\":\"If specified and supported by the platform, this will restrict traffic through the cloud-provider load-balancer will be restricted to the specified client IPs. This field will be ignored if the cloud-provider does not support the feature.\\\" More info: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"ports\":{\"description\":\"The list of ports that are exposed by this service. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies\",\"items\":{\"description\":\"ServicePort contains information on service's port.\",\"properties\":{\"name\":{\"description\":\"The name of this port within the service. This must be a DNS_LABEL. All ports within a ServiceSpec must have unique names. When considering the endpoints for a Service, this must match the 'name' field in the EndpointPort. Optional if only one ServicePort is defined on this service.\",\"type\":\"string\"},\"nodePort\":{\"description\":\"The port on each node on which this service is exposed when type=NodePort or LoadBalancer. Usually assigned by the system. If specified, it will be allocated to the service if unused or else creation of the service will fail. Default is to auto-allocate a port if the ServiceType of this Service requires one. More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport\",\"format\":\"int32\",\"type\":\"integer\"},\"port\":{\"description\":\"The port that will be exposed by this service.\",\"format\":\"int32\",\"type\":\"integer\"},\"protocol\":{\"description\":\"The IP protocol for this port. Supports \\\"TCP\\\", \\\"UDP\\\", and \\\"SCTP\\\". Default is TCP.\",\"type\":\"string\"},\"targetPort\":{\"anyOf\":[{\"type\":\"integer\"},{\"type\":\"string\"}],\"description\":\"Number or name of the port to access on the pods targeted by the service. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. If this is a string, it will be looked up as a named port in the target Pod's container ports. If this is not specified, the value of the 'port' field is used (an identity map). This field is ignored for services with clusterIP=None, and should be omitted or set equal to the 'port' field. More info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service\"}},\"required\":[\"port\"],\"type\":\"object\"},\"type\":\"array\"},\"publishNotReadyAddresses\":{\"description\":\"publishNotReadyAddresses, when set to true, indicates that DNS implementations must publish the notReadyAddresses of subsets for the Endpoints associated with the Service. The default value is false. The primary use case for setting this field is to use a StatefulSet's Headless Service to propagate SRV records for its Pods without respect to their readiness for purpose of peer discovery.\",\"type\":\"boolean\"},\"selector\":{\"additionalProperties\":{\"type\":\"string\"},\"description\":\"Route service traffic to pods with label keys and values matching this selector. If empty or not present, the service is assumed to have an external process managing its endpoints, which Kubernetes will not modify. Only applies to types ClusterIP, NodePort, and LoadBalancer. Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/\",\"type\":\"object\"},\"sessionAffinity\":{\"description\":\"Supports \\\"ClientIP\\\" and \\\"None\\\". Used to maintain session affinity. Enable client IP based session affinity. Must be ClientIP or None. Defaults to None. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies\",\"type\":\"string\"},\"sessionAffinityConfig\":{\"description\":\"sessionAffinityConfig contains the configurations of session affinity.\",\"properties\":{\"clientIP\":{\"description\":\"clientIP contains the configurations of Client IP based session affinity.\",\"properties\":{\"timeoutSeconds\":{\"description\":\"timeoutSeconds specifies the seconds of ClientIP type session sticky time. The value must be \\u003e0 \\u0026\\u0026 \\u003c=86400(for 1 day) if ServiceAffinity == \\\"ClientIP\\\". Default value is 10800(for 3 hours).\",\"format\":\"int32\",\"type\":\"integer\"}},\"type\":\"object\"}},\"type\":\"object\"},\"topologyKeys\":{\"description\":\"topologyKeys is a preference-order list of topology keys which implementations of services should use to preferentially sort endpoints when accessing this Service, it can not be used at the same time as externalTrafficPolicy=Local. Topology keys must be valid label keys and at most 16 keys may be specified. Endpoints are chosen based on the first topology key with available backends. If this field is specified and all entries have no backends that match the topology of the client, the service has no backends for that client and connections should fail. The special value \\\"*\\\" may be used to mean \\\"any topology\\\". This catch-all value, if used, only makes sense as the last value in the list. If this is not specified or empty, no topology constraints will be applied.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"type\":{\"description\":\"type determines how the Service is exposed. Defaults to ClusterIP. Valid options are ExternalName, ClusterIP, NodePort, and LoadBalancer. \\\"ExternalName\\\" maps to the specified externalName. \\\"ClusterIP\\\" allocates a cluster-internal IP address for load-balancing to endpoints. Endpoints are determined by the selector or if that is not specified, by manual construction of an Endpoints object. If clusterIP is \\\"None\\\", no virtual IP is allocated and the endpoints are published as a set of endpoints rather than a stable IP. \\\"NodePort\\\" builds on ClusterIP and allocates a port on every node which routes to the clusterIP. \\\"LoadBalancer\\\" builds on NodePort and creates an external load-balancer (if supported in the current cloud) which routes to the clusterIP. More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types\",\"type\":\"string\"}},\"type\":\"object\"}},\"type\":\"object\"},\"tls\":{\"description\":\"TLS defines options for configuring TLS for HTTP.\",\"properties\":{\"certificate\":{\"description\":\"Certificate is a reference to a Kubernetes secret that contains the certificate and private key for enabling TLS. The referenced secret should contain the following: \\n - `ca.crt`: The certificate authority (optional). - `tls.crt`: The certificate (or a chain). - `tls.key`: The private key to the first certificate in the certificate chain.\",\"properties\":{\"secretName\":{\"description\":\"SecretName is the name of the secret.\",\"type\":\"string\"}},\"type\":\"object\"},\"selfSignedCertificate\":{\"description\":\"SelfSignedCertificate allows configuring the self-signed certificate generated by the operator.\",\"properties\":{\"disabled\":{\"description\":\"Disabled indicates that the provisioning of the self-signed certificate should be disabled.\",\"type\":\"boolean\"},\"subjectAltNames\":{\"description\":\"SubjectAlternativeNames is a list of SANs to include in the generated HTTP TLS certificate.\",\"items\":{\"description\":\"SubjectAlternativeName represents a SAN entry in a x509 certificate.\",\"properties\":{\"dns\":{\"description\":\"DNS is the DNS name of the subject.\",\"type\":\"string\"},\"ip\":{\"description\":\"IP is the IP address of the subject.\",\"type\":\"string\"}},\"type\":\"object\"},\"type\":\"array\"}},\"type\":\"object\"}},\"type\":\"object\"}},\"type\":\"object\"},\"image\":{\"description\":\"Image is the Kibana Docker image to deploy.\",\"type\":\"string\"},\"podTemplate\":{\"description\":\"PodTemplate provides customisation options (labels, annotations, affinity rules, resource requests, and so on) for the Kibana pods\",\"type\":\"object\"},\"secureSettings\":{\"description\":\"SecureSettings is a list of references to Kubernetes secrets containing sensitive configuration options for Kibana. See: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-kibana.html#k8s-kibana-secure-settings\",\"items\":{\"description\":\"SecretSource defines a data source based on a Kubernetes Secret.\",\"properties\":{\"entries\":{\"description\":\"Entries define how to project each key-value pair in the secret to filesystem paths. If not defined, all keys will be projected to similarly named paths in the filesystem. If defined, only the specified keys will be projected to the corresponding paths.\",\"items\":{\"description\":\"KeyToPath defines how to map a key in a Secret object to a filesystem path.\",\"properties\":{\"key\":{\"description\":\"Key is the key contained in the secret.\",\"type\":\"string\"},\"path\":{\"description\":\"Path is the relative file path to map the key to. Path must not be an absolute file path and must not contain any \\\"..\\\" components.\",\"type\":\"string\"}},\"required\":[\"key\"],\"type\":\"object\"},\"type\":\"array\"},\"secretName\":{\"description\":\"SecretName is the name of the secret.\",\"type\":\"string\"}},\"required\":[\"secretName\"],\"type\":\"object\"},\"type\":\"array\"},\"serviceAccountName\":{\"description\":\"ServiceAccountName is used to check access from the current resource to a resource (eg. Elasticsearch) in a different namespace. Can only be used if ECK is enforcing RBAC on references.\",\"type\":\"string\"},\"version\":{\"description\":\"Version of Kibana.\",\"type\":\"string\"}},\"required\":[\"version\"],\"type\":\"object\"},\"status\":{\"description\":\"KibanaStatus defines the observed state of Kibana\",\"properties\":{\"associationStatus\":{\"description\":\"AssociationStatus is the status of an association resource.\",\"type\":\"string\"},\"availableNodes\":{\"format\":\"int32\",\"type\":\"integer\"},\"health\":{\"description\":\"KibanaHealth expresses the status of the Kibana instances.\",\"type\":\"string\"}},\"type\":\"object\"}}}},\"version\":\"v1\",\"versions\":[{\"name\":\"v1\",\"served\":true,\"storage\":true},{\"name\":\"v1beta1\",\"served\":true,\"storage\":false},{\"name\":\"v1alpha1\",\"served\":false,\"storage\":false}]},\"status\":{\"acceptedNames\":{\"kind\":\"\",\"plural\":\"\"},\"conditions\":[],\"storedVersions\":[]}}\n" }, "creationTimestamp": "2020-04-28T23:31:53Z", "generation": 1, "labels": { "velero.io/backup-name": "es", "velero.io/restore-name": "es-crds" }, "name": "kibanas.kibana.k8s.elastic.co", "resourceVersion": "1703552", "selfLink": "/apis/apiextensions.k8s.io/v1/customresourcedefinitions/kibanas.kibana.k8s.elastic.co", "uid": "95f42a77-654f-4380-a6b1-1fe2587f0713" }, "spec": { "conversion": { "strategy": "None" }, "group": "kibana.k8s.elastic.co", "names": { "categories": [ "elastic" ], "kind": "Kibana", "listKind": "KibanaList", "plural": "kibanas", "shortNames": [ "kb" ], "singular": "kibana" }, "preserveUnknownFields": true, "scope": "Namespaced", "versions": [ { "additionalPrinterColumns": [ { "jsonPath": ".status.health", "name": "health", "type": "string" }, { "description": "Available nodes", "jsonPath": ".status.availableNodes", "name": "nodes", "type": "integer" }, { "description": "Kibana version", "jsonPath": ".spec.version", "name": "version", "type": "string" }, { "jsonPath": ".metadata.creationTimestamp", "name": "age", "type": "date" } ], "name": "v1", "schema": { "openAPIV3Schema": { "description": "Kibana represents a Kibana resource in a Kubernetes cluster.", "properties": { "apiVersion": { "description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", "type": "string" }, "kind": { "description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", "type": "string" }, "metadata": { "type": "object" }, "spec": { "description": "KibanaSpec holds the specification of a Kibana instance.", "properties": { "config": { "description": "Config holds the Kibana configuration. See: https://www.elastic.co/guide/en/kibana/current/settings.html", "type": "object" }, "count": { "description": "Count of Kibana instances to deploy.", "format": "int32", "type": "integer" }, "elasticsearchRef": { "description": "ElasticsearchRef is a reference to an Elasticsearch cluster running in the same Kubernetes cluster.", "properties": { "name": { "description": "Name of the Kubernetes object.", "type": "string" }, "namespace": { "description": "Namespace of the Kubernetes object. If empty, defaults to the current namespace.", "type": "string" } }, "required": [ "name" ], "type": "object" }, "http": { "description": "HTTP holds the HTTP layer configuration for Kibana.", "properties": { "service": { "description": "Service defines the template for the associated Kubernetes Service object.", "properties": { "metadata": { "description": "ObjectMeta is the metadata of the service. The name and namespace provided here are managed by ECK and will be ignored.", "type": "object" }, "spec": { "description": "Spec is the specification of the service.", "properties": { "clusterIP": { "description": "clusterIP is the IP address of the service and is usually assigned randomly by the master. If an address is specified manually and is not in use by others, it will be allocated to the service; otherwise, creation of the service will fail. This field can not be changed through updates. Valid values are \"None\", empty string (\"\"), or a valid IP address. \"None\" can be specified for headless services when proxying is not required. Only applies to types ClusterIP, NodePort, and LoadBalancer. Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", "type": "string" }, "externalIPs": { "description": "externalIPs is a list of IP addresses for which nodes in the cluster will also accept traffic for this service. These IPs are not managed by Kubernetes. The user is responsible for ensuring that traffic arrives at a node with this IP. A common example is external load-balancers that are not part of the Kubernetes system.", "items": { "type": "string" }, "type": "array" }, "externalName": { "description": "externalName is the external reference that kubedns or equivalent will return as a CNAME record for this service. No proxying will be involved. Must be a valid RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) and requires Type to be ExternalName.", "type": "string" }, "externalTrafficPolicy": { "description": "externalTrafficPolicy denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints. \"Local\" preserves the client source IP and avoids a second hop for LoadBalancer and Nodeport type services, but risks potentially imbalanced traffic spreading. \"Cluster\" obscures the client source IP and may cause a second hop to another node, but should have good overall load-spreading.", "type": "string" }, "healthCheckNodePort": { "description": "healthCheckNodePort specifies the healthcheck nodePort for the service. If not specified, HealthCheckNodePort is created by the service api backend with the allocated nodePort. Will use user-specified nodePort value if specified by the client. Only effects when Type is set to LoadBalancer and ExternalTrafficPolicy is set to Local.", "format": "int32", "type": "integer" }, "ipFamily": { "description": "ipFamily specifies whether this Service has a preference for a particular IP family (e.g. IPv4 vs. IPv6). If a specific IP family is requested, the clusterIP field will be allocated from that family, if it is available in the cluster. If no IP family is requested, the cluster's primary IP family will be used. Other IP fields (loadBalancerIP, loadBalancerSourceRanges, externalIPs) and controllers which allocate external load-balancers should use the same IP family. Endpoints for this Service will be of this family. This field is immutable after creation. Assigning a ServiceIPFamily not available in the cluster (e.g. IPv6 in IPv4 only cluster) is an error condition and will fail during clusterIP assignment.", "type": "string" }, "loadBalancerIP": { "description": "Only applies to Service Type: LoadBalancer LoadBalancer will get created with the IP specified in this field. This feature depends on whether the underlying cloud-provider supports specifying the loadBalancerIP when a load balancer is created. This field will be ignored if the cloud-provider does not support the feature.", "type": "string" }, "loadBalancerSourceRanges": { "description": "If specified and supported by the platform, this will restrict traffic through the cloud-provider load-balancer will be restricted to the specified client IPs. This field will be ignored if the cloud-provider does not support the feature.\" More info: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/", "items": { "type": "string" }, "type": "array" }, "ports": { "description": "The list of ports that are exposed by this service. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", "items": { "description": "ServicePort contains information on service's port.", "properties": { "name": { "description": "The name of this port within the service. This must be a DNS_LABEL. All ports within a ServiceSpec must have unique names. When considering the endpoints for a Service, this must match the 'name' field in the EndpointPort. Optional if only one ServicePort is defined on this service.", "type": "string" }, "nodePort": { "description": "The port on each node on which this service is exposed when type=NodePort or LoadBalancer. Usually assigned by the system. If specified, it will be allocated to the service if unused or else creation of the service will fail. Default is to auto-allocate a port if the ServiceType of this Service requires one. More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport", "format": "int32", "type": "integer" }, "port": { "description": "The port that will be exposed by this service.", "format": "int32", "type": "integer" }, "protocol": { "description": "The IP protocol for this port. Supports \"TCP\", \"UDP\", and \"SCTP\". Default is TCP.", "type": "string" }, "targetPort": { "anyOf": [ { "type": "integer" }, { "type": "string" } ], "description": "Number or name of the port to access on the pods targeted by the service. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. If this is a string, it will be looked up as a named port in the target Pod's container ports. If this is not specified, the value of the 'port' field is used (an identity map). This field is ignored for services with clusterIP=None, and should be omitted or set equal to the 'port' field. More info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service" } }, "required": [ "port" ], "type": "object" }, "type": "array" }, "publishNotReadyAddresses": { "description": "publishNotReadyAddresses, when set to true, indicates that DNS implementations must publish the notReadyAddresses of subsets for the Endpoints associated with the Service. The default value is false. The primary use case for setting this field is to use a StatefulSet's Headless Service to propagate SRV records for its Pods without respect to their readiness for purpose of peer discovery.", "type": "boolean" }, "selector": { "additionalProperties": { "type": "string" }, "description": "Route service traffic to pods with label keys and values matching this selector. If empty or not present, the service is assumed to have an external process managing its endpoints, which Kubernetes will not modify. Only applies to types ClusterIP, NodePort, and LoadBalancer. Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/", "type": "object" }, "sessionAffinity": { "description": "Supports \"ClientIP\" and \"None\". Used to maintain session affinity. Enable client IP based session affinity. Must be ClientIP or None. Defaults to None. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", "type": "string" }, "sessionAffinityConfig": { "description": "sessionAffinityConfig contains the configurations of session affinity.", "properties": { "clientIP": { "description": "clientIP contains the configurations of Client IP based session affinity.", "properties": { "timeoutSeconds": { "description": "timeoutSeconds specifies the seconds of ClientIP type session sticky time. The value must be \u003e0 \u0026\u0026 \u003c=86400(for 1 day) if ServiceAffinity == \"ClientIP\". Default value is 10800(for 3 hours).", "format": "int32", "type": "integer" } }, "type": "object" } }, "type": "object" }, "topologyKeys": { "description": "topologyKeys is a preference-order list of topology keys which implementations of services should use to preferentially sort endpoints when accessing this Service, it can not be used at the same time as externalTrafficPolicy=Local. Topology keys must be valid label keys and at most 16 keys may be specified. Endpoints are chosen based on the first topology key with available backends. If this field is specified and all entries have no backends that match the topology of the client, the service has no backends for that client and connections should fail. The special value \"*\" may be used to mean \"any topology\". This catch-all value, if used, only makes sense as the last value in the list. If this is not specified or empty, no topology constraints will be applied.", "items": { "type": "string" }, "type": "array" }, "type": { "description": "type determines how the Service is exposed. Defaults to ClusterIP. Valid options are ExternalName, ClusterIP, NodePort, and LoadBalancer. \"ExternalName\" maps to the specified externalName. \"ClusterIP\" allocates a cluster-internal IP address for load-balancing to endpoints. Endpoints are determined by the selector or if that is not specified, by manual construction of an Endpoints object. If clusterIP is \"None\", no virtual IP is allocated and the endpoints are published as a set of endpoints rather than a stable IP. \"NodePort\" builds on ClusterIP and allocates a port on every node which routes to the clusterIP. \"LoadBalancer\" builds on NodePort and creates an external load-balancer (if supported in the current cloud) which routes to the clusterIP. More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types", "type": "string" } }, "type": "object" } }, "type": "object" }, "tls": { "description": "TLS defines options for configuring TLS for HTTP.", "properties": { "certificate": { "description": "Certificate is a reference to a Kubernetes secret that contains the certificate and private key for enabling TLS. The referenced secret should contain the following: \n - `ca.crt`: The certificate authority (optional). - `tls.crt`: The certificate (or a chain). - `tls.key`: The private key to the first certificate in the certificate chain.", "properties": { "secretName": { "description": "SecretName is the name of the secret.", "type": "string" } }, "type": "object" }, "selfSignedCertificate": { "description": "SelfSignedCertificate allows configuring the self-signed certificate generated by the operator.", "properties": { "disabled": { "description": "Disabled indicates that the provisioning of the self-signed certificate should be disabled.", "type": "boolean" }, "subjectAltNames": { "description": "SubjectAlternativeNames is a list of SANs to include in the generated HTTP TLS certificate.", "items": { "description": "SubjectAlternativeName represents a SAN entry in a x509 certificate.", "properties": { "dns": { "description": "DNS is the DNS name of the subject.", "type": "string" }, "ip": { "description": "IP is the IP address of the subject.", "type": "string" } }, "type": "object" }, "type": "array" } }, "type": "object" } }, "type": "object" } }, "type": "object" }, "image": { "description": "Image is the Kibana Docker image to deploy.", "type": "string" }, "podTemplate": { "description": "PodTemplate provides customisation options (labels, annotations, affinity rules, resource requests, and so on) for the Kibana pods", "type": "object" }, "secureSettings": { "description": "SecureSettings is a list of references to Kubernetes secrets containing sensitive configuration options for Kibana. See: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-kibana.html#k8s-kibana-secure-settings", "items": { "description": "SecretSource defines a data source based on a Kubernetes Secret.", "properties": { "entries": { "description": "Entries define how to project each key-value pair in the secret to filesystem paths. If not defined, all keys will be projected to similarly named paths in the filesystem. If defined, only the specified keys will be projected to the corresponding paths.", "items": { "description": "KeyToPath defines how to map a key in a Secret object to a filesystem path.", "properties": { "key": { "description": "Key is the key contained in the secret.", "type": "string" }, "path": { "description": "Path is the relative file path to map the key to. Path must not be an absolute file path and must not contain any \"..\" components.", "type": "string" } }, "required": [ "key" ], "type": "object" }, "type": "array" }, "secretName": { "description": "SecretName is the name of the secret.", "type": "string" } }, "required": [ "secretName" ], "type": "object" }, "type": "array" }, "serviceAccountName": { "description": "ServiceAccountName is used to check access from the current resource to a resource (eg. Elasticsearch) in a different namespace. Can only be used if ECK is enforcing RBAC on references.", "type": "string" }, "version": { "description": "Version of Kibana.", "type": "string" } }, "required": [ "version" ], "type": "object" }, "status": { "description": "KibanaStatus defines the observed state of Kibana", "properties": { "associationStatus": { "description": "AssociationStatus is the status of an association resource.", "type": "string" }, "availableNodes": { "format": "int32", "type": "integer" }, "health": { "description": "KibanaHealth expresses the status of the Kibana instances.", "type": "string" } }, "type": "object" } } } }, "served": true, "storage": true, "subresources": { "status": {} } }, { "additionalPrinterColumns": [ { "jsonPath": ".status.health", "name": "health", "type": "string" }, { "description": "Available nodes", "jsonPath": ".status.availableNodes", "name": "nodes", "type": "integer" }, { "description": "Kibana version", "jsonPath": ".spec.version", "name": "version", "type": "string" }, { "jsonPath": ".metadata.creationTimestamp", "name": "age", "type": "date" } ], "name": "v1beta1", "schema": { "openAPIV3Schema": { "description": "Kibana represents a Kibana resource in a Kubernetes cluster.", "properties": { "apiVersion": { "description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", "type": "string" }, "kind": { "description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", "type": "string" }, "metadata": { "type": "object" }, "spec": { "description": "KibanaSpec holds the specification of a Kibana instance.", "properties": { "config": { "description": "Config holds the Kibana configuration. See: https://www.elastic.co/guide/en/kibana/current/settings.html", "type": "object" }, "count": { "description": "Count of Kibana instances to deploy.", "format": "int32", "type": "integer" }, "elasticsearchRef": { "description": "ElasticsearchRef is a reference to an Elasticsearch cluster running in the same Kubernetes cluster.", "properties": { "name": { "description": "Name of the Kubernetes object.", "type": "string" }, "namespace": { "description": "Namespace of the Kubernetes object. If empty, defaults to the current namespace.", "type": "string" } }, "required": [ "name" ], "type": "object" }, "http": { "description": "HTTP holds the HTTP layer configuration for Kibana.", "properties": { "service": { "description": "Service defines the template for the associated Kubernetes Service object.", "properties": { "metadata": { "description": "ObjectMeta is the metadata of the service. The name and namespace provided here are managed by ECK and will be ignored.", "type": "object" }, "spec": { "description": "Spec is the specification of the service.", "properties": { "clusterIP": { "description": "clusterIP is the IP address of the service and is usually assigned randomly by the master. If an address is specified manually and is not in use by others, it will be allocated to the service; otherwise, creation of the service will fail. This field can not be changed through updates. Valid values are \"None\", empty string (\"\"), or a valid IP address. \"None\" can be specified for headless services when proxying is not required. Only applies to types ClusterIP, NodePort, and LoadBalancer. Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", "type": "string" }, "externalIPs": { "description": "externalIPs is a list of IP addresses for which nodes in the cluster will also accept traffic for this service. These IPs are not managed by Kubernetes. The user is responsible for ensuring that traffic arrives at a node with this IP. A common example is external load-balancers that are not part of the Kubernetes system.", "items": { "type": "string" }, "type": "array" }, "externalName": { "description": "externalName is the external reference that kubedns or equivalent will return as a CNAME record for this service. No proxying will be involved. Must be a valid RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) and requires Type to be ExternalName.", "type": "string" }, "externalTrafficPolicy": { "description": "externalTrafficPolicy denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints. \"Local\" preserves the client source IP and avoids a second hop for LoadBalancer and Nodeport type services, but risks potentially imbalanced traffic spreading. \"Cluster\" obscures the client source IP and may cause a second hop to another node, but should have good overall load-spreading.", "type": "string" }, "healthCheckNodePort": { "description": "healthCheckNodePort specifies the healthcheck nodePort for the service. If not specified, HealthCheckNodePort is created by the service api backend with the allocated nodePort. Will use user-specified nodePort value if specified by the client. Only effects when Type is set to LoadBalancer and ExternalTrafficPolicy is set to Local.", "format": "int32", "type": "integer" }, "ipFamily": { "description": "ipFamily specifies whether this Service has a preference for a particular IP family (e.g. IPv4 vs. IPv6). If a specific IP family is requested, the clusterIP field will be allocated from that family, if it is available in the cluster. If no IP family is requested, the cluster's primary IP family will be used. Other IP fields (loadBalancerIP, loadBalancerSourceRanges, externalIPs) and controllers which allocate external load-balancers should use the same IP family. Endpoints for this Service will be of this family. This field is immutable after creation. Assigning a ServiceIPFamily not available in the cluster (e.g. IPv6 in IPv4 only cluster) is an error condition and will fail during clusterIP assignment.", "type": "string" }, "loadBalancerIP": { "description": "Only applies to Service Type: LoadBalancer LoadBalancer will get created with the IP specified in this field. This feature depends on whether the underlying cloud-provider supports specifying the loadBalancerIP when a load balancer is created. This field will be ignored if the cloud-provider does not support the feature.", "type": "string" }, "loadBalancerSourceRanges": { "description": "If specified and supported by the platform, this will restrict traffic through the cloud-provider load-balancer will be restricted to the specified client IPs. This field will be ignored if the cloud-provider does not support the feature.\" More info: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/", "items": { "type": "string" }, "type": "array" }, "ports": { "description": "The list of ports that are exposed by this service. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", "items": { "description": "ServicePort contains information on service's port.", "properties": { "name": { "description": "The name of this port within the service. This must be a DNS_LABEL. All ports within a ServiceSpec must have unique names. When considering the endpoints for a Service, this must match the 'name' field in the EndpointPort. Optional if only one ServicePort is defined on this service.", "type": "string" }, "nodePort": { "description": "The port on each node on which this service is exposed when type=NodePort or LoadBalancer. Usually assigned by the system. If specified, it will be allocated to the service if unused or else creation of the service will fail. Default is to auto-allocate a port if the ServiceType of this Service requires one. More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport", "format": "int32", "type": "integer" }, "port": { "description": "The port that will be exposed by this service.", "format": "int32", "type": "integer" }, "protocol": { "description": "The IP protocol for this port. Supports \"TCP\", \"UDP\", and \"SCTP\". Default is TCP.", "type": "string" }, "targetPort": { "anyOf": [ { "type": "integer" }, { "type": "string" } ], "description": "Number or name of the port to access on the pods targeted by the service. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. If this is a string, it will be looked up as a named port in the target Pod's container ports. If this is not specified, the value of the 'port' field is used (an identity map). This field is ignored for services with clusterIP=None, and should be omitted or set equal to the 'port' field. More info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service" } }, "required": [ "port" ], "type": "object" }, "type": "array" }, "publishNotReadyAddresses": { "description": "publishNotReadyAddresses, when set to true, indicates that DNS implementations must publish the notReadyAddresses of subsets for the Endpoints associated with the Service. The default value is false. The primary use case for setting this field is to use a StatefulSet's Headless Service to propagate SRV records for its Pods without respect to their readiness for purpose of peer discovery.", "type": "boolean" }, "selector": { "additionalProperties": { "type": "string" }, "description": "Route service traffic to pods with label keys and values matching this selector. If empty or not present, the service is assumed to have an external process managing its endpoints, which Kubernetes will not modify. Only applies to types ClusterIP, NodePort, and LoadBalancer. Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/", "type": "object" }, "sessionAffinity": { "description": "Supports \"ClientIP\" and \"None\". Used to maintain session affinity. Enable client IP based session affinity. Must be ClientIP or None. Defaults to None. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", "type": "string" }, "sessionAffinityConfig": { "description": "sessionAffinityConfig contains the configurations of session affinity.", "properties": { "clientIP": { "description": "clientIP contains the configurations of Client IP based session affinity.", "properties": { "timeoutSeconds": { "description": "timeoutSeconds specifies the seconds of ClientIP type session sticky time. The value must be \u003e0 \u0026\u0026 \u003c=86400(for 1 day) if ServiceAffinity == \"ClientIP\". Default value is 10800(for 3 hours).", "format": "int32", "type": "integer" } }, "type": "object" } }, "type": "object" }, "topologyKeys": { "description": "topologyKeys is a preference-order list of topology keys which implementations of services should use to preferentially sort endpoints when accessing this Service, it can not be used at the same time as externalTrafficPolicy=Local. Topology keys must be valid label keys and at most 16 keys may be specified. Endpoints are chosen based on the first topology key with available backends. If this field is specified and all entries have no backends that match the topology of the client, the service has no backends for that client and connections should fail. The special value \"*\" may be used to mean \"any topology\". This catch-all value, if used, only makes sense as the last value in the list. If this is not specified or empty, no topology constraints will be applied.", "items": { "type": "string" }, "type": "array" }, "type": { "description": "type determines how the Service is exposed. Defaults to ClusterIP. Valid options are ExternalName, ClusterIP, NodePort, and LoadBalancer. \"ExternalName\" maps to the specified externalName. \"ClusterIP\" allocates a cluster-internal IP address for load-balancing to endpoints. Endpoints are determined by the selector or if that is not specified, by manual construction of an Endpoints object. If clusterIP is \"None\", no virtual IP is allocated and the endpoints are published as a set of endpoints rather than a stable IP. \"NodePort\" builds on ClusterIP and allocates a port on every node which routes to the clusterIP. \"LoadBalancer\" builds on NodePort and creates an external load-balancer (if supported in the current cloud) which routes to the clusterIP. More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types", "type": "string" } }, "type": "object" } }, "type": "object" }, "tls": { "description": "TLS defines options for configuring TLS for HTTP.", "properties": { "certificate": { "description": "Certificate is a reference to a Kubernetes secret that contains the certificate and private key for enabling TLS. The referenced secret should contain the following: \n - `ca.crt`: The certificate authority (optional). - `tls.crt`: The certificate (or a chain). - `tls.key`: The private key to the first certificate in the certificate chain.", "properties": { "secretName": { "description": "SecretName is the name of the secret.", "type": "string" } }, "type": "object" }, "selfSignedCertificate": { "description": "SelfSignedCertificate allows configuring the self-signed certificate generated by the operator.", "properties": { "disabled": { "description": "Disabled indicates that the provisioning of the self-signed certificate should be disabled.", "type": "boolean" }, "subjectAltNames": { "description": "SubjectAlternativeNames is a list of SANs to include in the generated HTTP TLS certificate.", "items": { "description": "SubjectAlternativeName represents a SAN entry in a x509 certificate.", "properties": { "dns": { "description": "DNS is the DNS name of the subject.", "type": "string" }, "ip": { "description": "IP is the IP address of the subject.", "type": "string" } }, "type": "object" }, "type": "array" } }, "type": "object" } }, "type": "object" } }, "type": "object" }, "image": { "description": "Image is the Kibana Docker image to deploy.", "type": "string" }, "podTemplate": { "description": "PodTemplate provides customisation options (labels, annotations, affinity rules, resource requests, and so on) for the Kibana pods", "type": "object" }, "secureSettings": { "description": "SecureSettings is a list of references to Kubernetes secrets containing sensitive configuration options for Kibana. See: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-kibana.html#k8s-kibana-secure-settings", "items": { "description": "SecretSource defines a data source based on a Kubernetes Secret.", "properties": { "entries": { "description": "Entries define how to project each key-value pair in the secret to filesystem paths. If not defined, all keys will be projected to similarly named paths in the filesystem. If defined, only the specified keys will be projected to the corresponding paths.", "items": { "description": "KeyToPath defines how to map a key in a Secret object to a filesystem path.", "properties": { "key": { "description": "Key is the key contained in the secret.", "type": "string" }, "path": { "description": "Path is the relative file path to map the key to. Path must not be an absolute file path and must not contain any \"..\" components.", "type": "string" } }, "required": [ "key" ], "type": "object" }, "type": "array" }, "secretName": { "description": "SecretName is the name of the secret.", "type": "string" } }, "required": [ "secretName" ], "type": "object" }, "type": "array" }, "serviceAccountName": { "description": "ServiceAccountName is used to check access from the current resource to a resource (eg. Elasticsearch) in a different namespace. Can only be used if ECK is enforcing RBAC on references.", "type": "string" }, "version": { "description": "Version of Kibana.", "type": "string" } }, "required": [ "version" ], "type": "object" }, "status": { "description": "KibanaStatus defines the observed state of Kibana", "properties": { "associationStatus": { "description": "AssociationStatus is the status of an association resource.", "type": "string" }, "availableNodes": { "format": "int32", "type": "integer" }, "health": { "description": "KibanaHealth expresses the status of the Kibana instances.", "type": "string" } }, "type": "object" } } } }, "served": true, "storage": false, "subresources": { "status": {} } }, { "additionalPrinterColumns": [ { "jsonPath": ".status.health", "name": "health", "type": "string" }, { "description": "Available nodes", "jsonPath": ".status.availableNodes", "name": "nodes", "type": "integer" }, { "description": "Kibana version", "jsonPath": ".spec.version", "name": "version", "type": "string" }, { "jsonPath": ".metadata.creationTimestamp", "name": "age", "type": "date" } ], "name": "v1alpha1", "schema": { "openAPIV3Schema": { "description": "Kibana represents a Kibana resource in a Kubernetes cluster.", "properties": { "apiVersion": { "description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", "type": "string" }, "kind": { "description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", "type": "string" }, "metadata": { "type": "object" }, "spec": { "description": "KibanaSpec holds the specification of a Kibana instance.", "properties": { "config": { "description": "Config holds the Kibana configuration. See: https://www.elastic.co/guide/en/kibana/current/settings.html", "type": "object" }, "count": { "description": "Count of Kibana instances to deploy.", "format": "int32", "type": "integer" }, "elasticsearchRef": { "description": "ElasticsearchRef is a reference to an Elasticsearch cluster running in the same Kubernetes cluster.", "properties": { "name": { "description": "Name of the Kubernetes object.", "type": "string" }, "namespace": { "description": "Namespace of the Kubernetes object. If empty, defaults to the current namespace.", "type": "string" } }, "required": [ "name" ], "type": "object" }, "http": { "description": "HTTP holds the HTTP layer configuration for Kibana.", "properties": { "service": { "description": "Service defines the template for the associated Kubernetes Service object.", "properties": { "metadata": { "description": "ObjectMeta is the metadata of the service. The name and namespace provided here are managed by ECK and will be ignored.", "type": "object" }, "spec": { "description": "Spec is the specification of the service.", "properties": { "clusterIP": { "description": "clusterIP is the IP address of the service and is usually assigned randomly by the master. If an address is specified manually and is not in use by others, it will be allocated to the service; otherwise, creation of the service will fail. This field can not be changed through updates. Valid values are \"None\", empty string (\"\"), or a valid IP address. \"None\" can be specified for headless services when proxying is not required. Only applies to types ClusterIP, NodePort, and LoadBalancer. Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", "type": "string" }, "externalIPs": { "description": "externalIPs is a list of IP addresses for which nodes in the cluster will also accept traffic for this service. These IPs are not managed by Kubernetes. The user is responsible for ensuring that traffic arrives at a node with this IP. A common example is external load-balancers that are not part of the Kubernetes system.", "items": { "type": "string" }, "type": "array" }, "externalName": { "description": "externalName is the external reference that kubedns or equivalent will return as a CNAME record for this service. No proxying will be involved. Must be a valid RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) and requires Type to be ExternalName.", "type": "string" }, "externalTrafficPolicy": { "description": "externalTrafficPolicy denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints. \"Local\" preserves the client source IP and avoids a second hop for LoadBalancer and Nodeport type services, but risks potentially imbalanced traffic spreading. \"Cluster\" obscures the client source IP and may cause a second hop to another node, but should have good overall load-spreading.", "type": "string" }, "healthCheckNodePort": { "description": "healthCheckNodePort specifies the healthcheck nodePort for the service. If not specified, HealthCheckNodePort is created by the service api backend with the allocated nodePort. Will use user-specified nodePort value if specified by the client. Only effects when Type is set to LoadBalancer and ExternalTrafficPolicy is set to Local.", "format": "int32", "type": "integer" }, "ipFamily": { "description": "ipFamily specifies whether this Service has a preference for a particular IP family (e.g. IPv4 vs. IPv6). If a specific IP family is requested, the clusterIP field will be allocated from that family, if it is available in the cluster. If no IP family is requested, the cluster's primary IP family will be used. Other IP fields (loadBalancerIP, loadBalancerSourceRanges, externalIPs) and controllers which allocate external load-balancers should use the same IP family. Endpoints for this Service will be of this family. This field is immutable after creation. Assigning a ServiceIPFamily not available in the cluster (e.g. IPv6 in IPv4 only cluster) is an error condition and will fail during clusterIP assignment.", "type": "string" }, "loadBalancerIP": { "description": "Only applies to Service Type: LoadBalancer LoadBalancer will get created with the IP specified in this field. This feature depends on whether the underlying cloud-provider supports specifying the loadBalancerIP when a load balancer is created. This field will be ignored if the cloud-provider does not support the feature.", "type": "string" }, "loadBalancerSourceRanges": { "description": "If specified and supported by the platform, this will restrict traffic through the cloud-provider load-balancer will be restricted to the specified client IPs. This field will be ignored if the cloud-provider does not support the feature.\" More info: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/", "items": { "type": "string" }, "type": "array" }, "ports": { "description": "The list of ports that are exposed by this service. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", "items": { "description": "ServicePort contains information on service's port.", "properties": { "name": { "description": "The name of this port within the service. This must be a DNS_LABEL. All ports within a ServiceSpec must have unique names. When considering the endpoints for a Service, this must match the 'name' field in the EndpointPort. Optional if only one ServicePort is defined on this service.", "type": "string" }, "nodePort": { "description": "The port on each node on which this service is exposed when type=NodePort or LoadBalancer. Usually assigned by the system. If specified, it will be allocated to the service if unused or else creation of the service will fail. Default is to auto-allocate a port if the ServiceType of this Service requires one. More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport", "format": "int32", "type": "integer" }, "port": { "description": "The port that will be exposed by this service.", "format": "int32", "type": "integer" }, "protocol": { "description": "The IP protocol for this port. Supports \"TCP\", \"UDP\", and \"SCTP\". Default is TCP.", "type": "string" }, "targetPort": { "anyOf": [ { "type": "integer" }, { "type": "string" } ], "description": "Number or name of the port to access on the pods targeted by the service. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. If this is a string, it will be looked up as a named port in the target Pod's container ports. If this is not specified, the value of the 'port' field is used (an identity map). This field is ignored for services with clusterIP=None, and should be omitted or set equal to the 'port' field. More info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service" } }, "required": [ "port" ], "type": "object" }, "type": "array" }, "publishNotReadyAddresses": { "description": "publishNotReadyAddresses, when set to true, indicates that DNS implementations must publish the notReadyAddresses of subsets for the Endpoints associated with the Service. The default value is false. The primary use case for setting this field is to use a StatefulSet's Headless Service to propagate SRV records for its Pods without respect to their readiness for purpose of peer discovery.", "type": "boolean" }, "selector": { "additionalProperties": { "type": "string" }, "description": "Route service traffic to pods with label keys and values matching this selector. If empty or not present, the service is assumed to have an external process managing its endpoints, which Kubernetes will not modify. Only applies to types ClusterIP, NodePort, and LoadBalancer. Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/", "type": "object" }, "sessionAffinity": { "description": "Supports \"ClientIP\" and \"None\". Used to maintain session affinity. Enable client IP based session affinity. Must be ClientIP or None. Defaults to None. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", "type": "string" }, "sessionAffinityConfig": { "description": "sessionAffinityConfig contains the configurations of session affinity.", "properties": { "clientIP": { "description": "clientIP contains the configurations of Client IP based session affinity.", "properties": { "timeoutSeconds": { "description": "timeoutSeconds specifies the seconds of ClientIP type session sticky time. The value must be \u003e0 \u0026\u0026 \u003c=86400(for 1 day) if ServiceAffinity == \"ClientIP\". Default value is 10800(for 3 hours).", "format": "int32", "type": "integer" } }, "type": "object" } }, "type": "object" }, "topologyKeys": { "description": "topologyKeys is a preference-order list of topology keys which implementations of services should use to preferentially sort endpoints when accessing this Service, it can not be used at the same time as externalTrafficPolicy=Local. Topology keys must be valid label keys and at most 16 keys may be specified. Endpoints are chosen based on the first topology key with available backends. If this field is specified and all entries have no backends that match the topology of the client, the service has no backends for that client and connections should fail. The special value \"*\" may be used to mean \"any topology\". This catch-all value, if used, only makes sense as the last value in the list. If this is not specified or empty, no topology constraints will be applied.", "items": { "type": "string" }, "type": "array" }, "type": { "description": "type determines how the Service is exposed. Defaults to ClusterIP. Valid options are ExternalName, ClusterIP, NodePort, and LoadBalancer. \"ExternalName\" maps to the specified externalName. \"ClusterIP\" allocates a cluster-internal IP address for load-balancing to endpoints. Endpoints are determined by the selector or if that is not specified, by manual construction of an Endpoints object. If clusterIP is \"None\", no virtual IP is allocated and the endpoints are published as a set of endpoints rather than a stable IP. \"NodePort\" builds on ClusterIP and allocates a port on every node which routes to the clusterIP. \"LoadBalancer\" builds on NodePort and creates an external load-balancer (if supported in the current cloud) which routes to the clusterIP. More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types", "type": "string" } }, "type": "object" } }, "type": "object" }, "tls": { "description": "TLS defines options for configuring TLS for HTTP.", "properties": { "certificate": { "description": "Certificate is a reference to a Kubernetes secret that contains the certificate and private key for enabling TLS. The referenced secret should contain the following: \n - `ca.crt`: The certificate authority (optional). - `tls.crt`: The certificate (or a chain). - `tls.key`: The private key to the first certificate in the certificate chain.", "properties": { "secretName": { "description": "SecretName is the name of the secret.", "type": "string" } }, "type": "object" }, "selfSignedCertificate": { "description": "SelfSignedCertificate allows configuring the self-signed certificate generated by the operator.", "properties": { "disabled": { "description": "Disabled indicates that the provisioning of the self-signed certificate should be disabled.", "type": "boolean" }, "subjectAltNames": { "description": "SubjectAlternativeNames is a list of SANs to include in the generated HTTP TLS certificate.", "items": { "description": "SubjectAlternativeName represents a SAN entry in a x509 certificate.", "properties": { "dns": { "description": "DNS is the DNS name of the subject.", "type": "string" }, "ip": { "description": "IP is the IP address of the subject.", "type": "string" } }, "type": "object" }, "type": "array" } }, "type": "object" } }, "type": "object" } }, "type": "object" }, "image": { "description": "Image is the Kibana Docker image to deploy.", "type": "string" }, "podTemplate": { "description": "PodTemplate provides customisation options (labels, annotations, affinity rules, resource requests, and so on) for the Kibana pods", "type": "object" }, "secureSettings": { "description": "SecureSettings is a list of references to Kubernetes secrets containing sensitive configuration options for Kibana. See: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-kibana.html#k8s-kibana-secure-settings", "items": { "description": "SecretSource defines a data source based on a Kubernetes Secret.", "properties": { "entries": { "description": "Entries define how to project each key-value pair in the secret to filesystem paths. If not defined, all keys will be projected to similarly named paths in the filesystem. If defined, only the specified keys will be projected to the corresponding paths.", "items": { "description": "KeyToPath defines how to map a key in a Secret object to a filesystem path.", "properties": { "key": { "description": "Key is the key contained in the secret.", "type": "string" }, "path": { "description": "Path is the relative file path to map the key to. Path must not be an absolute file path and must not contain any \"..\" components.", "type": "string" } }, "required": [ "key" ], "type": "object" }, "type": "array" }, "secretName": { "description": "SecretName is the name of the secret.", "type": "string" } }, "required": [ "secretName" ], "type": "object" }, "type": "array" }, "serviceAccountName": { "description": "ServiceAccountName is used to check access from the current resource to a resource (eg. Elasticsearch) in a different namespace. Can only be used if ECK is enforcing RBAC on references.", "type": "string" }, "version": { "description": "Version of Kibana.", "type": "string" } }, "required": [ "version" ], "type": "object" }, "status": { "description": "KibanaStatus defines the observed state of Kibana", "properties": { "associationStatus": { "description": "AssociationStatus is the status of an association resource.", "type": "string" }, "availableNodes": { "format": "int32", "type": "integer" }, "health": { "description": "KibanaHealth expresses the status of the Kibana instances.", "type": "string" } }, "type": "object" } } } }, "served": false, "storage": false, "subresources": { "status": {} } } ] }, "status": { "acceptedNames": { "categories": [ "elastic" ], "kind": "Kibana", "listKind": "KibanaList", "plural": "kibanas", "shortNames": [ "kb" ], "singular": "kibana" }, "conditions": [ { "lastTransitionTime": "2020-04-28T23:31:53Z", "message": "[spec.validation.openAPIV3Schema.properties[spec].properties[http].properties[service].properties[spec].properties[ports].items.properties[targetPort].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.type: Required value: must not be empty at the root]", "reason": "Violations", "status": "True", "type": "NonStructuralSchema" }, { "lastTransitionTime": "2020-04-28T23:31:53Z", "message": "no conflicts found", "reason": "NoConflicts", "status": "True", "type": "NamesAccepted" }, { "lastTransitionTime": "2020-04-28T23:31:53Z", "message": "the initial names have been accepted", "reason": "InitialNamesAccepted", "status": "True", "type": "Established" } ], "storedVersions": [ "v1" ] } } ================================================ FILE: pkg/backup/actions/testdata/v1/pprometheuses.monitoring.coreos.com.json ================================================ { "apiVersion": "apiextensions.k8s.io/v1", "kind": "CustomResourceDefinition", "metadata": { "annotations": { "controller-gen.kubebuilder.io/version": "v0.2.4", "kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"apiextensions.k8s.io/v1beta1\",\"kind\":\"CustomResourceDefinition\",\"metadata\":{\"annotations\":{\"controller-gen.kubebuilder.io/version\":\"v0.2.4\"},\"creationTimestamp\":null,\"name\":\"prometheuses.monitoring.coreos.com\"},\"spec\":{\"additionalPrinterColumns\":[{\"JSONPath\":\".spec.version\",\"description\":\"The version of Prometheus\",\"name\":\"Version\",\"type\":\"string\"},{\"JSONPath\":\".spec.replicas\",\"description\":\"The desired replicas number of Prometheuses\",\"name\":\"Replicas\",\"type\":\"integer\"},{\"JSONPath\":\".metadata.creationTimestamp\",\"name\":\"Age\",\"type\":\"date\"}],\"group\":\"monitoring.coreos.com\",\"names\":{\"kind\":\"Prometheus\",\"listKind\":\"PrometheusList\",\"plural\":\"prometheuses\",\"singular\":\"prometheus\"},\"preserveUnknownFields\":false,\"scope\":\"Namespaced\",\"subresources\":{},\"validation\":{\"openAPIV3Schema\":{\"description\":\"Prometheus defines a Prometheus deployment.\",\"properties\":{\"apiVersion\":{\"description\":\"APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\",\"type\":\"string\"},\"kind\":{\"description\":\"Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\",\"type\":\"string\"},\"metadata\":{\"type\":\"object\"},\"spec\":{\"description\":\"Specification of the desired behavior of the Prometheus cluster. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status\",\"properties\":{\"additionalAlertManagerConfigs\":{\"description\":\"AdditionalAlertManagerConfigs allows specifying a key of a Secret containing additional Prometheus AlertManager configurations. AlertManager configurations specified are appended to the configurations generated by the Prometheus Operator. Job configurations specified must have the form as specified in the official Prometheus documentation: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#alertmanager_config. As AlertManager configs are appended, the user is responsible to make sure it is valid. Note that using this feature may expose the possibility to break upgrades of Prometheus. It is advised to review Prometheus release notes to ensure that no incompatible AlertManager configs are going to break Prometheus after the upgrade.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or its key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"},\"additionalAlertRelabelConfigs\":{\"description\":\"AdditionalAlertRelabelConfigs allows specifying a key of a Secret containing additional Prometheus alert relabel configurations. Alert relabel configurations specified are appended to the configurations generated by the Prometheus Operator. Alert relabel configurations specified must have the form as specified in the official Prometheus documentation: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#alert_relabel_configs. As alert relabel configs are appended, the user is responsible to make sure it is valid. Note that using this feature may expose the possibility to break upgrades of Prometheus. It is advised to review Prometheus release notes to ensure that no incompatible alert relabel configs are going to break Prometheus after the upgrade.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or its key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"},\"additionalScrapeConfigs\":{\"description\":\"AdditionalScrapeConfigs allows specifying a key of a Secret containing additional Prometheus scrape configurations. Scrape configurations specified are appended to the configurations generated by the Prometheus Operator. Job configurations specified must have the form as specified in the official Prometheus documentation: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config. As scrape configs are appended, the user is responsible to make sure it is valid. Note that using this feature may expose the possibility to break upgrades of Prometheus. It is advised to review Prometheus release notes to ensure that no incompatible scrape configs are going to break Prometheus after the upgrade.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or its key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"},\"affinity\":{\"description\":\"If specified, the pod's scheduling constraints.\",\"properties\":{\"nodeAffinity\":{\"description\":\"Describes node affinity scheduling rules for the pod.\",\"properties\":{\"preferredDuringSchedulingIgnoredDuringExecution\":{\"description\":\"The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding \\\"weight\\\" to the sum if the node matches the corresponding matchExpressions; the node(s) with the highest sum are the most preferred.\",\"items\":{\"description\":\"An empty preferred scheduling term matches all objects with implicit weight 0 (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op).\",\"properties\":{\"preference\":{\"description\":\"A node selector term, associated with the corresponding weight.\",\"properties\":{\"matchExpressions\":{\"description\":\"A list of node selector requirements by node's labels.\",\"items\":{\"description\":\"A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"The label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.\",\"type\":\"string\"},\"values\":{\"description\":\"An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"],\"type\":\"object\"},\"type\":\"array\"},\"matchFields\":{\"description\":\"A list of node selector requirements by node's fields.\",\"items\":{\"description\":\"A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"The label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.\",\"type\":\"string\"},\"values\":{\"description\":\"An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"],\"type\":\"object\"},\"type\":\"array\"}},\"type\":\"object\"},\"weight\":{\"description\":\"Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100.\",\"format\":\"int32\",\"type\":\"integer\"}},\"required\":[\"preference\",\"weight\"],\"type\":\"object\"},\"type\":\"array\"},\"requiredDuringSchedulingIgnoredDuringExecution\":{\"description\":\"If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to an update), the system may or may not try to eventually evict the pod from its node.\",\"properties\":{\"nodeSelectorTerms\":{\"description\":\"Required. A list of node selector terms. The terms are ORed.\",\"items\":{\"description\":\"A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm.\",\"properties\":{\"matchExpressions\":{\"description\":\"A list of node selector requirements by node's labels.\",\"items\":{\"description\":\"A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"The label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.\",\"type\":\"string\"},\"values\":{\"description\":\"An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"],\"type\":\"object\"},\"type\":\"array\"},\"matchFields\":{\"description\":\"A list of node selector requirements by node's fields.\",\"items\":{\"description\":\"A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"The label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.\",\"type\":\"string\"},\"values\":{\"description\":\"An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"],\"type\":\"object\"},\"type\":\"array\"}},\"type\":\"object\"},\"type\":\"array\"}},\"required\":[\"nodeSelectorTerms\"],\"type\":\"object\"}},\"type\":\"object\"},\"podAffinity\":{\"description\":\"Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)).\",\"properties\":{\"preferredDuringSchedulingIgnoredDuringExecution\":{\"description\":\"The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding \\\"weight\\\" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred.\",\"items\":{\"description\":\"The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s)\",\"properties\":{\"podAffinityTerm\":{\"description\":\"Required. A pod affinity term, associated with the corresponding weight.\",\"properties\":{\"labelSelector\":{\"description\":\"A label query over a set of resources, in this case pods.\",\"properties\":{\"matchExpressions\":{\"description\":\"matchExpressions is a list of label selector requirements. The requirements are ANDed.\",\"items\":{\"description\":\"A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"key is the label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\",\"type\":\"string\"},\"values\":{\"description\":\"values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"],\"type\":\"object\"},\"type\":\"array\"},\"matchLabels\":{\"additionalProperties\":{\"type\":\"string\"},\"description\":\"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \\\"key\\\", the operator is \\\"In\\\", and the values array contains only \\\"value\\\". The requirements are ANDed.\",\"type\":\"object\"}},\"type\":\"object\"},\"namespaces\":{\"description\":\"namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \\\"this pod's namespace\\\"\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"topologyKey\":{\"description\":\"This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.\",\"type\":\"string\"}},\"required\":[\"topologyKey\"],\"type\":\"object\"},\"weight\":{\"description\":\"weight associated with matching the corresponding podAffinityTerm, in the range 1-100.\",\"format\":\"int32\",\"type\":\"integer\"}},\"required\":[\"podAffinityTerm\",\"weight\"],\"type\":\"object\"},\"type\":\"array\"},\"requiredDuringSchedulingIgnoredDuringExecution\":{\"description\":\"If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied.\",\"items\":{\"description\":\"Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key \\u003ctopologyKey\\u003e matches that of any node on which a pod of the set of pods is running\",\"properties\":{\"labelSelector\":{\"description\":\"A label query over a set of resources, in this case pods.\",\"properties\":{\"matchExpressions\":{\"description\":\"matchExpressions is a list of label selector requirements. The requirements are ANDed.\",\"items\":{\"description\":\"A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"key is the label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\",\"type\":\"string\"},\"values\":{\"description\":\"values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"],\"type\":\"object\"},\"type\":\"array\"},\"matchLabels\":{\"additionalProperties\":{\"type\":\"string\"},\"description\":\"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \\\"key\\\", the operator is \\\"In\\\", and the values array contains only \\\"value\\\". The requirements are ANDed.\",\"type\":\"object\"}},\"type\":\"object\"},\"namespaces\":{\"description\":\"namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \\\"this pod's namespace\\\"\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"topologyKey\":{\"description\":\"This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.\",\"type\":\"string\"}},\"required\":[\"topologyKey\"],\"type\":\"object\"},\"type\":\"array\"}},\"type\":\"object\"},\"podAntiAffinity\":{\"description\":\"Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)).\",\"properties\":{\"preferredDuringSchedulingIgnoredDuringExecution\":{\"description\":\"The scheduler will prefer to schedule pods to nodes that satisfy the anti-affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding \\\"weight\\\" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred.\",\"items\":{\"description\":\"The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s)\",\"properties\":{\"podAffinityTerm\":{\"description\":\"Required. A pod affinity term, associated with the corresponding weight.\",\"properties\":{\"labelSelector\":{\"description\":\"A label query over a set of resources, in this case pods.\",\"properties\":{\"matchExpressions\":{\"description\":\"matchExpressions is a list of label selector requirements. The requirements are ANDed.\",\"items\":{\"description\":\"A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"key is the label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\",\"type\":\"string\"},\"values\":{\"description\":\"values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"],\"type\":\"object\"},\"type\":\"array\"},\"matchLabels\":{\"additionalProperties\":{\"type\":\"string\"},\"description\":\"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \\\"key\\\", the operator is \\\"In\\\", and the values array contains only \\\"value\\\". The requirements are ANDed.\",\"type\":\"object\"}},\"type\":\"object\"},\"namespaces\":{\"description\":\"namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \\\"this pod's namespace\\\"\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"topologyKey\":{\"description\":\"This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.\",\"type\":\"string\"}},\"required\":[\"topologyKey\"],\"type\":\"object\"},\"weight\":{\"description\":\"weight associated with matching the corresponding podAffinityTerm, in the range 1-100.\",\"format\":\"int32\",\"type\":\"integer\"}},\"required\":[\"podAffinityTerm\",\"weight\"],\"type\":\"object\"},\"type\":\"array\"},\"requiredDuringSchedulingIgnoredDuringExecution\":{\"description\":\"If the anti-affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the anti-affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied.\",\"items\":{\"description\":\"Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key \\u003ctopologyKey\\u003e matches that of any node on which a pod of the set of pods is running\",\"properties\":{\"labelSelector\":{\"description\":\"A label query over a set of resources, in this case pods.\",\"properties\":{\"matchExpressions\":{\"description\":\"matchExpressions is a list of label selector requirements. The requirements are ANDed.\",\"items\":{\"description\":\"A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"key is the label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\",\"type\":\"string\"},\"values\":{\"description\":\"values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"],\"type\":\"object\"},\"type\":\"array\"},\"matchLabels\":{\"additionalProperties\":{\"type\":\"string\"},\"description\":\"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \\\"key\\\", the operator is \\\"In\\\", and the values array contains only \\\"value\\\". The requirements are ANDed.\",\"type\":\"object\"}},\"type\":\"object\"},\"namespaces\":{\"description\":\"namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \\\"this pod's namespace\\\"\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"topologyKey\":{\"description\":\"This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.\",\"type\":\"string\"}},\"required\":[\"topologyKey\"],\"type\":\"object\"},\"type\":\"array\"}},\"type\":\"object\"}},\"type\":\"object\"},\"alerting\":{\"description\":\"Define details regarding alerting.\",\"properties\":{\"alertmanagers\":{\"description\":\"AlertmanagerEndpoints Prometheus should fire alerts against.\",\"items\":{\"description\":\"AlertmanagerEndpoints defines a selection of a single Endpoints object containing alertmanager IPs to fire alerts against.\",\"properties\":{\"apiVersion\":{\"description\":\"Version of the Alertmanager API that Prometheus uses to send alerts. It can be \\\"v1\\\" or \\\"v2\\\".\",\"type\":\"string\"},\"bearerTokenFile\":{\"description\":\"BearerTokenFile to read from filesystem to use when authenticating to Alertmanager.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of Endpoints object in Namespace.\",\"type\":\"string\"},\"namespace\":{\"description\":\"Namespace of Endpoints object.\",\"type\":\"string\"},\"pathPrefix\":{\"description\":\"Prefix for the HTTP path alerts are pushed to.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"integer\"},{\"type\":\"string\"}],\"description\":\"Port the Alertmanager API is exposed on.\",\"x-kubernetes-int-or-string\":true},\"scheme\":{\"description\":\"Scheme to use when firing alerts.\",\"type\":\"string\"},\"tlsConfig\":{\"description\":\"TLS Config to use for alertmanager connection.\",\"properties\":{\"ca\":{\"description\":\"Struct containing the CA cert to use for the targets.\",\"properties\":{\"configMap\":{\"description\":\"ConfigMap containing data to use for the targets.\",\"properties\":{\"key\":{\"description\":\"The key to select.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the ConfigMap or its key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"},\"secret\":{\"description\":\"Secret containing data to use for the targets.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or its key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"}},\"type\":\"object\"},\"caFile\":{\"description\":\"Path to the CA cert in the Prometheus container to use for the targets.\",\"type\":\"string\"},\"cert\":{\"description\":\"Struct containing the client cert file for the targets.\",\"properties\":{\"configMap\":{\"description\":\"ConfigMap containing data to use for the targets.\",\"properties\":{\"key\":{\"description\":\"The key to select.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the ConfigMap or its key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"},\"secret\":{\"description\":\"Secret containing data to use for the targets.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or its key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"}},\"type\":\"object\"},\"certFile\":{\"description\":\"Path to the client cert file in the Prometheus container for the targets.\",\"type\":\"string\"},\"insecureSkipVerify\":{\"description\":\"Disable target certificate validation.\",\"type\":\"boolean\"},\"keyFile\":{\"description\":\"Path to the client key file in the Prometheus container for the targets.\",\"type\":\"string\"},\"keySecret\":{\"description\":\"Secret containing the client key file for the targets.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or its key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"},\"serverName\":{\"description\":\"Used to verify the hostname for the targets.\",\"type\":\"string\"}},\"type\":\"object\"}},\"required\":[\"name\",\"namespace\",\"port\"],\"type\":\"object\"},\"type\":\"array\"}},\"required\":[\"alertmanagers\"],\"type\":\"object\"},\"apiserverConfig\":{\"description\":\"APIServerConfig allows specifying a host and auth methods to access apiserver. If left empty, Prometheus is assumed to run inside of the cluster and will discover API servers automatically and use the pod's CA certificate and bearer token file at /var/run/secrets/kubernetes.io/serviceaccount/.\",\"properties\":{\"basicAuth\":{\"description\":\"BasicAuth allow an endpoint to authenticate over basic authentication\",\"properties\":{\"password\":{\"description\":\"The secret in the service monitor namespace that contains the password for authentication.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or its key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"},\"username\":{\"description\":\"The secret in the service monitor namespace that contains the username for authentication.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or its key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"}},\"type\":\"object\"},\"bearerToken\":{\"description\":\"Bearer token for accessing apiserver.\",\"type\":\"string\"},\"bearerTokenFile\":{\"description\":\"File to read bearer token for accessing apiserver.\",\"type\":\"string\"},\"host\":{\"description\":\"Host of apiserver. A valid string consisting of a hostname or IP followed by an optional port number\",\"type\":\"string\"},\"tlsConfig\":{\"description\":\"TLS Config to use for accessing apiserver.\",\"properties\":{\"ca\":{\"description\":\"Struct containing the CA cert to use for the targets.\",\"properties\":{\"configMap\":{\"description\":\"ConfigMap containing data to use for the targets.\",\"properties\":{\"key\":{\"description\":\"The key to select.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the ConfigMap or its key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"},\"secret\":{\"description\":\"Secret containing data to use for the targets.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or its key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"}},\"type\":\"object\"},\"caFile\":{\"description\":\"Path to the CA cert in the Prometheus container to use for the targets.\",\"type\":\"string\"},\"cert\":{\"description\":\"Struct containing the client cert file for the targets.\",\"properties\":{\"configMap\":{\"description\":\"ConfigMap containing data to use for the targets.\",\"properties\":{\"key\":{\"description\":\"The key to select.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the ConfigMap or its key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"},\"secret\":{\"description\":\"Secret containing data to use for the targets.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or its key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"}},\"type\":\"object\"},\"certFile\":{\"description\":\"Path to the client cert file in the Prometheus container for the targets.\",\"type\":\"string\"},\"insecureSkipVerify\":{\"description\":\"Disable target certificate validation.\",\"type\":\"boolean\"},\"keyFile\":{\"description\":\"Path to the client key file in the Prometheus container for the targets.\",\"type\":\"string\"},\"keySecret\":{\"description\":\"Secret containing the client key file for the targets.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or its key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"},\"serverName\":{\"description\":\"Used to verify the hostname for the targets.\",\"type\":\"string\"}},\"type\":\"object\"}},\"required\":[\"host\"],\"type\":\"object\"},\"arbitraryFSAccessThroughSMs\":{\"description\":\"ArbitraryFSAccessThroughSMs configures whether configuration based on a service monitor can access arbitrary files on the file system of the Prometheus container e.g. bearer token files.\",\"properties\":{\"deny\":{\"type\":\"boolean\"}},\"type\":\"object\"},\"baseImage\":{\"description\":\"Base image to use for a Prometheus deployment.\",\"type\":\"string\"},\"configMaps\":{\"description\":\"ConfigMaps is a list of ConfigMaps in the same namespace as the Prometheus object, which shall be mounted into the Prometheus Pods. The ConfigMaps are mounted into /etc/prometheus/configmaps/\\u003cconfigmap-name\\u003e.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"containers\":{\"description\":\"Containers allows injecting additional containers or modifying operator generated containers. This can be used to allow adding an authentication proxy to a Prometheus pod or to change the behavior of an operator generated container. Containers described here modify an operator generated container if they share the same name and modifications are done via a strategic merge patch. The current container names are: `prometheus`, `prometheus-config-reloader`, `rules-configmap-reloader`, and `thanos-sidecar`. Overriding containers is entirely outside the scope of what the maintainers will support and by doing so, you accept that this behaviour may break at any time without notice.\",\"items\":{\"description\":\"A single application container that you want to run within a pod.\",\"properties\":{\"args\":{\"description\":\"Arguments to the entrypoint. The docker image's CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"command\":{\"description\":\"Entrypoint array. Not executed within a shell. The docker image's ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"env\":{\"description\":\"List of environment variables to set in the container. Cannot be updated.\",\"items\":{\"description\":\"EnvVar represents an environment variable present in a Container.\",\"properties\":{\"name\":{\"description\":\"Name of the environment variable. Must be a C_IDENTIFIER.\",\"type\":\"string\"},\"value\":{\"description\":\"Variable references $(VAR_NAME) are expanded using the previous defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Defaults to \\\"\\\".\",\"type\":\"string\"},\"valueFrom\":{\"description\":\"Source for the environment variable's value. Cannot be used if value is not empty.\",\"properties\":{\"configMapKeyRef\":{\"description\":\"Selects a key of a ConfigMap.\",\"properties\":{\"key\":{\"description\":\"The key to select.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the ConfigMap or its key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"},\"fieldRef\":{\"description\":\"Selects a field of the pod: supports metadata.name, metadata.namespace, metadata.labels, metadata.annotations, spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.\",\"properties\":{\"apiVersion\":{\"description\":\"Version of the schema the FieldPath is written in terms of, defaults to \\\"v1\\\".\",\"type\":\"string\"},\"fieldPath\":{\"description\":\"Path of the field to select in the specified API version.\",\"type\":\"string\"}},\"required\":[\"fieldPath\"],\"type\":\"object\"},\"resourceFieldRef\":{\"description\":\"Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.\",\"properties\":{\"containerName\":{\"description\":\"Container name: required for volumes, optional for env vars\",\"type\":\"string\"},\"divisor\":{\"description\":\"Specifies the output format of the exposed resources, defaults to \\\"1\\\"\",\"type\":\"string\"},\"resource\":{\"description\":\"Required: resource to select\",\"type\":\"string\"}},\"required\":[\"resource\"],\"type\":\"object\"},\"secretKeyRef\":{\"description\":\"Selects a key of a secret in the pod's namespace\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or its key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"}},\"type\":\"object\"}},\"required\":[\"name\"],\"type\":\"object\"},\"type\":\"array\"},\"envFrom\":{\"description\":\"List of sources to populate environment variables in the container. The keys defined within a source must be a C_IDENTIFIER. All invalid keys will be reported as an event when the container is starting. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated.\",\"items\":{\"description\":\"EnvFromSource represents the source of a set of ConfigMaps\",\"properties\":{\"configMapRef\":{\"description\":\"The ConfigMap to select from\",\"properties\":{\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the ConfigMap must be defined\",\"type\":\"boolean\"}},\"type\":\"object\"},\"prefix\":{\"description\":\"An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER.\",\"type\":\"string\"},\"secretRef\":{\"description\":\"The Secret to select from\",\"properties\":{\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret must be defined\",\"type\":\"boolean\"}},\"type\":\"object\"}},\"type\":\"object\"},\"type\":\"array\"},\"image\":{\"description\":\"Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images This field is optional to allow higher level config management to default or override container images in workload controllers like Deployments and StatefulSets.\",\"type\":\"string\"},\"imagePullPolicy\":{\"description\":\"Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images\",\"type\":\"string\"},\"lifecycle\":{\"description\":\"Actions that the management system should take in response to container lifecycle events. Cannot be updated.\",\"properties\":{\"postStart\":{\"description\":\"PostStart is called immediately after a container is created. If the handler fails, the container is terminated and restarted according to its restart policy. Other management of the container blocks until the hook completes. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks\",\"properties\":{\"exec\":{\"description\":\"One and only one of the following should be specified. Exec specifies the action to take.\",\"properties\":{\"command\":{\"description\":\"Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"type\":\"object\"},\"httpGet\":{\"description\":\"HTTPGet specifies the http request to perform.\",\"properties\":{\"host\":{\"description\":\"Host name to connect to, defaults to the pod IP. You probably want to set \\\"Host\\\" in httpHeaders instead.\",\"type\":\"string\"},\"httpHeaders\":{\"description\":\"Custom headers to set in the request. HTTP allows repeated headers.\",\"items\":{\"description\":\"HTTPHeader describes a custom header to be used in HTTP probes\",\"properties\":{\"name\":{\"description\":\"The header field name\",\"type\":\"string\"},\"value\":{\"description\":\"The header field value\",\"type\":\"string\"}},\"required\":[\"name\",\"value\"],\"type\":\"object\"},\"type\":\"array\"},\"path\":{\"description\":\"Path to access on the HTTP server.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"integer\"},{\"type\":\"string\"}],\"description\":\"Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\",\"x-kubernetes-int-or-string\":true},\"scheme\":{\"description\":\"Scheme to use for connecting to the host. Defaults to HTTP.\",\"type\":\"string\"}},\"required\":[\"port\"],\"type\":\"object\"},\"tcpSocket\":{\"description\":\"TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook\",\"properties\":{\"host\":{\"description\":\"Optional: Host name to connect to, defaults to the pod IP.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"integer\"},{\"type\":\"string\"}],\"description\":\"Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\",\"x-kubernetes-int-or-string\":true}},\"required\":[\"port\"],\"type\":\"object\"}},\"type\":\"object\"},\"preStop\":{\"description\":\"PreStop is called immediately before a container is terminated due to an API request or management event such as liveness/startup probe failure, preemption, resource contention, etc. The handler is not called if the container crashes or exits. The reason for termination is passed to the handler. The Pod's termination grace period countdown begins before the PreStop hooked is executed. Regardless of the outcome of the handler, the container will eventually terminate within the Pod's termination grace period. Other management of the container blocks until the hook completes or until the termination grace period is reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks\",\"properties\":{\"exec\":{\"description\":\"One and only one of the following should be specified. Exec specifies the action to take.\",\"properties\":{\"command\":{\"description\":\"Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"type\":\"object\"},\"httpGet\":{\"description\":\"HTTPGet specifies the http request to perform.\",\"properties\":{\"host\":{\"description\":\"Host name to connect to, defaults to the pod IP. You probably want to set \\\"Host\\\" in httpHeaders instead.\",\"type\":\"string\"},\"httpHeaders\":{\"description\":\"Custom headers to set in the request. HTTP allows repeated headers.\",\"items\":{\"description\":\"HTTPHeader describes a custom header to be used in HTTP probes\",\"properties\":{\"name\":{\"description\":\"The header field name\",\"type\":\"string\"},\"value\":{\"description\":\"The header field value\",\"type\":\"string\"}},\"required\":[\"name\",\"value\"],\"type\":\"object\"},\"type\":\"array\"},\"path\":{\"description\":\"Path to access on the HTTP server.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"integer\"},{\"type\":\"string\"}],\"description\":\"Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\",\"x-kubernetes-int-or-string\":true},\"scheme\":{\"description\":\"Scheme to use for connecting to the host. Defaults to HTTP.\",\"type\":\"string\"}},\"required\":[\"port\"],\"type\":\"object\"},\"tcpSocket\":{\"description\":\"TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook\",\"properties\":{\"host\":{\"description\":\"Optional: Host name to connect to, defaults to the pod IP.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"integer\"},{\"type\":\"string\"}],\"description\":\"Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\",\"x-kubernetes-int-or-string\":true}},\"required\":[\"port\"],\"type\":\"object\"}},\"type\":\"object\"}},\"type\":\"object\"},\"livenessProbe\":{\"description\":\"Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\",\"properties\":{\"exec\":{\"description\":\"One and only one of the following should be specified. Exec specifies the action to take.\",\"properties\":{\"command\":{\"description\":\"Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"type\":\"object\"},\"failureThreshold\":{\"description\":\"Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.\",\"format\":\"int32\",\"type\":\"integer\"},\"httpGet\":{\"description\":\"HTTPGet specifies the http request to perform.\",\"properties\":{\"host\":{\"description\":\"Host name to connect to, defaults to the pod IP. You probably want to set \\\"Host\\\" in httpHeaders instead.\",\"type\":\"string\"},\"httpHeaders\":{\"description\":\"Custom headers to set in the request. HTTP allows repeated headers.\",\"items\":{\"description\":\"HTTPHeader describes a custom header to be used in HTTP probes\",\"properties\":{\"name\":{\"description\":\"The header field name\",\"type\":\"string\"},\"value\":{\"description\":\"The header field value\",\"type\":\"string\"}},\"required\":[\"name\",\"value\"],\"type\":\"object\"},\"type\":\"array\"},\"path\":{\"description\":\"Path to access on the HTTP server.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"integer\"},{\"type\":\"string\"}],\"description\":\"Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\",\"x-kubernetes-int-or-string\":true},\"scheme\":{\"description\":\"Scheme to use for connecting to the host. Defaults to HTTP.\",\"type\":\"string\"}},\"required\":[\"port\"],\"type\":\"object\"},\"initialDelaySeconds\":{\"description\":\"Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\",\"format\":\"int32\",\"type\":\"integer\"},\"periodSeconds\":{\"description\":\"How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.\",\"format\":\"int32\",\"type\":\"integer\"},\"successThreshold\":{\"description\":\"Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.\",\"format\":\"int32\",\"type\":\"integer\"},\"tcpSocket\":{\"description\":\"TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook\",\"properties\":{\"host\":{\"description\":\"Optional: Host name to connect to, defaults to the pod IP.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"integer\"},{\"type\":\"string\"}],\"description\":\"Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\",\"x-kubernetes-int-or-string\":true}},\"required\":[\"port\"],\"type\":\"object\"},\"timeoutSeconds\":{\"description\":\"Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\",\"format\":\"int32\",\"type\":\"integer\"}},\"type\":\"object\"},\"name\":{\"description\":\"Name of the container specified as a DNS_LABEL. Each container in a pod must have a unique name (DNS_LABEL). Cannot be updated.\",\"type\":\"string\"},\"ports\":{\"description\":\"List of ports to expose from the container. Exposing a port here gives the system additional information about the network connections a container uses, but is primarily informational. Not specifying a port here DOES NOT prevent that port from being exposed. Any port which is listening on the default \\\"0.0.0.0\\\" address inside a container will be accessible from the network. Cannot be updated.\",\"items\":{\"description\":\"ContainerPort represents a network port in a single container.\",\"properties\":{\"containerPort\":{\"description\":\"Number of port to expose on the pod's IP address. This must be a valid port number, 0 \\u003c x \\u003c 65536.\",\"format\":\"int32\",\"type\":\"integer\"},\"hostIP\":{\"description\":\"What host IP to bind the external port to.\",\"type\":\"string\"},\"hostPort\":{\"description\":\"Number of port to expose on the host. If specified, this must be a valid port number, 0 \\u003c x \\u003c 65536. If HostNetwork is specified, this must match ContainerPort. Most containers do not need this.\",\"format\":\"int32\",\"type\":\"integer\"},\"name\":{\"description\":\"If specified, this must be an IANA_SVC_NAME and unique within the pod. Each named port in a pod must have a unique name. Name for the port that can be referred to by services.\",\"type\":\"string\"},\"protocol\":{\"description\":\"Protocol for port. Must be UDP, TCP, or SCTP. Defaults to \\\"TCP\\\".\",\"type\":\"string\"}},\"required\":[\"containerPort\"],\"type\":\"object\"},\"type\":\"array\"},\"readinessProbe\":{\"description\":\"Periodic probe of container service readiness. Container will be removed from service endpoints if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\",\"properties\":{\"exec\":{\"description\":\"One and only one of the following should be specified. Exec specifies the action to take.\",\"properties\":{\"command\":{\"description\":\"Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"type\":\"object\"},\"failureThreshold\":{\"description\":\"Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.\",\"format\":\"int32\",\"type\":\"integer\"},\"httpGet\":{\"description\":\"HTTPGet specifies the http request to perform.\",\"properties\":{\"host\":{\"description\":\"Host name to connect to, defaults to the pod IP. You probably want to set \\\"Host\\\" in httpHeaders instead.\",\"type\":\"string\"},\"httpHeaders\":{\"description\":\"Custom headers to set in the request. HTTP allows repeated headers.\",\"items\":{\"description\":\"HTTPHeader describes a custom header to be used in HTTP probes\",\"properties\":{\"name\":{\"description\":\"The header field name\",\"type\":\"string\"},\"value\":{\"description\":\"The header field value\",\"type\":\"string\"}},\"required\":[\"name\",\"value\"],\"type\":\"object\"},\"type\":\"array\"},\"path\":{\"description\":\"Path to access on the HTTP server.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"integer\"},{\"type\":\"string\"}],\"description\":\"Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\",\"x-kubernetes-int-or-string\":true},\"scheme\":{\"description\":\"Scheme to use for connecting to the host. Defaults to HTTP.\",\"type\":\"string\"}},\"required\":[\"port\"],\"type\":\"object\"},\"initialDelaySeconds\":{\"description\":\"Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\",\"format\":\"int32\",\"type\":\"integer\"},\"periodSeconds\":{\"description\":\"How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.\",\"format\":\"int32\",\"type\":\"integer\"},\"successThreshold\":{\"description\":\"Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.\",\"format\":\"int32\",\"type\":\"integer\"},\"tcpSocket\":{\"description\":\"TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook\",\"properties\":{\"host\":{\"description\":\"Optional: Host name to connect to, defaults to the pod IP.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"integer\"},{\"type\":\"string\"}],\"description\":\"Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\",\"x-kubernetes-int-or-string\":true}},\"required\":[\"port\"],\"type\":\"object\"},\"timeoutSeconds\":{\"description\":\"Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\",\"format\":\"int32\",\"type\":\"integer\"}},\"type\":\"object\"},\"resources\":{\"description\":\"Compute Resources required by this container. Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/\",\"properties\":{\"limits\":{\"additionalProperties\":{\"type\":\"string\"},\"description\":\"Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/\",\"type\":\"object\"},\"requests\":{\"additionalProperties\":{\"type\":\"string\"},\"description\":\"Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/\",\"type\":\"object\"}},\"type\":\"object\"},\"securityContext\":{\"description\":\"Security options the pod should run with. More info: https://kubernetes.io/docs/concepts/policy/security-context/ More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/\",\"properties\":{\"allowPrivilegeEscalation\":{\"description\":\"AllowPrivilegeEscalation controls whether a process can gain more privileges than its parent process. This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged 2) has CAP_SYS_ADMIN\",\"type\":\"boolean\"},\"capabilities\":{\"description\":\"The capabilities to add/drop when running containers. Defaults to the default set of capabilities granted by the container runtime.\",\"properties\":{\"add\":{\"description\":\"Added capabilities\",\"items\":{\"description\":\"Capability represent POSIX capabilities type\",\"type\":\"string\"},\"type\":\"array\"},\"drop\":{\"description\":\"Removed capabilities\",\"items\":{\"description\":\"Capability represent POSIX capabilities type\",\"type\":\"string\"},\"type\":\"array\"}},\"type\":\"object\"},\"privileged\":{\"description\":\"Run container in privileged mode. Processes in privileged containers are essentially equivalent to root on the host. Defaults to false.\",\"type\":\"boolean\"},\"procMount\":{\"description\":\"procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled.\",\"type\":\"string\"},\"readOnlyRootFilesystem\":{\"description\":\"Whether this container has a read-only root filesystem. Default is false.\",\"type\":\"boolean\"},\"runAsGroup\":{\"description\":\"The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\",\"format\":\"int64\",\"type\":\"integer\"},\"runAsNonRoot\":{\"description\":\"Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\",\"type\":\"boolean\"},\"runAsUser\":{\"description\":\"The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\",\"format\":\"int64\",\"type\":\"integer\"},\"seLinuxOptions\":{\"description\":\"The SELinux context to be applied to the container. If unspecified, the container runtime will allocate a random SELinux context for each container. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\",\"properties\":{\"level\":{\"description\":\"Level is SELinux level label that applies to the container.\",\"type\":\"string\"},\"role\":{\"description\":\"Role is a SELinux role label that applies to the container.\",\"type\":\"string\"},\"type\":{\"description\":\"Type is a SELinux type label that applies to the container.\",\"type\":\"string\"},\"user\":{\"description\":\"User is a SELinux user label that applies to the container.\",\"type\":\"string\"}},\"type\":\"object\"},\"windowsOptions\":{\"description\":\"The Windows specific settings applied to all containers. If unspecified, the options from the PodSecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\",\"properties\":{\"gmsaCredentialSpec\":{\"description\":\"GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field.\",\"type\":\"string\"},\"gmsaCredentialSpecName\":{\"description\":\"GMSACredentialSpecName is the name of the GMSA credential spec to use.\",\"type\":\"string\"},\"runAsUserName\":{\"description\":\"The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\",\"type\":\"string\"}},\"type\":\"object\"}},\"type\":\"object\"},\"startupProbe\":{\"description\":\"StartupProbe indicates that the Pod has successfully initialized. If specified, no other probes are executed until this completes successfully. If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, when it might take a long time to load data or warm a cache, than during steady-state operation. This cannot be updated. This is a beta feature enabled by the StartupProbe feature flag. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\",\"properties\":{\"exec\":{\"description\":\"One and only one of the following should be specified. Exec specifies the action to take.\",\"properties\":{\"command\":{\"description\":\"Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"type\":\"object\"},\"failureThreshold\":{\"description\":\"Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.\",\"format\":\"int32\",\"type\":\"integer\"},\"httpGet\":{\"description\":\"HTTPGet specifies the http request to perform.\",\"properties\":{\"host\":{\"description\":\"Host name to connect to, defaults to the pod IP. You probably want to set \\\"Host\\\" in httpHeaders instead.\",\"type\":\"string\"},\"httpHeaders\":{\"description\":\"Custom headers to set in the request. HTTP allows repeated headers.\",\"items\":{\"description\":\"HTTPHeader describes a custom header to be used in HTTP probes\",\"properties\":{\"name\":{\"description\":\"The header field name\",\"type\":\"string\"},\"value\":{\"description\":\"The header field value\",\"type\":\"string\"}},\"required\":[\"name\",\"value\"],\"type\":\"object\"},\"type\":\"array\"},\"path\":{\"description\":\"Path to access on the HTTP server.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"integer\"},{\"type\":\"string\"}],\"description\":\"Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\",\"x-kubernetes-int-or-string\":true},\"scheme\":{\"description\":\"Scheme to use for connecting to the host. Defaults to HTTP.\",\"type\":\"string\"}},\"required\":[\"port\"],\"type\":\"object\"},\"initialDelaySeconds\":{\"description\":\"Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\",\"format\":\"int32\",\"type\":\"integer\"},\"periodSeconds\":{\"description\":\"How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.\",\"format\":\"int32\",\"type\":\"integer\"},\"successThreshold\":{\"description\":\"Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.\",\"format\":\"int32\",\"type\":\"integer\"},\"tcpSocket\":{\"description\":\"TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook\",\"properties\":{\"host\":{\"description\":\"Optional: Host name to connect to, defaults to the pod IP.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"integer\"},{\"type\":\"string\"}],\"description\":\"Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\",\"x-kubernetes-int-or-string\":true}},\"required\":[\"port\"],\"type\":\"object\"},\"timeoutSeconds\":{\"description\":\"Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\",\"format\":\"int32\",\"type\":\"integer\"}},\"type\":\"object\"},\"stdin\":{\"description\":\"Whether this container should allocate a buffer for stdin in the container runtime. If this is not set, reads from stdin in the container will always result in EOF. Default is false.\",\"type\":\"boolean\"},\"stdinOnce\":{\"description\":\"Whether the container runtime should close the stdin channel after it has been opened by a single attach. When stdin is true the stdin stream will remain open across multiple attach sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the first client attaches to stdin, and then remains open and accepts data until the client disconnects, at which time stdin is closed and remains closed until the container is restarted. If this flag is false, a container processes that reads from stdin will never receive an EOF. Default is false\",\"type\":\"boolean\"},\"terminationMessagePath\":{\"description\":\"Optional: Path at which the file to which the container's termination message will be written is mounted into the container's filesystem. Message written is intended to be brief final status, such as an assertion failure message. Will be truncated by the node if greater than 4096 bytes. The total message length across all containers will be limited to 12kb. Defaults to /dev/termination-log. Cannot be updated.\",\"type\":\"string\"},\"terminationMessagePolicy\":{\"description\":\"Indicate how the termination message should be populated. File will use the contents of terminationMessagePath to populate the container status message on both success and failure. FallbackToLogsOnError will use the last chunk of container log output if the termination message file is empty and the container exited with an error. The log output is limited to 2048 bytes or 80 lines, whichever is smaller. Defaults to File. Cannot be updated.\",\"type\":\"string\"},\"tty\":{\"description\":\"Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. Default is false.\",\"type\":\"boolean\"},\"volumeDevices\":{\"description\":\"volumeDevices is the list of block devices to be used by the container.\",\"items\":{\"description\":\"volumeDevice describes a mapping of a raw block device within a container.\",\"properties\":{\"devicePath\":{\"description\":\"devicePath is the path inside of the container that the device will be mapped to.\",\"type\":\"string\"},\"name\":{\"description\":\"name must match the name of a persistentVolumeClaim in the pod\",\"type\":\"string\"}},\"required\":[\"devicePath\",\"name\"],\"type\":\"object\"},\"type\":\"array\"},\"volumeMounts\":{\"description\":\"Pod volumes to mount into the container's filesystem. Cannot be updated.\",\"items\":{\"description\":\"VolumeMount describes a mounting of a Volume within a container.\",\"properties\":{\"mountPath\":{\"description\":\"Path within the container at which the volume should be mounted. Must not contain ':'.\",\"type\":\"string\"},\"mountPropagation\":{\"description\":\"mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10.\",\"type\":\"string\"},\"name\":{\"description\":\"This must match the Name of a Volume.\",\"type\":\"string\"},\"readOnly\":{\"description\":\"Mounted read-only if true, read-write otherwise (false or unspecified). Defaults to false.\",\"type\":\"boolean\"},\"subPath\":{\"description\":\"Path within the volume from which the container's volume should be mounted. Defaults to \\\"\\\" (volume's root).\",\"type\":\"string\"},\"subPathExpr\":{\"description\":\"Expanded path within the volume from which the container's volume should be mounted. Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. Defaults to \\\"\\\" (volume's root). SubPathExpr and SubPath are mutually exclusive.\",\"type\":\"string\"}},\"required\":[\"mountPath\",\"name\"],\"type\":\"object\"},\"type\":\"array\"},\"workingDir\":{\"description\":\"Container's working directory. If not specified, the container runtime's default will be used, which might be configured in the container image. Cannot be updated.\",\"type\":\"string\"}},\"required\":[\"name\"],\"type\":\"object\"},\"type\":\"array\"},\"disableCompaction\":{\"description\":\"Disable prometheus compaction.\",\"type\":\"boolean\"},\"enableAdminAPI\":{\"description\":\"Enable access to prometheus web admin API. Defaults to the value of `false`. WARNING: Enabling the admin APIs enables mutating endpoints, to delete data, shutdown Prometheus, and more. Enabling this should be done with care and the user is advised to add additional authentication authorization via a proxy to ensure only clients authorized to perform these actions can do so. For more information see https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-admin-apis\",\"type\":\"boolean\"},\"enforcedNamespaceLabel\":{\"description\":\"EnforcedNamespaceLabel enforces adding a namespace label of origin for each alert and metric that is user created. The label value will always be the namespace of the object that is being created.\",\"type\":\"string\"},\"evaluationInterval\":{\"description\":\"Interval between consecutive evaluations.\",\"type\":\"string\"},\"externalLabels\":{\"additionalProperties\":{\"type\":\"string\"},\"description\":\"The labels to add to any time series or alerts when communicating with external systems (federation, remote storage, Alertmanager).\",\"type\":\"object\"},\"externalUrl\":{\"description\":\"The external URL the Prometheus instances will be available under. This is necessary to generate correct URLs. This is necessary if Prometheus is not served from root of a DNS name.\",\"type\":\"string\"},\"ignoreNamespaceSelectors\":{\"description\":\"IgnoreNamespaceSelectors if set to true will ignore NamespaceSelector settings from the podmonitor and servicemonitor configs, and they will only discover endpoints within their current namespace. Defaults to false.\",\"type\":\"boolean\"},\"image\":{\"description\":\"Image if specified has precedence over baseImage, tag and sha combinations. Specifying the version is still necessary to ensure the Prometheus Operator knows what version of Prometheus is being configured.\",\"type\":\"string\"},\"imagePullSecrets\":{\"description\":\"An optional list of references to secrets in the same namespace to use for pulling prometheus and alertmanager images from registries see http://kubernetes.io/docs/user-guide/images#specifying-imagepullsecrets-on-a-pod\",\"items\":{\"description\":\"LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace.\",\"properties\":{\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"}},\"type\":\"object\"},\"type\":\"array\"},\"initContainers\":{\"description\":\"InitContainers allows adding initContainers to the pod definition. Those can be used to e.g. fetch secrets for injection into the Prometheus configuration from external sources. Any errors during the execution of an initContainer will lead to a restart of the Pod. More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ Using initContainers for any use case other then secret fetching is entirely outside the scope of what the maintainers will support and by doing so, you accept that this behaviour may break at any time without notice.\",\"items\":{\"description\":\"A single application container that you want to run within a pod.\",\"properties\":{\"args\":{\"description\":\"Arguments to the entrypoint. The docker image's CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"command\":{\"description\":\"Entrypoint array. Not executed within a shell. The docker image's ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"env\":{\"description\":\"List of environment variables to set in the container. Cannot be updated.\",\"items\":{\"description\":\"EnvVar represents an environment variable present in a Container.\",\"properties\":{\"name\":{\"description\":\"Name of the environment variable. Must be a C_IDENTIFIER.\",\"type\":\"string\"},\"value\":{\"description\":\"Variable references $(VAR_NAME) are expanded using the previous defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Defaults to \\\"\\\".\",\"type\":\"string\"},\"valueFrom\":{\"description\":\"Source for the environment variable's value. Cannot be used if value is not empty.\",\"properties\":{\"configMapKeyRef\":{\"description\":\"Selects a key of a ConfigMap.\",\"properties\":{\"key\":{\"description\":\"The key to select.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the ConfigMap or its key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"},\"fieldRef\":{\"description\":\"Selects a field of the pod: supports metadata.name, metadata.namespace, metadata.labels, metadata.annotations, spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.\",\"properties\":{\"apiVersion\":{\"description\":\"Version of the schema the FieldPath is written in terms of, defaults to \\\"v1\\\".\",\"type\":\"string\"},\"fieldPath\":{\"description\":\"Path of the field to select in the specified API version.\",\"type\":\"string\"}},\"required\":[\"fieldPath\"],\"type\":\"object\"},\"resourceFieldRef\":{\"description\":\"Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.\",\"properties\":{\"containerName\":{\"description\":\"Container name: required for volumes, optional for env vars\",\"type\":\"string\"},\"divisor\":{\"description\":\"Specifies the output format of the exposed resources, defaults to \\\"1\\\"\",\"type\":\"string\"},\"resource\":{\"description\":\"Required: resource to select\",\"type\":\"string\"}},\"required\":[\"resource\"],\"type\":\"object\"},\"secretKeyRef\":{\"description\":\"Selects a key of a secret in the pod's namespace\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or its key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"}},\"type\":\"object\"}},\"required\":[\"name\"],\"type\":\"object\"},\"type\":\"array\"},\"envFrom\":{\"description\":\"List of sources to populate environment variables in the container. The keys defined within a source must be a C_IDENTIFIER. All invalid keys will be reported as an event when the container is starting. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated.\",\"items\":{\"description\":\"EnvFromSource represents the source of a set of ConfigMaps\",\"properties\":{\"configMapRef\":{\"description\":\"The ConfigMap to select from\",\"properties\":{\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the ConfigMap must be defined\",\"type\":\"boolean\"}},\"type\":\"object\"},\"prefix\":{\"description\":\"An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER.\",\"type\":\"string\"},\"secretRef\":{\"description\":\"The Secret to select from\",\"properties\":{\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret must be defined\",\"type\":\"boolean\"}},\"type\":\"object\"}},\"type\":\"object\"},\"type\":\"array\"},\"image\":{\"description\":\"Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images This field is optional to allow higher level config management to default or override container images in workload controllers like Deployments and StatefulSets.\",\"type\":\"string\"},\"imagePullPolicy\":{\"description\":\"Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images\",\"type\":\"string\"},\"lifecycle\":{\"description\":\"Actions that the management system should take in response to container lifecycle events. Cannot be updated.\",\"properties\":{\"postStart\":{\"description\":\"PostStart is called immediately after a container is created. If the handler fails, the container is terminated and restarted according to its restart policy. Other management of the container blocks until the hook completes. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks\",\"properties\":{\"exec\":{\"description\":\"One and only one of the following should be specified. Exec specifies the action to take.\",\"properties\":{\"command\":{\"description\":\"Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"type\":\"object\"},\"httpGet\":{\"description\":\"HTTPGet specifies the http request to perform.\",\"properties\":{\"host\":{\"description\":\"Host name to connect to, defaults to the pod IP. You probably want to set \\\"Host\\\" in httpHeaders instead.\",\"type\":\"string\"},\"httpHeaders\":{\"description\":\"Custom headers to set in the request. HTTP allows repeated headers.\",\"items\":{\"description\":\"HTTPHeader describes a custom header to be used in HTTP probes\",\"properties\":{\"name\":{\"description\":\"The header field name\",\"type\":\"string\"},\"value\":{\"description\":\"The header field value\",\"type\":\"string\"}},\"required\":[\"name\",\"value\"],\"type\":\"object\"},\"type\":\"array\"},\"path\":{\"description\":\"Path to access on the HTTP server.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"integer\"},{\"type\":\"string\"}],\"description\":\"Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\",\"x-kubernetes-int-or-string\":true},\"scheme\":{\"description\":\"Scheme to use for connecting to the host. Defaults to HTTP.\",\"type\":\"string\"}},\"required\":[\"port\"],\"type\":\"object\"},\"tcpSocket\":{\"description\":\"TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook\",\"properties\":{\"host\":{\"description\":\"Optional: Host name to connect to, defaults to the pod IP.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"integer\"},{\"type\":\"string\"}],\"description\":\"Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\",\"x-kubernetes-int-or-string\":true}},\"required\":[\"port\"],\"type\":\"object\"}},\"type\":\"object\"},\"preStop\":{\"description\":\"PreStop is called immediately before a container is terminated due to an API request or management event such as liveness/startup probe failure, preemption, resource contention, etc. The handler is not called if the container crashes or exits. The reason for termination is passed to the handler. The Pod's termination grace period countdown begins before the PreStop hooked is executed. Regardless of the outcome of the handler, the container will eventually terminate within the Pod's termination grace period. Other management of the container blocks until the hook completes or until the termination grace period is reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks\",\"properties\":{\"exec\":{\"description\":\"One and only one of the following should be specified. Exec specifies the action to take.\",\"properties\":{\"command\":{\"description\":\"Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"type\":\"object\"},\"httpGet\":{\"description\":\"HTTPGet specifies the http request to perform.\",\"properties\":{\"host\":{\"description\":\"Host name to connect to, defaults to the pod IP. You probably want to set \\\"Host\\\" in httpHeaders instead.\",\"type\":\"string\"},\"httpHeaders\":{\"description\":\"Custom headers to set in the request. HTTP allows repeated headers.\",\"items\":{\"description\":\"HTTPHeader describes a custom header to be used in HTTP probes\",\"properties\":{\"name\":{\"description\":\"The header field name\",\"type\":\"string\"},\"value\":{\"description\":\"The header field value\",\"type\":\"string\"}},\"required\":[\"name\",\"value\"],\"type\":\"object\"},\"type\":\"array\"},\"path\":{\"description\":\"Path to access on the HTTP server.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"integer\"},{\"type\":\"string\"}],\"description\":\"Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\",\"x-kubernetes-int-or-string\":true},\"scheme\":{\"description\":\"Scheme to use for connecting to the host. Defaults to HTTP.\",\"type\":\"string\"}},\"required\":[\"port\"],\"type\":\"object\"},\"tcpSocket\":{\"description\":\"TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook\",\"properties\":{\"host\":{\"description\":\"Optional: Host name to connect to, defaults to the pod IP.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"integer\"},{\"type\":\"string\"}],\"description\":\"Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\",\"x-kubernetes-int-or-string\":true}},\"required\":[\"port\"],\"type\":\"object\"}},\"type\":\"object\"}},\"type\":\"object\"},\"livenessProbe\":{\"description\":\"Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\",\"properties\":{\"exec\":{\"description\":\"One and only one of the following should be specified. Exec specifies the action to take.\",\"properties\":{\"command\":{\"description\":\"Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"type\":\"object\"},\"failureThreshold\":{\"description\":\"Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.\",\"format\":\"int32\",\"type\":\"integer\"},\"httpGet\":{\"description\":\"HTTPGet specifies the http request to perform.\",\"properties\":{\"host\":{\"description\":\"Host name to connect to, defaults to the pod IP. You probably want to set \\\"Host\\\" in httpHeaders instead.\",\"type\":\"string\"},\"httpHeaders\":{\"description\":\"Custom headers to set in the request. HTTP allows repeated headers.\",\"items\":{\"description\":\"HTTPHeader describes a custom header to be used in HTTP probes\",\"properties\":{\"name\":{\"description\":\"The header field name\",\"type\":\"string\"},\"value\":{\"description\":\"The header field value\",\"type\":\"string\"}},\"required\":[\"name\",\"value\"],\"type\":\"object\"},\"type\":\"array\"},\"path\":{\"description\":\"Path to access on the HTTP server.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"integer\"},{\"type\":\"string\"}],\"description\":\"Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\",\"x-kubernetes-int-or-string\":true},\"scheme\":{\"description\":\"Scheme to use for connecting to the host. Defaults to HTTP.\",\"type\":\"string\"}},\"required\":[\"port\"],\"type\":\"object\"},\"initialDelaySeconds\":{\"description\":\"Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\",\"format\":\"int32\",\"type\":\"integer\"},\"periodSeconds\":{\"description\":\"How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.\",\"format\":\"int32\",\"type\":\"integer\"},\"successThreshold\":{\"description\":\"Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.\",\"format\":\"int32\",\"type\":\"integer\"},\"tcpSocket\":{\"description\":\"TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook\",\"properties\":{\"host\":{\"description\":\"Optional: Host name to connect to, defaults to the pod IP.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"integer\"},{\"type\":\"string\"}],\"description\":\"Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\",\"x-kubernetes-int-or-string\":true}},\"required\":[\"port\"],\"type\":\"object\"},\"timeoutSeconds\":{\"description\":\"Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\",\"format\":\"int32\",\"type\":\"integer\"}},\"type\":\"object\"},\"name\":{\"description\":\"Name of the container specified as a DNS_LABEL. Each container in a pod must have a unique name (DNS_LABEL). Cannot be updated.\",\"type\":\"string\"},\"ports\":{\"description\":\"List of ports to expose from the container. Exposing a port here gives the system additional information about the network connections a container uses, but is primarily informational. Not specifying a port here DOES NOT prevent that port from being exposed. Any port which is listening on the default \\\"0.0.0.0\\\" address inside a container will be accessible from the network. Cannot be updated.\",\"items\":{\"description\":\"ContainerPort represents a network port in a single container.\",\"properties\":{\"containerPort\":{\"description\":\"Number of port to expose on the pod's IP address. This must be a valid port number, 0 \\u003c x \\u003c 65536.\",\"format\":\"int32\",\"type\":\"integer\"},\"hostIP\":{\"description\":\"What host IP to bind the external port to.\",\"type\":\"string\"},\"hostPort\":{\"description\":\"Number of port to expose on the host. If specified, this must be a valid port number, 0 \\u003c x \\u003c 65536. If HostNetwork is specified, this must match ContainerPort. Most containers do not need this.\",\"format\":\"int32\",\"type\":\"integer\"},\"name\":{\"description\":\"If specified, this must be an IANA_SVC_NAME and unique within the pod. Each named port in a pod must have a unique name. Name for the port that can be referred to by services.\",\"type\":\"string\"},\"protocol\":{\"description\":\"Protocol for port. Must be UDP, TCP, or SCTP. Defaults to \\\"TCP\\\".\",\"type\":\"string\"}},\"required\":[\"containerPort\"],\"type\":\"object\"},\"type\":\"array\"},\"readinessProbe\":{\"description\":\"Periodic probe of container service readiness. Container will be removed from service endpoints if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\",\"properties\":{\"exec\":{\"description\":\"One and only one of the following should be specified. Exec specifies the action to take.\",\"properties\":{\"command\":{\"description\":\"Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"type\":\"object\"},\"failureThreshold\":{\"description\":\"Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.\",\"format\":\"int32\",\"type\":\"integer\"},\"httpGet\":{\"description\":\"HTTPGet specifies the http request to perform.\",\"properties\":{\"host\":{\"description\":\"Host name to connect to, defaults to the pod IP. You probably want to set \\\"Host\\\" in httpHeaders instead.\",\"type\":\"string\"},\"httpHeaders\":{\"description\":\"Custom headers to set in the request. HTTP allows repeated headers.\",\"items\":{\"description\":\"HTTPHeader describes a custom header to be used in HTTP probes\",\"properties\":{\"name\":{\"description\":\"The header field name\",\"type\":\"string\"},\"value\":{\"description\":\"The header field value\",\"type\":\"string\"}},\"required\":[\"name\",\"value\"],\"type\":\"object\"},\"type\":\"array\"},\"path\":{\"description\":\"Path to access on the HTTP server.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"integer\"},{\"type\":\"string\"}],\"description\":\"Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\",\"x-kubernetes-int-or-string\":true},\"scheme\":{\"description\":\"Scheme to use for connecting to the host. Defaults to HTTP.\",\"type\":\"string\"}},\"required\":[\"port\"],\"type\":\"object\"},\"initialDelaySeconds\":{\"description\":\"Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\",\"format\":\"int32\",\"type\":\"integer\"},\"periodSeconds\":{\"description\":\"How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.\",\"format\":\"int32\",\"type\":\"integer\"},\"successThreshold\":{\"description\":\"Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.\",\"format\":\"int32\",\"type\":\"integer\"},\"tcpSocket\":{\"description\":\"TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook\",\"properties\":{\"host\":{\"description\":\"Optional: Host name to connect to, defaults to the pod IP.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"integer\"},{\"type\":\"string\"}],\"description\":\"Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\",\"x-kubernetes-int-or-string\":true}},\"required\":[\"port\"],\"type\":\"object\"},\"timeoutSeconds\":{\"description\":\"Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\",\"format\":\"int32\",\"type\":\"integer\"}},\"type\":\"object\"},\"resources\":{\"description\":\"Compute Resources required by this container. Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/\",\"properties\":{\"limits\":{\"additionalProperties\":{\"type\":\"string\"},\"description\":\"Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/\",\"type\":\"object\"},\"requests\":{\"additionalProperties\":{\"type\":\"string\"},\"description\":\"Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/\",\"type\":\"object\"}},\"type\":\"object\"},\"securityContext\":{\"description\":\"Security options the pod should run with. More info: https://kubernetes.io/docs/concepts/policy/security-context/ More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/\",\"properties\":{\"allowPrivilegeEscalation\":{\"description\":\"AllowPrivilegeEscalation controls whether a process can gain more privileges than its parent process. This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged 2) has CAP_SYS_ADMIN\",\"type\":\"boolean\"},\"capabilities\":{\"description\":\"The capabilities to add/drop when running containers. Defaults to the default set of capabilities granted by the container runtime.\",\"properties\":{\"add\":{\"description\":\"Added capabilities\",\"items\":{\"description\":\"Capability represent POSIX capabilities type\",\"type\":\"string\"},\"type\":\"array\"},\"drop\":{\"description\":\"Removed capabilities\",\"items\":{\"description\":\"Capability represent POSIX capabilities type\",\"type\":\"string\"},\"type\":\"array\"}},\"type\":\"object\"},\"privileged\":{\"description\":\"Run container in privileged mode. Processes in privileged containers are essentially equivalent to root on the host. Defaults to false.\",\"type\":\"boolean\"},\"procMount\":{\"description\":\"procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled.\",\"type\":\"string\"},\"readOnlyRootFilesystem\":{\"description\":\"Whether this container has a read-only root filesystem. Default is false.\",\"type\":\"boolean\"},\"runAsGroup\":{\"description\":\"The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\",\"format\":\"int64\",\"type\":\"integer\"},\"runAsNonRoot\":{\"description\":\"Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\",\"type\":\"boolean\"},\"runAsUser\":{\"description\":\"The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\",\"format\":\"int64\",\"type\":\"integer\"},\"seLinuxOptions\":{\"description\":\"The SELinux context to be applied to the container. If unspecified, the container runtime will allocate a random SELinux context for each container. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\",\"properties\":{\"level\":{\"description\":\"Level is SELinux level label that applies to the container.\",\"type\":\"string\"},\"role\":{\"description\":\"Role is a SELinux role label that applies to the container.\",\"type\":\"string\"},\"type\":{\"description\":\"Type is a SELinux type label that applies to the container.\",\"type\":\"string\"},\"user\":{\"description\":\"User is a SELinux user label that applies to the container.\",\"type\":\"string\"}},\"type\":\"object\"},\"windowsOptions\":{\"description\":\"The Windows specific settings applied to all containers. If unspecified, the options from the PodSecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\",\"properties\":{\"gmsaCredentialSpec\":{\"description\":\"GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field.\",\"type\":\"string\"},\"gmsaCredentialSpecName\":{\"description\":\"GMSACredentialSpecName is the name of the GMSA credential spec to use.\",\"type\":\"string\"},\"runAsUserName\":{\"description\":\"The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\",\"type\":\"string\"}},\"type\":\"object\"}},\"type\":\"object\"},\"startupProbe\":{\"description\":\"StartupProbe indicates that the Pod has successfully initialized. If specified, no other probes are executed until this completes successfully. If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, when it might take a long time to load data or warm a cache, than during steady-state operation. This cannot be updated. This is a beta feature enabled by the StartupProbe feature flag. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\",\"properties\":{\"exec\":{\"description\":\"One and only one of the following should be specified. Exec specifies the action to take.\",\"properties\":{\"command\":{\"description\":\"Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"type\":\"object\"},\"failureThreshold\":{\"description\":\"Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.\",\"format\":\"int32\",\"type\":\"integer\"},\"httpGet\":{\"description\":\"HTTPGet specifies the http request to perform.\",\"properties\":{\"host\":{\"description\":\"Host name to connect to, defaults to the pod IP. You probably want to set \\\"Host\\\" in httpHeaders instead.\",\"type\":\"string\"},\"httpHeaders\":{\"description\":\"Custom headers to set in the request. HTTP allows repeated headers.\",\"items\":{\"description\":\"HTTPHeader describes a custom header to be used in HTTP probes\",\"properties\":{\"name\":{\"description\":\"The header field name\",\"type\":\"string\"},\"value\":{\"description\":\"The header field value\",\"type\":\"string\"}},\"required\":[\"name\",\"value\"],\"type\":\"object\"},\"type\":\"array\"},\"path\":{\"description\":\"Path to access on the HTTP server.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"integer\"},{\"type\":\"string\"}],\"description\":\"Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\",\"x-kubernetes-int-or-string\":true},\"scheme\":{\"description\":\"Scheme to use for connecting to the host. Defaults to HTTP.\",\"type\":\"string\"}},\"required\":[\"port\"],\"type\":\"object\"},\"initialDelaySeconds\":{\"description\":\"Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\",\"format\":\"int32\",\"type\":\"integer\"},\"periodSeconds\":{\"description\":\"How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.\",\"format\":\"int32\",\"type\":\"integer\"},\"successThreshold\":{\"description\":\"Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.\",\"format\":\"int32\",\"type\":\"integer\"},\"tcpSocket\":{\"description\":\"TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook\",\"properties\":{\"host\":{\"description\":\"Optional: Host name to connect to, defaults to the pod IP.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"integer\"},{\"type\":\"string\"}],\"description\":\"Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\",\"x-kubernetes-int-or-string\":true}},\"required\":[\"port\"],\"type\":\"object\"},\"timeoutSeconds\":{\"description\":\"Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\",\"format\":\"int32\",\"type\":\"integer\"}},\"type\":\"object\"},\"stdin\":{\"description\":\"Whether this container should allocate a buffer for stdin in the container runtime. If this is not set, reads from stdin in the container will always result in EOF. Default is false.\",\"type\":\"boolean\"},\"stdinOnce\":{\"description\":\"Whether the container runtime should close the stdin channel after it has been opened by a single attach. When stdin is true the stdin stream will remain open across multiple attach sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the first client attaches to stdin, and then remains open and accepts data until the client disconnects, at which time stdin is closed and remains closed until the container is restarted. If this flag is false, a container processes that reads from stdin will never receive an EOF. Default is false\",\"type\":\"boolean\"},\"terminationMessagePath\":{\"description\":\"Optional: Path at which the file to which the container's termination message will be written is mounted into the container's filesystem. Message written is intended to be brief final status, such as an assertion failure message. Will be truncated by the node if greater than 4096 bytes. The total message length across all containers will be limited to 12kb. Defaults to /dev/termination-log. Cannot be updated.\",\"type\":\"string\"},\"terminationMessagePolicy\":{\"description\":\"Indicate how the termination message should be populated. File will use the contents of terminationMessagePath to populate the container status message on both success and failure. FallbackToLogsOnError will use the last chunk of container log output if the termination message file is empty and the container exited with an error. The log output is limited to 2048 bytes or 80 lines, whichever is smaller. Defaults to File. Cannot be updated.\",\"type\":\"string\"},\"tty\":{\"description\":\"Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. Default is false.\",\"type\":\"boolean\"},\"volumeDevices\":{\"description\":\"volumeDevices is the list of block devices to be used by the container.\",\"items\":{\"description\":\"volumeDevice describes a mapping of a raw block device within a container.\",\"properties\":{\"devicePath\":{\"description\":\"devicePath is the path inside of the container that the device will be mapped to.\",\"type\":\"string\"},\"name\":{\"description\":\"name must match the name of a persistentVolumeClaim in the pod\",\"type\":\"string\"}},\"required\":[\"devicePath\",\"name\"],\"type\":\"object\"},\"type\":\"array\"},\"volumeMounts\":{\"description\":\"Pod volumes to mount into the container's filesystem. Cannot be updated.\",\"items\":{\"description\":\"VolumeMount describes a mounting of a Volume within a container.\",\"properties\":{\"mountPath\":{\"description\":\"Path within the container at which the volume should be mounted. Must not contain ':'.\",\"type\":\"string\"},\"mountPropagation\":{\"description\":\"mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10.\",\"type\":\"string\"},\"name\":{\"description\":\"This must match the Name of a Volume.\",\"type\":\"string\"},\"readOnly\":{\"description\":\"Mounted read-only if true, read-write otherwise (false or unspecified). Defaults to false.\",\"type\":\"boolean\"},\"subPath\":{\"description\":\"Path within the volume from which the container's volume should be mounted. Defaults to \\\"\\\" (volume's root).\",\"type\":\"string\"},\"subPathExpr\":{\"description\":\"Expanded path within the volume from which the container's volume should be mounted. Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. Defaults to \\\"\\\" (volume's root). SubPathExpr and SubPath are mutually exclusive.\",\"type\":\"string\"}},\"required\":[\"mountPath\",\"name\"],\"type\":\"object\"},\"type\":\"array\"},\"workingDir\":{\"description\":\"Container's working directory. If not specified, the container runtime's default will be used, which might be configured in the container image. Cannot be updated.\",\"type\":\"string\"}},\"required\":[\"name\"],\"type\":\"object\"},\"type\":\"array\"},\"listenLocal\":{\"description\":\"ListenLocal makes the Prometheus server listen on loopback, so that it does not bind against the Pod IP.\",\"type\":\"boolean\"},\"logFormat\":{\"description\":\"Log format for Prometheus to be configured with.\",\"type\":\"string\"},\"logLevel\":{\"description\":\"Log level for Prometheus to be configured with.\",\"type\":\"string\"},\"nodeSelector\":{\"additionalProperties\":{\"type\":\"string\"},\"description\":\"Define which Nodes the Pods are scheduled on.\",\"type\":\"object\"},\"overrideHonorLabels\":{\"description\":\"OverrideHonorLabels if set to true overrides all user configured honor_labels. If HonorLabels is set in ServiceMonitor or PodMonitor to true, this overrides honor_labels to false.\",\"type\":\"boolean\"},\"overrideHonorTimestamps\":{\"description\":\"OverrideHonorTimestamps allows to globally enforce honoring timestamps in all scrape configs.\",\"type\":\"boolean\"},\"paused\":{\"description\":\"When a Prometheus deployment is paused, no actions except for deletion will be performed on the underlying objects.\",\"type\":\"boolean\"},\"podMetadata\":{\"description\":\"PodMetadata configures Labels and Annotations which are propagated to the prometheus pods.\",\"properties\":{\"annotations\":{\"additionalProperties\":{\"type\":\"string\"},\"description\":\"Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations\",\"type\":\"object\"},\"labels\":{\"additionalProperties\":{\"type\":\"string\"},\"description\":\"Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels\",\"type\":\"object\"},\"name\":{\"description\":\"Name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names\",\"type\":\"string\"}},\"type\":\"object\"},\"podMonitorNamespaceSelector\":{\"description\":\"Namespaces to be selected for PodMonitor discovery. If nil, only check own namespace.\",\"properties\":{\"matchExpressions\":{\"description\":\"matchExpressions is a list of label selector requirements. The requirements are ANDed.\",\"items\":{\"description\":\"A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"key is the label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\",\"type\":\"string\"},\"values\":{\"description\":\"values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"],\"type\":\"object\"},\"type\":\"array\"},\"matchLabels\":{\"additionalProperties\":{\"type\":\"string\"},\"description\":\"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \\\"key\\\", the operator is \\\"In\\\", and the values array contains only \\\"value\\\". The requirements are ANDed.\",\"type\":\"object\"}},\"type\":\"object\"},\"podMonitorSelector\":{\"description\":\"*Experimental* PodMonitors to be selected for target discovery. *Deprecated:* if neither this nor serviceMonitorSelector are specified, configuration is unmanaged.\",\"properties\":{\"matchExpressions\":{\"description\":\"matchExpressions is a list of label selector requirements. The requirements are ANDed.\",\"items\":{\"description\":\"A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"key is the label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\",\"type\":\"string\"},\"values\":{\"description\":\"values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"],\"type\":\"object\"},\"type\":\"array\"},\"matchLabels\":{\"additionalProperties\":{\"type\":\"string\"},\"description\":\"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \\\"key\\\", the operator is \\\"In\\\", and the values array contains only \\\"value\\\". The requirements are ANDed.\",\"type\":\"object\"}},\"type\":\"object\"},\"portName\":{\"description\":\"Port name used for the pods and governing service. This defaults to web\",\"type\":\"string\"},\"priorityClassName\":{\"description\":\"Priority class assigned to the Pods\",\"type\":\"string\"},\"prometheusExternalLabelName\":{\"description\":\"Name of Prometheus external label used to denote Prometheus instance name. Defaults to the value of `prometheus`. External label will _not_ be added when value is set to empty string (`\\\"\\\"`).\",\"type\":\"string\"},\"query\":{\"description\":\"QuerySpec defines the query command line flags when starting Prometheus.\",\"properties\":{\"lookbackDelta\":{\"description\":\"The delta difference allowed for retrieving metrics during expression evaluations.\",\"type\":\"string\"},\"maxConcurrency\":{\"description\":\"Number of concurrent queries that can be run at once.\",\"format\":\"int32\",\"type\":\"integer\"},\"maxSamples\":{\"description\":\"Maximum number of samples a single query can load into memory. Note that queries will fail if they would load more samples than this into memory, so this also limits the number of samples a query can return.\",\"format\":\"int32\",\"type\":\"integer\"},\"timeout\":{\"description\":\"Maximum time a query may take before being aborted.\",\"type\":\"string\"}},\"type\":\"object\"},\"queryLogFile\":{\"description\":\"QueryLogFile specifies the file to which PromQL queries are logged. Note that this location must be writable, and can be persisted using an attached volume. Alternatively, the location can be set to a stdout location such as `/dev/stdout` to log querie information to the default Prometheus log stream. This is only available in versions of Prometheus \\u003e= 2.16.0. For more details, see the Prometheus docs (https://prometheus.io/docs/guides/query-log/)\",\"type\":\"string\"},\"remoteRead\":{\"description\":\"If specified, the remote_read spec. This is an experimental feature, it may change in any upcoming release in a breaking way.\",\"items\":{\"description\":\"RemoteReadSpec defines the remote_read configuration for prometheus.\",\"properties\":{\"basicAuth\":{\"description\":\"BasicAuth for the URL.\",\"properties\":{\"password\":{\"description\":\"The secret in the service monitor namespace that contains the password for authentication.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or its key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"},\"username\":{\"description\":\"The secret in the service monitor namespace that contains the username for authentication.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or its key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"}},\"type\":\"object\"},\"bearerToken\":{\"description\":\"bearer token for remote read.\",\"type\":\"string\"},\"bearerTokenFile\":{\"description\":\"File to read bearer token for remote read.\",\"type\":\"string\"},\"name\":{\"description\":\"The name of the remote read queue, must be unique if specified. The name is used in metrics and logging in order to differentiate read configurations. Only valid in Prometheus versions 2.15.0 and newer.\",\"type\":\"string\"},\"proxyUrl\":{\"description\":\"Optional ProxyURL\",\"type\":\"string\"},\"readRecent\":{\"description\":\"Whether reads should be made for queries for time ranges that the local storage should have complete data for.\",\"type\":\"boolean\"},\"remoteTimeout\":{\"description\":\"Timeout for requests to the remote read endpoint.\",\"type\":\"string\"},\"requiredMatchers\":{\"additionalProperties\":{\"type\":\"string\"},\"description\":\"An optional list of equality matchers which have to be present in a selector to query the remote read endpoint.\",\"type\":\"object\"},\"tlsConfig\":{\"description\":\"TLS Config to use for remote read.\",\"properties\":{\"ca\":{\"description\":\"Struct containing the CA cert to use for the targets.\",\"properties\":{\"configMap\":{\"description\":\"ConfigMap containing data to use for the targets.\",\"properties\":{\"key\":{\"description\":\"The key to select.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the ConfigMap or its key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"},\"secret\":{\"description\":\"Secret containing data to use for the targets.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or its key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"}},\"type\":\"object\"},\"caFile\":{\"description\":\"Path to the CA cert in the Prometheus container to use for the targets.\",\"type\":\"string\"},\"cert\":{\"description\":\"Struct containing the client cert file for the targets.\",\"properties\":{\"configMap\":{\"description\":\"ConfigMap containing data to use for the targets.\",\"properties\":{\"key\":{\"description\":\"The key to select.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the ConfigMap or its key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"},\"secret\":{\"description\":\"Secret containing data to use for the targets.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or its key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"}},\"type\":\"object\"},\"certFile\":{\"description\":\"Path to the client cert file in the Prometheus container for the targets.\",\"type\":\"string\"},\"insecureSkipVerify\":{\"description\":\"Disable target certificate validation.\",\"type\":\"boolean\"},\"keyFile\":{\"description\":\"Path to the client key file in the Prometheus container for the targets.\",\"type\":\"string\"},\"keySecret\":{\"description\":\"Secret containing the client key file for the targets.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or its key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"},\"serverName\":{\"description\":\"Used to verify the hostname for the targets.\",\"type\":\"string\"}},\"type\":\"object\"},\"url\":{\"description\":\"The URL of the endpoint to send samples to.\",\"type\":\"string\"}},\"required\":[\"url\"],\"type\":\"object\"},\"type\":\"array\"},\"remoteWrite\":{\"description\":\"If specified, the remote_write spec. This is an experimental feature, it may change in any upcoming release in a breaking way.\",\"items\":{\"description\":\"RemoteWriteSpec defines the remote_write configuration for prometheus.\",\"properties\":{\"basicAuth\":{\"description\":\"BasicAuth for the URL.\",\"properties\":{\"password\":{\"description\":\"The secret in the service monitor namespace that contains the password for authentication.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or its key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"},\"username\":{\"description\":\"The secret in the service monitor namespace that contains the username for authentication.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or its key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"}},\"type\":\"object\"},\"bearerToken\":{\"description\":\"File to read bearer token for remote write.\",\"type\":\"string\"},\"bearerTokenFile\":{\"description\":\"File to read bearer token for remote write.\",\"type\":\"string\"},\"name\":{\"description\":\"The name of the remote write queue, must be unique if specified. The name is used in metrics and logging in order to differentiate queues. Only valid in Prometheus versions 2.15.0 and newer.\",\"type\":\"string\"},\"proxyUrl\":{\"description\":\"Optional ProxyURL\",\"type\":\"string\"},\"queueConfig\":{\"description\":\"QueueConfig allows tuning of the remote write queue parameters.\",\"properties\":{\"batchSendDeadline\":{\"description\":\"BatchSendDeadline is the maximum time a sample will wait in buffer.\",\"type\":\"string\"},\"capacity\":{\"description\":\"Capacity is the number of samples to buffer per shard before we start dropping them.\",\"type\":\"integer\"},\"maxBackoff\":{\"description\":\"MaxBackoff is the maximum retry delay.\",\"type\":\"string\"},\"maxRetries\":{\"description\":\"MaxRetries is the maximum number of times to retry a batch on recoverable errors.\",\"type\":\"integer\"},\"maxSamplesPerSend\":{\"description\":\"MaxSamplesPerSend is the maximum number of samples per send.\",\"type\":\"integer\"},\"maxShards\":{\"description\":\"MaxShards is the maximum number of shards, i.e. amount of concurrency.\",\"type\":\"integer\"},\"minBackoff\":{\"description\":\"MinBackoff is the initial retry delay. Gets doubled for every retry.\",\"type\":\"string\"},\"minShards\":{\"description\":\"MinShards is the minimum number of shards, i.e. amount of concurrency.\",\"type\":\"integer\"}},\"type\":\"object\"},\"remoteTimeout\":{\"description\":\"Timeout for requests to the remote write endpoint.\",\"type\":\"string\"},\"tlsConfig\":{\"description\":\"TLS Config to use for remote write.\",\"properties\":{\"ca\":{\"description\":\"Struct containing the CA cert to use for the targets.\",\"properties\":{\"configMap\":{\"description\":\"ConfigMap containing data to use for the targets.\",\"properties\":{\"key\":{\"description\":\"The key to select.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the ConfigMap or its key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"},\"secret\":{\"description\":\"Secret containing data to use for the targets.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or its key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"}},\"type\":\"object\"},\"caFile\":{\"description\":\"Path to the CA cert in the Prometheus container to use for the targets.\",\"type\":\"string\"},\"cert\":{\"description\":\"Struct containing the client cert file for the targets.\",\"properties\":{\"configMap\":{\"description\":\"ConfigMap containing data to use for the targets.\",\"properties\":{\"key\":{\"description\":\"The key to select.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the ConfigMap or its key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"},\"secret\":{\"description\":\"Secret containing data to use for the targets.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or its key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"}},\"type\":\"object\"},\"certFile\":{\"description\":\"Path to the client cert file in the Prometheus container for the targets.\",\"type\":\"string\"},\"insecureSkipVerify\":{\"description\":\"Disable target certificate validation.\",\"type\":\"boolean\"},\"keyFile\":{\"description\":\"Path to the client key file in the Prometheus container for the targets.\",\"type\":\"string\"},\"keySecret\":{\"description\":\"Secret containing the client key file for the targets.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or its key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"},\"serverName\":{\"description\":\"Used to verify the hostname for the targets.\",\"type\":\"string\"}},\"type\":\"object\"},\"url\":{\"description\":\"The URL of the endpoint to send samples to.\",\"type\":\"string\"},\"writeRelabelConfigs\":{\"description\":\"The list of remote write relabel configurations.\",\"items\":{\"description\":\"RelabelConfig allows dynamic rewriting of the label set, being applied to samples before ingestion. It defines `\\u003cmetric_relabel_configs\\u003e`-section of Prometheus configuration. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs\",\"properties\":{\"action\":{\"description\":\"Action to perform based on regex matching. Default is 'replace'\",\"type\":\"string\"},\"modulus\":{\"description\":\"Modulus to take of the hash of the source label values.\",\"format\":\"int64\",\"type\":\"integer\"},\"regex\":{\"description\":\"Regular expression against which the extracted value is matched. Default is '(.*)'\",\"type\":\"string\"},\"replacement\":{\"description\":\"Replacement value against which a regex replace is performed if the regular expression matches. Regex capture groups are available. Default is '$1'\",\"type\":\"string\"},\"separator\":{\"description\":\"Separator placed between concatenated source label values. default is ';'.\",\"type\":\"string\"},\"sourceLabels\":{\"description\":\"The source labels select values from existing labels. Their content is concatenated using the configured separator and matched against the configured regular expression for the replace, keep, and drop actions.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"targetLabel\":{\"description\":\"Label to which the resulting value is written in a replace action. It is mandatory for replace actions. Regex capture groups are available.\",\"type\":\"string\"}},\"type\":\"object\"},\"type\":\"array\"}},\"required\":[\"url\"],\"type\":\"object\"},\"type\":\"array\"},\"replicaExternalLabelName\":{\"description\":\"Name of Prometheus external label used to denote replica name. Defaults to the value of `prometheus_replica`. External label will _not_ be added when value is set to empty string (`\\\"\\\"`).\",\"type\":\"string\"},\"replicas\":{\"description\":\"Number of instances to deploy for a Prometheus deployment.\",\"format\":\"int32\",\"type\":\"integer\"},\"resources\":{\"description\":\"Define resources requests and limits for single Pods.\",\"properties\":{\"limits\":{\"additionalProperties\":{\"type\":\"string\"},\"description\":\"Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/\",\"type\":\"object\"},\"requests\":{\"additionalProperties\":{\"type\":\"string\"},\"description\":\"Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/\",\"type\":\"object\"}},\"type\":\"object\"},\"retention\":{\"description\":\"Time duration Prometheus shall retain data for. Default is '24h', and must match the regular expression `[0-9]+(ms|s|m|h|d|w|y)` (milliseconds seconds minutes hours days weeks years).\",\"type\":\"string\"},\"retentionSize\":{\"description\":\"Maximum amount of disk space used by blocks.\",\"type\":\"string\"},\"routePrefix\":{\"description\":\"The route prefix Prometheus registers HTTP handlers for. This is useful, if using ExternalURL and a proxy is rewriting HTTP routes of a request, and the actual ExternalURL is still true, but the server serves requests under a different route prefix. For example for use with `kubectl proxy`.\",\"type\":\"string\"},\"ruleNamespaceSelector\":{\"description\":\"Namespaces to be selected for PrometheusRules discovery. If unspecified, only the same namespace as the Prometheus object is in is used.\",\"properties\":{\"matchExpressions\":{\"description\":\"matchExpressions is a list of label selector requirements. The requirements are ANDed.\",\"items\":{\"description\":\"A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"key is the label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\",\"type\":\"string\"},\"values\":{\"description\":\"values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"],\"type\":\"object\"},\"type\":\"array\"},\"matchLabels\":{\"additionalProperties\":{\"type\":\"string\"},\"description\":\"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \\\"key\\\", the operator is \\\"In\\\", and the values array contains only \\\"value\\\". The requirements are ANDed.\",\"type\":\"object\"}},\"type\":\"object\"},\"ruleSelector\":{\"description\":\"A selector to select which PrometheusRules to mount for loading alerting/recording rules from. Until (excluding) Prometheus Operator v0.24.0 Prometheus Operator will migrate any legacy rule ConfigMaps to PrometheusRule custom resources selected by RuleSelector. Make sure it does not match any config maps that you do not want to be migrated.\",\"properties\":{\"matchExpressions\":{\"description\":\"matchExpressions is a list of label selector requirements. The requirements are ANDed.\",\"items\":{\"description\":\"A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"key is the label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\",\"type\":\"string\"},\"values\":{\"description\":\"values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"],\"type\":\"object\"},\"type\":\"array\"},\"matchLabels\":{\"additionalProperties\":{\"type\":\"string\"},\"description\":\"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \\\"key\\\", the operator is \\\"In\\\", and the values array contains only \\\"value\\\". The requirements are ANDed.\",\"type\":\"object\"}},\"type\":\"object\"},\"rules\":{\"description\":\"/--rules.*/ command-line arguments.\",\"properties\":{\"alert\":{\"description\":\"/--rules.alert.*/ command-line arguments\",\"properties\":{\"forGracePeriod\":{\"description\":\"Minimum duration between alert and restored 'for' state. This is maintained only for alerts with configured 'for' time greater than grace period.\",\"type\":\"string\"},\"forOutageTolerance\":{\"description\":\"Max time to tolerate prometheus outage for restoring 'for' state of alert.\",\"type\":\"string\"},\"resendDelay\":{\"description\":\"Minimum amount of time to wait before resending an alert to Alertmanager.\",\"type\":\"string\"}},\"type\":\"object\"}},\"type\":\"object\"},\"scrapeInterval\":{\"description\":\"Interval between consecutive scrapes.\",\"type\":\"string\"},\"secrets\":{\"description\":\"Secrets is a list of Secrets in the same namespace as the Prometheus object, which shall be mounted into the Prometheus Pods. The Secrets are mounted into /etc/prometheus/secrets/\\u003csecret-name\\u003e.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"securityContext\":{\"description\":\"SecurityContext holds pod-level security attributes and common container settings. This defaults to the default PodSecurityContext.\",\"properties\":{\"fsGroup\":{\"description\":\"A special supplemental group that applies to all containers in a pod. Some volume types allow the Kubelet to change the ownership of that volume to be owned by the pod: \\n 1. The owning GID will be the FSGroup 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) 3. The permission bits are OR'd with rw-rw---- \\n If unset, the Kubelet will not modify the ownership and permissions of any volume.\",\"format\":\"int64\",\"type\":\"integer\"},\"fsGroupChangePolicy\":{\"description\":\"fsGroupChangePolicy defines behavior of changing ownership and permission of the volume before being exposed inside Pod. This field will only apply to volume types which support fsGroup based ownership(and permissions). It will have no effect on ephemeral volume types such as: secret, configmaps and emptydir. Valid values are \\\"OnRootMismatch\\\" and \\\"Always\\\". If not specified defaults to \\\"Always\\\".\",\"type\":\"string\"},\"runAsGroup\":{\"description\":\"The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container.\",\"format\":\"int64\",\"type\":\"integer\"},\"runAsNonRoot\":{\"description\":\"Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\",\"type\":\"boolean\"},\"runAsUser\":{\"description\":\"The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container.\",\"format\":\"int64\",\"type\":\"integer\"},\"seLinuxOptions\":{\"description\":\"The SELinux context to be applied to all containers. If unspecified, the container runtime will allocate a random SELinux context for each container. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container.\",\"properties\":{\"level\":{\"description\":\"Level is SELinux level label that applies to the container.\",\"type\":\"string\"},\"role\":{\"description\":\"Role is a SELinux role label that applies to the container.\",\"type\":\"string\"},\"type\":{\"description\":\"Type is a SELinux type label that applies to the container.\",\"type\":\"string\"},\"user\":{\"description\":\"User is a SELinux user label that applies to the container.\",\"type\":\"string\"}},\"type\":\"object\"},\"supplementalGroups\":{\"description\":\"A list of groups applied to the first process run in each container, in addition to the container's primary GID. If unspecified, no groups will be added to any container.\",\"items\":{\"format\":\"int64\",\"type\":\"integer\"},\"type\":\"array\"},\"sysctls\":{\"description\":\"Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported sysctls (by the container runtime) might fail to launch.\",\"items\":{\"description\":\"Sysctl defines a kernel parameter to be set\",\"properties\":{\"name\":{\"description\":\"Name of a property to set\",\"type\":\"string\"},\"value\":{\"description\":\"Value of a property to set\",\"type\":\"string\"}},\"required\":[\"name\",\"value\"],\"type\":\"object\"},\"type\":\"array\"},\"windowsOptions\":{\"description\":\"The Windows specific settings applied to all containers. If unspecified, the options within a container's SecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\",\"properties\":{\"gmsaCredentialSpec\":{\"description\":\"GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field.\",\"type\":\"string\"},\"gmsaCredentialSpecName\":{\"description\":\"GMSACredentialSpecName is the name of the GMSA credential spec to use.\",\"type\":\"string\"},\"runAsUserName\":{\"description\":\"The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\",\"type\":\"string\"}},\"type\":\"object\"}},\"type\":\"object\"},\"serviceAccountName\":{\"description\":\"ServiceAccountName is the name of the ServiceAccount to use to run the Prometheus Pods.\",\"type\":\"string\"},\"serviceMonitorNamespaceSelector\":{\"description\":\"Namespaces to be selected for ServiceMonitor discovery. If nil, only check own namespace.\",\"properties\":{\"matchExpressions\":{\"description\":\"matchExpressions is a list of label selector requirements. The requirements are ANDed.\",\"items\":{\"description\":\"A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"key is the label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\",\"type\":\"string\"},\"values\":{\"description\":\"values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"],\"type\":\"object\"},\"type\":\"array\"},\"matchLabels\":{\"additionalProperties\":{\"type\":\"string\"},\"description\":\"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \\\"key\\\", the operator is \\\"In\\\", and the values array contains only \\\"value\\\". The requirements are ANDed.\",\"type\":\"object\"}},\"type\":\"object\"},\"serviceMonitorSelector\":{\"description\":\"ServiceMonitors to be selected for target discovery. *Deprecated:* if neither this nor podMonitorSelector are specified, configuration is unmanaged.\",\"properties\":{\"matchExpressions\":{\"description\":\"matchExpressions is a list of label selector requirements. The requirements are ANDed.\",\"items\":{\"description\":\"A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"key is the label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\",\"type\":\"string\"},\"values\":{\"description\":\"values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"],\"type\":\"object\"},\"type\":\"array\"},\"matchLabels\":{\"additionalProperties\":{\"type\":\"string\"},\"description\":\"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \\\"key\\\", the operator is \\\"In\\\", and the values array contains only \\\"value\\\". The requirements are ANDed.\",\"type\":\"object\"}},\"type\":\"object\"},\"sha\":{\"description\":\"SHA of Prometheus container image to be deployed. Defaults to the value of `version`. Similar to a tag, but the SHA explicitly deploys an immutable container image. Version and Tag are ignored if SHA is set.\",\"type\":\"string\"},\"storage\":{\"description\":\"Storage spec to specify how storage shall be used.\",\"properties\":{\"disableMountSubPath\":{\"description\":\"Deprecated: subPath usage will be disabled by default in a future release, this option will become unnecessary. DisableMountSubPath allows to remove any subPath usage in volume mounts.\",\"type\":\"boolean\"},\"emptyDir\":{\"description\":\"EmptyDirVolumeSource to be used by the Prometheus StatefulSets. If specified, used in place of any volumeClaimTemplate. More info: https://kubernetes.io/docs/concepts/storage/volumes/#emptydir\",\"properties\":{\"medium\":{\"description\":\"What type of storage medium should back this directory. The default is \\\"\\\" which means to use the node's default medium. Must be an empty string (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir\",\"type\":\"string\"},\"sizeLimit\":{\"description\":\"Total amount of local storage required for this EmptyDir volume. The size limit is also applicable for memory medium. The maximum usage on memory medium EmptyDir would be the minimum value between the SizeLimit specified here and the sum of memory limits of all containers in a pod. The default is nil which means that the limit is undefined. More info: http://kubernetes.io/docs/user-guide/volumes#emptydir\",\"type\":\"string\"}},\"type\":\"object\"},\"volumeClaimTemplate\":{\"description\":\"A PVC spec to be used by the Prometheus StatefulSets.\",\"properties\":{\"apiVersion\":{\"description\":\"APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\",\"type\":\"string\"},\"kind\":{\"description\":\"Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\",\"type\":\"string\"},\"metadata\":{\"description\":\"EmbeddedMetadata contains metadata relevant to an EmbeddedResource.\",\"properties\":{\"annotations\":{\"additionalProperties\":{\"type\":\"string\"},\"description\":\"Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations\",\"type\":\"object\"},\"labels\":{\"additionalProperties\":{\"type\":\"string\"},\"description\":\"Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels\",\"type\":\"object\"},\"name\":{\"description\":\"Name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names\",\"type\":\"string\"}},\"type\":\"object\"},\"spec\":{\"description\":\"Spec defines the desired characteristics of a volume requested by a pod author. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims\",\"properties\":{\"accessModes\":{\"description\":\"AccessModes contains the desired access modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"dataSource\":{\"description\":\"This field can be used to specify either: * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot - Beta) * An existing PVC (PersistentVolumeClaim) * An existing custom resource/object that implements data population (Alpha) In order to use VolumeSnapshot object types, the appropriate feature gate must be enabled (VolumeSnapshotDataSource or AnyVolumeDataSource) If the provisioner or an external controller can support the specified data source, it will create a new volume based on the contents of the specified data source. If the specified data source is not supported, the volume will not be created and the failure will be reported as an event. In the future, we plan to support more data source types and the behavior of the provisioner may change.\",\"properties\":{\"apiGroup\":{\"description\":\"APIGroup is the group for the resource being referenced. If APIGroup is not specified, the specified Kind must be in the core API group. For any other third-party types, APIGroup is required.\",\"type\":\"string\"},\"kind\":{\"description\":\"Kind is the type of resource being referenced\",\"type\":\"string\"},\"name\":{\"description\":\"Name is the name of resource being referenced\",\"type\":\"string\"}},\"required\":[\"kind\",\"name\"],\"type\":\"object\"},\"resources\":{\"description\":\"Resources represents the minimum resources the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources\",\"properties\":{\"limits\":{\"additionalProperties\":{\"type\":\"string\"},\"description\":\"Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/\",\"type\":\"object\"},\"requests\":{\"additionalProperties\":{\"type\":\"string\"},\"description\":\"Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/\",\"type\":\"object\"}},\"type\":\"object\"},\"selector\":{\"description\":\"A label query over volumes to consider for binding.\",\"properties\":{\"matchExpressions\":{\"description\":\"matchExpressions is a list of label selector requirements. The requirements are ANDed.\",\"items\":{\"description\":\"A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"key is the label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\",\"type\":\"string\"},\"values\":{\"description\":\"values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"],\"type\":\"object\"},\"type\":\"array\"},\"matchLabels\":{\"additionalProperties\":{\"type\":\"string\"},\"description\":\"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \\\"key\\\", the operator is \\\"In\\\", and the values array contains only \\\"value\\\". The requirements are ANDed.\",\"type\":\"object\"}},\"type\":\"object\"},\"storageClassName\":{\"description\":\"Name of the StorageClass required by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1\",\"type\":\"string\"},\"volumeMode\":{\"description\":\"volumeMode defines what type of volume is required by the claim. Value of Filesystem is implied when not included in claim spec.\",\"type\":\"string\"},\"volumeName\":{\"description\":\"VolumeName is the binding reference to the PersistentVolume backing this claim.\",\"type\":\"string\"}},\"type\":\"object\"},\"status\":{\"description\":\"Status represents the current information/status of a persistent volume claim. Read-only. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims\",\"properties\":{\"accessModes\":{\"description\":\"AccessModes contains the actual access modes the volume backing the PVC has. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"capacity\":{\"additionalProperties\":{\"type\":\"string\"},\"description\":\"Represents the actual resources of the underlying volume.\",\"type\":\"object\"},\"conditions\":{\"description\":\"Current Condition of persistent volume claim. If underlying persistent volume is being resized then the Condition will be set to 'ResizeStarted'.\",\"items\":{\"description\":\"PersistentVolumeClaimCondition contains details about state of pvc\",\"properties\":{\"lastProbeTime\":{\"description\":\"Last time we probed the condition.\",\"format\":\"date-time\",\"type\":\"string\"},\"lastTransitionTime\":{\"description\":\"Last time the condition transitioned from one status to another.\",\"format\":\"date-time\",\"type\":\"string\"},\"message\":{\"description\":\"Human-readable message indicating details about last transition.\",\"type\":\"string\"},\"reason\":{\"description\":\"Unique, this should be a short, machine understandable string that gives the reason for condition's last transition. If it reports \\\"ResizeStarted\\\" that means the underlying persistent volume is being resized.\",\"type\":\"string\"},\"status\":{\"type\":\"string\"},\"type\":{\"description\":\"PersistentVolumeClaimConditionType is a valid value of PersistentVolumeClaimCondition.Type\",\"type\":\"string\"}},\"required\":[\"status\",\"type\"],\"type\":\"object\"},\"type\":\"array\"},\"phase\":{\"description\":\"Phase represents the current phase of PersistentVolumeClaim.\",\"type\":\"string\"}},\"type\":\"object\"}},\"type\":\"object\"}},\"type\":\"object\"},\"tag\":{\"description\":\"Tag of Prometheus container image to be deployed. Defaults to the value of `version`. Version is ignored if Tag is set.\",\"type\":\"string\"},\"thanos\":{\"description\":\"Thanos configuration allows configuring various aspects of a Prometheus server in a Thanos environment. \\n This section is experimental, it may change significantly without deprecation notice in any release. \\n This is experimental and may change significantly without backward compatibility in any release.\",\"properties\":{\"baseImage\":{\"description\":\"Thanos base image if other than default.\",\"type\":\"string\"},\"grpcServerTlsConfig\":{\"description\":\"GRPCServerTLSConfig configures the gRPC server from which Thanos Querier reads recorded rule data. Note: Currently only the CAFile, CertFile, and KeyFile fields are supported. Maps to the '--grpc-server-tls-*' CLI args.\",\"properties\":{\"ca\":{\"description\":\"Struct containing the CA cert to use for the targets.\",\"properties\":{\"configMap\":{\"description\":\"ConfigMap containing data to use for the targets.\",\"properties\":{\"key\":{\"description\":\"The key to select.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the ConfigMap or its key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"},\"secret\":{\"description\":\"Secret containing data to use for the targets.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or its key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"}},\"type\":\"object\"},\"caFile\":{\"description\":\"Path to the CA cert in the Prometheus container to use for the targets.\",\"type\":\"string\"},\"cert\":{\"description\":\"Struct containing the client cert file for the targets.\",\"properties\":{\"configMap\":{\"description\":\"ConfigMap containing data to use for the targets.\",\"properties\":{\"key\":{\"description\":\"The key to select.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the ConfigMap or its key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"},\"secret\":{\"description\":\"Secret containing data to use for the targets.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or its key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"}},\"type\":\"object\"},\"certFile\":{\"description\":\"Path to the client cert file in the Prometheus container for the targets.\",\"type\":\"string\"},\"insecureSkipVerify\":{\"description\":\"Disable target certificate validation.\",\"type\":\"boolean\"},\"keyFile\":{\"description\":\"Path to the client key file in the Prometheus container for the targets.\",\"type\":\"string\"},\"keySecret\":{\"description\":\"Secret containing the client key file for the targets.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or its key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"},\"serverName\":{\"description\":\"Used to verify the hostname for the targets.\",\"type\":\"string\"}},\"type\":\"object\"},\"image\":{\"description\":\"Image if specified has precedence over baseImage, tag and sha combinations. Specifying the version is still necessary to ensure the Prometheus Operator knows what version of Thanos is being configured.\",\"type\":\"string\"},\"listenLocal\":{\"description\":\"ListenLocal makes the Thanos sidecar listen on loopback, so that it does not bind against the Pod IP.\",\"type\":\"boolean\"},\"logFormat\":{\"description\":\"LogFormat for Thanos sidecar to be configured with.\",\"type\":\"string\"},\"logLevel\":{\"description\":\"LogLevel for Thanos sidecar to be configured with.\",\"type\":\"string\"},\"objectStorageConfig\":{\"description\":\"ObjectStorageConfig configures object storage in Thanos.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or its key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"},\"resources\":{\"description\":\"Resources defines the resource requirements for the Thanos sidecar. If not provided, no requests/limits will be set\",\"properties\":{\"limits\":{\"additionalProperties\":{\"type\":\"string\"},\"description\":\"Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/\",\"type\":\"object\"},\"requests\":{\"additionalProperties\":{\"type\":\"string\"},\"description\":\"Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/\",\"type\":\"object\"}},\"type\":\"object\"},\"sha\":{\"description\":\"SHA of Thanos container image to be deployed. Defaults to the value of `version`. Similar to a tag, but the SHA explicitly deploys an immutable container image. Version and Tag are ignored if SHA is set.\",\"type\":\"string\"},\"tag\":{\"description\":\"Tag of Thanos sidecar container image to be deployed. Defaults to the value of `version`. Version is ignored if Tag is set.\",\"type\":\"string\"},\"tracingConfig\":{\"description\":\"TracingConfig configures tracing in Thanos. This is an experimental feature, it may change in any upcoming release in a breaking way.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or its key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"},\"version\":{\"description\":\"Version describes the version of Thanos to use.\",\"type\":\"string\"}},\"type\":\"object\"},\"tolerations\":{\"description\":\"If specified, the pod's tolerations.\",\"items\":{\"description\":\"The pod this Toleration is attached to tolerates any taint that matches the triple \\u003ckey,value,effect\\u003e using the matching operator \\u003coperator\\u003e.\",\"properties\":{\"effect\":{\"description\":\"Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.\",\"type\":\"string\"},\"key\":{\"description\":\"Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys.\",\"type\":\"string\"},\"operator\":{\"description\":\"Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category.\",\"type\":\"string\"},\"tolerationSeconds\":{\"description\":\"TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system.\",\"format\":\"int64\",\"type\":\"integer\"},\"value\":{\"description\":\"Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string.\",\"type\":\"string\"}},\"type\":\"object\"},\"type\":\"array\"},\"version\":{\"description\":\"Version of Prometheus to be deployed.\",\"type\":\"string\"},\"volumeMounts\":{\"description\":\"VolumeMounts allows configuration of additional VolumeMounts on the output StatefulSet definition. VolumeMounts specified will be appended to other VolumeMounts in the prometheus container, that are generated as a result of StorageSpec objects.\",\"items\":{\"description\":\"VolumeMount describes a mounting of a Volume within a container.\",\"properties\":{\"mountPath\":{\"description\":\"Path within the container at which the volume should be mounted. Must not contain ':'.\",\"type\":\"string\"},\"mountPropagation\":{\"description\":\"mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10.\",\"type\":\"string\"},\"name\":{\"description\":\"This must match the Name of a Volume.\",\"type\":\"string\"},\"readOnly\":{\"description\":\"Mounted read-only if true, read-write otherwise (false or unspecified). Defaults to false.\",\"type\":\"boolean\"},\"subPath\":{\"description\":\"Path within the volume from which the container's volume should be mounted. Defaults to \\\"\\\" (volume's root).\",\"type\":\"string\"},\"subPathExpr\":{\"description\":\"Expanded path within the volume from which the container's volume should be mounted. Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. Defaults to \\\"\\\" (volume's root). SubPathExpr and SubPath are mutually exclusive.\",\"type\":\"string\"}},\"required\":[\"mountPath\",\"name\"],\"type\":\"object\"},\"type\":\"array\"},\"volumes\":{\"description\":\"Volumes allows configuration of additional volumes on the output StatefulSet definition. Volumes specified will be appended to other volumes that are generated as a result of StorageSpec objects.\",\"items\":{\"description\":\"Volume represents a named volume in a pod that may be accessed by any container in the pod.\",\"properties\":{\"awsElasticBlockStore\":{\"description\":\"AWSElasticBlockStore represents an AWS Disk resource that is attached to a kubelet's host machine and then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore\",\"properties\":{\"fsType\":{\"description\":\"Filesystem type of the volume that you want to mount. Tip: Ensure that the filesystem type is supported by the host operating system. Examples: \\\"ext4\\\", \\\"xfs\\\", \\\"ntfs\\\". Implicitly inferred to be \\\"ext4\\\" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore TODO: how do we prevent errors in the filesystem from compromising the machine\",\"type\":\"string\"},\"partition\":{\"description\":\"The partition in the volume that you want to mount. If omitted, the default is to mount by volume name. Examples: For volume /dev/sda1, you specify the partition as \\\"1\\\". Similarly, the volume partition for /dev/sda is \\\"0\\\" (or you can leave the property empty).\",\"format\":\"int32\",\"type\":\"integer\"},\"readOnly\":{\"description\":\"Specify \\\"true\\\" to force and set the ReadOnly property in VolumeMounts to \\\"true\\\". If omitted, the default is \\\"false\\\". More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore\",\"type\":\"boolean\"},\"volumeID\":{\"description\":\"Unique ID of the persistent disk resource in AWS (Amazon EBS volume). More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore\",\"type\":\"string\"}},\"required\":[\"volumeID\"],\"type\":\"object\"},\"azureDisk\":{\"description\":\"AzureDisk represents an Azure Data Disk mount on the host and bind mount to the pod.\",\"properties\":{\"cachingMode\":{\"description\":\"Host Caching mode: None, Read Only, Read Write.\",\"type\":\"string\"},\"diskName\":{\"description\":\"The Name of the data disk in the blob storage\",\"type\":\"string\"},\"diskURI\":{\"description\":\"The URI the data disk in the blob storage\",\"type\":\"string\"},\"fsType\":{\"description\":\"Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \\\"ext4\\\", \\\"xfs\\\", \\\"ntfs\\\". Implicitly inferred to be \\\"ext4\\\" if unspecified.\",\"type\":\"string\"},\"kind\":{\"description\":\"Expected values Shared: multiple blob disks per storage account Dedicated: single blob disk per storage account Managed: azure managed data disk (only in managed availability set). defaults to shared\",\"type\":\"string\"},\"readOnly\":{\"description\":\"Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.\",\"type\":\"boolean\"}},\"required\":[\"diskName\",\"diskURI\"],\"type\":\"object\"},\"azureFile\":{\"description\":\"AzureFile represents an Azure File Service mount on the host and bind mount to the pod.\",\"properties\":{\"readOnly\":{\"description\":\"Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.\",\"type\":\"boolean\"},\"secretName\":{\"description\":\"the name of secret that contains Azure Storage Account Name and Key\",\"type\":\"string\"},\"shareName\":{\"description\":\"Share Name\",\"type\":\"string\"}},\"required\":[\"secretName\",\"shareName\"],\"type\":\"object\"},\"cephfs\":{\"description\":\"CephFS represents a Ceph FS mount on the host that shares a pod's lifetime\",\"properties\":{\"monitors\":{\"description\":\"Required: Monitors is a collection of Ceph monitors More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"path\":{\"description\":\"Optional: Used as the mounted root, rather than the full Ceph tree, default is /\",\"type\":\"string\"},\"readOnly\":{\"description\":\"Optional: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it\",\"type\":\"boolean\"},\"secretFile\":{\"description\":\"Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it\",\"type\":\"string\"},\"secretRef\":{\"description\":\"Optional: SecretRef is reference to the authentication secret for User, default is empty. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it\",\"properties\":{\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"}},\"type\":\"object\"},\"user\":{\"description\":\"Optional: User is the rados user name, default is admin More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it\",\"type\":\"string\"}},\"required\":[\"monitors\"],\"type\":\"object\"},\"cinder\":{\"description\":\"Cinder represents a cinder volume attached and mounted on kubelets host machine. More info: https://examples.k8s.io/mysql-cinder-pd/README.md\",\"properties\":{\"fsType\":{\"description\":\"Filesystem type to mount. Must be a filesystem type supported by the host operating system. Examples: \\\"ext4\\\", \\\"xfs\\\", \\\"ntfs\\\". Implicitly inferred to be \\\"ext4\\\" if unspecified. More info: https://examples.k8s.io/mysql-cinder-pd/README.md\",\"type\":\"string\"},\"readOnly\":{\"description\":\"Optional: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. More info: https://examples.k8s.io/mysql-cinder-pd/README.md\",\"type\":\"boolean\"},\"secretRef\":{\"description\":\"Optional: points to a secret object containing parameters used to connect to OpenStack.\",\"properties\":{\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"}},\"type\":\"object\"},\"volumeID\":{\"description\":\"volume id used to identify the volume in cinder. More info: https://examples.k8s.io/mysql-cinder-pd/README.md\",\"type\":\"string\"}},\"required\":[\"volumeID\"],\"type\":\"object\"},\"configMap\":{\"description\":\"ConfigMap represents a configMap that should populate this volume\",\"properties\":{\"defaultMode\":{\"description\":\"Optional: mode bits to use on created files by default. Must be a value between 0 and 0777. Defaults to 0644. Directories within the path are not affected by this setting. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.\",\"format\":\"int32\",\"type\":\"integer\"},\"items\":{\"description\":\"If unspecified, each key-value pair in the Data field of the referenced ConfigMap will be projected into the volume as a file whose name is the key and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the ConfigMap, the volume setup will error unless it is marked optional. Paths must be relative and may not contain the '..' path or start with '..'.\",\"items\":{\"description\":\"Maps a string key to a path within a volume.\",\"properties\":{\"key\":{\"description\":\"The key to project.\",\"type\":\"string\"},\"mode\":{\"description\":\"Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.\",\"format\":\"int32\",\"type\":\"integer\"},\"path\":{\"description\":\"The relative path of the file to map the key to. May not be an absolute path. May not contain the path element '..'. May not start with the string '..'.\",\"type\":\"string\"}},\"required\":[\"key\",\"path\"],\"type\":\"object\"},\"type\":\"array\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the ConfigMap or its keys must be defined\",\"type\":\"boolean\"}},\"type\":\"object\"},\"csi\":{\"description\":\"CSI (Container Storage Interface) represents storage that is handled by an external CSI driver (Alpha feature).\",\"properties\":{\"driver\":{\"description\":\"Driver is the name of the CSI driver that handles this volume. Consult with your admin for the correct name as registered in the cluster.\",\"type\":\"string\"},\"fsType\":{\"description\":\"Filesystem type to mount. Ex. \\\"ext4\\\", \\\"xfs\\\", \\\"ntfs\\\". If not provided, the empty value is passed to the associated CSI driver which will determine the default filesystem to apply.\",\"type\":\"string\"},\"nodePublishSecretRef\":{\"description\":\"NodePublishSecretRef is a reference to the secret object containing sensitive information to pass to the CSI driver to complete the CSI NodePublishVolume and NodeUnpublishVolume calls. This field is optional, and may be empty if no secret is required. If the secret object contains more than one secret, all secret references are passed.\",\"properties\":{\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"}},\"type\":\"object\"},\"readOnly\":{\"description\":\"Specifies a read-only configuration for the volume. Defaults to false (read/write).\",\"type\":\"boolean\"},\"volumeAttributes\":{\"additionalProperties\":{\"type\":\"string\"},\"description\":\"VolumeAttributes stores driver-specific properties that are passed to the CSI driver. Consult your driver's documentation for supported values.\",\"type\":\"object\"}},\"required\":[\"driver\"],\"type\":\"object\"},\"downwardAPI\":{\"description\":\"DownwardAPI represents downward API about the pod that should populate this volume\",\"properties\":{\"defaultMode\":{\"description\":\"Optional: mode bits to use on created files by default. Must be a value between 0 and 0777. Defaults to 0644. Directories within the path are not affected by this setting. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.\",\"format\":\"int32\",\"type\":\"integer\"},\"items\":{\"description\":\"Items is a list of downward API volume file\",\"items\":{\"description\":\"DownwardAPIVolumeFile represents information to create the file containing the pod field\",\"properties\":{\"fieldRef\":{\"description\":\"Required: Selects a field of the pod: only annotations, labels, name and namespace are supported.\",\"properties\":{\"apiVersion\":{\"description\":\"Version of the schema the FieldPath is written in terms of, defaults to \\\"v1\\\".\",\"type\":\"string\"},\"fieldPath\":{\"description\":\"Path of the field to select in the specified API version.\",\"type\":\"string\"}},\"required\":[\"fieldPath\"],\"type\":\"object\"},\"mode\":{\"description\":\"Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.\",\"format\":\"int32\",\"type\":\"integer\"},\"path\":{\"description\":\"Required: Path is the relative path name of the file to be created. Must not be absolute or contain the '..' path. Must be utf-8 encoded. The first item of the relative path must not start with '..'\",\"type\":\"string\"},\"resourceFieldRef\":{\"description\":\"Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported.\",\"properties\":{\"containerName\":{\"description\":\"Container name: required for volumes, optional for env vars\",\"type\":\"string\"},\"divisor\":{\"description\":\"Specifies the output format of the exposed resources, defaults to \\\"1\\\"\",\"type\":\"string\"},\"resource\":{\"description\":\"Required: resource to select\",\"type\":\"string\"}},\"required\":[\"resource\"],\"type\":\"object\"}},\"required\":[\"path\"],\"type\":\"object\"},\"type\":\"array\"}},\"type\":\"object\"},\"emptyDir\":{\"description\":\"EmptyDir represents a temporary directory that shares a pod's lifetime. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir\",\"properties\":{\"medium\":{\"description\":\"What type of storage medium should back this directory. The default is \\\"\\\" which means to use the node's default medium. Must be an empty string (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir\",\"type\":\"string\"},\"sizeLimit\":{\"description\":\"Total amount of local storage required for this EmptyDir volume. The size limit is also applicable for memory medium. The maximum usage on memory medium EmptyDir would be the minimum value between the SizeLimit specified here and the sum of memory limits of all containers in a pod. The default is nil which means that the limit is undefined. More info: http://kubernetes.io/docs/user-guide/volumes#emptydir\",\"type\":\"string\"}},\"type\":\"object\"},\"fc\":{\"description\":\"FC represents a Fibre Channel resource that is attached to a kubelet's host machine and then exposed to the pod.\",\"properties\":{\"fsType\":{\"description\":\"Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \\\"ext4\\\", \\\"xfs\\\", \\\"ntfs\\\". Implicitly inferred to be \\\"ext4\\\" if unspecified. TODO: how do we prevent errors in the filesystem from compromising the machine\",\"type\":\"string\"},\"lun\":{\"description\":\"Optional: FC target lun number\",\"format\":\"int32\",\"type\":\"integer\"},\"readOnly\":{\"description\":\"Optional: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.\",\"type\":\"boolean\"},\"targetWWNs\":{\"description\":\"Optional: FC target worldwide names (WWNs)\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"wwids\":{\"description\":\"Optional: FC volume world wide identifiers (wwids) Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"type\":\"object\"},\"flexVolume\":{\"description\":\"FlexVolume represents a generic volume resource that is provisioned/attached using an exec based plugin.\",\"properties\":{\"driver\":{\"description\":\"Driver is the name of the driver to use for this volume.\",\"type\":\"string\"},\"fsType\":{\"description\":\"Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \\\"ext4\\\", \\\"xfs\\\", \\\"ntfs\\\". The default filesystem depends on FlexVolume script.\",\"type\":\"string\"},\"options\":{\"additionalProperties\":{\"type\":\"string\"},\"description\":\"Optional: Extra command options if any.\",\"type\":\"object\"},\"readOnly\":{\"description\":\"Optional: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.\",\"type\":\"boolean\"},\"secretRef\":{\"description\":\"Optional: SecretRef is reference to the secret object containing sensitive information to pass to the plugin scripts. This may be empty if no secret object is specified. If the secret object contains more than one secret, all secrets are passed to the plugin scripts.\",\"properties\":{\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"}},\"type\":\"object\"}},\"required\":[\"driver\"],\"type\":\"object\"},\"flocker\":{\"description\":\"Flocker represents a Flocker volume attached to a kubelet's host machine. This depends on the Flocker control service being running\",\"properties\":{\"datasetName\":{\"description\":\"Name of the dataset stored as metadata -\\u003e name on the dataset for Flocker should be considered as deprecated\",\"type\":\"string\"},\"datasetUUID\":{\"description\":\"UUID of the dataset. This is unique identifier of a Flocker dataset\",\"type\":\"string\"}},\"type\":\"object\"},\"gcePersistentDisk\":{\"description\":\"GCEPersistentDisk represents a GCE Disk resource that is attached to a kubelet's host machine and then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk\",\"properties\":{\"fsType\":{\"description\":\"Filesystem type of the volume that you want to mount. Tip: Ensure that the filesystem type is supported by the host operating system. Examples: \\\"ext4\\\", \\\"xfs\\\", \\\"ntfs\\\". Implicitly inferred to be \\\"ext4\\\" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk TODO: how do we prevent errors in the filesystem from compromising the machine\",\"type\":\"string\"},\"partition\":{\"description\":\"The partition in the volume that you want to mount. If omitted, the default is to mount by volume name. Examples: For volume /dev/sda1, you specify the partition as \\\"1\\\". Similarly, the volume partition for /dev/sda is \\\"0\\\" (or you can leave the property empty). More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk\",\"format\":\"int32\",\"type\":\"integer\"},\"pdName\":{\"description\":\"Unique name of the PD resource in GCE. Used to identify the disk in GCE. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk\",\"type\":\"string\"},\"readOnly\":{\"description\":\"ReadOnly here will force the ReadOnly setting in VolumeMounts. Defaults to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk\",\"type\":\"boolean\"}},\"required\":[\"pdName\"],\"type\":\"object\"},\"gitRepo\":{\"description\":\"GitRepo represents a git repository at a particular revision. DEPRECATED: GitRepo is deprecated. To provision a container with a git repo, mount an EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir into the Pod's container.\",\"properties\":{\"directory\":{\"description\":\"Target directory name. Must not contain or start with '..'. If '.' is supplied, the volume directory will be the git repository. Otherwise, if specified, the volume will contain the git repository in the subdirectory with the given name.\",\"type\":\"string\"},\"repository\":{\"description\":\"Repository URL\",\"type\":\"string\"},\"revision\":{\"description\":\"Commit hash for the specified revision.\",\"type\":\"string\"}},\"required\":[\"repository\"],\"type\":\"object\"},\"glusterfs\":{\"description\":\"Glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. More info: https://examples.k8s.io/volumes/glusterfs/README.md\",\"properties\":{\"endpoints\":{\"description\":\"EndpointsName is the endpoint name that details Glusterfs topology. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod\",\"type\":\"string\"},\"path\":{\"description\":\"Path is the Glusterfs volume path. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod\",\"type\":\"string\"},\"readOnly\":{\"description\":\"ReadOnly here will force the Glusterfs volume to be mounted with read-only permissions. Defaults to false. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod\",\"type\":\"boolean\"}},\"required\":[\"endpoints\",\"path\"],\"type\":\"object\"},\"hostPath\":{\"description\":\"HostPath represents a pre-existing file or directory on the host machine that is directly exposed to the container. This is generally used for system agents or other privileged things that are allowed to see the host machine. Most containers will NOT need this. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath --- TODO(jonesdl) We need to restrict who can use host directory mounts and who can/can not mount host directories as read/write.\",\"properties\":{\"path\":{\"description\":\"Path of the directory on the host. If the path is a symlink, it will follow the link to the real path. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath\",\"type\":\"string\"},\"type\":{\"description\":\"Type for HostPath Volume Defaults to \\\"\\\" More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath\",\"type\":\"string\"}},\"required\":[\"path\"],\"type\":\"object\"},\"iscsi\":{\"description\":\"ISCSI represents an ISCSI Disk resource that is attached to a kubelet's host machine and then exposed to the pod. More info: https://examples.k8s.io/volumes/iscsi/README.md\",\"properties\":{\"chapAuthDiscovery\":{\"description\":\"whether support iSCSI Discovery CHAP authentication\",\"type\":\"boolean\"},\"chapAuthSession\":{\"description\":\"whether support iSCSI Session CHAP authentication\",\"type\":\"boolean\"},\"fsType\":{\"description\":\"Filesystem type of the volume that you want to mount. Tip: Ensure that the filesystem type is supported by the host operating system. Examples: \\\"ext4\\\", \\\"xfs\\\", \\\"ntfs\\\". Implicitly inferred to be \\\"ext4\\\" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi TODO: how do we prevent errors in the filesystem from compromising the machine\",\"type\":\"string\"},\"initiatorName\":{\"description\":\"Custom iSCSI Initiator Name. If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface \\u003ctarget portal\\u003e:\\u003cvolume name\\u003e will be created for the connection.\",\"type\":\"string\"},\"iqn\":{\"description\":\"Target iSCSI Qualified Name.\",\"type\":\"string\"},\"iscsiInterface\":{\"description\":\"iSCSI Interface Name that uses an iSCSI transport. Defaults to 'default' (tcp).\",\"type\":\"string\"},\"lun\":{\"description\":\"iSCSI Target Lun number.\",\"format\":\"int32\",\"type\":\"integer\"},\"portals\":{\"description\":\"iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port is other than default (typically TCP ports 860 and 3260).\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"readOnly\":{\"description\":\"ReadOnly here will force the ReadOnly setting in VolumeMounts. Defaults to false.\",\"type\":\"boolean\"},\"secretRef\":{\"description\":\"CHAP Secret for iSCSI target and initiator authentication\",\"properties\":{\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"}},\"type\":\"object\"},\"targetPortal\":{\"description\":\"iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port is other than default (typically TCP ports 860 and 3260).\",\"type\":\"string\"}},\"required\":[\"iqn\",\"lun\",\"targetPortal\"],\"type\":\"object\"},\"name\":{\"description\":\"Volume's name. Must be a DNS_LABEL and unique within the pod. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\",\"type\":\"string\"},\"nfs\":{\"description\":\"NFS represents an NFS mount on the host that shares a pod's lifetime More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs\",\"properties\":{\"path\":{\"description\":\"Path that is exported by the NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs\",\"type\":\"string\"},\"readOnly\":{\"description\":\"ReadOnly here will force the NFS export to be mounted with read-only permissions. Defaults to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs\",\"type\":\"boolean\"},\"server\":{\"description\":\"Server is the hostname or IP address of the NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs\",\"type\":\"string\"}},\"required\":[\"path\",\"server\"],\"type\":\"object\"},\"persistentVolumeClaim\":{\"description\":\"PersistentVolumeClaimVolumeSource represents a reference to a PersistentVolumeClaim in the same namespace. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims\",\"properties\":{\"claimName\":{\"description\":\"ClaimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims\",\"type\":\"string\"},\"readOnly\":{\"description\":\"Will force the ReadOnly setting in VolumeMounts. Default false.\",\"type\":\"boolean\"}},\"required\":[\"claimName\"],\"type\":\"object\"},\"photonPersistentDisk\":{\"description\":\"PhotonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine\",\"properties\":{\"fsType\":{\"description\":\"Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \\\"ext4\\\", \\\"xfs\\\", \\\"ntfs\\\". Implicitly inferred to be \\\"ext4\\\" if unspecified.\",\"type\":\"string\"},\"pdID\":{\"description\":\"ID that identifies Photon Controller persistent disk\",\"type\":\"string\"}},\"required\":[\"pdID\"],\"type\":\"object\"},\"portworxVolume\":{\"description\":\"PortworxVolume represents a portworx volume attached and mounted on kubelets host machine\",\"properties\":{\"fsType\":{\"description\":\"FSType represents the filesystem type to mount Must be a filesystem type supported by the host operating system. Ex. \\\"ext4\\\", \\\"xfs\\\". Implicitly inferred to be \\\"ext4\\\" if unspecified.\",\"type\":\"string\"},\"readOnly\":{\"description\":\"Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.\",\"type\":\"boolean\"},\"volumeID\":{\"description\":\"VolumeID uniquely identifies a Portworx volume\",\"type\":\"string\"}},\"required\":[\"volumeID\"],\"type\":\"object\"},\"projected\":{\"description\":\"Items for all in one resources secrets, configmaps, and downward API\",\"properties\":{\"defaultMode\":{\"description\":\"Mode bits to use on created files by default. Must be a value between 0 and 0777. Directories within the path are not affected by this setting. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.\",\"format\":\"int32\",\"type\":\"integer\"},\"sources\":{\"description\":\"list of volume projections\",\"items\":{\"description\":\"Projection that may be projected along with other supported volume types\",\"properties\":{\"configMap\":{\"description\":\"information about the configMap data to project\",\"properties\":{\"items\":{\"description\":\"If unspecified, each key-value pair in the Data field of the referenced ConfigMap will be projected into the volume as a file whose name is the key and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the ConfigMap, the volume setup will error unless it is marked optional. Paths must be relative and may not contain the '..' path or start with '..'.\",\"items\":{\"description\":\"Maps a string key to a path within a volume.\",\"properties\":{\"key\":{\"description\":\"The key to project.\",\"type\":\"string\"},\"mode\":{\"description\":\"Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.\",\"format\":\"int32\",\"type\":\"integer\"},\"path\":{\"description\":\"The relative path of the file to map the key to. May not be an absolute path. May not contain the path element '..'. May not start with the string '..'.\",\"type\":\"string\"}},\"required\":[\"key\",\"path\"],\"type\":\"object\"},\"type\":\"array\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the ConfigMap or its keys must be defined\",\"type\":\"boolean\"}},\"type\":\"object\"},\"downwardAPI\":{\"description\":\"information about the downwardAPI data to project\",\"properties\":{\"items\":{\"description\":\"Items is a list of DownwardAPIVolume file\",\"items\":{\"description\":\"DownwardAPIVolumeFile represents information to create the file containing the pod field\",\"properties\":{\"fieldRef\":{\"description\":\"Required: Selects a field of the pod: only annotations, labels, name and namespace are supported.\",\"properties\":{\"apiVersion\":{\"description\":\"Version of the schema the FieldPath is written in terms of, defaults to \\\"v1\\\".\",\"type\":\"string\"},\"fieldPath\":{\"description\":\"Path of the field to select in the specified API version.\",\"type\":\"string\"}},\"required\":[\"fieldPath\"],\"type\":\"object\"},\"mode\":{\"description\":\"Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.\",\"format\":\"int32\",\"type\":\"integer\"},\"path\":{\"description\":\"Required: Path is the relative path name of the file to be created. Must not be absolute or contain the '..' path. Must be utf-8 encoded. The first item of the relative path must not start with '..'\",\"type\":\"string\"},\"resourceFieldRef\":{\"description\":\"Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported.\",\"properties\":{\"containerName\":{\"description\":\"Container name: required for volumes, optional for env vars\",\"type\":\"string\"},\"divisor\":{\"description\":\"Specifies the output format of the exposed resources, defaults to \\\"1\\\"\",\"type\":\"string\"},\"resource\":{\"description\":\"Required: resource to select\",\"type\":\"string\"}},\"required\":[\"resource\"],\"type\":\"object\"}},\"required\":[\"path\"],\"type\":\"object\"},\"type\":\"array\"}},\"type\":\"object\"},\"secret\":{\"description\":\"information about the secret data to project\",\"properties\":{\"items\":{\"description\":\"If unspecified, each key-value pair in the Data field of the referenced Secret will be projected into the volume as a file whose name is the key and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the Secret, the volume setup will error unless it is marked optional. Paths must be relative and may not contain the '..' path or start with '..'.\",\"items\":{\"description\":\"Maps a string key to a path within a volume.\",\"properties\":{\"key\":{\"description\":\"The key to project.\",\"type\":\"string\"},\"mode\":{\"description\":\"Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.\",\"format\":\"int32\",\"type\":\"integer\"},\"path\":{\"description\":\"The relative path of the file to map the key to. May not be an absolute path. May not contain the path element '..'. May not start with the string '..'.\",\"type\":\"string\"}},\"required\":[\"key\",\"path\"],\"type\":\"object\"},\"type\":\"array\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or its key must be defined\",\"type\":\"boolean\"}},\"type\":\"object\"},\"serviceAccountToken\":{\"description\":\"information about the serviceAccountToken data to project\",\"properties\":{\"audience\":{\"description\":\"Audience is the intended audience of the token. A recipient of a token must identify itself with an identifier specified in the audience of the token, and otherwise should reject the token. The audience defaults to the identifier of the apiserver.\",\"type\":\"string\"},\"expirationSeconds\":{\"description\":\"ExpirationSeconds is the requested duration of validity of the service account token. As the token approaches expiration, the kubelet volume plugin will proactively rotate the service account token. The kubelet will start trying to rotate the token if the token is older than 80 percent of its time to live or if the token is older than 24 hours.Defaults to 1 hour and must be at least 10 minutes.\",\"format\":\"int64\",\"type\":\"integer\"},\"path\":{\"description\":\"Path is the path relative to the mount point of the file to project the token into.\",\"type\":\"string\"}},\"required\":[\"path\"],\"type\":\"object\"}},\"type\":\"object\"},\"type\":\"array\"}},\"required\":[\"sources\"],\"type\":\"object\"},\"quobyte\":{\"description\":\"Quobyte represents a Quobyte mount on the host that shares a pod's lifetime\",\"properties\":{\"group\":{\"description\":\"Group to map volume access to Default is no group\",\"type\":\"string\"},\"readOnly\":{\"description\":\"ReadOnly here will force the Quobyte volume to be mounted with read-only permissions. Defaults to false.\",\"type\":\"boolean\"},\"registry\":{\"description\":\"Registry represents a single or multiple Quobyte Registry services specified as a string as host:port pair (multiple entries are separated with commas) which acts as the central registry for volumes\",\"type\":\"string\"},\"tenant\":{\"description\":\"Tenant owning the given Quobyte volume in the Backend Used with dynamically provisioned Quobyte volumes, value is set by the plugin\",\"type\":\"string\"},\"user\":{\"description\":\"User to map volume access to Defaults to serivceaccount user\",\"type\":\"string\"},\"volume\":{\"description\":\"Volume is a string that references an already created Quobyte volume by name.\",\"type\":\"string\"}},\"required\":[\"registry\",\"volume\"],\"type\":\"object\"},\"rbd\":{\"description\":\"RBD represents a Rados Block Device mount on the host that shares a pod's lifetime. More info: https://examples.k8s.io/volumes/rbd/README.md\",\"properties\":{\"fsType\":{\"description\":\"Filesystem type of the volume that you want to mount. Tip: Ensure that the filesystem type is supported by the host operating system. Examples: \\\"ext4\\\", \\\"xfs\\\", \\\"ntfs\\\". Implicitly inferred to be \\\"ext4\\\" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd TODO: how do we prevent errors in the filesystem from compromising the machine\",\"type\":\"string\"},\"image\":{\"description\":\"The rados image name. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it\",\"type\":\"string\"},\"keyring\":{\"description\":\"Keyring is the path to key ring for RBDUser. Default is /etc/ceph/keyring. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it\",\"type\":\"string\"},\"monitors\":{\"description\":\"A collection of Ceph monitors. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"pool\":{\"description\":\"The rados pool name. Default is rbd. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it\",\"type\":\"string\"},\"readOnly\":{\"description\":\"ReadOnly here will force the ReadOnly setting in VolumeMounts. Defaults to false. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it\",\"type\":\"boolean\"},\"secretRef\":{\"description\":\"SecretRef is name of the authentication secret for RBDUser. If provided overrides keyring. Default is nil. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it\",\"properties\":{\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"}},\"type\":\"object\"},\"user\":{\"description\":\"The rados user name. Default is admin. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it\",\"type\":\"string\"}},\"required\":[\"image\",\"monitors\"],\"type\":\"object\"},\"scaleIO\":{\"description\":\"ScaleIO represents a ScaleIO persistent volume attached and mounted on Kubernetes nodes.\",\"properties\":{\"fsType\":{\"description\":\"Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \\\"ext4\\\", \\\"xfs\\\", \\\"ntfs\\\". Default is \\\"xfs\\\".\",\"type\":\"string\"},\"gateway\":{\"description\":\"The host address of the ScaleIO API Gateway.\",\"type\":\"string\"},\"protectionDomain\":{\"description\":\"The name of the ScaleIO Protection Domain for the configured storage.\",\"type\":\"string\"},\"readOnly\":{\"description\":\"Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.\",\"type\":\"boolean\"},\"secretRef\":{\"description\":\"SecretRef references to the secret for ScaleIO user and other sensitive information. If this is not provided, Login operation will fail.\",\"properties\":{\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"}},\"type\":\"object\"},\"sslEnabled\":{\"description\":\"Flag to enable/disable SSL communication with Gateway, default false\",\"type\":\"boolean\"},\"storageMode\":{\"description\":\"Indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. Default is ThinProvisioned.\",\"type\":\"string\"},\"storagePool\":{\"description\":\"The ScaleIO Storage Pool associated with the protection domain.\",\"type\":\"string\"},\"system\":{\"description\":\"The name of the storage system as configured in ScaleIO.\",\"type\":\"string\"},\"volumeName\":{\"description\":\"The name of a volume already created in the ScaleIO system that is associated with this volume source.\",\"type\":\"string\"}},\"required\":[\"gateway\",\"secretRef\",\"system\"],\"type\":\"object\"},\"secret\":{\"description\":\"Secret represents a secret that should populate this volume. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret\",\"properties\":{\"defaultMode\":{\"description\":\"Optional: mode bits to use on created files by default. Must be a value between 0 and 0777. Defaults to 0644. Directories within the path are not affected by this setting. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.\",\"format\":\"int32\",\"type\":\"integer\"},\"items\":{\"description\":\"If unspecified, each key-value pair in the Data field of the referenced Secret will be projected into the volume as a file whose name is the key and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the Secret, the volume setup will error unless it is marked optional. Paths must be relative and may not contain the '..' path or start with '..'.\",\"items\":{\"description\":\"Maps a string key to a path within a volume.\",\"properties\":{\"key\":{\"description\":\"The key to project.\",\"type\":\"string\"},\"mode\":{\"description\":\"Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.\",\"format\":\"int32\",\"type\":\"integer\"},\"path\":{\"description\":\"The relative path of the file to map the key to. May not be an absolute path. May not contain the path element '..'. May not start with the string '..'.\",\"type\":\"string\"}},\"required\":[\"key\",\"path\"],\"type\":\"object\"},\"type\":\"array\"},\"optional\":{\"description\":\"Specify whether the Secret or its keys must be defined\",\"type\":\"boolean\"},\"secretName\":{\"description\":\"Name of the secret in the pod's namespace to use. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret\",\"type\":\"string\"}},\"type\":\"object\"},\"storageos\":{\"description\":\"StorageOS represents a StorageOS volume attached and mounted on Kubernetes nodes.\",\"properties\":{\"fsType\":{\"description\":\"Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \\\"ext4\\\", \\\"xfs\\\", \\\"ntfs\\\". Implicitly inferred to be \\\"ext4\\\" if unspecified.\",\"type\":\"string\"},\"readOnly\":{\"description\":\"Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.\",\"type\":\"boolean\"},\"secretRef\":{\"description\":\"SecretRef specifies the secret to use for obtaining the StorageOS API credentials. If not specified, default values will be attempted.\",\"properties\":{\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?\",\"type\":\"string\"}},\"type\":\"object\"},\"volumeName\":{\"description\":\"VolumeName is the human-readable name of the StorageOS volume. Volume names are only unique within a namespace.\",\"type\":\"string\"},\"volumeNamespace\":{\"description\":\"VolumeNamespace specifies the scope of the volume within StorageOS. If no namespace is specified then the Pod's namespace will be used. This allows the Kubernetes name scoping to be mirrored within StorageOS for tighter integration. Set VolumeName to any name to override the default behaviour. Set to \\\"default\\\" if you are not using namespaces within StorageOS. Namespaces that do not pre-exist within StorageOS will be created.\",\"type\":\"string\"}},\"type\":\"object\"},\"vsphereVolume\":{\"description\":\"VsphereVolume represents a vSphere volume attached and mounted on kubelets host machine\",\"properties\":{\"fsType\":{\"description\":\"Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \\\"ext4\\\", \\\"xfs\\\", \\\"ntfs\\\". Implicitly inferred to be \\\"ext4\\\" if unspecified.\",\"type\":\"string\"},\"storagePolicyID\":{\"description\":\"Storage Policy Based Management (SPBM) profile ID associated with the StoragePolicyName.\",\"type\":\"string\"},\"storagePolicyName\":{\"description\":\"Storage Policy Based Management (SPBM) profile name.\",\"type\":\"string\"},\"volumePath\":{\"description\":\"Path that identifies vSphere volume vmdk\",\"type\":\"string\"}},\"required\":[\"volumePath\"],\"type\":\"object\"}},\"required\":[\"name\"],\"type\":\"object\"},\"type\":\"array\"},\"walCompression\":{\"description\":\"Enable compression of the write-ahead log using Snappy. This flag is only available in versions of Prometheus \\u003e= 2.11.0.\",\"type\":\"boolean\"}},\"type\":\"object\"},\"status\":{\"description\":\"Most recent observed status of the Prometheus cluster. Read-only. Not included when requesting from the apiserver, only from the Prometheus Operator API itself. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status\",\"properties\":{\"availableReplicas\":{\"description\":\"Total number of available pods (ready for at least minReadySeconds) targeted by this Prometheus deployment.\",\"format\":\"int32\",\"type\":\"integer\"},\"paused\":{\"description\":\"Represents whether any actions on the underlying managed objects are being performed. Only delete actions will be performed.\",\"type\":\"boolean\"},\"replicas\":{\"description\":\"Total number of non-terminated pods targeted by this Prometheus deployment (their labels match the selector).\",\"format\":\"int32\",\"type\":\"integer\"},\"unavailableReplicas\":{\"description\":\"Total number of unavailable pods targeted by this Prometheus deployment.\",\"format\":\"int32\",\"type\":\"integer\"},\"updatedReplicas\":{\"description\":\"Total number of non-terminated pods targeted by this Prometheus deployment that have the desired version spec.\",\"format\":\"int32\",\"type\":\"integer\"}},\"required\":[\"availableReplicas\",\"paused\",\"replicas\",\"unavailableReplicas\",\"updatedReplicas\"],\"type\":\"object\"}},\"required\":[\"spec\"],\"type\":\"object\"}},\"version\":\"v1\",\"versions\":[{\"name\":\"v1\",\"served\":true,\"storage\":true}]},\"status\":{\"acceptedNames\":{\"kind\":\"\",\"plural\":\"\"},\"conditions\":[],\"storedVersions\":[]}}\n" }, "creationTimestamp": "2020-05-04T19:47:40Z", "generation": 1, "name": "prometheuses.monitoring.coreos.com", "resourceVersion": "1013", "selfLink": "/apis/apiextensions.k8s.io/v1/customresourcedefinitions/prometheuses.monitoring.coreos.com", "uid": "ff0864c8-712f-4b03-844c-339abc00cb15" }, "spec": { "conversion": { "strategy": "None" }, "group": "monitoring.coreos.com", "names": { "kind": "Prometheus", "listKind": "PrometheusList", "plural": "prometheuses", "singular": "prometheus" }, "scope": "Namespaced", "versions": [ { "additionalPrinterColumns": [ { "description": "The version of Prometheus", "jsonPath": ".spec.version", "name": "Version", "type": "string" }, { "description": "The desired replicas number of Prometheuses", "jsonPath": ".spec.replicas", "name": "Replicas", "type": "integer" }, { "jsonPath": ".metadata.creationTimestamp", "name": "Age", "type": "date" } ], "name": "v1", "schema": { "openAPIV3Schema": { "description": "Prometheus defines a Prometheus deployment.", "properties": { "apiVersion": { "description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", "type": "string" }, "kind": { "description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", "type": "string" }, "metadata": { "type": "object" }, "spec": { "description": "Specification of the desired behavior of the Prometheus cluster. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status", "properties": { "additionalAlertManagerConfigs": { "description": "AdditionalAlertManagerConfigs allows specifying a key of a Secret containing additional Prometheus AlertManager configurations. AlertManager configurations specified are appended to the configurations generated by the Prometheus Operator. Job configurations specified must have the form as specified in the official Prometheus documentation: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#alertmanager_config. As AlertManager configs are appended, the user is responsible to make sure it is valid. Note that using this feature may expose the possibility to break upgrades of Prometheus. It is advised to review Prometheus release notes to ensure that no incompatible AlertManager configs are going to break Prometheus after the upgrade.", "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" }, "optional": { "description": "Specify whether the Secret or its key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" }, "additionalAlertRelabelConfigs": { "description": "AdditionalAlertRelabelConfigs allows specifying a key of a Secret containing additional Prometheus alert relabel configurations. Alert relabel configurations specified are appended to the configurations generated by the Prometheus Operator. Alert relabel configurations specified must have the form as specified in the official Prometheus documentation: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#alert_relabel_configs. As alert relabel configs are appended, the user is responsible to make sure it is valid. Note that using this feature may expose the possibility to break upgrades of Prometheus. It is advised to review Prometheus release notes to ensure that no incompatible alert relabel configs are going to break Prometheus after the upgrade.", "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" }, "optional": { "description": "Specify whether the Secret or its key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" }, "additionalScrapeConfigs": { "description": "AdditionalScrapeConfigs allows specifying a key of a Secret containing additional Prometheus scrape configurations. Scrape configurations specified are appended to the configurations generated by the Prometheus Operator. Job configurations specified must have the form as specified in the official Prometheus documentation: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config. As scrape configs are appended, the user is responsible to make sure it is valid. Note that using this feature may expose the possibility to break upgrades of Prometheus. It is advised to review Prometheus release notes to ensure that no incompatible scrape configs are going to break Prometheus after the upgrade.", "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" }, "optional": { "description": "Specify whether the Secret or its key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" }, "affinity": { "description": "If specified, the pod's scheduling constraints.", "properties": { "nodeAffinity": { "description": "Describes node affinity scheduling rules for the pod.", "properties": { "preferredDuringSchedulingIgnoredDuringExecution": { "description": "The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding \"weight\" to the sum if the node matches the corresponding matchExpressions; the node(s) with the highest sum are the most preferred.", "items": { "description": "An empty preferred scheduling term matches all objects with implicit weight 0 (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op).", "properties": { "preference": { "description": "A node selector term, associated with the corresponding weight.", "properties": { "matchExpressions": { "description": "A list of node selector requirements by node's labels.", "items": { "description": "A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "properties": { "key": { "description": "The label key that the selector applies to.", "type": "string" }, "operator": { "description": "Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.", "type": "string" }, "values": { "description": "An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.", "items": { "type": "string" }, "type": "array" } }, "required": [ "key", "operator" ], "type": "object" }, "type": "array" }, "matchFields": { "description": "A list of node selector requirements by node's fields.", "items": { "description": "A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "properties": { "key": { "description": "The label key that the selector applies to.", "type": "string" }, "operator": { "description": "Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.", "type": "string" }, "values": { "description": "An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.", "items": { "type": "string" }, "type": "array" } }, "required": [ "key", "operator" ], "type": "object" }, "type": "array" } }, "type": "object" }, "weight": { "description": "Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100.", "format": "int32", "type": "integer" } }, "required": [ "preference", "weight" ], "type": "object" }, "type": "array" }, "requiredDuringSchedulingIgnoredDuringExecution": { "description": "If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to an update), the system may or may not try to eventually evict the pod from its node.", "properties": { "nodeSelectorTerms": { "description": "Required. A list of node selector terms. The terms are ORed.", "items": { "description": "A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm.", "properties": { "matchExpressions": { "description": "A list of node selector requirements by node's labels.", "items": { "description": "A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "properties": { "key": { "description": "The label key that the selector applies to.", "type": "string" }, "operator": { "description": "Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.", "type": "string" }, "values": { "description": "An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.", "items": { "type": "string" }, "type": "array" } }, "required": [ "key", "operator" ], "type": "object" }, "type": "array" }, "matchFields": { "description": "A list of node selector requirements by node's fields.", "items": { "description": "A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "properties": { "key": { "description": "The label key that the selector applies to.", "type": "string" }, "operator": { "description": "Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.", "type": "string" }, "values": { "description": "An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.", "items": { "type": "string" }, "type": "array" } }, "required": [ "key", "operator" ], "type": "object" }, "type": "array" } }, "type": "object" }, "type": "array" } }, "required": [ "nodeSelectorTerms" ], "type": "object" } }, "type": "object" }, "podAffinity": { "description": "Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)).", "properties": { "preferredDuringSchedulingIgnoredDuringExecution": { "description": "The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding \"weight\" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred.", "items": { "description": "The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s)", "properties": { "podAffinityTerm": { "description": "Required. A pod affinity term, associated with the corresponding weight.", "properties": { "labelSelector": { "description": "A label query over a set of resources, in this case pods.", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "items": { "type": "string" }, "type": "array" } }, "required": [ "key", "operator" ], "type": "object" }, "type": "array" }, "matchLabels": { "additionalProperties": { "type": "string" }, "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object" } }, "type": "object" }, "namespaces": { "description": "namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \"this pod's namespace\"", "items": { "type": "string" }, "type": "array" }, "topologyKey": { "description": "This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.", "type": "string" } }, "required": [ "topologyKey" ], "type": "object" }, "weight": { "description": "weight associated with matching the corresponding podAffinityTerm, in the range 1-100.", "format": "int32", "type": "integer" } }, "required": [ "podAffinityTerm", "weight" ], "type": "object" }, "type": "array" }, "requiredDuringSchedulingIgnoredDuringExecution": { "description": "If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied.", "items": { "description": "Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key \u003ctopologyKey\u003e matches that of any node on which a pod of the set of pods is running", "properties": { "labelSelector": { "description": "A label query over a set of resources, in this case pods.", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "items": { "type": "string" }, "type": "array" } }, "required": [ "key", "operator" ], "type": "object" }, "type": "array" }, "matchLabels": { "additionalProperties": { "type": "string" }, "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object" } }, "type": "object" }, "namespaces": { "description": "namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \"this pod's namespace\"", "items": { "type": "string" }, "type": "array" }, "topologyKey": { "description": "This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.", "type": "string" } }, "required": [ "topologyKey" ], "type": "object" }, "type": "array" } }, "type": "object" }, "podAntiAffinity": { "description": "Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)).", "properties": { "preferredDuringSchedulingIgnoredDuringExecution": { "description": "The scheduler will prefer to schedule pods to nodes that satisfy the anti-affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding \"weight\" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred.", "items": { "description": "The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s)", "properties": { "podAffinityTerm": { "description": "Required. A pod affinity term, associated with the corresponding weight.", "properties": { "labelSelector": { "description": "A label query over a set of resources, in this case pods.", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "items": { "type": "string" }, "type": "array" } }, "required": [ "key", "operator" ], "type": "object" }, "type": "array" }, "matchLabels": { "additionalProperties": { "type": "string" }, "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object" } }, "type": "object" }, "namespaces": { "description": "namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \"this pod's namespace\"", "items": { "type": "string" }, "type": "array" }, "topologyKey": { "description": "This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.", "type": "string" } }, "required": [ "topologyKey" ], "type": "object" }, "weight": { "description": "weight associated with matching the corresponding podAffinityTerm, in the range 1-100.", "format": "int32", "type": "integer" } }, "required": [ "podAffinityTerm", "weight" ], "type": "object" }, "type": "array" }, "requiredDuringSchedulingIgnoredDuringExecution": { "description": "If the anti-affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the anti-affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied.", "items": { "description": "Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key \u003ctopologyKey\u003e matches that of any node on which a pod of the set of pods is running", "properties": { "labelSelector": { "description": "A label query over a set of resources, in this case pods.", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "items": { "type": "string" }, "type": "array" } }, "required": [ "key", "operator" ], "type": "object" }, "type": "array" }, "matchLabels": { "additionalProperties": { "type": "string" }, "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object" } }, "type": "object" }, "namespaces": { "description": "namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \"this pod's namespace\"", "items": { "type": "string" }, "type": "array" }, "topologyKey": { "description": "This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.", "type": "string" } }, "required": [ "topologyKey" ], "type": "object" }, "type": "array" } }, "type": "object" } }, "type": "object" }, "alerting": { "description": "Define details regarding alerting.", "properties": { "alertmanagers": { "description": "AlertmanagerEndpoints Prometheus should fire alerts against.", "items": { "description": "AlertmanagerEndpoints defines a selection of a single Endpoints object containing alertmanager IPs to fire alerts against.", "properties": { "apiVersion": { "description": "Version of the Alertmanager API that Prometheus uses to send alerts. It can be \"v1\" or \"v2\".", "type": "string" }, "bearerTokenFile": { "description": "BearerTokenFile to read from filesystem to use when authenticating to Alertmanager.", "type": "string" }, "name": { "description": "Name of Endpoints object in Namespace.", "type": "string" }, "namespace": { "description": "Namespace of Endpoints object.", "type": "string" }, "pathPrefix": { "description": "Prefix for the HTTP path alerts are pushed to.", "type": "string" }, "port": { "anyOf": [ { "type": "integer" }, { "type": "string" } ], "description": "Port the Alertmanager API is exposed on.", "x-kubernetes-int-or-string": true }, "scheme": { "description": "Scheme to use when firing alerts.", "type": "string" }, "tlsConfig": { "description": "TLS Config to use for alertmanager connection.", "properties": { "ca": { "description": "Struct containing the CA cert to use for the targets.", "properties": { "configMap": { "description": "ConfigMap containing data to use for the targets.", "properties": { "key": { "description": "The key to select.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" }, "optional": { "description": "Specify whether the ConfigMap or its key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" }, "secret": { "description": "Secret containing data to use for the targets.", "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" }, "optional": { "description": "Specify whether the Secret or its key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" } }, "type": "object" }, "caFile": { "description": "Path to the CA cert in the Prometheus container to use for the targets.", "type": "string" }, "cert": { "description": "Struct containing the client cert file for the targets.", "properties": { "configMap": { "description": "ConfigMap containing data to use for the targets.", "properties": { "key": { "description": "The key to select.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" }, "optional": { "description": "Specify whether the ConfigMap or its key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" }, "secret": { "description": "Secret containing data to use for the targets.", "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" }, "optional": { "description": "Specify whether the Secret or its key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" } }, "type": "object" }, "certFile": { "description": "Path to the client cert file in the Prometheus container for the targets.", "type": "string" }, "insecureSkipVerify": { "description": "Disable target certificate validation.", "type": "boolean" }, "keyFile": { "description": "Path to the client key file in the Prometheus container for the targets.", "type": "string" }, "keySecret": { "description": "Secret containing the client key file for the targets.", "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" }, "optional": { "description": "Specify whether the Secret or its key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" }, "serverName": { "description": "Used to verify the hostname for the targets.", "type": "string" } }, "type": "object" } }, "required": [ "name", "namespace", "port" ], "type": "object" }, "type": "array" } }, "required": [ "alertmanagers" ], "type": "object" }, "apiserverConfig": { "description": "APIServerConfig allows specifying a host and auth methods to access apiserver. If left empty, Prometheus is assumed to run inside of the cluster and will discover API servers automatically and use the pod's CA certificate and bearer token file at /var/run/secrets/kubernetes.io/serviceaccount/.", "properties": { "basicAuth": { "description": "BasicAuth allow an endpoint to authenticate over basic authentication", "properties": { "password": { "description": "The secret in the service monitor namespace that contains the password for authentication.", "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" }, "optional": { "description": "Specify whether the Secret or its key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" }, "username": { "description": "The secret in the service monitor namespace that contains the username for authentication.", "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" }, "optional": { "description": "Specify whether the Secret or its key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" } }, "type": "object" }, "bearerToken": { "description": "Bearer token for accessing apiserver.", "type": "string" }, "bearerTokenFile": { "description": "File to read bearer token for accessing apiserver.", "type": "string" }, "host": { "description": "Host of apiserver. A valid string consisting of a hostname or IP followed by an optional port number", "type": "string" }, "tlsConfig": { "description": "TLS Config to use for accessing apiserver.", "properties": { "ca": { "description": "Struct containing the CA cert to use for the targets.", "properties": { "configMap": { "description": "ConfigMap containing data to use for the targets.", "properties": { "key": { "description": "The key to select.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" }, "optional": { "description": "Specify whether the ConfigMap or its key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" }, "secret": { "description": "Secret containing data to use for the targets.", "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" }, "optional": { "description": "Specify whether the Secret or its key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" } }, "type": "object" }, "caFile": { "description": "Path to the CA cert in the Prometheus container to use for the targets.", "type": "string" }, "cert": { "description": "Struct containing the client cert file for the targets.", "properties": { "configMap": { "description": "ConfigMap containing data to use for the targets.", "properties": { "key": { "description": "The key to select.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" }, "optional": { "description": "Specify whether the ConfigMap or its key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" }, "secret": { "description": "Secret containing data to use for the targets.", "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" }, "optional": { "description": "Specify whether the Secret or its key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" } }, "type": "object" }, "certFile": { "description": "Path to the client cert file in the Prometheus container for the targets.", "type": "string" }, "insecureSkipVerify": { "description": "Disable target certificate validation.", "type": "boolean" }, "keyFile": { "description": "Path to the client key file in the Prometheus container for the targets.", "type": "string" }, "keySecret": { "description": "Secret containing the client key file for the targets.", "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" }, "optional": { "description": "Specify whether the Secret or its key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" }, "serverName": { "description": "Used to verify the hostname for the targets.", "type": "string" } }, "type": "object" } }, "required": [ "host" ], "type": "object" }, "arbitraryFSAccessThroughSMs": { "description": "ArbitraryFSAccessThroughSMs configures whether configuration based on a service monitor can access arbitrary files on the file system of the Prometheus container e.g. bearer token files.", "properties": { "deny": { "type": "boolean" } }, "type": "object" }, "baseImage": { "description": "Base image to use for a Prometheus deployment.", "type": "string" }, "configMaps": { "description": "ConfigMaps is a list of ConfigMaps in the same namespace as the Prometheus object, which shall be mounted into the Prometheus Pods. The ConfigMaps are mounted into /etc/prometheus/configmaps/\u003cconfigmap-name\u003e.", "items": { "type": "string" }, "type": "array" }, "containers": { "description": "Containers allows injecting additional containers or modifying operator generated containers. This can be used to allow adding an authentication proxy to a Prometheus pod or to change the behavior of an operator generated container. Containers described here modify an operator generated container if they share the same name and modifications are done via a strategic merge patch. The current container names are: `prometheus`, `prometheus-config-reloader`, `rules-configmap-reloader`, and `thanos-sidecar`. Overriding containers is entirely outside the scope of what the maintainers will support and by doing so, you accept that this behaviour may break at any time without notice.", "items": { "description": "A single application container that you want to run within a pod.", "properties": { "args": { "description": "Arguments to the entrypoint. The docker image's CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell", "items": { "type": "string" }, "type": "array" }, "command": { "description": "Entrypoint array. Not executed within a shell. The docker image's ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell", "items": { "type": "string" }, "type": "array" }, "env": { "description": "List of environment variables to set in the container. Cannot be updated.", "items": { "description": "EnvVar represents an environment variable present in a Container.", "properties": { "name": { "description": "Name of the environment variable. Must be a C_IDENTIFIER.", "type": "string" }, "value": { "description": "Variable references $(VAR_NAME) are expanded using the previous defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Defaults to \"\".", "type": "string" }, "valueFrom": { "description": "Source for the environment variable's value. Cannot be used if value is not empty.", "properties": { "configMapKeyRef": { "description": "Selects a key of a ConfigMap.", "properties": { "key": { "description": "The key to select.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" }, "optional": { "description": "Specify whether the ConfigMap or its key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" }, "fieldRef": { "description": "Selects a field of the pod: supports metadata.name, metadata.namespace, metadata.labels, metadata.annotations, spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.", "properties": { "apiVersion": { "description": "Version of the schema the FieldPath is written in terms of, defaults to \"v1\".", "type": "string" }, "fieldPath": { "description": "Path of the field to select in the specified API version.", "type": "string" } }, "required": [ "fieldPath" ], "type": "object" }, "resourceFieldRef": { "description": "Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.", "properties": { "containerName": { "description": "Container name: required for volumes, optional for env vars", "type": "string" }, "divisor": { "description": "Specifies the output format of the exposed resources, defaults to \"1\"", "type": "string" }, "resource": { "description": "Required: resource to select", "type": "string" } }, "required": [ "resource" ], "type": "object" }, "secretKeyRef": { "description": "Selects a key of a secret in the pod's namespace", "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" }, "optional": { "description": "Specify whether the Secret or its key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" } }, "type": "object" } }, "required": [ "name" ], "type": "object" }, "type": "array" }, "envFrom": { "description": "List of sources to populate environment variables in the container. The keys defined within a source must be a C_IDENTIFIER. All invalid keys will be reported as an event when the container is starting. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated.", "items": { "description": "EnvFromSource represents the source of a set of ConfigMaps", "properties": { "configMapRef": { "description": "The ConfigMap to select from", "properties": { "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" }, "optional": { "description": "Specify whether the ConfigMap must be defined", "type": "boolean" } }, "type": "object" }, "prefix": { "description": "An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER.", "type": "string" }, "secretRef": { "description": "The Secret to select from", "properties": { "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" }, "optional": { "description": "Specify whether the Secret must be defined", "type": "boolean" } }, "type": "object" } }, "type": "object" }, "type": "array" }, "image": { "description": "Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images This field is optional to allow higher level config management to default or override container images in workload controllers like Deployments and StatefulSets.", "type": "string" }, "imagePullPolicy": { "description": "Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images", "type": "string" }, "lifecycle": { "description": "Actions that the management system should take in response to container lifecycle events. Cannot be updated.", "properties": { "postStart": { "description": "PostStart is called immediately after a container is created. If the handler fails, the container is terminated and restarted according to its restart policy. Other management of the container blocks until the hook completes. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks", "properties": { "exec": { "description": "One and only one of the following should be specified. Exec specifies the action to take.", "properties": { "command": { "description": "Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.", "items": { "type": "string" }, "type": "array" } }, "type": "object" }, "httpGet": { "description": "HTTPGet specifies the http request to perform.", "properties": { "host": { "description": "Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.", "type": "string" }, "httpHeaders": { "description": "Custom headers to set in the request. HTTP allows repeated headers.", "items": { "description": "HTTPHeader describes a custom header to be used in HTTP probes", "properties": { "name": { "description": "The header field name", "type": "string" }, "value": { "description": "The header field value", "type": "string" } }, "required": [ "name", "value" ], "type": "object" }, "type": "array" }, "path": { "description": "Path to access on the HTTP server.", "type": "string" }, "port": { "anyOf": [ { "type": "integer" }, { "type": "string" } ], "description": "Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.", "x-kubernetes-int-or-string": true }, "scheme": { "description": "Scheme to use for connecting to the host. Defaults to HTTP.", "type": "string" } }, "required": [ "port" ], "type": "object" }, "tcpSocket": { "description": "TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook", "properties": { "host": { "description": "Optional: Host name to connect to, defaults to the pod IP.", "type": "string" }, "port": { "anyOf": [ { "type": "integer" }, { "type": "string" } ], "description": "Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.", "x-kubernetes-int-or-string": true } }, "required": [ "port" ], "type": "object" } }, "type": "object" }, "preStop": { "description": "PreStop is called immediately before a container is terminated due to an API request or management event such as liveness/startup probe failure, preemption, resource contention, etc. The handler is not called if the container crashes or exits. The reason for termination is passed to the handler. The Pod's termination grace period countdown begins before the PreStop hooked is executed. Regardless of the outcome of the handler, the container will eventually terminate within the Pod's termination grace period. Other management of the container blocks until the hook completes or until the termination grace period is reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks", "properties": { "exec": { "description": "One and only one of the following should be specified. Exec specifies the action to take.", "properties": { "command": { "description": "Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.", "items": { "type": "string" }, "type": "array" } }, "type": "object" }, "httpGet": { "description": "HTTPGet specifies the http request to perform.", "properties": { "host": { "description": "Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.", "type": "string" }, "httpHeaders": { "description": "Custom headers to set in the request. HTTP allows repeated headers.", "items": { "description": "HTTPHeader describes a custom header to be used in HTTP probes", "properties": { "name": { "description": "The header field name", "type": "string" }, "value": { "description": "The header field value", "type": "string" } }, "required": [ "name", "value" ], "type": "object" }, "type": "array" }, "path": { "description": "Path to access on the HTTP server.", "type": "string" }, "port": { "anyOf": [ { "type": "integer" }, { "type": "string" } ], "description": "Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.", "x-kubernetes-int-or-string": true }, "scheme": { "description": "Scheme to use for connecting to the host. Defaults to HTTP.", "type": "string" } }, "required": [ "port" ], "type": "object" }, "tcpSocket": { "description": "TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook", "properties": { "host": { "description": "Optional: Host name to connect to, defaults to the pod IP.", "type": "string" }, "port": { "anyOf": [ { "type": "integer" }, { "type": "string" } ], "description": "Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.", "x-kubernetes-int-or-string": true } }, "required": [ "port" ], "type": "object" } }, "type": "object" } }, "type": "object" }, "livenessProbe": { "description": "Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes", "properties": { "exec": { "description": "One and only one of the following should be specified. Exec specifies the action to take.", "properties": { "command": { "description": "Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.", "items": { "type": "string" }, "type": "array" } }, "type": "object" }, "failureThreshold": { "description": "Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.", "format": "int32", "type": "integer" }, "httpGet": { "description": "HTTPGet specifies the http request to perform.", "properties": { "host": { "description": "Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.", "type": "string" }, "httpHeaders": { "description": "Custom headers to set in the request. HTTP allows repeated headers.", "items": { "description": "HTTPHeader describes a custom header to be used in HTTP probes", "properties": { "name": { "description": "The header field name", "type": "string" }, "value": { "description": "The header field value", "type": "string" } }, "required": [ "name", "value" ], "type": "object" }, "type": "array" }, "path": { "description": "Path to access on the HTTP server.", "type": "string" }, "port": { "anyOf": [ { "type": "integer" }, { "type": "string" } ], "description": "Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.", "x-kubernetes-int-or-string": true }, "scheme": { "description": "Scheme to use for connecting to the host. Defaults to HTTP.", "type": "string" } }, "required": [ "port" ], "type": "object" }, "initialDelaySeconds": { "description": "Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes", "format": "int32", "type": "integer" }, "periodSeconds": { "description": "How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.", "format": "int32", "type": "integer" }, "successThreshold": { "description": "Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.", "format": "int32", "type": "integer" }, "tcpSocket": { "description": "TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook", "properties": { "host": { "description": "Optional: Host name to connect to, defaults to the pod IP.", "type": "string" }, "port": { "anyOf": [ { "type": "integer" }, { "type": "string" } ], "description": "Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.", "x-kubernetes-int-or-string": true } }, "required": [ "port" ], "type": "object" }, "timeoutSeconds": { "description": "Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes", "format": "int32", "type": "integer" } }, "type": "object" }, "name": { "description": "Name of the container specified as a DNS_LABEL. Each container in a pod must have a unique name (DNS_LABEL). Cannot be updated.", "type": "string" }, "ports": { "description": "List of ports to expose from the container. Exposing a port here gives the system additional information about the network connections a container uses, but is primarily informational. Not specifying a port here DOES NOT prevent that port from being exposed. Any port which is listening on the default \"0.0.0.0\" address inside a container will be accessible from the network. Cannot be updated.", "items": { "description": "ContainerPort represents a network port in a single container.", "properties": { "containerPort": { "description": "Number of port to expose on the pod's IP address. This must be a valid port number, 0 \u003c x \u003c 65536.", "format": "int32", "type": "integer" }, "hostIP": { "description": "What host IP to bind the external port to.", "type": "string" }, "hostPort": { "description": "Number of port to expose on the host. If specified, this must be a valid port number, 0 \u003c x \u003c 65536. If HostNetwork is specified, this must match ContainerPort. Most containers do not need this.", "format": "int32", "type": "integer" }, "name": { "description": "If specified, this must be an IANA_SVC_NAME and unique within the pod. Each named port in a pod must have a unique name. Name for the port that can be referred to by services.", "type": "string" }, "protocol": { "description": "Protocol for port. Must be UDP, TCP, or SCTP. Defaults to \"TCP\".", "type": "string" } }, "required": [ "containerPort" ], "type": "object" }, "type": "array" }, "readinessProbe": { "description": "Periodic probe of container service readiness. Container will be removed from service endpoints if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes", "properties": { "exec": { "description": "One and only one of the following should be specified. Exec specifies the action to take.", "properties": { "command": { "description": "Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.", "items": { "type": "string" }, "type": "array" } }, "type": "object" }, "failureThreshold": { "description": "Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.", "format": "int32", "type": "integer" }, "httpGet": { "description": "HTTPGet specifies the http request to perform.", "properties": { "host": { "description": "Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.", "type": "string" }, "httpHeaders": { "description": "Custom headers to set in the request. HTTP allows repeated headers.", "items": { "description": "HTTPHeader describes a custom header to be used in HTTP probes", "properties": { "name": { "description": "The header field name", "type": "string" }, "value": { "description": "The header field value", "type": "string" } }, "required": [ "name", "value" ], "type": "object" }, "type": "array" }, "path": { "description": "Path to access on the HTTP server.", "type": "string" }, "port": { "anyOf": [ { "type": "integer" }, { "type": "string" } ], "description": "Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.", "x-kubernetes-int-or-string": true }, "scheme": { "description": "Scheme to use for connecting to the host. Defaults to HTTP.", "type": "string" } }, "required": [ "port" ], "type": "object" }, "initialDelaySeconds": { "description": "Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes", "format": "int32", "type": "integer" }, "periodSeconds": { "description": "How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.", "format": "int32", "type": "integer" }, "successThreshold": { "description": "Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.", "format": "int32", "type": "integer" }, "tcpSocket": { "description": "TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook", "properties": { "host": { "description": "Optional: Host name to connect to, defaults to the pod IP.", "type": "string" }, "port": { "anyOf": [ { "type": "integer" }, { "type": "string" } ], "description": "Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.", "x-kubernetes-int-or-string": true } }, "required": [ "port" ], "type": "object" }, "timeoutSeconds": { "description": "Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes", "format": "int32", "type": "integer" } }, "type": "object" }, "resources": { "description": "Compute Resources required by this container. Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", "properties": { "limits": { "additionalProperties": { "type": "string" }, "description": "Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", "type": "object" }, "requests": { "additionalProperties": { "type": "string" }, "description": "Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", "type": "object" } }, "type": "object" }, "securityContext": { "description": "Security options the pod should run with. More info: https://kubernetes.io/docs/concepts/policy/security-context/ More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/", "properties": { "allowPrivilegeEscalation": { "description": "AllowPrivilegeEscalation controls whether a process can gain more privileges than its parent process. This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged 2) has CAP_SYS_ADMIN", "type": "boolean" }, "capabilities": { "description": "The capabilities to add/drop when running containers. Defaults to the default set of capabilities granted by the container runtime.", "properties": { "add": { "description": "Added capabilities", "items": { "description": "Capability represent POSIX capabilities type", "type": "string" }, "type": "array" }, "drop": { "description": "Removed capabilities", "items": { "description": "Capability represent POSIX capabilities type", "type": "string" }, "type": "array" } }, "type": "object" }, "privileged": { "description": "Run container in privileged mode. Processes in privileged containers are essentially equivalent to root on the host. Defaults to false.", "type": "boolean" }, "procMount": { "description": "procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled.", "type": "string" }, "readOnlyRootFilesystem": { "description": "Whether this container has a read-only root filesystem. Default is false.", "type": "boolean" }, "runAsGroup": { "description": "The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.", "format": "int64", "type": "integer" }, "runAsNonRoot": { "description": "Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.", "type": "boolean" }, "runAsUser": { "description": "The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.", "format": "int64", "type": "integer" }, "seLinuxOptions": { "description": "The SELinux context to be applied to the container. If unspecified, the container runtime will allocate a random SELinux context for each container. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.", "properties": { "level": { "description": "Level is SELinux level label that applies to the container.", "type": "string" }, "role": { "description": "Role is a SELinux role label that applies to the container.", "type": "string" }, "type": { "description": "Type is a SELinux type label that applies to the container.", "type": "string" }, "user": { "description": "User is a SELinux user label that applies to the container.", "type": "string" } }, "type": "object" }, "windowsOptions": { "description": "The Windows specific settings applied to all containers. If unspecified, the options from the PodSecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.", "properties": { "gmsaCredentialSpec": { "description": "GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field.", "type": "string" }, "gmsaCredentialSpecName": { "description": "GMSACredentialSpecName is the name of the GMSA credential spec to use.", "type": "string" }, "runAsUserName": { "description": "The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.", "type": "string" } }, "type": "object" } }, "type": "object" }, "startupProbe": { "description": "StartupProbe indicates that the Pod has successfully initialized. If specified, no other probes are executed until this completes successfully. If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, when it might take a long time to load data or warm a cache, than during steady-state operation. This cannot be updated. This is a beta feature enabled by the StartupProbe feature flag. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes", "properties": { "exec": { "description": "One and only one of the following should be specified. Exec specifies the action to take.", "properties": { "command": { "description": "Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.", "items": { "type": "string" }, "type": "array" } }, "type": "object" }, "failureThreshold": { "description": "Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.", "format": "int32", "type": "integer" }, "httpGet": { "description": "HTTPGet specifies the http request to perform.", "properties": { "host": { "description": "Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.", "type": "string" }, "httpHeaders": { "description": "Custom headers to set in the request. HTTP allows repeated headers.", "items": { "description": "HTTPHeader describes a custom header to be used in HTTP probes", "properties": { "name": { "description": "The header field name", "type": "string" }, "value": { "description": "The header field value", "type": "string" } }, "required": [ "name", "value" ], "type": "object" }, "type": "array" }, "path": { "description": "Path to access on the HTTP server.", "type": "string" }, "port": { "anyOf": [ { "type": "integer" }, { "type": "string" } ], "description": "Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.", "x-kubernetes-int-or-string": true }, "scheme": { "description": "Scheme to use for connecting to the host. Defaults to HTTP.", "type": "string" } }, "required": [ "port" ], "type": "object" }, "initialDelaySeconds": { "description": "Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes", "format": "int32", "type": "integer" }, "periodSeconds": { "description": "How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.", "format": "int32", "type": "integer" }, "successThreshold": { "description": "Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.", "format": "int32", "type": "integer" }, "tcpSocket": { "description": "TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook", "properties": { "host": { "description": "Optional: Host name to connect to, defaults to the pod IP.", "type": "string" }, "port": { "anyOf": [ { "type": "integer" }, { "type": "string" } ], "description": "Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.", "x-kubernetes-int-or-string": true } }, "required": [ "port" ], "type": "object" }, "timeoutSeconds": { "description": "Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes", "format": "int32", "type": "integer" } }, "type": "object" }, "stdin": { "description": "Whether this container should allocate a buffer for stdin in the container runtime. If this is not set, reads from stdin in the container will always result in EOF. Default is false.", "type": "boolean" }, "stdinOnce": { "description": "Whether the container runtime should close the stdin channel after it has been opened by a single attach. When stdin is true the stdin stream will remain open across multiple attach sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the first client attaches to stdin, and then remains open and accepts data until the client disconnects, at which time stdin is closed and remains closed until the container is restarted. If this flag is false, a container processes that reads from stdin will never receive an EOF. Default is false", "type": "boolean" }, "terminationMessagePath": { "description": "Optional: Path at which the file to which the container's termination message will be written is mounted into the container's filesystem. Message written is intended to be brief final status, such as an assertion failure message. Will be truncated by the node if greater than 4096 bytes. The total message length across all containers will be limited to 12kb. Defaults to /dev/termination-log. Cannot be updated.", "type": "string" }, "terminationMessagePolicy": { "description": "Indicate how the termination message should be populated. File will use the contents of terminationMessagePath to populate the container status message on both success and failure. FallbackToLogsOnError will use the last chunk of container log output if the termination message file is empty and the container exited with an error. The log output is limited to 2048 bytes or 80 lines, whichever is smaller. Defaults to File. Cannot be updated.", "type": "string" }, "tty": { "description": "Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. Default is false.", "type": "boolean" }, "volumeDevices": { "description": "volumeDevices is the list of block devices to be used by the container.", "items": { "description": "volumeDevice describes a mapping of a raw block device within a container.", "properties": { "devicePath": { "description": "devicePath is the path inside of the container that the device will be mapped to.", "type": "string" }, "name": { "description": "name must match the name of a persistentVolumeClaim in the pod", "type": "string" } }, "required": [ "devicePath", "name" ], "type": "object" }, "type": "array" }, "volumeMounts": { "description": "Pod volumes to mount into the container's filesystem. Cannot be updated.", "items": { "description": "VolumeMount describes a mounting of a Volume within a container.", "properties": { "mountPath": { "description": "Path within the container at which the volume should be mounted. Must not contain ':'.", "type": "string" }, "mountPropagation": { "description": "mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10.", "type": "string" }, "name": { "description": "This must match the Name of a Volume.", "type": "string" }, "readOnly": { "description": "Mounted read-only if true, read-write otherwise (false or unspecified). Defaults to false.", "type": "boolean" }, "subPath": { "description": "Path within the volume from which the container's volume should be mounted. Defaults to \"\" (volume's root).", "type": "string" }, "subPathExpr": { "description": "Expanded path within the volume from which the container's volume should be mounted. Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. Defaults to \"\" (volume's root). SubPathExpr and SubPath are mutually exclusive.", "type": "string" } }, "required": [ "mountPath", "name" ], "type": "object" }, "type": "array" }, "workingDir": { "description": "Container's working directory. If not specified, the container runtime's default will be used, which might be configured in the container image. Cannot be updated.", "type": "string" } }, "required": [ "name" ], "type": "object" }, "type": "array" }, "disableCompaction": { "description": "Disable prometheus compaction.", "type": "boolean" }, "enableAdminAPI": { "description": "Enable access to prometheus web admin API. Defaults to the value of `false`. WARNING: Enabling the admin APIs enables mutating endpoints, to delete data, shutdown Prometheus, and more. Enabling this should be done with care and the user is advised to add additional authentication authorization via a proxy to ensure only clients authorized to perform these actions can do so. For more information see https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-admin-apis", "type": "boolean" }, "enforcedNamespaceLabel": { "description": "EnforcedNamespaceLabel enforces adding a namespace label of origin for each alert and metric that is user created. The label value will always be the namespace of the object that is being created.", "type": "string" }, "evaluationInterval": { "description": "Interval between consecutive evaluations.", "type": "string" }, "externalLabels": { "additionalProperties": { "type": "string" }, "description": "The labels to add to any time series or alerts when communicating with external systems (federation, remote storage, Alertmanager).", "type": "object" }, "externalUrl": { "description": "The external URL the Prometheus instances will be available under. This is necessary to generate correct URLs. This is necessary if Prometheus is not served from root of a DNS name.", "type": "string" }, "ignoreNamespaceSelectors": { "description": "IgnoreNamespaceSelectors if set to true will ignore NamespaceSelector settings from the podmonitor and servicemonitor configs, and they will only discover endpoints within their current namespace. Defaults to false.", "type": "boolean" }, "image": { "description": "Image if specified has precedence over baseImage, tag and sha combinations. Specifying the version is still necessary to ensure the Prometheus Operator knows what version of Prometheus is being configured.", "type": "string" }, "imagePullSecrets": { "description": "An optional list of references to secrets in the same namespace to use for pulling prometheus and alertmanager images from registries see http://kubernetes.io/docs/user-guide/images#specifying-imagepullsecrets-on-a-pod", "items": { "description": "LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace.", "properties": { "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" } }, "type": "object" }, "type": "array" }, "initContainers": { "description": "InitContainers allows adding initContainers to the pod definition. Those can be used to e.g. fetch secrets for injection into the Prometheus configuration from external sources. Any errors during the execution of an initContainer will lead to a restart of the Pod. More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ Using initContainers for any use case other then secret fetching is entirely outside the scope of what the maintainers will support and by doing so, you accept that this behaviour may break at any time without notice.", "items": { "description": "A single application container that you want to run within a pod.", "properties": { "args": { "description": "Arguments to the entrypoint. The docker image's CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell", "items": { "type": "string" }, "type": "array" }, "command": { "description": "Entrypoint array. Not executed within a shell. The docker image's ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell", "items": { "type": "string" }, "type": "array" }, "env": { "description": "List of environment variables to set in the container. Cannot be updated.", "items": { "description": "EnvVar represents an environment variable present in a Container.", "properties": { "name": { "description": "Name of the environment variable. Must be a C_IDENTIFIER.", "type": "string" }, "value": { "description": "Variable references $(VAR_NAME) are expanded using the previous defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Defaults to \"\".", "type": "string" }, "valueFrom": { "description": "Source for the environment variable's value. Cannot be used if value is not empty.", "properties": { "configMapKeyRef": { "description": "Selects a key of a ConfigMap.", "properties": { "key": { "description": "The key to select.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" }, "optional": { "description": "Specify whether the ConfigMap or its key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" }, "fieldRef": { "description": "Selects a field of the pod: supports metadata.name, metadata.namespace, metadata.labels, metadata.annotations, spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.", "properties": { "apiVersion": { "description": "Version of the schema the FieldPath is written in terms of, defaults to \"v1\".", "type": "string" }, "fieldPath": { "description": "Path of the field to select in the specified API version.", "type": "string" } }, "required": [ "fieldPath" ], "type": "object" }, "resourceFieldRef": { "description": "Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.", "properties": { "containerName": { "description": "Container name: required for volumes, optional for env vars", "type": "string" }, "divisor": { "description": "Specifies the output format of the exposed resources, defaults to \"1\"", "type": "string" }, "resource": { "description": "Required: resource to select", "type": "string" } }, "required": [ "resource" ], "type": "object" }, "secretKeyRef": { "description": "Selects a key of a secret in the pod's namespace", "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" }, "optional": { "description": "Specify whether the Secret or its key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" } }, "type": "object" } }, "required": [ "name" ], "type": "object" }, "type": "array" }, "envFrom": { "description": "List of sources to populate environment variables in the container. The keys defined within a source must be a C_IDENTIFIER. All invalid keys will be reported as an event when the container is starting. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated.", "items": { "description": "EnvFromSource represents the source of a set of ConfigMaps", "properties": { "configMapRef": { "description": "The ConfigMap to select from", "properties": { "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" }, "optional": { "description": "Specify whether the ConfigMap must be defined", "type": "boolean" } }, "type": "object" }, "prefix": { "description": "An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER.", "type": "string" }, "secretRef": { "description": "The Secret to select from", "properties": { "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" }, "optional": { "description": "Specify whether the Secret must be defined", "type": "boolean" } }, "type": "object" } }, "type": "object" }, "type": "array" }, "image": { "description": "Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images This field is optional to allow higher level config management to default or override container images in workload controllers like Deployments and StatefulSets.", "type": "string" }, "imagePullPolicy": { "description": "Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images", "type": "string" }, "lifecycle": { "description": "Actions that the management system should take in response to container lifecycle events. Cannot be updated.", "properties": { "postStart": { "description": "PostStart is called immediately after a container is created. If the handler fails, the container is terminated and restarted according to its restart policy. Other management of the container blocks until the hook completes. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks", "properties": { "exec": { "description": "One and only one of the following should be specified. Exec specifies the action to take.", "properties": { "command": { "description": "Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.", "items": { "type": "string" }, "type": "array" } }, "type": "object" }, "httpGet": { "description": "HTTPGet specifies the http request to perform.", "properties": { "host": { "description": "Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.", "type": "string" }, "httpHeaders": { "description": "Custom headers to set in the request. HTTP allows repeated headers.", "items": { "description": "HTTPHeader describes a custom header to be used in HTTP probes", "properties": { "name": { "description": "The header field name", "type": "string" }, "value": { "description": "The header field value", "type": "string" } }, "required": [ "name", "value" ], "type": "object" }, "type": "array" }, "path": { "description": "Path to access on the HTTP server.", "type": "string" }, "port": { "anyOf": [ { "type": "integer" }, { "type": "string" } ], "description": "Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.", "x-kubernetes-int-or-string": true }, "scheme": { "description": "Scheme to use for connecting to the host. Defaults to HTTP.", "type": "string" } }, "required": [ "port" ], "type": "object" }, "tcpSocket": { "description": "TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook", "properties": { "host": { "description": "Optional: Host name to connect to, defaults to the pod IP.", "type": "string" }, "port": { "anyOf": [ { "type": "integer" }, { "type": "string" } ], "description": "Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.", "x-kubernetes-int-or-string": true } }, "required": [ "port" ], "type": "object" } }, "type": "object" }, "preStop": { "description": "PreStop is called immediately before a container is terminated due to an API request or management event such as liveness/startup probe failure, preemption, resource contention, etc. The handler is not called if the container crashes or exits. The reason for termination is passed to the handler. The Pod's termination grace period countdown begins before the PreStop hooked is executed. Regardless of the outcome of the handler, the container will eventually terminate within the Pod's termination grace period. Other management of the container blocks until the hook completes or until the termination grace period is reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks", "properties": { "exec": { "description": "One and only one of the following should be specified. Exec specifies the action to take.", "properties": { "command": { "description": "Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.", "items": { "type": "string" }, "type": "array" } }, "type": "object" }, "httpGet": { "description": "HTTPGet specifies the http request to perform.", "properties": { "host": { "description": "Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.", "type": "string" }, "httpHeaders": { "description": "Custom headers to set in the request. HTTP allows repeated headers.", "items": { "description": "HTTPHeader describes a custom header to be used in HTTP probes", "properties": { "name": { "description": "The header field name", "type": "string" }, "value": { "description": "The header field value", "type": "string" } }, "required": [ "name", "value" ], "type": "object" }, "type": "array" }, "path": { "description": "Path to access on the HTTP server.", "type": "string" }, "port": { "anyOf": [ { "type": "integer" }, { "type": "string" } ], "description": "Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.", "x-kubernetes-int-or-string": true }, "scheme": { "description": "Scheme to use for connecting to the host. Defaults to HTTP.", "type": "string" } }, "required": [ "port" ], "type": "object" }, "tcpSocket": { "description": "TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook", "properties": { "host": { "description": "Optional: Host name to connect to, defaults to the pod IP.", "type": "string" }, "port": { "anyOf": [ { "type": "integer" }, { "type": "string" } ], "description": "Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.", "x-kubernetes-int-or-string": true } }, "required": [ "port" ], "type": "object" } }, "type": "object" } }, "type": "object" }, "livenessProbe": { "description": "Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes", "properties": { "exec": { "description": "One and only one of the following should be specified. Exec specifies the action to take.", "properties": { "command": { "description": "Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.", "items": { "type": "string" }, "type": "array" } }, "type": "object" }, "failureThreshold": { "description": "Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.", "format": "int32", "type": "integer" }, "httpGet": { "description": "HTTPGet specifies the http request to perform.", "properties": { "host": { "description": "Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.", "type": "string" }, "httpHeaders": { "description": "Custom headers to set in the request. HTTP allows repeated headers.", "items": { "description": "HTTPHeader describes a custom header to be used in HTTP probes", "properties": { "name": { "description": "The header field name", "type": "string" }, "value": { "description": "The header field value", "type": "string" } }, "required": [ "name", "value" ], "type": "object" }, "type": "array" }, "path": { "description": "Path to access on the HTTP server.", "type": "string" }, "port": { "anyOf": [ { "type": "integer" }, { "type": "string" } ], "description": "Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.", "x-kubernetes-int-or-string": true }, "scheme": { "description": "Scheme to use for connecting to the host. Defaults to HTTP.", "type": "string" } }, "required": [ "port" ], "type": "object" }, "initialDelaySeconds": { "description": "Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes", "format": "int32", "type": "integer" }, "periodSeconds": { "description": "How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.", "format": "int32", "type": "integer" }, "successThreshold": { "description": "Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.", "format": "int32", "type": "integer" }, "tcpSocket": { "description": "TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook", "properties": { "host": { "description": "Optional: Host name to connect to, defaults to the pod IP.", "type": "string" }, "port": { "anyOf": [ { "type": "integer" }, { "type": "string" } ], "description": "Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.", "x-kubernetes-int-or-string": true } }, "required": [ "port" ], "type": "object" }, "timeoutSeconds": { "description": "Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes", "format": "int32", "type": "integer" } }, "type": "object" }, "name": { "description": "Name of the container specified as a DNS_LABEL. Each container in a pod must have a unique name (DNS_LABEL). Cannot be updated.", "type": "string" }, "ports": { "description": "List of ports to expose from the container. Exposing a port here gives the system additional information about the network connections a container uses, but is primarily informational. Not specifying a port here DOES NOT prevent that port from being exposed. Any port which is listening on the default \"0.0.0.0\" address inside a container will be accessible from the network. Cannot be updated.", "items": { "description": "ContainerPort represents a network port in a single container.", "properties": { "containerPort": { "description": "Number of port to expose on the pod's IP address. This must be a valid port number, 0 \u003c x \u003c 65536.", "format": "int32", "type": "integer" }, "hostIP": { "description": "What host IP to bind the external port to.", "type": "string" }, "hostPort": { "description": "Number of port to expose on the host. If specified, this must be a valid port number, 0 \u003c x \u003c 65536. If HostNetwork is specified, this must match ContainerPort. Most containers do not need this.", "format": "int32", "type": "integer" }, "name": { "description": "If specified, this must be an IANA_SVC_NAME and unique within the pod. Each named port in a pod must have a unique name. Name for the port that can be referred to by services.", "type": "string" }, "protocol": { "description": "Protocol for port. Must be UDP, TCP, or SCTP. Defaults to \"TCP\".", "type": "string" } }, "required": [ "containerPort" ], "type": "object" }, "type": "array" }, "readinessProbe": { "description": "Periodic probe of container service readiness. Container will be removed from service endpoints if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes", "properties": { "exec": { "description": "One and only one of the following should be specified. Exec specifies the action to take.", "properties": { "command": { "description": "Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.", "items": { "type": "string" }, "type": "array" } }, "type": "object" }, "failureThreshold": { "description": "Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.", "format": "int32", "type": "integer" }, "httpGet": { "description": "HTTPGet specifies the http request to perform.", "properties": { "host": { "description": "Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.", "type": "string" }, "httpHeaders": { "description": "Custom headers to set in the request. HTTP allows repeated headers.", "items": { "description": "HTTPHeader describes a custom header to be used in HTTP probes", "properties": { "name": { "description": "The header field name", "type": "string" }, "value": { "description": "The header field value", "type": "string" } }, "required": [ "name", "value" ], "type": "object" }, "type": "array" }, "path": { "description": "Path to access on the HTTP server.", "type": "string" }, "port": { "anyOf": [ { "type": "integer" }, { "type": "string" } ], "description": "Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.", "x-kubernetes-int-or-string": true }, "scheme": { "description": "Scheme to use for connecting to the host. Defaults to HTTP.", "type": "string" } }, "required": [ "port" ], "type": "object" }, "initialDelaySeconds": { "description": "Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes", "format": "int32", "type": "integer" }, "periodSeconds": { "description": "How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.", "format": "int32", "type": "integer" }, "successThreshold": { "description": "Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.", "format": "int32", "type": "integer" }, "tcpSocket": { "description": "TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook", "properties": { "host": { "description": "Optional: Host name to connect to, defaults to the pod IP.", "type": "string" }, "port": { "anyOf": [ { "type": "integer" }, { "type": "string" } ], "description": "Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.", "x-kubernetes-int-or-string": true } }, "required": [ "port" ], "type": "object" }, "timeoutSeconds": { "description": "Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes", "format": "int32", "type": "integer" } }, "type": "object" }, "resources": { "description": "Compute Resources required by this container. Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", "properties": { "limits": { "additionalProperties": { "type": "string" }, "description": "Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", "type": "object" }, "requests": { "additionalProperties": { "type": "string" }, "description": "Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", "type": "object" } }, "type": "object" }, "securityContext": { "description": "Security options the pod should run with. More info: https://kubernetes.io/docs/concepts/policy/security-context/ More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/", "properties": { "allowPrivilegeEscalation": { "description": "AllowPrivilegeEscalation controls whether a process can gain more privileges than its parent process. This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged 2) has CAP_SYS_ADMIN", "type": "boolean" }, "capabilities": { "description": "The capabilities to add/drop when running containers. Defaults to the default set of capabilities granted by the container runtime.", "properties": { "add": { "description": "Added capabilities", "items": { "description": "Capability represent POSIX capabilities type", "type": "string" }, "type": "array" }, "drop": { "description": "Removed capabilities", "items": { "description": "Capability represent POSIX capabilities type", "type": "string" }, "type": "array" } }, "type": "object" }, "privileged": { "description": "Run container in privileged mode. Processes in privileged containers are essentially equivalent to root on the host. Defaults to false.", "type": "boolean" }, "procMount": { "description": "procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled.", "type": "string" }, "readOnlyRootFilesystem": { "description": "Whether this container has a read-only root filesystem. Default is false.", "type": "boolean" }, "runAsGroup": { "description": "The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.", "format": "int64", "type": "integer" }, "runAsNonRoot": { "description": "Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.", "type": "boolean" }, "runAsUser": { "description": "The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.", "format": "int64", "type": "integer" }, "seLinuxOptions": { "description": "The SELinux context to be applied to the container. If unspecified, the container runtime will allocate a random SELinux context for each container. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.", "properties": { "level": { "description": "Level is SELinux level label that applies to the container.", "type": "string" }, "role": { "description": "Role is a SELinux role label that applies to the container.", "type": "string" }, "type": { "description": "Type is a SELinux type label that applies to the container.", "type": "string" }, "user": { "description": "User is a SELinux user label that applies to the container.", "type": "string" } }, "type": "object" }, "windowsOptions": { "description": "The Windows specific settings applied to all containers. If unspecified, the options from the PodSecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.", "properties": { "gmsaCredentialSpec": { "description": "GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field.", "type": "string" }, "gmsaCredentialSpecName": { "description": "GMSACredentialSpecName is the name of the GMSA credential spec to use.", "type": "string" }, "runAsUserName": { "description": "The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.", "type": "string" } }, "type": "object" } }, "type": "object" }, "startupProbe": { "description": "StartupProbe indicates that the Pod has successfully initialized. If specified, no other probes are executed until this completes successfully. If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, when it might take a long time to load data or warm a cache, than during steady-state operation. This cannot be updated. This is a beta feature enabled by the StartupProbe feature flag. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes", "properties": { "exec": { "description": "One and only one of the following should be specified. Exec specifies the action to take.", "properties": { "command": { "description": "Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.", "items": { "type": "string" }, "type": "array" } }, "type": "object" }, "failureThreshold": { "description": "Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.", "format": "int32", "type": "integer" }, "httpGet": { "description": "HTTPGet specifies the http request to perform.", "properties": { "host": { "description": "Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.", "type": "string" }, "httpHeaders": { "description": "Custom headers to set in the request. HTTP allows repeated headers.", "items": { "description": "HTTPHeader describes a custom header to be used in HTTP probes", "properties": { "name": { "description": "The header field name", "type": "string" }, "value": { "description": "The header field value", "type": "string" } }, "required": [ "name", "value" ], "type": "object" }, "type": "array" }, "path": { "description": "Path to access on the HTTP server.", "type": "string" }, "port": { "anyOf": [ { "type": "integer" }, { "type": "string" } ], "description": "Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.", "x-kubernetes-int-or-string": true }, "scheme": { "description": "Scheme to use for connecting to the host. Defaults to HTTP.", "type": "string" } }, "required": [ "port" ], "type": "object" }, "initialDelaySeconds": { "description": "Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes", "format": "int32", "type": "integer" }, "periodSeconds": { "description": "How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.", "format": "int32", "type": "integer" }, "successThreshold": { "description": "Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.", "format": "int32", "type": "integer" }, "tcpSocket": { "description": "TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook", "properties": { "host": { "description": "Optional: Host name to connect to, defaults to the pod IP.", "type": "string" }, "port": { "anyOf": [ { "type": "integer" }, { "type": "string" } ], "description": "Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.", "x-kubernetes-int-or-string": true } }, "required": [ "port" ], "type": "object" }, "timeoutSeconds": { "description": "Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes", "format": "int32", "type": "integer" } }, "type": "object" }, "stdin": { "description": "Whether this container should allocate a buffer for stdin in the container runtime. If this is not set, reads from stdin in the container will always result in EOF. Default is false.", "type": "boolean" }, "stdinOnce": { "description": "Whether the container runtime should close the stdin channel after it has been opened by a single attach. When stdin is true the stdin stream will remain open across multiple attach sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the first client attaches to stdin, and then remains open and accepts data until the client disconnects, at which time stdin is closed and remains closed until the container is restarted. If this flag is false, a container processes that reads from stdin will never receive an EOF. Default is false", "type": "boolean" }, "terminationMessagePath": { "description": "Optional: Path at which the file to which the container's termination message will be written is mounted into the container's filesystem. Message written is intended to be brief final status, such as an assertion failure message. Will be truncated by the node if greater than 4096 bytes. The total message length across all containers will be limited to 12kb. Defaults to /dev/termination-log. Cannot be updated.", "type": "string" }, "terminationMessagePolicy": { "description": "Indicate how the termination message should be populated. File will use the contents of terminationMessagePath to populate the container status message on both success and failure. FallbackToLogsOnError will use the last chunk of container log output if the termination message file is empty and the container exited with an error. The log output is limited to 2048 bytes or 80 lines, whichever is smaller. Defaults to File. Cannot be updated.", "type": "string" }, "tty": { "description": "Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. Default is false.", "type": "boolean" }, "volumeDevices": { "description": "volumeDevices is the list of block devices to be used by the container.", "items": { "description": "volumeDevice describes a mapping of a raw block device within a container.", "properties": { "devicePath": { "description": "devicePath is the path inside of the container that the device will be mapped to.", "type": "string" }, "name": { "description": "name must match the name of a persistentVolumeClaim in the pod", "type": "string" } }, "required": [ "devicePath", "name" ], "type": "object" }, "type": "array" }, "volumeMounts": { "description": "Pod volumes to mount into the container's filesystem. Cannot be updated.", "items": { "description": "VolumeMount describes a mounting of a Volume within a container.", "properties": { "mountPath": { "description": "Path within the container at which the volume should be mounted. Must not contain ':'.", "type": "string" }, "mountPropagation": { "description": "mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10.", "type": "string" }, "name": { "description": "This must match the Name of a Volume.", "type": "string" }, "readOnly": { "description": "Mounted read-only if true, read-write otherwise (false or unspecified). Defaults to false.", "type": "boolean" }, "subPath": { "description": "Path within the volume from which the container's volume should be mounted. Defaults to \"\" (volume's root).", "type": "string" }, "subPathExpr": { "description": "Expanded path within the volume from which the container's volume should be mounted. Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. Defaults to \"\" (volume's root). SubPathExpr and SubPath are mutually exclusive.", "type": "string" } }, "required": [ "mountPath", "name" ], "type": "object" }, "type": "array" }, "workingDir": { "description": "Container's working directory. If not specified, the container runtime's default will be used, which might be configured in the container image. Cannot be updated.", "type": "string" } }, "required": [ "name" ], "type": "object" }, "type": "array" }, "listenLocal": { "description": "ListenLocal makes the Prometheus server listen on loopback, so that it does not bind against the Pod IP.", "type": "boolean" }, "logFormat": { "description": "Log format for Prometheus to be configured with.", "type": "string" }, "logLevel": { "description": "Log level for Prometheus to be configured with.", "type": "string" }, "nodeSelector": { "additionalProperties": { "type": "string" }, "description": "Define which Nodes the Pods are scheduled on.", "type": "object" }, "overrideHonorLabels": { "description": "OverrideHonorLabels if set to true overrides all user configured honor_labels. If HonorLabels is set in ServiceMonitor or PodMonitor to true, this overrides honor_labels to false.", "type": "boolean" }, "overrideHonorTimestamps": { "description": "OverrideHonorTimestamps allows to globally enforce honoring timestamps in all scrape configs.", "type": "boolean" }, "paused": { "description": "When a Prometheus deployment is paused, no actions except for deletion will be performed on the underlying objects.", "type": "boolean" }, "podMetadata": { "description": "PodMetadata configures Labels and Annotations which are propagated to the prometheus pods.", "properties": { "annotations": { "additionalProperties": { "type": "string" }, "description": "Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations", "type": "object" }, "labels": { "additionalProperties": { "type": "string" }, "description": "Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels", "type": "object" }, "name": { "description": "Name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names", "type": "string" } }, "type": "object" }, "podMonitorNamespaceSelector": { "description": "Namespaces to be selected for PodMonitor discovery. If nil, only check own namespace.", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "items": { "type": "string" }, "type": "array" } }, "required": [ "key", "operator" ], "type": "object" }, "type": "array" }, "matchLabels": { "additionalProperties": { "type": "string" }, "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object" } }, "type": "object" }, "podMonitorSelector": { "description": "*Experimental* PodMonitors to be selected for target discovery. *Deprecated:* if neither this nor serviceMonitorSelector are specified, configuration is unmanaged.", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "items": { "type": "string" }, "type": "array" } }, "required": [ "key", "operator" ], "type": "object" }, "type": "array" }, "matchLabels": { "additionalProperties": { "type": "string" }, "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object" } }, "type": "object" }, "portName": { "description": "Port name used for the pods and governing service. This defaults to web", "type": "string" }, "priorityClassName": { "description": "Priority class assigned to the Pods", "type": "string" }, "prometheusExternalLabelName": { "description": "Name of Prometheus external label used to denote Prometheus instance name. Defaults to the value of `prometheus`. External label will _not_ be added when value is set to empty string (`\"\"`).", "type": "string" }, "query": { "description": "QuerySpec defines the query command line flags when starting Prometheus.", "properties": { "lookbackDelta": { "description": "The delta difference allowed for retrieving metrics during expression evaluations.", "type": "string" }, "maxConcurrency": { "description": "Number of concurrent queries that can be run at once.", "format": "int32", "type": "integer" }, "maxSamples": { "description": "Maximum number of samples a single query can load into memory. Note that queries will fail if they would load more samples than this into memory, so this also limits the number of samples a query can return.", "format": "int32", "type": "integer" }, "timeout": { "description": "Maximum time a query may take before being aborted.", "type": "string" } }, "type": "object" }, "queryLogFile": { "description": "QueryLogFile specifies the file to which PromQL queries are logged. Note that this location must be writable, and can be persisted using an attached volume. Alternatively, the location can be set to a stdout location such as `/dev/stdout` to log querie information to the default Prometheus log stream. This is only available in versions of Prometheus \u003e= 2.16.0. For more details, see the Prometheus docs (https://prometheus.io/docs/guides/query-log/)", "type": "string" }, "remoteRead": { "description": "If specified, the remote_read spec. This is an experimental feature, it may change in any upcoming release in a breaking way.", "items": { "description": "RemoteReadSpec defines the remote_read configuration for prometheus.", "properties": { "basicAuth": { "description": "BasicAuth for the URL.", "properties": { "password": { "description": "The secret in the service monitor namespace that contains the password for authentication.", "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" }, "optional": { "description": "Specify whether the Secret or its key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" }, "username": { "description": "The secret in the service monitor namespace that contains the username for authentication.", "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" }, "optional": { "description": "Specify whether the Secret or its key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" } }, "type": "object" }, "bearerToken": { "description": "bearer token for remote read.", "type": "string" }, "bearerTokenFile": { "description": "File to read bearer token for remote read.", "type": "string" }, "name": { "description": "The name of the remote read queue, must be unique if specified. The name is used in metrics and logging in order to differentiate read configurations. Only valid in Prometheus versions 2.15.0 and newer.", "type": "string" }, "proxyUrl": { "description": "Optional ProxyURL", "type": "string" }, "readRecent": { "description": "Whether reads should be made for queries for time ranges that the local storage should have complete data for.", "type": "boolean" }, "remoteTimeout": { "description": "Timeout for requests to the remote read endpoint.", "type": "string" }, "requiredMatchers": { "additionalProperties": { "type": "string" }, "description": "An optional list of equality matchers which have to be present in a selector to query the remote read endpoint.", "type": "object" }, "tlsConfig": { "description": "TLS Config to use for remote read.", "properties": { "ca": { "description": "Struct containing the CA cert to use for the targets.", "properties": { "configMap": { "description": "ConfigMap containing data to use for the targets.", "properties": { "key": { "description": "The key to select.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" }, "optional": { "description": "Specify whether the ConfigMap or its key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" }, "secret": { "description": "Secret containing data to use for the targets.", "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" }, "optional": { "description": "Specify whether the Secret or its key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" } }, "type": "object" }, "caFile": { "description": "Path to the CA cert in the Prometheus container to use for the targets.", "type": "string" }, "cert": { "description": "Struct containing the client cert file for the targets.", "properties": { "configMap": { "description": "ConfigMap containing data to use for the targets.", "properties": { "key": { "description": "The key to select.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" }, "optional": { "description": "Specify whether the ConfigMap or its key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" }, "secret": { "description": "Secret containing data to use for the targets.", "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" }, "optional": { "description": "Specify whether the Secret or its key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" } }, "type": "object" }, "certFile": { "description": "Path to the client cert file in the Prometheus container for the targets.", "type": "string" }, "insecureSkipVerify": { "description": "Disable target certificate validation.", "type": "boolean" }, "keyFile": { "description": "Path to the client key file in the Prometheus container for the targets.", "type": "string" }, "keySecret": { "description": "Secret containing the client key file for the targets.", "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" }, "optional": { "description": "Specify whether the Secret or its key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" }, "serverName": { "description": "Used to verify the hostname for the targets.", "type": "string" } }, "type": "object" }, "url": { "description": "The URL of the endpoint to send samples to.", "type": "string" } }, "required": [ "url" ], "type": "object" }, "type": "array" }, "remoteWrite": { "description": "If specified, the remote_write spec. This is an experimental feature, it may change in any upcoming release in a breaking way.", "items": { "description": "RemoteWriteSpec defines the remote_write configuration for prometheus.", "properties": { "basicAuth": { "description": "BasicAuth for the URL.", "properties": { "password": { "description": "The secret in the service monitor namespace that contains the password for authentication.", "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" }, "optional": { "description": "Specify whether the Secret or its key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" }, "username": { "description": "The secret in the service monitor namespace that contains the username for authentication.", "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" }, "optional": { "description": "Specify whether the Secret or its key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" } }, "type": "object" }, "bearerToken": { "description": "File to read bearer token for remote write.", "type": "string" }, "bearerTokenFile": { "description": "File to read bearer token for remote write.", "type": "string" }, "name": { "description": "The name of the remote write queue, must be unique if specified. The name is used in metrics and logging in order to differentiate queues. Only valid in Prometheus versions 2.15.0 and newer.", "type": "string" }, "proxyUrl": { "description": "Optional ProxyURL", "type": "string" }, "queueConfig": { "description": "QueueConfig allows tuning of the remote write queue parameters.", "properties": { "batchSendDeadline": { "description": "BatchSendDeadline is the maximum time a sample will wait in buffer.", "type": "string" }, "capacity": { "description": "Capacity is the number of samples to buffer per shard before we start dropping them.", "type": "integer" }, "maxBackoff": { "description": "MaxBackoff is the maximum retry delay.", "type": "string" }, "maxRetries": { "description": "MaxRetries is the maximum number of times to retry a batch on recoverable errors.", "type": "integer" }, "maxSamplesPerSend": { "description": "MaxSamplesPerSend is the maximum number of samples per send.", "type": "integer" }, "maxShards": { "description": "MaxShards is the maximum number of shards, i.e. amount of concurrency.", "type": "integer" }, "minBackoff": { "description": "MinBackoff is the initial retry delay. Gets doubled for every retry.", "type": "string" }, "minShards": { "description": "MinShards is the minimum number of shards, i.e. amount of concurrency.", "type": "integer" } }, "type": "object" }, "remoteTimeout": { "description": "Timeout for requests to the remote write endpoint.", "type": "string" }, "tlsConfig": { "description": "TLS Config to use for remote write.", "properties": { "ca": { "description": "Struct containing the CA cert to use for the targets.", "properties": { "configMap": { "description": "ConfigMap containing data to use for the targets.", "properties": { "key": { "description": "The key to select.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" }, "optional": { "description": "Specify whether the ConfigMap or its key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" }, "secret": { "description": "Secret containing data to use for the targets.", "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" }, "optional": { "description": "Specify whether the Secret or its key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" } }, "type": "object" }, "caFile": { "description": "Path to the CA cert in the Prometheus container to use for the targets.", "type": "string" }, "cert": { "description": "Struct containing the client cert file for the targets.", "properties": { "configMap": { "description": "ConfigMap containing data to use for the targets.", "properties": { "key": { "description": "The key to select.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" }, "optional": { "description": "Specify whether the ConfigMap or its key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" }, "secret": { "description": "Secret containing data to use for the targets.", "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" }, "optional": { "description": "Specify whether the Secret or its key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" } }, "type": "object" }, "certFile": { "description": "Path to the client cert file in the Prometheus container for the targets.", "type": "string" }, "insecureSkipVerify": { "description": "Disable target certificate validation.", "type": "boolean" }, "keyFile": { "description": "Path to the client key file in the Prometheus container for the targets.", "type": "string" }, "keySecret": { "description": "Secret containing the client key file for the targets.", "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" }, "optional": { "description": "Specify whether the Secret or its key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" }, "serverName": { "description": "Used to verify the hostname for the targets.", "type": "string" } }, "type": "object" }, "url": { "description": "The URL of the endpoint to send samples to.", "type": "string" }, "writeRelabelConfigs": { "description": "The list of remote write relabel configurations.", "items": { "description": "RelabelConfig allows dynamic rewriting of the label set, being applied to samples before ingestion. It defines `\u003cmetric_relabel_configs\u003e`-section of Prometheus configuration. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs", "properties": { "action": { "description": "Action to perform based on regex matching. Default is 'replace'", "type": "string" }, "modulus": { "description": "Modulus to take of the hash of the source label values.", "format": "int64", "type": "integer" }, "regex": { "description": "Regular expression against which the extracted value is matched. Default is '(.*)'", "type": "string" }, "replacement": { "description": "Replacement value against which a regex replace is performed if the regular expression matches. Regex capture groups are available. Default is '$1'", "type": "string" }, "separator": { "description": "Separator placed between concatenated source label values. default is ';'.", "type": "string" }, "sourceLabels": { "description": "The source labels select values from existing labels. Their content is concatenated using the configured separator and matched against the configured regular expression for the replace, keep, and drop actions.", "items": { "type": "string" }, "type": "array" }, "targetLabel": { "description": "Label to which the resulting value is written in a replace action. It is mandatory for replace actions. Regex capture groups are available.", "type": "string" } }, "type": "object" }, "type": "array" } }, "required": [ "url" ], "type": "object" }, "type": "array" }, "replicaExternalLabelName": { "description": "Name of Prometheus external label used to denote replica name. Defaults to the value of `prometheus_replica`. External label will _not_ be added when value is set to empty string (`\"\"`).", "type": "string" }, "replicas": { "description": "Number of instances to deploy for a Prometheus deployment.", "format": "int32", "type": "integer" }, "resources": { "description": "Define resources requests and limits for single Pods.", "properties": { "limits": { "additionalProperties": { "type": "string" }, "description": "Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", "type": "object" }, "requests": { "additionalProperties": { "type": "string" }, "description": "Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", "type": "object" } }, "type": "object" }, "retention": { "description": "Time duration Prometheus shall retain data for. Default is '24h', and must match the regular expression `[0-9]+(ms|s|m|h|d|w|y)` (milliseconds seconds minutes hours days weeks years).", "type": "string" }, "retentionSize": { "description": "Maximum amount of disk space used by blocks.", "type": "string" }, "routePrefix": { "description": "The route prefix Prometheus registers HTTP handlers for. This is useful, if using ExternalURL and a proxy is rewriting HTTP routes of a request, and the actual ExternalURL is still true, but the server serves requests under a different route prefix. For example for use with `kubectl proxy`.", "type": "string" }, "ruleNamespaceSelector": { "description": "Namespaces to be selected for PrometheusRules discovery. If unspecified, only the same namespace as the Prometheus object is in is used.", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "items": { "type": "string" }, "type": "array" } }, "required": [ "key", "operator" ], "type": "object" }, "type": "array" }, "matchLabels": { "additionalProperties": { "type": "string" }, "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object" } }, "type": "object" }, "ruleSelector": { "description": "A selector to select which PrometheusRules to mount for loading alerting/recording rules from. Until (excluding) Prometheus Operator v0.24.0 Prometheus Operator will migrate any legacy rule ConfigMaps to PrometheusRule custom resources selected by RuleSelector. Make sure it does not match any config maps that you do not want to be migrated.", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "items": { "type": "string" }, "type": "array" } }, "required": [ "key", "operator" ], "type": "object" }, "type": "array" }, "matchLabels": { "additionalProperties": { "type": "string" }, "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object" } }, "type": "object" }, "rules": { "description": "/--rules.*/ command-line arguments.", "properties": { "alert": { "description": "/--rules.alert.*/ command-line arguments", "properties": { "forGracePeriod": { "description": "Minimum duration between alert and restored 'for' state. This is maintained only for alerts with configured 'for' time greater than grace period.", "type": "string" }, "forOutageTolerance": { "description": "Max time to tolerate prometheus outage for restoring 'for' state of alert.", "type": "string" }, "resendDelay": { "description": "Minimum amount of time to wait before resending an alert to Alertmanager.", "type": "string" } }, "type": "object" } }, "type": "object" }, "scrapeInterval": { "description": "Interval between consecutive scrapes.", "type": "string" }, "secrets": { "description": "Secrets is a list of Secrets in the same namespace as the Prometheus object, which shall be mounted into the Prometheus Pods. The Secrets are mounted into /etc/prometheus/secrets/\u003csecret-name\u003e.", "items": { "type": "string" }, "type": "array" }, "securityContext": { "description": "SecurityContext holds pod-level security attributes and common container settings. This defaults to the default PodSecurityContext.", "properties": { "fsGroup": { "description": "A special supplemental group that applies to all containers in a pod. Some volume types allow the Kubelet to change the ownership of that volume to be owned by the pod: \n 1. The owning GID will be the FSGroup 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) 3. The permission bits are OR'd with rw-rw---- \n If unset, the Kubelet will not modify the ownership and permissions of any volume.", "format": "int64", "type": "integer" }, "fsGroupChangePolicy": { "description": "fsGroupChangePolicy defines behavior of changing ownership and permission of the volume before being exposed inside Pod. This field will only apply to volume types which support fsGroup based ownership(and permissions). It will have no effect on ephemeral volume types such as: secret, configmaps and emptydir. Valid values are \"OnRootMismatch\" and \"Always\". If not specified defaults to \"Always\".", "type": "string" }, "runAsGroup": { "description": "The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container.", "format": "int64", "type": "integer" }, "runAsNonRoot": { "description": "Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.", "type": "boolean" }, "runAsUser": { "description": "The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container.", "format": "int64", "type": "integer" }, "seLinuxOptions": { "description": "The SELinux context to be applied to all containers. If unspecified, the container runtime will allocate a random SELinux context for each container. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container.", "properties": { "level": { "description": "Level is SELinux level label that applies to the container.", "type": "string" }, "role": { "description": "Role is a SELinux role label that applies to the container.", "type": "string" }, "type": { "description": "Type is a SELinux type label that applies to the container.", "type": "string" }, "user": { "description": "User is a SELinux user label that applies to the container.", "type": "string" } }, "type": "object" }, "supplementalGroups": { "description": "A list of groups applied to the first process run in each container, in addition to the container's primary GID. If unspecified, no groups will be added to any container.", "items": { "format": "int64", "type": "integer" }, "type": "array" }, "sysctls": { "description": "Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported sysctls (by the container runtime) might fail to launch.", "items": { "description": "Sysctl defines a kernel parameter to be set", "properties": { "name": { "description": "Name of a property to set", "type": "string" }, "value": { "description": "Value of a property to set", "type": "string" } }, "required": [ "name", "value" ], "type": "object" }, "type": "array" }, "windowsOptions": { "description": "The Windows specific settings applied to all containers. If unspecified, the options within a container's SecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.", "properties": { "gmsaCredentialSpec": { "description": "GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field.", "type": "string" }, "gmsaCredentialSpecName": { "description": "GMSACredentialSpecName is the name of the GMSA credential spec to use.", "type": "string" }, "runAsUserName": { "description": "The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.", "type": "string" } }, "type": "object" } }, "type": "object" }, "serviceAccountName": { "description": "ServiceAccountName is the name of the ServiceAccount to use to run the Prometheus Pods.", "type": "string" }, "serviceMonitorNamespaceSelector": { "description": "Namespaces to be selected for ServiceMonitor discovery. If nil, only check own namespace.", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "items": { "type": "string" }, "type": "array" } }, "required": [ "key", "operator" ], "type": "object" }, "type": "array" }, "matchLabels": { "additionalProperties": { "type": "string" }, "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object" } }, "type": "object" }, "serviceMonitorSelector": { "description": "ServiceMonitors to be selected for target discovery. *Deprecated:* if neither this nor podMonitorSelector are specified, configuration is unmanaged.", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "items": { "type": "string" }, "type": "array" } }, "required": [ "key", "operator" ], "type": "object" }, "type": "array" }, "matchLabels": { "additionalProperties": { "type": "string" }, "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object" } }, "type": "object" }, "sha": { "description": "SHA of Prometheus container image to be deployed. Defaults to the value of `version`. Similar to a tag, but the SHA explicitly deploys an immutable container image. Version and Tag are ignored if SHA is set.", "type": "string" }, "storage": { "description": "Storage spec to specify how storage shall be used.", "properties": { "disableMountSubPath": { "description": "Deprecated: subPath usage will be disabled by default in a future release, this option will become unnecessary. DisableMountSubPath allows to remove any subPath usage in volume mounts.", "type": "boolean" }, "emptyDir": { "description": "EmptyDirVolumeSource to be used by the Prometheus StatefulSets. If specified, used in place of any volumeClaimTemplate. More info: https://kubernetes.io/docs/concepts/storage/volumes/#emptydir", "properties": { "medium": { "description": "What type of storage medium should back this directory. The default is \"\" which means to use the node's default medium. Must be an empty string (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir", "type": "string" }, "sizeLimit": { "description": "Total amount of local storage required for this EmptyDir volume. The size limit is also applicable for memory medium. The maximum usage on memory medium EmptyDir would be the minimum value between the SizeLimit specified here and the sum of memory limits of all containers in a pod. The default is nil which means that the limit is undefined. More info: http://kubernetes.io/docs/user-guide/volumes#emptydir", "type": "string" } }, "type": "object" }, "volumeClaimTemplate": { "description": "A PVC spec to be used by the Prometheus StatefulSets.", "properties": { "apiVersion": { "description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", "type": "string" }, "kind": { "description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", "type": "string" }, "metadata": { "description": "EmbeddedMetadata contains metadata relevant to an EmbeddedResource.", "properties": { "annotations": { "additionalProperties": { "type": "string" }, "description": "Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations", "type": "object" }, "labels": { "additionalProperties": { "type": "string" }, "description": "Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels", "type": "object" }, "name": { "description": "Name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names", "type": "string" } }, "type": "object" }, "spec": { "description": "Spec defines the desired characteristics of a volume requested by a pod author. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims", "properties": { "accessModes": { "description": "AccessModes contains the desired access modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1", "items": { "type": "string" }, "type": "array" }, "dataSource": { "description": "This field can be used to specify either: * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot - Beta) * An existing PVC (PersistentVolumeClaim) * An existing custom resource/object that implements data population (Alpha) In order to use VolumeSnapshot object types, the appropriate feature gate must be enabled (VolumeSnapshotDataSource or AnyVolumeDataSource) If the provisioner or an external controller can support the specified data source, it will create a new volume based on the contents of the specified data source. If the specified data source is not supported, the volume will not be created and the failure will be reported as an event. In the future, we plan to support more data source types and the behavior of the provisioner may change.", "properties": { "apiGroup": { "description": "APIGroup is the group for the resource being referenced. If APIGroup is not specified, the specified Kind must be in the core API group. For any other third-party types, APIGroup is required.", "type": "string" }, "kind": { "description": "Kind is the type of resource being referenced", "type": "string" }, "name": { "description": "Name is the name of resource being referenced", "type": "string" } }, "required": [ "kind", "name" ], "type": "object" }, "resources": { "description": "Resources represents the minimum resources the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources", "properties": { "limits": { "additionalProperties": { "type": "string" }, "description": "Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", "type": "object" }, "requests": { "additionalProperties": { "type": "string" }, "description": "Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", "type": "object" } }, "type": "object" }, "selector": { "description": "A label query over volumes to consider for binding.", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "items": { "type": "string" }, "type": "array" } }, "required": [ "key", "operator" ], "type": "object" }, "type": "array" }, "matchLabels": { "additionalProperties": { "type": "string" }, "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object" } }, "type": "object" }, "storageClassName": { "description": "Name of the StorageClass required by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1", "type": "string" }, "volumeMode": { "description": "volumeMode defines what type of volume is required by the claim. Value of Filesystem is implied when not included in claim spec.", "type": "string" }, "volumeName": { "description": "VolumeName is the binding reference to the PersistentVolume backing this claim.", "type": "string" } }, "type": "object" }, "status": { "description": "Status represents the current information/status of a persistent volume claim. Read-only. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims", "properties": { "accessModes": { "description": "AccessModes contains the actual access modes the volume backing the PVC has. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1", "items": { "type": "string" }, "type": "array" }, "capacity": { "additionalProperties": { "type": "string" }, "description": "Represents the actual resources of the underlying volume.", "type": "object" }, "conditions": { "description": "Current Condition of persistent volume claim. If underlying persistent volume is being resized then the Condition will be set to 'ResizeStarted'.", "items": { "description": "PersistentVolumeClaimCondition contains details about state of pvc", "properties": { "lastProbeTime": { "description": "Last time we probed the condition.", "format": "date-time", "type": "string" }, "lastTransitionTime": { "description": "Last time the condition transitioned from one status to another.", "format": "date-time", "type": "string" }, "message": { "description": "Human-readable message indicating details about last transition.", "type": "string" }, "reason": { "description": "Unique, this should be a short, machine understandable string that gives the reason for condition's last transition. If it reports \"ResizeStarted\" that means the underlying persistent volume is being resized.", "type": "string" }, "status": { "type": "string" }, "type": { "description": "PersistentVolumeClaimConditionType is a valid value of PersistentVolumeClaimCondition.Type", "type": "string" } }, "required": [ "status", "type" ], "type": "object" }, "type": "array" }, "phase": { "description": "Phase represents the current phase of PersistentVolumeClaim.", "type": "string" } }, "type": "object" } }, "type": "object" } }, "type": "object" }, "tag": { "description": "Tag of Prometheus container image to be deployed. Defaults to the value of `version`. Version is ignored if Tag is set.", "type": "string" }, "thanos": { "description": "Thanos configuration allows configuring various aspects of a Prometheus server in a Thanos environment. \n This section is experimental, it may change significantly without deprecation notice in any release. \n This is experimental and may change significantly without backward compatibility in any release.", "properties": { "baseImage": { "description": "Thanos base image if other than default.", "type": "string" }, "grpcServerTlsConfig": { "description": "GRPCServerTLSConfig configures the gRPC server from which Thanos Querier reads recorded rule data. Note: Currently only the CAFile, CertFile, and KeyFile fields are supported. Maps to the '--grpc-server-tls-*' CLI args.", "properties": { "ca": { "description": "Struct containing the CA cert to use for the targets.", "properties": { "configMap": { "description": "ConfigMap containing data to use for the targets.", "properties": { "key": { "description": "The key to select.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" }, "optional": { "description": "Specify whether the ConfigMap or its key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" }, "secret": { "description": "Secret containing data to use for the targets.", "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" }, "optional": { "description": "Specify whether the Secret or its key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" } }, "type": "object" }, "caFile": { "description": "Path to the CA cert in the Prometheus container to use for the targets.", "type": "string" }, "cert": { "description": "Struct containing the client cert file for the targets.", "properties": { "configMap": { "description": "ConfigMap containing data to use for the targets.", "properties": { "key": { "description": "The key to select.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" }, "optional": { "description": "Specify whether the ConfigMap or its key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" }, "secret": { "description": "Secret containing data to use for the targets.", "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" }, "optional": { "description": "Specify whether the Secret or its key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" } }, "type": "object" }, "certFile": { "description": "Path to the client cert file in the Prometheus container for the targets.", "type": "string" }, "insecureSkipVerify": { "description": "Disable target certificate validation.", "type": "boolean" }, "keyFile": { "description": "Path to the client key file in the Prometheus container for the targets.", "type": "string" }, "keySecret": { "description": "Secret containing the client key file for the targets.", "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" }, "optional": { "description": "Specify whether the Secret or its key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" }, "serverName": { "description": "Used to verify the hostname for the targets.", "type": "string" } }, "type": "object" }, "image": { "description": "Image if specified has precedence over baseImage, tag and sha combinations. Specifying the version is still necessary to ensure the Prometheus Operator knows what version of Thanos is being configured.", "type": "string" }, "listenLocal": { "description": "ListenLocal makes the Thanos sidecar listen on loopback, so that it does not bind against the Pod IP.", "type": "boolean" }, "logFormat": { "description": "LogFormat for Thanos sidecar to be configured with.", "type": "string" }, "logLevel": { "description": "LogLevel for Thanos sidecar to be configured with.", "type": "string" }, "objectStorageConfig": { "description": "ObjectStorageConfig configures object storage in Thanos.", "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" }, "optional": { "description": "Specify whether the Secret or its key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" }, "resources": { "description": "Resources defines the resource requirements for the Thanos sidecar. If not provided, no requests/limits will be set", "properties": { "limits": { "additionalProperties": { "type": "string" }, "description": "Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", "type": "object" }, "requests": { "additionalProperties": { "type": "string" }, "description": "Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", "type": "object" } }, "type": "object" }, "sha": { "description": "SHA of Thanos container image to be deployed. Defaults to the value of `version`. Similar to a tag, but the SHA explicitly deploys an immutable container image. Version and Tag are ignored if SHA is set.", "type": "string" }, "tag": { "description": "Tag of Thanos sidecar container image to be deployed. Defaults to the value of `version`. Version is ignored if Tag is set.", "type": "string" }, "tracingConfig": { "description": "TracingConfig configures tracing in Thanos. This is an experimental feature, it may change in any upcoming release in a breaking way.", "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" }, "optional": { "description": "Specify whether the Secret or its key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" }, "version": { "description": "Version describes the version of Thanos to use.", "type": "string" } }, "type": "object" }, "tolerations": { "description": "If specified, the pod's tolerations.", "items": { "description": "The pod this Toleration is attached to tolerates any taint that matches the triple \u003ckey,value,effect\u003e using the matching operator \u003coperator\u003e.", "properties": { "effect": { "description": "Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.", "type": "string" }, "key": { "description": "Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys.", "type": "string" }, "operator": { "description": "Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category.", "type": "string" }, "tolerationSeconds": { "description": "TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system.", "format": "int64", "type": "integer" }, "value": { "description": "Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string.", "type": "string" } }, "type": "object" }, "type": "array" }, "version": { "description": "Version of Prometheus to be deployed.", "type": "string" }, "volumeMounts": { "description": "VolumeMounts allows configuration of additional VolumeMounts on the output StatefulSet definition. VolumeMounts specified will be appended to other VolumeMounts in the prometheus container, that are generated as a result of StorageSpec objects.", "items": { "description": "VolumeMount describes a mounting of a Volume within a container.", "properties": { "mountPath": { "description": "Path within the container at which the volume should be mounted. Must not contain ':'.", "type": "string" }, "mountPropagation": { "description": "mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10.", "type": "string" }, "name": { "description": "This must match the Name of a Volume.", "type": "string" }, "readOnly": { "description": "Mounted read-only if true, read-write otherwise (false or unspecified). Defaults to false.", "type": "boolean" }, "subPath": { "description": "Path within the volume from which the container's volume should be mounted. Defaults to \"\" (volume's root).", "type": "string" }, "subPathExpr": { "description": "Expanded path within the volume from which the container's volume should be mounted. Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. Defaults to \"\" (volume's root). SubPathExpr and SubPath are mutually exclusive.", "type": "string" } }, "required": [ "mountPath", "name" ], "type": "object" }, "type": "array" }, "volumes": { "description": "Volumes allows configuration of additional volumes on the output StatefulSet definition. Volumes specified will be appended to other volumes that are generated as a result of StorageSpec objects.", "items": { "description": "Volume represents a named volume in a pod that may be accessed by any container in the pod.", "properties": { "awsElasticBlockStore": { "description": "AWSElasticBlockStore represents an AWS Disk resource that is attached to a kubelet's host machine and then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore", "properties": { "fsType": { "description": "Filesystem type of the volume that you want to mount. Tip: Ensure that the filesystem type is supported by the host operating system. Examples: \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore TODO: how do we prevent errors in the filesystem from compromising the machine", "type": "string" }, "partition": { "description": "The partition in the volume that you want to mount. If omitted, the default is to mount by volume name. Examples: For volume /dev/sda1, you specify the partition as \"1\". Similarly, the volume partition for /dev/sda is \"0\" (or you can leave the property empty).", "format": "int32", "type": "integer" }, "readOnly": { "description": "Specify \"true\" to force and set the ReadOnly property in VolumeMounts to \"true\". If omitted, the default is \"false\". More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore", "type": "boolean" }, "volumeID": { "description": "Unique ID of the persistent disk resource in AWS (Amazon EBS volume). More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore", "type": "string" } }, "required": [ "volumeID" ], "type": "object" }, "azureDisk": { "description": "AzureDisk represents an Azure Data Disk mount on the host and bind mount to the pod.", "properties": { "cachingMode": { "description": "Host Caching mode: None, Read Only, Read Write.", "type": "string" }, "diskName": { "description": "The Name of the data disk in the blob storage", "type": "string" }, "diskURI": { "description": "The URI the data disk in the blob storage", "type": "string" }, "fsType": { "description": "Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified.", "type": "string" }, "kind": { "description": "Expected values Shared: multiple blob disks per storage account Dedicated: single blob disk per storage account Managed: azure managed data disk (only in managed availability set). defaults to shared", "type": "string" }, "readOnly": { "description": "Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.", "type": "boolean" } }, "required": [ "diskName", "diskURI" ], "type": "object" }, "azureFile": { "description": "AzureFile represents an Azure File Service mount on the host and bind mount to the pod.", "properties": { "readOnly": { "description": "Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.", "type": "boolean" }, "secretName": { "description": "the name of secret that contains Azure Storage Account Name and Key", "type": "string" }, "shareName": { "description": "Share Name", "type": "string" } }, "required": [ "secretName", "shareName" ], "type": "object" }, "cephfs": { "description": "CephFS represents a Ceph FS mount on the host that shares a pod's lifetime", "properties": { "monitors": { "description": "Required: Monitors is a collection of Ceph monitors More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it", "items": { "type": "string" }, "type": "array" }, "path": { "description": "Optional: Used as the mounted root, rather than the full Ceph tree, default is /", "type": "string" }, "readOnly": { "description": "Optional: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it", "type": "boolean" }, "secretFile": { "description": "Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it", "type": "string" }, "secretRef": { "description": "Optional: SecretRef is reference to the authentication secret for User, default is empty. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it", "properties": { "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" } }, "type": "object" }, "user": { "description": "Optional: User is the rados user name, default is admin More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it", "type": "string" } }, "required": [ "monitors" ], "type": "object" }, "cinder": { "description": "Cinder represents a cinder volume attached and mounted on kubelets host machine. More info: https://examples.k8s.io/mysql-cinder-pd/README.md", "properties": { "fsType": { "description": "Filesystem type to mount. Must be a filesystem type supported by the host operating system. Examples: \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified. More info: https://examples.k8s.io/mysql-cinder-pd/README.md", "type": "string" }, "readOnly": { "description": "Optional: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. More info: https://examples.k8s.io/mysql-cinder-pd/README.md", "type": "boolean" }, "secretRef": { "description": "Optional: points to a secret object containing parameters used to connect to OpenStack.", "properties": { "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" } }, "type": "object" }, "volumeID": { "description": "volume id used to identify the volume in cinder. More info: https://examples.k8s.io/mysql-cinder-pd/README.md", "type": "string" } }, "required": [ "volumeID" ], "type": "object" }, "configMap": { "description": "ConfigMap represents a configMap that should populate this volume", "properties": { "defaultMode": { "description": "Optional: mode bits to use on created files by default. Must be a value between 0 and 0777. Defaults to 0644. Directories within the path are not affected by this setting. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.", "format": "int32", "type": "integer" }, "items": { "description": "If unspecified, each key-value pair in the Data field of the referenced ConfigMap will be projected into the volume as a file whose name is the key and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the ConfigMap, the volume setup will error unless it is marked optional. Paths must be relative and may not contain the '..' path or start with '..'.", "items": { "description": "Maps a string key to a path within a volume.", "properties": { "key": { "description": "The key to project.", "type": "string" }, "mode": { "description": "Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.", "format": "int32", "type": "integer" }, "path": { "description": "The relative path of the file to map the key to. May not be an absolute path. May not contain the path element '..'. May not start with the string '..'.", "type": "string" } }, "required": [ "key", "path" ], "type": "object" }, "type": "array" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" }, "optional": { "description": "Specify whether the ConfigMap or its keys must be defined", "type": "boolean" } }, "type": "object" }, "csi": { "description": "CSI (Container Storage Interface) represents storage that is handled by an external CSI driver (Alpha feature).", "properties": { "driver": { "description": "Driver is the name of the CSI driver that handles this volume. Consult with your admin for the correct name as registered in the cluster.", "type": "string" }, "fsType": { "description": "Filesystem type to mount. Ex. \"ext4\", \"xfs\", \"ntfs\". If not provided, the empty value is passed to the associated CSI driver which will determine the default filesystem to apply.", "type": "string" }, "nodePublishSecretRef": { "description": "NodePublishSecretRef is a reference to the secret object containing sensitive information to pass to the CSI driver to complete the CSI NodePublishVolume and NodeUnpublishVolume calls. This field is optional, and may be empty if no secret is required. If the secret object contains more than one secret, all secret references are passed.", "properties": { "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" } }, "type": "object" }, "readOnly": { "description": "Specifies a read-only configuration for the volume. Defaults to false (read/write).", "type": "boolean" }, "volumeAttributes": { "additionalProperties": { "type": "string" }, "description": "VolumeAttributes stores driver-specific properties that are passed to the CSI driver. Consult your driver's documentation for supported values.", "type": "object" } }, "required": [ "driver" ], "type": "object" }, "downwardAPI": { "description": "DownwardAPI represents downward API about the pod that should populate this volume", "properties": { "defaultMode": { "description": "Optional: mode bits to use on created files by default. Must be a value between 0 and 0777. Defaults to 0644. Directories within the path are not affected by this setting. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.", "format": "int32", "type": "integer" }, "items": { "description": "Items is a list of downward API volume file", "items": { "description": "DownwardAPIVolumeFile represents information to create the file containing the pod field", "properties": { "fieldRef": { "description": "Required: Selects a field of the pod: only annotations, labels, name and namespace are supported.", "properties": { "apiVersion": { "description": "Version of the schema the FieldPath is written in terms of, defaults to \"v1\".", "type": "string" }, "fieldPath": { "description": "Path of the field to select in the specified API version.", "type": "string" } }, "required": [ "fieldPath" ], "type": "object" }, "mode": { "description": "Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.", "format": "int32", "type": "integer" }, "path": { "description": "Required: Path is the relative path name of the file to be created. Must not be absolute or contain the '..' path. Must be utf-8 encoded. The first item of the relative path must not start with '..'", "type": "string" }, "resourceFieldRef": { "description": "Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported.", "properties": { "containerName": { "description": "Container name: required for volumes, optional for env vars", "type": "string" }, "divisor": { "description": "Specifies the output format of the exposed resources, defaults to \"1\"", "type": "string" }, "resource": { "description": "Required: resource to select", "type": "string" } }, "required": [ "resource" ], "type": "object" } }, "required": [ "path" ], "type": "object" }, "type": "array" } }, "type": "object" }, "emptyDir": { "description": "EmptyDir represents a temporary directory that shares a pod's lifetime. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir", "properties": { "medium": { "description": "What type of storage medium should back this directory. The default is \"\" which means to use the node's default medium. Must be an empty string (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir", "type": "string" }, "sizeLimit": { "description": "Total amount of local storage required for this EmptyDir volume. The size limit is also applicable for memory medium. The maximum usage on memory medium EmptyDir would be the minimum value between the SizeLimit specified here and the sum of memory limits of all containers in a pod. The default is nil which means that the limit is undefined. More info: http://kubernetes.io/docs/user-guide/volumes#emptydir", "type": "string" } }, "type": "object" }, "fc": { "description": "FC represents a Fibre Channel resource that is attached to a kubelet's host machine and then exposed to the pod.", "properties": { "fsType": { "description": "Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified. TODO: how do we prevent errors in the filesystem from compromising the machine", "type": "string" }, "lun": { "description": "Optional: FC target lun number", "format": "int32", "type": "integer" }, "readOnly": { "description": "Optional: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.", "type": "boolean" }, "targetWWNs": { "description": "Optional: FC target worldwide names (WWNs)", "items": { "type": "string" }, "type": "array" }, "wwids": { "description": "Optional: FC volume world wide identifiers (wwids) Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously.", "items": { "type": "string" }, "type": "array" } }, "type": "object" }, "flexVolume": { "description": "FlexVolume represents a generic volume resource that is provisioned/attached using an exec based plugin.", "properties": { "driver": { "description": "Driver is the name of the driver to use for this volume.", "type": "string" }, "fsType": { "description": "Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". The default filesystem depends on FlexVolume script.", "type": "string" }, "options": { "additionalProperties": { "type": "string" }, "description": "Optional: Extra command options if any.", "type": "object" }, "readOnly": { "description": "Optional: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.", "type": "boolean" }, "secretRef": { "description": "Optional: SecretRef is reference to the secret object containing sensitive information to pass to the plugin scripts. This may be empty if no secret object is specified. If the secret object contains more than one secret, all secrets are passed to the plugin scripts.", "properties": { "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" } }, "type": "object" } }, "required": [ "driver" ], "type": "object" }, "flocker": { "description": "Flocker represents a Flocker volume attached to a kubelet's host machine. This depends on the Flocker control service being running", "properties": { "datasetName": { "description": "Name of the dataset stored as metadata -\u003e name on the dataset for Flocker should be considered as deprecated", "type": "string" }, "datasetUUID": { "description": "UUID of the dataset. This is unique identifier of a Flocker dataset", "type": "string" } }, "type": "object" }, "gcePersistentDisk": { "description": "GCEPersistentDisk represents a GCE Disk resource that is attached to a kubelet's host machine and then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk", "properties": { "fsType": { "description": "Filesystem type of the volume that you want to mount. Tip: Ensure that the filesystem type is supported by the host operating system. Examples: \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk TODO: how do we prevent errors in the filesystem from compromising the machine", "type": "string" }, "partition": { "description": "The partition in the volume that you want to mount. If omitted, the default is to mount by volume name. Examples: For volume /dev/sda1, you specify the partition as \"1\". Similarly, the volume partition for /dev/sda is \"0\" (or you can leave the property empty). More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk", "format": "int32", "type": "integer" }, "pdName": { "description": "Unique name of the PD resource in GCE. Used to identify the disk in GCE. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk", "type": "string" }, "readOnly": { "description": "ReadOnly here will force the ReadOnly setting in VolumeMounts. Defaults to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk", "type": "boolean" } }, "required": [ "pdName" ], "type": "object" }, "gitRepo": { "description": "GitRepo represents a git repository at a particular revision. DEPRECATED: GitRepo is deprecated. To provision a container with a git repo, mount an EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir into the Pod's container.", "properties": { "directory": { "description": "Target directory name. Must not contain or start with '..'. If '.' is supplied, the volume directory will be the git repository. Otherwise, if specified, the volume will contain the git repository in the subdirectory with the given name.", "type": "string" }, "repository": { "description": "Repository URL", "type": "string" }, "revision": { "description": "Commit hash for the specified revision.", "type": "string" } }, "required": [ "repository" ], "type": "object" }, "glusterfs": { "description": "Glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. More info: https://examples.k8s.io/volumes/glusterfs/README.md", "properties": { "endpoints": { "description": "EndpointsName is the endpoint name that details Glusterfs topology. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod", "type": "string" }, "path": { "description": "Path is the Glusterfs volume path. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod", "type": "string" }, "readOnly": { "description": "ReadOnly here will force the Glusterfs volume to be mounted with read-only permissions. Defaults to false. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod", "type": "boolean" } }, "required": [ "endpoints", "path" ], "type": "object" }, "hostPath": { "description": "HostPath represents a pre-existing file or directory on the host machine that is directly exposed to the container. This is generally used for system agents or other privileged things that are allowed to see the host machine. Most containers will NOT need this. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath --- TODO(jonesdl) We need to restrict who can use host directory mounts and who can/can not mount host directories as read/write.", "properties": { "path": { "description": "Path of the directory on the host. If the path is a symlink, it will follow the link to the real path. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath", "type": "string" }, "type": { "description": "Type for HostPath Volume Defaults to \"\" More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath", "type": "string" } }, "required": [ "path" ], "type": "object" }, "iscsi": { "description": "ISCSI represents an ISCSI Disk resource that is attached to a kubelet's host machine and then exposed to the pod. More info: https://examples.k8s.io/volumes/iscsi/README.md", "properties": { "chapAuthDiscovery": { "description": "whether support iSCSI Discovery CHAP authentication", "type": "boolean" }, "chapAuthSession": { "description": "whether support iSCSI Session CHAP authentication", "type": "boolean" }, "fsType": { "description": "Filesystem type of the volume that you want to mount. Tip: Ensure that the filesystem type is supported by the host operating system. Examples: \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi TODO: how do we prevent errors in the filesystem from compromising the machine", "type": "string" }, "initiatorName": { "description": "Custom iSCSI Initiator Name. If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface \u003ctarget portal\u003e:\u003cvolume name\u003e will be created for the connection.", "type": "string" }, "iqn": { "description": "Target iSCSI Qualified Name.", "type": "string" }, "iscsiInterface": { "description": "iSCSI Interface Name that uses an iSCSI transport. Defaults to 'default' (tcp).", "type": "string" }, "lun": { "description": "iSCSI Target Lun number.", "format": "int32", "type": "integer" }, "portals": { "description": "iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port is other than default (typically TCP ports 860 and 3260).", "items": { "type": "string" }, "type": "array" }, "readOnly": { "description": "ReadOnly here will force the ReadOnly setting in VolumeMounts. Defaults to false.", "type": "boolean" }, "secretRef": { "description": "CHAP Secret for iSCSI target and initiator authentication", "properties": { "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" } }, "type": "object" }, "targetPortal": { "description": "iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port is other than default (typically TCP ports 860 and 3260).", "type": "string" } }, "required": [ "iqn", "lun", "targetPortal" ], "type": "object" }, "name": { "description": "Volume's name. Must be a DNS_LABEL and unique within the pod. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", "type": "string" }, "nfs": { "description": "NFS represents an NFS mount on the host that shares a pod's lifetime More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs", "properties": { "path": { "description": "Path that is exported by the NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs", "type": "string" }, "readOnly": { "description": "ReadOnly here will force the NFS export to be mounted with read-only permissions. Defaults to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs", "type": "boolean" }, "server": { "description": "Server is the hostname or IP address of the NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs", "type": "string" } }, "required": [ "path", "server" ], "type": "object" }, "persistentVolumeClaim": { "description": "PersistentVolumeClaimVolumeSource represents a reference to a PersistentVolumeClaim in the same namespace. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims", "properties": { "claimName": { "description": "ClaimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims", "type": "string" }, "readOnly": { "description": "Will force the ReadOnly setting in VolumeMounts. Default false.", "type": "boolean" } }, "required": [ "claimName" ], "type": "object" }, "photonPersistentDisk": { "description": "PhotonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine", "properties": { "fsType": { "description": "Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified.", "type": "string" }, "pdID": { "description": "ID that identifies Photon Controller persistent disk", "type": "string" } }, "required": [ "pdID" ], "type": "object" }, "portworxVolume": { "description": "PortworxVolume represents a portworx volume attached and mounted on kubelets host machine", "properties": { "fsType": { "description": "FSType represents the filesystem type to mount Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\". Implicitly inferred to be \"ext4\" if unspecified.", "type": "string" }, "readOnly": { "description": "Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.", "type": "boolean" }, "volumeID": { "description": "VolumeID uniquely identifies a Portworx volume", "type": "string" } }, "required": [ "volumeID" ], "type": "object" }, "projected": { "description": "Items for all in one resources secrets, configmaps, and downward API", "properties": { "defaultMode": { "description": "Mode bits to use on created files by default. Must be a value between 0 and 0777. Directories within the path are not affected by this setting. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.", "format": "int32", "type": "integer" }, "sources": { "description": "list of volume projections", "items": { "description": "Projection that may be projected along with other supported volume types", "properties": { "configMap": { "description": "information about the configMap data to project", "properties": { "items": { "description": "If unspecified, each key-value pair in the Data field of the referenced ConfigMap will be projected into the volume as a file whose name is the key and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the ConfigMap, the volume setup will error unless it is marked optional. Paths must be relative and may not contain the '..' path or start with '..'.", "items": { "description": "Maps a string key to a path within a volume.", "properties": { "key": { "description": "The key to project.", "type": "string" }, "mode": { "description": "Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.", "format": "int32", "type": "integer" }, "path": { "description": "The relative path of the file to map the key to. May not be an absolute path. May not contain the path element '..'. May not start with the string '..'.", "type": "string" } }, "required": [ "key", "path" ], "type": "object" }, "type": "array" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" }, "optional": { "description": "Specify whether the ConfigMap or its keys must be defined", "type": "boolean" } }, "type": "object" }, "downwardAPI": { "description": "information about the downwardAPI data to project", "properties": { "items": { "description": "Items is a list of DownwardAPIVolume file", "items": { "description": "DownwardAPIVolumeFile represents information to create the file containing the pod field", "properties": { "fieldRef": { "description": "Required: Selects a field of the pod: only annotations, labels, name and namespace are supported.", "properties": { "apiVersion": { "description": "Version of the schema the FieldPath is written in terms of, defaults to \"v1\".", "type": "string" }, "fieldPath": { "description": "Path of the field to select in the specified API version.", "type": "string" } }, "required": [ "fieldPath" ], "type": "object" }, "mode": { "description": "Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.", "format": "int32", "type": "integer" }, "path": { "description": "Required: Path is the relative path name of the file to be created. Must not be absolute or contain the '..' path. Must be utf-8 encoded. The first item of the relative path must not start with '..'", "type": "string" }, "resourceFieldRef": { "description": "Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported.", "properties": { "containerName": { "description": "Container name: required for volumes, optional for env vars", "type": "string" }, "divisor": { "description": "Specifies the output format of the exposed resources, defaults to \"1\"", "type": "string" }, "resource": { "description": "Required: resource to select", "type": "string" } }, "required": [ "resource" ], "type": "object" } }, "required": [ "path" ], "type": "object" }, "type": "array" } }, "type": "object" }, "secret": { "description": "information about the secret data to project", "properties": { "items": { "description": "If unspecified, each key-value pair in the Data field of the referenced Secret will be projected into the volume as a file whose name is the key and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the Secret, the volume setup will error unless it is marked optional. Paths must be relative and may not contain the '..' path or start with '..'.", "items": { "description": "Maps a string key to a path within a volume.", "properties": { "key": { "description": "The key to project.", "type": "string" }, "mode": { "description": "Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.", "format": "int32", "type": "integer" }, "path": { "description": "The relative path of the file to map the key to. May not be an absolute path. May not contain the path element '..'. May not start with the string '..'.", "type": "string" } }, "required": [ "key", "path" ], "type": "object" }, "type": "array" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" }, "optional": { "description": "Specify whether the Secret or its key must be defined", "type": "boolean" } }, "type": "object" }, "serviceAccountToken": { "description": "information about the serviceAccountToken data to project", "properties": { "audience": { "description": "Audience is the intended audience of the token. A recipient of a token must identify itself with an identifier specified in the audience of the token, and otherwise should reject the token. The audience defaults to the identifier of the apiserver.", "type": "string" }, "expirationSeconds": { "description": "ExpirationSeconds is the requested duration of validity of the service account token. As the token approaches expiration, the kubelet volume plugin will proactively rotate the service account token. The kubelet will start trying to rotate the token if the token is older than 80 percent of its time to live or if the token is older than 24 hours.Defaults to 1 hour and must be at least 10 minutes.", "format": "int64", "type": "integer" }, "path": { "description": "Path is the path relative to the mount point of the file to project the token into.", "type": "string" } }, "required": [ "path" ], "type": "object" } }, "type": "object" }, "type": "array" } }, "required": [ "sources" ], "type": "object" }, "quobyte": { "description": "Quobyte represents a Quobyte mount on the host that shares a pod's lifetime", "properties": { "group": { "description": "Group to map volume access to Default is no group", "type": "string" }, "readOnly": { "description": "ReadOnly here will force the Quobyte volume to be mounted with read-only permissions. Defaults to false.", "type": "boolean" }, "registry": { "description": "Registry represents a single or multiple Quobyte Registry services specified as a string as host:port pair (multiple entries are separated with commas) which acts as the central registry for volumes", "type": "string" }, "tenant": { "description": "Tenant owning the given Quobyte volume in the Backend Used with dynamically provisioned Quobyte volumes, value is set by the plugin", "type": "string" }, "user": { "description": "User to map volume access to Defaults to serivceaccount user", "type": "string" }, "volume": { "description": "Volume is a string that references an already created Quobyte volume by name.", "type": "string" } }, "required": [ "registry", "volume" ], "type": "object" }, "rbd": { "description": "RBD represents a Rados Block Device mount on the host that shares a pod's lifetime. More info: https://examples.k8s.io/volumes/rbd/README.md", "properties": { "fsType": { "description": "Filesystem type of the volume that you want to mount. Tip: Ensure that the filesystem type is supported by the host operating system. Examples: \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd TODO: how do we prevent errors in the filesystem from compromising the machine", "type": "string" }, "image": { "description": "The rados image name. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it", "type": "string" }, "keyring": { "description": "Keyring is the path to key ring for RBDUser. Default is /etc/ceph/keyring. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it", "type": "string" }, "monitors": { "description": "A collection of Ceph monitors. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it", "items": { "type": "string" }, "type": "array" }, "pool": { "description": "The rados pool name. Default is rbd. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it", "type": "string" }, "readOnly": { "description": "ReadOnly here will force the ReadOnly setting in VolumeMounts. Defaults to false. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it", "type": "boolean" }, "secretRef": { "description": "SecretRef is name of the authentication secret for RBDUser. If provided overrides keyring. Default is nil. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it", "properties": { "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" } }, "type": "object" }, "user": { "description": "The rados user name. Default is admin. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it", "type": "string" } }, "required": [ "image", "monitors" ], "type": "object" }, "scaleIO": { "description": "ScaleIO represents a ScaleIO persistent volume attached and mounted on Kubernetes nodes.", "properties": { "fsType": { "description": "Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". Default is \"xfs\".", "type": "string" }, "gateway": { "description": "The host address of the ScaleIO API Gateway.", "type": "string" }, "protectionDomain": { "description": "The name of the ScaleIO Protection Domain for the configured storage.", "type": "string" }, "readOnly": { "description": "Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.", "type": "boolean" }, "secretRef": { "description": "SecretRef references to the secret for ScaleIO user and other sensitive information. If this is not provided, Login operation will fail.", "properties": { "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" } }, "type": "object" }, "sslEnabled": { "description": "Flag to enable/disable SSL communication with Gateway, default false", "type": "boolean" }, "storageMode": { "description": "Indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. Default is ThinProvisioned.", "type": "string" }, "storagePool": { "description": "The ScaleIO Storage Pool associated with the protection domain.", "type": "string" }, "system": { "description": "The name of the storage system as configured in ScaleIO.", "type": "string" }, "volumeName": { "description": "The name of a volume already created in the ScaleIO system that is associated with this volume source.", "type": "string" } }, "required": [ "gateway", "secretRef", "system" ], "type": "object" }, "secret": { "description": "Secret represents a secret that should populate this volume. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret", "properties": { "defaultMode": { "description": "Optional: mode bits to use on created files by default. Must be a value between 0 and 0777. Defaults to 0644. Directories within the path are not affected by this setting. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.", "format": "int32", "type": "integer" }, "items": { "description": "If unspecified, each key-value pair in the Data field of the referenced Secret will be projected into the volume as a file whose name is the key and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the Secret, the volume setup will error unless it is marked optional. Paths must be relative and may not contain the '..' path or start with '..'.", "items": { "description": "Maps a string key to a path within a volume.", "properties": { "key": { "description": "The key to project.", "type": "string" }, "mode": { "description": "Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.", "format": "int32", "type": "integer" }, "path": { "description": "The relative path of the file to map the key to. May not be an absolute path. May not contain the path element '..'. May not start with the string '..'.", "type": "string" } }, "required": [ "key", "path" ], "type": "object" }, "type": "array" }, "optional": { "description": "Specify whether the Secret or its keys must be defined", "type": "boolean" }, "secretName": { "description": "Name of the secret in the pod's namespace to use. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret", "type": "string" } }, "type": "object" }, "storageos": { "description": "StorageOS represents a StorageOS volume attached and mounted on Kubernetes nodes.", "properties": { "fsType": { "description": "Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified.", "type": "string" }, "readOnly": { "description": "Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.", "type": "boolean" }, "secretRef": { "description": "SecretRef specifies the secret to use for obtaining the StorageOS API credentials. If not specified, default values will be attempted.", "properties": { "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", "type": "string" } }, "type": "object" }, "volumeName": { "description": "VolumeName is the human-readable name of the StorageOS volume. Volume names are only unique within a namespace.", "type": "string" }, "volumeNamespace": { "description": "VolumeNamespace specifies the scope of the volume within StorageOS. If no namespace is specified then the Pod's namespace will be used. This allows the Kubernetes name scoping to be mirrored within StorageOS for tighter integration. Set VolumeName to any name to override the default behaviour. Set to \"default\" if you are not using namespaces within StorageOS. Namespaces that do not pre-exist within StorageOS will be created.", "type": "string" } }, "type": "object" }, "vsphereVolume": { "description": "VsphereVolume represents a vSphere volume attached and mounted on kubelets host machine", "properties": { "fsType": { "description": "Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified.", "type": "string" }, "storagePolicyID": { "description": "Storage Policy Based Management (SPBM) profile ID associated with the StoragePolicyName.", "type": "string" }, "storagePolicyName": { "description": "Storage Policy Based Management (SPBM) profile name.", "type": "string" }, "volumePath": { "description": "Path that identifies vSphere volume vmdk", "type": "string" } }, "required": [ "volumePath" ], "type": "object" } }, "required": [ "name" ], "type": "object" }, "type": "array" }, "walCompression": { "description": "Enable compression of the write-ahead log using Snappy. This flag is only available in versions of Prometheus \u003e= 2.11.0.", "type": "boolean" } }, "type": "object" }, "status": { "description": "Most recent observed status of the Prometheus cluster. Read-only. Not included when requesting from the apiserver, only from the Prometheus Operator API itself. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status", "properties": { "availableReplicas": { "description": "Total number of available pods (ready for at least minReadySeconds) targeted by this Prometheus deployment.", "format": "int32", "type": "integer" }, "paused": { "description": "Represents whether any actions on the underlying managed objects are being performed. Only delete actions will be performed.", "type": "boolean" }, "replicas": { "description": "Total number of non-terminated pods targeted by this Prometheus deployment (their labels match the selector).", "format": "int32", "type": "integer" }, "unavailableReplicas": { "description": "Total number of unavailable pods targeted by this Prometheus deployment.", "format": "int32", "type": "integer" }, "updatedReplicas": { "description": "Total number of non-terminated pods targeted by this Prometheus deployment that have the desired version spec.", "format": "int32", "type": "integer" } }, "required": [ "availableReplicas", "paused", "replicas", "unavailableReplicas", "updatedReplicas" ], "type": "object" } }, "required": [ "spec" ], "type": "object" } }, "served": true, "storage": true, "subresources": {} } ] }, "status": { "acceptedNames": { "kind": "Prometheus", "listKind": "PrometheusList", "plural": "prometheuses", "singular": "prometheus" }, "conditions": [ { "lastTransitionTime": "2020-05-04T19:47:40Z", "message": "no conflicts found", "reason": "NoConflicts", "status": "True", "type": "NamesAccepted" }, { "lastTransitionTime": "2020-05-04T19:47:40Z", "message": "the initial names have been accepted", "reason": "InitialNamesAccepted", "status": "True", "type": "Established" } ], "storedVersions": [ "v1" ] } } ================================================ FILE: pkg/backup/actions/testdata/v1/prometheuses.monitoring.coreos.com.json ================================================ { "apiVersion": "apiextensions.k8s.io/v1", "kind": "CustomResourceDefinition", "metadata": { "annotations": { "helm.sh/hook": "crd-install", "kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"apiextensions.k8s.io/v1beta1\",\"kind\":\"CustomResourceDefinition\",\"metadata\":{\"annotations\":{\"helm.sh/hook\":\"crd-install\"},\"creationTimestamp\":null,\"labels\":{\"app\":\"prometheus-operator\"},\"name\":\"prometheuses.monitoring.coreos.com\"},\"spec\":{\"group\":\"monitoring.coreos.com\",\"names\":{\"kind\":\"Prometheus\",\"plural\":\"prometheuses\"},\"scope\":\"Namespaced\",\"validation\":{\"openAPIV3Schema\":{\"properties\":{\"apiVersion\":{\"description\":\"APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources\",\"type\":\"string\"},\"kind\":{\"description\":\"Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds\",\"type\":\"string\"},\"spec\":{\"description\":\"PrometheusSpec is a specification of the desired behavior of the Prometheus cluster. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/api-conventions.md#spec-and-status\",\"properties\":{\"additionalAlertManagerConfigs\":{\"description\":\"SecretKeySelector selects a key of a Secret.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or it's key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"},\"additionalAlertRelabelConfigs\":{\"description\":\"SecretKeySelector selects a key of a Secret.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or it's key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"},\"additionalScrapeConfigs\":{\"description\":\"SecretKeySelector selects a key of a Secret.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or it's key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"},\"affinity\":{\"description\":\"Affinity is a group of affinity scheduling rules.\",\"properties\":{\"nodeAffinity\":{\"description\":\"Node affinity is a group of node affinity scheduling rules.\",\"properties\":{\"preferredDuringSchedulingIgnoredDuringExecution\":{\"description\":\"The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding \\\"weight\\\" to the sum if the node matches the corresponding matchExpressions; the node(s) with the highest sum are the most preferred.\",\"items\":{\"description\":\"An empty preferred scheduling term matches all objects with implicit weight 0 (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op).\",\"properties\":{\"preference\":{\"description\":\"A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm.\",\"properties\":{\"matchExpressions\":{\"description\":\"A list of node selector requirements by node's labels.\",\"items\":{\"description\":\"A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"The label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.\",\"type\":\"string\"},\"values\":{\"description\":\"An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"],\"type\":\"object\"},\"type\":\"array\"},\"matchFields\":{\"description\":\"A list of node selector requirements by node's fields.\",\"items\":{\"description\":\"A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"The label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.\",\"type\":\"string\"},\"values\":{\"description\":\"An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"],\"type\":\"object\"},\"type\":\"array\"}},\"type\":\"object\"},\"weight\":{\"description\":\"Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100.\",\"format\":\"int32\",\"type\":\"integer\"}},\"required\":[\"weight\",\"preference\"],\"type\":\"object\"},\"type\":\"array\"},\"requiredDuringSchedulingIgnoredDuringExecution\":{\"description\":\"A node selector represents the union of the results of one or more label queries over a set of nodes; that is, it represents the OR of the selectors represented by the node selector terms.\",\"properties\":{\"nodeSelectorTerms\":{\"description\":\"Required. A list of node selector terms. The terms are ORed.\",\"items\":{\"description\":\"A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm.\",\"properties\":{\"matchExpressions\":{\"description\":\"A list of node selector requirements by node's labels.\",\"items\":{\"description\":\"A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"The label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.\",\"type\":\"string\"},\"values\":{\"description\":\"An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"],\"type\":\"object\"},\"type\":\"array\"},\"matchFields\":{\"description\":\"A list of node selector requirements by node's fields.\",\"items\":{\"description\":\"A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"The label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.\",\"type\":\"string\"},\"values\":{\"description\":\"An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"],\"type\":\"object\"},\"type\":\"array\"}},\"type\":\"object\"},\"type\":\"array\"}},\"required\":[\"nodeSelectorTerms\"],\"type\":\"object\"}},\"type\":\"object\"},\"podAffinity\":{\"description\":\"Pod affinity is a group of inter pod affinity scheduling rules.\",\"properties\":{\"preferredDuringSchedulingIgnoredDuringExecution\":{\"description\":\"The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding \\\"weight\\\" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred.\",\"items\":{\"description\":\"The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s)\",\"properties\":{\"podAffinityTerm\":{\"description\":\"Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key \\u003ctopologyKey\\u003e matches that of any node on which a pod of the set of pods is running\",\"properties\":{\"labelSelector\":{\"description\":\"A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.\",\"properties\":{\"matchExpressions\":{\"description\":\"matchExpressions is a list of label selector requirements. The requirements are ANDed.\",\"items\":{\"description\":\"A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"key is the label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\",\"type\":\"string\"},\"values\":{\"description\":\"values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"],\"type\":\"object\"},\"type\":\"array\"},\"matchLabels\":{\"description\":\"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \\\"key\\\", the operator is \\\"In\\\", and the values array contains only \\\"value\\\". The requirements are ANDed.\",\"type\":\"object\"}},\"type\":\"object\"},\"namespaces\":{\"description\":\"namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \\\"this pod's namespace\\\"\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"topologyKey\":{\"description\":\"This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.\",\"type\":\"string\"}},\"required\":[\"topologyKey\"],\"type\":\"object\"},\"weight\":{\"description\":\"weight associated with matching the corresponding podAffinityTerm, in the range 1-100.\",\"format\":\"int32\",\"type\":\"integer\"}},\"required\":[\"weight\",\"podAffinityTerm\"],\"type\":\"object\"},\"type\":\"array\"},\"requiredDuringSchedulingIgnoredDuringExecution\":{\"description\":\"If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied.\",\"items\":{\"description\":\"Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key \\u003ctopologyKey\\u003e matches that of any node on which a pod of the set of pods is running\",\"properties\":{\"labelSelector\":{\"description\":\"A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.\",\"properties\":{\"matchExpressions\":{\"description\":\"matchExpressions is a list of label selector requirements. The requirements are ANDed.\",\"items\":{\"description\":\"A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"key is the label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\",\"type\":\"string\"},\"values\":{\"description\":\"values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"],\"type\":\"object\"},\"type\":\"array\"},\"matchLabels\":{\"description\":\"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \\\"key\\\", the operator is \\\"In\\\", and the values array contains only \\\"value\\\". The requirements are ANDed.\",\"type\":\"object\"}},\"type\":\"object\"},\"namespaces\":{\"description\":\"namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \\\"this pod's namespace\\\"\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"topologyKey\":{\"description\":\"This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.\",\"type\":\"string\"}},\"required\":[\"topologyKey\"],\"type\":\"object\"},\"type\":\"array\"}},\"type\":\"object\"},\"podAntiAffinity\":{\"description\":\"Pod anti affinity is a group of inter pod anti affinity scheduling rules.\",\"properties\":{\"preferredDuringSchedulingIgnoredDuringExecution\":{\"description\":\"The scheduler will prefer to schedule pods to nodes that satisfy the anti-affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding \\\"weight\\\" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred.\",\"items\":{\"description\":\"The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s)\",\"properties\":{\"podAffinityTerm\":{\"description\":\"Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key \\u003ctopologyKey\\u003e matches that of any node on which a pod of the set of pods is running\",\"properties\":{\"labelSelector\":{\"description\":\"A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.\",\"properties\":{\"matchExpressions\":{\"description\":\"matchExpressions is a list of label selector requirements. The requirements are ANDed.\",\"items\":{\"description\":\"A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"key is the label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\",\"type\":\"string\"},\"values\":{\"description\":\"values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"],\"type\":\"object\"},\"type\":\"array\"},\"matchLabels\":{\"description\":\"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \\\"key\\\", the operator is \\\"In\\\", and the values array contains only \\\"value\\\". The requirements are ANDed.\",\"type\":\"object\"}},\"type\":\"object\"},\"namespaces\":{\"description\":\"namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \\\"this pod's namespace\\\"\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"topologyKey\":{\"description\":\"This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.\",\"type\":\"string\"}},\"required\":[\"topologyKey\"],\"type\":\"object\"},\"weight\":{\"description\":\"weight associated with matching the corresponding podAffinityTerm, in the range 1-100.\",\"format\":\"int32\",\"type\":\"integer\"}},\"required\":[\"weight\",\"podAffinityTerm\"],\"type\":\"object\"},\"type\":\"array\"},\"requiredDuringSchedulingIgnoredDuringExecution\":{\"description\":\"If the anti-affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the anti-affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied.\",\"items\":{\"description\":\"Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key \\u003ctopologyKey\\u003e matches that of any node on which a pod of the set of pods is running\",\"properties\":{\"labelSelector\":{\"description\":\"A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.\",\"properties\":{\"matchExpressions\":{\"description\":\"matchExpressions is a list of label selector requirements. The requirements are ANDed.\",\"items\":{\"description\":\"A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"key is the label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\",\"type\":\"string\"},\"values\":{\"description\":\"values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"],\"type\":\"object\"},\"type\":\"array\"},\"matchLabels\":{\"description\":\"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \\\"key\\\", the operator is \\\"In\\\", and the values array contains only \\\"value\\\". The requirements are ANDed.\",\"type\":\"object\"}},\"type\":\"object\"},\"namespaces\":{\"description\":\"namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \\\"this pod's namespace\\\"\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"topologyKey\":{\"description\":\"This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.\",\"type\":\"string\"}},\"required\":[\"topologyKey\"],\"type\":\"object\"},\"type\":\"array\"}},\"type\":\"object\"}},\"type\":\"object\"},\"alerting\":{\"description\":\"AlertingSpec defines parameters for alerting configuration of Prometheus servers.\",\"properties\":{\"alertmanagers\":{\"description\":\"AlertmanagerEndpoints Prometheus should fire alerts against.\",\"items\":{\"description\":\"AlertmanagerEndpoints defines a selection of a single Endpoints object containing alertmanager IPs to fire alerts against.\",\"properties\":{\"bearerTokenFile\":{\"description\":\"BearerTokenFile to read from filesystem to use when authenticating to Alertmanager.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of Endpoints object in Namespace.\",\"type\":\"string\"},\"namespace\":{\"description\":\"Namespace of Endpoints object.\",\"type\":\"string\"},\"pathPrefix\":{\"description\":\"Prefix for the HTTP path alerts are pushed to.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"string\"},{\"type\":\"integer\"}]},\"scheme\":{\"description\":\"Scheme to use when firing alerts.\",\"type\":\"string\"},\"tlsConfig\":{\"description\":\"TLSConfig specifies TLS configuration parameters.\",\"properties\":{\"caFile\":{\"description\":\"The CA cert to use for the targets.\",\"type\":\"string\"},\"certFile\":{\"description\":\"The client cert file for the targets.\",\"type\":\"string\"},\"insecureSkipVerify\":{\"description\":\"Disable target certificate validation.\",\"type\":\"boolean\"},\"keyFile\":{\"description\":\"The client key file for the targets.\",\"type\":\"string\"},\"serverName\":{\"description\":\"Used to verify the hostname for the targets.\",\"type\":\"string\"}},\"type\":\"object\"}},\"required\":[\"namespace\",\"name\",\"port\"],\"type\":\"object\"},\"type\":\"array\"}},\"required\":[\"alertmanagers\"],\"type\":\"object\"},\"apiserverConfig\":{\"description\":\"APIServerConfig defines a host and auth methods to access apiserver. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config\",\"properties\":{\"basicAuth\":{\"description\":\"BasicAuth allow an endpoint to authenticate over basic authentication More info: https://prometheus.io/docs/operating/configuration/#endpoints\",\"properties\":{\"password\":{\"description\":\"SecretKeySelector selects a key of a Secret.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or it's key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"},\"username\":{\"description\":\"SecretKeySelector selects a key of a Secret.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or it's key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"}},\"type\":\"object\"},\"bearerToken\":{\"description\":\"Bearer token for accessing apiserver.\",\"type\":\"string\"},\"bearerTokenFile\":{\"description\":\"File to read bearer token for accessing apiserver.\",\"type\":\"string\"},\"host\":{\"description\":\"Host of apiserver. A valid string consisting of a hostname or IP followed by an optional port number\",\"type\":\"string\"},\"tlsConfig\":{\"description\":\"TLSConfig specifies TLS configuration parameters.\",\"properties\":{\"caFile\":{\"description\":\"The CA cert to use for the targets.\",\"type\":\"string\"},\"certFile\":{\"description\":\"The client cert file for the targets.\",\"type\":\"string\"},\"insecureSkipVerify\":{\"description\":\"Disable target certificate validation.\",\"type\":\"boolean\"},\"keyFile\":{\"description\":\"The client key file for the targets.\",\"type\":\"string\"},\"serverName\":{\"description\":\"Used to verify the hostname for the targets.\",\"type\":\"string\"}},\"type\":\"object\"}},\"required\":[\"host\"],\"type\":\"object\"},\"baseImage\":{\"description\":\"Base image to use for a Prometheus deployment.\",\"type\":\"string\"},\"configMaps\":{\"description\":\"ConfigMaps is a list of ConfigMaps in the same namespace as the Prometheus object, which shall be mounted into the Prometheus Pods. The ConfigMaps are mounted into /etc/prometheus/configmaps/\\u003cconfigmap-name\\u003e.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"containers\":{\"description\":\"Containers allows injecting additional containers or modifying operator generated containers. This can be used to allow adding an authentication proxy to a Prometheus pod or to change the behavior of an operator generated container. Containers described here modify an operator generated container if they share the same name and modifications are done via a strategic merge patch. The current container names are: `prometheus`, `prometheus-config-reloader`, `rules-configmap-reloader`, and `thanos-sidecar`. Overriding containers is entirely outside the scope of what the maintainers will support and by doing so, you accept that this behaviour may break at any time without notice.\",\"items\":{\"description\":\"A single application container that you want to run within a pod.\",\"properties\":{\"args\":{\"description\":\"Arguments to the entrypoint. The docker image's CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"command\":{\"description\":\"Entrypoint array. Not executed within a shell. The docker image's ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"env\":{\"description\":\"List of environment variables to set in the container. Cannot be updated.\",\"items\":{\"description\":\"EnvVar represents an environment variable present in a Container.\",\"properties\":{\"name\":{\"description\":\"Name of the environment variable. Must be a C_IDENTIFIER.\",\"type\":\"string\"},\"value\":{\"description\":\"Variable references $(VAR_NAME) are expanded using the previous defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Defaults to \\\"\\\".\",\"type\":\"string\"},\"valueFrom\":{\"description\":\"EnvVarSource represents a source for the value of an EnvVar.\",\"properties\":{\"configMapKeyRef\":{\"description\":\"Selects a key from a ConfigMap.\",\"properties\":{\"key\":{\"description\":\"The key to select.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the ConfigMap or it's key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"},\"fieldRef\":{\"description\":\"ObjectFieldSelector selects an APIVersioned field of an object.\",\"properties\":{\"apiVersion\":{\"description\":\"Version of the schema the FieldPath is written in terms of, defaults to \\\"v1\\\".\",\"type\":\"string\"},\"fieldPath\":{\"description\":\"Path of the field to select in the specified API version.\",\"type\":\"string\"}},\"required\":[\"fieldPath\"],\"type\":\"object\"},\"resourceFieldRef\":{\"description\":\"ResourceFieldSelector represents container resources (cpu, memory) and their output format\",\"properties\":{\"containerName\":{\"description\":\"Container name: required for volumes, optional for env vars\",\"type\":\"string\"},\"divisor\":{},\"resource\":{\"description\":\"Required: resource to select\",\"type\":\"string\"}},\"required\":[\"resource\"],\"type\":\"object\"},\"secretKeyRef\":{\"description\":\"SecretKeySelector selects a key of a Secret.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or it's key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"}},\"type\":\"object\"}},\"required\":[\"name\"],\"type\":\"object\"},\"type\":\"array\"},\"envFrom\":{\"description\":\"List of sources to populate environment variables in the container. The keys defined within a source must be a C_IDENTIFIER. All invalid keys will be reported as an event when the container is starting. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated.\",\"items\":{\"description\":\"EnvFromSource represents the source of a set of ConfigMaps\",\"properties\":{\"configMapRef\":{\"description\":\"ConfigMapEnvSource selects a ConfigMap to populate the environment variables with.\\n\\nThe contents of the target ConfigMap's Data field will represent the key-value pairs as environment variables.\",\"properties\":{\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the ConfigMap must be defined\",\"type\":\"boolean\"}},\"type\":\"object\"},\"prefix\":{\"description\":\"An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER.\",\"type\":\"string\"},\"secretRef\":{\"description\":\"SecretEnvSource selects a Secret to populate the environment variables with.\\n\\nThe contents of the target Secret's Data field will represent the key-value pairs as environment variables.\",\"properties\":{\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret must be defined\",\"type\":\"boolean\"}},\"type\":\"object\"}},\"type\":\"object\"},\"type\":\"array\"},\"image\":{\"description\":\"Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images This field is optional to allow higher level config management to default or override container images in workload controllers like Deployments and StatefulSets.\",\"type\":\"string\"},\"imagePullPolicy\":{\"description\":\"Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images\",\"type\":\"string\"},\"lifecycle\":{\"description\":\"Lifecycle describes actions that the management system should take in response to container lifecycle events. For the PostStart and PreStop lifecycle handlers, management of the container blocks until the action is complete, unless the container process fails, in which case the handler is aborted.\",\"properties\":{\"postStart\":{\"description\":\"Handler defines a specific action that should be taken\",\"properties\":{\"exec\":{\"description\":\"ExecAction describes a \\\"run in container\\\" action.\",\"properties\":{\"command\":{\"description\":\"Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"type\":\"object\"},\"httpGet\":{\"description\":\"HTTPGetAction describes an action based on HTTP Get requests.\",\"properties\":{\"host\":{\"description\":\"Host name to connect to, defaults to the pod IP. You probably want to set \\\"Host\\\" in httpHeaders instead.\",\"type\":\"string\"},\"httpHeaders\":{\"description\":\"Custom headers to set in the request. HTTP allows repeated headers.\",\"items\":{\"description\":\"HTTPHeader describes a custom header to be used in HTTP probes\",\"properties\":{\"name\":{\"description\":\"The header field name\",\"type\":\"string\"},\"value\":{\"description\":\"The header field value\",\"type\":\"string\"}},\"required\":[\"name\",\"value\"],\"type\":\"object\"},\"type\":\"array\"},\"path\":{\"description\":\"Path to access on the HTTP server.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"string\"},{\"type\":\"integer\"}]},\"scheme\":{\"description\":\"Scheme to use for connecting to the host. Defaults to HTTP.\",\"type\":\"string\"}},\"required\":[\"port\"],\"type\":\"object\"},\"tcpSocket\":{\"description\":\"TCPSocketAction describes an action based on opening a socket\",\"properties\":{\"host\":{\"description\":\"Optional: Host name to connect to, defaults to the pod IP.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"string\"},{\"type\":\"integer\"}]}},\"required\":[\"port\"],\"type\":\"object\"}},\"type\":\"object\"},\"preStop\":{\"description\":\"Handler defines a specific action that should be taken\",\"properties\":{\"exec\":{\"description\":\"ExecAction describes a \\\"run in container\\\" action.\",\"properties\":{\"command\":{\"description\":\"Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"type\":\"object\"},\"httpGet\":{\"description\":\"HTTPGetAction describes an action based on HTTP Get requests.\",\"properties\":{\"host\":{\"description\":\"Host name to connect to, defaults to the pod IP. You probably want to set \\\"Host\\\" in httpHeaders instead.\",\"type\":\"string\"},\"httpHeaders\":{\"description\":\"Custom headers to set in the request. HTTP allows repeated headers.\",\"items\":{\"description\":\"HTTPHeader describes a custom header to be used in HTTP probes\",\"properties\":{\"name\":{\"description\":\"The header field name\",\"type\":\"string\"},\"value\":{\"description\":\"The header field value\",\"type\":\"string\"}},\"required\":[\"name\",\"value\"],\"type\":\"object\"},\"type\":\"array\"},\"path\":{\"description\":\"Path to access on the HTTP server.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"string\"},{\"type\":\"integer\"}]},\"scheme\":{\"description\":\"Scheme to use for connecting to the host. Defaults to HTTP.\",\"type\":\"string\"}},\"required\":[\"port\"],\"type\":\"object\"},\"tcpSocket\":{\"description\":\"TCPSocketAction describes an action based on opening a socket\",\"properties\":{\"host\":{\"description\":\"Optional: Host name to connect to, defaults to the pod IP.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"string\"},{\"type\":\"integer\"}]}},\"required\":[\"port\"],\"type\":\"object\"}},\"type\":\"object\"}},\"type\":\"object\"},\"livenessProbe\":{\"description\":\"Probe describes a health check to be performed against a container to determine whether it is alive or ready to receive traffic.\",\"properties\":{\"exec\":{\"description\":\"ExecAction describes a \\\"run in container\\\" action.\",\"properties\":{\"command\":{\"description\":\"Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"type\":\"object\"},\"failureThreshold\":{\"description\":\"Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.\",\"format\":\"int32\",\"type\":\"integer\"},\"httpGet\":{\"description\":\"HTTPGetAction describes an action based on HTTP Get requests.\",\"properties\":{\"host\":{\"description\":\"Host name to connect to, defaults to the pod IP. You probably want to set \\\"Host\\\" in httpHeaders instead.\",\"type\":\"string\"},\"httpHeaders\":{\"description\":\"Custom headers to set in the request. HTTP allows repeated headers.\",\"items\":{\"description\":\"HTTPHeader describes a custom header to be used in HTTP probes\",\"properties\":{\"name\":{\"description\":\"The header field name\",\"type\":\"string\"},\"value\":{\"description\":\"The header field value\",\"type\":\"string\"}},\"required\":[\"name\",\"value\"],\"type\":\"object\"},\"type\":\"array\"},\"path\":{\"description\":\"Path to access on the HTTP server.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"string\"},{\"type\":\"integer\"}]},\"scheme\":{\"description\":\"Scheme to use for connecting to the host. Defaults to HTTP.\",\"type\":\"string\"}},\"required\":[\"port\"],\"type\":\"object\"},\"initialDelaySeconds\":{\"description\":\"Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\",\"format\":\"int32\",\"type\":\"integer\"},\"periodSeconds\":{\"description\":\"How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.\",\"format\":\"int32\",\"type\":\"integer\"},\"successThreshold\":{\"description\":\"Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness. Minimum value is 1.\",\"format\":\"int32\",\"type\":\"integer\"},\"tcpSocket\":{\"description\":\"TCPSocketAction describes an action based on opening a socket\",\"properties\":{\"host\":{\"description\":\"Optional: Host name to connect to, defaults to the pod IP.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"string\"},{\"type\":\"integer\"}]}},\"required\":[\"port\"],\"type\":\"object\"},\"timeoutSeconds\":{\"description\":\"Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\",\"format\":\"int32\",\"type\":\"integer\"}},\"type\":\"object\"},\"name\":{\"description\":\"Name of the container specified as a DNS_LABEL. Each container in a pod must have a unique name (DNS_LABEL). Cannot be updated.\",\"type\":\"string\"},\"ports\":{\"description\":\"List of ports to expose from the container. Exposing a port here gives the system additional information about the network connections a container uses, but is primarily informational. Not specifying a port here DOES NOT prevent that port from being exposed. Any port which is listening on the default \\\"0.0.0.0\\\" address inside a container will be accessible from the network. Cannot be updated.\",\"items\":{\"description\":\"ContainerPort represents a network port in a single container.\",\"properties\":{\"containerPort\":{\"description\":\"Number of port to expose on the pod's IP address. This must be a valid port number, 0 \\u003c x \\u003c 65536.\",\"format\":\"int32\",\"type\":\"integer\"},\"hostIP\":{\"description\":\"What host IP to bind the external port to.\",\"type\":\"string\"},\"hostPort\":{\"description\":\"Number of port to expose on the host. If specified, this must be a valid port number, 0 \\u003c x \\u003c 65536. If HostNetwork is specified, this must match ContainerPort. Most containers do not need this.\",\"format\":\"int32\",\"type\":\"integer\"},\"name\":{\"description\":\"If specified, this must be an IANA_SVC_NAME and unique within the pod. Each named port in a pod must have a unique name. Name for the port that can be referred to by services.\",\"type\":\"string\"},\"protocol\":{\"description\":\"Protocol for port. Must be UDP, TCP, or SCTP. Defaults to \\\"TCP\\\".\",\"type\":\"string\"}},\"required\":[\"containerPort\"],\"type\":\"object\"},\"type\":\"array\"},\"readinessProbe\":{\"description\":\"Probe describes a health check to be performed against a container to determine whether it is alive or ready to receive traffic.\",\"properties\":{\"exec\":{\"description\":\"ExecAction describes a \\\"run in container\\\" action.\",\"properties\":{\"command\":{\"description\":\"Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"type\":\"object\"},\"failureThreshold\":{\"description\":\"Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.\",\"format\":\"int32\",\"type\":\"integer\"},\"httpGet\":{\"description\":\"HTTPGetAction describes an action based on HTTP Get requests.\",\"properties\":{\"host\":{\"description\":\"Host name to connect to, defaults to the pod IP. You probably want to set \\\"Host\\\" in httpHeaders instead.\",\"type\":\"string\"},\"httpHeaders\":{\"description\":\"Custom headers to set in the request. HTTP allows repeated headers.\",\"items\":{\"description\":\"HTTPHeader describes a custom header to be used in HTTP probes\",\"properties\":{\"name\":{\"description\":\"The header field name\",\"type\":\"string\"},\"value\":{\"description\":\"The header field value\",\"type\":\"string\"}},\"required\":[\"name\",\"value\"],\"type\":\"object\"},\"type\":\"array\"},\"path\":{\"description\":\"Path to access on the HTTP server.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"string\"},{\"type\":\"integer\"}]},\"scheme\":{\"description\":\"Scheme to use for connecting to the host. Defaults to HTTP.\",\"type\":\"string\"}},\"required\":[\"port\"],\"type\":\"object\"},\"initialDelaySeconds\":{\"description\":\"Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\",\"format\":\"int32\",\"type\":\"integer\"},\"periodSeconds\":{\"description\":\"How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.\",\"format\":\"int32\",\"type\":\"integer\"},\"successThreshold\":{\"description\":\"Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness. Minimum value is 1.\",\"format\":\"int32\",\"type\":\"integer\"},\"tcpSocket\":{\"description\":\"TCPSocketAction describes an action based on opening a socket\",\"properties\":{\"host\":{\"description\":\"Optional: Host name to connect to, defaults to the pod IP.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"string\"},{\"type\":\"integer\"}]}},\"required\":[\"port\"],\"type\":\"object\"},\"timeoutSeconds\":{\"description\":\"Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\",\"format\":\"int32\",\"type\":\"integer\"}},\"type\":\"object\"},\"resources\":{\"description\":\"ResourceRequirements describes the compute resource requirements.\",\"properties\":{\"limits\":{\"description\":\"Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/\",\"type\":\"object\"},\"requests\":{\"description\":\"Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/\",\"type\":\"object\"}},\"type\":\"object\"},\"securityContext\":{\"description\":\"SecurityContext holds security configuration that will be applied to a container. Some fields are present in both SecurityContext and PodSecurityContext. When both are set, the values in SecurityContext take precedence.\",\"properties\":{\"allowPrivilegeEscalation\":{\"description\":\"AllowPrivilegeEscalation controls whether a process can gain more privileges than its parent process. This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged 2) has CAP_SYS_ADMIN\",\"type\":\"boolean\"},\"capabilities\":{\"description\":\"Adds and removes POSIX capabilities from running containers.\",\"properties\":{\"add\":{\"description\":\"Added capabilities\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"drop\":{\"description\":\"Removed capabilities\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"type\":\"object\"},\"privileged\":{\"description\":\"Run container in privileged mode. Processes in privileged containers are essentially equivalent to root on the host. Defaults to false.\",\"type\":\"boolean\"},\"procMount\":{\"description\":\"procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled.\",\"type\":\"string\"},\"readOnlyRootFilesystem\":{\"description\":\"Whether this container has a read-only root filesystem. Default is false.\",\"type\":\"boolean\"},\"runAsGroup\":{\"description\":\"The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\",\"format\":\"int64\",\"type\":\"integer\"},\"runAsNonRoot\":{\"description\":\"Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\",\"type\":\"boolean\"},\"runAsUser\":{\"description\":\"The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\",\"format\":\"int64\",\"type\":\"integer\"},\"seLinuxOptions\":{\"description\":\"SELinuxOptions are the labels to be applied to the container\",\"properties\":{\"level\":{\"description\":\"Level is SELinux level label that applies to the container.\",\"type\":\"string\"},\"role\":{\"description\":\"Role is a SELinux role label that applies to the container.\",\"type\":\"string\"},\"type\":{\"description\":\"Type is a SELinux type label that applies to the container.\",\"type\":\"string\"},\"user\":{\"description\":\"User is a SELinux user label that applies to the container.\",\"type\":\"string\"}},\"type\":\"object\"}},\"type\":\"object\"},\"stdin\":{\"description\":\"Whether this container should allocate a buffer for stdin in the container runtime. If this is not set, reads from stdin in the container will always result in EOF. Default is false.\",\"type\":\"boolean\"},\"stdinOnce\":{\"description\":\"Whether the container runtime should close the stdin channel after it has been opened by a single attach. When stdin is true the stdin stream will remain open across multiple attach sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the first client attaches to stdin, and then remains open and accepts data until the client disconnects, at which time stdin is closed and remains closed until the container is restarted. If this flag is false, a container processes that reads from stdin will never receive an EOF. Default is false\",\"type\":\"boolean\"},\"terminationMessagePath\":{\"description\":\"Optional: Path at which the file to which the container's termination message will be written is mounted into the container's filesystem. Message written is intended to be brief final status, such as an assertion failure message. Will be truncated by the node if greater than 4096 bytes. The total message length across all containers will be limited to 12kb. Defaults to /dev/termination-log. Cannot be updated.\",\"type\":\"string\"},\"terminationMessagePolicy\":{\"description\":\"Indicate how the termination message should be populated. File will use the contents of terminationMessagePath to populate the container status message on both success and failure. FallbackToLogsOnError will use the last chunk of container log output if the termination message file is empty and the container exited with an error. The log output is limited to 2048 bytes or 80 lines, whichever is smaller. Defaults to File. Cannot be updated.\",\"type\":\"string\"},\"tty\":{\"description\":\"Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. Default is false.\",\"type\":\"boolean\"},\"volumeDevices\":{\"description\":\"volumeDevices is the list of block devices to be used by the container. This is a beta feature.\",\"items\":{\"description\":\"volumeDevice describes a mapping of a raw block device within a container.\",\"properties\":{\"devicePath\":{\"description\":\"devicePath is the path inside of the container that the device will be mapped to.\",\"type\":\"string\"},\"name\":{\"description\":\"name must match the name of a persistentVolumeClaim in the pod\",\"type\":\"string\"}},\"required\":[\"name\",\"devicePath\"],\"type\":\"object\"},\"type\":\"array\"},\"volumeMounts\":{\"description\":\"Pod volumes to mount into the container's filesystem. Cannot be updated.\",\"items\":{\"description\":\"VolumeMount describes a mounting of a Volume within a container.\",\"properties\":{\"mountPath\":{\"description\":\"Path within the container at which the volume should be mounted. Must not contain ':'.\",\"type\":\"string\"},\"mountPropagation\":{\"description\":\"mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10.\",\"type\":\"string\"},\"name\":{\"description\":\"This must match the Name of a Volume.\",\"type\":\"string\"},\"readOnly\":{\"description\":\"Mounted read-only if true, read-write otherwise (false or unspecified). Defaults to false.\",\"type\":\"boolean\"},\"subPath\":{\"description\":\"Path within the volume from which the container's volume should be mounted. Defaults to \\\"\\\" (volume's root).\",\"type\":\"string\"},\"subPathExpr\":{\"description\":\"Expanded path within the volume from which the container's volume should be mounted. Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. Defaults to \\\"\\\" (volume's root). SubPathExpr and SubPath are mutually exclusive. This field is alpha in 1.14.\",\"type\":\"string\"}},\"required\":[\"name\",\"mountPath\"],\"type\":\"object\"},\"type\":\"array\"},\"workingDir\":{\"description\":\"Container's working directory. If not specified, the container runtime's default will be used, which might be configured in the container image. Cannot be updated.\",\"type\":\"string\"}},\"required\":[\"name\"],\"type\":\"object\"},\"type\":\"array\"},\"enableAdminAPI\":{\"description\":\"Enable access to prometheus web admin API. Defaults to the value of `false`. WARNING: Enabling the admin APIs enables mutating endpoints, to delete data, shutdown Prometheus, and more. Enabling this should be done with care and the user is advised to add additional authentication authorization via a proxy to ensure only clients authorized to perform these actions can do so. For more information see https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-admin-apis\",\"type\":\"boolean\"},\"evaluationInterval\":{\"description\":\"Interval between consecutive evaluations.\",\"type\":\"string\"},\"externalLabels\":{\"description\":\"The labels to add to any time series or alerts when communicating with external systems (federation, remote storage, Alertmanager).\",\"type\":\"object\"},\"externalUrl\":{\"description\":\"The external URL the Prometheus instances will be available under. This is necessary to generate correct URLs. This is necessary if Prometheus is not served from root of a DNS name.\",\"type\":\"string\"},\"image\":{\"description\":\"Image if specified has precedence over baseImage, tag and sha combinations. Specifying the version is still necessary to ensure the Prometheus Operator knows what version of Prometheus is being configured.\",\"type\":\"string\"},\"imagePullSecrets\":{\"description\":\"An optional list of references to secrets in the same namespace to use for pulling prometheus and alertmanager images from registries see http://kubernetes.io/docs/user-guide/images#specifying-imagepullsecrets-on-a-pod\",\"items\":{\"description\":\"LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace.\",\"properties\":{\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\",\"type\":\"string\"}},\"type\":\"object\"},\"type\":\"array\"},\"listenLocal\":{\"description\":\"ListenLocal makes the Prometheus server listen on loopback, so that it does not bind against the Pod IP.\",\"type\":\"boolean\"},\"logFormat\":{\"description\":\"Log format for Prometheus to be configured with.\",\"type\":\"string\"},\"logLevel\":{\"description\":\"Log level for Prometheus to be configured with.\",\"type\":\"string\"},\"nodeSelector\":{\"description\":\"Define which Nodes the Pods are scheduled on.\",\"type\":\"object\"},\"paused\":{\"description\":\"When a Prometheus deployment is paused, no actions except for deletion will be performed on the underlying objects.\",\"type\":\"boolean\"},\"podMetadata\":{\"description\":\"ObjectMeta is metadata that all persisted resources must have, which includes all objects users must create.\",\"properties\":{\"annotations\":{\"description\":\"Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations\",\"type\":\"object\"},\"clusterName\":{\"description\":\"The name of the cluster which the object belongs to. This is used to distinguish resources with same name and namespace in different clusters. This field is not set anywhere right now and apiserver is going to ignore it if set in create or update request.\",\"type\":\"string\"},\"creationTimestamp\":{\"description\":\"Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.\",\"format\":\"date-time\",\"type\":\"string\"},\"deletionGracePeriodSeconds\":{\"description\":\"Number of seconds allowed for this object to gracefully terminate before it will be removed from the system. Only set when deletionTimestamp is also set. May only be shortened. Read-only.\",\"format\":\"int64\",\"type\":\"integer\"},\"deletionTimestamp\":{\"description\":\"Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.\",\"format\":\"date-time\",\"type\":\"string\"},\"finalizers\":{\"description\":\"Must be empty before the object is deleted from the registry. Each entry is an identifier for the responsible component that will remove the entry from the list. If the deletionTimestamp of the object is non-nil, entries in this list can only be removed.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"generateName\":{\"description\":\"GenerateName is an optional prefix, used by the server, to generate a unique name ONLY IF the Name field has not been provided. If this field is used, the name returned to the client will be different than the name passed. This value will also be combined with a unique suffix. The provided value has the same validation rules as the Name field, and may be truncated by the length of the suffix required to make the value unique on the server.\\n\\nIf this field is specified and the generated name exists, the server will NOT return a 409 - instead, it will either return 201 Created or 500 with Reason ServerTimeout indicating a unique name could not be found in the time allotted, and the client should retry (optionally after the time indicated in the Retry-After header).\\n\\nApplied only if Name is not specified. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#idempotency\",\"type\":\"string\"},\"generation\":{\"description\":\"A sequence number representing a specific generation of the desired state. Populated by the system. Read-only.\",\"format\":\"int64\",\"type\":\"integer\"},\"initializers\":{\"description\":\"Initializers tracks the progress of initialization.\",\"properties\":{\"pending\":{\"description\":\"Pending is a list of initializers that must execute in order before this object is visible. When the last pending initializer is removed, and no failing result is set, the initializers struct will be set to nil and the object is considered as initialized and visible to all clients.\",\"items\":{\"description\":\"Initializer is information about an initializer that has not yet completed.\",\"properties\":{\"name\":{\"description\":\"name of the process that is responsible for initializing this object.\",\"type\":\"string\"}},\"required\":[\"name\"],\"type\":\"object\"},\"type\":\"array\"},\"result\":{\"description\":\"Status is a return value for calls that don't return other objects.\",\"properties\":{\"apiVersion\":{\"description\":\"APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources\",\"type\":\"string\"},\"code\":{\"description\":\"Suggested HTTP return code for this status, 0 if not set.\",\"format\":\"int32\",\"type\":\"integer\"},\"details\":{\"description\":\"StatusDetails is a set of additional properties that MAY be set by the server to provide additional information about a response. The Reason field of a Status object defines what attributes will be set. Clients must ignore fields that do not match the defined type of each attribute, and should assume that any attribute may be empty, invalid, or under defined.\",\"properties\":{\"causes\":{\"description\":\"The Causes array includes more details associated with the StatusReason failure. Not all StatusReasons may provide detailed causes.\",\"items\":{\"description\":\"StatusCause provides more information about an api.Status failure, including cases when multiple errors are encountered.\",\"properties\":{\"field\":{\"description\":\"The field of the resource that has caused this error, as named by its JSON serialization. May include dot and postfix notation for nested attributes. Arrays are zero-indexed. Fields may appear more than once in an array of causes due to fields having multiple errors. Optional.\\n\\nExamples:\\n \\\"name\\\" - the field \\\"name\\\" on the current resource\\n \\\"items[0].name\\\" - the field \\\"name\\\" on the first array entry in \\\"items\\\"\",\"type\":\"string\"},\"message\":{\"description\":\"A human-readable description of the cause of the error. This field may be presented as-is to a reader.\",\"type\":\"string\"},\"reason\":{\"description\":\"A machine-readable description of the cause of the error. If this value is empty there is no information available.\",\"type\":\"string\"}},\"type\":\"object\"},\"type\":\"array\"},\"group\":{\"description\":\"The group attribute of the resource associated with the status StatusReason.\",\"type\":\"string\"},\"kind\":{\"description\":\"The kind attribute of the resource associated with the status StatusReason. On some operations may differ from the requested resource Kind. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds\",\"type\":\"string\"},\"name\":{\"description\":\"The name attribute of the resource associated with the status StatusReason (when there is a single name which can be described).\",\"type\":\"string\"},\"retryAfterSeconds\":{\"description\":\"If specified, the time in seconds before the operation should be retried. Some errors may indicate the client must take an alternate action - for those errors this field may indicate how long to wait before taking the alternate action.\",\"format\":\"int32\",\"type\":\"integer\"},\"uid\":{\"description\":\"UID of the resource. (when there is a single resource which can be described). More info: http://kubernetes.io/docs/user-guide/identifiers#uids\",\"type\":\"string\"}},\"type\":\"object\"},\"kind\":{\"description\":\"Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds\",\"type\":\"string\"},\"message\":{\"description\":\"A human-readable description of the status of this operation.\",\"type\":\"string\"},\"metadata\":{\"description\":\"ListMeta describes metadata that synthetic resources must have, including lists and various status objects. A resource may have only one of {ObjectMeta, ListMeta}.\",\"properties\":{\"continue\":{\"description\":\"continue may be set if the user set a limit on the number of items returned, and indicates that the server has more data available. The value is opaque and may be used to issue another request to the endpoint that served this list to retrieve the next set of available objects. Continuing a consistent list may not be possible if the server configuration has changed or more than a few minutes have passed. The resourceVersion field returned when using this continue value will be identical to the value in the first response, unless you have received this token from an error message.\",\"type\":\"string\"},\"resourceVersion\":{\"description\":\"String that identifies the server's internal version of this object that can be used by clients to determine when objects have changed. Value must be treated as opaque by clients and passed unmodified back to the server. Populated by the system. Read-only. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency\",\"type\":\"string\"},\"selfLink\":{\"description\":\"selfLink is a URL representing this object. Populated by the system. Read-only.\",\"type\":\"string\"}},\"type\":\"object\"},\"reason\":{\"description\":\"A machine-readable description of why this operation is in the \\\"Failure\\\" status. If this value is empty there is no information available. A Reason clarifies an HTTP status code but does not override it.\",\"type\":\"string\"},\"status\":{\"description\":\"Status of the operation. One of: \\\"Success\\\" or \\\"Failure\\\". More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status\",\"type\":\"string\"}},\"type\":\"object\"}},\"required\":[\"pending\"],\"type\":\"object\"},\"labels\":{\"description\":\"Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels\",\"type\":\"object\"},\"managedFields\":{\"description\":\"ManagedFields maps workflow-id and version to the set of fields that are managed by that workflow. This is mostly for internal housekeeping, and users typically shouldn't need to set or understand this field. A workflow can be the user's name, a controller's name, or the name of a specific apply path like \\\"ci-cd\\\". The set of fields is always in the version that the workflow used when modifying the object.\\n\\nThis field is alpha and can be changed or removed without notice.\",\"items\":{\"description\":\"ManagedFieldsEntry is a workflow-id, a FieldSet and the group version of the resource that the fieldset applies to.\",\"properties\":{\"apiVersion\":{\"description\":\"APIVersion defines the version of this resource that this field set applies to. The format is \\\"group/version\\\" just like the top-level APIVersion field. It is necessary to track the version of a field set because it cannot be automatically converted.\",\"type\":\"string\"},\"fields\":{\"description\":\"Fields stores a set of fields in a data structure like a Trie. To understand how this is used, see: https://github.com/kubernetes-sigs/structured-merge-diff\",\"type\":\"object\"},\"manager\":{\"description\":\"Manager is an identifier of the workflow managing these fields.\",\"type\":\"string\"},\"operation\":{\"description\":\"Operation is the type of operation which lead to this ManagedFieldsEntry being created. The only valid values for this field are 'Apply' and 'Update'.\",\"type\":\"string\"},\"time\":{\"description\":\"Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.\",\"format\":\"date-time\",\"type\":\"string\"}},\"type\":\"object\"},\"type\":\"array\"},\"name\":{\"description\":\"Name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names\",\"type\":\"string\"},\"namespace\":{\"description\":\"Namespace defines the space within each name must be unique. An empty namespace is equivalent to the \\\"default\\\" namespace, but \\\"default\\\" is the canonical representation. Not all objects are required to be scoped to a namespace - the value of this field for those objects will be empty.\\n\\nMust be a DNS_LABEL. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/namespaces\",\"type\":\"string\"},\"ownerReferences\":{\"description\":\"List of objects depended by this object. If ALL objects in the list have been deleted, this object will be garbage collected. If this object is managed by a controller, then an entry in this list will point to this controller, with the controller field set to true. There cannot be more than one managing controller.\",\"items\":{\"description\":\"OwnerReference contains enough information to let you identify an owning object. An owning object must be in the same namespace as the dependent, or be cluster-scoped, so there is no namespace field.\",\"properties\":{\"apiVersion\":{\"description\":\"API version of the referent.\",\"type\":\"string\"},\"blockOwnerDeletion\":{\"description\":\"If true, AND if the owner has the \\\"foregroundDeletion\\\" finalizer, then the owner cannot be deleted from the key-value store until this reference is removed. Defaults to false. To set this field, a user needs \\\"delete\\\" permission of the owner, otherwise 422 (Unprocessable Entity) will be returned.\",\"type\":\"boolean\"},\"controller\":{\"description\":\"If true, this reference points to the managing controller.\",\"type\":\"boolean\"},\"kind\":{\"description\":\"Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#names\",\"type\":\"string\"},\"uid\":{\"description\":\"UID of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#uids\",\"type\":\"string\"}},\"required\":[\"apiVersion\",\"kind\",\"name\",\"uid\"],\"type\":\"object\"},\"type\":\"array\"},\"resourceVersion\":{\"description\":\"An opaque value that represents the internal version of this object that can be used by clients to determine when objects have changed. May be used for optimistic concurrency, change detection, and the watch operation on a resource or set of resources. Clients must treat these values as opaque and passed unmodified back to the server. They may only be valid for a particular resource or set of resources.\\n\\nPopulated by the system. Read-only. Value must be treated as opaque by clients and . More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency\",\"type\":\"string\"},\"selfLink\":{\"description\":\"SelfLink is a URL representing this object. Populated by the system. Read-only.\",\"type\":\"string\"},\"uid\":{\"description\":\"UID is the unique in time and space value for this object. It is typically generated by the server on successful creation of a resource and is not allowed to change on PUT operations.\\n\\nPopulated by the system. Read-only. More info: http://kubernetes.io/docs/user-guide/identifiers#uids\",\"type\":\"string\"}},\"type\":\"object\"},\"podMonitorNamespaceSelector\":{\"description\":\"A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.\",\"properties\":{\"matchExpressions\":{\"description\":\"matchExpressions is a list of label selector requirements. The requirements are ANDed.\",\"items\":{\"description\":\"A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"key is the label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\",\"type\":\"string\"},\"values\":{\"description\":\"values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"],\"type\":\"object\"},\"type\":\"array\"},\"matchLabels\":{\"description\":\"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \\\"key\\\", the operator is \\\"In\\\", and the values array contains only \\\"value\\\". The requirements are ANDed.\",\"type\":\"object\"}},\"type\":\"object\"},\"podMonitorSelector\":{\"description\":\"A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.\",\"properties\":{\"matchExpressions\":{\"description\":\"matchExpressions is a list of label selector requirements. The requirements are ANDed.\",\"items\":{\"description\":\"A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"key is the label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\",\"type\":\"string\"},\"values\":{\"description\":\"values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"],\"type\":\"object\"},\"type\":\"array\"},\"matchLabels\":{\"description\":\"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \\\"key\\\", the operator is \\\"In\\\", and the values array contains only \\\"value\\\". The requirements are ANDed.\",\"type\":\"object\"}},\"type\":\"object\"},\"priorityClassName\":{\"description\":\"Priority class assigned to the Pods\",\"type\":\"string\"},\"prometheusExternalLabelName\":{\"description\":\"Name of Prometheus external label used to denote Prometheus instance name. Defaults to the value of `prometheus`. External label will _not_ be added when value is set to empty string (`\\\"\\\"`).\",\"type\":\"string\"},\"query\":{\"description\":\"QuerySpec defines the query command line flags when starting Prometheus.\",\"properties\":{\"lookbackDelta\":{\"description\":\"The delta difference allowed for retrieving metrics during expression evaluations.\",\"type\":\"string\"},\"maxConcurrency\":{\"description\":\"Number of concurrent queries that can be run at once.\",\"format\":\"int32\",\"type\":\"integer\"},\"maxSamples\":{\"description\":\"Maximum number of samples a single query can load into memory. Note that queries will fail if they would load more samples than this into memory, so this also limits the number of samples a query can return.\",\"format\":\"int32\",\"type\":\"integer\"},\"timeout\":{\"description\":\"Maximum time a query may take before being aborted.\",\"type\":\"string\"}},\"type\":\"object\"},\"remoteRead\":{\"description\":\"If specified, the remote_read spec. This is an experimental feature, it may change in any upcoming release in a breaking way.\",\"items\":{\"description\":\"RemoteReadSpec defines the remote_read configuration for prometheus.\",\"properties\":{\"basicAuth\":{\"description\":\"BasicAuth allow an endpoint to authenticate over basic authentication More info: https://prometheus.io/docs/operating/configuration/#endpoints\",\"properties\":{\"password\":{\"description\":\"SecretKeySelector selects a key of a Secret.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or it's key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"},\"username\":{\"description\":\"SecretKeySelector selects a key of a Secret.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or it's key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"}},\"type\":\"object\"},\"bearerToken\":{\"description\":\"bearer token for remote read.\",\"type\":\"string\"},\"bearerTokenFile\":{\"description\":\"File to read bearer token for remote read.\",\"type\":\"string\"},\"proxyUrl\":{\"description\":\"Optional ProxyURL\",\"type\":\"string\"},\"readRecent\":{\"description\":\"Whether reads should be made for queries for time ranges that the local storage should have complete data for.\",\"type\":\"boolean\"},\"remoteTimeout\":{\"description\":\"Timeout for requests to the remote read endpoint.\",\"type\":\"string\"},\"requiredMatchers\":{\"description\":\"An optional list of equality matchers which have to be present in a selector to query the remote read endpoint.\",\"type\":\"object\"},\"tlsConfig\":{\"description\":\"TLSConfig specifies TLS configuration parameters.\",\"properties\":{\"caFile\":{\"description\":\"The CA cert to use for the targets.\",\"type\":\"string\"},\"certFile\":{\"description\":\"The client cert file for the targets.\",\"type\":\"string\"},\"insecureSkipVerify\":{\"description\":\"Disable target certificate validation.\",\"type\":\"boolean\"},\"keyFile\":{\"description\":\"The client key file for the targets.\",\"type\":\"string\"},\"serverName\":{\"description\":\"Used to verify the hostname for the targets.\",\"type\":\"string\"}},\"type\":\"object\"},\"url\":{\"description\":\"The URL of the endpoint to send samples to.\",\"type\":\"string\"}},\"required\":[\"url\"],\"type\":\"object\"},\"type\":\"array\"},\"remoteWrite\":{\"description\":\"If specified, the remote_write spec. This is an experimental feature, it may change in any upcoming release in a breaking way.\",\"items\":{\"description\":\"RemoteWriteSpec defines the remote_write configuration for prometheus.\",\"properties\":{\"basicAuth\":{\"description\":\"BasicAuth allow an endpoint to authenticate over basic authentication More info: https://prometheus.io/docs/operating/configuration/#endpoints\",\"properties\":{\"password\":{\"description\":\"SecretKeySelector selects a key of a Secret.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or it's key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"},\"username\":{\"description\":\"SecretKeySelector selects a key of a Secret.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or it's key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"}},\"type\":\"object\"},\"bearerToken\":{\"description\":\"File to read bearer token for remote write.\",\"type\":\"string\"},\"bearerTokenFile\":{\"description\":\"File to read bearer token for remote write.\",\"type\":\"string\"},\"proxyUrl\":{\"description\":\"Optional ProxyURL\",\"type\":\"string\"},\"queueConfig\":{\"description\":\"QueueConfig allows the tuning of remote_write queue_config parameters. This object is referenced in the RemoteWriteSpec object.\",\"properties\":{\"batchSendDeadline\":{\"description\":\"BatchSendDeadline is the maximum time a sample will wait in buffer.\",\"type\":\"string\"},\"capacity\":{\"description\":\"Capacity is the number of samples to buffer per shard before we start dropping them.\",\"format\":\"int32\",\"type\":\"integer\"},\"maxBackoff\":{\"description\":\"MaxBackoff is the maximum retry delay.\",\"type\":\"string\"},\"maxRetries\":{\"description\":\"MaxRetries is the maximum number of times to retry a batch on recoverable errors.\",\"format\":\"int32\",\"type\":\"integer\"},\"maxSamplesPerSend\":{\"description\":\"MaxSamplesPerSend is the maximum number of samples per send.\",\"format\":\"int32\",\"type\":\"integer\"},\"maxShards\":{\"description\":\"MaxShards is the maximum number of shards, i.e. amount of concurrency.\",\"format\":\"int32\",\"type\":\"integer\"},\"minBackoff\":{\"description\":\"MinBackoff is the initial retry delay. Gets doubled for every retry.\",\"type\":\"string\"},\"minShards\":{\"description\":\"MinShards is the minimum number of shards, i.e. amount of concurrency.\",\"format\":\"int32\",\"type\":\"integer\"}},\"type\":\"object\"},\"remoteTimeout\":{\"description\":\"Timeout for requests to the remote write endpoint.\",\"type\":\"string\"},\"tlsConfig\":{\"description\":\"TLSConfig specifies TLS configuration parameters.\",\"properties\":{\"caFile\":{\"description\":\"The CA cert to use for the targets.\",\"type\":\"string\"},\"certFile\":{\"description\":\"The client cert file for the targets.\",\"type\":\"string\"},\"insecureSkipVerify\":{\"description\":\"Disable target certificate validation.\",\"type\":\"boolean\"},\"keyFile\":{\"description\":\"The client key file for the targets.\",\"type\":\"string\"},\"serverName\":{\"description\":\"Used to verify the hostname for the targets.\",\"type\":\"string\"}},\"type\":\"object\"},\"url\":{\"description\":\"The URL of the endpoint to send samples to.\",\"type\":\"string\"},\"writeRelabelConfigs\":{\"description\":\"The list of remote write relabel configurations.\",\"items\":{\"description\":\"RelabelConfig allows dynamic rewriting of the label set, being applied to samples before ingestion. It defines `\\u003cmetric_relabel_configs\\u003e`-section of Prometheus configuration. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs\",\"properties\":{\"action\":{\"description\":\"Action to perform based on regex matching. Default is 'replace'\",\"type\":\"string\"},\"modulus\":{\"description\":\"Modulus to take of the hash of the source label values.\",\"format\":\"int64\",\"type\":\"integer\"},\"regex\":{\"description\":\"Regular expression against which the extracted value is matched. default is '(.*)'\",\"type\":\"string\"},\"replacement\":{\"description\":\"Replacement value against which a regex replace is performed if the regular expression matches. Regex capture groups are available. Default is '$1'\",\"type\":\"string\"},\"separator\":{\"description\":\"Separator placed between concatenated source label values. default is ';'.\",\"type\":\"string\"},\"sourceLabels\":{\"description\":\"The source labels select values from existing labels. Their content is concatenated using the configured separator and matched against the configured regular expression for the replace, keep, and drop actions.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"targetLabel\":{\"description\":\"Label to which the resulting value is written in a replace action. It is mandatory for replace actions. Regex capture groups are available.\",\"type\":\"string\"}},\"type\":\"object\"},\"type\":\"array\"}},\"required\":[\"url\"],\"type\":\"object\"},\"type\":\"array\"},\"replicaExternalLabelName\":{\"description\":\"Name of Prometheus external label used to denote replica name. Defaults to the value of `prometheus_replica`. External label will _not_ be added when value is set to empty string (`\\\"\\\"`).\",\"type\":\"string\"},\"replicas\":{\"description\":\"Number of instances to deploy for a Prometheus deployment.\",\"format\":\"int32\",\"type\":\"integer\"},\"resources\":{\"description\":\"ResourceRequirements describes the compute resource requirements.\",\"properties\":{\"limits\":{\"description\":\"Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/\",\"type\":\"object\"},\"requests\":{\"description\":\"Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/\",\"type\":\"object\"}},\"type\":\"object\"},\"retention\":{\"description\":\"Time duration Prometheus shall retain data for. Default is '24h', and must match the regular expression `[0-9]+(ms|s|m|h|d|w|y)` (milliseconds seconds minutes hours days weeks years).\",\"type\":\"string\"},\"retentionSize\":{\"description\":\"Maximum amount of disk space used by blocks.\",\"type\":\"string\"},\"routePrefix\":{\"description\":\"The route prefix Prometheus registers HTTP handlers for. This is useful, if using ExternalURL and a proxy is rewriting HTTP routes of a request, and the actual ExternalURL is still true, but the server serves requests under a different route prefix. For example for use with `kubectl proxy`.\",\"type\":\"string\"},\"ruleNamespaceSelector\":{\"description\":\"A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.\",\"properties\":{\"matchExpressions\":{\"description\":\"matchExpressions is a list of label selector requirements. The requirements are ANDed.\",\"items\":{\"description\":\"A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"key is the label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\",\"type\":\"string\"},\"values\":{\"description\":\"values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"],\"type\":\"object\"},\"type\":\"array\"},\"matchLabels\":{\"description\":\"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \\\"key\\\", the operator is \\\"In\\\", and the values array contains only \\\"value\\\". The requirements are ANDed.\",\"type\":\"object\"}},\"type\":\"object\"},\"ruleSelector\":{\"description\":\"A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.\",\"properties\":{\"matchExpressions\":{\"description\":\"matchExpressions is a list of label selector requirements. The requirements are ANDed.\",\"items\":{\"description\":\"A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"key is the label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\",\"type\":\"string\"},\"values\":{\"description\":\"values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"],\"type\":\"object\"},\"type\":\"array\"},\"matchLabels\":{\"description\":\"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \\\"key\\\", the operator is \\\"In\\\", and the values array contains only \\\"value\\\". The requirements are ANDed.\",\"type\":\"object\"}},\"type\":\"object\"},\"rules\":{\"description\":\"/--rules.*/ command-line arguments\",\"properties\":{\"alert\":{\"description\":\"/--rules.alert.*/ command-line arguments\",\"properties\":{\"forGracePeriod\":{\"description\":\"Minimum duration between alert and restored 'for' state. This is maintained only for alerts with configured 'for' time greater than grace period.\",\"type\":\"string\"},\"forOutageTolerance\":{\"description\":\"Max time to tolerate prometheus outage for restoring 'for' state of alert.\",\"type\":\"string\"},\"resendDelay\":{\"description\":\"Minimum amount of time to wait before resending an alert to Alertmanager.\",\"type\":\"string\"}},\"type\":\"object\"}},\"type\":\"object\"},\"scrapeInterval\":{\"description\":\"Interval between consecutive scrapes.\",\"type\":\"string\"},\"secrets\":{\"description\":\"Secrets is a list of Secrets in the same namespace as the Prometheus object, which shall be mounted into the Prometheus Pods. The Secrets are mounted into /etc/prometheus/secrets/\\u003csecret-name\\u003e.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"securityContext\":{\"description\":\"PodSecurityContext holds pod-level security attributes and common container settings. Some fields are also present in container.securityContext. Field values of container.securityContext take precedence over field values of PodSecurityContext.\",\"properties\":{\"fsGroup\":{\"description\":\"A special supplemental group that applies to all containers in a pod. Some volume types allow the Kubelet to change the ownership of that volume to be owned by the pod:\\n\\n1. The owning GID will be the FSGroup 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) 3. The permission bits are OR'd with rw-rw----\\n\\nIf unset, the Kubelet will not modify the ownership and permissions of any volume.\",\"format\":\"int64\",\"type\":\"integer\"},\"runAsGroup\":{\"description\":\"The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container.\",\"format\":\"int64\",\"type\":\"integer\"},\"runAsNonRoot\":{\"description\":\"Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\",\"type\":\"boolean\"},\"runAsUser\":{\"description\":\"The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container.\",\"format\":\"int64\",\"type\":\"integer\"},\"seLinuxOptions\":{\"description\":\"SELinuxOptions are the labels to be applied to the container\",\"properties\":{\"level\":{\"description\":\"Level is SELinux level label that applies to the container.\",\"type\":\"string\"},\"role\":{\"description\":\"Role is a SELinux role label that applies to the container.\",\"type\":\"string\"},\"type\":{\"description\":\"Type is a SELinux type label that applies to the container.\",\"type\":\"string\"},\"user\":{\"description\":\"User is a SELinux user label that applies to the container.\",\"type\":\"string\"}},\"type\":\"object\"},\"supplementalGroups\":{\"description\":\"A list of groups applied to the first process run in each container, in addition to the container's primary GID. If unspecified, no groups will be added to any container.\",\"items\":{\"format\":\"int64\",\"type\":\"integer\"},\"type\":\"array\"},\"sysctls\":{\"description\":\"Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported sysctls (by the container runtime) might fail to launch.\",\"items\":{\"description\":\"Sysctl defines a kernel parameter to be set\",\"properties\":{\"name\":{\"description\":\"Name of a property to set\",\"type\":\"string\"},\"value\":{\"description\":\"Value of a property to set\",\"type\":\"string\"}},\"required\":[\"name\",\"value\"],\"type\":\"object\"},\"type\":\"array\"}},\"type\":\"object\"},\"serviceAccountName\":{\"description\":\"ServiceAccountName is the name of the ServiceAccount to use to run the Prometheus Pods.\",\"type\":\"string\"},\"serviceMonitorNamespaceSelector\":{\"description\":\"A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.\",\"properties\":{\"matchExpressions\":{\"description\":\"matchExpressions is a list of label selector requirements. The requirements are ANDed.\",\"items\":{\"description\":\"A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"key is the label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\",\"type\":\"string\"},\"values\":{\"description\":\"values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"],\"type\":\"object\"},\"type\":\"array\"},\"matchLabels\":{\"description\":\"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \\\"key\\\", the operator is \\\"In\\\", and the values array contains only \\\"value\\\". The requirements are ANDed.\",\"type\":\"object\"}},\"type\":\"object\"},\"serviceMonitorSelector\":{\"description\":\"A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.\",\"properties\":{\"matchExpressions\":{\"description\":\"matchExpressions is a list of label selector requirements. The requirements are ANDed.\",\"items\":{\"description\":\"A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"key is the label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\",\"type\":\"string\"},\"values\":{\"description\":\"values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"],\"type\":\"object\"},\"type\":\"array\"},\"matchLabels\":{\"description\":\"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \\\"key\\\", the operator is \\\"In\\\", and the values array contains only \\\"value\\\". The requirements are ANDed.\",\"type\":\"object\"}},\"type\":\"object\"},\"sha\":{\"description\":\"SHA of Prometheus container image to be deployed. Defaults to the value of `version`. Similar to a tag, but the SHA explicitly deploys an immutable container image. Version and Tag are ignored if SHA is set.\",\"type\":\"string\"},\"storage\":{\"description\":\"StorageSpec defines the configured storage for a group Prometheus servers. If neither `emptyDir` nor `volumeClaimTemplate` is specified, then by default an [EmptyDir](https://kubernetes.io/docs/concepts/storage/volumes/#emptydir) will be used.\",\"properties\":{\"emptyDir\":{\"description\":\"Represents an empty directory for a pod. Empty directory volumes support ownership management and SELinux relabeling.\",\"properties\":{\"medium\":{\"description\":\"What type of storage medium should back this directory. The default is \\\"\\\" which means to use the node's default medium. Must be an empty string (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir\",\"type\":\"string\"},\"sizeLimit\":{}},\"type\":\"object\"},\"volumeClaimTemplate\":{\"description\":\"PersistentVolumeClaim is a user's request for and claim to a persistent volume\",\"properties\":{\"apiVersion\":{\"description\":\"APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources\",\"type\":\"string\"},\"kind\":{\"description\":\"Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds\",\"type\":\"string\"},\"metadata\":{\"description\":\"ObjectMeta is metadata that all persisted resources must have, which includes all objects users must create.\",\"properties\":{\"annotations\":{\"description\":\"Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations\",\"type\":\"object\"},\"clusterName\":{\"description\":\"The name of the cluster which the object belongs to. This is used to distinguish resources with same name and namespace in different clusters. This field is not set anywhere right now and apiserver is going to ignore it if set in create or update request.\",\"type\":\"string\"},\"creationTimestamp\":{\"description\":\"Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.\",\"format\":\"date-time\",\"type\":\"string\"},\"deletionGracePeriodSeconds\":{\"description\":\"Number of seconds allowed for this object to gracefully terminate before it will be removed from the system. Only set when deletionTimestamp is also set. May only be shortened. Read-only.\",\"format\":\"int64\",\"type\":\"integer\"},\"deletionTimestamp\":{\"description\":\"Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.\",\"format\":\"date-time\",\"type\":\"string\"},\"finalizers\":{\"description\":\"Must be empty before the object is deleted from the registry. Each entry is an identifier for the responsible component that will remove the entry from the list. If the deletionTimestamp of the object is non-nil, entries in this list can only be removed.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"generateName\":{\"description\":\"GenerateName is an optional prefix, used by the server, to generate a unique name ONLY IF the Name field has not been provided. If this field is used, the name returned to the client will be different than the name passed. This value will also be combined with a unique suffix. The provided value has the same validation rules as the Name field, and may be truncated by the length of the suffix required to make the value unique on the server.\\n\\nIf this field is specified and the generated name exists, the server will NOT return a 409 - instead, it will either return 201 Created or 500 with Reason ServerTimeout indicating a unique name could not be found in the time allotted, and the client should retry (optionally after the time indicated in the Retry-After header).\\n\\nApplied only if Name is not specified. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#idempotency\",\"type\":\"string\"},\"generation\":{\"description\":\"A sequence number representing a specific generation of the desired state. Populated by the system. Read-only.\",\"format\":\"int64\",\"type\":\"integer\"},\"initializers\":{\"description\":\"Initializers tracks the progress of initialization.\",\"properties\":{\"pending\":{\"description\":\"Pending is a list of initializers that must execute in order before this object is visible. When the last pending initializer is removed, and no failing result is set, the initializers struct will be set to nil and the object is considered as initialized and visible to all clients.\",\"items\":{\"description\":\"Initializer is information about an initializer that has not yet completed.\",\"properties\":{\"name\":{\"description\":\"name of the process that is responsible for initializing this object.\",\"type\":\"string\"}},\"required\":[\"name\"],\"type\":\"object\"},\"type\":\"array\"},\"result\":{\"description\":\"Status is a return value for calls that don't return other objects.\",\"properties\":{\"apiVersion\":{\"description\":\"APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources\",\"type\":\"string\"},\"code\":{\"description\":\"Suggested HTTP return code for this status, 0 if not set.\",\"format\":\"int32\",\"type\":\"integer\"},\"details\":{\"description\":\"StatusDetails is a set of additional properties that MAY be set by the server to provide additional information about a response. The Reason field of a Status object defines what attributes will be set. Clients must ignore fields that do not match the defined type of each attribute, and should assume that any attribute may be empty, invalid, or under defined.\",\"properties\":{\"causes\":{\"description\":\"The Causes array includes more details associated with the StatusReason failure. Not all StatusReasons may provide detailed causes.\",\"items\":{\"description\":\"StatusCause provides more information about an api.Status failure, including cases when multiple errors are encountered.\",\"properties\":{\"field\":{\"description\":\"The field of the resource that has caused this error, as named by its JSON serialization. May include dot and postfix notation for nested attributes. Arrays are zero-indexed. Fields may appear more than once in an array of causes due to fields having multiple errors. Optional.\\n\\nExamples:\\n \\\"name\\\" - the field \\\"name\\\" on the current resource\\n \\\"items[0].name\\\" - the field \\\"name\\\" on the first array entry in \\\"items\\\"\",\"type\":\"string\"},\"message\":{\"description\":\"A human-readable description of the cause of the error. This field may be presented as-is to a reader.\",\"type\":\"string\"},\"reason\":{\"description\":\"A machine-readable description of the cause of the error. If this value is empty there is no information available.\",\"type\":\"string\"}},\"type\":\"object\"},\"type\":\"array\"},\"group\":{\"description\":\"The group attribute of the resource associated with the status StatusReason.\",\"type\":\"string\"},\"kind\":{\"description\":\"The kind attribute of the resource associated with the status StatusReason. On some operations may differ from the requested resource Kind. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds\",\"type\":\"string\"},\"name\":{\"description\":\"The name attribute of the resource associated with the status StatusReason (when there is a single name which can be described).\",\"type\":\"string\"},\"retryAfterSeconds\":{\"description\":\"If specified, the time in seconds before the operation should be retried. Some errors may indicate the client must take an alternate action - for those errors this field may indicate how long to wait before taking the alternate action.\",\"format\":\"int32\",\"type\":\"integer\"},\"uid\":{\"description\":\"UID of the resource. (when there is a single resource which can be described). More info: http://kubernetes.io/docs/user-guide/identifiers#uids\",\"type\":\"string\"}},\"type\":\"object\"},\"kind\":{\"description\":\"Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds\",\"type\":\"string\"},\"message\":{\"description\":\"A human-readable description of the status of this operation.\",\"type\":\"string\"},\"metadata\":{\"description\":\"ListMeta describes metadata that synthetic resources must have, including lists and various status objects. A resource may have only one of {ObjectMeta, ListMeta}.\",\"properties\":{\"continue\":{\"description\":\"continue may be set if the user set a limit on the number of items returned, and indicates that the server has more data available. The value is opaque and may be used to issue another request to the endpoint that served this list to retrieve the next set of available objects. Continuing a consistent list may not be possible if the server configuration has changed or more than a few minutes have passed. The resourceVersion field returned when using this continue value will be identical to the value in the first response, unless you have received this token from an error message.\",\"type\":\"string\"},\"resourceVersion\":{\"description\":\"String that identifies the server's internal version of this object that can be used by clients to determine when objects have changed. Value must be treated as opaque by clients and passed unmodified back to the server. Populated by the system. Read-only. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency\",\"type\":\"string\"},\"selfLink\":{\"description\":\"selfLink is a URL representing this object. Populated by the system. Read-only.\",\"type\":\"string\"}},\"type\":\"object\"},\"reason\":{\"description\":\"A machine-readable description of why this operation is in the \\\"Failure\\\" status. If this value is empty there is no information available. A Reason clarifies an HTTP status code but does not override it.\",\"type\":\"string\"},\"status\":{\"description\":\"Status of the operation. One of: \\\"Success\\\" or \\\"Failure\\\". More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status\",\"type\":\"string\"}},\"type\":\"object\"}},\"required\":[\"pending\"],\"type\":\"object\"},\"labels\":{\"description\":\"Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels\",\"type\":\"object\"},\"managedFields\":{\"description\":\"ManagedFields maps workflow-id and version to the set of fields that are managed by that workflow. This is mostly for internal housekeeping, and users typically shouldn't need to set or understand this field. A workflow can be the user's name, a controller's name, or the name of a specific apply path like \\\"ci-cd\\\". The set of fields is always in the version that the workflow used when modifying the object.\\n\\nThis field is alpha and can be changed or removed without notice.\",\"items\":{\"description\":\"ManagedFieldsEntry is a workflow-id, a FieldSet and the group version of the resource that the fieldset applies to.\",\"properties\":{\"apiVersion\":{\"description\":\"APIVersion defines the version of this resource that this field set applies to. The format is \\\"group/version\\\" just like the top-level APIVersion field. It is necessary to track the version of a field set because it cannot be automatically converted.\",\"type\":\"string\"},\"fields\":{\"description\":\"Fields stores a set of fields in a data structure like a Trie. To understand how this is used, see: https://github.com/kubernetes-sigs/structured-merge-diff\",\"type\":\"object\"},\"manager\":{\"description\":\"Manager is an identifier of the workflow managing these fields.\",\"type\":\"string\"},\"operation\":{\"description\":\"Operation is the type of operation which lead to this ManagedFieldsEntry being created. The only valid values for this field are 'Apply' and 'Update'.\",\"type\":\"string\"},\"time\":{\"description\":\"Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.\",\"format\":\"date-time\",\"type\":\"string\"}},\"type\":\"object\"},\"type\":\"array\"},\"name\":{\"description\":\"Name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names\",\"type\":\"string\"},\"namespace\":{\"description\":\"Namespace defines the space within each name must be unique. An empty namespace is equivalent to the \\\"default\\\" namespace, but \\\"default\\\" is the canonical representation. Not all objects are required to be scoped to a namespace - the value of this field for those objects will be empty.\\n\\nMust be a DNS_LABEL. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/namespaces\",\"type\":\"string\"},\"ownerReferences\":{\"description\":\"List of objects depended by this object. If ALL objects in the list have been deleted, this object will be garbage collected. If this object is managed by a controller, then an entry in this list will point to this controller, with the controller field set to true. There cannot be more than one managing controller.\",\"items\":{\"description\":\"OwnerReference contains enough information to let you identify an owning object. An owning object must be in the same namespace as the dependent, or be cluster-scoped, so there is no namespace field.\",\"properties\":{\"apiVersion\":{\"description\":\"API version of the referent.\",\"type\":\"string\"},\"blockOwnerDeletion\":{\"description\":\"If true, AND if the owner has the \\\"foregroundDeletion\\\" finalizer, then the owner cannot be deleted from the key-value store until this reference is removed. Defaults to false. To set this field, a user needs \\\"delete\\\" permission of the owner, otherwise 422 (Unprocessable Entity) will be returned.\",\"type\":\"boolean\"},\"controller\":{\"description\":\"If true, this reference points to the managing controller.\",\"type\":\"boolean\"},\"kind\":{\"description\":\"Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#names\",\"type\":\"string\"},\"uid\":{\"description\":\"UID of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#uids\",\"type\":\"string\"}},\"required\":[\"apiVersion\",\"kind\",\"name\",\"uid\"],\"type\":\"object\"},\"type\":\"array\"},\"resourceVersion\":{\"description\":\"An opaque value that represents the internal version of this object that can be used by clients to determine when objects have changed. May be used for optimistic concurrency, change detection, and the watch operation on a resource or set of resources. Clients must treat these values as opaque and passed unmodified back to the server. They may only be valid for a particular resource or set of resources.\\n\\nPopulated by the system. Read-only. Value must be treated as opaque by clients and . More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency\",\"type\":\"string\"},\"selfLink\":{\"description\":\"SelfLink is a URL representing this object. Populated by the system. Read-only.\",\"type\":\"string\"},\"uid\":{\"description\":\"UID is the unique in time and space value for this object. It is typically generated by the server on successful creation of a resource and is not allowed to change on PUT operations.\\n\\nPopulated by the system. Read-only. More info: http://kubernetes.io/docs/user-guide/identifiers#uids\",\"type\":\"string\"}},\"type\":\"object\"},\"spec\":{\"description\":\"PersistentVolumeClaimSpec describes the common attributes of storage devices and allows a Source for provider-specific attributes\",\"properties\":{\"accessModes\":{\"description\":\"AccessModes contains the desired access modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"dataSource\":{\"description\":\"TypedLocalObjectReference contains enough information to let you locate the typed referenced object inside the same namespace.\",\"properties\":{\"apiGroup\":{\"description\":\"APIGroup is the group for the resource being referenced. If APIGroup is not specified, the specified Kind must be in the core API group. For any other third-party types, APIGroup is required.\",\"type\":\"string\"},\"kind\":{\"description\":\"Kind is the type of resource being referenced\",\"type\":\"string\"},\"name\":{\"description\":\"Name is the name of resource being referenced\",\"type\":\"string\"}},\"required\":[\"kind\",\"name\"],\"type\":\"object\"},\"resources\":{\"description\":\"ResourceRequirements describes the compute resource requirements.\",\"properties\":{\"limits\":{\"description\":\"Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/\",\"type\":\"object\"},\"requests\":{\"description\":\"Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/\",\"type\":\"object\"}},\"type\":\"object\"},\"selector\":{\"description\":\"A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.\",\"properties\":{\"matchExpressions\":{\"description\":\"matchExpressions is a list of label selector requirements. The requirements are ANDed.\",\"items\":{\"description\":\"A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"key is the label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\",\"type\":\"string\"},\"values\":{\"description\":\"values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"],\"type\":\"object\"},\"type\":\"array\"},\"matchLabels\":{\"description\":\"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \\\"key\\\", the operator is \\\"In\\\", and the values array contains only \\\"value\\\". The requirements are ANDed.\",\"type\":\"object\"}},\"type\":\"object\"},\"storageClassName\":{\"description\":\"Name of the StorageClass required by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1\",\"type\":\"string\"},\"volumeMode\":{\"description\":\"volumeMode defines what type of volume is required by the claim. Value of Filesystem is implied when not included in claim spec. This is a beta feature.\",\"type\":\"string\"},\"volumeName\":{\"description\":\"VolumeName is the binding reference to the PersistentVolume backing this claim.\",\"type\":\"string\"}},\"type\":\"object\"},\"status\":{\"description\":\"PersistentVolumeClaimStatus is the current status of a persistent volume claim.\",\"properties\":{\"accessModes\":{\"description\":\"AccessModes contains the actual access modes the volume backing the PVC has. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"capacity\":{\"description\":\"Represents the actual resources of the underlying volume.\",\"type\":\"object\"},\"conditions\":{\"description\":\"Current Condition of persistent volume claim. If underlying persistent volume is being resized then the Condition will be set to 'ResizeStarted'.\",\"items\":{\"description\":\"PersistentVolumeClaimCondition contains details about state of pvc\",\"properties\":{\"lastProbeTime\":{\"description\":\"Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.\",\"format\":\"date-time\",\"type\":\"string\"},\"lastTransitionTime\":{\"description\":\"Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.\",\"format\":\"date-time\",\"type\":\"string\"},\"message\":{\"description\":\"Human-readable message indicating details about last transition.\",\"type\":\"string\"},\"reason\":{\"description\":\"Unique, this should be a short, machine understandable string that gives the reason for condition's last transition. If it reports \\\"ResizeStarted\\\" that means the underlying persistent volume is being resized.\",\"type\":\"string\"},\"status\":{\"type\":\"string\"},\"type\":{\"type\":\"string\"}},\"required\":[\"type\",\"status\"],\"type\":\"object\"},\"type\":\"array\"},\"phase\":{\"description\":\"Phase represents the current phase of PersistentVolumeClaim.\",\"type\":\"string\"}},\"type\":\"object\"}},\"type\":\"object\"}},\"type\":\"object\"},\"tag\":{\"description\":\"Tag of Prometheus container image to be deployed. Defaults to the value of `version`. Version is ignored if Tag is set.\",\"type\":\"string\"},\"thanos\":{\"description\":\"ThanosSpec defines parameters for a Prometheus server within a Thanos deployment.\",\"properties\":{\"baseImage\":{\"description\":\"Thanos base image if other than default.\",\"type\":\"string\"},\"image\":{\"description\":\"Image if specified has precedence over baseImage, tag and sha combinations. Specifying the version is still necessary to ensure the Prometheus Operator knows what version of Thanos is being configured.\",\"type\":\"string\"},\"objectStorageConfig\":{\"description\":\"SecretKeySelector selects a key of a Secret.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or it's key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"},\"resources\":{\"description\":\"ResourceRequirements describes the compute resource requirements.\",\"properties\":{\"limits\":{\"description\":\"Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/\",\"type\":\"object\"},\"requests\":{\"description\":\"Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/\",\"type\":\"object\"}},\"type\":\"object\"},\"sha\":{\"description\":\"SHA of Thanos container image to be deployed. Defaults to the value of `version`. Similar to a tag, but the SHA explicitly deploys an immutable container image. Version and Tag are ignored if SHA is set.\",\"type\":\"string\"},\"tag\":{\"description\":\"Tag of Thanos sidecar container image to be deployed. Defaults to the value of `version`. Version is ignored if Tag is set.\",\"type\":\"string\"},\"version\":{\"description\":\"Version describes the version of Thanos to use.\",\"type\":\"string\"}},\"type\":\"object\"},\"tolerations\":{\"description\":\"If specified, the pod's tolerations.\",\"items\":{\"description\":\"The pod this Toleration is attached to tolerates any taint that matches the triple \\u003ckey,value,effect\\u003e using the matching operator \\u003coperator\\u003e.\",\"properties\":{\"effect\":{\"description\":\"Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.\",\"type\":\"string\"},\"key\":{\"description\":\"Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys.\",\"type\":\"string\"},\"operator\":{\"description\":\"Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category.\",\"type\":\"string\"},\"tolerationSeconds\":{\"description\":\"TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system.\",\"format\":\"int64\",\"type\":\"integer\"},\"value\":{\"description\":\"Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string.\",\"type\":\"string\"}},\"type\":\"object\"},\"type\":\"array\"},\"version\":{\"description\":\"Version of Prometheus to be deployed.\",\"type\":\"string\"},\"walCompression\":{\"description\":\"Enable compression of the write-ahead log using Snappy.\",\"type\":\"boolean\"}},\"type\":\"object\"},\"status\":{\"description\":\"PrometheusStatus is the most recent observed status of the Prometheus cluster. Read-only. Not included when requesting from the apiserver, only from the Prometheus Operator API itself. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/api-conventions.md#spec-and-status\",\"properties\":{\"availableReplicas\":{\"description\":\"Total number of available pods (ready for at least minReadySeconds) targeted by this Prometheus deployment.\",\"format\":\"int32\",\"type\":\"integer\"},\"paused\":{\"description\":\"Represents whether any actions on the underlying managed objects are being performed. Only delete actions will be performed.\",\"type\":\"boolean\"},\"replicas\":{\"description\":\"Total number of non-terminated pods targeted by this Prometheus deployment (their labels match the selector).\",\"format\":\"int32\",\"type\":\"integer\"},\"unavailableReplicas\":{\"description\":\"Total number of unavailable pods targeted by this Prometheus deployment.\",\"format\":\"int32\",\"type\":\"integer\"},\"updatedReplicas\":{\"description\":\"Total number of non-terminated pods targeted by this Prometheus deployment that have the desired version spec.\",\"format\":\"int32\",\"type\":\"integer\"}},\"required\":[\"paused\",\"replicas\",\"updatedReplicas\",\"availableReplicas\",\"unavailableReplicas\"],\"type\":\"object\"}},\"type\":\"object\"}},\"version\":\"v1\"}}\n" }, "creationTimestamp": "2020-05-05T16:58:10Z", "generation": 1, "labels": { "app": "prometheus-operator" }, "name": "prometheuses.monitoring.coreos.com", "resourceVersion": "207497", "selfLink": "/apis/apiextensions.k8s.io/v1/customresourcedefinitions/prometheuses.monitoring.coreos.com", "uid": "472b9025-d931-4d48-8a6f-77c3b7c33f6e" }, "spec": { "conversion": { "strategy": "None" }, "group": "monitoring.coreos.com", "names": { "kind": "Prometheus", "listKind": "PrometheusList", "plural": "prometheuses", "singular": "prometheus" }, "preserveUnknownFields": true, "scope": "Namespaced", "versions": [ { "name": "v1", "schema": { "openAPIV3Schema": { "properties": { "apiVersion": { "description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources", "type": "string" }, "kind": { "description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds", "type": "string" }, "spec": { "description": "PrometheusSpec is a specification of the desired behavior of the Prometheus cluster. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/api-conventions.md#spec-and-status", "properties": { "additionalAlertManagerConfigs": { "description": "SecretKeySelector selects a key of a Secret.", "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", "type": "string" }, "optional": { "description": "Specify whether the Secret or it's key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" }, "additionalAlertRelabelConfigs": { "description": "SecretKeySelector selects a key of a Secret.", "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", "type": "string" }, "optional": { "description": "Specify whether the Secret or it's key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" }, "additionalScrapeConfigs": { "description": "SecretKeySelector selects a key of a Secret.", "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", "type": "string" }, "optional": { "description": "Specify whether the Secret or it's key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" }, "affinity": { "description": "Affinity is a group of affinity scheduling rules.", "properties": { "nodeAffinity": { "description": "Node affinity is a group of node affinity scheduling rules.", "properties": { "preferredDuringSchedulingIgnoredDuringExecution": { "description": "The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding \"weight\" to the sum if the node matches the corresponding matchExpressions; the node(s) with the highest sum are the most preferred.", "items": { "description": "An empty preferred scheduling term matches all objects with implicit weight 0 (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op).", "properties": { "preference": { "description": "A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm.", "properties": { "matchExpressions": { "description": "A list of node selector requirements by node's labels.", "items": { "description": "A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "properties": { "key": { "description": "The label key that the selector applies to.", "type": "string" }, "operator": { "description": "Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.", "type": "string" }, "values": { "description": "An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.", "items": { "type": "string" }, "type": "array" } }, "required": [ "key", "operator" ], "type": "object" }, "type": "array" }, "matchFields": { "description": "A list of node selector requirements by node's fields.", "items": { "description": "A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "properties": { "key": { "description": "The label key that the selector applies to.", "type": "string" }, "operator": { "description": "Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.", "type": "string" }, "values": { "description": "An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.", "items": { "type": "string" }, "type": "array" } }, "required": [ "key", "operator" ], "type": "object" }, "type": "array" } }, "type": "object" }, "weight": { "description": "Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100.", "format": "int32", "type": "integer" } }, "required": [ "weight", "preference" ], "type": "object" }, "type": "array" }, "requiredDuringSchedulingIgnoredDuringExecution": { "description": "A node selector represents the union of the results of one or more label queries over a set of nodes; that is, it represents the OR of the selectors represented by the node selector terms.", "properties": { "nodeSelectorTerms": { "description": "Required. A list of node selector terms. The terms are ORed.", "items": { "description": "A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm.", "properties": { "matchExpressions": { "description": "A list of node selector requirements by node's labels.", "items": { "description": "A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "properties": { "key": { "description": "The label key that the selector applies to.", "type": "string" }, "operator": { "description": "Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.", "type": "string" }, "values": { "description": "An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.", "items": { "type": "string" }, "type": "array" } }, "required": [ "key", "operator" ], "type": "object" }, "type": "array" }, "matchFields": { "description": "A list of node selector requirements by node's fields.", "items": { "description": "A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "properties": { "key": { "description": "The label key that the selector applies to.", "type": "string" }, "operator": { "description": "Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.", "type": "string" }, "values": { "description": "An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.", "items": { "type": "string" }, "type": "array" } }, "required": [ "key", "operator" ], "type": "object" }, "type": "array" } }, "type": "object" }, "type": "array" } }, "required": [ "nodeSelectorTerms" ], "type": "object" } }, "type": "object" }, "podAffinity": { "description": "Pod affinity is a group of inter pod affinity scheduling rules.", "properties": { "preferredDuringSchedulingIgnoredDuringExecution": { "description": "The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding \"weight\" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred.", "items": { "description": "The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s)", "properties": { "podAffinityTerm": { "description": "Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key \u003ctopologyKey\u003e matches that of any node on which a pod of the set of pods is running", "properties": { "labelSelector": { "description": "A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "items": { "type": "string" }, "type": "array" } }, "required": [ "key", "operator" ], "type": "object" }, "type": "array" }, "matchLabels": { "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object" } }, "type": "object" }, "namespaces": { "description": "namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \"this pod's namespace\"", "items": { "type": "string" }, "type": "array" }, "topologyKey": { "description": "This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.", "type": "string" } }, "required": [ "topologyKey" ], "type": "object" }, "weight": { "description": "weight associated with matching the corresponding podAffinityTerm, in the range 1-100.", "format": "int32", "type": "integer" } }, "required": [ "weight", "podAffinityTerm" ], "type": "object" }, "type": "array" }, "requiredDuringSchedulingIgnoredDuringExecution": { "description": "If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied.", "items": { "description": "Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key \u003ctopologyKey\u003e matches that of any node on which a pod of the set of pods is running", "properties": { "labelSelector": { "description": "A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "items": { "type": "string" }, "type": "array" } }, "required": [ "key", "operator" ], "type": "object" }, "type": "array" }, "matchLabels": { "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object" } }, "type": "object" }, "namespaces": { "description": "namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \"this pod's namespace\"", "items": { "type": "string" }, "type": "array" }, "topologyKey": { "description": "This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.", "type": "string" } }, "required": [ "topologyKey" ], "type": "object" }, "type": "array" } }, "type": "object" }, "podAntiAffinity": { "description": "Pod anti affinity is a group of inter pod anti affinity scheduling rules.", "properties": { "preferredDuringSchedulingIgnoredDuringExecution": { "description": "The scheduler will prefer to schedule pods to nodes that satisfy the anti-affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding \"weight\" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred.", "items": { "description": "The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s)", "properties": { "podAffinityTerm": { "description": "Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key \u003ctopologyKey\u003e matches that of any node on which a pod of the set of pods is running", "properties": { "labelSelector": { "description": "A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "items": { "type": "string" }, "type": "array" } }, "required": [ "key", "operator" ], "type": "object" }, "type": "array" }, "matchLabels": { "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object" } }, "type": "object" }, "namespaces": { "description": "namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \"this pod's namespace\"", "items": { "type": "string" }, "type": "array" }, "topologyKey": { "description": "This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.", "type": "string" } }, "required": [ "topologyKey" ], "type": "object" }, "weight": { "description": "weight associated with matching the corresponding podAffinityTerm, in the range 1-100.", "format": "int32", "type": "integer" } }, "required": [ "weight", "podAffinityTerm" ], "type": "object" }, "type": "array" }, "requiredDuringSchedulingIgnoredDuringExecution": { "description": "If the anti-affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the anti-affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied.", "items": { "description": "Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key \u003ctopologyKey\u003e matches that of any node on which a pod of the set of pods is running", "properties": { "labelSelector": { "description": "A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "items": { "type": "string" }, "type": "array" } }, "required": [ "key", "operator" ], "type": "object" }, "type": "array" }, "matchLabels": { "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object" } }, "type": "object" }, "namespaces": { "description": "namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \"this pod's namespace\"", "items": { "type": "string" }, "type": "array" }, "topologyKey": { "description": "This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.", "type": "string" } }, "required": [ "topologyKey" ], "type": "object" }, "type": "array" } }, "type": "object" } }, "type": "object" }, "alerting": { "description": "AlertingSpec defines parameters for alerting configuration of Prometheus servers.", "properties": { "alertmanagers": { "description": "AlertmanagerEndpoints Prometheus should fire alerts against.", "items": { "description": "AlertmanagerEndpoints defines a selection of a single Endpoints object containing alertmanager IPs to fire alerts against.", "properties": { "bearerTokenFile": { "description": "BearerTokenFile to read from filesystem to use when authenticating to Alertmanager.", "type": "string" }, "name": { "description": "Name of Endpoints object in Namespace.", "type": "string" }, "namespace": { "description": "Namespace of Endpoints object.", "type": "string" }, "pathPrefix": { "description": "Prefix for the HTTP path alerts are pushed to.", "type": "string" }, "port": { "anyOf": [ { "type": "string" }, { "type": "integer" } ] }, "scheme": { "description": "Scheme to use when firing alerts.", "type": "string" }, "tlsConfig": { "description": "TLSConfig specifies TLS configuration parameters.", "properties": { "caFile": { "description": "The CA cert to use for the targets.", "type": "string" }, "certFile": { "description": "The client cert file for the targets.", "type": "string" }, "insecureSkipVerify": { "description": "Disable target certificate validation.", "type": "boolean" }, "keyFile": { "description": "The client key file for the targets.", "type": "string" }, "serverName": { "description": "Used to verify the hostname for the targets.", "type": "string" } }, "type": "object" } }, "required": [ "namespace", "name", "port" ], "type": "object" }, "type": "array" } }, "required": [ "alertmanagers" ], "type": "object" }, "apiserverConfig": { "description": "APIServerConfig defines a host and auth methods to access apiserver. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config", "properties": { "basicAuth": { "description": "BasicAuth allow an endpoint to authenticate over basic authentication More info: https://prometheus.io/docs/operating/configuration/#endpoints", "properties": { "password": { "description": "SecretKeySelector selects a key of a Secret.", "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", "type": "string" }, "optional": { "description": "Specify whether the Secret or it's key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" }, "username": { "description": "SecretKeySelector selects a key of a Secret.", "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", "type": "string" }, "optional": { "description": "Specify whether the Secret or it's key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" } }, "type": "object" }, "bearerToken": { "description": "Bearer token for accessing apiserver.", "type": "string" }, "bearerTokenFile": { "description": "File to read bearer token for accessing apiserver.", "type": "string" }, "host": { "description": "Host of apiserver. A valid string consisting of a hostname or IP followed by an optional port number", "type": "string" }, "tlsConfig": { "description": "TLSConfig specifies TLS configuration parameters.", "properties": { "caFile": { "description": "The CA cert to use for the targets.", "type": "string" }, "certFile": { "description": "The client cert file for the targets.", "type": "string" }, "insecureSkipVerify": { "description": "Disable target certificate validation.", "type": "boolean" }, "keyFile": { "description": "The client key file for the targets.", "type": "string" }, "serverName": { "description": "Used to verify the hostname for the targets.", "type": "string" } }, "type": "object" } }, "required": [ "host" ], "type": "object" }, "baseImage": { "description": "Base image to use for a Prometheus deployment.", "type": "string" }, "configMaps": { "description": "ConfigMaps is a list of ConfigMaps in the same namespace as the Prometheus object, which shall be mounted into the Prometheus Pods. The ConfigMaps are mounted into /etc/prometheus/configmaps/\u003cconfigmap-name\u003e.", "items": { "type": "string" }, "type": "array" }, "containers": { "description": "Containers allows injecting additional containers or modifying operator generated containers. This can be used to allow adding an authentication proxy to a Prometheus pod or to change the behavior of an operator generated container. Containers described here modify an operator generated container if they share the same name and modifications are done via a strategic merge patch. The current container names are: `prometheus`, `prometheus-config-reloader`, `rules-configmap-reloader`, and `thanos-sidecar`. Overriding containers is entirely outside the scope of what the maintainers will support and by doing so, you accept that this behaviour may break at any time without notice.", "items": { "description": "A single application container that you want to run within a pod.", "properties": { "args": { "description": "Arguments to the entrypoint. The docker image's CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell", "items": { "type": "string" }, "type": "array" }, "command": { "description": "Entrypoint array. Not executed within a shell. The docker image's ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell", "items": { "type": "string" }, "type": "array" }, "env": { "description": "List of environment variables to set in the container. Cannot be updated.", "items": { "description": "EnvVar represents an environment variable present in a Container.", "properties": { "name": { "description": "Name of the environment variable. Must be a C_IDENTIFIER.", "type": "string" }, "value": { "description": "Variable references $(VAR_NAME) are expanded using the previous defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Defaults to \"\".", "type": "string" }, "valueFrom": { "description": "EnvVarSource represents a source for the value of an EnvVar.", "properties": { "configMapKeyRef": { "description": "Selects a key from a ConfigMap.", "properties": { "key": { "description": "The key to select.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", "type": "string" }, "optional": { "description": "Specify whether the ConfigMap or it's key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" }, "fieldRef": { "description": "ObjectFieldSelector selects an APIVersioned field of an object.", "properties": { "apiVersion": { "description": "Version of the schema the FieldPath is written in terms of, defaults to \"v1\".", "type": "string" }, "fieldPath": { "description": "Path of the field to select in the specified API version.", "type": "string" } }, "required": [ "fieldPath" ], "type": "object" }, "resourceFieldRef": { "description": "ResourceFieldSelector represents container resources (cpu, memory) and their output format", "properties": { "containerName": { "description": "Container name: required for volumes, optional for env vars", "type": "string" }, "divisor": {}, "resource": { "description": "Required: resource to select", "type": "string" } }, "required": [ "resource" ], "type": "object" }, "secretKeyRef": { "description": "SecretKeySelector selects a key of a Secret.", "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", "type": "string" }, "optional": { "description": "Specify whether the Secret or it's key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" } }, "type": "object" } }, "required": [ "name" ], "type": "object" }, "type": "array" }, "envFrom": { "description": "List of sources to populate environment variables in the container. The keys defined within a source must be a C_IDENTIFIER. All invalid keys will be reported as an event when the container is starting. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated.", "items": { "description": "EnvFromSource represents the source of a set of ConfigMaps", "properties": { "configMapRef": { "description": "ConfigMapEnvSource selects a ConfigMap to populate the environment variables with.\n\nThe contents of the target ConfigMap's Data field will represent the key-value pairs as environment variables.", "properties": { "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", "type": "string" }, "optional": { "description": "Specify whether the ConfigMap must be defined", "type": "boolean" } }, "type": "object" }, "prefix": { "description": "An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER.", "type": "string" }, "secretRef": { "description": "SecretEnvSource selects a Secret to populate the environment variables with.\n\nThe contents of the target Secret's Data field will represent the key-value pairs as environment variables.", "properties": { "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", "type": "string" }, "optional": { "description": "Specify whether the Secret must be defined", "type": "boolean" } }, "type": "object" } }, "type": "object" }, "type": "array" }, "image": { "description": "Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images This field is optional to allow higher level config management to default or override container images in workload controllers like Deployments and StatefulSets.", "type": "string" }, "imagePullPolicy": { "description": "Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images", "type": "string" }, "lifecycle": { "description": "Lifecycle describes actions that the management system should take in response to container lifecycle events. For the PostStart and PreStop lifecycle handlers, management of the container blocks until the action is complete, unless the container process fails, in which case the handler is aborted.", "properties": { "postStart": { "description": "Handler defines a specific action that should be taken", "properties": { "exec": { "description": "ExecAction describes a \"run in container\" action.", "properties": { "command": { "description": "Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.", "items": { "type": "string" }, "type": "array" } }, "type": "object" }, "httpGet": { "description": "HTTPGetAction describes an action based on HTTP Get requests.", "properties": { "host": { "description": "Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.", "type": "string" }, "httpHeaders": { "description": "Custom headers to set in the request. HTTP allows repeated headers.", "items": { "description": "HTTPHeader describes a custom header to be used in HTTP probes", "properties": { "name": { "description": "The header field name", "type": "string" }, "value": { "description": "The header field value", "type": "string" } }, "required": [ "name", "value" ], "type": "object" }, "type": "array" }, "path": { "description": "Path to access on the HTTP server.", "type": "string" }, "port": { "anyOf": [ { "type": "string" }, { "type": "integer" } ] }, "scheme": { "description": "Scheme to use for connecting to the host. Defaults to HTTP.", "type": "string" } }, "required": [ "port" ], "type": "object" }, "tcpSocket": { "description": "TCPSocketAction describes an action based on opening a socket", "properties": { "host": { "description": "Optional: Host name to connect to, defaults to the pod IP.", "type": "string" }, "port": { "anyOf": [ { "type": "string" }, { "type": "integer" } ] } }, "required": [ "port" ], "type": "object" } }, "type": "object" }, "preStop": { "description": "Handler defines a specific action that should be taken", "properties": { "exec": { "description": "ExecAction describes a \"run in container\" action.", "properties": { "command": { "description": "Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.", "items": { "type": "string" }, "type": "array" } }, "type": "object" }, "httpGet": { "description": "HTTPGetAction describes an action based on HTTP Get requests.", "properties": { "host": { "description": "Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.", "type": "string" }, "httpHeaders": { "description": "Custom headers to set in the request. HTTP allows repeated headers.", "items": { "description": "HTTPHeader describes a custom header to be used in HTTP probes", "properties": { "name": { "description": "The header field name", "type": "string" }, "value": { "description": "The header field value", "type": "string" } }, "required": [ "name", "value" ], "type": "object" }, "type": "array" }, "path": { "description": "Path to access on the HTTP server.", "type": "string" }, "port": { "anyOf": [ { "type": "string" }, { "type": "integer" } ] }, "scheme": { "description": "Scheme to use for connecting to the host. Defaults to HTTP.", "type": "string" } }, "required": [ "port" ], "type": "object" }, "tcpSocket": { "description": "TCPSocketAction describes an action based on opening a socket", "properties": { "host": { "description": "Optional: Host name to connect to, defaults to the pod IP.", "type": "string" }, "port": { "anyOf": [ { "type": "string" }, { "type": "integer" } ] } }, "required": [ "port" ], "type": "object" } }, "type": "object" } }, "type": "object" }, "livenessProbe": { "description": "Probe describes a health check to be performed against a container to determine whether it is alive or ready to receive traffic.", "properties": { "exec": { "description": "ExecAction describes a \"run in container\" action.", "properties": { "command": { "description": "Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.", "items": { "type": "string" }, "type": "array" } }, "type": "object" }, "failureThreshold": { "description": "Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.", "format": "int32", "type": "integer" }, "httpGet": { "description": "HTTPGetAction describes an action based on HTTP Get requests.", "properties": { "host": { "description": "Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.", "type": "string" }, "httpHeaders": { "description": "Custom headers to set in the request. HTTP allows repeated headers.", "items": { "description": "HTTPHeader describes a custom header to be used in HTTP probes", "properties": { "name": { "description": "The header field name", "type": "string" }, "value": { "description": "The header field value", "type": "string" } }, "required": [ "name", "value" ], "type": "object" }, "type": "array" }, "path": { "description": "Path to access on the HTTP server.", "type": "string" }, "port": { "anyOf": [ { "type": "string" }, { "type": "integer" } ] }, "scheme": { "description": "Scheme to use for connecting to the host. Defaults to HTTP.", "type": "string" } }, "required": [ "port" ], "type": "object" }, "initialDelaySeconds": { "description": "Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes", "format": "int32", "type": "integer" }, "periodSeconds": { "description": "How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.", "format": "int32", "type": "integer" }, "successThreshold": { "description": "Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness. Minimum value is 1.", "format": "int32", "type": "integer" }, "tcpSocket": { "description": "TCPSocketAction describes an action based on opening a socket", "properties": { "host": { "description": "Optional: Host name to connect to, defaults to the pod IP.", "type": "string" }, "port": { "anyOf": [ { "type": "string" }, { "type": "integer" } ] } }, "required": [ "port" ], "type": "object" }, "timeoutSeconds": { "description": "Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes", "format": "int32", "type": "integer" } }, "type": "object" }, "name": { "description": "Name of the container specified as a DNS_LABEL. Each container in a pod must have a unique name (DNS_LABEL). Cannot be updated.", "type": "string" }, "ports": { "description": "List of ports to expose from the container. Exposing a port here gives the system additional information about the network connections a container uses, but is primarily informational. Not specifying a port here DOES NOT prevent that port from being exposed. Any port which is listening on the default \"0.0.0.0\" address inside a container will be accessible from the network. Cannot be updated.", "items": { "description": "ContainerPort represents a network port in a single container.", "properties": { "containerPort": { "description": "Number of port to expose on the pod's IP address. This must be a valid port number, 0 \u003c x \u003c 65536.", "format": "int32", "type": "integer" }, "hostIP": { "description": "What host IP to bind the external port to.", "type": "string" }, "hostPort": { "description": "Number of port to expose on the host. If specified, this must be a valid port number, 0 \u003c x \u003c 65536. If HostNetwork is specified, this must match ContainerPort. Most containers do not need this.", "format": "int32", "type": "integer" }, "name": { "description": "If specified, this must be an IANA_SVC_NAME and unique within the pod. Each named port in a pod must have a unique name. Name for the port that can be referred to by services.", "type": "string" }, "protocol": { "description": "Protocol for port. Must be UDP, TCP, or SCTP. Defaults to \"TCP\".", "type": "string" } }, "required": [ "containerPort" ], "type": "object" }, "type": "array" }, "readinessProbe": { "description": "Probe describes a health check to be performed against a container to determine whether it is alive or ready to receive traffic.", "properties": { "exec": { "description": "ExecAction describes a \"run in container\" action.", "properties": { "command": { "description": "Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.", "items": { "type": "string" }, "type": "array" } }, "type": "object" }, "failureThreshold": { "description": "Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.", "format": "int32", "type": "integer" }, "httpGet": { "description": "HTTPGetAction describes an action based on HTTP Get requests.", "properties": { "host": { "description": "Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.", "type": "string" }, "httpHeaders": { "description": "Custom headers to set in the request. HTTP allows repeated headers.", "items": { "description": "HTTPHeader describes a custom header to be used in HTTP probes", "properties": { "name": { "description": "The header field name", "type": "string" }, "value": { "description": "The header field value", "type": "string" } }, "required": [ "name", "value" ], "type": "object" }, "type": "array" }, "path": { "description": "Path to access on the HTTP server.", "type": "string" }, "port": { "anyOf": [ { "type": "string" }, { "type": "integer" } ] }, "scheme": { "description": "Scheme to use for connecting to the host. Defaults to HTTP.", "type": "string" } }, "required": [ "port" ], "type": "object" }, "initialDelaySeconds": { "description": "Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes", "format": "int32", "type": "integer" }, "periodSeconds": { "description": "How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.", "format": "int32", "type": "integer" }, "successThreshold": { "description": "Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness. Minimum value is 1.", "format": "int32", "type": "integer" }, "tcpSocket": { "description": "TCPSocketAction describes an action based on opening a socket", "properties": { "host": { "description": "Optional: Host name to connect to, defaults to the pod IP.", "type": "string" }, "port": { "anyOf": [ { "type": "string" }, { "type": "integer" } ] } }, "required": [ "port" ], "type": "object" }, "timeoutSeconds": { "description": "Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes", "format": "int32", "type": "integer" } }, "type": "object" }, "resources": { "description": "ResourceRequirements describes the compute resource requirements.", "properties": { "limits": { "description": "Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", "type": "object" }, "requests": { "description": "Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", "type": "object" } }, "type": "object" }, "securityContext": { "description": "SecurityContext holds security configuration that will be applied to a container. Some fields are present in both SecurityContext and PodSecurityContext. When both are set, the values in SecurityContext take precedence.", "properties": { "allowPrivilegeEscalation": { "description": "AllowPrivilegeEscalation controls whether a process can gain more privileges than its parent process. This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged 2) has CAP_SYS_ADMIN", "type": "boolean" }, "capabilities": { "description": "Adds and removes POSIX capabilities from running containers.", "properties": { "add": { "description": "Added capabilities", "items": { "type": "string" }, "type": "array" }, "drop": { "description": "Removed capabilities", "items": { "type": "string" }, "type": "array" } }, "type": "object" }, "privileged": { "description": "Run container in privileged mode. Processes in privileged containers are essentially equivalent to root on the host. Defaults to false.", "type": "boolean" }, "procMount": { "description": "procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled.", "type": "string" }, "readOnlyRootFilesystem": { "description": "Whether this container has a read-only root filesystem. Default is false.", "type": "boolean" }, "runAsGroup": { "description": "The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.", "format": "int64", "type": "integer" }, "runAsNonRoot": { "description": "Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.", "type": "boolean" }, "runAsUser": { "description": "The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.", "format": "int64", "type": "integer" }, "seLinuxOptions": { "description": "SELinuxOptions are the labels to be applied to the container", "properties": { "level": { "description": "Level is SELinux level label that applies to the container.", "type": "string" }, "role": { "description": "Role is a SELinux role label that applies to the container.", "type": "string" }, "type": { "description": "Type is a SELinux type label that applies to the container.", "type": "string" }, "user": { "description": "User is a SELinux user label that applies to the container.", "type": "string" } }, "type": "object" } }, "type": "object" }, "stdin": { "description": "Whether this container should allocate a buffer for stdin in the container runtime. If this is not set, reads from stdin in the container will always result in EOF. Default is false.", "type": "boolean" }, "stdinOnce": { "description": "Whether the container runtime should close the stdin channel after it has been opened by a single attach. When stdin is true the stdin stream will remain open across multiple attach sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the first client attaches to stdin, and then remains open and accepts data until the client disconnects, at which time stdin is closed and remains closed until the container is restarted. If this flag is false, a container processes that reads from stdin will never receive an EOF. Default is false", "type": "boolean" }, "terminationMessagePath": { "description": "Optional: Path at which the file to which the container's termination message will be written is mounted into the container's filesystem. Message written is intended to be brief final status, such as an assertion failure message. Will be truncated by the node if greater than 4096 bytes. The total message length across all containers will be limited to 12kb. Defaults to /dev/termination-log. Cannot be updated.", "type": "string" }, "terminationMessagePolicy": { "description": "Indicate how the termination message should be populated. File will use the contents of terminationMessagePath to populate the container status message on both success and failure. FallbackToLogsOnError will use the last chunk of container log output if the termination message file is empty and the container exited with an error. The log output is limited to 2048 bytes or 80 lines, whichever is smaller. Defaults to File. Cannot be updated.", "type": "string" }, "tty": { "description": "Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. Default is false.", "type": "boolean" }, "volumeDevices": { "description": "volumeDevices is the list of block devices to be used by the container. This is a beta feature.", "items": { "description": "volumeDevice describes a mapping of a raw block device within a container.", "properties": { "devicePath": { "description": "devicePath is the path inside of the container that the device will be mapped to.", "type": "string" }, "name": { "description": "name must match the name of a persistentVolumeClaim in the pod", "type": "string" } }, "required": [ "name", "devicePath" ], "type": "object" }, "type": "array" }, "volumeMounts": { "description": "Pod volumes to mount into the container's filesystem. Cannot be updated.", "items": { "description": "VolumeMount describes a mounting of a Volume within a container.", "properties": { "mountPath": { "description": "Path within the container at which the volume should be mounted. Must not contain ':'.", "type": "string" }, "mountPropagation": { "description": "mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10.", "type": "string" }, "name": { "description": "This must match the Name of a Volume.", "type": "string" }, "readOnly": { "description": "Mounted read-only if true, read-write otherwise (false or unspecified). Defaults to false.", "type": "boolean" }, "subPath": { "description": "Path within the volume from which the container's volume should be mounted. Defaults to \"\" (volume's root).", "type": "string" }, "subPathExpr": { "description": "Expanded path within the volume from which the container's volume should be mounted. Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. Defaults to \"\" (volume's root). SubPathExpr and SubPath are mutually exclusive. This field is alpha in 1.14.", "type": "string" } }, "required": [ "name", "mountPath" ], "type": "object" }, "type": "array" }, "workingDir": { "description": "Container's working directory. If not specified, the container runtime's default will be used, which might be configured in the container image. Cannot be updated.", "type": "string" } }, "required": [ "name" ], "type": "object" }, "type": "array" }, "enableAdminAPI": { "description": "Enable access to prometheus web admin API. Defaults to the value of `false`. WARNING: Enabling the admin APIs enables mutating endpoints, to delete data, shutdown Prometheus, and more. Enabling this should be done with care and the user is advised to add additional authentication authorization via a proxy to ensure only clients authorized to perform these actions can do so. For more information see https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-admin-apis", "type": "boolean" }, "evaluationInterval": { "description": "Interval between consecutive evaluations.", "type": "string" }, "externalLabels": { "description": "The labels to add to any time series or alerts when communicating with external systems (federation, remote storage, Alertmanager).", "type": "object" }, "externalUrl": { "description": "The external URL the Prometheus instances will be available under. This is necessary to generate correct URLs. This is necessary if Prometheus is not served from root of a DNS name.", "type": "string" }, "image": { "description": "Image if specified has precedence over baseImage, tag and sha combinations. Specifying the version is still necessary to ensure the Prometheus Operator knows what version of Prometheus is being configured.", "type": "string" }, "imagePullSecrets": { "description": "An optional list of references to secrets in the same namespace to use for pulling prometheus and alertmanager images from registries see http://kubernetes.io/docs/user-guide/images#specifying-imagepullsecrets-on-a-pod", "items": { "description": "LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace.", "properties": { "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", "type": "string" } }, "type": "object" }, "type": "array" }, "listenLocal": { "description": "ListenLocal makes the Prometheus server listen on loopback, so that it does not bind against the Pod IP.", "type": "boolean" }, "logFormat": { "description": "Log format for Prometheus to be configured with.", "type": "string" }, "logLevel": { "description": "Log level for Prometheus to be configured with.", "type": "string" }, "nodeSelector": { "description": "Define which Nodes the Pods are scheduled on.", "type": "object" }, "paused": { "description": "When a Prometheus deployment is paused, no actions except for deletion will be performed on the underlying objects.", "type": "boolean" }, "podMetadata": { "description": "ObjectMeta is metadata that all persisted resources must have, which includes all objects users must create.", "properties": { "annotations": { "description": "Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations", "type": "object" }, "clusterName": { "description": "The name of the cluster which the object belongs to. This is used to distinguish resources with same name and namespace in different clusters. This field is not set anywhere right now and apiserver is going to ignore it if set in create or update request.", "type": "string" }, "creationTimestamp": { "description": "Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.", "format": "date-time", "type": "string" }, "deletionGracePeriodSeconds": { "description": "Number of seconds allowed for this object to gracefully terminate before it will be removed from the system. Only set when deletionTimestamp is also set. May only be shortened. Read-only.", "format": "int64", "type": "integer" }, "deletionTimestamp": { "description": "Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.", "format": "date-time", "type": "string" }, "finalizers": { "description": "Must be empty before the object is deleted from the registry. Each entry is an identifier for the responsible component that will remove the entry from the list. If the deletionTimestamp of the object is non-nil, entries in this list can only be removed.", "items": { "type": "string" }, "type": "array" }, "generateName": { "description": "GenerateName is an optional prefix, used by the server, to generate a unique name ONLY IF the Name field has not been provided. If this field is used, the name returned to the client will be different than the name passed. This value will also be combined with a unique suffix. The provided value has the same validation rules as the Name field, and may be truncated by the length of the suffix required to make the value unique on the server.\n\nIf this field is specified and the generated name exists, the server will NOT return a 409 - instead, it will either return 201 Created or 500 with Reason ServerTimeout indicating a unique name could not be found in the time allotted, and the client should retry (optionally after the time indicated in the Retry-After header).\n\nApplied only if Name is not specified. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#idempotency", "type": "string" }, "generation": { "description": "A sequence number representing a specific generation of the desired state. Populated by the system. Read-only.", "format": "int64", "type": "integer" }, "initializers": { "description": "Initializers tracks the progress of initialization.", "properties": { "pending": { "description": "Pending is a list of initializers that must execute in order before this object is visible. When the last pending initializer is removed, and no failing result is set, the initializers struct will be set to nil and the object is considered as initialized and visible to all clients.", "items": { "description": "Initializer is information about an initializer that has not yet completed.", "properties": { "name": { "description": "name of the process that is responsible for initializing this object.", "type": "string" } }, "required": [ "name" ], "type": "object" }, "type": "array" }, "result": { "description": "Status is a return value for calls that don't return other objects.", "properties": { "apiVersion": { "description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources", "type": "string" }, "code": { "description": "Suggested HTTP return code for this status, 0 if not set.", "format": "int32", "type": "integer" }, "details": { "description": "StatusDetails is a set of additional properties that MAY be set by the server to provide additional information about a response. The Reason field of a Status object defines what attributes will be set. Clients must ignore fields that do not match the defined type of each attribute, and should assume that any attribute may be empty, invalid, or under defined.", "properties": { "causes": { "description": "The Causes array includes more details associated with the StatusReason failure. Not all StatusReasons may provide detailed causes.", "items": { "description": "StatusCause provides more information about an api.Status failure, including cases when multiple errors are encountered.", "properties": { "field": { "description": "The field of the resource that has caused this error, as named by its JSON serialization. May include dot and postfix notation for nested attributes. Arrays are zero-indexed. Fields may appear more than once in an array of causes due to fields having multiple errors. Optional.\n\nExamples:\n \"name\" - the field \"name\" on the current resource\n \"items[0].name\" - the field \"name\" on the first array entry in \"items\"", "type": "string" }, "message": { "description": "A human-readable description of the cause of the error. This field may be presented as-is to a reader.", "type": "string" }, "reason": { "description": "A machine-readable description of the cause of the error. If this value is empty there is no information available.", "type": "string" } }, "type": "object" }, "type": "array" }, "group": { "description": "The group attribute of the resource associated with the status StatusReason.", "type": "string" }, "kind": { "description": "The kind attribute of the resource associated with the status StatusReason. On some operations may differ from the requested resource Kind. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds", "type": "string" }, "name": { "description": "The name attribute of the resource associated with the status StatusReason (when there is a single name which can be described).", "type": "string" }, "retryAfterSeconds": { "description": "If specified, the time in seconds before the operation should be retried. Some errors may indicate the client must take an alternate action - for those errors this field may indicate how long to wait before taking the alternate action.", "format": "int32", "type": "integer" }, "uid": { "description": "UID of the resource. (when there is a single resource which can be described). More info: http://kubernetes.io/docs/user-guide/identifiers#uids", "type": "string" } }, "type": "object" }, "kind": { "description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds", "type": "string" }, "message": { "description": "A human-readable description of the status of this operation.", "type": "string" }, "metadata": { "description": "ListMeta describes metadata that synthetic resources must have, including lists and various status objects. A resource may have only one of {ObjectMeta, ListMeta}.", "properties": { "continue": { "description": "continue may be set if the user set a limit on the number of items returned, and indicates that the server has more data available. The value is opaque and may be used to issue another request to the endpoint that served this list to retrieve the next set of available objects. Continuing a consistent list may not be possible if the server configuration has changed or more than a few minutes have passed. The resourceVersion field returned when using this continue value will be identical to the value in the first response, unless you have received this token from an error message.", "type": "string" }, "resourceVersion": { "description": "String that identifies the server's internal version of this object that can be used by clients to determine when objects have changed. Value must be treated as opaque by clients and passed unmodified back to the server. Populated by the system. Read-only. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency", "type": "string" }, "selfLink": { "description": "selfLink is a URL representing this object. Populated by the system. Read-only.", "type": "string" } }, "type": "object" }, "reason": { "description": "A machine-readable description of why this operation is in the \"Failure\" status. If this value is empty there is no information available. A Reason clarifies an HTTP status code but does not override it.", "type": "string" }, "status": { "description": "Status of the operation. One of: \"Success\" or \"Failure\". More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status", "type": "string" } }, "type": "object" } }, "required": [ "pending" ], "type": "object" }, "labels": { "description": "Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels", "type": "object" }, "managedFields": { "description": "ManagedFields maps workflow-id and version to the set of fields that are managed by that workflow. This is mostly for internal housekeeping, and users typically shouldn't need to set or understand this field. A workflow can be the user's name, a controller's name, or the name of a specific apply path like \"ci-cd\". The set of fields is always in the version that the workflow used when modifying the object.\n\nThis field is alpha and can be changed or removed without notice.", "items": { "description": "ManagedFieldsEntry is a workflow-id, a FieldSet and the group version of the resource that the fieldset applies to.", "properties": { "apiVersion": { "description": "APIVersion defines the version of this resource that this field set applies to. The format is \"group/version\" just like the top-level APIVersion field. It is necessary to track the version of a field set because it cannot be automatically converted.", "type": "string" }, "fields": { "description": "Fields stores a set of fields in a data structure like a Trie. To understand how this is used, see: https://github.com/kubernetes-sigs/structured-merge-diff", "type": "object" }, "manager": { "description": "Manager is an identifier of the workflow managing these fields.", "type": "string" }, "operation": { "description": "Operation is the type of operation which lead to this ManagedFieldsEntry being created. The only valid values for this field are 'Apply' and 'Update'.", "type": "string" }, "time": { "description": "Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.", "format": "date-time", "type": "string" } }, "type": "object" }, "type": "array" }, "name": { "description": "Name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names", "type": "string" }, "namespace": { "description": "Namespace defines the space within each name must be unique. An empty namespace is equivalent to the \"default\" namespace, but \"default\" is the canonical representation. Not all objects are required to be scoped to a namespace - the value of this field for those objects will be empty.\n\nMust be a DNS_LABEL. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/namespaces", "type": "string" }, "ownerReferences": { "description": "List of objects depended by this object. If ALL objects in the list have been deleted, this object will be garbage collected. If this object is managed by a controller, then an entry in this list will point to this controller, with the controller field set to true. There cannot be more than one managing controller.", "items": { "description": "OwnerReference contains enough information to let you identify an owning object. An owning object must be in the same namespace as the dependent, or be cluster-scoped, so there is no namespace field.", "properties": { "apiVersion": { "description": "API version of the referent.", "type": "string" }, "blockOwnerDeletion": { "description": "If true, AND if the owner has the \"foregroundDeletion\" finalizer, then the owner cannot be deleted from the key-value store until this reference is removed. Defaults to false. To set this field, a user needs \"delete\" permission of the owner, otherwise 422 (Unprocessable Entity) will be returned.", "type": "boolean" }, "controller": { "description": "If true, this reference points to the managing controller.", "type": "boolean" }, "kind": { "description": "Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds", "type": "string" }, "name": { "description": "Name of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#names", "type": "string" }, "uid": { "description": "UID of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#uids", "type": "string" } }, "required": [ "apiVersion", "kind", "name", "uid" ], "type": "object" }, "type": "array" }, "resourceVersion": { "description": "An opaque value that represents the internal version of this object that can be used by clients to determine when objects have changed. May be used for optimistic concurrency, change detection, and the watch operation on a resource or set of resources. Clients must treat these values as opaque and passed unmodified back to the server. They may only be valid for a particular resource or set of resources.\n\nPopulated by the system. Read-only. Value must be treated as opaque by clients and . More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency", "type": "string" }, "selfLink": { "description": "SelfLink is a URL representing this object. Populated by the system. Read-only.", "type": "string" }, "uid": { "description": "UID is the unique in time and space value for this object. It is typically generated by the server on successful creation of a resource and is not allowed to change on PUT operations.\n\nPopulated by the system. Read-only. More info: http://kubernetes.io/docs/user-guide/identifiers#uids", "type": "string" } }, "type": "object" }, "podMonitorNamespaceSelector": { "description": "A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "items": { "type": "string" }, "type": "array" } }, "required": [ "key", "operator" ], "type": "object" }, "type": "array" }, "matchLabels": { "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object" } }, "type": "object" }, "podMonitorSelector": { "description": "A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "items": { "type": "string" }, "type": "array" } }, "required": [ "key", "operator" ], "type": "object" }, "type": "array" }, "matchLabels": { "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object" } }, "type": "object" }, "priorityClassName": { "description": "Priority class assigned to the Pods", "type": "string" }, "prometheusExternalLabelName": { "description": "Name of Prometheus external label used to denote Prometheus instance name. Defaults to the value of `prometheus`. External label will _not_ be added when value is set to empty string (`\"\"`).", "type": "string" }, "query": { "description": "QuerySpec defines the query command line flags when starting Prometheus.", "properties": { "lookbackDelta": { "description": "The delta difference allowed for retrieving metrics during expression evaluations.", "type": "string" }, "maxConcurrency": { "description": "Number of concurrent queries that can be run at once.", "format": "int32", "type": "integer" }, "maxSamples": { "description": "Maximum number of samples a single query can load into memory. Note that queries will fail if they would load more samples than this into memory, so this also limits the number of samples a query can return.", "format": "int32", "type": "integer" }, "timeout": { "description": "Maximum time a query may take before being aborted.", "type": "string" } }, "type": "object" }, "remoteRead": { "description": "If specified, the remote_read spec. This is an experimental feature, it may change in any upcoming release in a breaking way.", "items": { "description": "RemoteReadSpec defines the remote_read configuration for prometheus.", "properties": { "basicAuth": { "description": "BasicAuth allow an endpoint to authenticate over basic authentication More info: https://prometheus.io/docs/operating/configuration/#endpoints", "properties": { "password": { "description": "SecretKeySelector selects a key of a Secret.", "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", "type": "string" }, "optional": { "description": "Specify whether the Secret or it's key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" }, "username": { "description": "SecretKeySelector selects a key of a Secret.", "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", "type": "string" }, "optional": { "description": "Specify whether the Secret or it's key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" } }, "type": "object" }, "bearerToken": { "description": "bearer token for remote read.", "type": "string" }, "bearerTokenFile": { "description": "File to read bearer token for remote read.", "type": "string" }, "proxyUrl": { "description": "Optional ProxyURL", "type": "string" }, "readRecent": { "description": "Whether reads should be made for queries for time ranges that the local storage should have complete data for.", "type": "boolean" }, "remoteTimeout": { "description": "Timeout for requests to the remote read endpoint.", "type": "string" }, "requiredMatchers": { "description": "An optional list of equality matchers which have to be present in a selector to query the remote read endpoint.", "type": "object" }, "tlsConfig": { "description": "TLSConfig specifies TLS configuration parameters.", "properties": { "caFile": { "description": "The CA cert to use for the targets.", "type": "string" }, "certFile": { "description": "The client cert file for the targets.", "type": "string" }, "insecureSkipVerify": { "description": "Disable target certificate validation.", "type": "boolean" }, "keyFile": { "description": "The client key file for the targets.", "type": "string" }, "serverName": { "description": "Used to verify the hostname for the targets.", "type": "string" } }, "type": "object" }, "url": { "description": "The URL of the endpoint to send samples to.", "type": "string" } }, "required": [ "url" ], "type": "object" }, "type": "array" }, "remoteWrite": { "description": "If specified, the remote_write spec. This is an experimental feature, it may change in any upcoming release in a breaking way.", "items": { "description": "RemoteWriteSpec defines the remote_write configuration for prometheus.", "properties": { "basicAuth": { "description": "BasicAuth allow an endpoint to authenticate over basic authentication More info: https://prometheus.io/docs/operating/configuration/#endpoints", "properties": { "password": { "description": "SecretKeySelector selects a key of a Secret.", "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", "type": "string" }, "optional": { "description": "Specify whether the Secret or it's key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" }, "username": { "description": "SecretKeySelector selects a key of a Secret.", "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", "type": "string" }, "optional": { "description": "Specify whether the Secret or it's key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" } }, "type": "object" }, "bearerToken": { "description": "File to read bearer token for remote write.", "type": "string" }, "bearerTokenFile": { "description": "File to read bearer token for remote write.", "type": "string" }, "proxyUrl": { "description": "Optional ProxyURL", "type": "string" }, "queueConfig": { "description": "QueueConfig allows the tuning of remote_write queue_config parameters. This object is referenced in the RemoteWriteSpec object.", "properties": { "batchSendDeadline": { "description": "BatchSendDeadline is the maximum time a sample will wait in buffer.", "type": "string" }, "capacity": { "description": "Capacity is the number of samples to buffer per shard before we start dropping them.", "format": "int32", "type": "integer" }, "maxBackoff": { "description": "MaxBackoff is the maximum retry delay.", "type": "string" }, "maxRetries": { "description": "MaxRetries is the maximum number of times to retry a batch on recoverable errors.", "format": "int32", "type": "integer" }, "maxSamplesPerSend": { "description": "MaxSamplesPerSend is the maximum number of samples per send.", "format": "int32", "type": "integer" }, "maxShards": { "description": "MaxShards is the maximum number of shards, i.e. amount of concurrency.", "format": "int32", "type": "integer" }, "minBackoff": { "description": "MinBackoff is the initial retry delay. Gets doubled for every retry.", "type": "string" }, "minShards": { "description": "MinShards is the minimum number of shards, i.e. amount of concurrency.", "format": "int32", "type": "integer" } }, "type": "object" }, "remoteTimeout": { "description": "Timeout for requests to the remote write endpoint.", "type": "string" }, "tlsConfig": { "description": "TLSConfig specifies TLS configuration parameters.", "properties": { "caFile": { "description": "The CA cert to use for the targets.", "type": "string" }, "certFile": { "description": "The client cert file for the targets.", "type": "string" }, "insecureSkipVerify": { "description": "Disable target certificate validation.", "type": "boolean" }, "keyFile": { "description": "The client key file for the targets.", "type": "string" }, "serverName": { "description": "Used to verify the hostname for the targets.", "type": "string" } }, "type": "object" }, "url": { "description": "The URL of the endpoint to send samples to.", "type": "string" }, "writeRelabelConfigs": { "description": "The list of remote write relabel configurations.", "items": { "description": "RelabelConfig allows dynamic rewriting of the label set, being applied to samples before ingestion. It defines `\u003cmetric_relabel_configs\u003e`-section of Prometheus configuration. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs", "properties": { "action": { "description": "Action to perform based on regex matching. Default is 'replace'", "type": "string" }, "modulus": { "description": "Modulus to take of the hash of the source label values.", "format": "int64", "type": "integer" }, "regex": { "description": "Regular expression against which the extracted value is matched. default is '(.*)'", "type": "string" }, "replacement": { "description": "Replacement value against which a regex replace is performed if the regular expression matches. Regex capture groups are available. Default is '$1'", "type": "string" }, "separator": { "description": "Separator placed between concatenated source label values. default is ';'.", "type": "string" }, "sourceLabels": { "description": "The source labels select values from existing labels. Their content is concatenated using the configured separator and matched against the configured regular expression for the replace, keep, and drop actions.", "items": { "type": "string" }, "type": "array" }, "targetLabel": { "description": "Label to which the resulting value is written in a replace action. It is mandatory for replace actions. Regex capture groups are available.", "type": "string" } }, "type": "object" }, "type": "array" } }, "required": [ "url" ], "type": "object" }, "type": "array" }, "replicaExternalLabelName": { "description": "Name of Prometheus external label used to denote replica name. Defaults to the value of `prometheus_replica`. External label will _not_ be added when value is set to empty string (`\"\"`).", "type": "string" }, "replicas": { "description": "Number of instances to deploy for a Prometheus deployment.", "format": "int32", "type": "integer" }, "resources": { "description": "ResourceRequirements describes the compute resource requirements.", "properties": { "limits": { "description": "Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", "type": "object" }, "requests": { "description": "Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", "type": "object" } }, "type": "object" }, "retention": { "description": "Time duration Prometheus shall retain data for. Default is '24h', and must match the regular expression `[0-9]+(ms|s|m|h|d|w|y)` (milliseconds seconds minutes hours days weeks years).", "type": "string" }, "retentionSize": { "description": "Maximum amount of disk space used by blocks.", "type": "string" }, "routePrefix": { "description": "The route prefix Prometheus registers HTTP handlers for. This is useful, if using ExternalURL and a proxy is rewriting HTTP routes of a request, and the actual ExternalURL is still true, but the server serves requests under a different route prefix. For example for use with `kubectl proxy`.", "type": "string" }, "ruleNamespaceSelector": { "description": "A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "items": { "type": "string" }, "type": "array" } }, "required": [ "key", "operator" ], "type": "object" }, "type": "array" }, "matchLabels": { "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object" } }, "type": "object" }, "ruleSelector": { "description": "A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "items": { "type": "string" }, "type": "array" } }, "required": [ "key", "operator" ], "type": "object" }, "type": "array" }, "matchLabels": { "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object" } }, "type": "object" }, "rules": { "description": "/--rules.*/ command-line arguments", "properties": { "alert": { "description": "/--rules.alert.*/ command-line arguments", "properties": { "forGracePeriod": { "description": "Minimum duration between alert and restored 'for' state. This is maintained only for alerts with configured 'for' time greater than grace period.", "type": "string" }, "forOutageTolerance": { "description": "Max time to tolerate prometheus outage for restoring 'for' state of alert.", "type": "string" }, "resendDelay": { "description": "Minimum amount of time to wait before resending an alert to Alertmanager.", "type": "string" } }, "type": "object" } }, "type": "object" }, "scrapeInterval": { "description": "Interval between consecutive scrapes.", "type": "string" }, "secrets": { "description": "Secrets is a list of Secrets in the same namespace as the Prometheus object, which shall be mounted into the Prometheus Pods. The Secrets are mounted into /etc/prometheus/secrets/\u003csecret-name\u003e.", "items": { "type": "string" }, "type": "array" }, "securityContext": { "description": "PodSecurityContext holds pod-level security attributes and common container settings. Some fields are also present in container.securityContext. Field values of container.securityContext take precedence over field values of PodSecurityContext.", "properties": { "fsGroup": { "description": "A special supplemental group that applies to all containers in a pod. Some volume types allow the Kubelet to change the ownership of that volume to be owned by the pod:\n\n1. The owning GID will be the FSGroup 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) 3. The permission bits are OR'd with rw-rw----\n\nIf unset, the Kubelet will not modify the ownership and permissions of any volume.", "format": "int64", "type": "integer" }, "runAsGroup": { "description": "The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container.", "format": "int64", "type": "integer" }, "runAsNonRoot": { "description": "Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.", "type": "boolean" }, "runAsUser": { "description": "The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container.", "format": "int64", "type": "integer" }, "seLinuxOptions": { "description": "SELinuxOptions are the labels to be applied to the container", "properties": { "level": { "description": "Level is SELinux level label that applies to the container.", "type": "string" }, "role": { "description": "Role is a SELinux role label that applies to the container.", "type": "string" }, "type": { "description": "Type is a SELinux type label that applies to the container.", "type": "string" }, "user": { "description": "User is a SELinux user label that applies to the container.", "type": "string" } }, "type": "object" }, "supplementalGroups": { "description": "A list of groups applied to the first process run in each container, in addition to the container's primary GID. If unspecified, no groups will be added to any container.", "items": { "format": "int64", "type": "integer" }, "type": "array" }, "sysctls": { "description": "Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported sysctls (by the container runtime) might fail to launch.", "items": { "description": "Sysctl defines a kernel parameter to be set", "properties": { "name": { "description": "Name of a property to set", "type": "string" }, "value": { "description": "Value of a property to set", "type": "string" } }, "required": [ "name", "value" ], "type": "object" }, "type": "array" } }, "type": "object" }, "serviceAccountName": { "description": "ServiceAccountName is the name of the ServiceAccount to use to run the Prometheus Pods.", "type": "string" }, "serviceMonitorNamespaceSelector": { "description": "A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "items": { "type": "string" }, "type": "array" } }, "required": [ "key", "operator" ], "type": "object" }, "type": "array" }, "matchLabels": { "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object" } }, "type": "object" }, "serviceMonitorSelector": { "description": "A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "items": { "type": "string" }, "type": "array" } }, "required": [ "key", "operator" ], "type": "object" }, "type": "array" }, "matchLabels": { "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object" } }, "type": "object" }, "sha": { "description": "SHA of Prometheus container image to be deployed. Defaults to the value of `version`. Similar to a tag, but the SHA explicitly deploys an immutable container image. Version and Tag are ignored if SHA is set.", "type": "string" }, "storage": { "description": "StorageSpec defines the configured storage for a group Prometheus servers. If neither `emptyDir` nor `volumeClaimTemplate` is specified, then by default an [EmptyDir](https://kubernetes.io/docs/concepts/storage/volumes/#emptydir) will be used.", "properties": { "emptyDir": { "description": "Represents an empty directory for a pod. Empty directory volumes support ownership management and SELinux relabeling.", "properties": { "medium": { "description": "What type of storage medium should back this directory. The default is \"\" which means to use the node's default medium. Must be an empty string (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir", "type": "string" }, "sizeLimit": {} }, "type": "object" }, "volumeClaimTemplate": { "description": "PersistentVolumeClaim is a user's request for and claim to a persistent volume", "properties": { "apiVersion": { "description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources", "type": "string" }, "kind": { "description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds", "type": "string" }, "metadata": { "description": "ObjectMeta is metadata that all persisted resources must have, which includes all objects users must create.", "properties": { "annotations": { "description": "Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations", "type": "object" }, "clusterName": { "description": "The name of the cluster which the object belongs to. This is used to distinguish resources with same name and namespace in different clusters. This field is not set anywhere right now and apiserver is going to ignore it if set in create or update request.", "type": "string" }, "creationTimestamp": { "description": "Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.", "format": "date-time", "type": "string" }, "deletionGracePeriodSeconds": { "description": "Number of seconds allowed for this object to gracefully terminate before it will be removed from the system. Only set when deletionTimestamp is also set. May only be shortened. Read-only.", "format": "int64", "type": "integer" }, "deletionTimestamp": { "description": "Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.", "format": "date-time", "type": "string" }, "finalizers": { "description": "Must be empty before the object is deleted from the registry. Each entry is an identifier for the responsible component that will remove the entry from the list. If the deletionTimestamp of the object is non-nil, entries in this list can only be removed.", "items": { "type": "string" }, "type": "array" }, "generateName": { "description": "GenerateName is an optional prefix, used by the server, to generate a unique name ONLY IF the Name field has not been provided. If this field is used, the name returned to the client will be different than the name passed. This value will also be combined with a unique suffix. The provided value has the same validation rules as the Name field, and may be truncated by the length of the suffix required to make the value unique on the server.\n\nIf this field is specified and the generated name exists, the server will NOT return a 409 - instead, it will either return 201 Created or 500 with Reason ServerTimeout indicating a unique name could not be found in the time allotted, and the client should retry (optionally after the time indicated in the Retry-After header).\n\nApplied only if Name is not specified. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#idempotency", "type": "string" }, "generation": { "description": "A sequence number representing a specific generation of the desired state. Populated by the system. Read-only.", "format": "int64", "type": "integer" }, "initializers": { "description": "Initializers tracks the progress of initialization.", "properties": { "pending": { "description": "Pending is a list of initializers that must execute in order before this object is visible. When the last pending initializer is removed, and no failing result is set, the initializers struct will be set to nil and the object is considered as initialized and visible to all clients.", "items": { "description": "Initializer is information about an initializer that has not yet completed.", "properties": { "name": { "description": "name of the process that is responsible for initializing this object.", "type": "string" } }, "required": [ "name" ], "type": "object" }, "type": "array" }, "result": { "description": "Status is a return value for calls that don't return other objects.", "properties": { "apiVersion": { "description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources", "type": "string" }, "code": { "description": "Suggested HTTP return code for this status, 0 if not set.", "format": "int32", "type": "integer" }, "details": { "description": "StatusDetails is a set of additional properties that MAY be set by the server to provide additional information about a response. The Reason field of a Status object defines what attributes will be set. Clients must ignore fields that do not match the defined type of each attribute, and should assume that any attribute may be empty, invalid, or under defined.", "properties": { "causes": { "description": "The Causes array includes more details associated with the StatusReason failure. Not all StatusReasons may provide detailed causes.", "items": { "description": "StatusCause provides more information about an api.Status failure, including cases when multiple errors are encountered.", "properties": { "field": { "description": "The field of the resource that has caused this error, as named by its JSON serialization. May include dot and postfix notation for nested attributes. Arrays are zero-indexed. Fields may appear more than once in an array of causes due to fields having multiple errors. Optional.\n\nExamples:\n \"name\" - the field \"name\" on the current resource\n \"items[0].name\" - the field \"name\" on the first array entry in \"items\"", "type": "string" }, "message": { "description": "A human-readable description of the cause of the error. This field may be presented as-is to a reader.", "type": "string" }, "reason": { "description": "A machine-readable description of the cause of the error. If this value is empty there is no information available.", "type": "string" } }, "type": "object" }, "type": "array" }, "group": { "description": "The group attribute of the resource associated with the status StatusReason.", "type": "string" }, "kind": { "description": "The kind attribute of the resource associated with the status StatusReason. On some operations may differ from the requested resource Kind. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds", "type": "string" }, "name": { "description": "The name attribute of the resource associated with the status StatusReason (when there is a single name which can be described).", "type": "string" }, "retryAfterSeconds": { "description": "If specified, the time in seconds before the operation should be retried. Some errors may indicate the client must take an alternate action - for those errors this field may indicate how long to wait before taking the alternate action.", "format": "int32", "type": "integer" }, "uid": { "description": "UID of the resource. (when there is a single resource which can be described). More info: http://kubernetes.io/docs/user-guide/identifiers#uids", "type": "string" } }, "type": "object" }, "kind": { "description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds", "type": "string" }, "message": { "description": "A human-readable description of the status of this operation.", "type": "string" }, "metadata": { "description": "ListMeta describes metadata that synthetic resources must have, including lists and various status objects. A resource may have only one of {ObjectMeta, ListMeta}.", "properties": { "continue": { "description": "continue may be set if the user set a limit on the number of items returned, and indicates that the server has more data available. The value is opaque and may be used to issue another request to the endpoint that served this list to retrieve the next set of available objects. Continuing a consistent list may not be possible if the server configuration has changed or more than a few minutes have passed. The resourceVersion field returned when using this continue value will be identical to the value in the first response, unless you have received this token from an error message.", "type": "string" }, "resourceVersion": { "description": "String that identifies the server's internal version of this object that can be used by clients to determine when objects have changed. Value must be treated as opaque by clients and passed unmodified back to the server. Populated by the system. Read-only. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency", "type": "string" }, "selfLink": { "description": "selfLink is a URL representing this object. Populated by the system. Read-only.", "type": "string" } }, "type": "object" }, "reason": { "description": "A machine-readable description of why this operation is in the \"Failure\" status. If this value is empty there is no information available. A Reason clarifies an HTTP status code but does not override it.", "type": "string" }, "status": { "description": "Status of the operation. One of: \"Success\" or \"Failure\". More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status", "type": "string" } }, "type": "object" } }, "required": [ "pending" ], "type": "object" }, "labels": { "description": "Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels", "type": "object" }, "managedFields": { "description": "ManagedFields maps workflow-id and version to the set of fields that are managed by that workflow. This is mostly for internal housekeeping, and users typically shouldn't need to set or understand this field. A workflow can be the user's name, a controller's name, or the name of a specific apply path like \"ci-cd\". The set of fields is always in the version that the workflow used when modifying the object.\n\nThis field is alpha and can be changed or removed without notice.", "items": { "description": "ManagedFieldsEntry is a workflow-id, a FieldSet and the group version of the resource that the fieldset applies to.", "properties": { "apiVersion": { "description": "APIVersion defines the version of this resource that this field set applies to. The format is \"group/version\" just like the top-level APIVersion field. It is necessary to track the version of a field set because it cannot be automatically converted.", "type": "string" }, "fields": { "description": "Fields stores a set of fields in a data structure like a Trie. To understand how this is used, see: https://github.com/kubernetes-sigs/structured-merge-diff", "type": "object" }, "manager": { "description": "Manager is an identifier of the workflow managing these fields.", "type": "string" }, "operation": { "description": "Operation is the type of operation which lead to this ManagedFieldsEntry being created. The only valid values for this field are 'Apply' and 'Update'.", "type": "string" }, "time": { "description": "Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.", "format": "date-time", "type": "string" } }, "type": "object" }, "type": "array" }, "name": { "description": "Name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names", "type": "string" }, "namespace": { "description": "Namespace defines the space within each name must be unique. An empty namespace is equivalent to the \"default\" namespace, but \"default\" is the canonical representation. Not all objects are required to be scoped to a namespace - the value of this field for those objects will be empty.\n\nMust be a DNS_LABEL. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/namespaces", "type": "string" }, "ownerReferences": { "description": "List of objects depended by this object. If ALL objects in the list have been deleted, this object will be garbage collected. If this object is managed by a controller, then an entry in this list will point to this controller, with the controller field set to true. There cannot be more than one managing controller.", "items": { "description": "OwnerReference contains enough information to let you identify an owning object. An owning object must be in the same namespace as the dependent, or be cluster-scoped, so there is no namespace field.", "properties": { "apiVersion": { "description": "API version of the referent.", "type": "string" }, "blockOwnerDeletion": { "description": "If true, AND if the owner has the \"foregroundDeletion\" finalizer, then the owner cannot be deleted from the key-value store until this reference is removed. Defaults to false. To set this field, a user needs \"delete\" permission of the owner, otherwise 422 (Unprocessable Entity) will be returned.", "type": "boolean" }, "controller": { "description": "If true, this reference points to the managing controller.", "type": "boolean" }, "kind": { "description": "Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds", "type": "string" }, "name": { "description": "Name of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#names", "type": "string" }, "uid": { "description": "UID of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#uids", "type": "string" } }, "required": [ "apiVersion", "kind", "name", "uid" ], "type": "object" }, "type": "array" }, "resourceVersion": { "description": "An opaque value that represents the internal version of this object that can be used by clients to determine when objects have changed. May be used for optimistic concurrency, change detection, and the watch operation on a resource or set of resources. Clients must treat these values as opaque and passed unmodified back to the server. They may only be valid for a particular resource or set of resources.\n\nPopulated by the system. Read-only. Value must be treated as opaque by clients and . More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency", "type": "string" }, "selfLink": { "description": "SelfLink is a URL representing this object. Populated by the system. Read-only.", "type": "string" }, "uid": { "description": "UID is the unique in time and space value for this object. It is typically generated by the server on successful creation of a resource and is not allowed to change on PUT operations.\n\nPopulated by the system. Read-only. More info: http://kubernetes.io/docs/user-guide/identifiers#uids", "type": "string" } }, "type": "object" }, "spec": { "description": "PersistentVolumeClaimSpec describes the common attributes of storage devices and allows a Source for provider-specific attributes", "properties": { "accessModes": { "description": "AccessModes contains the desired access modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1", "items": { "type": "string" }, "type": "array" }, "dataSource": { "description": "TypedLocalObjectReference contains enough information to let you locate the typed referenced object inside the same namespace.", "properties": { "apiGroup": { "description": "APIGroup is the group for the resource being referenced. If APIGroup is not specified, the specified Kind must be in the core API group. For any other third-party types, APIGroup is required.", "type": "string" }, "kind": { "description": "Kind is the type of resource being referenced", "type": "string" }, "name": { "description": "Name is the name of resource being referenced", "type": "string" } }, "required": [ "kind", "name" ], "type": "object" }, "resources": { "description": "ResourceRequirements describes the compute resource requirements.", "properties": { "limits": { "description": "Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", "type": "object" }, "requests": { "description": "Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", "type": "object" } }, "type": "object" }, "selector": { "description": "A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "items": { "type": "string" }, "type": "array" } }, "required": [ "key", "operator" ], "type": "object" }, "type": "array" }, "matchLabels": { "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object" } }, "type": "object" }, "storageClassName": { "description": "Name of the StorageClass required by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1", "type": "string" }, "volumeMode": { "description": "volumeMode defines what type of volume is required by the claim. Value of Filesystem is implied when not included in claim spec. This is a beta feature.", "type": "string" }, "volumeName": { "description": "VolumeName is the binding reference to the PersistentVolume backing this claim.", "type": "string" } }, "type": "object" }, "status": { "description": "PersistentVolumeClaimStatus is the current status of a persistent volume claim.", "properties": { "accessModes": { "description": "AccessModes contains the actual access modes the volume backing the PVC has. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1", "items": { "type": "string" }, "type": "array" }, "capacity": { "description": "Represents the actual resources of the underlying volume.", "type": "object" }, "conditions": { "description": "Current Condition of persistent volume claim. If underlying persistent volume is being resized then the Condition will be set to 'ResizeStarted'.", "items": { "description": "PersistentVolumeClaimCondition contains details about state of pvc", "properties": { "lastProbeTime": { "description": "Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.", "format": "date-time", "type": "string" }, "lastTransitionTime": { "description": "Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.", "format": "date-time", "type": "string" }, "message": { "description": "Human-readable message indicating details about last transition.", "type": "string" }, "reason": { "description": "Unique, this should be a short, machine understandable string that gives the reason for condition's last transition. If it reports \"ResizeStarted\" that means the underlying persistent volume is being resized.", "type": "string" }, "status": { "type": "string" }, "type": { "type": "string" } }, "required": [ "type", "status" ], "type": "object" }, "type": "array" }, "phase": { "description": "Phase represents the current phase of PersistentVolumeClaim.", "type": "string" } }, "type": "object" } }, "type": "object" } }, "type": "object" }, "tag": { "description": "Tag of Prometheus container image to be deployed. Defaults to the value of `version`. Version is ignored if Tag is set.", "type": "string" }, "thanos": { "description": "ThanosSpec defines parameters for a Prometheus server within a Thanos deployment.", "properties": { "baseImage": { "description": "Thanos base image if other than default.", "type": "string" }, "image": { "description": "Image if specified has precedence over baseImage, tag and sha combinations. Specifying the version is still necessary to ensure the Prometheus Operator knows what version of Thanos is being configured.", "type": "string" }, "objectStorageConfig": { "description": "SecretKeySelector selects a key of a Secret.", "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", "type": "string" }, "optional": { "description": "Specify whether the Secret or it's key must be defined", "type": "boolean" } }, "required": [ "key" ], "type": "object" }, "resources": { "description": "ResourceRequirements describes the compute resource requirements.", "properties": { "limits": { "description": "Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", "type": "object" }, "requests": { "description": "Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", "type": "object" } }, "type": "object" }, "sha": { "description": "SHA of Thanos container image to be deployed. Defaults to the value of `version`. Similar to a tag, but the SHA explicitly deploys an immutable container image. Version and Tag are ignored if SHA is set.", "type": "string" }, "tag": { "description": "Tag of Thanos sidecar container image to be deployed. Defaults to the value of `version`. Version is ignored if Tag is set.", "type": "string" }, "version": { "description": "Version describes the version of Thanos to use.", "type": "string" } }, "type": "object" }, "tolerations": { "description": "If specified, the pod's tolerations.", "items": { "description": "The pod this Toleration is attached to tolerates any taint that matches the triple \u003ckey,value,effect\u003e using the matching operator \u003coperator\u003e.", "properties": { "effect": { "description": "Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.", "type": "string" }, "key": { "description": "Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys.", "type": "string" }, "operator": { "description": "Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category.", "type": "string" }, "tolerationSeconds": { "description": "TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system.", "format": "int64", "type": "integer" }, "value": { "description": "Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string.", "type": "string" } }, "type": "object" }, "type": "array" }, "version": { "description": "Version of Prometheus to be deployed.", "type": "string" }, "walCompression": { "description": "Enable compression of the write-ahead log using Snappy.", "type": "boolean" } }, "type": "object" }, "status": { "description": "PrometheusStatus is the most recent observed status of the Prometheus cluster. Read-only. Not included when requesting from the apiserver, only from the Prometheus Operator API itself. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/api-conventions.md#spec-and-status", "properties": { "availableReplicas": { "description": "Total number of available pods (ready for at least minReadySeconds) targeted by this Prometheus deployment.", "format": "int32", "type": "integer" }, "paused": { "description": "Represents whether any actions on the underlying managed objects are being performed. Only delete actions will be performed.", "type": "boolean" }, "replicas": { "description": "Total number of non-terminated pods targeted by this Prometheus deployment (their labels match the selector).", "format": "int32", "type": "integer" }, "unavailableReplicas": { "description": "Total number of unavailable pods targeted by this Prometheus deployment.", "format": "int32", "type": "integer" }, "updatedReplicas": { "description": "Total number of non-terminated pods targeted by this Prometheus deployment that have the desired version spec.", "format": "int32", "type": "integer" } }, "required": [ "paused", "replicas", "updatedReplicas", "availableReplicas", "unavailableReplicas" ], "type": "object" } }, "type": "object" } }, "served": true, "storage": true } ] }, "status": { "acceptedNames": { "kind": "Prometheus", "listKind": "PrometheusList", "plural": "prometheuses", "singular": "prometheus" }, "conditions": [ { "lastTransitionTime": "2020-05-05T16:58:10Z", "message": "no conflicts found", "reason": "NoConflicts", "status": "True", "type": "NamesAccepted" }, { "lastTransitionTime": "2020-05-05T16:58:10Z", "message": "the initial names have been accepted", "reason": "InitialNamesAccepted", "status": "True", "type": "Established" }, { "lastTransitionTime": "2020-05-05T16:58:10Z", "message": "[spec.validation.openAPIV3Schema.properties[spec].properties[alerting].properties[alertmanagers].items.properties[port].anyOf[0].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[alerting].properties[alertmanagers].items.properties[port].anyOf[1].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[alerting].properties[alertmanagers].items.properties[port].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[env].items.properties[valueFrom].properties[resourceFieldRef].properties[divisor].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[postStart].properties[httpGet].properties[port].anyOf[0].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[postStart].properties[httpGet].properties[port].anyOf[1].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[postStart].properties[httpGet].properties[port].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[postStart].properties[tcpSocket].properties[port].anyOf[0].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[postStart].properties[tcpSocket].properties[port].anyOf[1].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[postStart].properties[tcpSocket].properties[port].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[preStop].properties[httpGet].properties[port].anyOf[0].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[preStop].properties[httpGet].properties[port].anyOf[1].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[preStop].properties[httpGet].properties[port].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[preStop].properties[tcpSocket].properties[port].anyOf[0].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[preStop].properties[tcpSocket].properties[port].anyOf[1].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[preStop].properties[tcpSocket].properties[port].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[livenessProbe].properties[httpGet].properties[port].anyOf[0].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[livenessProbe].properties[httpGet].properties[port].anyOf[1].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[livenessProbe].properties[httpGet].properties[port].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[livenessProbe].properties[tcpSocket].properties[port].anyOf[0].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[livenessProbe].properties[tcpSocket].properties[port].anyOf[1].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[livenessProbe].properties[tcpSocket].properties[port].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[readinessProbe].properties[httpGet].properties[port].anyOf[0].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[readinessProbe].properties[httpGet].properties[port].anyOf[1].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[readinessProbe].properties[httpGet].properties[port].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[readinessProbe].properties[tcpSocket].properties[port].anyOf[0].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[readinessProbe].properties[tcpSocket].properties[port].anyOf[1].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[readinessProbe].properties[tcpSocket].properties[port].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[storage].properties[emptyDir].properties[sizeLimit].type: Required value: must not be empty for specified object fields]", "reason": "Violations", "status": "True", "type": "NonStructuralSchema" } ], "storedVersions": [ "v1" ] } } ================================================ FILE: pkg/backup/actions/testdata/v1beta1/alertmanagers.monitoring.coreos.com.json ================================================ { "kind": "CustomResourceDefinition", "apiVersion": "apiextensions.k8s.io/v1beta1", "metadata": { "name": "alertmanagers.monitoring.coreos.com", "selfLink": "/apis/apiextensions.k8s.io/v1beta1/customresourcedefinitions/alertmanagers.monitoring.coreos.com", "uid": "768a7255-d97a-4762-a400-f359fb24f4c8", "resourceVersion": "206434", "generation": 1, "creationTimestamp": "2020-05-05T16:51:39Z", "labels": { "app": "prometheus-operator" }, "annotations": { "helm.sh/hook": "crd-install", "kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"apiextensions.k8s.io/v1beta1\",\"kind\":\"CustomResourceDefinition\",\"metadata\":{\"annotations\":{\"helm.sh/hook\":\"crd-install\"},\"creationTimestamp\":null,\"labels\":{\"app\":\"prometheus-operator\"},\"name\":\"alertmanagers.monitoring.coreos.com\"},\"spec\":{\"group\":\"monitoring.coreos.com\",\"names\":{\"kind\":\"Alertmanager\",\"plural\":\"alertmanagers\"},\"scope\":\"Namespaced\",\"validation\":{\"openAPIV3Schema\":{\"properties\":{\"apiVersion\":{\"description\":\"APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources\",\"type\":\"string\"},\"kind\":{\"description\":\"Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds\",\"type\":\"string\"},\"spec\":{\"description\":\"AlertmanagerSpec is a specification of the desired behavior of the Alertmanager cluster. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/api-conventions.md#spec-and-status\",\"properties\":{\"additionalPeers\":{\"description\":\"AdditionalPeers allows injecting a set of additional Alertmanagers to peer with to form a highly available cluster.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"affinity\":{\"description\":\"Affinity is a group of affinity scheduling rules.\",\"properties\":{\"nodeAffinity\":{\"description\":\"Node affinity is a group of node affinity scheduling rules.\",\"properties\":{\"preferredDuringSchedulingIgnoredDuringExecution\":{\"description\":\"The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding \\\"weight\\\" to the sum if the node matches the corresponding matchExpressions; the node(s) with the highest sum are the most preferred.\",\"items\":{\"description\":\"An empty preferred scheduling term matches all objects with implicit weight 0 (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op).\",\"properties\":{\"preference\":{\"description\":\"A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm.\",\"properties\":{\"matchExpressions\":{\"description\":\"A list of node selector requirements by node's labels.\",\"items\":{\"description\":\"A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"The label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.\",\"type\":\"string\"},\"values\":{\"description\":\"An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"]},\"type\":\"array\"},\"matchFields\":{\"description\":\"A list of node selector requirements by node's fields.\",\"items\":{\"description\":\"A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"The label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.\",\"type\":\"string\"},\"values\":{\"description\":\"An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"]},\"type\":\"array\"}}},\"weight\":{\"description\":\"Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100.\",\"format\":\"int32\",\"type\":\"integer\"}},\"required\":[\"weight\",\"preference\"]},\"type\":\"array\"},\"requiredDuringSchedulingIgnoredDuringExecution\":{\"description\":\"A node selector represents the union of the results of one or more label queries over a set of nodes; that is, it represents the OR of the selectors represented by the node selector terms.\",\"properties\":{\"nodeSelectorTerms\":{\"description\":\"Required. A list of node selector terms. The terms are ORed.\",\"items\":{\"description\":\"A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm.\",\"properties\":{\"matchExpressions\":{\"description\":\"A list of node selector requirements by node's labels.\",\"items\":{\"description\":\"A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"The label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.\",\"type\":\"string\"},\"values\":{\"description\":\"An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"]},\"type\":\"array\"},\"matchFields\":{\"description\":\"A list of node selector requirements by node's fields.\",\"items\":{\"description\":\"A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"The label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.\",\"type\":\"string\"},\"values\":{\"description\":\"An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"]},\"type\":\"array\"}}},\"type\":\"array\"}},\"required\":[\"nodeSelectorTerms\"]}}},\"podAffinity\":{\"description\":\"Pod affinity is a group of inter pod affinity scheduling rules.\",\"properties\":{\"preferredDuringSchedulingIgnoredDuringExecution\":{\"description\":\"The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding \\\"weight\\\" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred.\",\"items\":{\"description\":\"The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s)\",\"properties\":{\"podAffinityTerm\":{\"description\":\"Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key \\u003ctopologyKey\\u003e matches that of any node on which a pod of the set of pods is running\",\"properties\":{\"labelSelector\":{\"description\":\"A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.\",\"properties\":{\"matchExpressions\":{\"description\":\"matchExpressions is a list of label selector requirements. The requirements are ANDed.\",\"items\":{\"description\":\"A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"key is the label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\",\"type\":\"string\"},\"values\":{\"description\":\"values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"]},\"type\":\"array\"},\"matchLabels\":{\"description\":\"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \\\"key\\\", the operator is \\\"In\\\", and the values array contains only \\\"value\\\". The requirements are ANDed.\",\"type\":\"object\"}}},\"namespaces\":{\"description\":\"namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \\\"this pod's namespace\\\"\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"topologyKey\":{\"description\":\"This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.\",\"type\":\"string\"}},\"required\":[\"topologyKey\"]},\"weight\":{\"description\":\"weight associated with matching the corresponding podAffinityTerm, in the range 1-100.\",\"format\":\"int32\",\"type\":\"integer\"}},\"required\":[\"weight\",\"podAffinityTerm\"]},\"type\":\"array\"},\"requiredDuringSchedulingIgnoredDuringExecution\":{\"description\":\"If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied.\",\"items\":{\"description\":\"Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key \\u003ctopologyKey\\u003e matches that of any node on which a pod of the set of pods is running\",\"properties\":{\"labelSelector\":{\"description\":\"A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.\",\"properties\":{\"matchExpressions\":{\"description\":\"matchExpressions is a list of label selector requirements. The requirements are ANDed.\",\"items\":{\"description\":\"A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"key is the label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\",\"type\":\"string\"},\"values\":{\"description\":\"values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"]},\"type\":\"array\"},\"matchLabels\":{\"description\":\"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \\\"key\\\", the operator is \\\"In\\\", and the values array contains only \\\"value\\\". The requirements are ANDed.\",\"type\":\"object\"}}},\"namespaces\":{\"description\":\"namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \\\"this pod's namespace\\\"\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"topologyKey\":{\"description\":\"This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.\",\"type\":\"string\"}},\"required\":[\"topologyKey\"]},\"type\":\"array\"}}},\"podAntiAffinity\":{\"description\":\"Pod anti affinity is a group of inter pod anti affinity scheduling rules.\",\"properties\":{\"preferredDuringSchedulingIgnoredDuringExecution\":{\"description\":\"The scheduler will prefer to schedule pods to nodes that satisfy the anti-affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding \\\"weight\\\" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred.\",\"items\":{\"description\":\"The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s)\",\"properties\":{\"podAffinityTerm\":{\"description\":\"Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key \\u003ctopologyKey\\u003e matches that of any node on which a pod of the set of pods is running\",\"properties\":{\"labelSelector\":{\"description\":\"A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.\",\"properties\":{\"matchExpressions\":{\"description\":\"matchExpressions is a list of label selector requirements. The requirements are ANDed.\",\"items\":{\"description\":\"A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"key is the label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\",\"type\":\"string\"},\"values\":{\"description\":\"values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"]},\"type\":\"array\"},\"matchLabels\":{\"description\":\"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \\\"key\\\", the operator is \\\"In\\\", and the values array contains only \\\"value\\\". The requirements are ANDed.\",\"type\":\"object\"}}},\"namespaces\":{\"description\":\"namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \\\"this pod's namespace\\\"\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"topologyKey\":{\"description\":\"This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.\",\"type\":\"string\"}},\"required\":[\"topologyKey\"]},\"weight\":{\"description\":\"weight associated with matching the corresponding podAffinityTerm, in the range 1-100.\",\"format\":\"int32\",\"type\":\"integer\"}},\"required\":[\"weight\",\"podAffinityTerm\"]},\"type\":\"array\"},\"requiredDuringSchedulingIgnoredDuringExecution\":{\"description\":\"If the anti-affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the anti-affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied.\",\"items\":{\"description\":\"Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key \\u003ctopologyKey\\u003e matches that of any node on which a pod of the set of pods is running\",\"properties\":{\"labelSelector\":{\"description\":\"A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.\",\"properties\":{\"matchExpressions\":{\"description\":\"matchExpressions is a list of label selector requirements. The requirements are ANDed.\",\"items\":{\"description\":\"A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"key is the label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\",\"type\":\"string\"},\"values\":{\"description\":\"values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"]},\"type\":\"array\"},\"matchLabels\":{\"description\":\"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \\\"key\\\", the operator is \\\"In\\\", and the values array contains only \\\"value\\\". The requirements are ANDed.\",\"type\":\"object\"}}},\"namespaces\":{\"description\":\"namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \\\"this pod's namespace\\\"\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"topologyKey\":{\"description\":\"This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.\",\"type\":\"string\"}},\"required\":[\"topologyKey\"]},\"type\":\"array\"}}}}},\"baseImage\":{\"description\":\"Base image that is used to deploy pods, without tag.\",\"type\":\"string\"},\"configMaps\":{\"description\":\"ConfigMaps is a list of ConfigMaps in the same namespace as the Alertmanager object, which shall be mounted into the Alertmanager Pods. The ConfigMaps are mounted into /etc/alertmanager/configmaps/\\u003cconfigmap-name\\u003e.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"configSecret\":{\"description\":\"ConfigSecret is the name of a Kubernetes Secret in the same namespace as the Alertmanager object, which contains configuration for this Alertmanager instance. Defaults to 'alertmanager-' The secret is mounted into /etc/alertmanager/config.\",\"type\":\"string\"},\"containers\":{\"description\":\"Containers allows injecting additional containers. This is meant to allow adding an authentication proxy to an Alertmanager pod.\",\"items\":{\"description\":\"A single application container that you want to run within a pod.\",\"properties\":{\"args\":{\"description\":\"Arguments to the entrypoint. The docker image's CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"command\":{\"description\":\"Entrypoint array. Not executed within a shell. The docker image's ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"env\":{\"description\":\"List of environment variables to set in the container. Cannot be updated.\",\"items\":{\"description\":\"EnvVar represents an environment variable present in a Container.\",\"properties\":{\"name\":{\"description\":\"Name of the environment variable. Must be a C_IDENTIFIER.\",\"type\":\"string\"},\"value\":{\"description\":\"Variable references $(VAR_NAME) are expanded using the previous defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Defaults to \\\"\\\".\",\"type\":\"string\"},\"valueFrom\":{\"description\":\"EnvVarSource represents a source for the value of an EnvVar.\",\"properties\":{\"configMapKeyRef\":{\"description\":\"Selects a key from a ConfigMap.\",\"properties\":{\"key\":{\"description\":\"The key to select.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the ConfigMap or it's key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"]},\"fieldRef\":{\"description\":\"ObjectFieldSelector selects an APIVersioned field of an object.\",\"properties\":{\"apiVersion\":{\"description\":\"Version of the schema the FieldPath is written in terms of, defaults to \\\"v1\\\".\",\"type\":\"string\"},\"fieldPath\":{\"description\":\"Path of the field to select in the specified API version.\",\"type\":\"string\"}},\"required\":[\"fieldPath\"]},\"resourceFieldRef\":{\"description\":\"ResourceFieldSelector represents container resources (cpu, memory) and their output format\",\"properties\":{\"containerName\":{\"description\":\"Container name: required for volumes, optional for env vars\",\"type\":\"string\"},\"divisor\":{},\"resource\":{\"description\":\"Required: resource to select\",\"type\":\"string\"}},\"required\":[\"resource\"]},\"secretKeyRef\":{\"description\":\"SecretKeySelector selects a key of a Secret.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or it's key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"]}}}},\"required\":[\"name\"]},\"type\":\"array\"},\"envFrom\":{\"description\":\"List of sources to populate environment variables in the container. The keys defined within a source must be a C_IDENTIFIER. All invalid keys will be reported as an event when the container is starting. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated.\",\"items\":{\"description\":\"EnvFromSource represents the source of a set of ConfigMaps\",\"properties\":{\"configMapRef\":{\"description\":\"ConfigMapEnvSource selects a ConfigMap to populate the environment variables with.\\nThe contents of the target ConfigMap's Data field will represent the key-value pairs as environment variables.\",\"properties\":{\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the ConfigMap must be defined\",\"type\":\"boolean\"}}},\"prefix\":{\"description\":\"An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER.\",\"type\":\"string\"},\"secretRef\":{\"description\":\"SecretEnvSource selects a Secret to populate the environment variables with.\\nThe contents of the target Secret's Data field will represent the key-value pairs as environment variables.\",\"properties\":{\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret must be defined\",\"type\":\"boolean\"}}}}},\"type\":\"array\"},\"image\":{\"description\":\"Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images This field is optional to allow higher level config management to default or override container images in workload controllers like Deployments and StatefulSets.\",\"type\":\"string\"},\"imagePullPolicy\":{\"description\":\"Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images\",\"type\":\"string\"},\"lifecycle\":{\"description\":\"Lifecycle describes actions that the management system should take in response to container lifecycle events. For the PostStart and PreStop lifecycle handlers, management of the container blocks until the action is complete, unless the container process fails, in which case the handler is aborted.\",\"properties\":{\"postStart\":{\"description\":\"Handler defines a specific action that should be taken\",\"properties\":{\"exec\":{\"description\":\"ExecAction describes a \\\"run in container\\\" action.\",\"properties\":{\"command\":{\"description\":\"Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}}},\"httpGet\":{\"description\":\"HTTPGetAction describes an action based on HTTP Get requests.\",\"properties\":{\"host\":{\"description\":\"Host name to connect to, defaults to the pod IP. You probably want to set \\\"Host\\\" in httpHeaders instead.\",\"type\":\"string\"},\"httpHeaders\":{\"description\":\"Custom headers to set in the request. HTTP allows repeated headers.\",\"items\":{\"description\":\"HTTPHeader describes a custom header to be used in HTTP probes\",\"properties\":{\"name\":{\"description\":\"The header field name\",\"type\":\"string\"},\"value\":{\"description\":\"The header field value\",\"type\":\"string\"}},\"required\":[\"name\",\"value\"]},\"type\":\"array\"},\"path\":{\"description\":\"Path to access on the HTTP server.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"string\"},{\"type\":\"integer\"}]},\"scheme\":{\"description\":\"Scheme to use for connecting to the host. Defaults to HTTP.\",\"type\":\"string\"}},\"required\":[\"port\"]},\"tcpSocket\":{\"description\":\"TCPSocketAction describes an action based on opening a socket\",\"properties\":{\"host\":{\"description\":\"Optional: Host name to connect to, defaults to the pod IP.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"string\"},{\"type\":\"integer\"}]}},\"required\":[\"port\"]}}},\"preStop\":{\"description\":\"Handler defines a specific action that should be taken\",\"properties\":{\"exec\":{\"description\":\"ExecAction describes a \\\"run in container\\\" action.\",\"properties\":{\"command\":{\"description\":\"Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}}},\"httpGet\":{\"description\":\"HTTPGetAction describes an action based on HTTP Get requests.\",\"properties\":{\"host\":{\"description\":\"Host name to connect to, defaults to the pod IP. You probably want to set \\\"Host\\\" in httpHeaders instead.\",\"type\":\"string\"},\"httpHeaders\":{\"description\":\"Custom headers to set in the request. HTTP allows repeated headers.\",\"items\":{\"description\":\"HTTPHeader describes a custom header to be used in HTTP probes\",\"properties\":{\"name\":{\"description\":\"The header field name\",\"type\":\"string\"},\"value\":{\"description\":\"The header field value\",\"type\":\"string\"}},\"required\":[\"name\",\"value\"]},\"type\":\"array\"},\"path\":{\"description\":\"Path to access on the HTTP server.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"string\"},{\"type\":\"integer\"}]},\"scheme\":{\"description\":\"Scheme to use for connecting to the host. Defaults to HTTP.\",\"type\":\"string\"}},\"required\":[\"port\"]},\"tcpSocket\":{\"description\":\"TCPSocketAction describes an action based on opening a socket\",\"properties\":{\"host\":{\"description\":\"Optional: Host name to connect to, defaults to the pod IP.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"string\"},{\"type\":\"integer\"}]}},\"required\":[\"port\"]}}}}},\"livenessProbe\":{\"description\":\"Probe describes a health check to be performed against a container to determine whether it is alive or ready to receive traffic.\",\"properties\":{\"exec\":{\"description\":\"ExecAction describes a \\\"run in container\\\" action.\",\"properties\":{\"command\":{\"description\":\"Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}}},\"failureThreshold\":{\"description\":\"Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.\",\"format\":\"int32\",\"type\":\"integer\"},\"httpGet\":{\"description\":\"HTTPGetAction describes an action based on HTTP Get requests.\",\"properties\":{\"host\":{\"description\":\"Host name to connect to, defaults to the pod IP. You probably want to set \\\"Host\\\" in httpHeaders instead.\",\"type\":\"string\"},\"httpHeaders\":{\"description\":\"Custom headers to set in the request. HTTP allows repeated headers.\",\"items\":{\"description\":\"HTTPHeader describes a custom header to be used in HTTP probes\",\"properties\":{\"name\":{\"description\":\"The header field name\",\"type\":\"string\"},\"value\":{\"description\":\"The header field value\",\"type\":\"string\"}},\"required\":[\"name\",\"value\"]},\"type\":\"array\"},\"path\":{\"description\":\"Path to access on the HTTP server.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"string\"},{\"type\":\"integer\"}]},\"scheme\":{\"description\":\"Scheme to use for connecting to the host. Defaults to HTTP.\",\"type\":\"string\"}},\"required\":[\"port\"]},\"initialDelaySeconds\":{\"description\":\"Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\",\"format\":\"int32\",\"type\":\"integer\"},\"periodSeconds\":{\"description\":\"How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.\",\"format\":\"int32\",\"type\":\"integer\"},\"successThreshold\":{\"description\":\"Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness. Minimum value is 1.\",\"format\":\"int32\",\"type\":\"integer\"},\"tcpSocket\":{\"description\":\"TCPSocketAction describes an action based on opening a socket\",\"properties\":{\"host\":{\"description\":\"Optional: Host name to connect to, defaults to the pod IP.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"string\"},{\"type\":\"integer\"}]}},\"required\":[\"port\"]},\"timeoutSeconds\":{\"description\":\"Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\",\"format\":\"int32\",\"type\":\"integer\"}}},\"name\":{\"description\":\"Name of the container specified as a DNS_LABEL. Each container in a pod must have a unique name (DNS_LABEL). Cannot be updated.\",\"type\":\"string\"},\"ports\":{\"description\":\"List of ports to expose from the container. Exposing a port here gives the system additional information about the network connections a container uses, but is primarily informational. Not specifying a port here DOES NOT prevent that port from being exposed. Any port which is listening on the default \\\"0.0.0.0\\\" address inside a container will be accessible from the network. Cannot be updated.\",\"items\":{\"description\":\"ContainerPort represents a network port in a single container.\",\"properties\":{\"containerPort\":{\"description\":\"Number of port to expose on the pod's IP address. This must be a valid port number, 0 \\u003c x \\u003c 65536.\",\"format\":\"int32\",\"type\":\"integer\"},\"hostIP\":{\"description\":\"What host IP to bind the external port to.\",\"type\":\"string\"},\"hostPort\":{\"description\":\"Number of port to expose on the host. If specified, this must be a valid port number, 0 \\u003c x \\u003c 65536. If HostNetwork is specified, this must match ContainerPort. Most containers do not need this.\",\"format\":\"int32\",\"type\":\"integer\"},\"name\":{\"description\":\"If specified, this must be an IANA_SVC_NAME and unique within the pod. Each named port in a pod must have a unique name. Name for the port that can be referred to by services.\",\"type\":\"string\"},\"protocol\":{\"description\":\"Protocol for port. Must be UDP, TCP, or SCTP. Defaults to \\\"TCP\\\".\",\"type\":\"string\"}},\"required\":[\"containerPort\"]},\"type\":\"array\"},\"readinessProbe\":{\"description\":\"Probe describes a health check to be performed against a container to determine whether it is alive or ready to receive traffic.\",\"properties\":{\"exec\":{\"description\":\"ExecAction describes a \\\"run in container\\\" action.\",\"properties\":{\"command\":{\"description\":\"Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}}},\"failureThreshold\":{\"description\":\"Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.\",\"format\":\"int32\",\"type\":\"integer\"},\"httpGet\":{\"description\":\"HTTPGetAction describes an action based on HTTP Get requests.\",\"properties\":{\"host\":{\"description\":\"Host name to connect to, defaults to the pod IP. You probably want to set \\\"Host\\\" in httpHeaders instead.\",\"type\":\"string\"},\"httpHeaders\":{\"description\":\"Custom headers to set in the request. HTTP allows repeated headers.\",\"items\":{\"description\":\"HTTPHeader describes a custom header to be used in HTTP probes\",\"properties\":{\"name\":{\"description\":\"The header field name\",\"type\":\"string\"},\"value\":{\"description\":\"The header field value\",\"type\":\"string\"}},\"required\":[\"name\",\"value\"]},\"type\":\"array\"},\"path\":{\"description\":\"Path to access on the HTTP server.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"string\"},{\"type\":\"integer\"}]},\"scheme\":{\"description\":\"Scheme to use for connecting to the host. Defaults to HTTP.\",\"type\":\"string\"}},\"required\":[\"port\"]},\"initialDelaySeconds\":{\"description\":\"Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\",\"format\":\"int32\",\"type\":\"integer\"},\"periodSeconds\":{\"description\":\"How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.\",\"format\":\"int32\",\"type\":\"integer\"},\"successThreshold\":{\"description\":\"Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness. Minimum value is 1.\",\"format\":\"int32\",\"type\":\"integer\"},\"tcpSocket\":{\"description\":\"TCPSocketAction describes an action based on opening a socket\",\"properties\":{\"host\":{\"description\":\"Optional: Host name to connect to, defaults to the pod IP.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"string\"},{\"type\":\"integer\"}]}},\"required\":[\"port\"]},\"timeoutSeconds\":{\"description\":\"Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\",\"format\":\"int32\",\"type\":\"integer\"}}},\"resources\":{\"description\":\"ResourceRequirements describes the compute resource requirements.\",\"properties\":{\"limits\":{\"description\":\"Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/\",\"type\":\"object\"},\"requests\":{\"description\":\"Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/\",\"type\":\"object\"}}},\"securityContext\":{\"description\":\"SecurityContext holds security configuration that will be applied to a container. Some fields are present in both SecurityContext and PodSecurityContext. When both are set, the values in SecurityContext take precedence.\",\"properties\":{\"allowPrivilegeEscalation\":{\"description\":\"AllowPrivilegeEscalation controls whether a process can gain more privileges than its parent process. This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged 2) has CAP_SYS_ADMIN\",\"type\":\"boolean\"},\"capabilities\":{\"description\":\"Adds and removes POSIX capabilities from running containers.\",\"properties\":{\"add\":{\"description\":\"Added capabilities\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"drop\":{\"description\":\"Removed capabilities\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}}},\"privileged\":{\"description\":\"Run container in privileged mode. Processes in privileged containers are essentially equivalent to root on the host. Defaults to false.\",\"type\":\"boolean\"},\"procMount\":{\"description\":\"procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled.\",\"type\":\"string\"},\"readOnlyRootFilesystem\":{\"description\":\"Whether this container has a read-only root filesystem. Default is false.\",\"type\":\"boolean\"},\"runAsGroup\":{\"description\":\"The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\",\"format\":\"int64\",\"type\":\"integer\"},\"runAsNonRoot\":{\"description\":\"Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\",\"type\":\"boolean\"},\"runAsUser\":{\"description\":\"The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\",\"format\":\"int64\",\"type\":\"integer\"},\"seLinuxOptions\":{\"description\":\"SELinuxOptions are the labels to be applied to the container\",\"properties\":{\"level\":{\"description\":\"Level is SELinux level label that applies to the container.\",\"type\":\"string\"},\"role\":{\"description\":\"Role is a SELinux role label that applies to the container.\",\"type\":\"string\"},\"type\":{\"description\":\"Type is a SELinux type label that applies to the container.\",\"type\":\"string\"},\"user\":{\"description\":\"User is a SELinux user label that applies to the container.\",\"type\":\"string\"}}}}},\"stdin\":{\"description\":\"Whether this container should allocate a buffer for stdin in the container runtime. If this is not set, reads from stdin in the container will always result in EOF. Default is false.\",\"type\":\"boolean\"},\"stdinOnce\":{\"description\":\"Whether the container runtime should close the stdin channel after it has been opened by a single attach. When stdin is true the stdin stream will remain open across multiple attach sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the first client attaches to stdin, and then remains open and accepts data until the client disconnects, at which time stdin is closed and remains closed until the container is restarted. If this flag is false, a container processes that reads from stdin will never receive an EOF. Default is false\",\"type\":\"boolean\"},\"terminationMessagePath\":{\"description\":\"Optional: Path at which the file to which the container's termination message will be written is mounted into the container's filesystem. Message written is intended to be brief final status, such as an assertion failure message. Will be truncated by the node if greater than 4096 bytes. The total message length across all containers will be limited to 12kb. Defaults to /dev/termination-log. Cannot be updated.\",\"type\":\"string\"},\"terminationMessagePolicy\":{\"description\":\"Indicate how the termination message should be populated. File will use the contents of terminationMessagePath to populate the container status message on both success and failure. FallbackToLogsOnError will use the last chunk of container log output if the termination message file is empty and the container exited with an error. The log output is limited to 2048 bytes or 80 lines, whichever is smaller. Defaults to File. Cannot be updated.\",\"type\":\"string\"},\"tty\":{\"description\":\"Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. Default is false.\",\"type\":\"boolean\"},\"volumeDevices\":{\"description\":\"volumeDevices is the list of block devices to be used by the container. This is a beta feature.\",\"items\":{\"description\":\"volumeDevice describes a mapping of a raw block device within a container.\",\"properties\":{\"devicePath\":{\"description\":\"devicePath is the path inside of the container that the device will be mapped to.\",\"type\":\"string\"},\"name\":{\"description\":\"name must match the name of a persistentVolumeClaim in the pod\",\"type\":\"string\"}},\"required\":[\"name\",\"devicePath\"]},\"type\":\"array\"},\"volumeMounts\":{\"description\":\"Pod volumes to mount into the container's filesystem. Cannot be updated.\",\"items\":{\"description\":\"VolumeMount describes a mounting of a Volume within a container.\",\"properties\":{\"mountPath\":{\"description\":\"Path within the container at which the volume should be mounted. Must not contain ':'.\",\"type\":\"string\"},\"mountPropagation\":{\"description\":\"mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10.\",\"type\":\"string\"},\"name\":{\"description\":\"This must match the Name of a Volume.\",\"type\":\"string\"},\"readOnly\":{\"description\":\"Mounted read-only if true, read-write otherwise (false or unspecified). Defaults to false.\",\"type\":\"boolean\"},\"subPath\":{\"description\":\"Path within the volume from which the container's volume should be mounted. Defaults to \\\"\\\" (volume's root).\",\"type\":\"string\"}},\"required\":[\"name\",\"mountPath\"]},\"type\":\"array\"},\"workingDir\":{\"description\":\"Container's working directory. If not specified, the container runtime's default will be used, which might be configured in the container image. Cannot be updated.\",\"type\":\"string\"}},\"required\":[\"name\"]},\"type\":\"array\"},\"externalUrl\":{\"description\":\"The external URL the Alertmanager instances will be available under. This is necessary to generate correct URLs. This is necessary if Alertmanager is not served from root of a DNS name.\",\"type\":\"string\"},\"image\":{\"description\":\"Image if specified has precedence over baseImage, tag and sha combinations. Specifying the version is still necessary to ensure the Prometheus Operator knows what version of Alertmanager is being configured.\",\"type\":\"string\"},\"imagePullSecrets\":{\"description\":\"An optional list of references to secrets in the same namespace to use for pulling prometheus and alertmanager images from registries see http://kubernetes.io/docs/user-guide/images#specifying-imagepullsecrets-on-a-pod\",\"items\":{\"description\":\"LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace.\",\"properties\":{\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\",\"type\":\"string\"}}},\"type\":\"array\"},\"listenLocal\":{\"description\":\"ListenLocal makes the Alertmanager server listen on loopback, so that it does not bind against the Pod IP. Note this is only for the Alertmanager UI, not the gossip communication.\",\"type\":\"boolean\"},\"logLevel\":{\"description\":\"Log level for Alertmanager to be configured with.\",\"type\":\"string\"},\"nodeSelector\":{\"description\":\"Define which Nodes the Pods are scheduled on.\",\"type\":\"object\"},\"paused\":{\"description\":\"If set to true all actions on the underlying managed objects are not goint to be performed, except for delete actions.\",\"type\":\"boolean\"},\"podMetadata\":{\"description\":\"ObjectMeta is metadata that all persisted resources must have, which includes all objects users must create.\",\"properties\":{\"annotations\":{\"description\":\"Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations\",\"type\":\"object\"},\"clusterName\":{\"description\":\"The name of the cluster which the object belongs to. This is used to distinguish resources with same name and namespace in different clusters. This field is not set anywhere right now and apiserver is going to ignore it if set in create or update request.\",\"type\":\"string\"},\"creationTimestamp\":{\"description\":\"Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.\",\"format\":\"date-time\",\"type\":\"string\"},\"deletionGracePeriodSeconds\":{\"description\":\"Number of seconds allowed for this object to gracefully terminate before it will be removed from the system. Only set when deletionTimestamp is also set. May only be shortened. Read-only.\",\"format\":\"int64\",\"type\":\"integer\"},\"deletionTimestamp\":{\"description\":\"Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.\",\"format\":\"date-time\",\"type\":\"string\"},\"finalizers\":{\"description\":\"Must be empty before the object is deleted from the registry. Each entry is an identifier for the responsible component that will remove the entry from the list. If the deletionTimestamp of the object is non-nil, entries in this list can only be removed.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"generateName\":{\"description\":\"GenerateName is an optional prefix, used by the server, to generate a unique name ONLY IF the Name field has not been provided. If this field is used, the name returned to the client will be different than the name passed. This value will also be combined with a unique suffix. The provided value has the same validation rules as the Name field, and may be truncated by the length of the suffix required to make the value unique on the server.\\nIf this field is specified and the generated name exists, the server will NOT return a 409 - instead, it will either return 201 Created or 500 with Reason ServerTimeout indicating a unique name could not be found in the time allotted, and the client should retry (optionally after the time indicated in the Retry-After header).\\nApplied only if Name is not specified. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#idempotency\",\"type\":\"string\"},\"generation\":{\"description\":\"A sequence number representing a specific generation of the desired state. Populated by the system. Read-only.\",\"format\":\"int64\",\"type\":\"integer\"},\"initializers\":{\"description\":\"Initializers tracks the progress of initialization.\",\"properties\":{\"pending\":{\"description\":\"Pending is a list of initializers that must execute in order before this object is visible. When the last pending initializer is removed, and no failing result is set, the initializers struct will be set to nil and the object is considered as initialized and visible to all clients.\",\"items\":{\"description\":\"Initializer is information about an initializer that has not yet completed.\",\"properties\":{\"name\":{\"description\":\"name of the process that is responsible for initializing this object.\",\"type\":\"string\"}},\"required\":[\"name\"]},\"type\":\"array\"},\"result\":{\"description\":\"Status is a return value for calls that don't return other objects.\",\"properties\":{\"apiVersion\":{\"description\":\"APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources\",\"type\":\"string\"},\"code\":{\"description\":\"Suggested HTTP return code for this status, 0 if not set.\",\"format\":\"int32\",\"type\":\"integer\"},\"details\":{\"description\":\"StatusDetails is a set of additional properties that MAY be set by the server to provide additional information about a response. The Reason field of a Status object defines what attributes will be set. Clients must ignore fields that do not match the defined type of each attribute, and should assume that any attribute may be empty, invalid, or under defined.\",\"properties\":{\"causes\":{\"description\":\"The Causes array includes more details associated with the StatusReason failure. Not all StatusReasons may provide detailed causes.\",\"items\":{\"description\":\"StatusCause provides more information about an api.Status failure, including cases when multiple errors are encountered.\",\"properties\":{\"field\":{\"description\":\"The field of the resource that has caused this error, as named by its JSON serialization. May include dot and postfix notation for nested attributes. Arrays are zero-indexed. Fields may appear more than once in an array of causes due to fields having multiple errors. Optional.\\nExamples:\\n \\\"name\\\" - the field \\\"name\\\" on the current resource\\n \\\"items[0].name\\\" - the field \\\"name\\\" on the first array entry in \\\"items\\\"\",\"type\":\"string\"},\"message\":{\"description\":\"A human-readable description of the cause of the error. This field may be presented as-is to a reader.\",\"type\":\"string\"},\"reason\":{\"description\":\"A machine-readable description of the cause of the error. If this value is empty there is no information available.\",\"type\":\"string\"}}},\"type\":\"array\"},\"group\":{\"description\":\"The group attribute of the resource associated with the status StatusReason.\",\"type\":\"string\"},\"kind\":{\"description\":\"The kind attribute of the resource associated with the status StatusReason. On some operations may differ from the requested resource Kind. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds\",\"type\":\"string\"},\"name\":{\"description\":\"The name attribute of the resource associated with the status StatusReason (when there is a single name which can be described).\",\"type\":\"string\"},\"retryAfterSeconds\":{\"description\":\"If specified, the time in seconds before the operation should be retried. Some errors may indicate the client must take an alternate action - for those errors this field may indicate how long to wait before taking the alternate action.\",\"format\":\"int32\",\"type\":\"integer\"},\"uid\":{\"description\":\"UID of the resource. (when there is a single resource which can be described). More info: http://kubernetes.io/docs/user-guide/identifiers#uids\",\"type\":\"string\"}}},\"kind\":{\"description\":\"Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds\",\"type\":\"string\"},\"message\":{\"description\":\"A human-readable description of the status of this operation.\",\"type\":\"string\"},\"metadata\":{\"description\":\"ListMeta describes metadata that synthetic resources must have, including lists and various status objects. A resource may have only one of {ObjectMeta, ListMeta}.\",\"properties\":{\"continue\":{\"description\":\"continue may be set if the user set a limit on the number of items returned, and indicates that the server has more data available. The value is opaque and may be used to issue another request to the endpoint that served this list to retrieve the next set of available objects. Continuing a consistent list may not be possible if the server configuration has changed or more than a few minutes have passed. The resourceVersion field returned when using this continue value will be identical to the value in the first response, unless you have received this token from an error message.\",\"type\":\"string\"},\"resourceVersion\":{\"description\":\"String that identifies the server's internal version of this object that can be used by clients to determine when objects have changed. Value must be treated as opaque by clients and passed unmodified back to the server. Populated by the system. Read-only. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency\",\"type\":\"string\"},\"selfLink\":{\"description\":\"selfLink is a URL representing this object. Populated by the system. Read-only.\",\"type\":\"string\"}}},\"reason\":{\"description\":\"A machine-readable description of why this operation is in the \\\"Failure\\\" status. If this value is empty there is no information available. A Reason clarifies an HTTP status code but does not override it.\",\"type\":\"string\"},\"status\":{\"description\":\"Status of the operation. One of: \\\"Success\\\" or \\\"Failure\\\". More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status\",\"type\":\"string\"}}}},\"required\":[\"pending\"]},\"labels\":{\"description\":\"Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels\",\"type\":\"object\"},\"name\":{\"description\":\"Name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names\",\"type\":\"string\"},\"namespace\":{\"description\":\"Namespace defines the space within each name must be unique. An empty namespace is equivalent to the \\\"default\\\" namespace, but \\\"default\\\" is the canonical representation. Not all objects are required to be scoped to a namespace - the value of this field for those objects will be empty.\\nMust be a DNS_LABEL. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/namespaces\",\"type\":\"string\"},\"ownerReferences\":{\"description\":\"List of objects depended by this object. If ALL objects in the list have been deleted, this object will be garbage collected. If this object is managed by a controller, then an entry in this list will point to this controller, with the controller field set to true. There cannot be more than one managing controller.\",\"items\":{\"description\":\"OwnerReference contains enough information to let you identify an owning object. An owning object must be in the same namespace as the dependent, or be cluster-scoped, so there is no namespace field.\",\"properties\":{\"apiVersion\":{\"description\":\"API version of the referent.\",\"type\":\"string\"},\"blockOwnerDeletion\":{\"description\":\"If true, AND if the owner has the \\\"foregroundDeletion\\\" finalizer, then the owner cannot be deleted from the key-value store until this reference is removed. Defaults to false. To set this field, a user needs \\\"delete\\\" permission of the owner, otherwise 422 (Unprocessable Entity) will be returned.\",\"type\":\"boolean\"},\"controller\":{\"description\":\"If true, this reference points to the managing controller.\",\"type\":\"boolean\"},\"kind\":{\"description\":\"Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#names\",\"type\":\"string\"},\"uid\":{\"description\":\"UID of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#uids\",\"type\":\"string\"}},\"required\":[\"apiVersion\",\"kind\",\"name\",\"uid\"]},\"type\":\"array\"},\"resourceVersion\":{\"description\":\"An opaque value that represents the internal version of this object that can be used by clients to determine when objects have changed. May be used for optimistic concurrency, change detection, and the watch operation on a resource or set of resources. Clients must treat these values as opaque and passed unmodified back to the server. They may only be valid for a particular resource or set of resources.\\nPopulated by the system. Read-only. Value must be treated as opaque by clients and . More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency\",\"type\":\"string\"},\"selfLink\":{\"description\":\"SelfLink is a URL representing this object. Populated by the system. Read-only.\",\"type\":\"string\"},\"uid\":{\"description\":\"UID is the unique in time and space value for this object. It is typically generated by the server on successful creation of a resource and is not allowed to change on PUT operations.\\nPopulated by the system. Read-only. More info: http://kubernetes.io/docs/user-guide/identifiers#uids\",\"type\":\"string\"}}},\"priorityClassName\":{\"description\":\"Priority class assigned to the Pods\",\"type\":\"string\"},\"replicas\":{\"description\":\"Size is the expected size of the alertmanager cluster. The controller will eventually make the size of the running cluster equal to the expected size.\",\"format\":\"int32\",\"type\":\"integer\"},\"resources\":{\"description\":\"ResourceRequirements describes the compute resource requirements.\",\"properties\":{\"limits\":{\"description\":\"Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/\",\"type\":\"object\"},\"requests\":{\"description\":\"Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/\",\"type\":\"object\"}}},\"retention\":{\"description\":\"Time duration Alertmanager shall retain data for. Default is '120h', and must match the regular expression `[0-9]+(ms|s|m|h)` (milliseconds seconds minutes hours).\",\"type\":\"string\"},\"routePrefix\":{\"description\":\"The route prefix Alertmanager registers HTTP handlers for. This is useful, if using ExternalURL and a proxy is rewriting HTTP routes of a request, and the actual ExternalURL is still true, but the server serves requests under a different route prefix. For example for use with `kubectl proxy`.\",\"type\":\"string\"},\"secrets\":{\"description\":\"Secrets is a list of Secrets in the same namespace as the Alertmanager object, which shall be mounted into the Alertmanager Pods. The Secrets are mounted into /etc/alertmanager/secrets/\\u003csecret-name\\u003e.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"securityContext\":{\"description\":\"PodSecurityContext holds pod-level security attributes and common container settings. Some fields are also present in container.securityContext. Field values of container.securityContext take precedence over field values of PodSecurityContext.\",\"properties\":{\"fsGroup\":{\"description\":\"A special supplemental group that applies to all containers in a pod. Some volume types allow the Kubelet to change the ownership of that volume to be owned by the pod:\\n1. The owning GID will be the FSGroup 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) 3. The permission bits are OR'd with rw-rw----\\nIf unset, the Kubelet will not modify the ownership and permissions of any volume.\",\"format\":\"int64\",\"type\":\"integer\"},\"runAsGroup\":{\"description\":\"The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container.\",\"format\":\"int64\",\"type\":\"integer\"},\"runAsNonRoot\":{\"description\":\"Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\",\"type\":\"boolean\"},\"runAsUser\":{\"description\":\"The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container.\",\"format\":\"int64\",\"type\":\"integer\"},\"seLinuxOptions\":{\"description\":\"SELinuxOptions are the labels to be applied to the container\",\"properties\":{\"level\":{\"description\":\"Level is SELinux level label that applies to the container.\",\"type\":\"string\"},\"role\":{\"description\":\"Role is a SELinux role label that applies to the container.\",\"type\":\"string\"},\"type\":{\"description\":\"Type is a SELinux type label that applies to the container.\",\"type\":\"string\"},\"user\":{\"description\":\"User is a SELinux user label that applies to the container.\",\"type\":\"string\"}}},\"supplementalGroups\":{\"description\":\"A list of groups applied to the first process run in each container, in addition to the container's primary GID. If unspecified, no groups will be added to any container.\",\"items\":{\"format\":\"int64\",\"type\":\"integer\"},\"type\":\"array\"},\"sysctls\":{\"description\":\"Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported sysctls (by the container runtime) might fail to launch.\",\"items\":{\"description\":\"Sysctl defines a kernel parameter to be set\",\"properties\":{\"name\":{\"description\":\"Name of a property to set\",\"type\":\"string\"},\"value\":{\"description\":\"Value of a property to set\",\"type\":\"string\"}},\"required\":[\"name\",\"value\"]},\"type\":\"array\"}}},\"serviceAccountName\":{\"description\":\"ServiceAccountName is the name of the ServiceAccount to use to run the Prometheus Pods.\",\"type\":\"string\"},\"sha\":{\"description\":\"SHA of Alertmanager container image to be deployed. Defaults to the value of `version`. Similar to a tag, but the SHA explicitly deploys an immutable container image. Version and Tag are ignored if SHA is set.\",\"type\":\"string\"},\"storage\":{\"description\":\"StorageSpec defines the configured storage for a group Prometheus servers. If neither `emptyDir` nor `volumeClaimTemplate` is specified, then by default an [EmptyDir](https://kubernetes.io/docs/concepts/storage/volumes/#emptydir) will be used.\",\"properties\":{\"emptyDir\":{\"description\":\"Represents an empty directory for a pod. Empty directory volumes support ownership management and SELinux relabeling.\",\"properties\":{\"medium\":{\"description\":\"What type of storage medium should back this directory. The default is \\\"\\\" which means to use the node's default medium. Must be an empty string (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir\",\"type\":\"string\"},\"sizeLimit\":{}}},\"volumeClaimTemplate\":{\"description\":\"PersistentVolumeClaim is a user's request for and claim to a persistent volume\",\"properties\":{\"apiVersion\":{\"description\":\"APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources\",\"type\":\"string\"},\"kind\":{\"description\":\"Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds\",\"type\":\"string\"},\"metadata\":{\"description\":\"ObjectMeta is metadata that all persisted resources must have, which includes all objects users must create.\",\"properties\":{\"annotations\":{\"description\":\"Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations\",\"type\":\"object\"},\"clusterName\":{\"description\":\"The name of the cluster which the object belongs to. This is used to distinguish resources with same name and namespace in different clusters. This field is not set anywhere right now and apiserver is going to ignore it if set in create or update request.\",\"type\":\"string\"},\"creationTimestamp\":{\"description\":\"Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.\",\"format\":\"date-time\",\"type\":\"string\"},\"deletionGracePeriodSeconds\":{\"description\":\"Number of seconds allowed for this object to gracefully terminate before it will be removed from the system. Only set when deletionTimestamp is also set. May only be shortened. Read-only.\",\"format\":\"int64\",\"type\":\"integer\"},\"deletionTimestamp\":{\"description\":\"Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.\",\"format\":\"date-time\",\"type\":\"string\"},\"finalizers\":{\"description\":\"Must be empty before the object is deleted from the registry. Each entry is an identifier for the responsible component that will remove the entry from the list. If the deletionTimestamp of the object is non-nil, entries in this list can only be removed.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"generateName\":{\"description\":\"GenerateName is an optional prefix, used by the server, to generate a unique name ONLY IF the Name field has not been provided. If this field is used, the name returned to the client will be different than the name passed. This value will also be combined with a unique suffix. The provided value has the same validation rules as the Name field, and may be truncated by the length of the suffix required to make the value unique on the server.\\nIf this field is specified and the generated name exists, the server will NOT return a 409 - instead, it will either return 201 Created or 500 with Reason ServerTimeout indicating a unique name could not be found in the time allotted, and the client should retry (optionally after the time indicated in the Retry-After header).\\nApplied only if Name is not specified. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#idempotency\",\"type\":\"string\"},\"generation\":{\"description\":\"A sequence number representing a specific generation of the desired state. Populated by the system. Read-only.\",\"format\":\"int64\",\"type\":\"integer\"},\"initializers\":{\"description\":\"Initializers tracks the progress of initialization.\",\"properties\":{\"pending\":{\"description\":\"Pending is a list of initializers that must execute in order before this object is visible. When the last pending initializer is removed, and no failing result is set, the initializers struct will be set to nil and the object is considered as initialized and visible to all clients.\",\"items\":{\"description\":\"Initializer is information about an initializer that has not yet completed.\",\"properties\":{\"name\":{\"description\":\"name of the process that is responsible for initializing this object.\",\"type\":\"string\"}},\"required\":[\"name\"]},\"type\":\"array\"},\"result\":{\"description\":\"Status is a return value for calls that don't return other objects.\",\"properties\":{\"apiVersion\":{\"description\":\"APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources\",\"type\":\"string\"},\"code\":{\"description\":\"Suggested HTTP return code for this status, 0 if not set.\",\"format\":\"int32\",\"type\":\"integer\"},\"details\":{\"description\":\"StatusDetails is a set of additional properties that MAY be set by the server to provide additional information about a response. The Reason field of a Status object defines what attributes will be set. Clients must ignore fields that do not match the defined type of each attribute, and should assume that any attribute may be empty, invalid, or under defined.\",\"properties\":{\"causes\":{\"description\":\"The Causes array includes more details associated with the StatusReason failure. Not all StatusReasons may provide detailed causes.\",\"items\":{\"description\":\"StatusCause provides more information about an api.Status failure, including cases when multiple errors are encountered.\",\"properties\":{\"field\":{\"description\":\"The field of the resource that has caused this error, as named by its JSON serialization. May include dot and postfix notation for nested attributes. Arrays are zero-indexed. Fields may appear more than once in an array of causes due to fields having multiple errors. Optional.\\nExamples:\\n \\\"name\\\" - the field \\\"name\\\" on the current resource\\n \\\"items[0].name\\\" - the field \\\"name\\\" on the first array entry in \\\"items\\\"\",\"type\":\"string\"},\"message\":{\"description\":\"A human-readable description of the cause of the error. This field may be presented as-is to a reader.\",\"type\":\"string\"},\"reason\":{\"description\":\"A machine-readable description of the cause of the error. If this value is empty there is no information available.\",\"type\":\"string\"}}},\"type\":\"array\"},\"group\":{\"description\":\"The group attribute of the resource associated with the status StatusReason.\",\"type\":\"string\"},\"kind\":{\"description\":\"The kind attribute of the resource associated with the status StatusReason. On some operations may differ from the requested resource Kind. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds\",\"type\":\"string\"},\"name\":{\"description\":\"The name attribute of the resource associated with the status StatusReason (when there is a single name which can be described).\",\"type\":\"string\"},\"retryAfterSeconds\":{\"description\":\"If specified, the time in seconds before the operation should be retried. Some errors may indicate the client must take an alternate action - for those errors this field may indicate how long to wait before taking the alternate action.\",\"format\":\"int32\",\"type\":\"integer\"},\"uid\":{\"description\":\"UID of the resource. (when there is a single resource which can be described). More info: http://kubernetes.io/docs/user-guide/identifiers#uids\",\"type\":\"string\"}}},\"kind\":{\"description\":\"Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds\",\"type\":\"string\"},\"message\":{\"description\":\"A human-readable description of the status of this operation.\",\"type\":\"string\"},\"metadata\":{\"description\":\"ListMeta describes metadata that synthetic resources must have, including lists and various status objects. A resource may have only one of {ObjectMeta, ListMeta}.\",\"properties\":{\"continue\":{\"description\":\"continue may be set if the user set a limit on the number of items returned, and indicates that the server has more data available. The value is opaque and may be used to issue another request to the endpoint that served this list to retrieve the next set of available objects. Continuing a consistent list may not be possible if the server configuration has changed or more than a few minutes have passed. The resourceVersion field returned when using this continue value will be identical to the value in the first response, unless you have received this token from an error message.\",\"type\":\"string\"},\"resourceVersion\":{\"description\":\"String that identifies the server's internal version of this object that can be used by clients to determine when objects have changed. Value must be treated as opaque by clients and passed unmodified back to the server. Populated by the system. Read-only. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency\",\"type\":\"string\"},\"selfLink\":{\"description\":\"selfLink is a URL representing this object. Populated by the system. Read-only.\",\"type\":\"string\"}}},\"reason\":{\"description\":\"A machine-readable description of why this operation is in the \\\"Failure\\\" status. If this value is empty there is no information available. A Reason clarifies an HTTP status code but does not override it.\",\"type\":\"string\"},\"status\":{\"description\":\"Status of the operation. One of: \\\"Success\\\" or \\\"Failure\\\". More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status\",\"type\":\"string\"}}}},\"required\":[\"pending\"]},\"labels\":{\"description\":\"Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels\",\"type\":\"object\"},\"name\":{\"description\":\"Name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names\",\"type\":\"string\"},\"namespace\":{\"description\":\"Namespace defines the space within each name must be unique. An empty namespace is equivalent to the \\\"default\\\" namespace, but \\\"default\\\" is the canonical representation. Not all objects are required to be scoped to a namespace - the value of this field for those objects will be empty.\\nMust be a DNS_LABEL. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/namespaces\",\"type\":\"string\"},\"ownerReferences\":{\"description\":\"List of objects depended by this object. If ALL objects in the list have been deleted, this object will be garbage collected. If this object is managed by a controller, then an entry in this list will point to this controller, with the controller field set to true. There cannot be more than one managing controller.\",\"items\":{\"description\":\"OwnerReference contains enough information to let you identify an owning object. An owning object must be in the same namespace as the dependent, or be cluster-scoped, so there is no namespace field.\",\"properties\":{\"apiVersion\":{\"description\":\"API version of the referent.\",\"type\":\"string\"},\"blockOwnerDeletion\":{\"description\":\"If true, AND if the owner has the \\\"foregroundDeletion\\\" finalizer, then the owner cannot be deleted from the key-value store until this reference is removed. Defaults to false. To set this field, a user needs \\\"delete\\\" permission of the owner, otherwise 422 (Unprocessable Entity) will be returned.\",\"type\":\"boolean\"},\"controller\":{\"description\":\"If true, this reference points to the managing controller.\",\"type\":\"boolean\"},\"kind\":{\"description\":\"Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#names\",\"type\":\"string\"},\"uid\":{\"description\":\"UID of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#uids\",\"type\":\"string\"}},\"required\":[\"apiVersion\",\"kind\",\"name\",\"uid\"]},\"type\":\"array\"},\"resourceVersion\":{\"description\":\"An opaque value that represents the internal version of this object that can be used by clients to determine when objects have changed. May be used for optimistic concurrency, change detection, and the watch operation on a resource or set of resources. Clients must treat these values as opaque and passed unmodified back to the server. They may only be valid for a particular resource or set of resources.\\nPopulated by the system. Read-only. Value must be treated as opaque by clients and . More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency\",\"type\":\"string\"},\"selfLink\":{\"description\":\"SelfLink is a URL representing this object. Populated by the system. Read-only.\",\"type\":\"string\"},\"uid\":{\"description\":\"UID is the unique in time and space value for this object. It is typically generated by the server on successful creation of a resource and is not allowed to change on PUT operations.\\nPopulated by the system. Read-only. More info: http://kubernetes.io/docs/user-guide/identifiers#uids\",\"type\":\"string\"}}},\"spec\":{\"description\":\"PersistentVolumeClaimSpec describes the common attributes of storage devices and allows a Source for provider-specific attributes\",\"properties\":{\"accessModes\":{\"description\":\"AccessModes contains the desired access modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"dataSource\":{\"description\":\"TypedLocalObjectReference contains enough information to let you locate the typed referenced object inside the same namespace.\",\"properties\":{\"apiGroup\":{\"description\":\"APIGroup is the group for the resource being referenced. If APIGroup is not specified, the specified Kind must be in the core API group. For any other third-party types, APIGroup is required.\",\"type\":\"string\"},\"kind\":{\"description\":\"Kind is the type of resource being referenced\",\"type\":\"string\"},\"name\":{\"description\":\"Name is the name of resource being referenced\",\"type\":\"string\"}},\"required\":[\"kind\",\"name\"]},\"resources\":{\"description\":\"ResourceRequirements describes the compute resource requirements.\",\"properties\":{\"limits\":{\"description\":\"Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/\",\"type\":\"object\"},\"requests\":{\"description\":\"Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/\",\"type\":\"object\"}}},\"selector\":{\"description\":\"A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.\",\"properties\":{\"matchExpressions\":{\"description\":\"matchExpressions is a list of label selector requirements. The requirements are ANDed.\",\"items\":{\"description\":\"A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"key is the label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\",\"type\":\"string\"},\"values\":{\"description\":\"values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"]},\"type\":\"array\"},\"matchLabels\":{\"description\":\"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \\\"key\\\", the operator is \\\"In\\\", and the values array contains only \\\"value\\\". The requirements are ANDed.\",\"type\":\"object\"}}},\"storageClassName\":{\"description\":\"Name of the StorageClass required by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1\",\"type\":\"string\"},\"volumeMode\":{\"description\":\"volumeMode defines what type of volume is required by the claim. Value of Filesystem is implied when not included in claim spec. This is a beta feature.\",\"type\":\"string\"},\"volumeName\":{\"description\":\"VolumeName is the binding reference to the PersistentVolume backing this claim.\",\"type\":\"string\"}}},\"status\":{\"description\":\"PersistentVolumeClaimStatus is the current status of a persistent volume claim.\",\"properties\":{\"accessModes\":{\"description\":\"AccessModes contains the actual access modes the volume backing the PVC has. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"capacity\":{\"description\":\"Represents the actual resources of the underlying volume.\",\"type\":\"object\"},\"conditions\":{\"description\":\"Current Condition of persistent volume claim. If underlying persistent volume is being resized then the Condition will be set to 'ResizeStarted'.\",\"items\":{\"description\":\"PersistentVolumeClaimCondition contains details about state of pvc\",\"properties\":{\"lastProbeTime\":{\"description\":\"Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.\",\"format\":\"date-time\",\"type\":\"string\"},\"lastTransitionTime\":{\"description\":\"Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.\",\"format\":\"date-time\",\"type\":\"string\"},\"message\":{\"description\":\"Human-readable message indicating details about last transition.\",\"type\":\"string\"},\"reason\":{\"description\":\"Unique, this should be a short, machine understandable string that gives the reason for condition's last transition. If it reports \\\"ResizeStarted\\\" that means the underlying persistent volume is being resized.\",\"type\":\"string\"},\"status\":{\"type\":\"string\"},\"type\":{\"type\":\"string\"}},\"required\":[\"type\",\"status\"]},\"type\":\"array\"},\"phase\":{\"description\":\"Phase represents the current phase of PersistentVolumeClaim.\",\"type\":\"string\"}}}}}}},\"tag\":{\"description\":\"Tag of Alertmanager container image to be deployed. Defaults to the value of `version`. Version is ignored if Tag is set.\",\"type\":\"string\"},\"tolerations\":{\"description\":\"If specified, the pod's tolerations.\",\"items\":{\"description\":\"The pod this Toleration is attached to tolerates any taint that matches the triple \\u003ckey,value,effect\\u003e using the matching operator \\u003coperator\\u003e.\",\"properties\":{\"effect\":{\"description\":\"Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.\",\"type\":\"string\"},\"key\":{\"description\":\"Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys.\",\"type\":\"string\"},\"operator\":{\"description\":\"Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category.\",\"type\":\"string\"},\"tolerationSeconds\":{\"description\":\"TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system.\",\"format\":\"int64\",\"type\":\"integer\"},\"value\":{\"description\":\"Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string.\",\"type\":\"string\"}}},\"type\":\"array\"},\"version\":{\"description\":\"Version the cluster should be on.\",\"type\":\"string\"}}},\"status\":{\"description\":\"AlertmanagerStatus is the most recent observed status of the Alertmanager cluster. Read-only. Not included when requesting from the apiserver, only from the Prometheus Operator API itself. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/api-conventions.md#spec-and-status\",\"properties\":{\"availableReplicas\":{\"description\":\"Total number of available pods (ready for at least minReadySeconds) targeted by this Alertmanager cluster.\",\"format\":\"int32\",\"type\":\"integer\"},\"paused\":{\"description\":\"Represents whether any actions on the underlying managed objects are being performed. Only delete actions will be performed.\",\"type\":\"boolean\"},\"replicas\":{\"description\":\"Total number of non-terminated pods targeted by this Alertmanager cluster (their labels match the selector).\",\"format\":\"int32\",\"type\":\"integer\"},\"unavailableReplicas\":{\"description\":\"Total number of unavailable pods targeted by this Alertmanager cluster.\",\"format\":\"int32\",\"type\":\"integer\"},\"updatedReplicas\":{\"description\":\"Total number of non-terminated pods targeted by this Alertmanager cluster that have the desired version spec.\",\"format\":\"int32\",\"type\":\"integer\"}},\"required\":[\"paused\",\"replicas\",\"updatedReplicas\",\"availableReplicas\",\"unavailableReplicas\"]}}}},\"version\":\"v1\"}}\n" } }, "spec": { "group": "monitoring.coreos.com", "version": "v1", "names": { "plural": "alertmanagers", "singular": "alertmanager", "kind": "Alertmanager", "listKind": "AlertmanagerList" }, "scope": "Namespaced", "validation": { "openAPIV3Schema": { "properties": { "apiVersion": { "description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources", "type": "string" }, "kind": { "description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds", "type": "string" }, "spec": { "description": "AlertmanagerSpec is a specification of the desired behavior of the Alertmanager cluster. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/api-conventions.md#spec-and-status", "properties": { "additionalPeers": { "description": "AdditionalPeers allows injecting a set of additional Alertmanagers to peer with to form a highly available cluster.", "type": "array", "items": { "type": "string" } }, "affinity": { "description": "Affinity is a group of affinity scheduling rules.", "properties": { "nodeAffinity": { "description": "Node affinity is a group of node affinity scheduling rules.", "properties": { "preferredDuringSchedulingIgnoredDuringExecution": { "description": "The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding \"weight\" to the sum if the node matches the corresponding matchExpressions; the node(s) with the highest sum are the most preferred.", "type": "array", "items": { "description": "An empty preferred scheduling term matches all objects with implicit weight 0 (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op).", "required": [ "weight", "preference" ], "properties": { "preference": { "description": "A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm.", "properties": { "matchExpressions": { "description": "A list of node selector requirements by node's labels.", "type": "array", "items": { "description": "A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "required": [ "key", "operator" ], "properties": { "key": { "description": "The label key that the selector applies to.", "type": "string" }, "operator": { "description": "Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.", "type": "string" }, "values": { "description": "An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.", "type": "array", "items": { "type": "string" } } } } }, "matchFields": { "description": "A list of node selector requirements by node's fields.", "type": "array", "items": { "description": "A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "required": [ "key", "operator" ], "properties": { "key": { "description": "The label key that the selector applies to.", "type": "string" }, "operator": { "description": "Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.", "type": "string" }, "values": { "description": "An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.", "type": "array", "items": { "type": "string" } } } } } } }, "weight": { "description": "Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100.", "type": "integer", "format": "int32" } } } }, "requiredDuringSchedulingIgnoredDuringExecution": { "description": "A node selector represents the union of the results of one or more label queries over a set of nodes; that is, it represents the OR of the selectors represented by the node selector terms.", "required": [ "nodeSelectorTerms" ], "properties": { "nodeSelectorTerms": { "description": "Required. A list of node selector terms. The terms are ORed.", "type": "array", "items": { "description": "A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm.", "properties": { "matchExpressions": { "description": "A list of node selector requirements by node's labels.", "type": "array", "items": { "description": "A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "required": [ "key", "operator" ], "properties": { "key": { "description": "The label key that the selector applies to.", "type": "string" }, "operator": { "description": "Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.", "type": "string" }, "values": { "description": "An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.", "type": "array", "items": { "type": "string" } } } } }, "matchFields": { "description": "A list of node selector requirements by node's fields.", "type": "array", "items": { "description": "A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "required": [ "key", "operator" ], "properties": { "key": { "description": "The label key that the selector applies to.", "type": "string" }, "operator": { "description": "Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.", "type": "string" }, "values": { "description": "An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.", "type": "array", "items": { "type": "string" } } } } } } } } } } } }, "podAffinity": { "description": "Pod affinity is a group of inter pod affinity scheduling rules.", "properties": { "preferredDuringSchedulingIgnoredDuringExecution": { "description": "The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding \"weight\" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred.", "type": "array", "items": { "description": "The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s)", "required": [ "weight", "podAffinityTerm" ], "properties": { "podAffinityTerm": { "description": "Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running", "required": [ "topologyKey" ], "properties": { "labelSelector": { "description": "A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "type": "array", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "required": [ "key", "operator" ], "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "type": "array", "items": { "type": "string" } } } } }, "matchLabels": { "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object" } } }, "namespaces": { "description": "namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \"this pod's namespace\"", "type": "array", "items": { "type": "string" } }, "topologyKey": { "description": "This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.", "type": "string" } } }, "weight": { "description": "weight associated with matching the corresponding podAffinityTerm, in the range 1-100.", "type": "integer", "format": "int32" } } } }, "requiredDuringSchedulingIgnoredDuringExecution": { "description": "If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied.", "type": "array", "items": { "description": "Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running", "required": [ "topologyKey" ], "properties": { "labelSelector": { "description": "A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "type": "array", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "required": [ "key", "operator" ], "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "type": "array", "items": { "type": "string" } } } } }, "matchLabels": { "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object" } } }, "namespaces": { "description": "namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \"this pod's namespace\"", "type": "array", "items": { "type": "string" } }, "topologyKey": { "description": "This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.", "type": "string" } } } } } }, "podAntiAffinity": { "description": "Pod anti affinity is a group of inter pod anti affinity scheduling rules.", "properties": { "preferredDuringSchedulingIgnoredDuringExecution": { "description": "The scheduler will prefer to schedule pods to nodes that satisfy the anti-affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding \"weight\" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred.", "type": "array", "items": { "description": "The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s)", "required": [ "weight", "podAffinityTerm" ], "properties": { "podAffinityTerm": { "description": "Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running", "required": [ "topologyKey" ], "properties": { "labelSelector": { "description": "A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "type": "array", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "required": [ "key", "operator" ], "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "type": "array", "items": { "type": "string" } } } } }, "matchLabels": { "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object" } } }, "namespaces": { "description": "namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \"this pod's namespace\"", "type": "array", "items": { "type": "string" } }, "topologyKey": { "description": "This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.", "type": "string" } } }, "weight": { "description": "weight associated with matching the corresponding podAffinityTerm, in the range 1-100.", "type": "integer", "format": "int32" } } } }, "requiredDuringSchedulingIgnoredDuringExecution": { "description": "If the anti-affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the anti-affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied.", "type": "array", "items": { "description": "Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running", "required": [ "topologyKey" ], "properties": { "labelSelector": { "description": "A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "type": "array", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "required": [ "key", "operator" ], "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "type": "array", "items": { "type": "string" } } } } }, "matchLabels": { "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object" } } }, "namespaces": { "description": "namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \"this pod's namespace\"", "type": "array", "items": { "type": "string" } }, "topologyKey": { "description": "This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.", "type": "string" } } } } } } } }, "baseImage": { "description": "Base image that is used to deploy pods, without tag.", "type": "string" }, "configMaps": { "description": "ConfigMaps is a list of ConfigMaps in the same namespace as the Alertmanager object, which shall be mounted into the Alertmanager Pods. The ConfigMaps are mounted into /etc/alertmanager/configmaps/.", "type": "array", "items": { "type": "string" } }, "configSecret": { "description": "ConfigSecret is the name of a Kubernetes Secret in the same namespace as the Alertmanager object, which contains configuration for this Alertmanager instance. Defaults to 'alertmanager-' The secret is mounted into /etc/alertmanager/config.", "type": "string" }, "containers": { "description": "Containers allows injecting additional containers. This is meant to allow adding an authentication proxy to an Alertmanager pod.", "type": "array", "items": { "description": "A single application container that you want to run within a pod.", "required": [ "name" ], "properties": { "args": { "description": "Arguments to the entrypoint. The docker image's CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell", "type": "array", "items": { "type": "string" } }, "command": { "description": "Entrypoint array. Not executed within a shell. The docker image's ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell", "type": "array", "items": { "type": "string" } }, "env": { "description": "List of environment variables to set in the container. Cannot be updated.", "type": "array", "items": { "description": "EnvVar represents an environment variable present in a Container.", "required": [ "name" ], "properties": { "name": { "description": "Name of the environment variable. Must be a C_IDENTIFIER.", "type": "string" }, "value": { "description": "Variable references $(VAR_NAME) are expanded using the previous defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Defaults to \"\".", "type": "string" }, "valueFrom": { "description": "EnvVarSource represents a source for the value of an EnvVar.", "properties": { "configMapKeyRef": { "description": "Selects a key from a ConfigMap.", "required": [ "key" ], "properties": { "key": { "description": "The key to select.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", "type": "string" }, "optional": { "description": "Specify whether the ConfigMap or it's key must be defined", "type": "boolean" } } }, "fieldRef": { "description": "ObjectFieldSelector selects an APIVersioned field of an object.", "required": [ "fieldPath" ], "properties": { "apiVersion": { "description": "Version of the schema the FieldPath is written in terms of, defaults to \"v1\".", "type": "string" }, "fieldPath": { "description": "Path of the field to select in the specified API version.", "type": "string" } } }, "resourceFieldRef": { "description": "ResourceFieldSelector represents container resources (cpu, memory) and their output format", "required": [ "resource" ], "properties": { "containerName": { "description": "Container name: required for volumes, optional for env vars", "type": "string" }, "divisor": {}, "resource": { "description": "Required: resource to select", "type": "string" } } }, "secretKeyRef": { "description": "SecretKeySelector selects a key of a Secret.", "required": [ "key" ], "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", "type": "string" }, "optional": { "description": "Specify whether the Secret or it's key must be defined", "type": "boolean" } } } } } } } }, "envFrom": { "description": "List of sources to populate environment variables in the container. The keys defined within a source must be a C_IDENTIFIER. All invalid keys will be reported as an event when the container is starting. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated.", "type": "array", "items": { "description": "EnvFromSource represents the source of a set of ConfigMaps", "properties": { "configMapRef": { "description": "ConfigMapEnvSource selects a ConfigMap to populate the environment variables with.\nThe contents of the target ConfigMap's Data field will represent the key-value pairs as environment variables.", "properties": { "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", "type": "string" }, "optional": { "description": "Specify whether the ConfigMap must be defined", "type": "boolean" } } }, "prefix": { "description": "An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER.", "type": "string" }, "secretRef": { "description": "SecretEnvSource selects a Secret to populate the environment variables with.\nThe contents of the target Secret's Data field will represent the key-value pairs as environment variables.", "properties": { "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", "type": "string" }, "optional": { "description": "Specify whether the Secret must be defined", "type": "boolean" } } } } } }, "image": { "description": "Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images This field is optional to allow higher level config management to default or override container images in workload controllers like Deployments and StatefulSets.", "type": "string" }, "imagePullPolicy": { "description": "Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images", "type": "string" }, "lifecycle": { "description": "Lifecycle describes actions that the management system should take in response to container lifecycle events. For the PostStart and PreStop lifecycle handlers, management of the container blocks until the action is complete, unless the container process fails, in which case the handler is aborted.", "properties": { "postStart": { "description": "Handler defines a specific action that should be taken", "properties": { "exec": { "description": "ExecAction describes a \"run in container\" action.", "properties": { "command": { "description": "Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.", "type": "array", "items": { "type": "string" } } } }, "httpGet": { "description": "HTTPGetAction describes an action based on HTTP Get requests.", "required": [ "port" ], "properties": { "host": { "description": "Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.", "type": "string" }, "httpHeaders": { "description": "Custom headers to set in the request. HTTP allows repeated headers.", "type": "array", "items": { "description": "HTTPHeader describes a custom header to be used in HTTP probes", "required": [ "name", "value" ], "properties": { "name": { "description": "The header field name", "type": "string" }, "value": { "description": "The header field value", "type": "string" } } } }, "path": { "description": "Path to access on the HTTP server.", "type": "string" }, "port": { "anyOf": [ { "type": "string" }, { "type": "integer" } ] }, "scheme": { "description": "Scheme to use for connecting to the host. Defaults to HTTP.", "type": "string" } } }, "tcpSocket": { "description": "TCPSocketAction describes an action based on opening a socket", "required": [ "port" ], "properties": { "host": { "description": "Optional: Host name to connect to, defaults to the pod IP.", "type": "string" }, "port": { "anyOf": [ { "type": "string" }, { "type": "integer" } ] } } } } }, "preStop": { "description": "Handler defines a specific action that should be taken", "properties": { "exec": { "description": "ExecAction describes a \"run in container\" action.", "properties": { "command": { "description": "Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.", "type": "array", "items": { "type": "string" } } } }, "httpGet": { "description": "HTTPGetAction describes an action based on HTTP Get requests.", "required": [ "port" ], "properties": { "host": { "description": "Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.", "type": "string" }, "httpHeaders": { "description": "Custom headers to set in the request. HTTP allows repeated headers.", "type": "array", "items": { "description": "HTTPHeader describes a custom header to be used in HTTP probes", "required": [ "name", "value" ], "properties": { "name": { "description": "The header field name", "type": "string" }, "value": { "description": "The header field value", "type": "string" } } } }, "path": { "description": "Path to access on the HTTP server.", "type": "string" }, "port": { "anyOf": [ { "type": "string" }, { "type": "integer" } ] }, "scheme": { "description": "Scheme to use for connecting to the host. Defaults to HTTP.", "type": "string" } } }, "tcpSocket": { "description": "TCPSocketAction describes an action based on opening a socket", "required": [ "port" ], "properties": { "host": { "description": "Optional: Host name to connect to, defaults to the pod IP.", "type": "string" }, "port": { "anyOf": [ { "type": "string" }, { "type": "integer" } ] } } } } } } }, "livenessProbe": { "description": "Probe describes a health check to be performed against a container to determine whether it is alive or ready to receive traffic.", "properties": { "exec": { "description": "ExecAction describes a \"run in container\" action.", "properties": { "command": { "description": "Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.", "type": "array", "items": { "type": "string" } } } }, "failureThreshold": { "description": "Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.", "type": "integer", "format": "int32" }, "httpGet": { "description": "HTTPGetAction describes an action based on HTTP Get requests.", "required": [ "port" ], "properties": { "host": { "description": "Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.", "type": "string" }, "httpHeaders": { "description": "Custom headers to set in the request. HTTP allows repeated headers.", "type": "array", "items": { "description": "HTTPHeader describes a custom header to be used in HTTP probes", "required": [ "name", "value" ], "properties": { "name": { "description": "The header field name", "type": "string" }, "value": { "description": "The header field value", "type": "string" } } } }, "path": { "description": "Path to access on the HTTP server.", "type": "string" }, "port": { "anyOf": [ { "type": "string" }, { "type": "integer" } ] }, "scheme": { "description": "Scheme to use for connecting to the host. Defaults to HTTP.", "type": "string" } } }, "initialDelaySeconds": { "description": "Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes", "type": "integer", "format": "int32" }, "periodSeconds": { "description": "How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.", "type": "integer", "format": "int32" }, "successThreshold": { "description": "Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness. Minimum value is 1.", "type": "integer", "format": "int32" }, "tcpSocket": { "description": "TCPSocketAction describes an action based on opening a socket", "required": [ "port" ], "properties": { "host": { "description": "Optional: Host name to connect to, defaults to the pod IP.", "type": "string" }, "port": { "anyOf": [ { "type": "string" }, { "type": "integer" } ] } } }, "timeoutSeconds": { "description": "Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes", "type": "integer", "format": "int32" } } }, "name": { "description": "Name of the container specified as a DNS_LABEL. Each container in a pod must have a unique name (DNS_LABEL). Cannot be updated.", "type": "string" }, "ports": { "description": "List of ports to expose from the container. Exposing a port here gives the system additional information about the network connections a container uses, but is primarily informational. Not specifying a port here DOES NOT prevent that port from being exposed. Any port which is listening on the default \"0.0.0.0\" address inside a container will be accessible from the network. Cannot be updated.", "type": "array", "items": { "description": "ContainerPort represents a network port in a single container.", "required": [ "containerPort" ], "properties": { "containerPort": { "description": "Number of port to expose on the pod's IP address. This must be a valid port number, 0 < x < 65536.", "type": "integer", "format": "int32" }, "hostIP": { "description": "What host IP to bind the external port to.", "type": "string" }, "hostPort": { "description": "Number of port to expose on the host. If specified, this must be a valid port number, 0 < x < 65536. If HostNetwork is specified, this must match ContainerPort. Most containers do not need this.", "type": "integer", "format": "int32" }, "name": { "description": "If specified, this must be an IANA_SVC_NAME and unique within the pod. Each named port in a pod must have a unique name. Name for the port that can be referred to by services.", "type": "string" }, "protocol": { "description": "Protocol for port. Must be UDP, TCP, or SCTP. Defaults to \"TCP\".", "type": "string" } } } }, "readinessProbe": { "description": "Probe describes a health check to be performed against a container to determine whether it is alive or ready to receive traffic.", "properties": { "exec": { "description": "ExecAction describes a \"run in container\" action.", "properties": { "command": { "description": "Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.", "type": "array", "items": { "type": "string" } } } }, "failureThreshold": { "description": "Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.", "type": "integer", "format": "int32" }, "httpGet": { "description": "HTTPGetAction describes an action based on HTTP Get requests.", "required": [ "port" ], "properties": { "host": { "description": "Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.", "type": "string" }, "httpHeaders": { "description": "Custom headers to set in the request. HTTP allows repeated headers.", "type": "array", "items": { "description": "HTTPHeader describes a custom header to be used in HTTP probes", "required": [ "name", "value" ], "properties": { "name": { "description": "The header field name", "type": "string" }, "value": { "description": "The header field value", "type": "string" } } } }, "path": { "description": "Path to access on the HTTP server.", "type": "string" }, "port": { "anyOf": [ { "type": "string" }, { "type": "integer" } ] }, "scheme": { "description": "Scheme to use for connecting to the host. Defaults to HTTP.", "type": "string" } } }, "initialDelaySeconds": { "description": "Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes", "type": "integer", "format": "int32" }, "periodSeconds": { "description": "How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.", "type": "integer", "format": "int32" }, "successThreshold": { "description": "Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness. Minimum value is 1.", "type": "integer", "format": "int32" }, "tcpSocket": { "description": "TCPSocketAction describes an action based on opening a socket", "required": [ "port" ], "properties": { "host": { "description": "Optional: Host name to connect to, defaults to the pod IP.", "type": "string" }, "port": { "anyOf": [ { "type": "string" }, { "type": "integer" } ] } } }, "timeoutSeconds": { "description": "Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes", "type": "integer", "format": "int32" } } }, "resources": { "description": "ResourceRequirements describes the compute resource requirements.", "properties": { "limits": { "description": "Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", "type": "object" }, "requests": { "description": "Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", "type": "object" } } }, "securityContext": { "description": "SecurityContext holds security configuration that will be applied to a container. Some fields are present in both SecurityContext and PodSecurityContext. When both are set, the values in SecurityContext take precedence.", "properties": { "allowPrivilegeEscalation": { "description": "AllowPrivilegeEscalation controls whether a process can gain more privileges than its parent process. This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged 2) has CAP_SYS_ADMIN", "type": "boolean" }, "capabilities": { "description": "Adds and removes POSIX capabilities from running containers.", "properties": { "add": { "description": "Added capabilities", "type": "array", "items": { "type": "string" } }, "drop": { "description": "Removed capabilities", "type": "array", "items": { "type": "string" } } } }, "privileged": { "description": "Run container in privileged mode. Processes in privileged containers are essentially equivalent to root on the host. Defaults to false.", "type": "boolean" }, "procMount": { "description": "procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled.", "type": "string" }, "readOnlyRootFilesystem": { "description": "Whether this container has a read-only root filesystem. Default is false.", "type": "boolean" }, "runAsGroup": { "description": "The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.", "type": "integer", "format": "int64" }, "runAsNonRoot": { "description": "Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.", "type": "boolean" }, "runAsUser": { "description": "The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.", "type": "integer", "format": "int64" }, "seLinuxOptions": { "description": "SELinuxOptions are the labels to be applied to the container", "properties": { "level": { "description": "Level is SELinux level label that applies to the container.", "type": "string" }, "role": { "description": "Role is a SELinux role label that applies to the container.", "type": "string" }, "type": { "description": "Type is a SELinux type label that applies to the container.", "type": "string" }, "user": { "description": "User is a SELinux user label that applies to the container.", "type": "string" } } } } }, "stdin": { "description": "Whether this container should allocate a buffer for stdin in the container runtime. If this is not set, reads from stdin in the container will always result in EOF. Default is false.", "type": "boolean" }, "stdinOnce": { "description": "Whether the container runtime should close the stdin channel after it has been opened by a single attach. When stdin is true the stdin stream will remain open across multiple attach sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the first client attaches to stdin, and then remains open and accepts data until the client disconnects, at which time stdin is closed and remains closed until the container is restarted. If this flag is false, a container processes that reads from stdin will never receive an EOF. Default is false", "type": "boolean" }, "terminationMessagePath": { "description": "Optional: Path at which the file to which the container's termination message will be written is mounted into the container's filesystem. Message written is intended to be brief final status, such as an assertion failure message. Will be truncated by the node if greater than 4096 bytes. The total message length across all containers will be limited to 12kb. Defaults to /dev/termination-log. Cannot be updated.", "type": "string" }, "terminationMessagePolicy": { "description": "Indicate how the termination message should be populated. File will use the contents of terminationMessagePath to populate the container status message on both success and failure. FallbackToLogsOnError will use the last chunk of container log output if the termination message file is empty and the container exited with an error. The log output is limited to 2048 bytes or 80 lines, whichever is smaller. Defaults to File. Cannot be updated.", "type": "string" }, "tty": { "description": "Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. Default is false.", "type": "boolean" }, "volumeDevices": { "description": "volumeDevices is the list of block devices to be used by the container. This is a beta feature.", "type": "array", "items": { "description": "volumeDevice describes a mapping of a raw block device within a container.", "required": [ "name", "devicePath" ], "properties": { "devicePath": { "description": "devicePath is the path inside of the container that the device will be mapped to.", "type": "string" }, "name": { "description": "name must match the name of a persistentVolumeClaim in the pod", "type": "string" } } } }, "volumeMounts": { "description": "Pod volumes to mount into the container's filesystem. Cannot be updated.", "type": "array", "items": { "description": "VolumeMount describes a mounting of a Volume within a container.", "required": [ "name", "mountPath" ], "properties": { "mountPath": { "description": "Path within the container at which the volume should be mounted. Must not contain ':'.", "type": "string" }, "mountPropagation": { "description": "mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10.", "type": "string" }, "name": { "description": "This must match the Name of a Volume.", "type": "string" }, "readOnly": { "description": "Mounted read-only if true, read-write otherwise (false or unspecified). Defaults to false.", "type": "boolean" }, "subPath": { "description": "Path within the volume from which the container's volume should be mounted. Defaults to \"\" (volume's root).", "type": "string" } } } }, "workingDir": { "description": "Container's working directory. If not specified, the container runtime's default will be used, which might be configured in the container image. Cannot be updated.", "type": "string" } } } }, "externalUrl": { "description": "The external URL the Alertmanager instances will be available under. This is necessary to generate correct URLs. This is necessary if Alertmanager is not served from root of a DNS name.", "type": "string" }, "image": { "description": "Image if specified has precedence over baseImage, tag and sha combinations. Specifying the version is still necessary to ensure the Prometheus Operator knows what version of Alertmanager is being configured.", "type": "string" }, "imagePullSecrets": { "description": "An optional list of references to secrets in the same namespace to use for pulling prometheus and alertmanager images from registries see http://kubernetes.io/docs/user-guide/images#specifying-imagepullsecrets-on-a-pod", "type": "array", "items": { "description": "LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace.", "properties": { "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", "type": "string" } } } }, "listenLocal": { "description": "ListenLocal makes the Alertmanager server listen on loopback, so that it does not bind against the Pod IP. Note this is only for the Alertmanager UI, not the gossip communication.", "type": "boolean" }, "logLevel": { "description": "Log level for Alertmanager to be configured with.", "type": "string" }, "nodeSelector": { "description": "Define which Nodes the Pods are scheduled on.", "type": "object" }, "paused": { "description": "If set to true all actions on the underlying managed objects are not goint to be performed, except for delete actions.", "type": "boolean" }, "podMetadata": { "description": "ObjectMeta is metadata that all persisted resources must have, which includes all objects users must create.", "properties": { "annotations": { "description": "Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations", "type": "object" }, "clusterName": { "description": "The name of the cluster which the object belongs to. This is used to distinguish resources with same name and namespace in different clusters. This field is not set anywhere right now and apiserver is going to ignore it if set in create or update request.", "type": "string" }, "creationTimestamp": { "description": "Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.", "type": "string", "format": "date-time" }, "deletionGracePeriodSeconds": { "description": "Number of seconds allowed for this object to gracefully terminate before it will be removed from the system. Only set when deletionTimestamp is also set. May only be shortened. Read-only.", "type": "integer", "format": "int64" }, "deletionTimestamp": { "description": "Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.", "type": "string", "format": "date-time" }, "finalizers": { "description": "Must be empty before the object is deleted from the registry. Each entry is an identifier for the responsible component that will remove the entry from the list. If the deletionTimestamp of the object is non-nil, entries in this list can only be removed.", "type": "array", "items": { "type": "string" } }, "generateName": { "description": "GenerateName is an optional prefix, used by the server, to generate a unique name ONLY IF the Name field has not been provided. If this field is used, the name returned to the client will be different than the name passed. This value will also be combined with a unique suffix. The provided value has the same validation rules as the Name field, and may be truncated by the length of the suffix required to make the value unique on the server.\nIf this field is specified and the generated name exists, the server will NOT return a 409 - instead, it will either return 201 Created or 500 with Reason ServerTimeout indicating a unique name could not be found in the time allotted, and the client should retry (optionally after the time indicated in the Retry-After header).\nApplied only if Name is not specified. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#idempotency", "type": "string" }, "generation": { "description": "A sequence number representing a specific generation of the desired state. Populated by the system. Read-only.", "type": "integer", "format": "int64" }, "initializers": { "description": "Initializers tracks the progress of initialization.", "required": [ "pending" ], "properties": { "pending": { "description": "Pending is a list of initializers that must execute in order before this object is visible. When the last pending initializer is removed, and no failing result is set, the initializers struct will be set to nil and the object is considered as initialized and visible to all clients.", "type": "array", "items": { "description": "Initializer is information about an initializer that has not yet completed.", "required": [ "name" ], "properties": { "name": { "description": "name of the process that is responsible for initializing this object.", "type": "string" } } } }, "result": { "description": "Status is a return value for calls that don't return other objects.", "properties": { "apiVersion": { "description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources", "type": "string" }, "code": { "description": "Suggested HTTP return code for this status, 0 if not set.", "type": "integer", "format": "int32" }, "details": { "description": "StatusDetails is a set of additional properties that MAY be set by the server to provide additional information about a response. The Reason field of a Status object defines what attributes will be set. Clients must ignore fields that do not match the defined type of each attribute, and should assume that any attribute may be empty, invalid, or under defined.", "properties": { "causes": { "description": "The Causes array includes more details associated with the StatusReason failure. Not all StatusReasons may provide detailed causes.", "type": "array", "items": { "description": "StatusCause provides more information about an api.Status failure, including cases when multiple errors are encountered.", "properties": { "field": { "description": "The field of the resource that has caused this error, as named by its JSON serialization. May include dot and postfix notation for nested attributes. Arrays are zero-indexed. Fields may appear more than once in an array of causes due to fields having multiple errors. Optional.\nExamples:\n \"name\" - the field \"name\" on the current resource\n \"items[0].name\" - the field \"name\" on the first array entry in \"items\"", "type": "string" }, "message": { "description": "A human-readable description of the cause of the error. This field may be presented as-is to a reader.", "type": "string" }, "reason": { "description": "A machine-readable description of the cause of the error. If this value is empty there is no information available.", "type": "string" } } } }, "group": { "description": "The group attribute of the resource associated with the status StatusReason.", "type": "string" }, "kind": { "description": "The kind attribute of the resource associated with the status StatusReason. On some operations may differ from the requested resource Kind. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds", "type": "string" }, "name": { "description": "The name attribute of the resource associated with the status StatusReason (when there is a single name which can be described).", "type": "string" }, "retryAfterSeconds": { "description": "If specified, the time in seconds before the operation should be retried. Some errors may indicate the client must take an alternate action - for those errors this field may indicate how long to wait before taking the alternate action.", "type": "integer", "format": "int32" }, "uid": { "description": "UID of the resource. (when there is a single resource which can be described). More info: http://kubernetes.io/docs/user-guide/identifiers#uids", "type": "string" } } }, "kind": { "description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds", "type": "string" }, "message": { "description": "A human-readable description of the status of this operation.", "type": "string" }, "metadata": { "description": "ListMeta describes metadata that synthetic resources must have, including lists and various status objects. A resource may have only one of {ObjectMeta, ListMeta}.", "properties": { "continue": { "description": "continue may be set if the user set a limit on the number of items returned, and indicates that the server has more data available. The value is opaque and may be used to issue another request to the endpoint that served this list to retrieve the next set of available objects. Continuing a consistent list may not be possible if the server configuration has changed or more than a few minutes have passed. The resourceVersion field returned when using this continue value will be identical to the value in the first response, unless you have received this token from an error message.", "type": "string" }, "resourceVersion": { "description": "String that identifies the server's internal version of this object that can be used by clients to determine when objects have changed. Value must be treated as opaque by clients and passed unmodified back to the server. Populated by the system. Read-only. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency", "type": "string" }, "selfLink": { "description": "selfLink is a URL representing this object. Populated by the system. Read-only.", "type": "string" } } }, "reason": { "description": "A machine-readable description of why this operation is in the \"Failure\" status. If this value is empty there is no information available. A Reason clarifies an HTTP status code but does not override it.", "type": "string" }, "status": { "description": "Status of the operation. One of: \"Success\" or \"Failure\". More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status", "type": "string" } } } } }, "labels": { "description": "Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels", "type": "object" }, "name": { "description": "Name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names", "type": "string" }, "namespace": { "description": "Namespace defines the space within each name must be unique. An empty namespace is equivalent to the \"default\" namespace, but \"default\" is the canonical representation. Not all objects are required to be scoped to a namespace - the value of this field for those objects will be empty.\nMust be a DNS_LABEL. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/namespaces", "type": "string" }, "ownerReferences": { "description": "List of objects depended by this object. If ALL objects in the list have been deleted, this object will be garbage collected. If this object is managed by a controller, then an entry in this list will point to this controller, with the controller field set to true. There cannot be more than one managing controller.", "type": "array", "items": { "description": "OwnerReference contains enough information to let you identify an owning object. An owning object must be in the same namespace as the dependent, or be cluster-scoped, so there is no namespace field.", "required": [ "apiVersion", "kind", "name", "uid" ], "properties": { "apiVersion": { "description": "API version of the referent.", "type": "string" }, "blockOwnerDeletion": { "description": "If true, AND if the owner has the \"foregroundDeletion\" finalizer, then the owner cannot be deleted from the key-value store until this reference is removed. Defaults to false. To set this field, a user needs \"delete\" permission of the owner, otherwise 422 (Unprocessable Entity) will be returned.", "type": "boolean" }, "controller": { "description": "If true, this reference points to the managing controller.", "type": "boolean" }, "kind": { "description": "Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds", "type": "string" }, "name": { "description": "Name of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#names", "type": "string" }, "uid": { "description": "UID of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#uids", "type": "string" } } } }, "resourceVersion": { "description": "An opaque value that represents the internal version of this object that can be used by clients to determine when objects have changed. May be used for optimistic concurrency, change detection, and the watch operation on a resource or set of resources. Clients must treat these values as opaque and passed unmodified back to the server. They may only be valid for a particular resource or set of resources.\nPopulated by the system. Read-only. Value must be treated as opaque by clients and . More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency", "type": "string" }, "selfLink": { "description": "SelfLink is a URL representing this object. Populated by the system. Read-only.", "type": "string" }, "uid": { "description": "UID is the unique in time and space value for this object. It is typically generated by the server on successful creation of a resource and is not allowed to change on PUT operations.\nPopulated by the system. Read-only. More info: http://kubernetes.io/docs/user-guide/identifiers#uids", "type": "string" } } }, "priorityClassName": { "description": "Priority class assigned to the Pods", "type": "string" }, "replicas": { "description": "Size is the expected size of the alertmanager cluster. The controller will eventually make the size of the running cluster equal to the expected size.", "type": "integer", "format": "int32" }, "resources": { "description": "ResourceRequirements describes the compute resource requirements.", "properties": { "limits": { "description": "Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", "type": "object" }, "requests": { "description": "Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", "type": "object" } } }, "retention": { "description": "Time duration Alertmanager shall retain data for. Default is '120h', and must match the regular expression `[0-9]+(ms|s|m|h)` (milliseconds seconds minutes hours).", "type": "string" }, "routePrefix": { "description": "The route prefix Alertmanager registers HTTP handlers for. This is useful, if using ExternalURL and a proxy is rewriting HTTP routes of a request, and the actual ExternalURL is still true, but the server serves requests under a different route prefix. For example for use with `kubectl proxy`.", "type": "string" }, "secrets": { "description": "Secrets is a list of Secrets in the same namespace as the Alertmanager object, which shall be mounted into the Alertmanager Pods. The Secrets are mounted into /etc/alertmanager/secrets/.", "type": "array", "items": { "type": "string" } }, "securityContext": { "description": "PodSecurityContext holds pod-level security attributes and common container settings. Some fields are also present in container.securityContext. Field values of container.securityContext take precedence over field values of PodSecurityContext.", "properties": { "fsGroup": { "description": "A special supplemental group that applies to all containers in a pod. Some volume types allow the Kubelet to change the ownership of that volume to be owned by the pod:\n1. The owning GID will be the FSGroup 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) 3. The permission bits are OR'd with rw-rw----\nIf unset, the Kubelet will not modify the ownership and permissions of any volume.", "type": "integer", "format": "int64" }, "runAsGroup": { "description": "The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container.", "type": "integer", "format": "int64" }, "runAsNonRoot": { "description": "Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.", "type": "boolean" }, "runAsUser": { "description": "The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container.", "type": "integer", "format": "int64" }, "seLinuxOptions": { "description": "SELinuxOptions are the labels to be applied to the container", "properties": { "level": { "description": "Level is SELinux level label that applies to the container.", "type": "string" }, "role": { "description": "Role is a SELinux role label that applies to the container.", "type": "string" }, "type": { "description": "Type is a SELinux type label that applies to the container.", "type": "string" }, "user": { "description": "User is a SELinux user label that applies to the container.", "type": "string" } } }, "supplementalGroups": { "description": "A list of groups applied to the first process run in each container, in addition to the container's primary GID. If unspecified, no groups will be added to any container.", "type": "array", "items": { "type": "integer", "format": "int64" } }, "sysctls": { "description": "Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported sysctls (by the container runtime) might fail to launch.", "type": "array", "items": { "description": "Sysctl defines a kernel parameter to be set", "required": [ "name", "value" ], "properties": { "name": { "description": "Name of a property to set", "type": "string" }, "value": { "description": "Value of a property to set", "type": "string" } } } } } }, "serviceAccountName": { "description": "ServiceAccountName is the name of the ServiceAccount to use to run the Prometheus Pods.", "type": "string" }, "sha": { "description": "SHA of Alertmanager container image to be deployed. Defaults to the value of `version`. Similar to a tag, but the SHA explicitly deploys an immutable container image. Version and Tag are ignored if SHA is set.", "type": "string" }, "storage": { "description": "StorageSpec defines the configured storage for a group Prometheus servers. If neither `emptyDir` nor `volumeClaimTemplate` is specified, then by default an [EmptyDir](https://kubernetes.io/docs/concepts/storage/volumes/#emptydir) will be used.", "properties": { "emptyDir": { "description": "Represents an empty directory for a pod. Empty directory volumes support ownership management and SELinux relabeling.", "properties": { "medium": { "description": "What type of storage medium should back this directory. The default is \"\" which means to use the node's default medium. Must be an empty string (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir", "type": "string" }, "sizeLimit": {} } }, "volumeClaimTemplate": { "description": "PersistentVolumeClaim is a user's request for and claim to a persistent volume", "properties": { "apiVersion": { "description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources", "type": "string" }, "kind": { "description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds", "type": "string" }, "metadata": { "description": "ObjectMeta is metadata that all persisted resources must have, which includes all objects users must create.", "properties": { "annotations": { "description": "Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations", "type": "object" }, "clusterName": { "description": "The name of the cluster which the object belongs to. This is used to distinguish resources with same name and namespace in different clusters. This field is not set anywhere right now and apiserver is going to ignore it if set in create or update request.", "type": "string" }, "creationTimestamp": { "description": "Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.", "type": "string", "format": "date-time" }, "deletionGracePeriodSeconds": { "description": "Number of seconds allowed for this object to gracefully terminate before it will be removed from the system. Only set when deletionTimestamp is also set. May only be shortened. Read-only.", "type": "integer", "format": "int64" }, "deletionTimestamp": { "description": "Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.", "type": "string", "format": "date-time" }, "finalizers": { "description": "Must be empty before the object is deleted from the registry. Each entry is an identifier for the responsible component that will remove the entry from the list. If the deletionTimestamp of the object is non-nil, entries in this list can only be removed.", "type": "array", "items": { "type": "string" } }, "generateName": { "description": "GenerateName is an optional prefix, used by the server, to generate a unique name ONLY IF the Name field has not been provided. If this field is used, the name returned to the client will be different than the name passed. This value will also be combined with a unique suffix. The provided value has the same validation rules as the Name field, and may be truncated by the length of the suffix required to make the value unique on the server.\nIf this field is specified and the generated name exists, the server will NOT return a 409 - instead, it will either return 201 Created or 500 with Reason ServerTimeout indicating a unique name could not be found in the time allotted, and the client should retry (optionally after the time indicated in the Retry-After header).\nApplied only if Name is not specified. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#idempotency", "type": "string" }, "generation": { "description": "A sequence number representing a specific generation of the desired state. Populated by the system. Read-only.", "type": "integer", "format": "int64" }, "initializers": { "description": "Initializers tracks the progress of initialization.", "required": [ "pending" ], "properties": { "pending": { "description": "Pending is a list of initializers that must execute in order before this object is visible. When the last pending initializer is removed, and no failing result is set, the initializers struct will be set to nil and the object is considered as initialized and visible to all clients.", "type": "array", "items": { "description": "Initializer is information about an initializer that has not yet completed.", "required": [ "name" ], "properties": { "name": { "description": "name of the process that is responsible for initializing this object.", "type": "string" } } } }, "result": { "description": "Status is a return value for calls that don't return other objects.", "properties": { "apiVersion": { "description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources", "type": "string" }, "code": { "description": "Suggested HTTP return code for this status, 0 if not set.", "type": "integer", "format": "int32" }, "details": { "description": "StatusDetails is a set of additional properties that MAY be set by the server to provide additional information about a response. The Reason field of a Status object defines what attributes will be set. Clients must ignore fields that do not match the defined type of each attribute, and should assume that any attribute may be empty, invalid, or under defined.", "properties": { "causes": { "description": "The Causes array includes more details associated with the StatusReason failure. Not all StatusReasons may provide detailed causes.", "type": "array", "items": { "description": "StatusCause provides more information about an api.Status failure, including cases when multiple errors are encountered.", "properties": { "field": { "description": "The field of the resource that has caused this error, as named by its JSON serialization. May include dot and postfix notation for nested attributes. Arrays are zero-indexed. Fields may appear more than once in an array of causes due to fields having multiple errors. Optional.\nExamples:\n \"name\" - the field \"name\" on the current resource\n \"items[0].name\" - the field \"name\" on the first array entry in \"items\"", "type": "string" }, "message": { "description": "A human-readable description of the cause of the error. This field may be presented as-is to a reader.", "type": "string" }, "reason": { "description": "A machine-readable description of the cause of the error. If this value is empty there is no information available.", "type": "string" } } } }, "group": { "description": "The group attribute of the resource associated with the status StatusReason.", "type": "string" }, "kind": { "description": "The kind attribute of the resource associated with the status StatusReason. On some operations may differ from the requested resource Kind. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds", "type": "string" }, "name": { "description": "The name attribute of the resource associated with the status StatusReason (when there is a single name which can be described).", "type": "string" }, "retryAfterSeconds": { "description": "If specified, the time in seconds before the operation should be retried. Some errors may indicate the client must take an alternate action - for those errors this field may indicate how long to wait before taking the alternate action.", "type": "integer", "format": "int32" }, "uid": { "description": "UID of the resource. (when there is a single resource which can be described). More info: http://kubernetes.io/docs/user-guide/identifiers#uids", "type": "string" } } }, "kind": { "description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds", "type": "string" }, "message": { "description": "A human-readable description of the status of this operation.", "type": "string" }, "metadata": { "description": "ListMeta describes metadata that synthetic resources must have, including lists and various status objects. A resource may have only one of {ObjectMeta, ListMeta}.", "properties": { "continue": { "description": "continue may be set if the user set a limit on the number of items returned, and indicates that the server has more data available. The value is opaque and may be used to issue another request to the endpoint that served this list to retrieve the next set of available objects. Continuing a consistent list may not be possible if the server configuration has changed or more than a few minutes have passed. The resourceVersion field returned when using this continue value will be identical to the value in the first response, unless you have received this token from an error message.", "type": "string" }, "resourceVersion": { "description": "String that identifies the server's internal version of this object that can be used by clients to determine when objects have changed. Value must be treated as opaque by clients and passed unmodified back to the server. Populated by the system. Read-only. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency", "type": "string" }, "selfLink": { "description": "selfLink is a URL representing this object. Populated by the system. Read-only.", "type": "string" } } }, "reason": { "description": "A machine-readable description of why this operation is in the \"Failure\" status. If this value is empty there is no information available. A Reason clarifies an HTTP status code but does not override it.", "type": "string" }, "status": { "description": "Status of the operation. One of: \"Success\" or \"Failure\". More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status", "type": "string" } } } } }, "labels": { "description": "Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels", "type": "object" }, "name": { "description": "Name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names", "type": "string" }, "namespace": { "description": "Namespace defines the space within each name must be unique. An empty namespace is equivalent to the \"default\" namespace, but \"default\" is the canonical representation. Not all objects are required to be scoped to a namespace - the value of this field for those objects will be empty.\nMust be a DNS_LABEL. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/namespaces", "type": "string" }, "ownerReferences": { "description": "List of objects depended by this object. If ALL objects in the list have been deleted, this object will be garbage collected. If this object is managed by a controller, then an entry in this list will point to this controller, with the controller field set to true. There cannot be more than one managing controller.", "type": "array", "items": { "description": "OwnerReference contains enough information to let you identify an owning object. An owning object must be in the same namespace as the dependent, or be cluster-scoped, so there is no namespace field.", "required": [ "apiVersion", "kind", "name", "uid" ], "properties": { "apiVersion": { "description": "API version of the referent.", "type": "string" }, "blockOwnerDeletion": { "description": "If true, AND if the owner has the \"foregroundDeletion\" finalizer, then the owner cannot be deleted from the key-value store until this reference is removed. Defaults to false. To set this field, a user needs \"delete\" permission of the owner, otherwise 422 (Unprocessable Entity) will be returned.", "type": "boolean" }, "controller": { "description": "If true, this reference points to the managing controller.", "type": "boolean" }, "kind": { "description": "Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds", "type": "string" }, "name": { "description": "Name of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#names", "type": "string" }, "uid": { "description": "UID of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#uids", "type": "string" } } } }, "resourceVersion": { "description": "An opaque value that represents the internal version of this object that can be used by clients to determine when objects have changed. May be used for optimistic concurrency, change detection, and the watch operation on a resource or set of resources. Clients must treat these values as opaque and passed unmodified back to the server. They may only be valid for a particular resource or set of resources.\nPopulated by the system. Read-only. Value must be treated as opaque by clients and . More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency", "type": "string" }, "selfLink": { "description": "SelfLink is a URL representing this object. Populated by the system. Read-only.", "type": "string" }, "uid": { "description": "UID is the unique in time and space value for this object. It is typically generated by the server on successful creation of a resource and is not allowed to change on PUT operations.\nPopulated by the system. Read-only. More info: http://kubernetes.io/docs/user-guide/identifiers#uids", "type": "string" } } }, "spec": { "description": "PersistentVolumeClaimSpec describes the common attributes of storage devices and allows a Source for provider-specific attributes", "properties": { "accessModes": { "description": "AccessModes contains the desired access modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1", "type": "array", "items": { "type": "string" } }, "dataSource": { "description": "TypedLocalObjectReference contains enough information to let you locate the typed referenced object inside the same namespace.", "required": [ "kind", "name" ], "properties": { "apiGroup": { "description": "APIGroup is the group for the resource being referenced. If APIGroup is not specified, the specified Kind must be in the core API group. For any other third-party types, APIGroup is required.", "type": "string" }, "kind": { "description": "Kind is the type of resource being referenced", "type": "string" }, "name": { "description": "Name is the name of resource being referenced", "type": "string" } } }, "resources": { "description": "ResourceRequirements describes the compute resource requirements.", "properties": { "limits": { "description": "Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", "type": "object" }, "requests": { "description": "Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", "type": "object" } } }, "selector": { "description": "A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "type": "array", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "required": [ "key", "operator" ], "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "type": "array", "items": { "type": "string" } } } } }, "matchLabels": { "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object" } } }, "storageClassName": { "description": "Name of the StorageClass required by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1", "type": "string" }, "volumeMode": { "description": "volumeMode defines what type of volume is required by the claim. Value of Filesystem is implied when not included in claim spec. This is a beta feature.", "type": "string" }, "volumeName": { "description": "VolumeName is the binding reference to the PersistentVolume backing this claim.", "type": "string" } } }, "status": { "description": "PersistentVolumeClaimStatus is the current status of a persistent volume claim.", "properties": { "accessModes": { "description": "AccessModes contains the actual access modes the volume backing the PVC has. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1", "type": "array", "items": { "type": "string" } }, "capacity": { "description": "Represents the actual resources of the underlying volume.", "type": "object" }, "conditions": { "description": "Current Condition of persistent volume claim. If underlying persistent volume is being resized then the Condition will be set to 'ResizeStarted'.", "type": "array", "items": { "description": "PersistentVolumeClaimCondition contains details about state of pvc", "required": [ "type", "status" ], "properties": { "lastProbeTime": { "description": "Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.", "type": "string", "format": "date-time" }, "lastTransitionTime": { "description": "Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.", "type": "string", "format": "date-time" }, "message": { "description": "Human-readable message indicating details about last transition.", "type": "string" }, "reason": { "description": "Unique, this should be a short, machine understandable string that gives the reason for condition's last transition. If it reports \"ResizeStarted\" that means the underlying persistent volume is being resized.", "type": "string" }, "status": { "type": "string" }, "type": { "type": "string" } } } }, "phase": { "description": "Phase represents the current phase of PersistentVolumeClaim.", "type": "string" } } } } } } }, "tag": { "description": "Tag of Alertmanager container image to be deployed. Defaults to the value of `version`. Version is ignored if Tag is set.", "type": "string" }, "tolerations": { "description": "If specified, the pod's tolerations.", "type": "array", "items": { "description": "The pod this Toleration is attached to tolerates any taint that matches the triple using the matching operator .", "properties": { "effect": { "description": "Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.", "type": "string" }, "key": { "description": "Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys.", "type": "string" }, "operator": { "description": "Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category.", "type": "string" }, "tolerationSeconds": { "description": "TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system.", "type": "integer", "format": "int64" }, "value": { "description": "Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string.", "type": "string" } } } }, "version": { "description": "Version the cluster should be on.", "type": "string" } } }, "status": { "description": "AlertmanagerStatus is the most recent observed status of the Alertmanager cluster. Read-only. Not included when requesting from the apiserver, only from the Prometheus Operator API itself. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/api-conventions.md#spec-and-status", "required": [ "paused", "replicas", "updatedReplicas", "availableReplicas", "unavailableReplicas" ], "properties": { "availableReplicas": { "description": "Total number of available pods (ready for at least minReadySeconds) targeted by this Alertmanager cluster.", "type": "integer", "format": "int32" }, "paused": { "description": "Represents whether any actions on the underlying managed objects are being performed. Only delete actions will be performed.", "type": "boolean" }, "replicas": { "description": "Total number of non-terminated pods targeted by this Alertmanager cluster (their labels match the selector).", "type": "integer", "format": "int32" }, "unavailableReplicas": { "description": "Total number of unavailable pods targeted by this Alertmanager cluster.", "type": "integer", "format": "int32" }, "updatedReplicas": { "description": "Total number of non-terminated pods targeted by this Alertmanager cluster that have the desired version spec.", "type": "integer", "format": "int32" } } } } } }, "versions": [ { "name": "v1", "served": true, "storage": true } ], "conversion": { "strategy": "None" }, "preserveUnknownFields": true }, "status": { "conditions": [ { "type": "NamesAccepted", "status": "True", "lastTransitionTime": "2020-05-05T16:51:39Z", "reason": "NoConflicts", "message": "no conflicts found" }, { "type": "Established", "status": "True", "lastTransitionTime": "2020-05-05T16:51:39Z", "reason": "InitialNamesAccepted", "message": "the initial names have been accepted" }, { "type": "NonStructuralSchema", "status": "True", "lastTransitionTime": "2020-05-05T16:51:39Z", "reason": "Violations", "message": "[spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[nodeAffinity].properties[preferredDuringSchedulingIgnoredDuringExecution].items.properties[preference].properties[matchExpressions].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[nodeAffinity].properties[preferredDuringSchedulingIgnoredDuringExecution].items.properties[preference].properties[matchFields].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[nodeAffinity].properties[preferredDuringSchedulingIgnoredDuringExecution].items.properties[preference].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[nodeAffinity].properties[preferredDuringSchedulingIgnoredDuringExecution].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[nodeAffinity].properties[requiredDuringSchedulingIgnoredDuringExecution].properties[nodeSelectorTerms].items.properties[matchExpressions].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[nodeAffinity].properties[requiredDuringSchedulingIgnoredDuringExecution].properties[nodeSelectorTerms].items.properties[matchFields].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[nodeAffinity].properties[requiredDuringSchedulingIgnoredDuringExecution].properties[nodeSelectorTerms].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[nodeAffinity].properties[requiredDuringSchedulingIgnoredDuringExecution].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[nodeAffinity].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[podAffinity].properties[preferredDuringSchedulingIgnoredDuringExecution].items.properties[podAffinityTerm].properties[labelSelector].properties[matchExpressions].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[podAffinity].properties[preferredDuringSchedulingIgnoredDuringExecution].items.properties[podAffinityTerm].properties[labelSelector].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[podAffinity].properties[preferredDuringSchedulingIgnoredDuringExecution].items.properties[podAffinityTerm].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[podAffinity].properties[preferredDuringSchedulingIgnoredDuringExecution].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[podAffinity].properties[requiredDuringSchedulingIgnoredDuringExecution].items.properties[labelSelector].properties[matchExpressions].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[podAffinity].properties[requiredDuringSchedulingIgnoredDuringExecution].items.properties[labelSelector].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[podAffinity].properties[requiredDuringSchedulingIgnoredDuringExecution].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[podAffinity].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[podAntiAffinity].properties[preferredDuringSchedulingIgnoredDuringExecution].items.properties[podAffinityTerm].properties[labelSelector].properties[matchExpressions].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[podAntiAffinity].properties[preferredDuringSchedulingIgnoredDuringExecution].items.properties[podAffinityTerm].properties[labelSelector].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[podAntiAffinity].properties[preferredDuringSchedulingIgnoredDuringExecution].items.properties[podAffinityTerm].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[podAntiAffinity].properties[preferredDuringSchedulingIgnoredDuringExecution].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[podAntiAffinity].properties[requiredDuringSchedulingIgnoredDuringExecution].items.properties[labelSelector].properties[matchExpressions].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[podAntiAffinity].properties[requiredDuringSchedulingIgnoredDuringExecution].items.properties[labelSelector].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[podAntiAffinity].properties[requiredDuringSchedulingIgnoredDuringExecution].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].properties[podAntiAffinity].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[affinity].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[envFrom].items.properties[configMapRef].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[envFrom].items.properties[secretRef].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[envFrom].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[env].items.properties[valueFrom].properties[configMapKeyRef].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[env].items.properties[valueFrom].properties[fieldRef].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[env].items.properties[valueFrom].properties[resourceFieldRef].properties[divisor].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[env].items.properties[valueFrom].properties[resourceFieldRef].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[env].items.properties[valueFrom].properties[secretKeyRef].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[env].items.properties[valueFrom].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[env].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[postStart].properties[exec].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[postStart].properties[httpGet].properties[httpHeaders].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[postStart].properties[httpGet].properties[port].anyOf[0].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[postStart].properties[httpGet].properties[port].anyOf[1].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[postStart].properties[httpGet].properties[port].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[postStart].properties[httpGet].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[postStart].properties[tcpSocket].properties[port].anyOf[0].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[postStart].properties[tcpSocket].properties[port].anyOf[1].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[postStart].properties[tcpSocket].properties[port].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[postStart].properties[tcpSocket].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[postStart].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[preStop].properties[exec].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[preStop].properties[httpGet].properties[httpHeaders].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[preStop].properties[httpGet].properties[port].anyOf[0].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[preStop].properties[httpGet].properties[port].anyOf[1].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[preStop].properties[httpGet].properties[port].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[preStop].properties[httpGet].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[preStop].properties[tcpSocket].properties[port].anyOf[0].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[preStop].properties[tcpSocket].properties[port].anyOf[1].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[preStop].properties[tcpSocket].properties[port].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[preStop].properties[tcpSocket].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[preStop].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[livenessProbe].properties[exec].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[livenessProbe].properties[httpGet].properties[httpHeaders].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[livenessProbe].properties[httpGet].properties[port].anyOf[0].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[livenessProbe].properties[httpGet].properties[port].anyOf[1].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[livenessProbe].properties[httpGet].properties[port].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[livenessProbe].properties[httpGet].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[livenessProbe].properties[tcpSocket].properties[port].anyOf[0].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[livenessProbe].properties[tcpSocket].properties[port].anyOf[1].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[livenessProbe].properties[tcpSocket].properties[port].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[livenessProbe].properties[tcpSocket].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[livenessProbe].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[ports].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[readinessProbe].properties[exec].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[readinessProbe].properties[httpGet].properties[httpHeaders].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[readinessProbe].properties[httpGet].properties[port].anyOf[0].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[readinessProbe].properties[httpGet].properties[port].anyOf[1].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[readinessProbe].properties[httpGet].properties[port].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[readinessProbe].properties[httpGet].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[readinessProbe].properties[tcpSocket].properties[port].anyOf[0].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[readinessProbe].properties[tcpSocket].properties[port].anyOf[1].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[readinessProbe].properties[tcpSocket].properties[port].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[readinessProbe].properties[tcpSocket].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[readinessProbe].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[resources].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[securityContext].properties[capabilities].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[securityContext].properties[seLinuxOptions].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[securityContext].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[volumeDevices].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[volumeMounts].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[imagePullSecrets].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[podMetadata].properties[initializers].properties[pending].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[podMetadata].properties[initializers].properties[result].properties[details].properties[causes].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[podMetadata].properties[initializers].properties[result].properties[details].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[podMetadata].properties[initializers].properties[result].properties[metadata].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[podMetadata].properties[initializers].properties[result].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[podMetadata].properties[initializers].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[podMetadata].properties[ownerReferences].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[podMetadata].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[resources].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[securityContext].properties[seLinuxOptions].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[securityContext].properties[sysctls].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[securityContext].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[storage].properties[emptyDir].properties[sizeLimit].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[storage].properties[emptyDir].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[storage].properties[volumeClaimTemplate].properties[metadata].properties[initializers].properties[pending].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[storage].properties[volumeClaimTemplate].properties[metadata].properties[initializers].properties[result].properties[details].properties[causes].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[storage].properties[volumeClaimTemplate].properties[metadata].properties[initializers].properties[result].properties[details].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[storage].properties[volumeClaimTemplate].properties[metadata].properties[initializers].properties[result].properties[metadata].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[storage].properties[volumeClaimTemplate].properties[metadata].properties[initializers].properties[result].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[storage].properties[volumeClaimTemplate].properties[metadata].properties[initializers].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[storage].properties[volumeClaimTemplate].properties[metadata].properties[ownerReferences].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[storage].properties[volumeClaimTemplate].properties[metadata].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[storage].properties[volumeClaimTemplate].properties[spec].properties[dataSource].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[storage].properties[volumeClaimTemplate].properties[spec].properties[resources].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[storage].properties[volumeClaimTemplate].properties[spec].properties[selector].properties[matchExpressions].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[storage].properties[volumeClaimTemplate].properties[spec].properties[selector].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[storage].properties[volumeClaimTemplate].properties[spec].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[storage].properties[volumeClaimTemplate].properties[status].properties[conditions].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].properties[storage].properties[volumeClaimTemplate].properties[status].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[storage].properties[volumeClaimTemplate].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[storage].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[tolerations].items.type: Required value: must not be empty for specified array items, spec.validation.openAPIV3Schema.properties[spec].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[status].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.type: Required value: must not be empty at the root]" } ], "acceptedNames": { "plural": "alertmanagers", "singular": "alertmanager", "kind": "Alertmanager", "listKind": "AlertmanagerList" }, "storedVersions": [ "v1" ] } } ================================================ FILE: pkg/backup/actions/testdata/v1beta1/elasticsearches.elasticsearch.k8s.elastic.co.json ================================================ { "kind": "CustomResourceDefinition", "apiVersion": "apiextensions.k8s.io/v1beta1", "metadata": { "name": "elasticsearches.elasticsearch.k8s.elastic.co", "selfLink": "/apis/apiextensions.k8s.io/v1beta1/customresourcedefinitions/elasticsearches.elasticsearch.k8s.elastic.co", "uid": "e8596856-29ae-47e4-8b14-5f7f027adf4a", "resourceVersion": "1703536", "generation": 1, "creationTimestamp": "2020-04-28T23:31:51Z", "labels": { "velero.io/backup-name": "es", "velero.io/restore-name": "es-crds" }, "annotations": { "controller-gen.kubebuilder.io/version": "v0.2.5", "kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"apiextensions.k8s.io/v1beta1\",\"kind\":\"CustomResourceDefinition\",\"metadata\":{\"annotations\":{\"controller-gen.kubebuilder.io/version\":\"v0.2.5\"},\"creationTimestamp\":null,\"name\":\"elasticsearches.elasticsearch.k8s.elastic.co\"},\"spec\":{\"additionalPrinterColumns\":[{\"JSONPath\":\".status.health\",\"name\":\"health\",\"type\":\"string\"},{\"JSONPath\":\".status.availableNodes\",\"description\":\"Available nodes\",\"name\":\"nodes\",\"type\":\"integer\"},{\"JSONPath\":\".spec.version\",\"description\":\"Elasticsearch version\",\"name\":\"version\",\"type\":\"string\"},{\"JSONPath\":\".status.phase\",\"name\":\"phase\",\"type\":\"string\"},{\"JSONPath\":\".metadata.creationTimestamp\",\"name\":\"age\",\"type\":\"date\"}],\"group\":\"elasticsearch.k8s.elastic.co\",\"names\":{\"categories\":[\"elastic\"],\"kind\":\"Elasticsearch\",\"listKind\":\"ElasticsearchList\",\"plural\":\"elasticsearches\",\"shortNames\":[\"es\"],\"singular\":\"elasticsearch\"},\"scope\":\"Namespaced\",\"subresources\":{\"status\":{}},\"validation\":{\"openAPIV3Schema\":{\"description\":\"Elasticsearch represents an Elasticsearch resource in a Kubernetes cluster.\",\"properties\":{\"apiVersion\":{\"description\":\"APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\",\"type\":\"string\"},\"kind\":{\"description\":\"Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\",\"type\":\"string\"},\"metadata\":{\"type\":\"object\"},\"spec\":{\"description\":\"ElasticsearchSpec holds the specification of an Elasticsearch cluster.\",\"properties\":{\"auth\":{\"description\":\"Auth contains user authentication and authorization security settings for Elasticsearch.\",\"properties\":{\"fileRealm\":{\"description\":\"FileRealm to propagate to the Elasticsearch cluster.\",\"items\":{\"description\":\"FileRealmSource references users to create in the Elasticsearch cluster.\",\"properties\":{\"secretName\":{\"description\":\"SecretName is the name of the secret.\",\"type\":\"string\"}},\"type\":\"object\"},\"type\":\"array\"},\"roles\":{\"description\":\"Roles to propagate to the Elasticsearch cluster.\",\"items\":{\"description\":\"RoleSource references roles to create in the Elasticsearch cluster.\",\"properties\":{\"secretName\":{\"description\":\"SecretName is the name of the secret.\",\"type\":\"string\"}},\"type\":\"object\"},\"type\":\"array\"}},\"type\":\"object\"},\"http\":{\"description\":\"HTTP holds HTTP layer settings for Elasticsearch.\",\"properties\":{\"service\":{\"description\":\"Service defines the template for the associated Kubernetes Service object.\",\"properties\":{\"metadata\":{\"description\":\"ObjectMeta is the metadata of the service. The name and namespace provided here are managed by ECK and will be ignored.\",\"type\":\"object\"},\"spec\":{\"description\":\"Spec is the specification of the service.\",\"properties\":{\"clusterIP\":{\"description\":\"clusterIP is the IP address of the service and is usually assigned randomly by the master. If an address is specified manually and is not in use by others, it will be allocated to the service; otherwise, creation of the service will fail. This field can not be changed through updates. Valid values are \\\"None\\\", empty string (\\\"\\\"), or a valid IP address. \\\"None\\\" can be specified for headless services when proxying is not required. Only applies to types ClusterIP, NodePort, and LoadBalancer. Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies\",\"type\":\"string\"},\"externalIPs\":{\"description\":\"externalIPs is a list of IP addresses for which nodes in the cluster will also accept traffic for this service. These IPs are not managed by Kubernetes. The user is responsible for ensuring that traffic arrives at a node with this IP. A common example is external load-balancers that are not part of the Kubernetes system.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"externalName\":{\"description\":\"externalName is the external reference that kubedns or equivalent will return as a CNAME record for this service. No proxying will be involved. Must be a valid RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) and requires Type to be ExternalName.\",\"type\":\"string\"},\"externalTrafficPolicy\":{\"description\":\"externalTrafficPolicy denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints. \\\"Local\\\" preserves the client source IP and avoids a second hop for LoadBalancer and Nodeport type services, but risks potentially imbalanced traffic spreading. \\\"Cluster\\\" obscures the client source IP and may cause a second hop to another node, but should have good overall load-spreading.\",\"type\":\"string\"},\"healthCheckNodePort\":{\"description\":\"healthCheckNodePort specifies the healthcheck nodePort for the service. If not specified, HealthCheckNodePort is created by the service api backend with the allocated nodePort. Will use user-specified nodePort value if specified by the client. Only effects when Type is set to LoadBalancer and ExternalTrafficPolicy is set to Local.\",\"format\":\"int32\",\"type\":\"integer\"},\"ipFamily\":{\"description\":\"ipFamily specifies whether this Service has a preference for a particular IP family (e.g. IPv4 vs. IPv6). If a specific IP family is requested, the clusterIP field will be allocated from that family, if it is available in the cluster. If no IP family is requested, the cluster's primary IP family will be used. Other IP fields (loadBalancerIP, loadBalancerSourceRanges, externalIPs) and controllers which allocate external load-balancers should use the same IP family. Endpoints for this Service will be of this family. This field is immutable after creation. Assigning a ServiceIPFamily not available in the cluster (e.g. IPv6 in IPv4 only cluster) is an error condition and will fail during clusterIP assignment.\",\"type\":\"string\"},\"loadBalancerIP\":{\"description\":\"Only applies to Service Type: LoadBalancer LoadBalancer will get created with the IP specified in this field. This feature depends on whether the underlying cloud-provider supports specifying the loadBalancerIP when a load balancer is created. This field will be ignored if the cloud-provider does not support the feature.\",\"type\":\"string\"},\"loadBalancerSourceRanges\":{\"description\":\"If specified and supported by the platform, this will restrict traffic through the cloud-provider load-balancer will be restricted to the specified client IPs. This field will be ignored if the cloud-provider does not support the feature.\\\" More info: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"ports\":{\"description\":\"The list of ports that are exposed by this service. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies\",\"items\":{\"description\":\"ServicePort contains information on service's port.\",\"properties\":{\"name\":{\"description\":\"The name of this port within the service. This must be a DNS_LABEL. All ports within a ServiceSpec must have unique names. When considering the endpoints for a Service, this must match the 'name' field in the EndpointPort. Optional if only one ServicePort is defined on this service.\",\"type\":\"string\"},\"nodePort\":{\"description\":\"The port on each node on which this service is exposed when type=NodePort or LoadBalancer. Usually assigned by the system. If specified, it will be allocated to the service if unused or else creation of the service will fail. Default is to auto-allocate a port if the ServiceType of this Service requires one. More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport\",\"format\":\"int32\",\"type\":\"integer\"},\"port\":{\"description\":\"The port that will be exposed by this service.\",\"format\":\"int32\",\"type\":\"integer\"},\"protocol\":{\"description\":\"The IP protocol for this port. Supports \\\"TCP\\\", \\\"UDP\\\", and \\\"SCTP\\\". Default is TCP.\",\"type\":\"string\"},\"targetPort\":{\"anyOf\":[{\"type\":\"integer\"},{\"type\":\"string\"}],\"description\":\"Number or name of the port to access on the pods targeted by the service. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. If this is a string, it will be looked up as a named port in the target Pod's container ports. If this is not specified, the value of the 'port' field is used (an identity map). This field is ignored for services with clusterIP=None, and should be omitted or set equal to the 'port' field. More info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service\"}},\"required\":[\"port\"],\"type\":\"object\"},\"type\":\"array\"},\"publishNotReadyAddresses\":{\"description\":\"publishNotReadyAddresses, when set to true, indicates that DNS implementations must publish the notReadyAddresses of subsets for the Endpoints associated with the Service. The default value is false. The primary use case for setting this field is to use a StatefulSet's Headless Service to propagate SRV records for its Pods without respect to their readiness for purpose of peer discovery.\",\"type\":\"boolean\"},\"selector\":{\"additionalProperties\":{\"type\":\"string\"},\"description\":\"Route service traffic to pods with label keys and values matching this selector. If empty or not present, the service is assumed to have an external process managing its endpoints, which Kubernetes will not modify. Only applies to types ClusterIP, NodePort, and LoadBalancer. Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/\",\"type\":\"object\"},\"sessionAffinity\":{\"description\":\"Supports \\\"ClientIP\\\" and \\\"None\\\". Used to maintain session affinity. Enable client IP based session affinity. Must be ClientIP or None. Defaults to None. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies\",\"type\":\"string\"},\"sessionAffinityConfig\":{\"description\":\"sessionAffinityConfig contains the configurations of session affinity.\",\"properties\":{\"clientIP\":{\"description\":\"clientIP contains the configurations of Client IP based session affinity.\",\"properties\":{\"timeoutSeconds\":{\"description\":\"timeoutSeconds specifies the seconds of ClientIP type session sticky time. The value must be \\u003e0 \\u0026\\u0026 \\u003c=86400(for 1 day) if ServiceAffinity == \\\"ClientIP\\\". Default value is 10800(for 3 hours).\",\"format\":\"int32\",\"type\":\"integer\"}},\"type\":\"object\"}},\"type\":\"object\"},\"topologyKeys\":{\"description\":\"topologyKeys is a preference-order list of topology keys which implementations of services should use to preferentially sort endpoints when accessing this Service, it can not be used at the same time as externalTrafficPolicy=Local. Topology keys must be valid label keys and at most 16 keys may be specified. Endpoints are chosen based on the first topology key with available backends. If this field is specified and all entries have no backends that match the topology of the client, the service has no backends for that client and connections should fail. The special value \\\"*\\\" may be used to mean \\\"any topology\\\". This catch-all value, if used, only makes sense as the last value in the list. If this is not specified or empty, no topology constraints will be applied.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"type\":{\"description\":\"type determines how the Service is exposed. Defaults to ClusterIP. Valid options are ExternalName, ClusterIP, NodePort, and LoadBalancer. \\\"ExternalName\\\" maps to the specified externalName. \\\"ClusterIP\\\" allocates a cluster-internal IP address for load-balancing to endpoints. Endpoints are determined by the selector or if that is not specified, by manual construction of an Endpoints object. If clusterIP is \\\"None\\\", no virtual IP is allocated and the endpoints are published as a set of endpoints rather than a stable IP. \\\"NodePort\\\" builds on ClusterIP and allocates a port on every node which routes to the clusterIP. \\\"LoadBalancer\\\" builds on NodePort and creates an external load-balancer (if supported in the current cloud) which routes to the clusterIP. More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types\",\"type\":\"string\"}},\"type\":\"object\"}},\"type\":\"object\"},\"tls\":{\"description\":\"TLS defines options for configuring TLS for HTTP.\",\"properties\":{\"certificate\":{\"description\":\"Certificate is a reference to a Kubernetes secret that contains the certificate and private key for enabling TLS. The referenced secret should contain the following: \\n - `ca.crt`: The certificate authority (optional). - `tls.crt`: The certificate (or a chain). - `tls.key`: The private key to the first certificate in the certificate chain.\",\"properties\":{\"secretName\":{\"description\":\"SecretName is the name of the secret.\",\"type\":\"string\"}},\"type\":\"object\"},\"selfSignedCertificate\":{\"description\":\"SelfSignedCertificate allows configuring the self-signed certificate generated by the operator.\",\"properties\":{\"disabled\":{\"description\":\"Disabled indicates that the provisioning of the self-signed certificate should be disabled.\",\"type\":\"boolean\"},\"subjectAltNames\":{\"description\":\"SubjectAlternativeNames is a list of SANs to include in the generated HTTP TLS certificate.\",\"items\":{\"description\":\"SubjectAlternativeName represents a SAN entry in a x509 certificate.\",\"properties\":{\"dns\":{\"description\":\"DNS is the DNS name of the subject.\",\"type\":\"string\"},\"ip\":{\"description\":\"IP is the IP address of the subject.\",\"type\":\"string\"}},\"type\":\"object\"},\"type\":\"array\"}},\"type\":\"object\"}},\"type\":\"object\"}},\"type\":\"object\"},\"image\":{\"description\":\"Image is the Elasticsearch Docker image to deploy.\",\"type\":\"string\"},\"nodeSets\":{\"description\":\"NodeSets allow specifying groups of Elasticsearch nodes sharing the same configuration and Pod templates. See: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-orchestration.html\",\"items\":{\"description\":\"NodeSet is the specification for a group of Elasticsearch nodes sharing the same configuration and a Pod template.\",\"properties\":{\"config\":{\"description\":\"Config holds the Elasticsearch configuration.\",\"type\":\"object\"},\"count\":{\"description\":\"Count of Elasticsearch nodes to deploy.\",\"format\":\"int32\",\"minimum\":1,\"type\":\"integer\"},\"name\":{\"description\":\"Name of this set of nodes. Becomes a part of the Elasticsearch node.name setting.\",\"maxLength\":23,\"pattern\":\"[a-zA-Z0-9-]+\",\"type\":\"string\"},\"podTemplate\":{\"description\":\"PodTemplate provides customisation options (labels, annotations, affinity rules, resource requests, and so on) for the Pods belonging to this NodeSet.\",\"type\":\"object\"},\"volumeClaimTemplates\":{\"description\":\"VolumeClaimTemplates is a list of persistent volume claims to be used by each Pod in this NodeSet. Every claim in this list must have a matching volumeMount in one of the containers defined in the PodTemplate. Items defined here take precedence over any default claims added by the operator with the same name. See: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-volume-claim-templates.html\",\"items\":{\"description\":\"PersistentVolumeClaim is a user's request for and claim to a persistent volume\",\"properties\":{\"apiVersion\":{\"description\":\"APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\",\"type\":\"string\"},\"kind\":{\"description\":\"Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\",\"type\":\"string\"},\"metadata\":{\"description\":\"Standard object's metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata\",\"type\":\"object\"},\"spec\":{\"description\":\"Spec defines the desired characteristics of a volume requested by a pod author. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims\",\"properties\":{\"accessModes\":{\"description\":\"AccessModes contains the desired access modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"dataSource\":{\"description\":\"This field requires the VolumeSnapshotDataSource alpha feature gate to be enabled and currently VolumeSnapshot is the only supported data source. If the provisioner can support VolumeSnapshot data source, it will create a new volume and data will be restored to the volume at the same time. If the provisioner does not support VolumeSnapshot data source, volume will not be created and the failure will be reported as an event. In the future, we plan to support more data source types and the behavior of the provisioner may change.\",\"properties\":{\"apiGroup\":{\"description\":\"APIGroup is the group for the resource being referenced. If APIGroup is not specified, the specified Kind must be in the core API group. For any other third-party types, APIGroup is required.\",\"type\":\"string\"},\"kind\":{\"description\":\"Kind is the type of resource being referenced\",\"type\":\"string\"},\"name\":{\"description\":\"Name is the name of resource being referenced\",\"type\":\"string\"}},\"required\":[\"kind\",\"name\"],\"type\":\"object\"},\"resources\":{\"description\":\"Resources represents the minimum resources the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources\",\"properties\":{\"limits\":{\"additionalProperties\":{\"anyOf\":[{\"type\":\"integer\"},{\"type\":\"string\"}],\"pattern\":\"^(\\\\+|-)?(([0-9]+(\\\\.[0-9]*)?)|(\\\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\\\+|-)?(([0-9]+(\\\\.[0-9]*)?)|(\\\\.[0-9]+))))?$\"},\"description\":\"Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/\",\"type\":\"object\"},\"requests\":{\"additionalProperties\":{\"anyOf\":[{\"type\":\"integer\"},{\"type\":\"string\"}],\"pattern\":\"^(\\\\+|-)?(([0-9]+(\\\\.[0-9]*)?)|(\\\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\\\+|-)?(([0-9]+(\\\\.[0-9]*)?)|(\\\\.[0-9]+))))?$\"},\"description\":\"Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/\",\"type\":\"object\"}},\"type\":\"object\"},\"selector\":{\"description\":\"A label query over volumes to consider for binding.\",\"properties\":{\"matchExpressions\":{\"description\":\"matchExpressions is a list of label selector requirements. The requirements are ANDed.\",\"items\":{\"description\":\"A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"key is the label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\",\"type\":\"string\"},\"values\":{\"description\":\"values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"],\"type\":\"object\"},\"type\":\"array\"},\"matchLabels\":{\"additionalProperties\":{\"type\":\"string\"},\"description\":\"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \\\"key\\\", the operator is \\\"In\\\", and the values array contains only \\\"value\\\". The requirements are ANDed.\",\"type\":\"object\"}},\"type\":\"object\"},\"storageClassName\":{\"description\":\"Name of the StorageClass required by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1\",\"type\":\"string\"},\"volumeMode\":{\"description\":\"volumeMode defines what type of volume is required by the claim. Value of Filesystem is implied when not included in claim spec. This is a beta feature.\",\"type\":\"string\"},\"volumeName\":{\"description\":\"VolumeName is the binding reference to the PersistentVolume backing this claim.\",\"type\":\"string\"}},\"type\":\"object\"},\"status\":{\"description\":\"Status represents the current information/status of a persistent volume claim. Read-only. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims\",\"properties\":{\"accessModes\":{\"description\":\"AccessModes contains the actual access modes the volume backing the PVC has. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"capacity\":{\"additionalProperties\":{\"anyOf\":[{\"type\":\"integer\"},{\"type\":\"string\"}],\"pattern\":\"^(\\\\+|-)?(([0-9]+(\\\\.[0-9]*)?)|(\\\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\\\+|-)?(([0-9]+(\\\\.[0-9]*)?)|(\\\\.[0-9]+))))?$\"},\"description\":\"Represents the actual resources of the underlying volume.\",\"type\":\"object\"},\"conditions\":{\"description\":\"Current Condition of persistent volume claim. If underlying persistent volume is being resized then the Condition will be set to 'ResizeStarted'.\",\"items\":{\"description\":\"PersistentVolumeClaimCondition contains details about state of pvc\",\"properties\":{\"lastProbeTime\":{\"description\":\"Last time we probed the condition.\",\"format\":\"date-time\",\"type\":\"string\"},\"lastTransitionTime\":{\"description\":\"Last time the condition transitioned from one status to another.\",\"format\":\"date-time\",\"type\":\"string\"},\"message\":{\"description\":\"Human-readable message indicating details about last transition.\",\"type\":\"string\"},\"reason\":{\"description\":\"Unique, this should be a short, machine understandable string that gives the reason for condition's last transition. If it reports \\\"ResizeStarted\\\" that means the underlying persistent volume is being resized.\",\"type\":\"string\"},\"status\":{\"type\":\"string\"},\"type\":{\"description\":\"PersistentVolumeClaimConditionType is a valid value of PersistentVolumeClaimCondition.Type\",\"type\":\"string\"}},\"required\":[\"status\",\"type\"],\"type\":\"object\"},\"type\":\"array\"},\"phase\":{\"description\":\"Phase represents the current phase of PersistentVolumeClaim.\",\"type\":\"string\"}},\"type\":\"object\"}},\"type\":\"object\"},\"type\":\"array\"}},\"required\":[\"count\",\"name\"],\"type\":\"object\"},\"minItems\":1,\"type\":\"array\"},\"podDisruptionBudget\":{\"description\":\"PodDisruptionBudget provides access to the default pod disruption budget for the Elasticsearch cluster. The default budget selects all cluster pods and sets `maxUnavailable` to 1. To disable, set `PodDisruptionBudget` to the empty value (`{}` in YAML).\",\"properties\":{\"metadata\":{\"description\":\"ObjectMeta is the metadata of the PDB. The name and namespace provided here are managed by ECK and will be ignored.\",\"type\":\"object\"},\"spec\":{\"description\":\"Spec is the specification of the PDB.\",\"properties\":{\"maxUnavailable\":{\"anyOf\":[{\"type\":\"integer\"},{\"type\":\"string\"}],\"description\":\"An eviction is allowed if at most \\\"maxUnavailable\\\" pods selected by \\\"selector\\\" are unavailable after the eviction, i.e. even in absence of the evicted pod. For example, one can prevent all voluntary evictions by specifying 0. This is a mutually exclusive setting with \\\"minAvailable\\\".\"},\"minAvailable\":{\"anyOf\":[{\"type\":\"integer\"},{\"type\":\"string\"}],\"description\":\"An eviction is allowed if at least \\\"minAvailable\\\" pods selected by \\\"selector\\\" will still be available after the eviction, i.e. even in the absence of the evicted pod. So for example you can prevent all voluntary evictions by specifying \\\"100%\\\".\"},\"selector\":{\"description\":\"Label query over pods whose evictions are managed by the disruption budget.\",\"properties\":{\"matchExpressions\":{\"description\":\"matchExpressions is a list of label selector requirements. The requirements are ANDed.\",\"items\":{\"description\":\"A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"key is the label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\",\"type\":\"string\"},\"values\":{\"description\":\"values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"],\"type\":\"object\"},\"type\":\"array\"},\"matchLabels\":{\"additionalProperties\":{\"type\":\"string\"},\"description\":\"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \\\"key\\\", the operator is \\\"In\\\", and the values array contains only \\\"value\\\". The requirements are ANDed.\",\"type\":\"object\"}},\"type\":\"object\"}},\"type\":\"object\"}},\"type\":\"object\"},\"remoteClusters\":{\"description\":\"RemoteClusters enables you to establish uni-directional connections to a remote Elasticsearch cluster.\",\"items\":{\"description\":\"RemoteCluster declares a remote Elasticsearch cluster connection.\",\"properties\":{\"elasticsearchRef\":{\"description\":\"ElasticsearchRef is a reference to an Elasticsearch cluster running within the same k8s cluster.\",\"properties\":{\"name\":{\"description\":\"Name of the Kubernetes object.\",\"type\":\"string\"},\"namespace\":{\"description\":\"Namespace of the Kubernetes object. If empty, defaults to the current namespace.\",\"type\":\"string\"}},\"required\":[\"name\"],\"type\":\"object\"},\"name\":{\"description\":\"Name is the name of the remote cluster as it is set in the Elasticsearch settings. The name is expected to be unique for each remote clusters.\",\"minLength\":1,\"type\":\"string\"}},\"required\":[\"name\"],\"type\":\"object\"},\"type\":\"array\"},\"secureSettings\":{\"description\":\"SecureSettings is a list of references to Kubernetes secrets containing sensitive configuration options for Elasticsearch. See: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-es-secure-settings.html\",\"items\":{\"description\":\"SecretSource defines a data source based on a Kubernetes Secret.\",\"properties\":{\"entries\":{\"description\":\"Entries define how to project each key-value pair in the secret to filesystem paths. If not defined, all keys will be projected to similarly named paths in the filesystem. If defined, only the specified keys will be projected to the corresponding paths.\",\"items\":{\"description\":\"KeyToPath defines how to map a key in a Secret object to a filesystem path.\",\"properties\":{\"key\":{\"description\":\"Key is the key contained in the secret.\",\"type\":\"string\"},\"path\":{\"description\":\"Path is the relative file path to map the key to. Path must not be an absolute file path and must not contain any \\\"..\\\" components.\",\"type\":\"string\"}},\"required\":[\"key\"],\"type\":\"object\"},\"type\":\"array\"},\"secretName\":{\"description\":\"SecretName is the name of the secret.\",\"type\":\"string\"}},\"required\":[\"secretName\"],\"type\":\"object\"},\"type\":\"array\"},\"serviceAccountName\":{\"description\":\"ServiceAccountName is used to check access from the current resource to a resource (eg. a remote Elasticsearch cluster) in a different namespace. Can only be used if ECK is enforcing RBAC on references.\",\"type\":\"string\"},\"transport\":{\"description\":\"Transport holds transport layer settings for Elasticsearch.\",\"properties\":{\"service\":{\"description\":\"Service defines the template for the associated Kubernetes Service object.\",\"properties\":{\"metadata\":{\"description\":\"ObjectMeta is the metadata of the service. The name and namespace provided here are managed by ECK and will be ignored.\",\"type\":\"object\"},\"spec\":{\"description\":\"Spec is the specification of the service.\",\"properties\":{\"clusterIP\":{\"description\":\"clusterIP is the IP address of the service and is usually assigned randomly by the master. If an address is specified manually and is not in use by others, it will be allocated to the service; otherwise, creation of the service will fail. This field can not be changed through updates. Valid values are \\\"None\\\", empty string (\\\"\\\"), or a valid IP address. \\\"None\\\" can be specified for headless services when proxying is not required. Only applies to types ClusterIP, NodePort, and LoadBalancer. Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies\",\"type\":\"string\"},\"externalIPs\":{\"description\":\"externalIPs is a list of IP addresses for which nodes in the cluster will also accept traffic for this service. These IPs are not managed by Kubernetes. The user is responsible for ensuring that traffic arrives at a node with this IP. A common example is external load-balancers that are not part of the Kubernetes system.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"externalName\":{\"description\":\"externalName is the external reference that kubedns or equivalent will return as a CNAME record for this service. No proxying will be involved. Must be a valid RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) and requires Type to be ExternalName.\",\"type\":\"string\"},\"externalTrafficPolicy\":{\"description\":\"externalTrafficPolicy denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints. \\\"Local\\\" preserves the client source IP and avoids a second hop for LoadBalancer and Nodeport type services, but risks potentially imbalanced traffic spreading. \\\"Cluster\\\" obscures the client source IP and may cause a second hop to another node, but should have good overall load-spreading.\",\"type\":\"string\"},\"healthCheckNodePort\":{\"description\":\"healthCheckNodePort specifies the healthcheck nodePort for the service. If not specified, HealthCheckNodePort is created by the service api backend with the allocated nodePort. Will use user-specified nodePort value if specified by the client. Only effects when Type is set to LoadBalancer and ExternalTrafficPolicy is set to Local.\",\"format\":\"int32\",\"type\":\"integer\"},\"ipFamily\":{\"description\":\"ipFamily specifies whether this Service has a preference for a particular IP family (e.g. IPv4 vs. IPv6). If a specific IP family is requested, the clusterIP field will be allocated from that family, if it is available in the cluster. If no IP family is requested, the cluster's primary IP family will be used. Other IP fields (loadBalancerIP, loadBalancerSourceRanges, externalIPs) and controllers which allocate external load-balancers should use the same IP family. Endpoints for this Service will be of this family. This field is immutable after creation. Assigning a ServiceIPFamily not available in the cluster (e.g. IPv6 in IPv4 only cluster) is an error condition and will fail during clusterIP assignment.\",\"type\":\"string\"},\"loadBalancerIP\":{\"description\":\"Only applies to Service Type: LoadBalancer LoadBalancer will get created with the IP specified in this field. This feature depends on whether the underlying cloud-provider supports specifying the loadBalancerIP when a load balancer is created. This field will be ignored if the cloud-provider does not support the feature.\",\"type\":\"string\"},\"loadBalancerSourceRanges\":{\"description\":\"If specified and supported by the platform, this will restrict traffic through the cloud-provider load-balancer will be restricted to the specified client IPs. This field will be ignored if the cloud-provider does not support the feature.\\\" More info: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"ports\":{\"description\":\"The list of ports that are exposed by this service. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies\",\"items\":{\"description\":\"ServicePort contains information on service's port.\",\"properties\":{\"name\":{\"description\":\"The name of this port within the service. This must be a DNS_LABEL. All ports within a ServiceSpec must have unique names. When considering the endpoints for a Service, this must match the 'name' field in the EndpointPort. Optional if only one ServicePort is defined on this service.\",\"type\":\"string\"},\"nodePort\":{\"description\":\"The port on each node on which this service is exposed when type=NodePort or LoadBalancer. Usually assigned by the system. If specified, it will be allocated to the service if unused or else creation of the service will fail. Default is to auto-allocate a port if the ServiceType of this Service requires one. More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport\",\"format\":\"int32\",\"type\":\"integer\"},\"port\":{\"description\":\"The port that will be exposed by this service.\",\"format\":\"int32\",\"type\":\"integer\"},\"protocol\":{\"description\":\"The IP protocol for this port. Supports \\\"TCP\\\", \\\"UDP\\\", and \\\"SCTP\\\". Default is TCP.\",\"type\":\"string\"},\"targetPort\":{\"anyOf\":[{\"type\":\"integer\"},{\"type\":\"string\"}],\"description\":\"Number or name of the port to access on the pods targeted by the service. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. If this is a string, it will be looked up as a named port in the target Pod's container ports. If this is not specified, the value of the 'port' field is used (an identity map). This field is ignored for services with clusterIP=None, and should be omitted or set equal to the 'port' field. More info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service\"}},\"required\":[\"port\"],\"type\":\"object\"},\"type\":\"array\"},\"publishNotReadyAddresses\":{\"description\":\"publishNotReadyAddresses, when set to true, indicates that DNS implementations must publish the notReadyAddresses of subsets for the Endpoints associated with the Service. The default value is false. The primary use case for setting this field is to use a StatefulSet's Headless Service to propagate SRV records for its Pods without respect to their readiness for purpose of peer discovery.\",\"type\":\"boolean\"},\"selector\":{\"additionalProperties\":{\"type\":\"string\"},\"description\":\"Route service traffic to pods with label keys and values matching this selector. If empty or not present, the service is assumed to have an external process managing its endpoints, which Kubernetes will not modify. Only applies to types ClusterIP, NodePort, and LoadBalancer. Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/\",\"type\":\"object\"},\"sessionAffinity\":{\"description\":\"Supports \\\"ClientIP\\\" and \\\"None\\\". Used to maintain session affinity. Enable client IP based session affinity. Must be ClientIP or None. Defaults to None. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies\",\"type\":\"string\"},\"sessionAffinityConfig\":{\"description\":\"sessionAffinityConfig contains the configurations of session affinity.\",\"properties\":{\"clientIP\":{\"description\":\"clientIP contains the configurations of Client IP based session affinity.\",\"properties\":{\"timeoutSeconds\":{\"description\":\"timeoutSeconds specifies the seconds of ClientIP type session sticky time. The value must be \\u003e0 \\u0026\\u0026 \\u003c=86400(for 1 day) if ServiceAffinity == \\\"ClientIP\\\". Default value is 10800(for 3 hours).\",\"format\":\"int32\",\"type\":\"integer\"}},\"type\":\"object\"}},\"type\":\"object\"},\"topologyKeys\":{\"description\":\"topologyKeys is a preference-order list of topology keys which implementations of services should use to preferentially sort endpoints when accessing this Service, it can not be used at the same time as externalTrafficPolicy=Local. Topology keys must be valid label keys and at most 16 keys may be specified. Endpoints are chosen based on the first topology key with available backends. If this field is specified and all entries have no backends that match the topology of the client, the service has no backends for that client and connections should fail. The special value \\\"*\\\" may be used to mean \\\"any topology\\\". This catch-all value, if used, only makes sense as the last value in the list. If this is not specified or empty, no topology constraints will be applied.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"type\":{\"description\":\"type determines how the Service is exposed. Defaults to ClusterIP. Valid options are ExternalName, ClusterIP, NodePort, and LoadBalancer. \\\"ExternalName\\\" maps to the specified externalName. \\\"ClusterIP\\\" allocates a cluster-internal IP address for load-balancing to endpoints. Endpoints are determined by the selector or if that is not specified, by manual construction of an Endpoints object. If clusterIP is \\\"None\\\", no virtual IP is allocated and the endpoints are published as a set of endpoints rather than a stable IP. \\\"NodePort\\\" builds on ClusterIP and allocates a port on every node which routes to the clusterIP. \\\"LoadBalancer\\\" builds on NodePort and creates an external load-balancer (if supported in the current cloud) which routes to the clusterIP. More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types\",\"type\":\"string\"}},\"type\":\"object\"}},\"type\":\"object\"}},\"type\":\"object\"},\"updateStrategy\":{\"description\":\"UpdateStrategy specifies how updates to the cluster should be performed.\",\"properties\":{\"changeBudget\":{\"description\":\"ChangeBudget defines the constraints to consider when applying changes to the Elasticsearch cluster.\",\"properties\":{\"maxSurge\":{\"description\":\"MaxSurge is the maximum number of new pods that can be created exceeding the original number of pods defined in the specification. MaxSurge is only taken into consideration when scaling up. Setting a negative value will disable the restriction. Defaults to unbounded if not specified.\",\"format\":\"int32\",\"type\":\"integer\"},\"maxUnavailable\":{\"description\":\"MaxUnavailable is the maximum number of pods that can be unavailable (not ready) during the update due to circumstances under the control of the operator. Setting a negative value will disable this restriction. Defaults to 1 if not specified.\",\"format\":\"int32\",\"type\":\"integer\"}},\"type\":\"object\"}},\"type\":\"object\"},\"version\":{\"description\":\"Version of Elasticsearch.\",\"type\":\"string\"}},\"required\":[\"nodeSets\",\"version\"],\"type\":\"object\"},\"status\":{\"description\":\"ElasticsearchStatus defines the observed state of Elasticsearch\",\"properties\":{\"availableNodes\":{\"format\":\"int32\",\"type\":\"integer\"},\"health\":{\"description\":\"ElasticsearchHealth is the health of the cluster as returned by the health API.\",\"type\":\"string\"},\"phase\":{\"description\":\"ElasticsearchOrchestrationPhase is the phase Elasticsearch is in from the controller point of view.\",\"type\":\"string\"}},\"type\":\"object\"}}}},\"version\":\"v1\",\"versions\":[{\"name\":\"v1\",\"served\":true,\"storage\":true},{\"name\":\"v1beta1\",\"served\":true,\"storage\":false},{\"name\":\"v1alpha1\",\"served\":false,\"storage\":false}]},\"status\":{\"acceptedNames\":{\"kind\":\"\",\"plural\":\"\"},\"conditions\":[],\"storedVersions\":[]}}\n" } }, "spec": { "group": "elasticsearch.k8s.elastic.co", "version": "v1", "names": { "plural": "elasticsearches", "singular": "elasticsearch", "shortNames": [ "es" ], "kind": "Elasticsearch", "listKind": "ElasticsearchList", "categories": [ "elastic" ] }, "scope": "Namespaced", "validation": { "openAPIV3Schema": { "description": "Elasticsearch represents an Elasticsearch resource in a Kubernetes cluster.", "properties": { "apiVersion": { "description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", "type": "string" }, "kind": { "description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", "type": "string" }, "metadata": { "type": "object" }, "spec": { "description": "ElasticsearchSpec holds the specification of an Elasticsearch cluster.", "type": "object", "required": [ "nodeSets", "version" ], "properties": { "auth": { "description": "Auth contains user authentication and authorization security settings for Elasticsearch.", "type": "object", "properties": { "fileRealm": { "description": "FileRealm to propagate to the Elasticsearch cluster.", "type": "array", "items": { "description": "FileRealmSource references users to create in the Elasticsearch cluster.", "type": "object", "properties": { "secretName": { "description": "SecretName is the name of the secret.", "type": "string" } } } }, "roles": { "description": "Roles to propagate to the Elasticsearch cluster.", "type": "array", "items": { "description": "RoleSource references roles to create in the Elasticsearch cluster.", "type": "object", "properties": { "secretName": { "description": "SecretName is the name of the secret.", "type": "string" } } } } } }, "http": { "description": "HTTP holds HTTP layer settings for Elasticsearch.", "type": "object", "properties": { "service": { "description": "Service defines the template for the associated Kubernetes Service object.", "type": "object", "properties": { "metadata": { "description": "ObjectMeta is the metadata of the service. The name and namespace provided here are managed by ECK and will be ignored.", "type": "object" }, "spec": { "description": "Spec is the specification of the service.", "type": "object", "properties": { "clusterIP": { "description": "clusterIP is the IP address of the service and is usually assigned randomly by the master. If an address is specified manually and is not in use by others, it will be allocated to the service; otherwise, creation of the service will fail. This field can not be changed through updates. Valid values are \"None\", empty string (\"\"), or a valid IP address. \"None\" can be specified for headless services when proxying is not required. Only applies to types ClusterIP, NodePort, and LoadBalancer. Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", "type": "string" }, "externalIPs": { "description": "externalIPs is a list of IP addresses for which nodes in the cluster will also accept traffic for this service. These IPs are not managed by Kubernetes. The user is responsible for ensuring that traffic arrives at a node with this IP. A common example is external load-balancers that are not part of the Kubernetes system.", "type": "array", "items": { "type": "string" } }, "externalName": { "description": "externalName is the external reference that kubedns or equivalent will return as a CNAME record for this service. No proxying will be involved. Must be a valid RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) and requires Type to be ExternalName.", "type": "string" }, "externalTrafficPolicy": { "description": "externalTrafficPolicy denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints. \"Local\" preserves the client source IP and avoids a second hop for LoadBalancer and Nodeport type services, but risks potentially imbalanced traffic spreading. \"Cluster\" obscures the client source IP and may cause a second hop to another node, but should have good overall load-spreading.", "type": "string" }, "healthCheckNodePort": { "description": "healthCheckNodePort specifies the healthcheck nodePort for the service. If not specified, HealthCheckNodePort is created by the service api backend with the allocated nodePort. Will use user-specified nodePort value if specified by the client. Only effects when Type is set to LoadBalancer and ExternalTrafficPolicy is set to Local.", "type": "integer", "format": "int32" }, "ipFamily": { "description": "ipFamily specifies whether this Service has a preference for a particular IP family (e.g. IPv4 vs. IPv6). If a specific IP family is requested, the clusterIP field will be allocated from that family, if it is available in the cluster. If no IP family is requested, the cluster's primary IP family will be used. Other IP fields (loadBalancerIP, loadBalancerSourceRanges, externalIPs) and controllers which allocate external load-balancers should use the same IP family. Endpoints for this Service will be of this family. This field is immutable after creation. Assigning a ServiceIPFamily not available in the cluster (e.g. IPv6 in IPv4 only cluster) is an error condition and will fail during clusterIP assignment.", "type": "string" }, "loadBalancerIP": { "description": "Only applies to Service Type: LoadBalancer LoadBalancer will get created with the IP specified in this field. This feature depends on whether the underlying cloud-provider supports specifying the loadBalancerIP when a load balancer is created. This field will be ignored if the cloud-provider does not support the feature.", "type": "string" }, "loadBalancerSourceRanges": { "description": "If specified and supported by the platform, this will restrict traffic through the cloud-provider load-balancer will be restricted to the specified client IPs. This field will be ignored if the cloud-provider does not support the feature.\" More info: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/", "type": "array", "items": { "type": "string" } }, "ports": { "description": "The list of ports that are exposed by this service. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", "type": "array", "items": { "description": "ServicePort contains information on service's port.", "type": "object", "required": [ "port" ], "properties": { "name": { "description": "The name of this port within the service. This must be a DNS_LABEL. All ports within a ServiceSpec must have unique names. When considering the endpoints for a Service, this must match the 'name' field in the EndpointPort. Optional if only one ServicePort is defined on this service.", "type": "string" }, "nodePort": { "description": "The port on each node on which this service is exposed when type=NodePort or LoadBalancer. Usually assigned by the system. If specified, it will be allocated to the service if unused or else creation of the service will fail. Default is to auto-allocate a port if the ServiceType of this Service requires one. More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport", "type": "integer", "format": "int32" }, "port": { "description": "The port that will be exposed by this service.", "type": "integer", "format": "int32" }, "protocol": { "description": "The IP protocol for this port. Supports \"TCP\", \"UDP\", and \"SCTP\". Default is TCP.", "type": "string" }, "targetPort": { "description": "Number or name of the port to access on the pods targeted by the service. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. If this is a string, it will be looked up as a named port in the target Pod's container ports. If this is not specified, the value of the 'port' field is used (an identity map). This field is ignored for services with clusterIP=None, and should be omitted or set equal to the 'port' field. More info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service", "anyOf": [ { "type": "integer" }, { "type": "string" } ] } } } }, "publishNotReadyAddresses": { "description": "publishNotReadyAddresses, when set to true, indicates that DNS implementations must publish the notReadyAddresses of subsets for the Endpoints associated with the Service. The default value is false. The primary use case for setting this field is to use a StatefulSet's Headless Service to propagate SRV records for its Pods without respect to their readiness for purpose of peer discovery.", "type": "boolean" }, "selector": { "description": "Route service traffic to pods with label keys and values matching this selector. If empty or not present, the service is assumed to have an external process managing its endpoints, which Kubernetes will not modify. Only applies to types ClusterIP, NodePort, and LoadBalancer. Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/", "type": "object", "additionalProperties": { "type": "string" } }, "sessionAffinity": { "description": "Supports \"ClientIP\" and \"None\". Used to maintain session affinity. Enable client IP based session affinity. Must be ClientIP or None. Defaults to None. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", "type": "string" }, "sessionAffinityConfig": { "description": "sessionAffinityConfig contains the configurations of session affinity.", "type": "object", "properties": { "clientIP": { "description": "clientIP contains the configurations of Client IP based session affinity.", "type": "object", "properties": { "timeoutSeconds": { "description": "timeoutSeconds specifies the seconds of ClientIP type session sticky time. The value must be >0 && <=86400(for 1 day) if ServiceAffinity == \"ClientIP\". Default value is 10800(for 3 hours).", "type": "integer", "format": "int32" } } } } }, "topologyKeys": { "description": "topologyKeys is a preference-order list of topology keys which implementations of services should use to preferentially sort endpoints when accessing this Service, it can not be used at the same time as externalTrafficPolicy=Local. Topology keys must be valid label keys and at most 16 keys may be specified. Endpoints are chosen based on the first topology key with available backends. If this field is specified and all entries have no backends that match the topology of the client, the service has no backends for that client and connections should fail. The special value \"*\" may be used to mean \"any topology\". This catch-all value, if used, only makes sense as the last value in the list. If this is not specified or empty, no topology constraints will be applied.", "type": "array", "items": { "type": "string" } }, "type": { "description": "type determines how the Service is exposed. Defaults to ClusterIP. Valid options are ExternalName, ClusterIP, NodePort, and LoadBalancer. \"ExternalName\" maps to the specified externalName. \"ClusterIP\" allocates a cluster-internal IP address for load-balancing to endpoints. Endpoints are determined by the selector or if that is not specified, by manual construction of an Endpoints object. If clusterIP is \"None\", no virtual IP is allocated and the endpoints are published as a set of endpoints rather than a stable IP. \"NodePort\" builds on ClusterIP and allocates a port on every node which routes to the clusterIP. \"LoadBalancer\" builds on NodePort and creates an external load-balancer (if supported in the current cloud) which routes to the clusterIP. More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types", "type": "string" } } } } }, "tls": { "description": "TLS defines options for configuring TLS for HTTP.", "type": "object", "properties": { "certificate": { "description": "Certificate is a reference to a Kubernetes secret that contains the certificate and private key for enabling TLS. The referenced secret should contain the following: \n - `ca.crt`: The certificate authority (optional). - `tls.crt`: The certificate (or a chain). - `tls.key`: The private key to the first certificate in the certificate chain.", "type": "object", "properties": { "secretName": { "description": "SecretName is the name of the secret.", "type": "string" } } }, "selfSignedCertificate": { "description": "SelfSignedCertificate allows configuring the self-signed certificate generated by the operator.", "type": "object", "properties": { "disabled": { "description": "Disabled indicates that the provisioning of the self-signed certificate should be disabled.", "type": "boolean" }, "subjectAltNames": { "description": "SubjectAlternativeNames is a list of SANs to include in the generated HTTP TLS certificate.", "type": "array", "items": { "description": "SubjectAlternativeName represents a SAN entry in a x509 certificate.", "type": "object", "properties": { "dns": { "description": "DNS is the DNS name of the subject.", "type": "string" }, "ip": { "description": "IP is the IP address of the subject.", "type": "string" } } } } } } } } } }, "image": { "description": "Image is the Elasticsearch Docker image to deploy.", "type": "string" }, "nodeSets": { "description": "NodeSets allow specifying groups of Elasticsearch nodes sharing the same configuration and Pod templates. See: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-orchestration.html", "type": "array", "minItems": 1, "items": { "description": "NodeSet is the specification for a group of Elasticsearch nodes sharing the same configuration and a Pod template.", "type": "object", "required": [ "count", "name" ], "properties": { "config": { "description": "Config holds the Elasticsearch configuration.", "type": "object" }, "count": { "description": "Count of Elasticsearch nodes to deploy.", "type": "integer", "format": "int32", "minimum": 1 }, "name": { "description": "Name of this set of nodes. Becomes a part of the Elasticsearch node.name setting.", "type": "string", "maxLength": 23, "pattern": "[a-zA-Z0-9-]+" }, "podTemplate": { "description": "PodTemplate provides customisation options (labels, annotations, affinity rules, resource requests, and so on) for the Pods belonging to this NodeSet.", "type": "object" }, "volumeClaimTemplates": { "description": "VolumeClaimTemplates is a list of persistent volume claims to be used by each Pod in this NodeSet. Every claim in this list must have a matching volumeMount in one of the containers defined in the PodTemplate. Items defined here take precedence over any default claims added by the operator with the same name. See: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-volume-claim-templates.html", "type": "array", "items": { "description": "PersistentVolumeClaim is a user's request for and claim to a persistent volume", "type": "object", "properties": { "apiVersion": { "description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", "type": "string" }, "kind": { "description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", "type": "string" }, "metadata": { "description": "Standard object's metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata", "type": "object" }, "spec": { "description": "Spec defines the desired characteristics of a volume requested by a pod author. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims", "type": "object", "properties": { "accessModes": { "description": "AccessModes contains the desired access modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1", "type": "array", "items": { "type": "string" } }, "dataSource": { "description": "This field requires the VolumeSnapshotDataSource alpha feature gate to be enabled and currently VolumeSnapshot is the only supported data source. If the provisioner can support VolumeSnapshot data source, it will create a new volume and data will be restored to the volume at the same time. If the provisioner does not support VolumeSnapshot data source, volume will not be created and the failure will be reported as an event. In the future, we plan to support more data source types and the behavior of the provisioner may change.", "type": "object", "required": [ "kind", "name" ], "properties": { "apiGroup": { "description": "APIGroup is the group for the resource being referenced. If APIGroup is not specified, the specified Kind must be in the core API group. For any other third-party types, APIGroup is required.", "type": "string" }, "kind": { "description": "Kind is the type of resource being referenced", "type": "string" }, "name": { "description": "Name is the name of resource being referenced", "type": "string" } } }, "resources": { "description": "Resources represents the minimum resources the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources", "type": "object", "properties": { "limits": { "description": "Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", "type": "object", "additionalProperties": { "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$", "anyOf": [ { "type": "integer" }, { "type": "string" } ] } }, "requests": { "description": "Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", "type": "object", "additionalProperties": { "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$", "anyOf": [ { "type": "integer" }, { "type": "string" } ] } } } }, "selector": { "description": "A label query over volumes to consider for binding.", "type": "object", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "type": "array", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "type": "object", "required": [ "key", "operator" ], "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "type": "array", "items": { "type": "string" } } } } }, "matchLabels": { "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object", "additionalProperties": { "type": "string" } } } }, "storageClassName": { "description": "Name of the StorageClass required by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1", "type": "string" }, "volumeMode": { "description": "volumeMode defines what type of volume is required by the claim. Value of Filesystem is implied when not included in claim spec. This is a beta feature.", "type": "string" }, "volumeName": { "description": "VolumeName is the binding reference to the PersistentVolume backing this claim.", "type": "string" } } }, "status": { "description": "Status represents the current information/status of a persistent volume claim. Read-only. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims", "type": "object", "properties": { "accessModes": { "description": "AccessModes contains the actual access modes the volume backing the PVC has. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1", "type": "array", "items": { "type": "string" } }, "capacity": { "description": "Represents the actual resources of the underlying volume.", "type": "object", "additionalProperties": { "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$", "anyOf": [ { "type": "integer" }, { "type": "string" } ] } }, "conditions": { "description": "Current Condition of persistent volume claim. If underlying persistent volume is being resized then the Condition will be set to 'ResizeStarted'.", "type": "array", "items": { "description": "PersistentVolumeClaimCondition contains details about state of pvc", "type": "object", "required": [ "status", "type" ], "properties": { "lastProbeTime": { "description": "Last time we probed the condition.", "type": "string", "format": "date-time" }, "lastTransitionTime": { "description": "Last time the condition transitioned from one status to another.", "type": "string", "format": "date-time" }, "message": { "description": "Human-readable message indicating details about last transition.", "type": "string" }, "reason": { "description": "Unique, this should be a short, machine understandable string that gives the reason for condition's last transition. If it reports \"ResizeStarted\" that means the underlying persistent volume is being resized.", "type": "string" }, "status": { "type": "string" }, "type": { "description": "PersistentVolumeClaimConditionType is a valid value of PersistentVolumeClaimCondition.Type", "type": "string" } } } }, "phase": { "description": "Phase represents the current phase of PersistentVolumeClaim.", "type": "string" } } } } } } } } }, "podDisruptionBudget": { "description": "PodDisruptionBudget provides access to the default pod disruption budget for the Elasticsearch cluster. The default budget selects all cluster pods and sets `maxUnavailable` to 1. To disable, set `PodDisruptionBudget` to the empty value (`{}` in YAML).", "type": "object", "properties": { "metadata": { "description": "ObjectMeta is the metadata of the PDB. The name and namespace provided here are managed by ECK and will be ignored.", "type": "object" }, "spec": { "description": "Spec is the specification of the PDB.", "type": "object", "properties": { "maxUnavailable": { "description": "An eviction is allowed if at most \"maxUnavailable\" pods selected by \"selector\" are unavailable after the eviction, i.e. even in absence of the evicted pod. For example, one can prevent all voluntary evictions by specifying 0. This is a mutually exclusive setting with \"minAvailable\".", "anyOf": [ { "type": "integer" }, { "type": "string" } ] }, "minAvailable": { "description": "An eviction is allowed if at least \"minAvailable\" pods selected by \"selector\" will still be available after the eviction, i.e. even in the absence of the evicted pod. So for example you can prevent all voluntary evictions by specifying \"100%\".", "anyOf": [ { "type": "integer" }, { "type": "string" } ] }, "selector": { "description": "Label query over pods whose evictions are managed by the disruption budget.", "type": "object", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "type": "array", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "type": "object", "required": [ "key", "operator" ], "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "type": "array", "items": { "type": "string" } } } } }, "matchLabels": { "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object", "additionalProperties": { "type": "string" } } } } } } } }, "remoteClusters": { "description": "RemoteClusters enables you to establish uni-directional connections to a remote Elasticsearch cluster.", "type": "array", "items": { "description": "RemoteCluster declares a remote Elasticsearch cluster connection.", "type": "object", "required": [ "name" ], "properties": { "elasticsearchRef": { "description": "ElasticsearchRef is a reference to an Elasticsearch cluster running within the same k8s cluster.", "type": "object", "required": [ "name" ], "properties": { "name": { "description": "Name of the Kubernetes object.", "type": "string" }, "namespace": { "description": "Namespace of the Kubernetes object. If empty, defaults to the current namespace.", "type": "string" } } }, "name": { "description": "Name is the name of the remote cluster as it is set in the Elasticsearch settings. The name is expected to be unique for each remote clusters.", "type": "string", "minLength": 1 } } } }, "secureSettings": { "description": "SecureSettings is a list of references to Kubernetes secrets containing sensitive configuration options for Elasticsearch. See: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-es-secure-settings.html", "type": "array", "items": { "description": "SecretSource defines a data source based on a Kubernetes Secret.", "type": "object", "required": [ "secretName" ], "properties": { "entries": { "description": "Entries define how to project each key-value pair in the secret to filesystem paths. If not defined, all keys will be projected to similarly named paths in the filesystem. If defined, only the specified keys will be projected to the corresponding paths.", "type": "array", "items": { "description": "KeyToPath defines how to map a key in a Secret object to a filesystem path.", "type": "object", "required": [ "key" ], "properties": { "key": { "description": "Key is the key contained in the secret.", "type": "string" }, "path": { "description": "Path is the relative file path to map the key to. Path must not be an absolute file path and must not contain any \"..\" components.", "type": "string" } } } }, "secretName": { "description": "SecretName is the name of the secret.", "type": "string" } } } }, "serviceAccountName": { "description": "ServiceAccountName is used to check access from the current resource to a resource (eg. a remote Elasticsearch cluster) in a different namespace. Can only be used if ECK is enforcing RBAC on references.", "type": "string" }, "transport": { "description": "Transport holds transport layer settings for Elasticsearch.", "type": "object", "properties": { "service": { "description": "Service defines the template for the associated Kubernetes Service object.", "type": "object", "properties": { "metadata": { "description": "ObjectMeta is the metadata of the service. The name and namespace provided here are managed by ECK and will be ignored.", "type": "object" }, "spec": { "description": "Spec is the specification of the service.", "type": "object", "properties": { "clusterIP": { "description": "clusterIP is the IP address of the service and is usually assigned randomly by the master. If an address is specified manually and is not in use by others, it will be allocated to the service; otherwise, creation of the service will fail. This field can not be changed through updates. Valid values are \"None\", empty string (\"\"), or a valid IP address. \"None\" can be specified for headless services when proxying is not required. Only applies to types ClusterIP, NodePort, and LoadBalancer. Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", "type": "string" }, "externalIPs": { "description": "externalIPs is a list of IP addresses for which nodes in the cluster will also accept traffic for this service. These IPs are not managed by Kubernetes. The user is responsible for ensuring that traffic arrives at a node with this IP. A common example is external load-balancers that are not part of the Kubernetes system.", "type": "array", "items": { "type": "string" } }, "externalName": { "description": "externalName is the external reference that kubedns or equivalent will return as a CNAME record for this service. No proxying will be involved. Must be a valid RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) and requires Type to be ExternalName.", "type": "string" }, "externalTrafficPolicy": { "description": "externalTrafficPolicy denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints. \"Local\" preserves the client source IP and avoids a second hop for LoadBalancer and Nodeport type services, but risks potentially imbalanced traffic spreading. \"Cluster\" obscures the client source IP and may cause a second hop to another node, but should have good overall load-spreading.", "type": "string" }, "healthCheckNodePort": { "description": "healthCheckNodePort specifies the healthcheck nodePort for the service. If not specified, HealthCheckNodePort is created by the service api backend with the allocated nodePort. Will use user-specified nodePort value if specified by the client. Only effects when Type is set to LoadBalancer and ExternalTrafficPolicy is set to Local.", "type": "integer", "format": "int32" }, "ipFamily": { "description": "ipFamily specifies whether this Service has a preference for a particular IP family (e.g. IPv4 vs. IPv6). If a specific IP family is requested, the clusterIP field will be allocated from that family, if it is available in the cluster. If no IP family is requested, the cluster's primary IP family will be used. Other IP fields (loadBalancerIP, loadBalancerSourceRanges, externalIPs) and controllers which allocate external load-balancers should use the same IP family. Endpoints for this Service will be of this family. This field is immutable after creation. Assigning a ServiceIPFamily not available in the cluster (e.g. IPv6 in IPv4 only cluster) is an error condition and will fail during clusterIP assignment.", "type": "string" }, "loadBalancerIP": { "description": "Only applies to Service Type: LoadBalancer LoadBalancer will get created with the IP specified in this field. This feature depends on whether the underlying cloud-provider supports specifying the loadBalancerIP when a load balancer is created. This field will be ignored if the cloud-provider does not support the feature.", "type": "string" }, "loadBalancerSourceRanges": { "description": "If specified and supported by the platform, this will restrict traffic through the cloud-provider load-balancer will be restricted to the specified client IPs. This field will be ignored if the cloud-provider does not support the feature.\" More info: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/", "type": "array", "items": { "type": "string" } }, "ports": { "description": "The list of ports that are exposed by this service. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", "type": "array", "items": { "description": "ServicePort contains information on service's port.", "type": "object", "required": [ "port" ], "properties": { "name": { "description": "The name of this port within the service. This must be a DNS_LABEL. All ports within a ServiceSpec must have unique names. When considering the endpoints for a Service, this must match the 'name' field in the EndpointPort. Optional if only one ServicePort is defined on this service.", "type": "string" }, "nodePort": { "description": "The port on each node on which this service is exposed when type=NodePort or LoadBalancer. Usually assigned by the system. If specified, it will be allocated to the service if unused or else creation of the service will fail. Default is to auto-allocate a port if the ServiceType of this Service requires one. More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport", "type": "integer", "format": "int32" }, "port": { "description": "The port that will be exposed by this service.", "type": "integer", "format": "int32" }, "protocol": { "description": "The IP protocol for this port. Supports \"TCP\", \"UDP\", and \"SCTP\". Default is TCP.", "type": "string" }, "targetPort": { "description": "Number or name of the port to access on the pods targeted by the service. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. If this is a string, it will be looked up as a named port in the target Pod's container ports. If this is not specified, the value of the 'port' field is used (an identity map). This field is ignored for services with clusterIP=None, and should be omitted or set equal to the 'port' field. More info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service", "anyOf": [ { "type": "integer" }, { "type": "string" } ] } } } }, "publishNotReadyAddresses": { "description": "publishNotReadyAddresses, when set to true, indicates that DNS implementations must publish the notReadyAddresses of subsets for the Endpoints associated with the Service. The default value is false. The primary use case for setting this field is to use a StatefulSet's Headless Service to propagate SRV records for its Pods without respect to their readiness for purpose of peer discovery.", "type": "boolean" }, "selector": { "description": "Route service traffic to pods with label keys and values matching this selector. If empty or not present, the service is assumed to have an external process managing its endpoints, which Kubernetes will not modify. Only applies to types ClusterIP, NodePort, and LoadBalancer. Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/", "type": "object", "additionalProperties": { "type": "string" } }, "sessionAffinity": { "description": "Supports \"ClientIP\" and \"None\". Used to maintain session affinity. Enable client IP based session affinity. Must be ClientIP or None. Defaults to None. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", "type": "string" }, "sessionAffinityConfig": { "description": "sessionAffinityConfig contains the configurations of session affinity.", "type": "object", "properties": { "clientIP": { "description": "clientIP contains the configurations of Client IP based session affinity.", "type": "object", "properties": { "timeoutSeconds": { "description": "timeoutSeconds specifies the seconds of ClientIP type session sticky time. The value must be >0 && <=86400(for 1 day) if ServiceAffinity == \"ClientIP\". Default value is 10800(for 3 hours).", "type": "integer", "format": "int32" } } } } }, "topologyKeys": { "description": "topologyKeys is a preference-order list of topology keys which implementations of services should use to preferentially sort endpoints when accessing this Service, it can not be used at the same time as externalTrafficPolicy=Local. Topology keys must be valid label keys and at most 16 keys may be specified. Endpoints are chosen based on the first topology key with available backends. If this field is specified and all entries have no backends that match the topology of the client, the service has no backends for that client and connections should fail. The special value \"*\" may be used to mean \"any topology\". This catch-all value, if used, only makes sense as the last value in the list. If this is not specified or empty, no topology constraints will be applied.", "type": "array", "items": { "type": "string" } }, "type": { "description": "type determines how the Service is exposed. Defaults to ClusterIP. Valid options are ExternalName, ClusterIP, NodePort, and LoadBalancer. \"ExternalName\" maps to the specified externalName. \"ClusterIP\" allocates a cluster-internal IP address for load-balancing to endpoints. Endpoints are determined by the selector or if that is not specified, by manual construction of an Endpoints object. If clusterIP is \"None\", no virtual IP is allocated and the endpoints are published as a set of endpoints rather than a stable IP. \"NodePort\" builds on ClusterIP and allocates a port on every node which routes to the clusterIP. \"LoadBalancer\" builds on NodePort and creates an external load-balancer (if supported in the current cloud) which routes to the clusterIP. More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types", "type": "string" } } } } } } }, "updateStrategy": { "description": "UpdateStrategy specifies how updates to the cluster should be performed.", "type": "object", "properties": { "changeBudget": { "description": "ChangeBudget defines the constraints to consider when applying changes to the Elasticsearch cluster.", "type": "object", "properties": { "maxSurge": { "description": "MaxSurge is the maximum number of new pods that can be created exceeding the original number of pods defined in the specification. MaxSurge is only taken into consideration when scaling up. Setting a negative value will disable the restriction. Defaults to unbounded if not specified.", "type": "integer", "format": "int32" }, "maxUnavailable": { "description": "MaxUnavailable is the maximum number of pods that can be unavailable (not ready) during the update due to circumstances under the control of the operator. Setting a negative value will disable this restriction. Defaults to 1 if not specified.", "type": "integer", "format": "int32" } } } } }, "version": { "description": "Version of Elasticsearch.", "type": "string" } } }, "status": { "description": "ElasticsearchStatus defines the observed state of Elasticsearch", "type": "object", "properties": { "availableNodes": { "type": "integer", "format": "int32" }, "health": { "description": "ElasticsearchHealth is the health of the cluster as returned by the health API.", "type": "string" }, "phase": { "description": "ElasticsearchOrchestrationPhase is the phase Elasticsearch is in from the controller point of view.", "type": "string" } } } } } }, "subresources": { "status": {} }, "versions": [ { "name": "v1", "served": true, "storage": true }, { "name": "v1beta1", "served": true, "storage": false }, { "name": "v1alpha1", "served": false, "storage": false } ], "additionalPrinterColumns": [ { "name": "health", "type": "string", "JSONPath": ".status.health" }, { "name": "nodes", "type": "integer", "description": "Available nodes", "JSONPath": ".status.availableNodes" }, { "name": "version", "type": "string", "description": "Elasticsearch version", "JSONPath": ".spec.version" }, { "name": "phase", "type": "string", "JSONPath": ".status.phase" }, { "name": "age", "type": "date", "JSONPath": ".metadata.creationTimestamp" } ], "conversion": { "strategy": "None" }, "preserveUnknownFields": true }, "status": { "conditions": [ { "type": "NonStructuralSchema", "status": "True", "lastTransitionTime": "2020-04-28T23:31:51Z", "reason": "Violations", "message": "[spec.validation.openAPIV3Schema.properties[spec].properties[http].properties[service].properties[spec].properties[ports].items.properties[targetPort].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[nodeSets].items.properties[volumeClaimTemplates].items.properties[spec].properties[resources].properties[limits].additionalProperties.type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[nodeSets].items.properties[volumeClaimTemplates].items.properties[spec].properties[resources].properties[requests].additionalProperties.type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[nodeSets].items.properties[volumeClaimTemplates].items.properties[status].properties[capacity].additionalProperties.type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[podDisruptionBudget].properties[spec].properties[maxUnavailable].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[podDisruptionBudget].properties[spec].properties[minAvailable].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[transport].properties[service].properties[spec].properties[ports].items.properties[targetPort].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.type: Required value: must not be empty at the root]" }, { "type": "NamesAccepted", "status": "True", "lastTransitionTime": "2020-04-28T23:31:51Z", "reason": "NoConflicts", "message": "no conflicts found" }, { "type": "Established", "status": "True", "lastTransitionTime": "2020-04-28T23:31:51Z", "reason": "InitialNamesAccepted", "message": "the initial names have been accepted" } ], "acceptedNames": { "plural": "elasticsearches", "singular": "elasticsearch", "shortNames": [ "es" ], "kind": "Elasticsearch", "listKind": "ElasticsearchList", "categories": [ "elastic" ] }, "storedVersions": [ "v1" ] } } ================================================ FILE: pkg/backup/actions/testdata/v1beta1/gcpsamples.gcp.stacks.crossplane.io.json ================================================ { "kind": "CustomResourceDefinition", "apiVersion": "apiextensions.k8s.io/v1beta1", "metadata": { "name": "gcpsamples.gcp.stacks.crossplane.io", "selfLink": "/apis/apiextensions.k8s.io/v1beta1/customresourcedefinitions/gcpsamples.gcp.stacks.crossplane.io", "uid": "c0bbac74-acab-4620-b628-1d5f91b19040", "resourceVersion": "5567", "generation": 1, "creationTimestamp": "2020-04-20T17:27:56Z", "annotations": { "kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"apiextensions.k8s.io/v1beta1\",\"kind\":\"CustomResourceDefinition\",\"metadata\":{\"annotations\":{},\"creationTimestamp\":\"2020-04-20T16:57:37Z\",\"generation\":1,\"name\":\"gcpsamples.gcp.stacks.crossplane.io\",\"resourceVersion\":\"549\",\"selfLink\":\"/apis/apiextensions.k8s.io/v1/customresourcedefinitions/gcpsamples.gcp.stacks.crossplane.io\",\"uid\":\"db5f4321-3226-44b0-8247-66fd7ef59dc8\"},\"spec\":{\"conversion\":{\"strategy\":\"None\"},\"group\":\"gcp.stacks.crossplane.io\",\"names\":{\"kind\":\"GCPSample\",\"listKind\":\"GCPSampleList\",\"plural\":\"gcpsamples\",\"singular\":\"gcpsample\"},\"preserveUnknownFields\":true,\"scope\":\"Cluster\",\"versions\":[{\"name\":\"v1alpha1\",\"served\":true,\"storage\":true}]},\"status\":{\"acceptedNames\":{\"kind\":\"GCPSample\",\"listKind\":\"GCPSampleList\",\"plural\":\"gcpsamples\",\"singular\":\"gcpsample\"},\"conditions\":[{\"lastTransitionTime\":\"2020-04-20T16:57:37Z\",\"message\":\"no conflicts found\",\"reason\":\"NoConflicts\",\"status\":\"True\",\"type\":\"NamesAccepted\"},{\"lastTransitionTime\":\"2020-04-20T16:57:37Z\",\"message\":\"the initial names have been accepted\",\"reason\":\"InitialNamesAccepted\",\"status\":\"True\",\"type\":\"Established\"}],\"storedVersions\":[\"v1alpha1\"]}}\n" } }, "spec": { "group": "gcp.stacks.crossplane.io", "version": "v1alpha1", "names": { "plural": "gcpsamples", "singular": "gcpsample", "kind": "GCPSample", "listKind": "GCPSampleList" }, "scope": "Cluster", "versions": [ { "name": "v1alpha1", "served": true, "storage": true } ], "conversion": { "strategy": "None" }, "preserveUnknownFields": true }, "status": { "conditions": [ { "type": "NamesAccepted", "status": "True", "lastTransitionTime": "2020-04-20T17:27:56Z", "reason": "NoConflicts", "message": "no conflicts found" }, { "type": "Established", "status": "True", "lastTransitionTime": "2020-04-20T17:27:56Z", "reason": "InitialNamesAccepted", "message": "the initial names have been accepted" } ], "acceptedNames": { "plural": "gcpsamples", "singular": "gcpsample", "kind": "GCPSample", "listKind": "GCPSampleList" }, "storedVersions": [ "v1alpha1" ] } } ================================================ FILE: pkg/backup/actions/testdata/v1beta1/kibanas.kibana.k8s.elastic.co.json ================================================ { "kind": "CustomResourceDefinition", "apiVersion": "apiextensions.k8s.io/v1beta1", "metadata": { "name": "kibanas.kibana.k8s.elastic.co", "selfLink": "/apis/apiextensions.k8s.io/v1beta1/customresourcedefinitions/kibanas.kibana.k8s.elastic.co", "uid": "95f42a77-654f-4380-a6b1-1fe2587f0713", "resourceVersion": "1703552", "generation": 1, "creationTimestamp": "2020-04-28T23:31:53Z", "labels": { "velero.io/backup-name": "es", "velero.io/restore-name": "es-crds" }, "annotations": { "controller-gen.kubebuilder.io/version": "v0.2.5", "kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"apiextensions.k8s.io/v1beta1\",\"kind\":\"CustomResourceDefinition\",\"metadata\":{\"annotations\":{\"controller-gen.kubebuilder.io/version\":\"v0.2.5\"},\"creationTimestamp\":null,\"name\":\"kibanas.kibana.k8s.elastic.co\"},\"spec\":{\"additionalPrinterColumns\":[{\"JSONPath\":\".status.health\",\"name\":\"health\",\"type\":\"string\"},{\"JSONPath\":\".status.availableNodes\",\"description\":\"Available nodes\",\"name\":\"nodes\",\"type\":\"integer\"},{\"JSONPath\":\".spec.version\",\"description\":\"Kibana version\",\"name\":\"version\",\"type\":\"string\"},{\"JSONPath\":\".metadata.creationTimestamp\",\"name\":\"age\",\"type\":\"date\"}],\"group\":\"kibana.k8s.elastic.co\",\"names\":{\"categories\":[\"elastic\"],\"kind\":\"Kibana\",\"listKind\":\"KibanaList\",\"plural\":\"kibanas\",\"shortNames\":[\"kb\"],\"singular\":\"kibana\"},\"scope\":\"Namespaced\",\"subresources\":{\"status\":{}},\"validation\":{\"openAPIV3Schema\":{\"description\":\"Kibana represents a Kibana resource in a Kubernetes cluster.\",\"properties\":{\"apiVersion\":{\"description\":\"APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\",\"type\":\"string\"},\"kind\":{\"description\":\"Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\",\"type\":\"string\"},\"metadata\":{\"type\":\"object\"},\"spec\":{\"description\":\"KibanaSpec holds the specification of a Kibana instance.\",\"properties\":{\"config\":{\"description\":\"Config holds the Kibana configuration. See: https://www.elastic.co/guide/en/kibana/current/settings.html\",\"type\":\"object\"},\"count\":{\"description\":\"Count of Kibana instances to deploy.\",\"format\":\"int32\",\"type\":\"integer\"},\"elasticsearchRef\":{\"description\":\"ElasticsearchRef is a reference to an Elasticsearch cluster running in the same Kubernetes cluster.\",\"properties\":{\"name\":{\"description\":\"Name of the Kubernetes object.\",\"type\":\"string\"},\"namespace\":{\"description\":\"Namespace of the Kubernetes object. If empty, defaults to the current namespace.\",\"type\":\"string\"}},\"required\":[\"name\"],\"type\":\"object\"},\"http\":{\"description\":\"HTTP holds the HTTP layer configuration for Kibana.\",\"properties\":{\"service\":{\"description\":\"Service defines the template for the associated Kubernetes Service object.\",\"properties\":{\"metadata\":{\"description\":\"ObjectMeta is the metadata of the service. The name and namespace provided here are managed by ECK and will be ignored.\",\"type\":\"object\"},\"spec\":{\"description\":\"Spec is the specification of the service.\",\"properties\":{\"clusterIP\":{\"description\":\"clusterIP is the IP address of the service and is usually assigned randomly by the master. If an address is specified manually and is not in use by others, it will be allocated to the service; otherwise, creation of the service will fail. This field can not be changed through updates. Valid values are \\\"None\\\", empty string (\\\"\\\"), or a valid IP address. \\\"None\\\" can be specified for headless services when proxying is not required. Only applies to types ClusterIP, NodePort, and LoadBalancer. Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies\",\"type\":\"string\"},\"externalIPs\":{\"description\":\"externalIPs is a list of IP addresses for which nodes in the cluster will also accept traffic for this service. These IPs are not managed by Kubernetes. The user is responsible for ensuring that traffic arrives at a node with this IP. A common example is external load-balancers that are not part of the Kubernetes system.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"externalName\":{\"description\":\"externalName is the external reference that kubedns or equivalent will return as a CNAME record for this service. No proxying will be involved. Must be a valid RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) and requires Type to be ExternalName.\",\"type\":\"string\"},\"externalTrafficPolicy\":{\"description\":\"externalTrafficPolicy denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints. \\\"Local\\\" preserves the client source IP and avoids a second hop for LoadBalancer and Nodeport type services, but risks potentially imbalanced traffic spreading. \\\"Cluster\\\" obscures the client source IP and may cause a second hop to another node, but should have good overall load-spreading.\",\"type\":\"string\"},\"healthCheckNodePort\":{\"description\":\"healthCheckNodePort specifies the healthcheck nodePort for the service. If not specified, HealthCheckNodePort is created by the service api backend with the allocated nodePort. Will use user-specified nodePort value if specified by the client. Only effects when Type is set to LoadBalancer and ExternalTrafficPolicy is set to Local.\",\"format\":\"int32\",\"type\":\"integer\"},\"ipFamily\":{\"description\":\"ipFamily specifies whether this Service has a preference for a particular IP family (e.g. IPv4 vs. IPv6). If a specific IP family is requested, the clusterIP field will be allocated from that family, if it is available in the cluster. If no IP family is requested, the cluster's primary IP family will be used. Other IP fields (loadBalancerIP, loadBalancerSourceRanges, externalIPs) and controllers which allocate external load-balancers should use the same IP family. Endpoints for this Service will be of this family. This field is immutable after creation. Assigning a ServiceIPFamily not available in the cluster (e.g. IPv6 in IPv4 only cluster) is an error condition and will fail during clusterIP assignment.\",\"type\":\"string\"},\"loadBalancerIP\":{\"description\":\"Only applies to Service Type: LoadBalancer LoadBalancer will get created with the IP specified in this field. This feature depends on whether the underlying cloud-provider supports specifying the loadBalancerIP when a load balancer is created. This field will be ignored if the cloud-provider does not support the feature.\",\"type\":\"string\"},\"loadBalancerSourceRanges\":{\"description\":\"If specified and supported by the platform, this will restrict traffic through the cloud-provider load-balancer will be restricted to the specified client IPs. This field will be ignored if the cloud-provider does not support the feature.\\\" More info: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"ports\":{\"description\":\"The list of ports that are exposed by this service. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies\",\"items\":{\"description\":\"ServicePort contains information on service's port.\",\"properties\":{\"name\":{\"description\":\"The name of this port within the service. This must be a DNS_LABEL. All ports within a ServiceSpec must have unique names. When considering the endpoints for a Service, this must match the 'name' field in the EndpointPort. Optional if only one ServicePort is defined on this service.\",\"type\":\"string\"},\"nodePort\":{\"description\":\"The port on each node on which this service is exposed when type=NodePort or LoadBalancer. Usually assigned by the system. If specified, it will be allocated to the service if unused or else creation of the service will fail. Default is to auto-allocate a port if the ServiceType of this Service requires one. More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport\",\"format\":\"int32\",\"type\":\"integer\"},\"port\":{\"description\":\"The port that will be exposed by this service.\",\"format\":\"int32\",\"type\":\"integer\"},\"protocol\":{\"description\":\"The IP protocol for this port. Supports \\\"TCP\\\", \\\"UDP\\\", and \\\"SCTP\\\". Default is TCP.\",\"type\":\"string\"},\"targetPort\":{\"anyOf\":[{\"type\":\"integer\"},{\"type\":\"string\"}],\"description\":\"Number or name of the port to access on the pods targeted by the service. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. If this is a string, it will be looked up as a named port in the target Pod's container ports. If this is not specified, the value of the 'port' field is used (an identity map). This field is ignored for services with clusterIP=None, and should be omitted or set equal to the 'port' field. More info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service\"}},\"required\":[\"port\"],\"type\":\"object\"},\"type\":\"array\"},\"publishNotReadyAddresses\":{\"description\":\"publishNotReadyAddresses, when set to true, indicates that DNS implementations must publish the notReadyAddresses of subsets for the Endpoints associated with the Service. The default value is false. The primary use case for setting this field is to use a StatefulSet's Headless Service to propagate SRV records for its Pods without respect to their readiness for purpose of peer discovery.\",\"type\":\"boolean\"},\"selector\":{\"additionalProperties\":{\"type\":\"string\"},\"description\":\"Route service traffic to pods with label keys and values matching this selector. If empty or not present, the service is assumed to have an external process managing its endpoints, which Kubernetes will not modify. Only applies to types ClusterIP, NodePort, and LoadBalancer. Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/\",\"type\":\"object\"},\"sessionAffinity\":{\"description\":\"Supports \\\"ClientIP\\\" and \\\"None\\\". Used to maintain session affinity. Enable client IP based session affinity. Must be ClientIP or None. Defaults to None. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies\",\"type\":\"string\"},\"sessionAffinityConfig\":{\"description\":\"sessionAffinityConfig contains the configurations of session affinity.\",\"properties\":{\"clientIP\":{\"description\":\"clientIP contains the configurations of Client IP based session affinity.\",\"properties\":{\"timeoutSeconds\":{\"description\":\"timeoutSeconds specifies the seconds of ClientIP type session sticky time. The value must be \\u003e0 \\u0026\\u0026 \\u003c=86400(for 1 day) if ServiceAffinity == \\\"ClientIP\\\". Default value is 10800(for 3 hours).\",\"format\":\"int32\",\"type\":\"integer\"}},\"type\":\"object\"}},\"type\":\"object\"},\"topologyKeys\":{\"description\":\"topologyKeys is a preference-order list of topology keys which implementations of services should use to preferentially sort endpoints when accessing this Service, it can not be used at the same time as externalTrafficPolicy=Local. Topology keys must be valid label keys and at most 16 keys may be specified. Endpoints are chosen based on the first topology key with available backends. If this field is specified and all entries have no backends that match the topology of the client, the service has no backends for that client and connections should fail. The special value \\\"*\\\" may be used to mean \\\"any topology\\\". This catch-all value, if used, only makes sense as the last value in the list. If this is not specified or empty, no topology constraints will be applied.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"type\":{\"description\":\"type determines how the Service is exposed. Defaults to ClusterIP. Valid options are ExternalName, ClusterIP, NodePort, and LoadBalancer. \\\"ExternalName\\\" maps to the specified externalName. \\\"ClusterIP\\\" allocates a cluster-internal IP address for load-balancing to endpoints. Endpoints are determined by the selector or if that is not specified, by manual construction of an Endpoints object. If clusterIP is \\\"None\\\", no virtual IP is allocated and the endpoints are published as a set of endpoints rather than a stable IP. \\\"NodePort\\\" builds on ClusterIP and allocates a port on every node which routes to the clusterIP. \\\"LoadBalancer\\\" builds on NodePort and creates an external load-balancer (if supported in the current cloud) which routes to the clusterIP. More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types\",\"type\":\"string\"}},\"type\":\"object\"}},\"type\":\"object\"},\"tls\":{\"description\":\"TLS defines options for configuring TLS for HTTP.\",\"properties\":{\"certificate\":{\"description\":\"Certificate is a reference to a Kubernetes secret that contains the certificate and private key for enabling TLS. The referenced secret should contain the following: \\n - `ca.crt`: The certificate authority (optional). - `tls.crt`: The certificate (or a chain). - `tls.key`: The private key to the first certificate in the certificate chain.\",\"properties\":{\"secretName\":{\"description\":\"SecretName is the name of the secret.\",\"type\":\"string\"}},\"type\":\"object\"},\"selfSignedCertificate\":{\"description\":\"SelfSignedCertificate allows configuring the self-signed certificate generated by the operator.\",\"properties\":{\"disabled\":{\"description\":\"Disabled indicates that the provisioning of the self-signed certificate should be disabled.\",\"type\":\"boolean\"},\"subjectAltNames\":{\"description\":\"SubjectAlternativeNames is a list of SANs to include in the generated HTTP TLS certificate.\",\"items\":{\"description\":\"SubjectAlternativeName represents a SAN entry in a x509 certificate.\",\"properties\":{\"dns\":{\"description\":\"DNS is the DNS name of the subject.\",\"type\":\"string\"},\"ip\":{\"description\":\"IP is the IP address of the subject.\",\"type\":\"string\"}},\"type\":\"object\"},\"type\":\"array\"}},\"type\":\"object\"}},\"type\":\"object\"}},\"type\":\"object\"},\"image\":{\"description\":\"Image is the Kibana Docker image to deploy.\",\"type\":\"string\"},\"podTemplate\":{\"description\":\"PodTemplate provides customisation options (labels, annotations, affinity rules, resource requests, and so on) for the Kibana pods\",\"type\":\"object\"},\"secureSettings\":{\"description\":\"SecureSettings is a list of references to Kubernetes secrets containing sensitive configuration options for Kibana. See: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-kibana.html#k8s-kibana-secure-settings\",\"items\":{\"description\":\"SecretSource defines a data source based on a Kubernetes Secret.\",\"properties\":{\"entries\":{\"description\":\"Entries define how to project each key-value pair in the secret to filesystem paths. If not defined, all keys will be projected to similarly named paths in the filesystem. If defined, only the specified keys will be projected to the corresponding paths.\",\"items\":{\"description\":\"KeyToPath defines how to map a key in a Secret object to a filesystem path.\",\"properties\":{\"key\":{\"description\":\"Key is the key contained in the secret.\",\"type\":\"string\"},\"path\":{\"description\":\"Path is the relative file path to map the key to. Path must not be an absolute file path and must not contain any \\\"..\\\" components.\",\"type\":\"string\"}},\"required\":[\"key\"],\"type\":\"object\"},\"type\":\"array\"},\"secretName\":{\"description\":\"SecretName is the name of the secret.\",\"type\":\"string\"}},\"required\":[\"secretName\"],\"type\":\"object\"},\"type\":\"array\"},\"serviceAccountName\":{\"description\":\"ServiceAccountName is used to check access from the current resource to a resource (eg. Elasticsearch) in a different namespace. Can only be used if ECK is enforcing RBAC on references.\",\"type\":\"string\"},\"version\":{\"description\":\"Version of Kibana.\",\"type\":\"string\"}},\"required\":[\"version\"],\"type\":\"object\"},\"status\":{\"description\":\"KibanaStatus defines the observed state of Kibana\",\"properties\":{\"associationStatus\":{\"description\":\"AssociationStatus is the status of an association resource.\",\"type\":\"string\"},\"availableNodes\":{\"format\":\"int32\",\"type\":\"integer\"},\"health\":{\"description\":\"KibanaHealth expresses the status of the Kibana instances.\",\"type\":\"string\"}},\"type\":\"object\"}}}},\"version\":\"v1\",\"versions\":[{\"name\":\"v1\",\"served\":true,\"storage\":true},{\"name\":\"v1beta1\",\"served\":true,\"storage\":false},{\"name\":\"v1alpha1\",\"served\":false,\"storage\":false}]},\"status\":{\"acceptedNames\":{\"kind\":\"\",\"plural\":\"\"},\"conditions\":[],\"storedVersions\":[]}}\n" } }, "spec": { "group": "kibana.k8s.elastic.co", "version": "v1", "names": { "plural": "kibanas", "singular": "kibana", "shortNames": [ "kb" ], "kind": "Kibana", "listKind": "KibanaList", "categories": [ "elastic" ] }, "scope": "Namespaced", "validation": { "openAPIV3Schema": { "description": "Kibana represents a Kibana resource in a Kubernetes cluster.", "properties": { "apiVersion": { "description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", "type": "string" }, "kind": { "description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", "type": "string" }, "metadata": { "type": "object" }, "spec": { "description": "KibanaSpec holds the specification of a Kibana instance.", "type": "object", "required": [ "version" ], "properties": { "config": { "description": "Config holds the Kibana configuration. See: https://www.elastic.co/guide/en/kibana/current/settings.html", "type": "object" }, "count": { "description": "Count of Kibana instances to deploy.", "type": "integer", "format": "int32" }, "elasticsearchRef": { "description": "ElasticsearchRef is a reference to an Elasticsearch cluster running in the same Kubernetes cluster.", "type": "object", "required": [ "name" ], "properties": { "name": { "description": "Name of the Kubernetes object.", "type": "string" }, "namespace": { "description": "Namespace of the Kubernetes object. If empty, defaults to the current namespace.", "type": "string" } } }, "http": { "description": "HTTP holds the HTTP layer configuration for Kibana.", "type": "object", "properties": { "service": { "description": "Service defines the template for the associated Kubernetes Service object.", "type": "object", "properties": { "metadata": { "description": "ObjectMeta is the metadata of the service. The name and namespace provided here are managed by ECK and will be ignored.", "type": "object" }, "spec": { "description": "Spec is the specification of the service.", "type": "object", "properties": { "clusterIP": { "description": "clusterIP is the IP address of the service and is usually assigned randomly by the master. If an address is specified manually and is not in use by others, it will be allocated to the service; otherwise, creation of the service will fail. This field can not be changed through updates. Valid values are \"None\", empty string (\"\"), or a valid IP address. \"None\" can be specified for headless services when proxying is not required. Only applies to types ClusterIP, NodePort, and LoadBalancer. Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", "type": "string" }, "externalIPs": { "description": "externalIPs is a list of IP addresses for which nodes in the cluster will also accept traffic for this service. These IPs are not managed by Kubernetes. The user is responsible for ensuring that traffic arrives at a node with this IP. A common example is external load-balancers that are not part of the Kubernetes system.", "type": "array", "items": { "type": "string" } }, "externalName": { "description": "externalName is the external reference that kubedns or equivalent will return as a CNAME record for this service. No proxying will be involved. Must be a valid RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) and requires Type to be ExternalName.", "type": "string" }, "externalTrafficPolicy": { "description": "externalTrafficPolicy denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints. \"Local\" preserves the client source IP and avoids a second hop for LoadBalancer and Nodeport type services, but risks potentially imbalanced traffic spreading. \"Cluster\" obscures the client source IP and may cause a second hop to another node, but should have good overall load-spreading.", "type": "string" }, "healthCheckNodePort": { "description": "healthCheckNodePort specifies the healthcheck nodePort for the service. If not specified, HealthCheckNodePort is created by the service api backend with the allocated nodePort. Will use user-specified nodePort value if specified by the client. Only effects when Type is set to LoadBalancer and ExternalTrafficPolicy is set to Local.", "type": "integer", "format": "int32" }, "ipFamily": { "description": "ipFamily specifies whether this Service has a preference for a particular IP family (e.g. IPv4 vs. IPv6). If a specific IP family is requested, the clusterIP field will be allocated from that family, if it is available in the cluster. If no IP family is requested, the cluster's primary IP family will be used. Other IP fields (loadBalancerIP, loadBalancerSourceRanges, externalIPs) and controllers which allocate external load-balancers should use the same IP family. Endpoints for this Service will be of this family. This field is immutable after creation. Assigning a ServiceIPFamily not available in the cluster (e.g. IPv6 in IPv4 only cluster) is an error condition and will fail during clusterIP assignment.", "type": "string" }, "loadBalancerIP": { "description": "Only applies to Service Type: LoadBalancer LoadBalancer will get created with the IP specified in this field. This feature depends on whether the underlying cloud-provider supports specifying the loadBalancerIP when a load balancer is created. This field will be ignored if the cloud-provider does not support the feature.", "type": "string" }, "loadBalancerSourceRanges": { "description": "If specified and supported by the platform, this will restrict traffic through the cloud-provider load-balancer will be restricted to the specified client IPs. This field will be ignored if the cloud-provider does not support the feature.\" More info: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/", "type": "array", "items": { "type": "string" } }, "ports": { "description": "The list of ports that are exposed by this service. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", "type": "array", "items": { "description": "ServicePort contains information on service's port.", "type": "object", "required": [ "port" ], "properties": { "name": { "description": "The name of this port within the service. This must be a DNS_LABEL. All ports within a ServiceSpec must have unique names. When considering the endpoints for a Service, this must match the 'name' field in the EndpointPort. Optional if only one ServicePort is defined on this service.", "type": "string" }, "nodePort": { "description": "The port on each node on which this service is exposed when type=NodePort or LoadBalancer. Usually assigned by the system. If specified, it will be allocated to the service if unused or else creation of the service will fail. Default is to auto-allocate a port if the ServiceType of this Service requires one. More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport", "type": "integer", "format": "int32" }, "port": { "description": "The port that will be exposed by this service.", "type": "integer", "format": "int32" }, "protocol": { "description": "The IP protocol for this port. Supports \"TCP\", \"UDP\", and \"SCTP\". Default is TCP.", "type": "string" }, "targetPort": { "description": "Number or name of the port to access on the pods targeted by the service. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. If this is a string, it will be looked up as a named port in the target Pod's container ports. If this is not specified, the value of the 'port' field is used (an identity map). This field is ignored for services with clusterIP=None, and should be omitted or set equal to the 'port' field. More info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service", "anyOf": [ { "type": "integer" }, { "type": "string" } ] } } } }, "publishNotReadyAddresses": { "description": "publishNotReadyAddresses, when set to true, indicates that DNS implementations must publish the notReadyAddresses of subsets for the Endpoints associated with the Service. The default value is false. The primary use case for setting this field is to use a StatefulSet's Headless Service to propagate SRV records for its Pods without respect to their readiness for purpose of peer discovery.", "type": "boolean" }, "selector": { "description": "Route service traffic to pods with label keys and values matching this selector. If empty or not present, the service is assumed to have an external process managing its endpoints, which Kubernetes will not modify. Only applies to types ClusterIP, NodePort, and LoadBalancer. Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/", "type": "object", "additionalProperties": { "type": "string" } }, "sessionAffinity": { "description": "Supports \"ClientIP\" and \"None\". Used to maintain session affinity. Enable client IP based session affinity. Must be ClientIP or None. Defaults to None. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", "type": "string" }, "sessionAffinityConfig": { "description": "sessionAffinityConfig contains the configurations of session affinity.", "type": "object", "properties": { "clientIP": { "description": "clientIP contains the configurations of Client IP based session affinity.", "type": "object", "properties": { "timeoutSeconds": { "description": "timeoutSeconds specifies the seconds of ClientIP type session sticky time. The value must be >0 && <=86400(for 1 day) if ServiceAffinity == \"ClientIP\". Default value is 10800(for 3 hours).", "type": "integer", "format": "int32" } } } } }, "topologyKeys": { "description": "topologyKeys is a preference-order list of topology keys which implementations of services should use to preferentially sort endpoints when accessing this Service, it can not be used at the same time as externalTrafficPolicy=Local. Topology keys must be valid label keys and at most 16 keys may be specified. Endpoints are chosen based on the first topology key with available backends. If this field is specified and all entries have no backends that match the topology of the client, the service has no backends for that client and connections should fail. The special value \"*\" may be used to mean \"any topology\". This catch-all value, if used, only makes sense as the last value in the list. If this is not specified or empty, no topology constraints will be applied.", "type": "array", "items": { "type": "string" } }, "type": { "description": "type determines how the Service is exposed. Defaults to ClusterIP. Valid options are ExternalName, ClusterIP, NodePort, and LoadBalancer. \"ExternalName\" maps to the specified externalName. \"ClusterIP\" allocates a cluster-internal IP address for load-balancing to endpoints. Endpoints are determined by the selector or if that is not specified, by manual construction of an Endpoints object. If clusterIP is \"None\", no virtual IP is allocated and the endpoints are published as a set of endpoints rather than a stable IP. \"NodePort\" builds on ClusterIP and allocates a port on every node which routes to the clusterIP. \"LoadBalancer\" builds on NodePort and creates an external load-balancer (if supported in the current cloud) which routes to the clusterIP. More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types", "type": "string" } } } } }, "tls": { "description": "TLS defines options for configuring TLS for HTTP.", "type": "object", "properties": { "certificate": { "description": "Certificate is a reference to a Kubernetes secret that contains the certificate and private key for enabling TLS. The referenced secret should contain the following: \n - `ca.crt`: The certificate authority (optional). - `tls.crt`: The certificate (or a chain). - `tls.key`: The private key to the first certificate in the certificate chain.", "type": "object", "properties": { "secretName": { "description": "SecretName is the name of the secret.", "type": "string" } } }, "selfSignedCertificate": { "description": "SelfSignedCertificate allows configuring the self-signed certificate generated by the operator.", "type": "object", "properties": { "disabled": { "description": "Disabled indicates that the provisioning of the self-signed certificate should be disabled.", "type": "boolean" }, "subjectAltNames": { "description": "SubjectAlternativeNames is a list of SANs to include in the generated HTTP TLS certificate.", "type": "array", "items": { "description": "SubjectAlternativeName represents a SAN entry in a x509 certificate.", "type": "object", "properties": { "dns": { "description": "DNS is the DNS name of the subject.", "type": "string" }, "ip": { "description": "IP is the IP address of the subject.", "type": "string" } } } } } } } } } }, "image": { "description": "Image is the Kibana Docker image to deploy.", "type": "string" }, "podTemplate": { "description": "PodTemplate provides customisation options (labels, annotations, affinity rules, resource requests, and so on) for the Kibana pods", "type": "object" }, "secureSettings": { "description": "SecureSettings is a list of references to Kubernetes secrets containing sensitive configuration options for Kibana. See: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-kibana.html#k8s-kibana-secure-settings", "type": "array", "items": { "description": "SecretSource defines a data source based on a Kubernetes Secret.", "type": "object", "required": [ "secretName" ], "properties": { "entries": { "description": "Entries define how to project each key-value pair in the secret to filesystem paths. If not defined, all keys will be projected to similarly named paths in the filesystem. If defined, only the specified keys will be projected to the corresponding paths.", "type": "array", "items": { "description": "KeyToPath defines how to map a key in a Secret object to a filesystem path.", "type": "object", "required": [ "key" ], "properties": { "key": { "description": "Key is the key contained in the secret.", "type": "string" }, "path": { "description": "Path is the relative file path to map the key to. Path must not be an absolute file path and must not contain any \"..\" components.", "type": "string" } } } }, "secretName": { "description": "SecretName is the name of the secret.", "type": "string" } } } }, "serviceAccountName": { "description": "ServiceAccountName is used to check access from the current resource to a resource (eg. Elasticsearch) in a different namespace. Can only be used if ECK is enforcing RBAC on references.", "type": "string" }, "version": { "description": "Version of Kibana.", "type": "string" } } }, "status": { "description": "KibanaStatus defines the observed state of Kibana", "type": "object", "properties": { "associationStatus": { "description": "AssociationStatus is the status of an association resource.", "type": "string" }, "availableNodes": { "type": "integer", "format": "int32" }, "health": { "description": "KibanaHealth expresses the status of the Kibana instances.", "type": "string" } } } } } }, "subresources": { "status": {} }, "versions": [ { "name": "v1", "served": true, "storage": true }, { "name": "v1beta1", "served": true, "storage": false }, { "name": "v1alpha1", "served": false, "storage": false } ], "additionalPrinterColumns": [ { "name": "health", "type": "string", "JSONPath": ".status.health" }, { "name": "nodes", "type": "integer", "description": "Available nodes", "JSONPath": ".status.availableNodes" }, { "name": "version", "type": "string", "description": "Kibana version", "JSONPath": ".spec.version" }, { "name": "age", "type": "date", "JSONPath": ".metadata.creationTimestamp" } ], "conversion": { "strategy": "None" }, "preserveUnknownFields": true }, "status": { "conditions": [ { "type": "NonStructuralSchema", "status": "True", "lastTransitionTime": "2020-04-28T23:31:53Z", "reason": "Violations", "message": "[spec.validation.openAPIV3Schema.properties[spec].properties[http].properties[service].properties[spec].properties[ports].items.properties[targetPort].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.type: Required value: must not be empty at the root]" }, { "type": "NamesAccepted", "status": "True", "lastTransitionTime": "2020-04-28T23:31:53Z", "reason": "NoConflicts", "message": "no conflicts found" }, { "type": "Established", "status": "True", "lastTransitionTime": "2020-04-28T23:31:53Z", "reason": "InitialNamesAccepted", "message": "the initial names have been accepted" } ], "acceptedNames": { "plural": "kibanas", "singular": "kibana", "shortNames": [ "kb" ], "kind": "Kibana", "listKind": "KibanaList", "categories": [ "elastic" ] }, "storedVersions": [ "v1" ] } } ================================================ FILE: pkg/backup/actions/testdata/v1beta1/prometheuses.monitoring.coreos.com.json ================================================ { "kind": "CustomResourceDefinition", "apiVersion": "apiextensions.k8s.io/v1beta1", "metadata": { "name": "prometheuses.monitoring.coreos.com", "selfLink": "/apis/apiextensions.k8s.io/v1beta1/customresourcedefinitions/prometheuses.monitoring.coreos.com", "uid": "472b9025-d931-4d48-8a6f-77c3b7c33f6e", "resourceVersion": "207497", "generation": 1, "creationTimestamp": "2020-05-05T16:58:10Z", "labels": { "app": "prometheus-operator" }, "annotations": { "helm.sh/hook": "crd-install", "kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"apiextensions.k8s.io/v1beta1\",\"kind\":\"CustomResourceDefinition\",\"metadata\":{\"annotations\":{\"helm.sh/hook\":\"crd-install\"},\"creationTimestamp\":null,\"labels\":{\"app\":\"prometheus-operator\"},\"name\":\"prometheuses.monitoring.coreos.com\"},\"spec\":{\"group\":\"monitoring.coreos.com\",\"names\":{\"kind\":\"Prometheus\",\"plural\":\"prometheuses\"},\"scope\":\"Namespaced\",\"validation\":{\"openAPIV3Schema\":{\"properties\":{\"apiVersion\":{\"description\":\"APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources\",\"type\":\"string\"},\"kind\":{\"description\":\"Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds\",\"type\":\"string\"},\"spec\":{\"description\":\"PrometheusSpec is a specification of the desired behavior of the Prometheus cluster. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/api-conventions.md#spec-and-status\",\"properties\":{\"additionalAlertManagerConfigs\":{\"description\":\"SecretKeySelector selects a key of a Secret.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or it's key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"},\"additionalAlertRelabelConfigs\":{\"description\":\"SecretKeySelector selects a key of a Secret.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or it's key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"},\"additionalScrapeConfigs\":{\"description\":\"SecretKeySelector selects a key of a Secret.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or it's key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"},\"affinity\":{\"description\":\"Affinity is a group of affinity scheduling rules.\",\"properties\":{\"nodeAffinity\":{\"description\":\"Node affinity is a group of node affinity scheduling rules.\",\"properties\":{\"preferredDuringSchedulingIgnoredDuringExecution\":{\"description\":\"The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding \\\"weight\\\" to the sum if the node matches the corresponding matchExpressions; the node(s) with the highest sum are the most preferred.\",\"items\":{\"description\":\"An empty preferred scheduling term matches all objects with implicit weight 0 (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op).\",\"properties\":{\"preference\":{\"description\":\"A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm.\",\"properties\":{\"matchExpressions\":{\"description\":\"A list of node selector requirements by node's labels.\",\"items\":{\"description\":\"A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"The label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.\",\"type\":\"string\"},\"values\":{\"description\":\"An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"],\"type\":\"object\"},\"type\":\"array\"},\"matchFields\":{\"description\":\"A list of node selector requirements by node's fields.\",\"items\":{\"description\":\"A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"The label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.\",\"type\":\"string\"},\"values\":{\"description\":\"An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"],\"type\":\"object\"},\"type\":\"array\"}},\"type\":\"object\"},\"weight\":{\"description\":\"Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100.\",\"format\":\"int32\",\"type\":\"integer\"}},\"required\":[\"weight\",\"preference\"],\"type\":\"object\"},\"type\":\"array\"},\"requiredDuringSchedulingIgnoredDuringExecution\":{\"description\":\"A node selector represents the union of the results of one or more label queries over a set of nodes; that is, it represents the OR of the selectors represented by the node selector terms.\",\"properties\":{\"nodeSelectorTerms\":{\"description\":\"Required. A list of node selector terms. The terms are ORed.\",\"items\":{\"description\":\"A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm.\",\"properties\":{\"matchExpressions\":{\"description\":\"A list of node selector requirements by node's labels.\",\"items\":{\"description\":\"A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"The label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.\",\"type\":\"string\"},\"values\":{\"description\":\"An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"],\"type\":\"object\"},\"type\":\"array\"},\"matchFields\":{\"description\":\"A list of node selector requirements by node's fields.\",\"items\":{\"description\":\"A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"The label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.\",\"type\":\"string\"},\"values\":{\"description\":\"An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"],\"type\":\"object\"},\"type\":\"array\"}},\"type\":\"object\"},\"type\":\"array\"}},\"required\":[\"nodeSelectorTerms\"],\"type\":\"object\"}},\"type\":\"object\"},\"podAffinity\":{\"description\":\"Pod affinity is a group of inter pod affinity scheduling rules.\",\"properties\":{\"preferredDuringSchedulingIgnoredDuringExecution\":{\"description\":\"The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding \\\"weight\\\" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred.\",\"items\":{\"description\":\"The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s)\",\"properties\":{\"podAffinityTerm\":{\"description\":\"Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key \\u003ctopologyKey\\u003e matches that of any node on which a pod of the set of pods is running\",\"properties\":{\"labelSelector\":{\"description\":\"A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.\",\"properties\":{\"matchExpressions\":{\"description\":\"matchExpressions is a list of label selector requirements. The requirements are ANDed.\",\"items\":{\"description\":\"A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"key is the label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\",\"type\":\"string\"},\"values\":{\"description\":\"values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"],\"type\":\"object\"},\"type\":\"array\"},\"matchLabels\":{\"description\":\"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \\\"key\\\", the operator is \\\"In\\\", and the values array contains only \\\"value\\\". The requirements are ANDed.\",\"type\":\"object\"}},\"type\":\"object\"},\"namespaces\":{\"description\":\"namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \\\"this pod's namespace\\\"\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"topologyKey\":{\"description\":\"This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.\",\"type\":\"string\"}},\"required\":[\"topologyKey\"],\"type\":\"object\"},\"weight\":{\"description\":\"weight associated with matching the corresponding podAffinityTerm, in the range 1-100.\",\"format\":\"int32\",\"type\":\"integer\"}},\"required\":[\"weight\",\"podAffinityTerm\"],\"type\":\"object\"},\"type\":\"array\"},\"requiredDuringSchedulingIgnoredDuringExecution\":{\"description\":\"If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied.\",\"items\":{\"description\":\"Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key \\u003ctopologyKey\\u003e matches that of any node on which a pod of the set of pods is running\",\"properties\":{\"labelSelector\":{\"description\":\"A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.\",\"properties\":{\"matchExpressions\":{\"description\":\"matchExpressions is a list of label selector requirements. The requirements are ANDed.\",\"items\":{\"description\":\"A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"key is the label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\",\"type\":\"string\"},\"values\":{\"description\":\"values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"],\"type\":\"object\"},\"type\":\"array\"},\"matchLabels\":{\"description\":\"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \\\"key\\\", the operator is \\\"In\\\", and the values array contains only \\\"value\\\". The requirements are ANDed.\",\"type\":\"object\"}},\"type\":\"object\"},\"namespaces\":{\"description\":\"namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \\\"this pod's namespace\\\"\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"topologyKey\":{\"description\":\"This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.\",\"type\":\"string\"}},\"required\":[\"topologyKey\"],\"type\":\"object\"},\"type\":\"array\"}},\"type\":\"object\"},\"podAntiAffinity\":{\"description\":\"Pod anti affinity is a group of inter pod anti affinity scheduling rules.\",\"properties\":{\"preferredDuringSchedulingIgnoredDuringExecution\":{\"description\":\"The scheduler will prefer to schedule pods to nodes that satisfy the anti-affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding \\\"weight\\\" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred.\",\"items\":{\"description\":\"The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s)\",\"properties\":{\"podAffinityTerm\":{\"description\":\"Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key \\u003ctopologyKey\\u003e matches that of any node on which a pod of the set of pods is running\",\"properties\":{\"labelSelector\":{\"description\":\"A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.\",\"properties\":{\"matchExpressions\":{\"description\":\"matchExpressions is a list of label selector requirements. The requirements are ANDed.\",\"items\":{\"description\":\"A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"key is the label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\",\"type\":\"string\"},\"values\":{\"description\":\"values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"],\"type\":\"object\"},\"type\":\"array\"},\"matchLabels\":{\"description\":\"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \\\"key\\\", the operator is \\\"In\\\", and the values array contains only \\\"value\\\". The requirements are ANDed.\",\"type\":\"object\"}},\"type\":\"object\"},\"namespaces\":{\"description\":\"namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \\\"this pod's namespace\\\"\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"topologyKey\":{\"description\":\"This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.\",\"type\":\"string\"}},\"required\":[\"topologyKey\"],\"type\":\"object\"},\"weight\":{\"description\":\"weight associated with matching the corresponding podAffinityTerm, in the range 1-100.\",\"format\":\"int32\",\"type\":\"integer\"}},\"required\":[\"weight\",\"podAffinityTerm\"],\"type\":\"object\"},\"type\":\"array\"},\"requiredDuringSchedulingIgnoredDuringExecution\":{\"description\":\"If the anti-affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the anti-affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied.\",\"items\":{\"description\":\"Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key \\u003ctopologyKey\\u003e matches that of any node on which a pod of the set of pods is running\",\"properties\":{\"labelSelector\":{\"description\":\"A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.\",\"properties\":{\"matchExpressions\":{\"description\":\"matchExpressions is a list of label selector requirements. The requirements are ANDed.\",\"items\":{\"description\":\"A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"key is the label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\",\"type\":\"string\"},\"values\":{\"description\":\"values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"],\"type\":\"object\"},\"type\":\"array\"},\"matchLabels\":{\"description\":\"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \\\"key\\\", the operator is \\\"In\\\", and the values array contains only \\\"value\\\". The requirements are ANDed.\",\"type\":\"object\"}},\"type\":\"object\"},\"namespaces\":{\"description\":\"namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \\\"this pod's namespace\\\"\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"topologyKey\":{\"description\":\"This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.\",\"type\":\"string\"}},\"required\":[\"topologyKey\"],\"type\":\"object\"},\"type\":\"array\"}},\"type\":\"object\"}},\"type\":\"object\"},\"alerting\":{\"description\":\"AlertingSpec defines parameters for alerting configuration of Prometheus servers.\",\"properties\":{\"alertmanagers\":{\"description\":\"AlertmanagerEndpoints Prometheus should fire alerts against.\",\"items\":{\"description\":\"AlertmanagerEndpoints defines a selection of a single Endpoints object containing alertmanager IPs to fire alerts against.\",\"properties\":{\"bearerTokenFile\":{\"description\":\"BearerTokenFile to read from filesystem to use when authenticating to Alertmanager.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of Endpoints object in Namespace.\",\"type\":\"string\"},\"namespace\":{\"description\":\"Namespace of Endpoints object.\",\"type\":\"string\"},\"pathPrefix\":{\"description\":\"Prefix for the HTTP path alerts are pushed to.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"string\"},{\"type\":\"integer\"}]},\"scheme\":{\"description\":\"Scheme to use when firing alerts.\",\"type\":\"string\"},\"tlsConfig\":{\"description\":\"TLSConfig specifies TLS configuration parameters.\",\"properties\":{\"caFile\":{\"description\":\"The CA cert to use for the targets.\",\"type\":\"string\"},\"certFile\":{\"description\":\"The client cert file for the targets.\",\"type\":\"string\"},\"insecureSkipVerify\":{\"description\":\"Disable target certificate validation.\",\"type\":\"boolean\"},\"keyFile\":{\"description\":\"The client key file for the targets.\",\"type\":\"string\"},\"serverName\":{\"description\":\"Used to verify the hostname for the targets.\",\"type\":\"string\"}},\"type\":\"object\"}},\"required\":[\"namespace\",\"name\",\"port\"],\"type\":\"object\"},\"type\":\"array\"}},\"required\":[\"alertmanagers\"],\"type\":\"object\"},\"apiserverConfig\":{\"description\":\"APIServerConfig defines a host and auth methods to access apiserver. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config\",\"properties\":{\"basicAuth\":{\"description\":\"BasicAuth allow an endpoint to authenticate over basic authentication More info: https://prometheus.io/docs/operating/configuration/#endpoints\",\"properties\":{\"password\":{\"description\":\"SecretKeySelector selects a key of a Secret.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or it's key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"},\"username\":{\"description\":\"SecretKeySelector selects a key of a Secret.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or it's key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"}},\"type\":\"object\"},\"bearerToken\":{\"description\":\"Bearer token for accessing apiserver.\",\"type\":\"string\"},\"bearerTokenFile\":{\"description\":\"File to read bearer token for accessing apiserver.\",\"type\":\"string\"},\"host\":{\"description\":\"Host of apiserver. A valid string consisting of a hostname or IP followed by an optional port number\",\"type\":\"string\"},\"tlsConfig\":{\"description\":\"TLSConfig specifies TLS configuration parameters.\",\"properties\":{\"caFile\":{\"description\":\"The CA cert to use for the targets.\",\"type\":\"string\"},\"certFile\":{\"description\":\"The client cert file for the targets.\",\"type\":\"string\"},\"insecureSkipVerify\":{\"description\":\"Disable target certificate validation.\",\"type\":\"boolean\"},\"keyFile\":{\"description\":\"The client key file for the targets.\",\"type\":\"string\"},\"serverName\":{\"description\":\"Used to verify the hostname for the targets.\",\"type\":\"string\"}},\"type\":\"object\"}},\"required\":[\"host\"],\"type\":\"object\"},\"baseImage\":{\"description\":\"Base image to use for a Prometheus deployment.\",\"type\":\"string\"},\"configMaps\":{\"description\":\"ConfigMaps is a list of ConfigMaps in the same namespace as the Prometheus object, which shall be mounted into the Prometheus Pods. The ConfigMaps are mounted into /etc/prometheus/configmaps/\\u003cconfigmap-name\\u003e.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"containers\":{\"description\":\"Containers allows injecting additional containers or modifying operator generated containers. This can be used to allow adding an authentication proxy to a Prometheus pod or to change the behavior of an operator generated container. Containers described here modify an operator generated container if they share the same name and modifications are done via a strategic merge patch. The current container names are: `prometheus`, `prometheus-config-reloader`, `rules-configmap-reloader`, and `thanos-sidecar`. Overriding containers is entirely outside the scope of what the maintainers will support and by doing so, you accept that this behaviour may break at any time without notice.\",\"items\":{\"description\":\"A single application container that you want to run within a pod.\",\"properties\":{\"args\":{\"description\":\"Arguments to the entrypoint. The docker image's CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"command\":{\"description\":\"Entrypoint array. Not executed within a shell. The docker image's ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"env\":{\"description\":\"List of environment variables to set in the container. Cannot be updated.\",\"items\":{\"description\":\"EnvVar represents an environment variable present in a Container.\",\"properties\":{\"name\":{\"description\":\"Name of the environment variable. Must be a C_IDENTIFIER.\",\"type\":\"string\"},\"value\":{\"description\":\"Variable references $(VAR_NAME) are expanded using the previous defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Defaults to \\\"\\\".\",\"type\":\"string\"},\"valueFrom\":{\"description\":\"EnvVarSource represents a source for the value of an EnvVar.\",\"properties\":{\"configMapKeyRef\":{\"description\":\"Selects a key from a ConfigMap.\",\"properties\":{\"key\":{\"description\":\"The key to select.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the ConfigMap or it's key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"},\"fieldRef\":{\"description\":\"ObjectFieldSelector selects an APIVersioned field of an object.\",\"properties\":{\"apiVersion\":{\"description\":\"Version of the schema the FieldPath is written in terms of, defaults to \\\"v1\\\".\",\"type\":\"string\"},\"fieldPath\":{\"description\":\"Path of the field to select in the specified API version.\",\"type\":\"string\"}},\"required\":[\"fieldPath\"],\"type\":\"object\"},\"resourceFieldRef\":{\"description\":\"ResourceFieldSelector represents container resources (cpu, memory) and their output format\",\"properties\":{\"containerName\":{\"description\":\"Container name: required for volumes, optional for env vars\",\"type\":\"string\"},\"divisor\":{},\"resource\":{\"description\":\"Required: resource to select\",\"type\":\"string\"}},\"required\":[\"resource\"],\"type\":\"object\"},\"secretKeyRef\":{\"description\":\"SecretKeySelector selects a key of a Secret.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or it's key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"}},\"type\":\"object\"}},\"required\":[\"name\"],\"type\":\"object\"},\"type\":\"array\"},\"envFrom\":{\"description\":\"List of sources to populate environment variables in the container. The keys defined within a source must be a C_IDENTIFIER. All invalid keys will be reported as an event when the container is starting. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated.\",\"items\":{\"description\":\"EnvFromSource represents the source of a set of ConfigMaps\",\"properties\":{\"configMapRef\":{\"description\":\"ConfigMapEnvSource selects a ConfigMap to populate the environment variables with.\\n\\nThe contents of the target ConfigMap's Data field will represent the key-value pairs as environment variables.\",\"properties\":{\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the ConfigMap must be defined\",\"type\":\"boolean\"}},\"type\":\"object\"},\"prefix\":{\"description\":\"An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER.\",\"type\":\"string\"},\"secretRef\":{\"description\":\"SecretEnvSource selects a Secret to populate the environment variables with.\\n\\nThe contents of the target Secret's Data field will represent the key-value pairs as environment variables.\",\"properties\":{\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret must be defined\",\"type\":\"boolean\"}},\"type\":\"object\"}},\"type\":\"object\"},\"type\":\"array\"},\"image\":{\"description\":\"Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images This field is optional to allow higher level config management to default or override container images in workload controllers like Deployments and StatefulSets.\",\"type\":\"string\"},\"imagePullPolicy\":{\"description\":\"Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images\",\"type\":\"string\"},\"lifecycle\":{\"description\":\"Lifecycle describes actions that the management system should take in response to container lifecycle events. For the PostStart and PreStop lifecycle handlers, management of the container blocks until the action is complete, unless the container process fails, in which case the handler is aborted.\",\"properties\":{\"postStart\":{\"description\":\"Handler defines a specific action that should be taken\",\"properties\":{\"exec\":{\"description\":\"ExecAction describes a \\\"run in container\\\" action.\",\"properties\":{\"command\":{\"description\":\"Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"type\":\"object\"},\"httpGet\":{\"description\":\"HTTPGetAction describes an action based on HTTP Get requests.\",\"properties\":{\"host\":{\"description\":\"Host name to connect to, defaults to the pod IP. You probably want to set \\\"Host\\\" in httpHeaders instead.\",\"type\":\"string\"},\"httpHeaders\":{\"description\":\"Custom headers to set in the request. HTTP allows repeated headers.\",\"items\":{\"description\":\"HTTPHeader describes a custom header to be used in HTTP probes\",\"properties\":{\"name\":{\"description\":\"The header field name\",\"type\":\"string\"},\"value\":{\"description\":\"The header field value\",\"type\":\"string\"}},\"required\":[\"name\",\"value\"],\"type\":\"object\"},\"type\":\"array\"},\"path\":{\"description\":\"Path to access on the HTTP server.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"string\"},{\"type\":\"integer\"}]},\"scheme\":{\"description\":\"Scheme to use for connecting to the host. Defaults to HTTP.\",\"type\":\"string\"}},\"required\":[\"port\"],\"type\":\"object\"},\"tcpSocket\":{\"description\":\"TCPSocketAction describes an action based on opening a socket\",\"properties\":{\"host\":{\"description\":\"Optional: Host name to connect to, defaults to the pod IP.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"string\"},{\"type\":\"integer\"}]}},\"required\":[\"port\"],\"type\":\"object\"}},\"type\":\"object\"},\"preStop\":{\"description\":\"Handler defines a specific action that should be taken\",\"properties\":{\"exec\":{\"description\":\"ExecAction describes a \\\"run in container\\\" action.\",\"properties\":{\"command\":{\"description\":\"Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"type\":\"object\"},\"httpGet\":{\"description\":\"HTTPGetAction describes an action based on HTTP Get requests.\",\"properties\":{\"host\":{\"description\":\"Host name to connect to, defaults to the pod IP. You probably want to set \\\"Host\\\" in httpHeaders instead.\",\"type\":\"string\"},\"httpHeaders\":{\"description\":\"Custom headers to set in the request. HTTP allows repeated headers.\",\"items\":{\"description\":\"HTTPHeader describes a custom header to be used in HTTP probes\",\"properties\":{\"name\":{\"description\":\"The header field name\",\"type\":\"string\"},\"value\":{\"description\":\"The header field value\",\"type\":\"string\"}},\"required\":[\"name\",\"value\"],\"type\":\"object\"},\"type\":\"array\"},\"path\":{\"description\":\"Path to access on the HTTP server.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"string\"},{\"type\":\"integer\"}]},\"scheme\":{\"description\":\"Scheme to use for connecting to the host. Defaults to HTTP.\",\"type\":\"string\"}},\"required\":[\"port\"],\"type\":\"object\"},\"tcpSocket\":{\"description\":\"TCPSocketAction describes an action based on opening a socket\",\"properties\":{\"host\":{\"description\":\"Optional: Host name to connect to, defaults to the pod IP.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"string\"},{\"type\":\"integer\"}]}},\"required\":[\"port\"],\"type\":\"object\"}},\"type\":\"object\"}},\"type\":\"object\"},\"livenessProbe\":{\"description\":\"Probe describes a health check to be performed against a container to determine whether it is alive or ready to receive traffic.\",\"properties\":{\"exec\":{\"description\":\"ExecAction describes a \\\"run in container\\\" action.\",\"properties\":{\"command\":{\"description\":\"Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"type\":\"object\"},\"failureThreshold\":{\"description\":\"Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.\",\"format\":\"int32\",\"type\":\"integer\"},\"httpGet\":{\"description\":\"HTTPGetAction describes an action based on HTTP Get requests.\",\"properties\":{\"host\":{\"description\":\"Host name to connect to, defaults to the pod IP. You probably want to set \\\"Host\\\" in httpHeaders instead.\",\"type\":\"string\"},\"httpHeaders\":{\"description\":\"Custom headers to set in the request. HTTP allows repeated headers.\",\"items\":{\"description\":\"HTTPHeader describes a custom header to be used in HTTP probes\",\"properties\":{\"name\":{\"description\":\"The header field name\",\"type\":\"string\"},\"value\":{\"description\":\"The header field value\",\"type\":\"string\"}},\"required\":[\"name\",\"value\"],\"type\":\"object\"},\"type\":\"array\"},\"path\":{\"description\":\"Path to access on the HTTP server.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"string\"},{\"type\":\"integer\"}]},\"scheme\":{\"description\":\"Scheme to use for connecting to the host. Defaults to HTTP.\",\"type\":\"string\"}},\"required\":[\"port\"],\"type\":\"object\"},\"initialDelaySeconds\":{\"description\":\"Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\",\"format\":\"int32\",\"type\":\"integer\"},\"periodSeconds\":{\"description\":\"How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.\",\"format\":\"int32\",\"type\":\"integer\"},\"successThreshold\":{\"description\":\"Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness. Minimum value is 1.\",\"format\":\"int32\",\"type\":\"integer\"},\"tcpSocket\":{\"description\":\"TCPSocketAction describes an action based on opening a socket\",\"properties\":{\"host\":{\"description\":\"Optional: Host name to connect to, defaults to the pod IP.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"string\"},{\"type\":\"integer\"}]}},\"required\":[\"port\"],\"type\":\"object\"},\"timeoutSeconds\":{\"description\":\"Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\",\"format\":\"int32\",\"type\":\"integer\"}},\"type\":\"object\"},\"name\":{\"description\":\"Name of the container specified as a DNS_LABEL. Each container in a pod must have a unique name (DNS_LABEL). Cannot be updated.\",\"type\":\"string\"},\"ports\":{\"description\":\"List of ports to expose from the container. Exposing a port here gives the system additional information about the network connections a container uses, but is primarily informational. Not specifying a port here DOES NOT prevent that port from being exposed. Any port which is listening on the default \\\"0.0.0.0\\\" address inside a container will be accessible from the network. Cannot be updated.\",\"items\":{\"description\":\"ContainerPort represents a network port in a single container.\",\"properties\":{\"containerPort\":{\"description\":\"Number of port to expose on the pod's IP address. This must be a valid port number, 0 \\u003c x \\u003c 65536.\",\"format\":\"int32\",\"type\":\"integer\"},\"hostIP\":{\"description\":\"What host IP to bind the external port to.\",\"type\":\"string\"},\"hostPort\":{\"description\":\"Number of port to expose on the host. If specified, this must be a valid port number, 0 \\u003c x \\u003c 65536. If HostNetwork is specified, this must match ContainerPort. Most containers do not need this.\",\"format\":\"int32\",\"type\":\"integer\"},\"name\":{\"description\":\"If specified, this must be an IANA_SVC_NAME and unique within the pod. Each named port in a pod must have a unique name. Name for the port that can be referred to by services.\",\"type\":\"string\"},\"protocol\":{\"description\":\"Protocol for port. Must be UDP, TCP, or SCTP. Defaults to \\\"TCP\\\".\",\"type\":\"string\"}},\"required\":[\"containerPort\"],\"type\":\"object\"},\"type\":\"array\"},\"readinessProbe\":{\"description\":\"Probe describes a health check to be performed against a container to determine whether it is alive or ready to receive traffic.\",\"properties\":{\"exec\":{\"description\":\"ExecAction describes a \\\"run in container\\\" action.\",\"properties\":{\"command\":{\"description\":\"Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"type\":\"object\"},\"failureThreshold\":{\"description\":\"Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.\",\"format\":\"int32\",\"type\":\"integer\"},\"httpGet\":{\"description\":\"HTTPGetAction describes an action based on HTTP Get requests.\",\"properties\":{\"host\":{\"description\":\"Host name to connect to, defaults to the pod IP. You probably want to set \\\"Host\\\" in httpHeaders instead.\",\"type\":\"string\"},\"httpHeaders\":{\"description\":\"Custom headers to set in the request. HTTP allows repeated headers.\",\"items\":{\"description\":\"HTTPHeader describes a custom header to be used in HTTP probes\",\"properties\":{\"name\":{\"description\":\"The header field name\",\"type\":\"string\"},\"value\":{\"description\":\"The header field value\",\"type\":\"string\"}},\"required\":[\"name\",\"value\"],\"type\":\"object\"},\"type\":\"array\"},\"path\":{\"description\":\"Path to access on the HTTP server.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"string\"},{\"type\":\"integer\"}]},\"scheme\":{\"description\":\"Scheme to use for connecting to the host. Defaults to HTTP.\",\"type\":\"string\"}},\"required\":[\"port\"],\"type\":\"object\"},\"initialDelaySeconds\":{\"description\":\"Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\",\"format\":\"int32\",\"type\":\"integer\"},\"periodSeconds\":{\"description\":\"How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.\",\"format\":\"int32\",\"type\":\"integer\"},\"successThreshold\":{\"description\":\"Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness. Minimum value is 1.\",\"format\":\"int32\",\"type\":\"integer\"},\"tcpSocket\":{\"description\":\"TCPSocketAction describes an action based on opening a socket\",\"properties\":{\"host\":{\"description\":\"Optional: Host name to connect to, defaults to the pod IP.\",\"type\":\"string\"},\"port\":{\"anyOf\":[{\"type\":\"string\"},{\"type\":\"integer\"}]}},\"required\":[\"port\"],\"type\":\"object\"},\"timeoutSeconds\":{\"description\":\"Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\",\"format\":\"int32\",\"type\":\"integer\"}},\"type\":\"object\"},\"resources\":{\"description\":\"ResourceRequirements describes the compute resource requirements.\",\"properties\":{\"limits\":{\"description\":\"Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/\",\"type\":\"object\"},\"requests\":{\"description\":\"Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/\",\"type\":\"object\"}},\"type\":\"object\"},\"securityContext\":{\"description\":\"SecurityContext holds security configuration that will be applied to a container. Some fields are present in both SecurityContext and PodSecurityContext. When both are set, the values in SecurityContext take precedence.\",\"properties\":{\"allowPrivilegeEscalation\":{\"description\":\"AllowPrivilegeEscalation controls whether a process can gain more privileges than its parent process. This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged 2) has CAP_SYS_ADMIN\",\"type\":\"boolean\"},\"capabilities\":{\"description\":\"Adds and removes POSIX capabilities from running containers.\",\"properties\":{\"add\":{\"description\":\"Added capabilities\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"drop\":{\"description\":\"Removed capabilities\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"type\":\"object\"},\"privileged\":{\"description\":\"Run container in privileged mode. Processes in privileged containers are essentially equivalent to root on the host. Defaults to false.\",\"type\":\"boolean\"},\"procMount\":{\"description\":\"procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled.\",\"type\":\"string\"},\"readOnlyRootFilesystem\":{\"description\":\"Whether this container has a read-only root filesystem. Default is false.\",\"type\":\"boolean\"},\"runAsGroup\":{\"description\":\"The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\",\"format\":\"int64\",\"type\":\"integer\"},\"runAsNonRoot\":{\"description\":\"Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\",\"type\":\"boolean\"},\"runAsUser\":{\"description\":\"The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\",\"format\":\"int64\",\"type\":\"integer\"},\"seLinuxOptions\":{\"description\":\"SELinuxOptions are the labels to be applied to the container\",\"properties\":{\"level\":{\"description\":\"Level is SELinux level label that applies to the container.\",\"type\":\"string\"},\"role\":{\"description\":\"Role is a SELinux role label that applies to the container.\",\"type\":\"string\"},\"type\":{\"description\":\"Type is a SELinux type label that applies to the container.\",\"type\":\"string\"},\"user\":{\"description\":\"User is a SELinux user label that applies to the container.\",\"type\":\"string\"}},\"type\":\"object\"}},\"type\":\"object\"},\"stdin\":{\"description\":\"Whether this container should allocate a buffer for stdin in the container runtime. If this is not set, reads from stdin in the container will always result in EOF. Default is false.\",\"type\":\"boolean\"},\"stdinOnce\":{\"description\":\"Whether the container runtime should close the stdin channel after it has been opened by a single attach. When stdin is true the stdin stream will remain open across multiple attach sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the first client attaches to stdin, and then remains open and accepts data until the client disconnects, at which time stdin is closed and remains closed until the container is restarted. If this flag is false, a container processes that reads from stdin will never receive an EOF. Default is false\",\"type\":\"boolean\"},\"terminationMessagePath\":{\"description\":\"Optional: Path at which the file to which the container's termination message will be written is mounted into the container's filesystem. Message written is intended to be brief final status, such as an assertion failure message. Will be truncated by the node if greater than 4096 bytes. The total message length across all containers will be limited to 12kb. Defaults to /dev/termination-log. Cannot be updated.\",\"type\":\"string\"},\"terminationMessagePolicy\":{\"description\":\"Indicate how the termination message should be populated. File will use the contents of terminationMessagePath to populate the container status message on both success and failure. FallbackToLogsOnError will use the last chunk of container log output if the termination message file is empty and the container exited with an error. The log output is limited to 2048 bytes or 80 lines, whichever is smaller. Defaults to File. Cannot be updated.\",\"type\":\"string\"},\"tty\":{\"description\":\"Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. Default is false.\",\"type\":\"boolean\"},\"volumeDevices\":{\"description\":\"volumeDevices is the list of block devices to be used by the container. This is a beta feature.\",\"items\":{\"description\":\"volumeDevice describes a mapping of a raw block device within a container.\",\"properties\":{\"devicePath\":{\"description\":\"devicePath is the path inside of the container that the device will be mapped to.\",\"type\":\"string\"},\"name\":{\"description\":\"name must match the name of a persistentVolumeClaim in the pod\",\"type\":\"string\"}},\"required\":[\"name\",\"devicePath\"],\"type\":\"object\"},\"type\":\"array\"},\"volumeMounts\":{\"description\":\"Pod volumes to mount into the container's filesystem. Cannot be updated.\",\"items\":{\"description\":\"VolumeMount describes a mounting of a Volume within a container.\",\"properties\":{\"mountPath\":{\"description\":\"Path within the container at which the volume should be mounted. Must not contain ':'.\",\"type\":\"string\"},\"mountPropagation\":{\"description\":\"mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10.\",\"type\":\"string\"},\"name\":{\"description\":\"This must match the Name of a Volume.\",\"type\":\"string\"},\"readOnly\":{\"description\":\"Mounted read-only if true, read-write otherwise (false or unspecified). Defaults to false.\",\"type\":\"boolean\"},\"subPath\":{\"description\":\"Path within the volume from which the container's volume should be mounted. Defaults to \\\"\\\" (volume's root).\",\"type\":\"string\"},\"subPathExpr\":{\"description\":\"Expanded path within the volume from which the container's volume should be mounted. Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. Defaults to \\\"\\\" (volume's root). SubPathExpr and SubPath are mutually exclusive. This field is alpha in 1.14.\",\"type\":\"string\"}},\"required\":[\"name\",\"mountPath\"],\"type\":\"object\"},\"type\":\"array\"},\"workingDir\":{\"description\":\"Container's working directory. If not specified, the container runtime's default will be used, which might be configured in the container image. Cannot be updated.\",\"type\":\"string\"}},\"required\":[\"name\"],\"type\":\"object\"},\"type\":\"array\"},\"enableAdminAPI\":{\"description\":\"Enable access to prometheus web admin API. Defaults to the value of `false`. WARNING: Enabling the admin APIs enables mutating endpoints, to delete data, shutdown Prometheus, and more. Enabling this should be done with care and the user is advised to add additional authentication authorization via a proxy to ensure only clients authorized to perform these actions can do so. For more information see https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-admin-apis\",\"type\":\"boolean\"},\"evaluationInterval\":{\"description\":\"Interval between consecutive evaluations.\",\"type\":\"string\"},\"externalLabels\":{\"description\":\"The labels to add to any time series or alerts when communicating with external systems (federation, remote storage, Alertmanager).\",\"type\":\"object\"},\"externalUrl\":{\"description\":\"The external URL the Prometheus instances will be available under. This is necessary to generate correct URLs. This is necessary if Prometheus is not served from root of a DNS name.\",\"type\":\"string\"},\"image\":{\"description\":\"Image if specified has precedence over baseImage, tag and sha combinations. Specifying the version is still necessary to ensure the Prometheus Operator knows what version of Prometheus is being configured.\",\"type\":\"string\"},\"imagePullSecrets\":{\"description\":\"An optional list of references to secrets in the same namespace to use for pulling prometheus and alertmanager images from registries see http://kubernetes.io/docs/user-guide/images#specifying-imagepullsecrets-on-a-pod\",\"items\":{\"description\":\"LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace.\",\"properties\":{\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\",\"type\":\"string\"}},\"type\":\"object\"},\"type\":\"array\"},\"listenLocal\":{\"description\":\"ListenLocal makes the Prometheus server listen on loopback, so that it does not bind against the Pod IP.\",\"type\":\"boolean\"},\"logFormat\":{\"description\":\"Log format for Prometheus to be configured with.\",\"type\":\"string\"},\"logLevel\":{\"description\":\"Log level for Prometheus to be configured with.\",\"type\":\"string\"},\"nodeSelector\":{\"description\":\"Define which Nodes the Pods are scheduled on.\",\"type\":\"object\"},\"paused\":{\"description\":\"When a Prometheus deployment is paused, no actions except for deletion will be performed on the underlying objects.\",\"type\":\"boolean\"},\"podMetadata\":{\"description\":\"ObjectMeta is metadata that all persisted resources must have, which includes all objects users must create.\",\"properties\":{\"annotations\":{\"description\":\"Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations\",\"type\":\"object\"},\"clusterName\":{\"description\":\"The name of the cluster which the object belongs to. This is used to distinguish resources with same name and namespace in different clusters. This field is not set anywhere right now and apiserver is going to ignore it if set in create or update request.\",\"type\":\"string\"},\"creationTimestamp\":{\"description\":\"Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.\",\"format\":\"date-time\",\"type\":\"string\"},\"deletionGracePeriodSeconds\":{\"description\":\"Number of seconds allowed for this object to gracefully terminate before it will be removed from the system. Only set when deletionTimestamp is also set. May only be shortened. Read-only.\",\"format\":\"int64\",\"type\":\"integer\"},\"deletionTimestamp\":{\"description\":\"Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.\",\"format\":\"date-time\",\"type\":\"string\"},\"finalizers\":{\"description\":\"Must be empty before the object is deleted from the registry. Each entry is an identifier for the responsible component that will remove the entry from the list. If the deletionTimestamp of the object is non-nil, entries in this list can only be removed.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"generateName\":{\"description\":\"GenerateName is an optional prefix, used by the server, to generate a unique name ONLY IF the Name field has not been provided. If this field is used, the name returned to the client will be different than the name passed. This value will also be combined with a unique suffix. The provided value has the same validation rules as the Name field, and may be truncated by the length of the suffix required to make the value unique on the server.\\n\\nIf this field is specified and the generated name exists, the server will NOT return a 409 - instead, it will either return 201 Created or 500 with Reason ServerTimeout indicating a unique name could not be found in the time allotted, and the client should retry (optionally after the time indicated in the Retry-After header).\\n\\nApplied only if Name is not specified. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#idempotency\",\"type\":\"string\"},\"generation\":{\"description\":\"A sequence number representing a specific generation of the desired state. Populated by the system. Read-only.\",\"format\":\"int64\",\"type\":\"integer\"},\"initializers\":{\"description\":\"Initializers tracks the progress of initialization.\",\"properties\":{\"pending\":{\"description\":\"Pending is a list of initializers that must execute in order before this object is visible. When the last pending initializer is removed, and no failing result is set, the initializers struct will be set to nil and the object is considered as initialized and visible to all clients.\",\"items\":{\"description\":\"Initializer is information about an initializer that has not yet completed.\",\"properties\":{\"name\":{\"description\":\"name of the process that is responsible for initializing this object.\",\"type\":\"string\"}},\"required\":[\"name\"],\"type\":\"object\"},\"type\":\"array\"},\"result\":{\"description\":\"Status is a return value for calls that don't return other objects.\",\"properties\":{\"apiVersion\":{\"description\":\"APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources\",\"type\":\"string\"},\"code\":{\"description\":\"Suggested HTTP return code for this status, 0 if not set.\",\"format\":\"int32\",\"type\":\"integer\"},\"details\":{\"description\":\"StatusDetails is a set of additional properties that MAY be set by the server to provide additional information about a response. The Reason field of a Status object defines what attributes will be set. Clients must ignore fields that do not match the defined type of each attribute, and should assume that any attribute may be empty, invalid, or under defined.\",\"properties\":{\"causes\":{\"description\":\"The Causes array includes more details associated with the StatusReason failure. Not all StatusReasons may provide detailed causes.\",\"items\":{\"description\":\"StatusCause provides more information about an api.Status failure, including cases when multiple errors are encountered.\",\"properties\":{\"field\":{\"description\":\"The field of the resource that has caused this error, as named by its JSON serialization. May include dot and postfix notation for nested attributes. Arrays are zero-indexed. Fields may appear more than once in an array of causes due to fields having multiple errors. Optional.\\n\\nExamples:\\n \\\"name\\\" - the field \\\"name\\\" on the current resource\\n \\\"items[0].name\\\" - the field \\\"name\\\" on the first array entry in \\\"items\\\"\",\"type\":\"string\"},\"message\":{\"description\":\"A human-readable description of the cause of the error. This field may be presented as-is to a reader.\",\"type\":\"string\"},\"reason\":{\"description\":\"A machine-readable description of the cause of the error. If this value is empty there is no information available.\",\"type\":\"string\"}},\"type\":\"object\"},\"type\":\"array\"},\"group\":{\"description\":\"The group attribute of the resource associated with the status StatusReason.\",\"type\":\"string\"},\"kind\":{\"description\":\"The kind attribute of the resource associated with the status StatusReason. On some operations may differ from the requested resource Kind. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds\",\"type\":\"string\"},\"name\":{\"description\":\"The name attribute of the resource associated with the status StatusReason (when there is a single name which can be described).\",\"type\":\"string\"},\"retryAfterSeconds\":{\"description\":\"If specified, the time in seconds before the operation should be retried. Some errors may indicate the client must take an alternate action - for those errors this field may indicate how long to wait before taking the alternate action.\",\"format\":\"int32\",\"type\":\"integer\"},\"uid\":{\"description\":\"UID of the resource. (when there is a single resource which can be described). More info: http://kubernetes.io/docs/user-guide/identifiers#uids\",\"type\":\"string\"}},\"type\":\"object\"},\"kind\":{\"description\":\"Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds\",\"type\":\"string\"},\"message\":{\"description\":\"A human-readable description of the status of this operation.\",\"type\":\"string\"},\"metadata\":{\"description\":\"ListMeta describes metadata that synthetic resources must have, including lists and various status objects. A resource may have only one of {ObjectMeta, ListMeta}.\",\"properties\":{\"continue\":{\"description\":\"continue may be set if the user set a limit on the number of items returned, and indicates that the server has more data available. The value is opaque and may be used to issue another request to the endpoint that served this list to retrieve the next set of available objects. Continuing a consistent list may not be possible if the server configuration has changed or more than a few minutes have passed. The resourceVersion field returned when using this continue value will be identical to the value in the first response, unless you have received this token from an error message.\",\"type\":\"string\"},\"resourceVersion\":{\"description\":\"String that identifies the server's internal version of this object that can be used by clients to determine when objects have changed. Value must be treated as opaque by clients and passed unmodified back to the server. Populated by the system. Read-only. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency\",\"type\":\"string\"},\"selfLink\":{\"description\":\"selfLink is a URL representing this object. Populated by the system. Read-only.\",\"type\":\"string\"}},\"type\":\"object\"},\"reason\":{\"description\":\"A machine-readable description of why this operation is in the \\\"Failure\\\" status. If this value is empty there is no information available. A Reason clarifies an HTTP status code but does not override it.\",\"type\":\"string\"},\"status\":{\"description\":\"Status of the operation. One of: \\\"Success\\\" or \\\"Failure\\\". More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status\",\"type\":\"string\"}},\"type\":\"object\"}},\"required\":[\"pending\"],\"type\":\"object\"},\"labels\":{\"description\":\"Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels\",\"type\":\"object\"},\"managedFields\":{\"description\":\"ManagedFields maps workflow-id and version to the set of fields that are managed by that workflow. This is mostly for internal housekeeping, and users typically shouldn't need to set or understand this field. A workflow can be the user's name, a controller's name, or the name of a specific apply path like \\\"ci-cd\\\". The set of fields is always in the version that the workflow used when modifying the object.\\n\\nThis field is alpha and can be changed or removed without notice.\",\"items\":{\"description\":\"ManagedFieldsEntry is a workflow-id, a FieldSet and the group version of the resource that the fieldset applies to.\",\"properties\":{\"apiVersion\":{\"description\":\"APIVersion defines the version of this resource that this field set applies to. The format is \\\"group/version\\\" just like the top-level APIVersion field. It is necessary to track the version of a field set because it cannot be automatically converted.\",\"type\":\"string\"},\"fields\":{\"description\":\"Fields stores a set of fields in a data structure like a Trie. To understand how this is used, see: https://github.com/kubernetes-sigs/structured-merge-diff\",\"type\":\"object\"},\"manager\":{\"description\":\"Manager is an identifier of the workflow managing these fields.\",\"type\":\"string\"},\"operation\":{\"description\":\"Operation is the type of operation which lead to this ManagedFieldsEntry being created. The only valid values for this field are 'Apply' and 'Update'.\",\"type\":\"string\"},\"time\":{\"description\":\"Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.\",\"format\":\"date-time\",\"type\":\"string\"}},\"type\":\"object\"},\"type\":\"array\"},\"name\":{\"description\":\"Name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names\",\"type\":\"string\"},\"namespace\":{\"description\":\"Namespace defines the space within each name must be unique. An empty namespace is equivalent to the \\\"default\\\" namespace, but \\\"default\\\" is the canonical representation. Not all objects are required to be scoped to a namespace - the value of this field for those objects will be empty.\\n\\nMust be a DNS_LABEL. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/namespaces\",\"type\":\"string\"},\"ownerReferences\":{\"description\":\"List of objects depended by this object. If ALL objects in the list have been deleted, this object will be garbage collected. If this object is managed by a controller, then an entry in this list will point to this controller, with the controller field set to true. There cannot be more than one managing controller.\",\"items\":{\"description\":\"OwnerReference contains enough information to let you identify an owning object. An owning object must be in the same namespace as the dependent, or be cluster-scoped, so there is no namespace field.\",\"properties\":{\"apiVersion\":{\"description\":\"API version of the referent.\",\"type\":\"string\"},\"blockOwnerDeletion\":{\"description\":\"If true, AND if the owner has the \\\"foregroundDeletion\\\" finalizer, then the owner cannot be deleted from the key-value store until this reference is removed. Defaults to false. To set this field, a user needs \\\"delete\\\" permission of the owner, otherwise 422 (Unprocessable Entity) will be returned.\",\"type\":\"boolean\"},\"controller\":{\"description\":\"If true, this reference points to the managing controller.\",\"type\":\"boolean\"},\"kind\":{\"description\":\"Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#names\",\"type\":\"string\"},\"uid\":{\"description\":\"UID of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#uids\",\"type\":\"string\"}},\"required\":[\"apiVersion\",\"kind\",\"name\",\"uid\"],\"type\":\"object\"},\"type\":\"array\"},\"resourceVersion\":{\"description\":\"An opaque value that represents the internal version of this object that can be used by clients to determine when objects have changed. May be used for optimistic concurrency, change detection, and the watch operation on a resource or set of resources. Clients must treat these values as opaque and passed unmodified back to the server. They may only be valid for a particular resource or set of resources.\\n\\nPopulated by the system. Read-only. Value must be treated as opaque by clients and . More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency\",\"type\":\"string\"},\"selfLink\":{\"description\":\"SelfLink is a URL representing this object. Populated by the system. Read-only.\",\"type\":\"string\"},\"uid\":{\"description\":\"UID is the unique in time and space value for this object. It is typically generated by the server on successful creation of a resource and is not allowed to change on PUT operations.\\n\\nPopulated by the system. Read-only. More info: http://kubernetes.io/docs/user-guide/identifiers#uids\",\"type\":\"string\"}},\"type\":\"object\"},\"podMonitorNamespaceSelector\":{\"description\":\"A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.\",\"properties\":{\"matchExpressions\":{\"description\":\"matchExpressions is a list of label selector requirements. The requirements are ANDed.\",\"items\":{\"description\":\"A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"key is the label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\",\"type\":\"string\"},\"values\":{\"description\":\"values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"],\"type\":\"object\"},\"type\":\"array\"},\"matchLabels\":{\"description\":\"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \\\"key\\\", the operator is \\\"In\\\", and the values array contains only \\\"value\\\". The requirements are ANDed.\",\"type\":\"object\"}},\"type\":\"object\"},\"podMonitorSelector\":{\"description\":\"A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.\",\"properties\":{\"matchExpressions\":{\"description\":\"matchExpressions is a list of label selector requirements. The requirements are ANDed.\",\"items\":{\"description\":\"A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"key is the label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\",\"type\":\"string\"},\"values\":{\"description\":\"values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"],\"type\":\"object\"},\"type\":\"array\"},\"matchLabels\":{\"description\":\"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \\\"key\\\", the operator is \\\"In\\\", and the values array contains only \\\"value\\\". The requirements are ANDed.\",\"type\":\"object\"}},\"type\":\"object\"},\"priorityClassName\":{\"description\":\"Priority class assigned to the Pods\",\"type\":\"string\"},\"prometheusExternalLabelName\":{\"description\":\"Name of Prometheus external label used to denote Prometheus instance name. Defaults to the value of `prometheus`. External label will _not_ be added when value is set to empty string (`\\\"\\\"`).\",\"type\":\"string\"},\"query\":{\"description\":\"QuerySpec defines the query command line flags when starting Prometheus.\",\"properties\":{\"lookbackDelta\":{\"description\":\"The delta difference allowed for retrieving metrics during expression evaluations.\",\"type\":\"string\"},\"maxConcurrency\":{\"description\":\"Number of concurrent queries that can be run at once.\",\"format\":\"int32\",\"type\":\"integer\"},\"maxSamples\":{\"description\":\"Maximum number of samples a single query can load into memory. Note that queries will fail if they would load more samples than this into memory, so this also limits the number of samples a query can return.\",\"format\":\"int32\",\"type\":\"integer\"},\"timeout\":{\"description\":\"Maximum time a query may take before being aborted.\",\"type\":\"string\"}},\"type\":\"object\"},\"remoteRead\":{\"description\":\"If specified, the remote_read spec. This is an experimental feature, it may change in any upcoming release in a breaking way.\",\"items\":{\"description\":\"RemoteReadSpec defines the remote_read configuration for prometheus.\",\"properties\":{\"basicAuth\":{\"description\":\"BasicAuth allow an endpoint to authenticate over basic authentication More info: https://prometheus.io/docs/operating/configuration/#endpoints\",\"properties\":{\"password\":{\"description\":\"SecretKeySelector selects a key of a Secret.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or it's key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"},\"username\":{\"description\":\"SecretKeySelector selects a key of a Secret.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or it's key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"}},\"type\":\"object\"},\"bearerToken\":{\"description\":\"bearer token for remote read.\",\"type\":\"string\"},\"bearerTokenFile\":{\"description\":\"File to read bearer token for remote read.\",\"type\":\"string\"},\"proxyUrl\":{\"description\":\"Optional ProxyURL\",\"type\":\"string\"},\"readRecent\":{\"description\":\"Whether reads should be made for queries for time ranges that the local storage should have complete data for.\",\"type\":\"boolean\"},\"remoteTimeout\":{\"description\":\"Timeout for requests to the remote read endpoint.\",\"type\":\"string\"},\"requiredMatchers\":{\"description\":\"An optional list of equality matchers which have to be present in a selector to query the remote read endpoint.\",\"type\":\"object\"},\"tlsConfig\":{\"description\":\"TLSConfig specifies TLS configuration parameters.\",\"properties\":{\"caFile\":{\"description\":\"The CA cert to use for the targets.\",\"type\":\"string\"},\"certFile\":{\"description\":\"The client cert file for the targets.\",\"type\":\"string\"},\"insecureSkipVerify\":{\"description\":\"Disable target certificate validation.\",\"type\":\"boolean\"},\"keyFile\":{\"description\":\"The client key file for the targets.\",\"type\":\"string\"},\"serverName\":{\"description\":\"Used to verify the hostname for the targets.\",\"type\":\"string\"}},\"type\":\"object\"},\"url\":{\"description\":\"The URL of the endpoint to send samples to.\",\"type\":\"string\"}},\"required\":[\"url\"],\"type\":\"object\"},\"type\":\"array\"},\"remoteWrite\":{\"description\":\"If specified, the remote_write spec. This is an experimental feature, it may change in any upcoming release in a breaking way.\",\"items\":{\"description\":\"RemoteWriteSpec defines the remote_write configuration for prometheus.\",\"properties\":{\"basicAuth\":{\"description\":\"BasicAuth allow an endpoint to authenticate over basic authentication More info: https://prometheus.io/docs/operating/configuration/#endpoints\",\"properties\":{\"password\":{\"description\":\"SecretKeySelector selects a key of a Secret.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or it's key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"},\"username\":{\"description\":\"SecretKeySelector selects a key of a Secret.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or it's key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"}},\"type\":\"object\"},\"bearerToken\":{\"description\":\"File to read bearer token for remote write.\",\"type\":\"string\"},\"bearerTokenFile\":{\"description\":\"File to read bearer token for remote write.\",\"type\":\"string\"},\"proxyUrl\":{\"description\":\"Optional ProxyURL\",\"type\":\"string\"},\"queueConfig\":{\"description\":\"QueueConfig allows the tuning of remote_write queue_config parameters. This object is referenced in the RemoteWriteSpec object.\",\"properties\":{\"batchSendDeadline\":{\"description\":\"BatchSendDeadline is the maximum time a sample will wait in buffer.\",\"type\":\"string\"},\"capacity\":{\"description\":\"Capacity is the number of samples to buffer per shard before we start dropping them.\",\"format\":\"int32\",\"type\":\"integer\"},\"maxBackoff\":{\"description\":\"MaxBackoff is the maximum retry delay.\",\"type\":\"string\"},\"maxRetries\":{\"description\":\"MaxRetries is the maximum number of times to retry a batch on recoverable errors.\",\"format\":\"int32\",\"type\":\"integer\"},\"maxSamplesPerSend\":{\"description\":\"MaxSamplesPerSend is the maximum number of samples per send.\",\"format\":\"int32\",\"type\":\"integer\"},\"maxShards\":{\"description\":\"MaxShards is the maximum number of shards, i.e. amount of concurrency.\",\"format\":\"int32\",\"type\":\"integer\"},\"minBackoff\":{\"description\":\"MinBackoff is the initial retry delay. Gets doubled for every retry.\",\"type\":\"string\"},\"minShards\":{\"description\":\"MinShards is the minimum number of shards, i.e. amount of concurrency.\",\"format\":\"int32\",\"type\":\"integer\"}},\"type\":\"object\"},\"remoteTimeout\":{\"description\":\"Timeout for requests to the remote write endpoint.\",\"type\":\"string\"},\"tlsConfig\":{\"description\":\"TLSConfig specifies TLS configuration parameters.\",\"properties\":{\"caFile\":{\"description\":\"The CA cert to use for the targets.\",\"type\":\"string\"},\"certFile\":{\"description\":\"The client cert file for the targets.\",\"type\":\"string\"},\"insecureSkipVerify\":{\"description\":\"Disable target certificate validation.\",\"type\":\"boolean\"},\"keyFile\":{\"description\":\"The client key file for the targets.\",\"type\":\"string\"},\"serverName\":{\"description\":\"Used to verify the hostname for the targets.\",\"type\":\"string\"}},\"type\":\"object\"},\"url\":{\"description\":\"The URL of the endpoint to send samples to.\",\"type\":\"string\"},\"writeRelabelConfigs\":{\"description\":\"The list of remote write relabel configurations.\",\"items\":{\"description\":\"RelabelConfig allows dynamic rewriting of the label set, being applied to samples before ingestion. It defines `\\u003cmetric_relabel_configs\\u003e`-section of Prometheus configuration. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs\",\"properties\":{\"action\":{\"description\":\"Action to perform based on regex matching. Default is 'replace'\",\"type\":\"string\"},\"modulus\":{\"description\":\"Modulus to take of the hash of the source label values.\",\"format\":\"int64\",\"type\":\"integer\"},\"regex\":{\"description\":\"Regular expression against which the extracted value is matched. default is '(.*)'\",\"type\":\"string\"},\"replacement\":{\"description\":\"Replacement value against which a regex replace is performed if the regular expression matches. Regex capture groups are available. Default is '$1'\",\"type\":\"string\"},\"separator\":{\"description\":\"Separator placed between concatenated source label values. default is ';'.\",\"type\":\"string\"},\"sourceLabels\":{\"description\":\"The source labels select values from existing labels. Their content is concatenated using the configured separator and matched against the configured regular expression for the replace, keep, and drop actions.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"targetLabel\":{\"description\":\"Label to which the resulting value is written in a replace action. It is mandatory for replace actions. Regex capture groups are available.\",\"type\":\"string\"}},\"type\":\"object\"},\"type\":\"array\"}},\"required\":[\"url\"],\"type\":\"object\"},\"type\":\"array\"},\"replicaExternalLabelName\":{\"description\":\"Name of Prometheus external label used to denote replica name. Defaults to the value of `prometheus_replica`. External label will _not_ be added when value is set to empty string (`\\\"\\\"`).\",\"type\":\"string\"},\"replicas\":{\"description\":\"Number of instances to deploy for a Prometheus deployment.\",\"format\":\"int32\",\"type\":\"integer\"},\"resources\":{\"description\":\"ResourceRequirements describes the compute resource requirements.\",\"properties\":{\"limits\":{\"description\":\"Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/\",\"type\":\"object\"},\"requests\":{\"description\":\"Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/\",\"type\":\"object\"}},\"type\":\"object\"},\"retention\":{\"description\":\"Time duration Prometheus shall retain data for. Default is '24h', and must match the regular expression `[0-9]+(ms|s|m|h|d|w|y)` (milliseconds seconds minutes hours days weeks years).\",\"type\":\"string\"},\"retentionSize\":{\"description\":\"Maximum amount of disk space used by blocks.\",\"type\":\"string\"},\"routePrefix\":{\"description\":\"The route prefix Prometheus registers HTTP handlers for. This is useful, if using ExternalURL and a proxy is rewriting HTTP routes of a request, and the actual ExternalURL is still true, but the server serves requests under a different route prefix. For example for use with `kubectl proxy`.\",\"type\":\"string\"},\"ruleNamespaceSelector\":{\"description\":\"A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.\",\"properties\":{\"matchExpressions\":{\"description\":\"matchExpressions is a list of label selector requirements. The requirements are ANDed.\",\"items\":{\"description\":\"A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"key is the label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\",\"type\":\"string\"},\"values\":{\"description\":\"values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"],\"type\":\"object\"},\"type\":\"array\"},\"matchLabels\":{\"description\":\"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \\\"key\\\", the operator is \\\"In\\\", and the values array contains only \\\"value\\\". The requirements are ANDed.\",\"type\":\"object\"}},\"type\":\"object\"},\"ruleSelector\":{\"description\":\"A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.\",\"properties\":{\"matchExpressions\":{\"description\":\"matchExpressions is a list of label selector requirements. The requirements are ANDed.\",\"items\":{\"description\":\"A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"key is the label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\",\"type\":\"string\"},\"values\":{\"description\":\"values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"],\"type\":\"object\"},\"type\":\"array\"},\"matchLabels\":{\"description\":\"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \\\"key\\\", the operator is \\\"In\\\", and the values array contains only \\\"value\\\". The requirements are ANDed.\",\"type\":\"object\"}},\"type\":\"object\"},\"rules\":{\"description\":\"/--rules.*/ command-line arguments\",\"properties\":{\"alert\":{\"description\":\"/--rules.alert.*/ command-line arguments\",\"properties\":{\"forGracePeriod\":{\"description\":\"Minimum duration between alert and restored 'for' state. This is maintained only for alerts with configured 'for' time greater than grace period.\",\"type\":\"string\"},\"forOutageTolerance\":{\"description\":\"Max time to tolerate prometheus outage for restoring 'for' state of alert.\",\"type\":\"string\"},\"resendDelay\":{\"description\":\"Minimum amount of time to wait before resending an alert to Alertmanager.\",\"type\":\"string\"}},\"type\":\"object\"}},\"type\":\"object\"},\"scrapeInterval\":{\"description\":\"Interval between consecutive scrapes.\",\"type\":\"string\"},\"secrets\":{\"description\":\"Secrets is a list of Secrets in the same namespace as the Prometheus object, which shall be mounted into the Prometheus Pods. The Secrets are mounted into /etc/prometheus/secrets/\\u003csecret-name\\u003e.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"securityContext\":{\"description\":\"PodSecurityContext holds pod-level security attributes and common container settings. Some fields are also present in container.securityContext. Field values of container.securityContext take precedence over field values of PodSecurityContext.\",\"properties\":{\"fsGroup\":{\"description\":\"A special supplemental group that applies to all containers in a pod. Some volume types allow the Kubelet to change the ownership of that volume to be owned by the pod:\\n\\n1. The owning GID will be the FSGroup 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) 3. The permission bits are OR'd with rw-rw----\\n\\nIf unset, the Kubelet will not modify the ownership and permissions of any volume.\",\"format\":\"int64\",\"type\":\"integer\"},\"runAsGroup\":{\"description\":\"The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container.\",\"format\":\"int64\",\"type\":\"integer\"},\"runAsNonRoot\":{\"description\":\"Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\",\"type\":\"boolean\"},\"runAsUser\":{\"description\":\"The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container.\",\"format\":\"int64\",\"type\":\"integer\"},\"seLinuxOptions\":{\"description\":\"SELinuxOptions are the labels to be applied to the container\",\"properties\":{\"level\":{\"description\":\"Level is SELinux level label that applies to the container.\",\"type\":\"string\"},\"role\":{\"description\":\"Role is a SELinux role label that applies to the container.\",\"type\":\"string\"},\"type\":{\"description\":\"Type is a SELinux type label that applies to the container.\",\"type\":\"string\"},\"user\":{\"description\":\"User is a SELinux user label that applies to the container.\",\"type\":\"string\"}},\"type\":\"object\"},\"supplementalGroups\":{\"description\":\"A list of groups applied to the first process run in each container, in addition to the container's primary GID. If unspecified, no groups will be added to any container.\",\"items\":{\"format\":\"int64\",\"type\":\"integer\"},\"type\":\"array\"},\"sysctls\":{\"description\":\"Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported sysctls (by the container runtime) might fail to launch.\",\"items\":{\"description\":\"Sysctl defines a kernel parameter to be set\",\"properties\":{\"name\":{\"description\":\"Name of a property to set\",\"type\":\"string\"},\"value\":{\"description\":\"Value of a property to set\",\"type\":\"string\"}},\"required\":[\"name\",\"value\"],\"type\":\"object\"},\"type\":\"array\"}},\"type\":\"object\"},\"serviceAccountName\":{\"description\":\"ServiceAccountName is the name of the ServiceAccount to use to run the Prometheus Pods.\",\"type\":\"string\"},\"serviceMonitorNamespaceSelector\":{\"description\":\"A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.\",\"properties\":{\"matchExpressions\":{\"description\":\"matchExpressions is a list of label selector requirements. The requirements are ANDed.\",\"items\":{\"description\":\"A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"key is the label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\",\"type\":\"string\"},\"values\":{\"description\":\"values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"],\"type\":\"object\"},\"type\":\"array\"},\"matchLabels\":{\"description\":\"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \\\"key\\\", the operator is \\\"In\\\", and the values array contains only \\\"value\\\". The requirements are ANDed.\",\"type\":\"object\"}},\"type\":\"object\"},\"serviceMonitorSelector\":{\"description\":\"A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.\",\"properties\":{\"matchExpressions\":{\"description\":\"matchExpressions is a list of label selector requirements. The requirements are ANDed.\",\"items\":{\"description\":\"A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"key is the label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\",\"type\":\"string\"},\"values\":{\"description\":\"values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"],\"type\":\"object\"},\"type\":\"array\"},\"matchLabels\":{\"description\":\"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \\\"key\\\", the operator is \\\"In\\\", and the values array contains only \\\"value\\\". The requirements are ANDed.\",\"type\":\"object\"}},\"type\":\"object\"},\"sha\":{\"description\":\"SHA of Prometheus container image to be deployed. Defaults to the value of `version`. Similar to a tag, but the SHA explicitly deploys an immutable container image. Version and Tag are ignored if SHA is set.\",\"type\":\"string\"},\"storage\":{\"description\":\"StorageSpec defines the configured storage for a group Prometheus servers. If neither `emptyDir` nor `volumeClaimTemplate` is specified, then by default an [EmptyDir](https://kubernetes.io/docs/concepts/storage/volumes/#emptydir) will be used.\",\"properties\":{\"emptyDir\":{\"description\":\"Represents an empty directory for a pod. Empty directory volumes support ownership management and SELinux relabeling.\",\"properties\":{\"medium\":{\"description\":\"What type of storage medium should back this directory. The default is \\\"\\\" which means to use the node's default medium. Must be an empty string (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir\",\"type\":\"string\"},\"sizeLimit\":{}},\"type\":\"object\"},\"volumeClaimTemplate\":{\"description\":\"PersistentVolumeClaim is a user's request for and claim to a persistent volume\",\"properties\":{\"apiVersion\":{\"description\":\"APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources\",\"type\":\"string\"},\"kind\":{\"description\":\"Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds\",\"type\":\"string\"},\"metadata\":{\"description\":\"ObjectMeta is metadata that all persisted resources must have, which includes all objects users must create.\",\"properties\":{\"annotations\":{\"description\":\"Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations\",\"type\":\"object\"},\"clusterName\":{\"description\":\"The name of the cluster which the object belongs to. This is used to distinguish resources with same name and namespace in different clusters. This field is not set anywhere right now and apiserver is going to ignore it if set in create or update request.\",\"type\":\"string\"},\"creationTimestamp\":{\"description\":\"Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.\",\"format\":\"date-time\",\"type\":\"string\"},\"deletionGracePeriodSeconds\":{\"description\":\"Number of seconds allowed for this object to gracefully terminate before it will be removed from the system. Only set when deletionTimestamp is also set. May only be shortened. Read-only.\",\"format\":\"int64\",\"type\":\"integer\"},\"deletionTimestamp\":{\"description\":\"Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.\",\"format\":\"date-time\",\"type\":\"string\"},\"finalizers\":{\"description\":\"Must be empty before the object is deleted from the registry. Each entry is an identifier for the responsible component that will remove the entry from the list. If the deletionTimestamp of the object is non-nil, entries in this list can only be removed.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"generateName\":{\"description\":\"GenerateName is an optional prefix, used by the server, to generate a unique name ONLY IF the Name field has not been provided. If this field is used, the name returned to the client will be different than the name passed. This value will also be combined with a unique suffix. The provided value has the same validation rules as the Name field, and may be truncated by the length of the suffix required to make the value unique on the server.\\n\\nIf this field is specified and the generated name exists, the server will NOT return a 409 - instead, it will either return 201 Created or 500 with Reason ServerTimeout indicating a unique name could not be found in the time allotted, and the client should retry (optionally after the time indicated in the Retry-After header).\\n\\nApplied only if Name is not specified. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#idempotency\",\"type\":\"string\"},\"generation\":{\"description\":\"A sequence number representing a specific generation of the desired state. Populated by the system. Read-only.\",\"format\":\"int64\",\"type\":\"integer\"},\"initializers\":{\"description\":\"Initializers tracks the progress of initialization.\",\"properties\":{\"pending\":{\"description\":\"Pending is a list of initializers that must execute in order before this object is visible. When the last pending initializer is removed, and no failing result is set, the initializers struct will be set to nil and the object is considered as initialized and visible to all clients.\",\"items\":{\"description\":\"Initializer is information about an initializer that has not yet completed.\",\"properties\":{\"name\":{\"description\":\"name of the process that is responsible for initializing this object.\",\"type\":\"string\"}},\"required\":[\"name\"],\"type\":\"object\"},\"type\":\"array\"},\"result\":{\"description\":\"Status is a return value for calls that don't return other objects.\",\"properties\":{\"apiVersion\":{\"description\":\"APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources\",\"type\":\"string\"},\"code\":{\"description\":\"Suggested HTTP return code for this status, 0 if not set.\",\"format\":\"int32\",\"type\":\"integer\"},\"details\":{\"description\":\"StatusDetails is a set of additional properties that MAY be set by the server to provide additional information about a response. The Reason field of a Status object defines what attributes will be set. Clients must ignore fields that do not match the defined type of each attribute, and should assume that any attribute may be empty, invalid, or under defined.\",\"properties\":{\"causes\":{\"description\":\"The Causes array includes more details associated with the StatusReason failure. Not all StatusReasons may provide detailed causes.\",\"items\":{\"description\":\"StatusCause provides more information about an api.Status failure, including cases when multiple errors are encountered.\",\"properties\":{\"field\":{\"description\":\"The field of the resource that has caused this error, as named by its JSON serialization. May include dot and postfix notation for nested attributes. Arrays are zero-indexed. Fields may appear more than once in an array of causes due to fields having multiple errors. Optional.\\n\\nExamples:\\n \\\"name\\\" - the field \\\"name\\\" on the current resource\\n \\\"items[0].name\\\" - the field \\\"name\\\" on the first array entry in \\\"items\\\"\",\"type\":\"string\"},\"message\":{\"description\":\"A human-readable description of the cause of the error. This field may be presented as-is to a reader.\",\"type\":\"string\"},\"reason\":{\"description\":\"A machine-readable description of the cause of the error. If this value is empty there is no information available.\",\"type\":\"string\"}},\"type\":\"object\"},\"type\":\"array\"},\"group\":{\"description\":\"The group attribute of the resource associated with the status StatusReason.\",\"type\":\"string\"},\"kind\":{\"description\":\"The kind attribute of the resource associated with the status StatusReason. On some operations may differ from the requested resource Kind. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds\",\"type\":\"string\"},\"name\":{\"description\":\"The name attribute of the resource associated with the status StatusReason (when there is a single name which can be described).\",\"type\":\"string\"},\"retryAfterSeconds\":{\"description\":\"If specified, the time in seconds before the operation should be retried. Some errors may indicate the client must take an alternate action - for those errors this field may indicate how long to wait before taking the alternate action.\",\"format\":\"int32\",\"type\":\"integer\"},\"uid\":{\"description\":\"UID of the resource. (when there is a single resource which can be described). More info: http://kubernetes.io/docs/user-guide/identifiers#uids\",\"type\":\"string\"}},\"type\":\"object\"},\"kind\":{\"description\":\"Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds\",\"type\":\"string\"},\"message\":{\"description\":\"A human-readable description of the status of this operation.\",\"type\":\"string\"},\"metadata\":{\"description\":\"ListMeta describes metadata that synthetic resources must have, including lists and various status objects. A resource may have only one of {ObjectMeta, ListMeta}.\",\"properties\":{\"continue\":{\"description\":\"continue may be set if the user set a limit on the number of items returned, and indicates that the server has more data available. The value is opaque and may be used to issue another request to the endpoint that served this list to retrieve the next set of available objects. Continuing a consistent list may not be possible if the server configuration has changed or more than a few minutes have passed. The resourceVersion field returned when using this continue value will be identical to the value in the first response, unless you have received this token from an error message.\",\"type\":\"string\"},\"resourceVersion\":{\"description\":\"String that identifies the server's internal version of this object that can be used by clients to determine when objects have changed. Value must be treated as opaque by clients and passed unmodified back to the server. Populated by the system. Read-only. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency\",\"type\":\"string\"},\"selfLink\":{\"description\":\"selfLink is a URL representing this object. Populated by the system. Read-only.\",\"type\":\"string\"}},\"type\":\"object\"},\"reason\":{\"description\":\"A machine-readable description of why this operation is in the \\\"Failure\\\" status. If this value is empty there is no information available. A Reason clarifies an HTTP status code but does not override it.\",\"type\":\"string\"},\"status\":{\"description\":\"Status of the operation. One of: \\\"Success\\\" or \\\"Failure\\\". More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status\",\"type\":\"string\"}},\"type\":\"object\"}},\"required\":[\"pending\"],\"type\":\"object\"},\"labels\":{\"description\":\"Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels\",\"type\":\"object\"},\"managedFields\":{\"description\":\"ManagedFields maps workflow-id and version to the set of fields that are managed by that workflow. This is mostly for internal housekeeping, and users typically shouldn't need to set or understand this field. A workflow can be the user's name, a controller's name, or the name of a specific apply path like \\\"ci-cd\\\". The set of fields is always in the version that the workflow used when modifying the object.\\n\\nThis field is alpha and can be changed or removed without notice.\",\"items\":{\"description\":\"ManagedFieldsEntry is a workflow-id, a FieldSet and the group version of the resource that the fieldset applies to.\",\"properties\":{\"apiVersion\":{\"description\":\"APIVersion defines the version of this resource that this field set applies to. The format is \\\"group/version\\\" just like the top-level APIVersion field. It is necessary to track the version of a field set because it cannot be automatically converted.\",\"type\":\"string\"},\"fields\":{\"description\":\"Fields stores a set of fields in a data structure like a Trie. To understand how this is used, see: https://github.com/kubernetes-sigs/structured-merge-diff\",\"type\":\"object\"},\"manager\":{\"description\":\"Manager is an identifier of the workflow managing these fields.\",\"type\":\"string\"},\"operation\":{\"description\":\"Operation is the type of operation which lead to this ManagedFieldsEntry being created. The only valid values for this field are 'Apply' and 'Update'.\",\"type\":\"string\"},\"time\":{\"description\":\"Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.\",\"format\":\"date-time\",\"type\":\"string\"}},\"type\":\"object\"},\"type\":\"array\"},\"name\":{\"description\":\"Name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names\",\"type\":\"string\"},\"namespace\":{\"description\":\"Namespace defines the space within each name must be unique. An empty namespace is equivalent to the \\\"default\\\" namespace, but \\\"default\\\" is the canonical representation. Not all objects are required to be scoped to a namespace - the value of this field for those objects will be empty.\\n\\nMust be a DNS_LABEL. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/namespaces\",\"type\":\"string\"},\"ownerReferences\":{\"description\":\"List of objects depended by this object. If ALL objects in the list have been deleted, this object will be garbage collected. If this object is managed by a controller, then an entry in this list will point to this controller, with the controller field set to true. There cannot be more than one managing controller.\",\"items\":{\"description\":\"OwnerReference contains enough information to let you identify an owning object. An owning object must be in the same namespace as the dependent, or be cluster-scoped, so there is no namespace field.\",\"properties\":{\"apiVersion\":{\"description\":\"API version of the referent.\",\"type\":\"string\"},\"blockOwnerDeletion\":{\"description\":\"If true, AND if the owner has the \\\"foregroundDeletion\\\" finalizer, then the owner cannot be deleted from the key-value store until this reference is removed. Defaults to false. To set this field, a user needs \\\"delete\\\" permission of the owner, otherwise 422 (Unprocessable Entity) will be returned.\",\"type\":\"boolean\"},\"controller\":{\"description\":\"If true, this reference points to the managing controller.\",\"type\":\"boolean\"},\"kind\":{\"description\":\"Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#names\",\"type\":\"string\"},\"uid\":{\"description\":\"UID of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#uids\",\"type\":\"string\"}},\"required\":[\"apiVersion\",\"kind\",\"name\",\"uid\"],\"type\":\"object\"},\"type\":\"array\"},\"resourceVersion\":{\"description\":\"An opaque value that represents the internal version of this object that can be used by clients to determine when objects have changed. May be used for optimistic concurrency, change detection, and the watch operation on a resource or set of resources. Clients must treat these values as opaque and passed unmodified back to the server. They may only be valid for a particular resource or set of resources.\\n\\nPopulated by the system. Read-only. Value must be treated as opaque by clients and . More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency\",\"type\":\"string\"},\"selfLink\":{\"description\":\"SelfLink is a URL representing this object. Populated by the system. Read-only.\",\"type\":\"string\"},\"uid\":{\"description\":\"UID is the unique in time and space value for this object. It is typically generated by the server on successful creation of a resource and is not allowed to change on PUT operations.\\n\\nPopulated by the system. Read-only. More info: http://kubernetes.io/docs/user-guide/identifiers#uids\",\"type\":\"string\"}},\"type\":\"object\"},\"spec\":{\"description\":\"PersistentVolumeClaimSpec describes the common attributes of storage devices and allows a Source for provider-specific attributes\",\"properties\":{\"accessModes\":{\"description\":\"AccessModes contains the desired access modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"dataSource\":{\"description\":\"TypedLocalObjectReference contains enough information to let you locate the typed referenced object inside the same namespace.\",\"properties\":{\"apiGroup\":{\"description\":\"APIGroup is the group for the resource being referenced. If APIGroup is not specified, the specified Kind must be in the core API group. For any other third-party types, APIGroup is required.\",\"type\":\"string\"},\"kind\":{\"description\":\"Kind is the type of resource being referenced\",\"type\":\"string\"},\"name\":{\"description\":\"Name is the name of resource being referenced\",\"type\":\"string\"}},\"required\":[\"kind\",\"name\"],\"type\":\"object\"},\"resources\":{\"description\":\"ResourceRequirements describes the compute resource requirements.\",\"properties\":{\"limits\":{\"description\":\"Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/\",\"type\":\"object\"},\"requests\":{\"description\":\"Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/\",\"type\":\"object\"}},\"type\":\"object\"},\"selector\":{\"description\":\"A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.\",\"properties\":{\"matchExpressions\":{\"description\":\"matchExpressions is a list of label selector requirements. The requirements are ANDed.\",\"items\":{\"description\":\"A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\",\"properties\":{\"key\":{\"description\":\"key is the label key that the selector applies to.\",\"type\":\"string\"},\"operator\":{\"description\":\"operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\",\"type\":\"string\"},\"values\":{\"description\":\"values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\",\"items\":{\"type\":\"string\"},\"type\":\"array\"}},\"required\":[\"key\",\"operator\"],\"type\":\"object\"},\"type\":\"array\"},\"matchLabels\":{\"description\":\"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \\\"key\\\", the operator is \\\"In\\\", and the values array contains only \\\"value\\\". The requirements are ANDed.\",\"type\":\"object\"}},\"type\":\"object\"},\"storageClassName\":{\"description\":\"Name of the StorageClass required by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1\",\"type\":\"string\"},\"volumeMode\":{\"description\":\"volumeMode defines what type of volume is required by the claim. Value of Filesystem is implied when not included in claim spec. This is a beta feature.\",\"type\":\"string\"},\"volumeName\":{\"description\":\"VolumeName is the binding reference to the PersistentVolume backing this claim.\",\"type\":\"string\"}},\"type\":\"object\"},\"status\":{\"description\":\"PersistentVolumeClaimStatus is the current status of a persistent volume claim.\",\"properties\":{\"accessModes\":{\"description\":\"AccessModes contains the actual access modes the volume backing the PVC has. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1\",\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"capacity\":{\"description\":\"Represents the actual resources of the underlying volume.\",\"type\":\"object\"},\"conditions\":{\"description\":\"Current Condition of persistent volume claim. If underlying persistent volume is being resized then the Condition will be set to 'ResizeStarted'.\",\"items\":{\"description\":\"PersistentVolumeClaimCondition contains details about state of pvc\",\"properties\":{\"lastProbeTime\":{\"description\":\"Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.\",\"format\":\"date-time\",\"type\":\"string\"},\"lastTransitionTime\":{\"description\":\"Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.\",\"format\":\"date-time\",\"type\":\"string\"},\"message\":{\"description\":\"Human-readable message indicating details about last transition.\",\"type\":\"string\"},\"reason\":{\"description\":\"Unique, this should be a short, machine understandable string that gives the reason for condition's last transition. If it reports \\\"ResizeStarted\\\" that means the underlying persistent volume is being resized.\",\"type\":\"string\"},\"status\":{\"type\":\"string\"},\"type\":{\"type\":\"string\"}},\"required\":[\"type\",\"status\"],\"type\":\"object\"},\"type\":\"array\"},\"phase\":{\"description\":\"Phase represents the current phase of PersistentVolumeClaim.\",\"type\":\"string\"}},\"type\":\"object\"}},\"type\":\"object\"}},\"type\":\"object\"},\"tag\":{\"description\":\"Tag of Prometheus container image to be deployed. Defaults to the value of `version`. Version is ignored if Tag is set.\",\"type\":\"string\"},\"thanos\":{\"description\":\"ThanosSpec defines parameters for a Prometheus server within a Thanos deployment.\",\"properties\":{\"baseImage\":{\"description\":\"Thanos base image if other than default.\",\"type\":\"string\"},\"image\":{\"description\":\"Image if specified has precedence over baseImage, tag and sha combinations. Specifying the version is still necessary to ensure the Prometheus Operator knows what version of Thanos is being configured.\",\"type\":\"string\"},\"objectStorageConfig\":{\"description\":\"SecretKeySelector selects a key of a Secret.\",\"properties\":{\"key\":{\"description\":\"The key of the secret to select from. Must be a valid secret key.\",\"type\":\"string\"},\"name\":{\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\",\"type\":\"string\"},\"optional\":{\"description\":\"Specify whether the Secret or it's key must be defined\",\"type\":\"boolean\"}},\"required\":[\"key\"],\"type\":\"object\"},\"resources\":{\"description\":\"ResourceRequirements describes the compute resource requirements.\",\"properties\":{\"limits\":{\"description\":\"Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/\",\"type\":\"object\"},\"requests\":{\"description\":\"Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/\",\"type\":\"object\"}},\"type\":\"object\"},\"sha\":{\"description\":\"SHA of Thanos container image to be deployed. Defaults to the value of `version`. Similar to a tag, but the SHA explicitly deploys an immutable container image. Version and Tag are ignored if SHA is set.\",\"type\":\"string\"},\"tag\":{\"description\":\"Tag of Thanos sidecar container image to be deployed. Defaults to the value of `version`. Version is ignored if Tag is set.\",\"type\":\"string\"},\"version\":{\"description\":\"Version describes the version of Thanos to use.\",\"type\":\"string\"}},\"type\":\"object\"},\"tolerations\":{\"description\":\"If specified, the pod's tolerations.\",\"items\":{\"description\":\"The pod this Toleration is attached to tolerates any taint that matches the triple \\u003ckey,value,effect\\u003e using the matching operator \\u003coperator\\u003e.\",\"properties\":{\"effect\":{\"description\":\"Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.\",\"type\":\"string\"},\"key\":{\"description\":\"Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys.\",\"type\":\"string\"},\"operator\":{\"description\":\"Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category.\",\"type\":\"string\"},\"tolerationSeconds\":{\"description\":\"TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system.\",\"format\":\"int64\",\"type\":\"integer\"},\"value\":{\"description\":\"Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string.\",\"type\":\"string\"}},\"type\":\"object\"},\"type\":\"array\"},\"version\":{\"description\":\"Version of Prometheus to be deployed.\",\"type\":\"string\"},\"walCompression\":{\"description\":\"Enable compression of the write-ahead log using Snappy.\",\"type\":\"boolean\"}},\"type\":\"object\"},\"status\":{\"description\":\"PrometheusStatus is the most recent observed status of the Prometheus cluster. Read-only. Not included when requesting from the apiserver, only from the Prometheus Operator API itself. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/api-conventions.md#spec-and-status\",\"properties\":{\"availableReplicas\":{\"description\":\"Total number of available pods (ready for at least minReadySeconds) targeted by this Prometheus deployment.\",\"format\":\"int32\",\"type\":\"integer\"},\"paused\":{\"description\":\"Represents whether any actions on the underlying managed objects are being performed. Only delete actions will be performed.\",\"type\":\"boolean\"},\"replicas\":{\"description\":\"Total number of non-terminated pods targeted by this Prometheus deployment (their labels match the selector).\",\"format\":\"int32\",\"type\":\"integer\"},\"unavailableReplicas\":{\"description\":\"Total number of unavailable pods targeted by this Prometheus deployment.\",\"format\":\"int32\",\"type\":\"integer\"},\"updatedReplicas\":{\"description\":\"Total number of non-terminated pods targeted by this Prometheus deployment that have the desired version spec.\",\"format\":\"int32\",\"type\":\"integer\"}},\"required\":[\"paused\",\"replicas\",\"updatedReplicas\",\"availableReplicas\",\"unavailableReplicas\"],\"type\":\"object\"}},\"type\":\"object\"}},\"version\":\"v1\"}}\n" } }, "spec": { "group": "monitoring.coreos.com", "version": "v1", "names": { "plural": "prometheuses", "singular": "prometheus", "kind": "Prometheus", "listKind": "PrometheusList" }, "scope": "Namespaced", "validation": { "openAPIV3Schema": { "type": "object", "properties": { "apiVersion": { "description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources", "type": "string" }, "kind": { "description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds", "type": "string" }, "spec": { "description": "PrometheusSpec is a specification of the desired behavior of the Prometheus cluster. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/api-conventions.md#spec-and-status", "type": "object", "properties": { "additionalAlertManagerConfigs": { "description": "SecretKeySelector selects a key of a Secret.", "type": "object", "required": [ "key" ], "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", "type": "string" }, "optional": { "description": "Specify whether the Secret or it's key must be defined", "type": "boolean" } } }, "additionalAlertRelabelConfigs": { "description": "SecretKeySelector selects a key of a Secret.", "type": "object", "required": [ "key" ], "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", "type": "string" }, "optional": { "description": "Specify whether the Secret or it's key must be defined", "type": "boolean" } } }, "additionalScrapeConfigs": { "description": "SecretKeySelector selects a key of a Secret.", "type": "object", "required": [ "key" ], "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", "type": "string" }, "optional": { "description": "Specify whether the Secret or it's key must be defined", "type": "boolean" } } }, "affinity": { "description": "Affinity is a group of affinity scheduling rules.", "type": "object", "properties": { "nodeAffinity": { "description": "Node affinity is a group of node affinity scheduling rules.", "type": "object", "properties": { "preferredDuringSchedulingIgnoredDuringExecution": { "description": "The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding \"weight\" to the sum if the node matches the corresponding matchExpressions; the node(s) with the highest sum are the most preferred.", "type": "array", "items": { "description": "An empty preferred scheduling term matches all objects with implicit weight 0 (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op).", "type": "object", "required": [ "weight", "preference" ], "properties": { "preference": { "description": "A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm.", "type": "object", "properties": { "matchExpressions": { "description": "A list of node selector requirements by node's labels.", "type": "array", "items": { "description": "A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "type": "object", "required": [ "key", "operator" ], "properties": { "key": { "description": "The label key that the selector applies to.", "type": "string" }, "operator": { "description": "Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.", "type": "string" }, "values": { "description": "An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.", "type": "array", "items": { "type": "string" } } } } }, "matchFields": { "description": "A list of node selector requirements by node's fields.", "type": "array", "items": { "description": "A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "type": "object", "required": [ "key", "operator" ], "properties": { "key": { "description": "The label key that the selector applies to.", "type": "string" }, "operator": { "description": "Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.", "type": "string" }, "values": { "description": "An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.", "type": "array", "items": { "type": "string" } } } } } } }, "weight": { "description": "Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100.", "type": "integer", "format": "int32" } } } }, "requiredDuringSchedulingIgnoredDuringExecution": { "description": "A node selector represents the union of the results of one or more label queries over a set of nodes; that is, it represents the OR of the selectors represented by the node selector terms.", "type": "object", "required": [ "nodeSelectorTerms" ], "properties": { "nodeSelectorTerms": { "description": "Required. A list of node selector terms. The terms are ORed.", "type": "array", "items": { "description": "A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm.", "type": "object", "properties": { "matchExpressions": { "description": "A list of node selector requirements by node's labels.", "type": "array", "items": { "description": "A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "type": "object", "required": [ "key", "operator" ], "properties": { "key": { "description": "The label key that the selector applies to.", "type": "string" }, "operator": { "description": "Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.", "type": "string" }, "values": { "description": "An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.", "type": "array", "items": { "type": "string" } } } } }, "matchFields": { "description": "A list of node selector requirements by node's fields.", "type": "array", "items": { "description": "A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "type": "object", "required": [ "key", "operator" ], "properties": { "key": { "description": "The label key that the selector applies to.", "type": "string" }, "operator": { "description": "Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.", "type": "string" }, "values": { "description": "An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.", "type": "array", "items": { "type": "string" } } } } } } } } } } } }, "podAffinity": { "description": "Pod affinity is a group of inter pod affinity scheduling rules.", "type": "object", "properties": { "preferredDuringSchedulingIgnoredDuringExecution": { "description": "The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding \"weight\" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred.", "type": "array", "items": { "description": "The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s)", "type": "object", "required": [ "weight", "podAffinityTerm" ], "properties": { "podAffinityTerm": { "description": "Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running", "type": "object", "required": [ "topologyKey" ], "properties": { "labelSelector": { "description": "A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.", "type": "object", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "type": "array", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "type": "object", "required": [ "key", "operator" ], "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "type": "array", "items": { "type": "string" } } } } }, "matchLabels": { "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object" } } }, "namespaces": { "description": "namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \"this pod's namespace\"", "type": "array", "items": { "type": "string" } }, "topologyKey": { "description": "This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.", "type": "string" } } }, "weight": { "description": "weight associated with matching the corresponding podAffinityTerm, in the range 1-100.", "type": "integer", "format": "int32" } } } }, "requiredDuringSchedulingIgnoredDuringExecution": { "description": "If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied.", "type": "array", "items": { "description": "Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running", "type": "object", "required": [ "topologyKey" ], "properties": { "labelSelector": { "description": "A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.", "type": "object", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "type": "array", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "type": "object", "required": [ "key", "operator" ], "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "type": "array", "items": { "type": "string" } } } } }, "matchLabels": { "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object" } } }, "namespaces": { "description": "namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \"this pod's namespace\"", "type": "array", "items": { "type": "string" } }, "topologyKey": { "description": "This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.", "type": "string" } } } } } }, "podAntiAffinity": { "description": "Pod anti affinity is a group of inter pod anti affinity scheduling rules.", "type": "object", "properties": { "preferredDuringSchedulingIgnoredDuringExecution": { "description": "The scheduler will prefer to schedule pods to nodes that satisfy the anti-affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding \"weight\" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred.", "type": "array", "items": { "description": "The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s)", "type": "object", "required": [ "weight", "podAffinityTerm" ], "properties": { "podAffinityTerm": { "description": "Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running", "type": "object", "required": [ "topologyKey" ], "properties": { "labelSelector": { "description": "A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.", "type": "object", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "type": "array", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "type": "object", "required": [ "key", "operator" ], "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "type": "array", "items": { "type": "string" } } } } }, "matchLabels": { "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object" } } }, "namespaces": { "description": "namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \"this pod's namespace\"", "type": "array", "items": { "type": "string" } }, "topologyKey": { "description": "This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.", "type": "string" } } }, "weight": { "description": "weight associated with matching the corresponding podAffinityTerm, in the range 1-100.", "type": "integer", "format": "int32" } } } }, "requiredDuringSchedulingIgnoredDuringExecution": { "description": "If the anti-affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the anti-affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied.", "type": "array", "items": { "description": "Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running", "type": "object", "required": [ "topologyKey" ], "properties": { "labelSelector": { "description": "A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.", "type": "object", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "type": "array", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "type": "object", "required": [ "key", "operator" ], "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "type": "array", "items": { "type": "string" } } } } }, "matchLabels": { "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object" } } }, "namespaces": { "description": "namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \"this pod's namespace\"", "type": "array", "items": { "type": "string" } }, "topologyKey": { "description": "This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.", "type": "string" } } } } } } } }, "alerting": { "description": "AlertingSpec defines parameters for alerting configuration of Prometheus servers.", "type": "object", "required": [ "alertmanagers" ], "properties": { "alertmanagers": { "description": "AlertmanagerEndpoints Prometheus should fire alerts against.", "type": "array", "items": { "description": "AlertmanagerEndpoints defines a selection of a single Endpoints object containing alertmanager IPs to fire alerts against.", "type": "object", "required": [ "namespace", "name", "port" ], "properties": { "bearerTokenFile": { "description": "BearerTokenFile to read from filesystem to use when authenticating to Alertmanager.", "type": "string" }, "name": { "description": "Name of Endpoints object in Namespace.", "type": "string" }, "namespace": { "description": "Namespace of Endpoints object.", "type": "string" }, "pathPrefix": { "description": "Prefix for the HTTP path alerts are pushed to.", "type": "string" }, "port": { "anyOf": [ { "type": "string" }, { "type": "integer" } ] }, "scheme": { "description": "Scheme to use when firing alerts.", "type": "string" }, "tlsConfig": { "description": "TLSConfig specifies TLS configuration parameters.", "type": "object", "properties": { "caFile": { "description": "The CA cert to use for the targets.", "type": "string" }, "certFile": { "description": "The client cert file for the targets.", "type": "string" }, "insecureSkipVerify": { "description": "Disable target certificate validation.", "type": "boolean" }, "keyFile": { "description": "The client key file for the targets.", "type": "string" }, "serverName": { "description": "Used to verify the hostname for the targets.", "type": "string" } } } } } } } }, "apiserverConfig": { "description": "APIServerConfig defines a host and auth methods to access apiserver. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config", "type": "object", "required": [ "host" ], "properties": { "basicAuth": { "description": "BasicAuth allow an endpoint to authenticate over basic authentication More info: https://prometheus.io/docs/operating/configuration/#endpoints", "type": "object", "properties": { "password": { "description": "SecretKeySelector selects a key of a Secret.", "type": "object", "required": [ "key" ], "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", "type": "string" }, "optional": { "description": "Specify whether the Secret or it's key must be defined", "type": "boolean" } } }, "username": { "description": "SecretKeySelector selects a key of a Secret.", "type": "object", "required": [ "key" ], "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", "type": "string" }, "optional": { "description": "Specify whether the Secret or it's key must be defined", "type": "boolean" } } } } }, "bearerToken": { "description": "Bearer token for accessing apiserver.", "type": "string" }, "bearerTokenFile": { "description": "File to read bearer token for accessing apiserver.", "type": "string" }, "host": { "description": "Host of apiserver. A valid string consisting of a hostname or IP followed by an optional port number", "type": "string" }, "tlsConfig": { "description": "TLSConfig specifies TLS configuration parameters.", "type": "object", "properties": { "caFile": { "description": "The CA cert to use for the targets.", "type": "string" }, "certFile": { "description": "The client cert file for the targets.", "type": "string" }, "insecureSkipVerify": { "description": "Disable target certificate validation.", "type": "boolean" }, "keyFile": { "description": "The client key file for the targets.", "type": "string" }, "serverName": { "description": "Used to verify the hostname for the targets.", "type": "string" } } } } }, "baseImage": { "description": "Base image to use for a Prometheus deployment.", "type": "string" }, "configMaps": { "description": "ConfigMaps is a list of ConfigMaps in the same namespace as the Prometheus object, which shall be mounted into the Prometheus Pods. The ConfigMaps are mounted into /etc/prometheus/configmaps/.", "type": "array", "items": { "type": "string" } }, "containers": { "description": "Containers allows injecting additional containers or modifying operator generated containers. This can be used to allow adding an authentication proxy to a Prometheus pod or to change the behavior of an operator generated container. Containers described here modify an operator generated container if they share the same name and modifications are done via a strategic merge patch. The current container names are: `prometheus`, `prometheus-config-reloader`, `rules-configmap-reloader`, and `thanos-sidecar`. Overriding containers is entirely outside the scope of what the maintainers will support and by doing so, you accept that this behaviour may break at any time without notice.", "type": "array", "items": { "description": "A single application container that you want to run within a pod.", "type": "object", "required": [ "name" ], "properties": { "args": { "description": "Arguments to the entrypoint. The docker image's CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell", "type": "array", "items": { "type": "string" } }, "command": { "description": "Entrypoint array. Not executed within a shell. The docker image's ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell", "type": "array", "items": { "type": "string" } }, "env": { "description": "List of environment variables to set in the container. Cannot be updated.", "type": "array", "items": { "description": "EnvVar represents an environment variable present in a Container.", "type": "object", "required": [ "name" ], "properties": { "name": { "description": "Name of the environment variable. Must be a C_IDENTIFIER.", "type": "string" }, "value": { "description": "Variable references $(VAR_NAME) are expanded using the previous defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Defaults to \"\".", "type": "string" }, "valueFrom": { "description": "EnvVarSource represents a source for the value of an EnvVar.", "type": "object", "properties": { "configMapKeyRef": { "description": "Selects a key from a ConfigMap.", "type": "object", "required": [ "key" ], "properties": { "key": { "description": "The key to select.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", "type": "string" }, "optional": { "description": "Specify whether the ConfigMap or it's key must be defined", "type": "boolean" } } }, "fieldRef": { "description": "ObjectFieldSelector selects an APIVersioned field of an object.", "type": "object", "required": [ "fieldPath" ], "properties": { "apiVersion": { "description": "Version of the schema the FieldPath is written in terms of, defaults to \"v1\".", "type": "string" }, "fieldPath": { "description": "Path of the field to select in the specified API version.", "type": "string" } } }, "resourceFieldRef": { "description": "ResourceFieldSelector represents container resources (cpu, memory) and their output format", "type": "object", "required": [ "resource" ], "properties": { "containerName": { "description": "Container name: required for volumes, optional for env vars", "type": "string" }, "divisor": {}, "resource": { "description": "Required: resource to select", "type": "string" } } }, "secretKeyRef": { "description": "SecretKeySelector selects a key of a Secret.", "type": "object", "required": [ "key" ], "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", "type": "string" }, "optional": { "description": "Specify whether the Secret or it's key must be defined", "type": "boolean" } } } } } } } }, "envFrom": { "description": "List of sources to populate environment variables in the container. The keys defined within a source must be a C_IDENTIFIER. All invalid keys will be reported as an event when the container is starting. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated.", "type": "array", "items": { "description": "EnvFromSource represents the source of a set of ConfigMaps", "type": "object", "properties": { "configMapRef": { "description": "ConfigMapEnvSource selects a ConfigMap to populate the environment variables with.\n\nThe contents of the target ConfigMap's Data field will represent the key-value pairs as environment variables.", "type": "object", "properties": { "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", "type": "string" }, "optional": { "description": "Specify whether the ConfigMap must be defined", "type": "boolean" } } }, "prefix": { "description": "An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER.", "type": "string" }, "secretRef": { "description": "SecretEnvSource selects a Secret to populate the environment variables with.\n\nThe contents of the target Secret's Data field will represent the key-value pairs as environment variables.", "type": "object", "properties": { "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", "type": "string" }, "optional": { "description": "Specify whether the Secret must be defined", "type": "boolean" } } } } } }, "image": { "description": "Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images This field is optional to allow higher level config management to default or override container images in workload controllers like Deployments and StatefulSets.", "type": "string" }, "imagePullPolicy": { "description": "Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images", "type": "string" }, "lifecycle": { "description": "Lifecycle describes actions that the management system should take in response to container lifecycle events. For the PostStart and PreStop lifecycle handlers, management of the container blocks until the action is complete, unless the container process fails, in which case the handler is aborted.", "type": "object", "properties": { "postStart": { "description": "Handler defines a specific action that should be taken", "type": "object", "properties": { "exec": { "description": "ExecAction describes a \"run in container\" action.", "type": "object", "properties": { "command": { "description": "Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.", "type": "array", "items": { "type": "string" } } } }, "httpGet": { "description": "HTTPGetAction describes an action based on HTTP Get requests.", "type": "object", "required": [ "port" ], "properties": { "host": { "description": "Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.", "type": "string" }, "httpHeaders": { "description": "Custom headers to set in the request. HTTP allows repeated headers.", "type": "array", "items": { "description": "HTTPHeader describes a custom header to be used in HTTP probes", "type": "object", "required": [ "name", "value" ], "properties": { "name": { "description": "The header field name", "type": "string" }, "value": { "description": "The header field value", "type": "string" } } } }, "path": { "description": "Path to access on the HTTP server.", "type": "string" }, "port": { "anyOf": [ { "type": "string" }, { "type": "integer" } ] }, "scheme": { "description": "Scheme to use for connecting to the host. Defaults to HTTP.", "type": "string" } } }, "tcpSocket": { "description": "TCPSocketAction describes an action based on opening a socket", "type": "object", "required": [ "port" ], "properties": { "host": { "description": "Optional: Host name to connect to, defaults to the pod IP.", "type": "string" }, "port": { "anyOf": [ { "type": "string" }, { "type": "integer" } ] } } } } }, "preStop": { "description": "Handler defines a specific action that should be taken", "type": "object", "properties": { "exec": { "description": "ExecAction describes a \"run in container\" action.", "type": "object", "properties": { "command": { "description": "Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.", "type": "array", "items": { "type": "string" } } } }, "httpGet": { "description": "HTTPGetAction describes an action based on HTTP Get requests.", "type": "object", "required": [ "port" ], "properties": { "host": { "description": "Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.", "type": "string" }, "httpHeaders": { "description": "Custom headers to set in the request. HTTP allows repeated headers.", "type": "array", "items": { "description": "HTTPHeader describes a custom header to be used in HTTP probes", "type": "object", "required": [ "name", "value" ], "properties": { "name": { "description": "The header field name", "type": "string" }, "value": { "description": "The header field value", "type": "string" } } } }, "path": { "description": "Path to access on the HTTP server.", "type": "string" }, "port": { "anyOf": [ { "type": "string" }, { "type": "integer" } ] }, "scheme": { "description": "Scheme to use for connecting to the host. Defaults to HTTP.", "type": "string" } } }, "tcpSocket": { "description": "TCPSocketAction describes an action based on opening a socket", "type": "object", "required": [ "port" ], "properties": { "host": { "description": "Optional: Host name to connect to, defaults to the pod IP.", "type": "string" }, "port": { "anyOf": [ { "type": "string" }, { "type": "integer" } ] } } } } } } }, "livenessProbe": { "description": "Probe describes a health check to be performed against a container to determine whether it is alive or ready to receive traffic.", "type": "object", "properties": { "exec": { "description": "ExecAction describes a \"run in container\" action.", "type": "object", "properties": { "command": { "description": "Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.", "type": "array", "items": { "type": "string" } } } }, "failureThreshold": { "description": "Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.", "type": "integer", "format": "int32" }, "httpGet": { "description": "HTTPGetAction describes an action based on HTTP Get requests.", "type": "object", "required": [ "port" ], "properties": { "host": { "description": "Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.", "type": "string" }, "httpHeaders": { "description": "Custom headers to set in the request. HTTP allows repeated headers.", "type": "array", "items": { "description": "HTTPHeader describes a custom header to be used in HTTP probes", "type": "object", "required": [ "name", "value" ], "properties": { "name": { "description": "The header field name", "type": "string" }, "value": { "description": "The header field value", "type": "string" } } } }, "path": { "description": "Path to access on the HTTP server.", "type": "string" }, "port": { "anyOf": [ { "type": "string" }, { "type": "integer" } ] }, "scheme": { "description": "Scheme to use for connecting to the host. Defaults to HTTP.", "type": "string" } } }, "initialDelaySeconds": { "description": "Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes", "type": "integer", "format": "int32" }, "periodSeconds": { "description": "How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.", "type": "integer", "format": "int32" }, "successThreshold": { "description": "Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness. Minimum value is 1.", "type": "integer", "format": "int32" }, "tcpSocket": { "description": "TCPSocketAction describes an action based on opening a socket", "type": "object", "required": [ "port" ], "properties": { "host": { "description": "Optional: Host name to connect to, defaults to the pod IP.", "type": "string" }, "port": { "anyOf": [ { "type": "string" }, { "type": "integer" } ] } } }, "timeoutSeconds": { "description": "Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes", "type": "integer", "format": "int32" } } }, "name": { "description": "Name of the container specified as a DNS_LABEL. Each container in a pod must have a unique name (DNS_LABEL). Cannot be updated.", "type": "string" }, "ports": { "description": "List of ports to expose from the container. Exposing a port here gives the system additional information about the network connections a container uses, but is primarily informational. Not specifying a port here DOES NOT prevent that port from being exposed. Any port which is listening on the default \"0.0.0.0\" address inside a container will be accessible from the network. Cannot be updated.", "type": "array", "items": { "description": "ContainerPort represents a network port in a single container.", "type": "object", "required": [ "containerPort" ], "properties": { "containerPort": { "description": "Number of port to expose on the pod's IP address. This must be a valid port number, 0 < x < 65536.", "type": "integer", "format": "int32" }, "hostIP": { "description": "What host IP to bind the external port to.", "type": "string" }, "hostPort": { "description": "Number of port to expose on the host. If specified, this must be a valid port number, 0 < x < 65536. If HostNetwork is specified, this must match ContainerPort. Most containers do not need this.", "type": "integer", "format": "int32" }, "name": { "description": "If specified, this must be an IANA_SVC_NAME and unique within the pod. Each named port in a pod must have a unique name. Name for the port that can be referred to by services.", "type": "string" }, "protocol": { "description": "Protocol for port. Must be UDP, TCP, or SCTP. Defaults to \"TCP\".", "type": "string" } } } }, "readinessProbe": { "description": "Probe describes a health check to be performed against a container to determine whether it is alive or ready to receive traffic.", "type": "object", "properties": { "exec": { "description": "ExecAction describes a \"run in container\" action.", "type": "object", "properties": { "command": { "description": "Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.", "type": "array", "items": { "type": "string" } } } }, "failureThreshold": { "description": "Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.", "type": "integer", "format": "int32" }, "httpGet": { "description": "HTTPGetAction describes an action based on HTTP Get requests.", "type": "object", "required": [ "port" ], "properties": { "host": { "description": "Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.", "type": "string" }, "httpHeaders": { "description": "Custom headers to set in the request. HTTP allows repeated headers.", "type": "array", "items": { "description": "HTTPHeader describes a custom header to be used in HTTP probes", "type": "object", "required": [ "name", "value" ], "properties": { "name": { "description": "The header field name", "type": "string" }, "value": { "description": "The header field value", "type": "string" } } } }, "path": { "description": "Path to access on the HTTP server.", "type": "string" }, "port": { "anyOf": [ { "type": "string" }, { "type": "integer" } ] }, "scheme": { "description": "Scheme to use for connecting to the host. Defaults to HTTP.", "type": "string" } } }, "initialDelaySeconds": { "description": "Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes", "type": "integer", "format": "int32" }, "periodSeconds": { "description": "How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.", "type": "integer", "format": "int32" }, "successThreshold": { "description": "Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness. Minimum value is 1.", "type": "integer", "format": "int32" }, "tcpSocket": { "description": "TCPSocketAction describes an action based on opening a socket", "type": "object", "required": [ "port" ], "properties": { "host": { "description": "Optional: Host name to connect to, defaults to the pod IP.", "type": "string" }, "port": { "anyOf": [ { "type": "string" }, { "type": "integer" } ] } } }, "timeoutSeconds": { "description": "Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes", "type": "integer", "format": "int32" } } }, "resources": { "description": "ResourceRequirements describes the compute resource requirements.", "type": "object", "properties": { "limits": { "description": "Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", "type": "object" }, "requests": { "description": "Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", "type": "object" } } }, "securityContext": { "description": "SecurityContext holds security configuration that will be applied to a container. Some fields are present in both SecurityContext and PodSecurityContext. When both are set, the values in SecurityContext take precedence.", "type": "object", "properties": { "allowPrivilegeEscalation": { "description": "AllowPrivilegeEscalation controls whether a process can gain more privileges than its parent process. This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged 2) has CAP_SYS_ADMIN", "type": "boolean" }, "capabilities": { "description": "Adds and removes POSIX capabilities from running containers.", "type": "object", "properties": { "add": { "description": "Added capabilities", "type": "array", "items": { "type": "string" } }, "drop": { "description": "Removed capabilities", "type": "array", "items": { "type": "string" } } } }, "privileged": { "description": "Run container in privileged mode. Processes in privileged containers are essentially equivalent to root on the host. Defaults to false.", "type": "boolean" }, "procMount": { "description": "procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled.", "type": "string" }, "readOnlyRootFilesystem": { "description": "Whether this container has a read-only root filesystem. Default is false.", "type": "boolean" }, "runAsGroup": { "description": "The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.", "type": "integer", "format": "int64" }, "runAsNonRoot": { "description": "Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.", "type": "boolean" }, "runAsUser": { "description": "The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.", "type": "integer", "format": "int64" }, "seLinuxOptions": { "description": "SELinuxOptions are the labels to be applied to the container", "type": "object", "properties": { "level": { "description": "Level is SELinux level label that applies to the container.", "type": "string" }, "role": { "description": "Role is a SELinux role label that applies to the container.", "type": "string" }, "type": { "description": "Type is a SELinux type label that applies to the container.", "type": "string" }, "user": { "description": "User is a SELinux user label that applies to the container.", "type": "string" } } } } }, "stdin": { "description": "Whether this container should allocate a buffer for stdin in the container runtime. If this is not set, reads from stdin in the container will always result in EOF. Default is false.", "type": "boolean" }, "stdinOnce": { "description": "Whether the container runtime should close the stdin channel after it has been opened by a single attach. When stdin is true the stdin stream will remain open across multiple attach sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the first client attaches to stdin, and then remains open and accepts data until the client disconnects, at which time stdin is closed and remains closed until the container is restarted. If this flag is false, a container processes that reads from stdin will never receive an EOF. Default is false", "type": "boolean" }, "terminationMessagePath": { "description": "Optional: Path at which the file to which the container's termination message will be written is mounted into the container's filesystem. Message written is intended to be brief final status, such as an assertion failure message. Will be truncated by the node if greater than 4096 bytes. The total message length across all containers will be limited to 12kb. Defaults to /dev/termination-log. Cannot be updated.", "type": "string" }, "terminationMessagePolicy": { "description": "Indicate how the termination message should be populated. File will use the contents of terminationMessagePath to populate the container status message on both success and failure. FallbackToLogsOnError will use the last chunk of container log output if the termination message file is empty and the container exited with an error. The log output is limited to 2048 bytes or 80 lines, whichever is smaller. Defaults to File. Cannot be updated.", "type": "string" }, "tty": { "description": "Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. Default is false.", "type": "boolean" }, "volumeDevices": { "description": "volumeDevices is the list of block devices to be used by the container. This is a beta feature.", "type": "array", "items": { "description": "volumeDevice describes a mapping of a raw block device within a container.", "type": "object", "required": [ "name", "devicePath" ], "properties": { "devicePath": { "description": "devicePath is the path inside of the container that the device will be mapped to.", "type": "string" }, "name": { "description": "name must match the name of a persistentVolumeClaim in the pod", "type": "string" } } } }, "volumeMounts": { "description": "Pod volumes to mount into the container's filesystem. Cannot be updated.", "type": "array", "items": { "description": "VolumeMount describes a mounting of a Volume within a container.", "type": "object", "required": [ "name", "mountPath" ], "properties": { "mountPath": { "description": "Path within the container at which the volume should be mounted. Must not contain ':'.", "type": "string" }, "mountPropagation": { "description": "mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10.", "type": "string" }, "name": { "description": "This must match the Name of a Volume.", "type": "string" }, "readOnly": { "description": "Mounted read-only if true, read-write otherwise (false or unspecified). Defaults to false.", "type": "boolean" }, "subPath": { "description": "Path within the volume from which the container's volume should be mounted. Defaults to \"\" (volume's root).", "type": "string" }, "subPathExpr": { "description": "Expanded path within the volume from which the container's volume should be mounted. Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. Defaults to \"\" (volume's root). SubPathExpr and SubPath are mutually exclusive. This field is alpha in 1.14.", "type": "string" } } } }, "workingDir": { "description": "Container's working directory. If not specified, the container runtime's default will be used, which might be configured in the container image. Cannot be updated.", "type": "string" } } } }, "enableAdminAPI": { "description": "Enable access to prometheus web admin API. Defaults to the value of `false`. WARNING: Enabling the admin APIs enables mutating endpoints, to delete data, shutdown Prometheus, and more. Enabling this should be done with care and the user is advised to add additional authentication authorization via a proxy to ensure only clients authorized to perform these actions can do so. For more information see https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-admin-apis", "type": "boolean" }, "evaluationInterval": { "description": "Interval between consecutive evaluations.", "type": "string" }, "externalLabels": { "description": "The labels to add to any time series or alerts when communicating with external systems (federation, remote storage, Alertmanager).", "type": "object" }, "externalUrl": { "description": "The external URL the Prometheus instances will be available under. This is necessary to generate correct URLs. This is necessary if Prometheus is not served from root of a DNS name.", "type": "string" }, "image": { "description": "Image if specified has precedence over baseImage, tag and sha combinations. Specifying the version is still necessary to ensure the Prometheus Operator knows what version of Prometheus is being configured.", "type": "string" }, "imagePullSecrets": { "description": "An optional list of references to secrets in the same namespace to use for pulling prometheus and alertmanager images from registries see http://kubernetes.io/docs/user-guide/images#specifying-imagepullsecrets-on-a-pod", "type": "array", "items": { "description": "LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace.", "type": "object", "properties": { "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", "type": "string" } } } }, "listenLocal": { "description": "ListenLocal makes the Prometheus server listen on loopback, so that it does not bind against the Pod IP.", "type": "boolean" }, "logFormat": { "description": "Log format for Prometheus to be configured with.", "type": "string" }, "logLevel": { "description": "Log level for Prometheus to be configured with.", "type": "string" }, "nodeSelector": { "description": "Define which Nodes the Pods are scheduled on.", "type": "object" }, "paused": { "description": "When a Prometheus deployment is paused, no actions except for deletion will be performed on the underlying objects.", "type": "boolean" }, "podMetadata": { "description": "ObjectMeta is metadata that all persisted resources must have, which includes all objects users must create.", "type": "object", "properties": { "annotations": { "description": "Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations", "type": "object" }, "clusterName": { "description": "The name of the cluster which the object belongs to. This is used to distinguish resources with same name and namespace in different clusters. This field is not set anywhere right now and apiserver is going to ignore it if set in create or update request.", "type": "string" }, "creationTimestamp": { "description": "Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.", "type": "string", "format": "date-time" }, "deletionGracePeriodSeconds": { "description": "Number of seconds allowed for this object to gracefully terminate before it will be removed from the system. Only set when deletionTimestamp is also set. May only be shortened. Read-only.", "type": "integer", "format": "int64" }, "deletionTimestamp": { "description": "Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.", "type": "string", "format": "date-time" }, "finalizers": { "description": "Must be empty before the object is deleted from the registry. Each entry is an identifier for the responsible component that will remove the entry from the list. If the deletionTimestamp of the object is non-nil, entries in this list can only be removed.", "type": "array", "items": { "type": "string" } }, "generateName": { "description": "GenerateName is an optional prefix, used by the server, to generate a unique name ONLY IF the Name field has not been provided. If this field is used, the name returned to the client will be different than the name passed. This value will also be combined with a unique suffix. The provided value has the same validation rules as the Name field, and may be truncated by the length of the suffix required to make the value unique on the server.\n\nIf this field is specified and the generated name exists, the server will NOT return a 409 - instead, it will either return 201 Created or 500 with Reason ServerTimeout indicating a unique name could not be found in the time allotted, and the client should retry (optionally after the time indicated in the Retry-After header).\n\nApplied only if Name is not specified. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#idempotency", "type": "string" }, "generation": { "description": "A sequence number representing a specific generation of the desired state. Populated by the system. Read-only.", "type": "integer", "format": "int64" }, "initializers": { "description": "Initializers tracks the progress of initialization.", "type": "object", "required": [ "pending" ], "properties": { "pending": { "description": "Pending is a list of initializers that must execute in order before this object is visible. When the last pending initializer is removed, and no failing result is set, the initializers struct will be set to nil and the object is considered as initialized and visible to all clients.", "type": "array", "items": { "description": "Initializer is information about an initializer that has not yet completed.", "type": "object", "required": [ "name" ], "properties": { "name": { "description": "name of the process that is responsible for initializing this object.", "type": "string" } } } }, "result": { "description": "Status is a return value for calls that don't return other objects.", "type": "object", "properties": { "apiVersion": { "description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources", "type": "string" }, "code": { "description": "Suggested HTTP return code for this status, 0 if not set.", "type": "integer", "format": "int32" }, "details": { "description": "StatusDetails is a set of additional properties that MAY be set by the server to provide additional information about a response. The Reason field of a Status object defines what attributes will be set. Clients must ignore fields that do not match the defined type of each attribute, and should assume that any attribute may be empty, invalid, or under defined.", "type": "object", "properties": { "causes": { "description": "The Causes array includes more details associated with the StatusReason failure. Not all StatusReasons may provide detailed causes.", "type": "array", "items": { "description": "StatusCause provides more information about an api.Status failure, including cases when multiple errors are encountered.", "type": "object", "properties": { "field": { "description": "The field of the resource that has caused this error, as named by its JSON serialization. May include dot and postfix notation for nested attributes. Arrays are zero-indexed. Fields may appear more than once in an array of causes due to fields having multiple errors. Optional.\n\nExamples:\n \"name\" - the field \"name\" on the current resource\n \"items[0].name\" - the field \"name\" on the first array entry in \"items\"", "type": "string" }, "message": { "description": "A human-readable description of the cause of the error. This field may be presented as-is to a reader.", "type": "string" }, "reason": { "description": "A machine-readable description of the cause of the error. If this value is empty there is no information available.", "type": "string" } } } }, "group": { "description": "The group attribute of the resource associated with the status StatusReason.", "type": "string" }, "kind": { "description": "The kind attribute of the resource associated with the status StatusReason. On some operations may differ from the requested resource Kind. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds", "type": "string" }, "name": { "description": "The name attribute of the resource associated with the status StatusReason (when there is a single name which can be described).", "type": "string" }, "retryAfterSeconds": { "description": "If specified, the time in seconds before the operation should be retried. Some errors may indicate the client must take an alternate action - for those errors this field may indicate how long to wait before taking the alternate action.", "type": "integer", "format": "int32" }, "uid": { "description": "UID of the resource. (when there is a single resource which can be described). More info: http://kubernetes.io/docs/user-guide/identifiers#uids", "type": "string" } } }, "kind": { "description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds", "type": "string" }, "message": { "description": "A human-readable description of the status of this operation.", "type": "string" }, "metadata": { "description": "ListMeta describes metadata that synthetic resources must have, including lists and various status objects. A resource may have only one of {ObjectMeta, ListMeta}.", "type": "object", "properties": { "continue": { "description": "continue may be set if the user set a limit on the number of items returned, and indicates that the server has more data available. The value is opaque and may be used to issue another request to the endpoint that served this list to retrieve the next set of available objects. Continuing a consistent list may not be possible if the server configuration has changed or more than a few minutes have passed. The resourceVersion field returned when using this continue value will be identical to the value in the first response, unless you have received this token from an error message.", "type": "string" }, "resourceVersion": { "description": "String that identifies the server's internal version of this object that can be used by clients to determine when objects have changed. Value must be treated as opaque by clients and passed unmodified back to the server. Populated by the system. Read-only. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency", "type": "string" }, "selfLink": { "description": "selfLink is a URL representing this object. Populated by the system. Read-only.", "type": "string" } } }, "reason": { "description": "A machine-readable description of why this operation is in the \"Failure\" status. If this value is empty there is no information available. A Reason clarifies an HTTP status code but does not override it.", "type": "string" }, "status": { "description": "Status of the operation. One of: \"Success\" or \"Failure\". More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status", "type": "string" } } } } }, "labels": { "description": "Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels", "type": "object" }, "managedFields": { "description": "ManagedFields maps workflow-id and version to the set of fields that are managed by that workflow. This is mostly for internal housekeeping, and users typically shouldn't need to set or understand this field. A workflow can be the user's name, a controller's name, or the name of a specific apply path like \"ci-cd\". The set of fields is always in the version that the workflow used when modifying the object.\n\nThis field is alpha and can be changed or removed without notice.", "type": "array", "items": { "description": "ManagedFieldsEntry is a workflow-id, a FieldSet and the group version of the resource that the fieldset applies to.", "type": "object", "properties": { "apiVersion": { "description": "APIVersion defines the version of this resource that this field set applies to. The format is \"group/version\" just like the top-level APIVersion field. It is necessary to track the version of a field set because it cannot be automatically converted.", "type": "string" }, "fields": { "description": "Fields stores a set of fields in a data structure like a Trie. To understand how this is used, see: https://github.com/kubernetes-sigs/structured-merge-diff", "type": "object" }, "manager": { "description": "Manager is an identifier of the workflow managing these fields.", "type": "string" }, "operation": { "description": "Operation is the type of operation which lead to this ManagedFieldsEntry being created. The only valid values for this field are 'Apply' and 'Update'.", "type": "string" }, "time": { "description": "Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.", "type": "string", "format": "date-time" } } } }, "name": { "description": "Name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names", "type": "string" }, "namespace": { "description": "Namespace defines the space within each name must be unique. An empty namespace is equivalent to the \"default\" namespace, but \"default\" is the canonical representation. Not all objects are required to be scoped to a namespace - the value of this field for those objects will be empty.\n\nMust be a DNS_LABEL. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/namespaces", "type": "string" }, "ownerReferences": { "description": "List of objects depended by this object. If ALL objects in the list have been deleted, this object will be garbage collected. If this object is managed by a controller, then an entry in this list will point to this controller, with the controller field set to true. There cannot be more than one managing controller.", "type": "array", "items": { "description": "OwnerReference contains enough information to let you identify an owning object. An owning object must be in the same namespace as the dependent, or be cluster-scoped, so there is no namespace field.", "type": "object", "required": [ "apiVersion", "kind", "name", "uid" ], "properties": { "apiVersion": { "description": "API version of the referent.", "type": "string" }, "blockOwnerDeletion": { "description": "If true, AND if the owner has the \"foregroundDeletion\" finalizer, then the owner cannot be deleted from the key-value store until this reference is removed. Defaults to false. To set this field, a user needs \"delete\" permission of the owner, otherwise 422 (Unprocessable Entity) will be returned.", "type": "boolean" }, "controller": { "description": "If true, this reference points to the managing controller.", "type": "boolean" }, "kind": { "description": "Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds", "type": "string" }, "name": { "description": "Name of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#names", "type": "string" }, "uid": { "description": "UID of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#uids", "type": "string" } } } }, "resourceVersion": { "description": "An opaque value that represents the internal version of this object that can be used by clients to determine when objects have changed. May be used for optimistic concurrency, change detection, and the watch operation on a resource or set of resources. Clients must treat these values as opaque and passed unmodified back to the server. They may only be valid for a particular resource or set of resources.\n\nPopulated by the system. Read-only. Value must be treated as opaque by clients and . More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency", "type": "string" }, "selfLink": { "description": "SelfLink is a URL representing this object. Populated by the system. Read-only.", "type": "string" }, "uid": { "description": "UID is the unique in time and space value for this object. It is typically generated by the server on successful creation of a resource and is not allowed to change on PUT operations.\n\nPopulated by the system. Read-only. More info: http://kubernetes.io/docs/user-guide/identifiers#uids", "type": "string" } } }, "podMonitorNamespaceSelector": { "description": "A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.", "type": "object", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "type": "array", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "type": "object", "required": [ "key", "operator" ], "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "type": "array", "items": { "type": "string" } } } } }, "matchLabels": { "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object" } } }, "podMonitorSelector": { "description": "A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.", "type": "object", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "type": "array", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "type": "object", "required": [ "key", "operator" ], "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "type": "array", "items": { "type": "string" } } } } }, "matchLabels": { "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object" } } }, "priorityClassName": { "description": "Priority class assigned to the Pods", "type": "string" }, "prometheusExternalLabelName": { "description": "Name of Prometheus external label used to denote Prometheus instance name. Defaults to the value of `prometheus`. External label will _not_ be added when value is set to empty string (`\"\"`).", "type": "string" }, "query": { "description": "QuerySpec defines the query command line flags when starting Prometheus.", "type": "object", "properties": { "lookbackDelta": { "description": "The delta difference allowed for retrieving metrics during expression evaluations.", "type": "string" }, "maxConcurrency": { "description": "Number of concurrent queries that can be run at once.", "type": "integer", "format": "int32" }, "maxSamples": { "description": "Maximum number of samples a single query can load into memory. Note that queries will fail if they would load more samples than this into memory, so this also limits the number of samples a query can return.", "type": "integer", "format": "int32" }, "timeout": { "description": "Maximum time a query may take before being aborted.", "type": "string" } } }, "remoteRead": { "description": "If specified, the remote_read spec. This is an experimental feature, it may change in any upcoming release in a breaking way.", "type": "array", "items": { "description": "RemoteReadSpec defines the remote_read configuration for prometheus.", "type": "object", "required": [ "url" ], "properties": { "basicAuth": { "description": "BasicAuth allow an endpoint to authenticate over basic authentication More info: https://prometheus.io/docs/operating/configuration/#endpoints", "type": "object", "properties": { "password": { "description": "SecretKeySelector selects a key of a Secret.", "type": "object", "required": [ "key" ], "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", "type": "string" }, "optional": { "description": "Specify whether the Secret or it's key must be defined", "type": "boolean" } } }, "username": { "description": "SecretKeySelector selects a key of a Secret.", "type": "object", "required": [ "key" ], "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", "type": "string" }, "optional": { "description": "Specify whether the Secret or it's key must be defined", "type": "boolean" } } } } }, "bearerToken": { "description": "bearer token for remote read.", "type": "string" }, "bearerTokenFile": { "description": "File to read bearer token for remote read.", "type": "string" }, "proxyUrl": { "description": "Optional ProxyURL", "type": "string" }, "readRecent": { "description": "Whether reads should be made for queries for time ranges that the local storage should have complete data for.", "type": "boolean" }, "remoteTimeout": { "description": "Timeout for requests to the remote read endpoint.", "type": "string" }, "requiredMatchers": { "description": "An optional list of equality matchers which have to be present in a selector to query the remote read endpoint.", "type": "object" }, "tlsConfig": { "description": "TLSConfig specifies TLS configuration parameters.", "type": "object", "properties": { "caFile": { "description": "The CA cert to use for the targets.", "type": "string" }, "certFile": { "description": "The client cert file for the targets.", "type": "string" }, "insecureSkipVerify": { "description": "Disable target certificate validation.", "type": "boolean" }, "keyFile": { "description": "The client key file for the targets.", "type": "string" }, "serverName": { "description": "Used to verify the hostname for the targets.", "type": "string" } } }, "url": { "description": "The URL of the endpoint to send samples to.", "type": "string" } } } }, "remoteWrite": { "description": "If specified, the remote_write spec. This is an experimental feature, it may change in any upcoming release in a breaking way.", "type": "array", "items": { "description": "RemoteWriteSpec defines the remote_write configuration for prometheus.", "type": "object", "required": [ "url" ], "properties": { "basicAuth": { "description": "BasicAuth allow an endpoint to authenticate over basic authentication More info: https://prometheus.io/docs/operating/configuration/#endpoints", "type": "object", "properties": { "password": { "description": "SecretKeySelector selects a key of a Secret.", "type": "object", "required": [ "key" ], "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", "type": "string" }, "optional": { "description": "Specify whether the Secret or it's key must be defined", "type": "boolean" } } }, "username": { "description": "SecretKeySelector selects a key of a Secret.", "type": "object", "required": [ "key" ], "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", "type": "string" }, "optional": { "description": "Specify whether the Secret or it's key must be defined", "type": "boolean" } } } } }, "bearerToken": { "description": "File to read bearer token for remote write.", "type": "string" }, "bearerTokenFile": { "description": "File to read bearer token for remote write.", "type": "string" }, "proxyUrl": { "description": "Optional ProxyURL", "type": "string" }, "queueConfig": { "description": "QueueConfig allows the tuning of remote_write queue_config parameters. This object is referenced in the RemoteWriteSpec object.", "type": "object", "properties": { "batchSendDeadline": { "description": "BatchSendDeadline is the maximum time a sample will wait in buffer.", "type": "string" }, "capacity": { "description": "Capacity is the number of samples to buffer per shard before we start dropping them.", "type": "integer", "format": "int32" }, "maxBackoff": { "description": "MaxBackoff is the maximum retry delay.", "type": "string" }, "maxRetries": { "description": "MaxRetries is the maximum number of times to retry a batch on recoverable errors.", "type": "integer", "format": "int32" }, "maxSamplesPerSend": { "description": "MaxSamplesPerSend is the maximum number of samples per send.", "type": "integer", "format": "int32" }, "maxShards": { "description": "MaxShards is the maximum number of shards, i.e. amount of concurrency.", "type": "integer", "format": "int32" }, "minBackoff": { "description": "MinBackoff is the initial retry delay. Gets doubled for every retry.", "type": "string" }, "minShards": { "description": "MinShards is the minimum number of shards, i.e. amount of concurrency.", "type": "integer", "format": "int32" } } }, "remoteTimeout": { "description": "Timeout for requests to the remote write endpoint.", "type": "string" }, "tlsConfig": { "description": "TLSConfig specifies TLS configuration parameters.", "type": "object", "properties": { "caFile": { "description": "The CA cert to use for the targets.", "type": "string" }, "certFile": { "description": "The client cert file for the targets.", "type": "string" }, "insecureSkipVerify": { "description": "Disable target certificate validation.", "type": "boolean" }, "keyFile": { "description": "The client key file for the targets.", "type": "string" }, "serverName": { "description": "Used to verify the hostname for the targets.", "type": "string" } } }, "url": { "description": "The URL of the endpoint to send samples to.", "type": "string" }, "writeRelabelConfigs": { "description": "The list of remote write relabel configurations.", "type": "array", "items": { "description": "RelabelConfig allows dynamic rewriting of the label set, being applied to samples before ingestion. It defines ``-section of Prometheus configuration. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs", "type": "object", "properties": { "action": { "description": "Action to perform based on regex matching. Default is 'replace'", "type": "string" }, "modulus": { "description": "Modulus to take of the hash of the source label values.", "type": "integer", "format": "int64" }, "regex": { "description": "Regular expression against which the extracted value is matched. default is '(.*)'", "type": "string" }, "replacement": { "description": "Replacement value against which a regex replace is performed if the regular expression matches. Regex capture groups are available. Default is '$1'", "type": "string" }, "separator": { "description": "Separator placed between concatenated source label values. default is ';'.", "type": "string" }, "sourceLabels": { "description": "The source labels select values from existing labels. Their content is concatenated using the configured separator and matched against the configured regular expression for the replace, keep, and drop actions.", "type": "array", "items": { "type": "string" } }, "targetLabel": { "description": "Label to which the resulting value is written in a replace action. It is mandatory for replace actions. Regex capture groups are available.", "type": "string" } } } } } } }, "replicaExternalLabelName": { "description": "Name of Prometheus external label used to denote replica name. Defaults to the value of `prometheus_replica`. External label will _not_ be added when value is set to empty string (`\"\"`).", "type": "string" }, "replicas": { "description": "Number of instances to deploy for a Prometheus deployment.", "type": "integer", "format": "int32" }, "resources": { "description": "ResourceRequirements describes the compute resource requirements.", "type": "object", "properties": { "limits": { "description": "Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", "type": "object" }, "requests": { "description": "Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", "type": "object" } } }, "retention": { "description": "Time duration Prometheus shall retain data for. Default is '24h', and must match the regular expression `[0-9]+(ms|s|m|h|d|w|y)` (milliseconds seconds minutes hours days weeks years).", "type": "string" }, "retentionSize": { "description": "Maximum amount of disk space used by blocks.", "type": "string" }, "routePrefix": { "description": "The route prefix Prometheus registers HTTP handlers for. This is useful, if using ExternalURL and a proxy is rewriting HTTP routes of a request, and the actual ExternalURL is still true, but the server serves requests under a different route prefix. For example for use with `kubectl proxy`.", "type": "string" }, "ruleNamespaceSelector": { "description": "A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.", "type": "object", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "type": "array", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "type": "object", "required": [ "key", "operator" ], "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "type": "array", "items": { "type": "string" } } } } }, "matchLabels": { "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object" } } }, "ruleSelector": { "description": "A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.", "type": "object", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "type": "array", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "type": "object", "required": [ "key", "operator" ], "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "type": "array", "items": { "type": "string" } } } } }, "matchLabels": { "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object" } } }, "rules": { "description": "/--rules.*/ command-line arguments", "type": "object", "properties": { "alert": { "description": "/--rules.alert.*/ command-line arguments", "type": "object", "properties": { "forGracePeriod": { "description": "Minimum duration between alert and restored 'for' state. This is maintained only for alerts with configured 'for' time greater than grace period.", "type": "string" }, "forOutageTolerance": { "description": "Max time to tolerate prometheus outage for restoring 'for' state of alert.", "type": "string" }, "resendDelay": { "description": "Minimum amount of time to wait before resending an alert to Alertmanager.", "type": "string" } } } } }, "scrapeInterval": { "description": "Interval between consecutive scrapes.", "type": "string" }, "secrets": { "description": "Secrets is a list of Secrets in the same namespace as the Prometheus object, which shall be mounted into the Prometheus Pods. The Secrets are mounted into /etc/prometheus/secrets/.", "type": "array", "items": { "type": "string" } }, "securityContext": { "description": "PodSecurityContext holds pod-level security attributes and common container settings. Some fields are also present in container.securityContext. Field values of container.securityContext take precedence over field values of PodSecurityContext.", "type": "object", "properties": { "fsGroup": { "description": "A special supplemental group that applies to all containers in a pod. Some volume types allow the Kubelet to change the ownership of that volume to be owned by the pod:\n\n1. The owning GID will be the FSGroup 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) 3. The permission bits are OR'd with rw-rw----\n\nIf unset, the Kubelet will not modify the ownership and permissions of any volume.", "type": "integer", "format": "int64" }, "runAsGroup": { "description": "The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container.", "type": "integer", "format": "int64" }, "runAsNonRoot": { "description": "Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.", "type": "boolean" }, "runAsUser": { "description": "The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container.", "type": "integer", "format": "int64" }, "seLinuxOptions": { "description": "SELinuxOptions are the labels to be applied to the container", "type": "object", "properties": { "level": { "description": "Level is SELinux level label that applies to the container.", "type": "string" }, "role": { "description": "Role is a SELinux role label that applies to the container.", "type": "string" }, "type": { "description": "Type is a SELinux type label that applies to the container.", "type": "string" }, "user": { "description": "User is a SELinux user label that applies to the container.", "type": "string" } } }, "supplementalGroups": { "description": "A list of groups applied to the first process run in each container, in addition to the container's primary GID. If unspecified, no groups will be added to any container.", "type": "array", "items": { "type": "integer", "format": "int64" } }, "sysctls": { "description": "Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported sysctls (by the container runtime) might fail to launch.", "type": "array", "items": { "description": "Sysctl defines a kernel parameter to be set", "type": "object", "required": [ "name", "value" ], "properties": { "name": { "description": "Name of a property to set", "type": "string" }, "value": { "description": "Value of a property to set", "type": "string" } } } } } }, "serviceAccountName": { "description": "ServiceAccountName is the name of the ServiceAccount to use to run the Prometheus Pods.", "type": "string" }, "serviceMonitorNamespaceSelector": { "description": "A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.", "type": "object", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "type": "array", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "type": "object", "required": [ "key", "operator" ], "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "type": "array", "items": { "type": "string" } } } } }, "matchLabels": { "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object" } } }, "serviceMonitorSelector": { "description": "A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.", "type": "object", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "type": "array", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "type": "object", "required": [ "key", "operator" ], "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "type": "array", "items": { "type": "string" } } } } }, "matchLabels": { "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object" } } }, "sha": { "description": "SHA of Prometheus container image to be deployed. Defaults to the value of `version`. Similar to a tag, but the SHA explicitly deploys an immutable container image. Version and Tag are ignored if SHA is set.", "type": "string" }, "storage": { "description": "StorageSpec defines the configured storage for a group Prometheus servers. If neither `emptyDir` nor `volumeClaimTemplate` is specified, then by default an [EmptyDir](https://kubernetes.io/docs/concepts/storage/volumes/#emptydir) will be used.", "type": "object", "properties": { "emptyDir": { "description": "Represents an empty directory for a pod. Empty directory volumes support ownership management and SELinux relabeling.", "type": "object", "properties": { "medium": { "description": "What type of storage medium should back this directory. The default is \"\" which means to use the node's default medium. Must be an empty string (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir", "type": "string" }, "sizeLimit": {} } }, "volumeClaimTemplate": { "description": "PersistentVolumeClaim is a user's request for and claim to a persistent volume", "type": "object", "properties": { "apiVersion": { "description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources", "type": "string" }, "kind": { "description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds", "type": "string" }, "metadata": { "description": "ObjectMeta is metadata that all persisted resources must have, which includes all objects users must create.", "type": "object", "properties": { "annotations": { "description": "Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations", "type": "object" }, "clusterName": { "description": "The name of the cluster which the object belongs to. This is used to distinguish resources with same name and namespace in different clusters. This field is not set anywhere right now and apiserver is going to ignore it if set in create or update request.", "type": "string" }, "creationTimestamp": { "description": "Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.", "type": "string", "format": "date-time" }, "deletionGracePeriodSeconds": { "description": "Number of seconds allowed for this object to gracefully terminate before it will be removed from the system. Only set when deletionTimestamp is also set. May only be shortened. Read-only.", "type": "integer", "format": "int64" }, "deletionTimestamp": { "description": "Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.", "type": "string", "format": "date-time" }, "finalizers": { "description": "Must be empty before the object is deleted from the registry. Each entry is an identifier for the responsible component that will remove the entry from the list. If the deletionTimestamp of the object is non-nil, entries in this list can only be removed.", "type": "array", "items": { "type": "string" } }, "generateName": { "description": "GenerateName is an optional prefix, used by the server, to generate a unique name ONLY IF the Name field has not been provided. If this field is used, the name returned to the client will be different than the name passed. This value will also be combined with a unique suffix. The provided value has the same validation rules as the Name field, and may be truncated by the length of the suffix required to make the value unique on the server.\n\nIf this field is specified and the generated name exists, the server will NOT return a 409 - instead, it will either return 201 Created or 500 with Reason ServerTimeout indicating a unique name could not be found in the time allotted, and the client should retry (optionally after the time indicated in the Retry-After header).\n\nApplied only if Name is not specified. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#idempotency", "type": "string" }, "generation": { "description": "A sequence number representing a specific generation of the desired state. Populated by the system. Read-only.", "type": "integer", "format": "int64" }, "initializers": { "description": "Initializers tracks the progress of initialization.", "type": "object", "required": [ "pending" ], "properties": { "pending": { "description": "Pending is a list of initializers that must execute in order before this object is visible. When the last pending initializer is removed, and no failing result is set, the initializers struct will be set to nil and the object is considered as initialized and visible to all clients.", "type": "array", "items": { "description": "Initializer is information about an initializer that has not yet completed.", "type": "object", "required": [ "name" ], "properties": { "name": { "description": "name of the process that is responsible for initializing this object.", "type": "string" } } } }, "result": { "description": "Status is a return value for calls that don't return other objects.", "type": "object", "properties": { "apiVersion": { "description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources", "type": "string" }, "code": { "description": "Suggested HTTP return code for this status, 0 if not set.", "type": "integer", "format": "int32" }, "details": { "description": "StatusDetails is a set of additional properties that MAY be set by the server to provide additional information about a response. The Reason field of a Status object defines what attributes will be set. Clients must ignore fields that do not match the defined type of each attribute, and should assume that any attribute may be empty, invalid, or under defined.", "type": "object", "properties": { "causes": { "description": "The Causes array includes more details associated with the StatusReason failure. Not all StatusReasons may provide detailed causes.", "type": "array", "items": { "description": "StatusCause provides more information about an api.Status failure, including cases when multiple errors are encountered.", "type": "object", "properties": { "field": { "description": "The field of the resource that has caused this error, as named by its JSON serialization. May include dot and postfix notation for nested attributes. Arrays are zero-indexed. Fields may appear more than once in an array of causes due to fields having multiple errors. Optional.\n\nExamples:\n \"name\" - the field \"name\" on the current resource\n \"items[0].name\" - the field \"name\" on the first array entry in \"items\"", "type": "string" }, "message": { "description": "A human-readable description of the cause of the error. This field may be presented as-is to a reader.", "type": "string" }, "reason": { "description": "A machine-readable description of the cause of the error. If this value is empty there is no information available.", "type": "string" } } } }, "group": { "description": "The group attribute of the resource associated with the status StatusReason.", "type": "string" }, "kind": { "description": "The kind attribute of the resource associated with the status StatusReason. On some operations may differ from the requested resource Kind. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds", "type": "string" }, "name": { "description": "The name attribute of the resource associated with the status StatusReason (when there is a single name which can be described).", "type": "string" }, "retryAfterSeconds": { "description": "If specified, the time in seconds before the operation should be retried. Some errors may indicate the client must take an alternate action - for those errors this field may indicate how long to wait before taking the alternate action.", "type": "integer", "format": "int32" }, "uid": { "description": "UID of the resource. (when there is a single resource which can be described). More info: http://kubernetes.io/docs/user-guide/identifiers#uids", "type": "string" } } }, "kind": { "description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds", "type": "string" }, "message": { "description": "A human-readable description of the status of this operation.", "type": "string" }, "metadata": { "description": "ListMeta describes metadata that synthetic resources must have, including lists and various status objects. A resource may have only one of {ObjectMeta, ListMeta}.", "type": "object", "properties": { "continue": { "description": "continue may be set if the user set a limit on the number of items returned, and indicates that the server has more data available. The value is opaque and may be used to issue another request to the endpoint that served this list to retrieve the next set of available objects. Continuing a consistent list may not be possible if the server configuration has changed or more than a few minutes have passed. The resourceVersion field returned when using this continue value will be identical to the value in the first response, unless you have received this token from an error message.", "type": "string" }, "resourceVersion": { "description": "String that identifies the server's internal version of this object that can be used by clients to determine when objects have changed. Value must be treated as opaque by clients and passed unmodified back to the server. Populated by the system. Read-only. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency", "type": "string" }, "selfLink": { "description": "selfLink is a URL representing this object. Populated by the system. Read-only.", "type": "string" } } }, "reason": { "description": "A machine-readable description of why this operation is in the \"Failure\" status. If this value is empty there is no information available. A Reason clarifies an HTTP status code but does not override it.", "type": "string" }, "status": { "description": "Status of the operation. One of: \"Success\" or \"Failure\". More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status", "type": "string" } } } } }, "labels": { "description": "Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels", "type": "object" }, "managedFields": { "description": "ManagedFields maps workflow-id and version to the set of fields that are managed by that workflow. This is mostly for internal housekeeping, and users typically shouldn't need to set or understand this field. A workflow can be the user's name, a controller's name, or the name of a specific apply path like \"ci-cd\". The set of fields is always in the version that the workflow used when modifying the object.\n\nThis field is alpha and can be changed or removed without notice.", "type": "array", "items": { "description": "ManagedFieldsEntry is a workflow-id, a FieldSet and the group version of the resource that the fieldset applies to.", "type": "object", "properties": { "apiVersion": { "description": "APIVersion defines the version of this resource that this field set applies to. The format is \"group/version\" just like the top-level APIVersion field. It is necessary to track the version of a field set because it cannot be automatically converted.", "type": "string" }, "fields": { "description": "Fields stores a set of fields in a data structure like a Trie. To understand how this is used, see: https://github.com/kubernetes-sigs/structured-merge-diff", "type": "object" }, "manager": { "description": "Manager is an identifier of the workflow managing these fields.", "type": "string" }, "operation": { "description": "Operation is the type of operation which lead to this ManagedFieldsEntry being created. The only valid values for this field are 'Apply' and 'Update'.", "type": "string" }, "time": { "description": "Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.", "type": "string", "format": "date-time" } } } }, "name": { "description": "Name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names", "type": "string" }, "namespace": { "description": "Namespace defines the space within each name must be unique. An empty namespace is equivalent to the \"default\" namespace, but \"default\" is the canonical representation. Not all objects are required to be scoped to a namespace - the value of this field for those objects will be empty.\n\nMust be a DNS_LABEL. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/namespaces", "type": "string" }, "ownerReferences": { "description": "List of objects depended by this object. If ALL objects in the list have been deleted, this object will be garbage collected. If this object is managed by a controller, then an entry in this list will point to this controller, with the controller field set to true. There cannot be more than one managing controller.", "type": "array", "items": { "description": "OwnerReference contains enough information to let you identify an owning object. An owning object must be in the same namespace as the dependent, or be cluster-scoped, so there is no namespace field.", "type": "object", "required": [ "apiVersion", "kind", "name", "uid" ], "properties": { "apiVersion": { "description": "API version of the referent.", "type": "string" }, "blockOwnerDeletion": { "description": "If true, AND if the owner has the \"foregroundDeletion\" finalizer, then the owner cannot be deleted from the key-value store until this reference is removed. Defaults to false. To set this field, a user needs \"delete\" permission of the owner, otherwise 422 (Unprocessable Entity) will be returned.", "type": "boolean" }, "controller": { "description": "If true, this reference points to the managing controller.", "type": "boolean" }, "kind": { "description": "Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds", "type": "string" }, "name": { "description": "Name of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#names", "type": "string" }, "uid": { "description": "UID of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#uids", "type": "string" } } } }, "resourceVersion": { "description": "An opaque value that represents the internal version of this object that can be used by clients to determine when objects have changed. May be used for optimistic concurrency, change detection, and the watch operation on a resource or set of resources. Clients must treat these values as opaque and passed unmodified back to the server. They may only be valid for a particular resource or set of resources.\n\nPopulated by the system. Read-only. Value must be treated as opaque by clients and . More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency", "type": "string" }, "selfLink": { "description": "SelfLink is a URL representing this object. Populated by the system. Read-only.", "type": "string" }, "uid": { "description": "UID is the unique in time and space value for this object. It is typically generated by the server on successful creation of a resource and is not allowed to change on PUT operations.\n\nPopulated by the system. Read-only. More info: http://kubernetes.io/docs/user-guide/identifiers#uids", "type": "string" } } }, "spec": { "description": "PersistentVolumeClaimSpec describes the common attributes of storage devices and allows a Source for provider-specific attributes", "type": "object", "properties": { "accessModes": { "description": "AccessModes contains the desired access modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1", "type": "array", "items": { "type": "string" } }, "dataSource": { "description": "TypedLocalObjectReference contains enough information to let you locate the typed referenced object inside the same namespace.", "type": "object", "required": [ "kind", "name" ], "properties": { "apiGroup": { "description": "APIGroup is the group for the resource being referenced. If APIGroup is not specified, the specified Kind must be in the core API group. For any other third-party types, APIGroup is required.", "type": "string" }, "kind": { "description": "Kind is the type of resource being referenced", "type": "string" }, "name": { "description": "Name is the name of resource being referenced", "type": "string" } } }, "resources": { "description": "ResourceRequirements describes the compute resource requirements.", "type": "object", "properties": { "limits": { "description": "Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", "type": "object" }, "requests": { "description": "Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", "type": "object" } } }, "selector": { "description": "A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.", "type": "object", "properties": { "matchExpressions": { "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "type": "array", "items": { "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", "type": "object", "required": [ "key", "operator" ], "properties": { "key": { "description": "key is the label key that the selector applies to.", "type": "string" }, "operator": { "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", "type": "string" }, "values": { "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", "type": "array", "items": { "type": "string" } } } } }, "matchLabels": { "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", "type": "object" } } }, "storageClassName": { "description": "Name of the StorageClass required by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1", "type": "string" }, "volumeMode": { "description": "volumeMode defines what type of volume is required by the claim. Value of Filesystem is implied when not included in claim spec. This is a beta feature.", "type": "string" }, "volumeName": { "description": "VolumeName is the binding reference to the PersistentVolume backing this claim.", "type": "string" } } }, "status": { "description": "PersistentVolumeClaimStatus is the current status of a persistent volume claim.", "type": "object", "properties": { "accessModes": { "description": "AccessModes contains the actual access modes the volume backing the PVC has. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1", "type": "array", "items": { "type": "string" } }, "capacity": { "description": "Represents the actual resources of the underlying volume.", "type": "object" }, "conditions": { "description": "Current Condition of persistent volume claim. If underlying persistent volume is being resized then the Condition will be set to 'ResizeStarted'.", "type": "array", "items": { "description": "PersistentVolumeClaimCondition contains details about state of pvc", "type": "object", "required": [ "type", "status" ], "properties": { "lastProbeTime": { "description": "Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.", "type": "string", "format": "date-time" }, "lastTransitionTime": { "description": "Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.", "type": "string", "format": "date-time" }, "message": { "description": "Human-readable message indicating details about last transition.", "type": "string" }, "reason": { "description": "Unique, this should be a short, machine understandable string that gives the reason for condition's last transition. If it reports \"ResizeStarted\" that means the underlying persistent volume is being resized.", "type": "string" }, "status": { "type": "string" }, "type": { "type": "string" } } } }, "phase": { "description": "Phase represents the current phase of PersistentVolumeClaim.", "type": "string" } } } } } } }, "tag": { "description": "Tag of Prometheus container image to be deployed. Defaults to the value of `version`. Version is ignored if Tag is set.", "type": "string" }, "thanos": { "description": "ThanosSpec defines parameters for a Prometheus server within a Thanos deployment.", "type": "object", "properties": { "baseImage": { "description": "Thanos base image if other than default.", "type": "string" }, "image": { "description": "Image if specified has precedence over baseImage, tag and sha combinations. Specifying the version is still necessary to ensure the Prometheus Operator knows what version of Thanos is being configured.", "type": "string" }, "objectStorageConfig": { "description": "SecretKeySelector selects a key of a Secret.", "type": "object", "required": [ "key" ], "properties": { "key": { "description": "The key of the secret to select from. Must be a valid secret key.", "type": "string" }, "name": { "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", "type": "string" }, "optional": { "description": "Specify whether the Secret or it's key must be defined", "type": "boolean" } } }, "resources": { "description": "ResourceRequirements describes the compute resource requirements.", "type": "object", "properties": { "limits": { "description": "Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", "type": "object" }, "requests": { "description": "Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", "type": "object" } } }, "sha": { "description": "SHA of Thanos container image to be deployed. Defaults to the value of `version`. Similar to a tag, but the SHA explicitly deploys an immutable container image. Version and Tag are ignored if SHA is set.", "type": "string" }, "tag": { "description": "Tag of Thanos sidecar container image to be deployed. Defaults to the value of `version`. Version is ignored if Tag is set.", "type": "string" }, "version": { "description": "Version describes the version of Thanos to use.", "type": "string" } } }, "tolerations": { "description": "If specified, the pod's tolerations.", "type": "array", "items": { "description": "The pod this Toleration is attached to tolerates any taint that matches the triple using the matching operator .", "type": "object", "properties": { "effect": { "description": "Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.", "type": "string" }, "key": { "description": "Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys.", "type": "string" }, "operator": { "description": "Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category.", "type": "string" }, "tolerationSeconds": { "description": "TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system.", "type": "integer", "format": "int64" }, "value": { "description": "Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string.", "type": "string" } } } }, "version": { "description": "Version of Prometheus to be deployed.", "type": "string" }, "walCompression": { "description": "Enable compression of the write-ahead log using Snappy.", "type": "boolean" } } }, "status": { "description": "PrometheusStatus is the most recent observed status of the Prometheus cluster. Read-only. Not included when requesting from the apiserver, only from the Prometheus Operator API itself. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/api-conventions.md#spec-and-status", "type": "object", "required": [ "paused", "replicas", "updatedReplicas", "availableReplicas", "unavailableReplicas" ], "properties": { "availableReplicas": { "description": "Total number of available pods (ready for at least minReadySeconds) targeted by this Prometheus deployment.", "type": "integer", "format": "int32" }, "paused": { "description": "Represents whether any actions on the underlying managed objects are being performed. Only delete actions will be performed.", "type": "boolean" }, "replicas": { "description": "Total number of non-terminated pods targeted by this Prometheus deployment (their labels match the selector).", "type": "integer", "format": "int32" }, "unavailableReplicas": { "description": "Total number of unavailable pods targeted by this Prometheus deployment.", "type": "integer", "format": "int32" }, "updatedReplicas": { "description": "Total number of non-terminated pods targeted by this Prometheus deployment that have the desired version spec.", "type": "integer", "format": "int32" } } } } } }, "versions": [ { "name": "v1", "served": true, "storage": true } ], "conversion": { "strategy": "None" }, "preserveUnknownFields": true }, "status": { "conditions": [ { "type": "NamesAccepted", "status": "True", "lastTransitionTime": "2020-05-05T16:58:10Z", "reason": "NoConflicts", "message": "no conflicts found" }, { "type": "Established", "status": "True", "lastTransitionTime": "2020-05-05T16:58:10Z", "reason": "InitialNamesAccepted", "message": "the initial names have been accepted" }, { "type": "NonStructuralSchema", "status": "True", "lastTransitionTime": "2020-05-05T16:58:10Z", "reason": "Violations", "message": "[spec.validation.openAPIV3Schema.properties[spec].properties[alerting].properties[alertmanagers].items.properties[port].anyOf[0].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[alerting].properties[alertmanagers].items.properties[port].anyOf[1].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[alerting].properties[alertmanagers].items.properties[port].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[env].items.properties[valueFrom].properties[resourceFieldRef].properties[divisor].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[postStart].properties[httpGet].properties[port].anyOf[0].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[postStart].properties[httpGet].properties[port].anyOf[1].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[postStart].properties[httpGet].properties[port].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[postStart].properties[tcpSocket].properties[port].anyOf[0].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[postStart].properties[tcpSocket].properties[port].anyOf[1].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[postStart].properties[tcpSocket].properties[port].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[preStop].properties[httpGet].properties[port].anyOf[0].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[preStop].properties[httpGet].properties[port].anyOf[1].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[preStop].properties[httpGet].properties[port].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[preStop].properties[tcpSocket].properties[port].anyOf[0].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[preStop].properties[tcpSocket].properties[port].anyOf[1].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[lifecycle].properties[preStop].properties[tcpSocket].properties[port].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[livenessProbe].properties[httpGet].properties[port].anyOf[0].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[livenessProbe].properties[httpGet].properties[port].anyOf[1].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[livenessProbe].properties[httpGet].properties[port].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[livenessProbe].properties[tcpSocket].properties[port].anyOf[0].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[livenessProbe].properties[tcpSocket].properties[port].anyOf[1].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[livenessProbe].properties[tcpSocket].properties[port].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[readinessProbe].properties[httpGet].properties[port].anyOf[0].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[readinessProbe].properties[httpGet].properties[port].anyOf[1].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[readinessProbe].properties[httpGet].properties[port].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[readinessProbe].properties[tcpSocket].properties[port].anyOf[0].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[readinessProbe].properties[tcpSocket].properties[port].anyOf[1].type: Forbidden: must be empty to be structural, spec.validation.openAPIV3Schema.properties[spec].properties[containers].items.properties[readinessProbe].properties[tcpSocket].properties[port].type: Required value: must not be empty for specified object fields, spec.validation.openAPIV3Schema.properties[spec].properties[storage].properties[emptyDir].properties[sizeLimit].type: Required value: must not be empty for specified object fields]" } ], "acceptedNames": { "plural": "prometheuses", "singular": "prometheus", "kind": "Prometheus", "listKind": "PrometheusList" }, "storedVersions": [ "v1" ] } } ================================================ FILE: pkg/backup/backed_up_items_map.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backup import ( "fmt" "sort" "sync" ) // backedUpItemsMap keeps track of the items already backed up for the current Velero Backup type backedUpItemsMap struct { *sync.RWMutex backedUpItems map[itemKey]struct{} totalItems map[itemKey]struct{} } func NewBackedUpItemsMap() *backedUpItemsMap { return &backedUpItemsMap{ RWMutex: &sync.RWMutex{}, backedUpItems: make(map[itemKey]struct{}), totalItems: make(map[itemKey]struct{}), } } func (m *backedUpItemsMap) CopyItemMap() map[itemKey]struct{} { m.RLock() defer m.RUnlock() returnMap := make(map[itemKey]struct{}, len(m.backedUpItems)) for key, val := range m.backedUpItems { returnMap[key] = val } return returnMap } // ResourceMap returns a map of the backed up items. // For each map entry, the key is the resource type, // and the value is a list of namespaced names for the resource. func (m *backedUpItemsMap) ResourceMap() map[string][]string { m.RLock() defer m.RUnlock() resources := map[string][]string{} for i := range m.backedUpItems { entry := i.name if i.namespace != "" { entry = fmt.Sprintf("%s/%s", i.namespace, i.name) } resources[i.resource] = append(resources[i.resource], entry) } // sort namespace/name entries for each GVK for _, v := range resources { sort.Strings(v) } return resources } func (m *backedUpItemsMap) Len() int { m.RLock() defer m.RUnlock() return len(m.backedUpItems) } func (m *backedUpItemsMap) BackedUpAndTotalLen() (int, int) { m.RLock() defer m.RUnlock() return len(m.backedUpItems), len(m.totalItems) } func (m *backedUpItemsMap) Has(key itemKey) bool { m.RLock() defer m.RUnlock() _, exists := m.backedUpItems[key] return exists } func (m *backedUpItemsMap) AddItem(key itemKey) { m.Lock() defer m.Unlock() m.backedUpItems[key] = struct{}{} m.totalItems[key] = struct{}{} } func (m *backedUpItemsMap) DeleteItem(key itemKey) { m.Lock() defer m.Unlock() delete(m.backedUpItems, key) delete(m.totalItems, key) } func (m *backedUpItemsMap) AddItemToTotal(key itemKey) { m.Lock() defer m.Unlock() m.totalItems[key] = struct{}{} } ================================================ FILE: pkg/backup/backup.go ================================================ /* Copyright the Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backup import ( "archive/tar" "bytes" "compress/gzip" "context" "encoding/json" "fmt" "io" "os" "path/filepath" "sync" "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" kubeerrs "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/wait" kbclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/internal/hook" "github.com/vmware-tanzu/velero/internal/volume" "github.com/vmware-tanzu/velero/internal/volumehelper" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/discovery" "github.com/vmware-tanzu/velero/pkg/itemblock" "github.com/vmware-tanzu/velero/pkg/itemoperation" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/persistence" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" "github.com/vmware-tanzu/velero/pkg/plugin/framework" "github.com/vmware-tanzu/velero/pkg/plugin/velero" biav2 "github.com/vmware-tanzu/velero/pkg/plugin/velero/backupitemaction/v2" ibav1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/itemblockaction/v1" vsv1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/volumesnapshotter/v1" "github.com/vmware-tanzu/velero/pkg/podexec" "github.com/vmware-tanzu/velero/pkg/podvolume" "github.com/vmware-tanzu/velero/pkg/util/boolptr" "github.com/vmware-tanzu/velero/pkg/util/collections" "github.com/vmware-tanzu/velero/pkg/util/kube" ) // BackupVersion is the current backup major version for Velero. // Deprecated, use BackupFormatVersion const BackupVersion = 1 // BackupFormatVersion is the current backup version for Velero, including major, minor, and patch. const BackupFormatVersion = "1.1.0" // ArgoCD managed by namespace label key const ArgoCDManagedByNamespaceLabel = "argocd.argoproj.io/managed-by" // Backupper performs backups. type Backupper interface { // Backup takes a backup using the specification in the velerov1api.Backup and writes backup and log data // to the given writers. Backup( logger logrus.FieldLogger, backup *Request, backupFile io.Writer, actions []biav2.BackupItemAction, itemBlockActions []ibav1.ItemBlockAction, volumeSnapshotterGetter VolumeSnapshotterGetter, ) error BackupWithResolvers( log logrus.FieldLogger, backupRequest *Request, backupFile io.Writer, backupItemActionResolver framework.BackupItemActionResolverV2, itemBlockActionResolver framework.ItemBlockActionResolver, volumeSnapshotterGetter VolumeSnapshotterGetter, ) error FinalizeBackup( log logrus.FieldLogger, backupRequest *Request, inBackupFile io.Reader, outBackupFile io.Writer, backupItemActionResolver framework.BackupItemActionResolverV2, asyncBIAOperations []*itemoperation.BackupOperation, backupStore persistence.BackupStore, ) error } // kubernetesBackupper implements Backupper. type kubernetesBackupper struct { kbClient kbclient.Client dynamicFactory client.DynamicFactory discoveryHelper discovery.Helper podCommandExecutor podexec.PodCommandExecutor podVolumeBackupperFactory podvolume.BackupperFactory podVolumeTimeout time.Duration defaultVolumesToFsBackup bool clientPageSize int uploaderType string pluginManager func(logrus.FieldLogger) clientmgmt.Manager backupStoreGetter persistence.ObjectBackupStoreGetter } func (i *itemKey) String() string { return fmt.Sprintf("resource=%s,namespace=%s,name=%s", i.resource, i.namespace, i.name) } func cohabitatingResources() map[string]*cohabitatingResource { return map[string]*cohabitatingResource{ "deployments": newCohabitatingResource("deployments", "extensions", "apps"), "daemonsets": newCohabitatingResource("daemonsets", "extensions", "apps"), "replicasets": newCohabitatingResource("replicasets", "extensions", "apps"), "networkpolicies": newCohabitatingResource("networkpolicies", "extensions", "networking.k8s.io"), "events": newCohabitatingResource("events", "", "events.k8s.io"), } } // NewKubernetesBackupper creates a new kubernetesBackupper. func NewKubernetesBackupper( kbClient kbclient.Client, discoveryHelper discovery.Helper, dynamicFactory client.DynamicFactory, podCommandExecutor podexec.PodCommandExecutor, podVolumeBackupperFactory podvolume.BackupperFactory, podVolumeTimeout time.Duration, defaultVolumesToFsBackup bool, clientPageSize int, uploaderType string, pluginManager func(logrus.FieldLogger) clientmgmt.Manager, backupStoreGetter persistence.ObjectBackupStoreGetter, ) (Backupper, error) { return &kubernetesBackupper{ kbClient: kbClient, discoveryHelper: discoveryHelper, dynamicFactory: dynamicFactory, podCommandExecutor: podCommandExecutor, podVolumeBackupperFactory: podVolumeBackupperFactory, podVolumeTimeout: podVolumeTimeout, defaultVolumesToFsBackup: defaultVolumesToFsBackup, clientPageSize: clientPageSize, uploaderType: uploaderType, pluginManager: pluginManager, backupStoreGetter: backupStoreGetter, }, nil } // getNamespaceIncludesExcludesAndArgoCDNamespaces returns an IncludesExcludes list containing which namespaces to // include and exclude from the backup and a list of namespaces managed by ArgoCD. func getNamespaceIncludesExcludesAndArgoCDNamespaces(backup *velerov1api.Backup, kbClient kbclient.Client) (*collections.NamespaceIncludesExcludes, []string, error) { nsList := corev1api.NamespaceList{} activeNamespaces := []string{} nsManagedByArgoCD := []string{} if err := kbClient.List(context.Background(), &nsList); err != nil { return nil, nsManagedByArgoCD, err } for _, ns := range nsList.Items { activeNamespaces = append(activeNamespaces, ns.Name) } // Set ActiveNamespaces first, then set includes/excludes includesExcludes := collections.NewNamespaceIncludesExcludes(). ActiveNamespaces(activeNamespaces). Includes(backup.Spec.IncludedNamespaces...). Excludes(backup.Spec.ExcludedNamespaces...) // Expand wildcards if needed if err := includesExcludes.ExpandIncludesExcludes(); err != nil { return nil, []string{}, err } // Check for ArgoCD managed namespaces in the namespaces that will be included for _, ns := range nsList.Items { nsLabels := ns.GetLabels() if len(nsLabels[ArgoCDManagedByNamespaceLabel]) > 0 && includesExcludes.ShouldInclude(ns.Name) { nsManagedByArgoCD = append(nsManagedByArgoCD, ns.Name) } } return includesExcludes, nsManagedByArgoCD, nil } func getResourceHooks(hookSpecs []velerov1api.BackupResourceHookSpec, discoveryHelper discovery.Helper) ([]hook.ResourceHook, error) { resourceHooks := make([]hook.ResourceHook, 0, len(hookSpecs)) for _, s := range hookSpecs { h, err := getResourceHook(s, discoveryHelper) if err != nil { return []hook.ResourceHook{}, err } resourceHooks = append(resourceHooks, h) } return resourceHooks, nil } func getResourceHook(hookSpec velerov1api.BackupResourceHookSpec, discoveryHelper discovery.Helper) (hook.ResourceHook, error) { h := hook.ResourceHook{ Name: hookSpec.Name, Selector: hook.ResourceHookSelector{ Namespaces: collections.NewIncludesExcludes().Includes(hookSpec.IncludedNamespaces...).Excludes(hookSpec.ExcludedNamespaces...), Resources: collections.GetResourceIncludesExcludes(discoveryHelper, hookSpec.IncludedResources, hookSpec.ExcludedResources), }, Pre: hookSpec.PreHooks, Post: hookSpec.PostHooks, } if hookSpec.LabelSelector != nil { labelSelector, err := metav1.LabelSelectorAsSelector(hookSpec.LabelSelector) if err != nil { return hook.ResourceHook{}, errors.WithStack(err) } h.Selector.LabelSelector = labelSelector } return h, nil } type VolumeSnapshotterGetter interface { GetVolumeSnapshotter(name string) (vsv1.VolumeSnapshotter, error) } // Backup backs up the items specified in the Backup, placing them in a gzip-compressed tar file // written to backupFile. The finalized velerov1api.Backup is written to metadata. Any error that represents // a complete backup failure is returned. Errors that constitute partial failures (i.e. failures to // back up individual resources that don't prevent the backup from continuing to be processed) are logged // to the backup log. func (kb *kubernetesBackupper) Backup(log logrus.FieldLogger, backupRequest *Request, backupFile io.Writer, actions []biav2.BackupItemAction, itemBlockActions []ibav1.ItemBlockAction, volumeSnapshotterGetter VolumeSnapshotterGetter) error { backupItemActions := framework.NewBackupItemActionResolverV2(actions) itemBlockActionResolver := framework.NewItemBlockActionResolver(itemBlockActions) return kb.BackupWithResolvers(log, backupRequest, backupFile, backupItemActions, itemBlockActionResolver, volumeSnapshotterGetter) } func (kb *kubernetesBackupper) BackupWithResolvers( log logrus.FieldLogger, backupRequest *Request, backupFile io.Writer, backupItemActionResolver framework.BackupItemActionResolverV2, itemBlockActionResolver framework.ItemBlockActionResolver, volumeSnapshotterGetter VolumeSnapshotterGetter, ) error { gzippedData := gzip.NewWriter(backupFile) defer gzippedData.Close() tw := NewTarWriter(tar.NewWriter(gzippedData)) defer tw.Close() log.Info("Writing backup version file") if err := kb.writeBackupVersion(tw); err != nil { return errors.WithStack(err) } var err error var nsManagedByArgoCD []string backupRequest.NamespaceIncludesExcludes, nsManagedByArgoCD, err = getNamespaceIncludesExcludesAndArgoCDNamespaces(backupRequest.Backup, kb.kbClient) if err != nil { log.WithError(err).Errorf("error getting namespace includes/excludes") return err } if backupRequest.NamespaceIncludesExcludes.IsWildcardExpanded() { expandedIncludes := backupRequest.NamespaceIncludesExcludes.GetIncludes() expandedExcludes := backupRequest.NamespaceIncludesExcludes.GetExcludes() // Get the final namespace list after wildcard expansion wildcardResult, err := backupRequest.NamespaceIncludesExcludes.ResolveNamespaceList() if err != nil { log.WithError(err).Errorf("error resolving namespace list") return err } log.WithFields(logrus.Fields{ "expandedIncludes": expandedIncludes, "expandedExcludes": expandedExcludes, "wildcardResult": wildcardResult, "includedCount": len(expandedIncludes), "excludedCount": len(expandedExcludes), "resultCount": len(wildcardResult), }).Info("Successfully expanded wildcard patterns") } log.Infof("Including namespaces: %s", backupRequest.NamespaceIncludesExcludes.IncludesString()) log.Infof("Excluding namespaces: %s", backupRequest.NamespaceIncludesExcludes.ExcludesString()) // check if there are any namespaces included in the backup which are managed by argoCD // We will check for the existence of a ArgoCD label in the includedNamespaces and add a warning // so that users are at least aware about the existence of argoCD managed ns in their backup // Related Issue: https://github.com/vmware-tanzu/velero/issues/7905 if len(nsManagedByArgoCD) > 0 { log.Warnf("backup operation may encounter complications and potentially produce undesirable results due to the inclusion of namespaces %v managed by ArgoCD in the backup.", nsManagedByArgoCD) } if collections.UseOldResourceFilters(backupRequest.Spec) { backupRequest.ResourceIncludesExcludes = collections.GetGlobalResourceIncludesExcludes(kb.discoveryHelper, log, backupRequest.Spec.IncludedResources, backupRequest.Spec.ExcludedResources, backupRequest.Spec.IncludeClusterResources, *backupRequest.NamespaceIncludesExcludes) } else { srie := collections.GetScopeResourceIncludesExcludes(kb.discoveryHelper, log, backupRequest.Spec.IncludedNamespaceScopedResources, backupRequest.Spec.ExcludedNamespaceScopedResources, backupRequest.Spec.IncludedClusterScopedResources, backupRequest.Spec.ExcludedClusterScopedResources, *backupRequest.NamespaceIncludesExcludes, ) if backupRequest.ResPolicies != nil { srie.CombineWithPolicy(backupRequest.ResPolicies.GetIncludeExcludePolicy()) } backupRequest.ResourceIncludesExcludes = srie } log.Infof("Backing up all volumes using pod volume backup: %t", boolptr.IsSetToTrue(backupRequest.Backup.Spec.DefaultVolumesToFsBackup)) backupRequest.ResourceHooks, err = getResourceHooks(backupRequest.Spec.Hooks.Resources, kb.discoveryHelper) if err != nil { log.WithError(errors.WithStack(err)).Debugf("Error from getResourceHooks") return err } backupRequest.ResolvedActions, err = backupItemActionResolver.ResolveActions(kb.discoveryHelper, log) if err != nil { log.WithError(errors.WithStack(err)).Debugf("Error from backupItemActionResolver.ResolveActions") return err } backupRequest.ResolvedItemBlockActions, err = itemBlockActionResolver.ResolveActions(kb.discoveryHelper, log) if err != nil { log.WithError(errors.WithStack(err)).Errorf("Error from itemBlockActionResolver.ResolveActions") return err } podVolumeTimeout := kb.podVolumeTimeout if val := backupRequest.Annotations[velerov1api.PodVolumeOperationTimeoutAnnotation]; val != "" { parsed, err := time.ParseDuration(val) if err != nil { log.WithError(errors.WithStack(err)).Errorf("Unable to parse pod volume timeout annotation %s, using server value.", val) } else { podVolumeTimeout = parsed } } var podVolumeCancelFunc context.CancelFunc podVolumeContext, podVolumeCancelFunc := context.WithTimeout(context.Background(), podVolumeTimeout) defer podVolumeCancelFunc() var podVolumeBackupper podvolume.Backupper if kb.podVolumeBackupperFactory != nil { podVolumeBackupper, err = kb.podVolumeBackupperFactory.NewBackupper(podVolumeContext, log, backupRequest.Backup, kb.uploaderType) if err != nil { log.WithError(errors.WithStack(err)).Debugf("Error from NewBackupper") return errors.WithStack(err) } } // set up a temp dir for the itemCollector to use to temporarily // store items as they're scraped from the API. tempDir, err := os.MkdirTemp("", "") if err != nil { return errors.Wrap(err, "error creating temp dir for backup") } defer os.RemoveAll(tempDir) collector := &itemCollector{ log: log, backupRequest: backupRequest, discoveryHelper: kb.discoveryHelper, dynamicFactory: kb.dynamicFactory, cohabitatingResources: cohabitatingResources(), dir: tempDir, pageSize: kb.clientPageSize, } items := collector.getAllItems() log.WithField("progress", "").Infof("Collected %d items matching the backup spec from the Kubernetes API (actual number of items backed up may be more or less depending on velero.io/exclude-from-backup annotation, plugins returning additional related items to back up, etc.)", len(items)) updated := backupRequest.Backup.DeepCopy() if updated.Status.Progress == nil { updated.Status.Progress = &velerov1api.BackupProgress{} } updated.Status.Progress.TotalItems = len(items) if err := kube.PatchResource(backupRequest.Backup, updated, kb.kbClient); err != nil { log.WithError(errors.WithStack((err))).Warn("Got error trying to update backup's status.progress.totalItems") } backupRequest.Status.Progress = &velerov1api.BackupProgress{TotalItems: len(items)} // Resolve namespaces for PVC-to-Pod cache building in volumehelper. // See issue #9179 for details. namespaces, err := backupRequest.NamespaceIncludesExcludes.ResolveNamespaceList() if err != nil { log.WithError(err).Error("Failed to resolve namespace list for PVC-to-Pod cache") return err } volumeHelperImpl, err := volumehelper.NewVolumeHelperImplWithNamespaces( backupRequest.ResPolicies, backupRequest.Spec.SnapshotVolumes, log, kb.kbClient, boolptr.IsSetToTrue(backupRequest.Spec.DefaultVolumesToFsBackup), !backupRequest.ResourceIncludesExcludes.ShouldInclude(kuberesource.PersistentVolumeClaims.String()), namespaces, ) if err != nil { log.WithError(err).Error("Failed to build PVC-to-Pod cache for volume policy lookups") return err } itemBackupper := &itemBackupper{ backupRequest: backupRequest, tarWriter: tw, dynamicFactory: kb.dynamicFactory, kbClient: kb.kbClient, discoveryHelper: kb.discoveryHelper, podVolumeBackupper: podVolumeBackupper, podVolumeContext: podVolumeContext, podVolumeSnapshotTracker: podvolume.NewTracker(), volumeSnapshotterCache: NewVolumeSnapshotterCache(volumeSnapshotterGetter), itemHookHandler: &hook.DefaultItemHookHandler{ PodCommandExecutor: kb.podCommandExecutor, }, hookTracker: hook.NewHookTracker(), volumeHelperImpl: volumeHelperImpl, kubernetesBackupper: kb, } // helper struct to send current progress between the main // backup loop and the gouroutine that periodically patches // the backup CR with progress updates type progressUpdate struct { totalItems, itemsBackedUp int } // the main backup process will send on this channel once // for every item it processes. update := make(chan progressUpdate) // the main backup process will send on this channel when // it's done sending progress updates quit := make(chan struct{}) // This is the progress updater goroutine that receives // progress updates on the 'update' channel. It patches // the backup CR with progress updates at most every second, // but it will not issue a patch if it hasn't received a new // update since the previous patch. This goroutine exits // when it receives on the 'quit' channel. go func() { ticker := time.NewTicker(1 * time.Second) var lastUpdate *progressUpdate for { select { case <-quit: ticker.Stop() return case val := <-update: lastUpdate = &val case <-ticker.C: if lastUpdate != nil { updated := backupRequest.Backup.DeepCopy() if updated.Status.Progress == nil { updated.Status.Progress = &velerov1api.BackupProgress{} } updated.Status.Progress.TotalItems = lastUpdate.totalItems updated.Status.Progress.ItemsBackedUp = lastUpdate.itemsBackedUp if err := kube.PatchResource(backupRequest.Backup, updated, kb.kbClient); err != nil { log.WithError(errors.WithStack((err))).Warn("Got error trying to update backup's status.progress") } backupRequest.Status.Progress = &velerov1api.BackupProgress{TotalItems: lastUpdate.totalItems, ItemsBackedUp: lastUpdate.itemsBackedUp} lastUpdate = nil } } } }() responseCtx, responseCancel := context.WithCancel(context.Background()) backedUpGroupResources := map[schema.GroupResource]bool{} // Maps items in the item list from GR+NamespacedName to a slice of pointers to kubernetesResources // We need the slice value since if the EnableAPIGroupVersions feature flag is set, there may // be more than one resource to back up for the given item. itemsMap := make(map[velero.ResourceIdentifier][]*kubernetesResource) for i := range items { key := velero.ResourceIdentifier{ GroupResource: items[i].groupResource, Namespace: items[i].namespace, Name: items[i].name, } itemsMap[key] = append(itemsMap[key], items[i]) // add to total items for progress reporting if items[i].kind != "" { backupRequest.BackedUpItems.AddItemToTotal(itemKey{ resource: fmt.Sprintf("%s/%s", items[i].preferredGVR.GroupVersion().String(), items[i].kind), namespace: items[i].namespace, name: items[i].name, }) } } var itemBlock *BackupItemBlock itemBlockReturn := make(chan ItemBlockReturn, 100) wg := &sync.WaitGroup{} // Handle returns from worker pool processing ItemBlocks go func() { for { select { case response := <-itemBlockReturn: // process each BackupItemBlock response func() { defer wg.Done() if response.err != nil { log.WithError(errors.WithStack((response.err))).Error("Got error in BackupItemBlock.") } for _, backedUpGR := range response.resources { backedUpGroupResources[backedUpGR] = true } // We could eventually track which itemBlocks have finished // using response.itemBlock // updated total is computed as "how many items we've backed up so far, // plus how many items are processed but not yet backed up plus how many // we know of that are remaining to be processed" backedUpItems, totalItems := backupRequest.BackedUpItems.BackedUpAndTotalLen() // send a progress update update <- progressUpdate{ totalItems: totalItems, itemsBackedUp: backedUpItems, } if len(response.itemBlock.Items) > 0 { log.WithFields(map[string]any{ "progress": "", "kind": response.itemBlock.Items[0].Item.GroupVersionKind().GroupKind().String(), "namespace": response.itemBlock.Items[0].Item.GetNamespace(), "name": response.itemBlock.Items[0].Item.GetName(), }).Infof("Backed up %d items out of an estimated total of %d (estimate will change throughout the backup)", backedUpItems, totalItems) } }() case <-responseCtx.Done(): return } } }() for i := range items { log.WithFields(map[string]any{ "progress": "", "resource": items[i].groupResource.String(), "namespace": items[i].namespace, "name": items[i].name, }).Infof("Processing item") // Skip if this item has already been processed (in a block or previously excluded) if items[i].inItemBlockOrExcluded { log.Debugf("Not creating new ItemBlock for %s %s/%s because it's already in an ItemBlock", items[i].groupResource.String(), items[i].namespace, items[i].name) } else { if itemBlock == nil { itemBlock = NewBackupItemBlock(log, itemBackupper) } var newBlockItem *unstructured.Unstructured // If the EnableAPIGroupVersions feature flag is set, there could be multiple versions // of this item to be backed up. Include all of them in the same ItemBlock key := velero.ResourceIdentifier{ GroupResource: items[i].groupResource, Namespace: items[i].namespace, Name: items[i].name, } allVersionsOfItem := itemsMap[key] for _, itemVersion := range allVersionsOfItem { unstructured := itemBlock.addKubernetesResource(itemVersion, log) if newBlockItem == nil { newBlockItem = unstructured } } // call GetRelatedItems, add found items to block if not in block, recursively until no more items if newBlockItem != nil { kb.executeItemBlockActions(log, newBlockItem, items[i].groupResource, items[i].name, items[i].namespace, itemsMap, itemBlock) } } // We skip calling backupItemBlock here so that we will add the next item to the current ItemBlock if: // 1) This is not the last item to be processed // 2) Both current and next item are ordered resources // 3) Both current and next item are for the same GroupResource addNextToBlock := i < len(items)-1 && items[i].orderedResource && items[i+1].orderedResource && items[i].groupResource == items[i+1].groupResource if itemBlock != nil && len(itemBlock.Items) > 0 && !addNextToBlock { log.Infof("Backing Up Item Block including %s %s/%s (%v items in block)", items[i].groupResource.String(), items[i].namespace, items[i].name, len(itemBlock.Items)) wg.Add(1) backupRequest.WorkerPool.GetInputChannel() <- ItemBlockInput{ itemBlock: itemBlock, returnChan: itemBlockReturn, } itemBlock = nil } } done := make(chan struct{}) go func() { defer close(done) wg.Wait() }() // Wait for all the ItemBlocks to be processed select { case <-done: log.Info("done processing ItemBlocks") case <-responseCtx.Done(): log.Info("ItemBlock processing canceled") } // cancel response-processing goroutine responseCancel() // no more progress updates will be sent on the 'update' channel quit <- struct{}{} // back up CRD(this is a CRD definition of the resource, it's a CRD instance) for resource if found. // We should only need to do this if we've backed up at least one item for the resource // and the CRD type(this is the CRD type itself) is neither included or excluded. // When it's included, the resource's CRD is already handled. When it's excluded, no need to check. if !backupRequest.ResourceIncludesExcludes.ShouldExclude(kuberesource.CustomResourceDefinitions.String()) && !backupRequest.ResourceIncludesExcludes.ShouldInclude(kuberesource.CustomResourceDefinitions.String()) { for gr := range backedUpGroupResources { kb.backupCRD(log, gr, itemBackupper) } } processedPVBs := itemBackupper.podVolumeBackupper.WaitAllPodVolumesProcessed(log) backupRequest.PodVolumeBackups = append(backupRequest.PodVolumeBackups, processedPVBs...) // do a final update on progress since we may have just added some CRDs and may not have updated // for the last few processed items. updated = backupRequest.Backup.DeepCopy() if updated.Status.Progress == nil { updated.Status.Progress = &velerov1api.BackupProgress{} } backedUpItems := backupRequest.BackedUpItems.Len() updated.Status.Progress.TotalItems = backedUpItems updated.Status.Progress.ItemsBackedUp = backedUpItems // update the hooks execution status if updated.Status.HookStatus == nil { updated.Status.HookStatus = &velerov1api.HookStatus{} } updated.Status.HookStatus.HooksAttempted, updated.Status.HookStatus.HooksFailed = itemBackupper.hookTracker.Stat() log.Debugf("hookAttempted: %d, hookFailed: %d", updated.Status.HookStatus.HooksAttempted, updated.Status.HookStatus.HooksFailed) if err := kube.PatchResource(backupRequest.Backup, updated, kb.kbClient); err != nil { log.WithError(errors.WithStack((err))).Warn("Got error trying to update backup's status.progress and hook status") } if skippedPVSummary, err := json.Marshal(backupRequest.SkippedPVTracker.Summary()); err != nil { log.WithError(errors.WithStack(err)).Warn("Fail to generate skipped PV summary.") } else { log.Infof("Summary for skipped PVs: %s", skippedPVSummary) } backupRequest.Status.Progress = &velerov1api.BackupProgress{TotalItems: backedUpItems, ItemsBackedUp: backedUpItems} log.WithField("progress", "").Infof("Backed up a total of %d items", backedUpItems) return nil } func (kb *kubernetesBackupper) executeItemBlockActions( log logrus.FieldLogger, obj runtime.Unstructured, groupResource schema.GroupResource, name, namespace string, itemsMap map[velero.ResourceIdentifier][]*kubernetesResource, itemBlock *BackupItemBlock, ) { metadata, err := meta.Accessor(obj) if err != nil { log.WithError(errors.WithStack(err)).Warn("Failed to get object metadata.") return } for _, action := range itemBlock.itemBackupper.backupRequest.ResolvedItemBlockActions { if !action.ShouldUse(groupResource, namespace, metadata, log) { continue } log.Info("Executing ItemBlock action") relatedItems, err := action.GetRelatedItems(obj, itemBlock.itemBackupper.backupRequest.Backup) if err != nil { log.Error(errors.Wrapf(err, "error executing ItemBlock action (groupResource=%s, namespace=%s, name=%s)", groupResource.String(), namespace, name)) continue } for _, relatedItem := range relatedItems { var newBlockItem *unstructured.Unstructured // Look for item in itemsMap itemsToAdd := itemsMap[relatedItem] // if item is in the item collector list, we'll have at least one element. // If EnableAPIGroupVersions is set, we may have more than one. // If we get an unstructured obj back from addKubernetesResource, then it wasn't // already in a block and we recursively look for related items in the returned item. if len(itemsToAdd) > 0 { for _, itemToAdd := range itemsToAdd { unstructured := itemBlock.addKubernetesResource(itemToAdd, log) if newBlockItem == nil { newBlockItem = unstructured } } if newBlockItem != nil { kb.executeItemBlockActions(log, newBlockItem, relatedItem.GroupResource, relatedItem.Name, relatedItem.Namespace, itemsMap, itemBlock) } continue } // Item wasn't found in item collector list, get from cluster gvr, resource, err := itemBlock.itemBackupper.discoveryHelper.ResourceFor(relatedItem.GroupResource.WithVersion("")) if err != nil { log.Error(errors.Wrapf(err, "Unable to obtain gvr and resource for related item %s %s/%s", relatedItem.GroupResource.String(), relatedItem.Namespace, relatedItem.Name)) continue } client, err := itemBlock.itemBackupper.dynamicFactory.ClientForGroupVersionResource(gvr.GroupVersion(), resource, relatedItem.Namespace) if err != nil { log.Error(errors.Wrapf(err, "Unable to obtain client for gvr %s %s (%s)", gvr.GroupVersion(), resource.Name, relatedItem.Namespace)) continue } item, err := client.Get(relatedItem.Name, metav1.GetOptions{}) if apierrors.IsNotFound(err) { log.WithFields(logrus.Fields{ "groupResource": relatedItem.GroupResource, "namespace": relatedItem.Namespace, "name": relatedItem.Name, }).Warnf("Related item was not found in Kubernetes API, can't add to item block") continue } if err != nil { log.Error(errors.Wrapf(err, "Error while trying to get related item %s %s/%s from cluster", relatedItem.GroupResource.String(), relatedItem.Namespace, relatedItem.Name)) continue } itemsMap[relatedItem] = append(itemsMap[relatedItem], &kubernetesResource{ groupResource: relatedItem.GroupResource, preferredGVR: gvr, namespace: relatedItem.Namespace, name: relatedItem.Name, inItemBlockOrExcluded: true, }) relatedItemMetadata, err := meta.Accessor(item) if err != nil { log.WithError(errors.WithStack(err)).Warn("Failed to get object metadata.") continue } // Don't add to ItemBlock if item is excluded // itemInclusionChecks logs the reason if !itemBlock.itemBackupper.itemInclusionChecks(log, false, relatedItemMetadata, item, relatedItem.GroupResource) { continue } log.Infof("adding %s %s/%s to ItemBlock", relatedItem.GroupResource, relatedItem.Namespace, relatedItem.Name) itemBlock.AddUnstructured(relatedItem.GroupResource, item, gvr) kb.executeItemBlockActions(log, item, relatedItem.GroupResource, relatedItem.Name, relatedItem.Namespace, itemsMap, itemBlock) } } } func (kb *kubernetesBackupper) backupItemBlock(itemBlock *BackupItemBlock) []schema.GroupResource { // find pods in ItemBlock // filter pods based on whether they still need to be backed up // this list will be used to run pre/post hooks var preHookPods []itemblock.ItemBlockItem itemBlock.Log.Debug("Executing pre hooks") for _, item := range itemBlock.Items { if item.Gr == kuberesource.Pods { key, err := kb.getItemKey(item) if err != nil { itemBlock.Log.WithError(errors.WithStack(err)).Error("Error accessing pod metadata") continue } // Don't run hooks if pod has already been backed up if !itemBlock.itemBackupper.backupRequest.BackedUpItems.Has(key) { preHookPods = append(preHookPods, item) } } } postHookPods, failedPods, errs := kb.handleItemBlockPreHooks(itemBlock, preHookPods) for i, pod := range failedPods { itemBlock.Log.WithError(errs[i]).WithField("name", pod.Item.GetName()).Error("Error running pre hooks for pod") // if pre hook fails, flag pod as backed-up and move on key, err := kb.getItemKey(pod) if err != nil { itemBlock.Log.WithError(errors.WithStack(err)).Error("Error accessing pod metadata") continue } itemBlock.itemBackupper.backupRequest.BackedUpItems.AddItem(key) } itemBlock.Log.Debug("Backing up items in BackupItemBlock") var grList []schema.GroupResource for _, item := range itemBlock.Items { if backedUp := kb.backupItem(itemBlock.Log, item.Gr, itemBlock.itemBackupper, item.Item, item.PreferredGVR, itemBlock); backedUp { grList = append(grList, item.Gr) } } if len(postHookPods) > 0 { itemBlock.Log.Debug("Executing post hooks") kb.handleItemBlockPostHooks(itemBlock, postHookPods) } return grList } func (kb *kubernetesBackupper) getItemKey(item itemblock.ItemBlockItem) (itemKey, error) { metadata, err := meta.Accessor(item.Item) if err != nil { return itemKey{}, err } key := itemKey{ resource: resourceKey(item.Item), namespace: metadata.GetNamespace(), name: metadata.GetName(), } return key, nil } func (kb *kubernetesBackupper) handleItemBlockPreHooks(itemBlock *BackupItemBlock, hookPods []itemblock.ItemBlockItem) ([]itemblock.ItemBlockItem, []itemblock.ItemBlockItem, []error) { var successPods []itemblock.ItemBlockItem var failedPods []itemblock.ItemBlockItem var errs []error for _, pod := range hookPods { err := itemBlock.itemBackupper.itemHookHandler.HandleHooks(itemBlock.Log, pod.Gr, pod.Item, itemBlock.itemBackupper.backupRequest.ResourceHooks, hook.PhasePre, itemBlock.itemBackupper.hookTracker) if err == nil { successPods = append(successPods, pod) } else { failedPods = append(failedPods, pod) errs = append(errs, err) } } return successPods, failedPods, errs } // The hooks cannot execute until the PVBs to be processed func (kb *kubernetesBackupper) handleItemBlockPostHooks(itemBlock *BackupItemBlock, hookPods []itemblock.ItemBlockItem) { log := itemBlock.Log // the post hooks will not execute until all PVBs of the item block pods are processed if err := kb.waitUntilPVBsProcessed(itemBlock.itemBackupper.podVolumeContext, log, itemBlock, hookPods); err != nil { log.WithError(err).Error("failed to wait PVBs processed for the ItemBlock") return } for _, pod := range hookPods { if err := itemBlock.itemBackupper.itemHookHandler.HandleHooks(itemBlock.Log, pod.Gr, pod.Item, itemBlock.itemBackupper.backupRequest.ResourceHooks, hook.PhasePost, itemBlock.itemBackupper.hookTracker); err != nil { log.WithError(err).WithField("name", pod.Item.GetName()).Error("Error running post hooks for pod") } } } // wait all PVBs of the item block pods to be processed func (kb *kubernetesBackupper) waitUntilPVBsProcessed(ctx context.Context, log logrus.FieldLogger, itemBlock *BackupItemBlock, pods []itemblock.ItemBlockItem) error { pvbMap := map[*velerov1api.PodVolumeBackup]bool{} for _, pod := range pods { namespace, name := pod.Item.GetNamespace(), pod.Item.GetName() pvbs, err := itemBlock.itemBackupper.podVolumeBackupper.ListPodVolumeBackupsByPod(namespace, name) if err != nil { return errors.Wrapf(err, "failed to list PodVolumeBackups for pod %s/%s", namespace, name) } for _, pvb := range pvbs { pvbMap[pvb] = pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseCompleted || pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseFailed || pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseCanceled } } checkFunc := func(context.Context) (done bool, err error) { allProcessed := true for pvb, processed := range pvbMap { if processed { continue } updatedPVB, err := itemBlock.itemBackupper.podVolumeBackupper.GetPodVolumeBackupByPodAndVolume(pvb.Spec.Pod.Namespace, pvb.Spec.Pod.Name, pvb.Spec.Volume) if err != nil { allProcessed = false log.Infof("failed to get PVB: %v", err) continue } if updatedPVB.Status.Phase == velerov1api.PodVolumeBackupPhaseCompleted || updatedPVB.Status.Phase == velerov1api.PodVolumeBackupPhaseFailed || updatedPVB.Status.Phase == velerov1api.PodVolumeBackupPhaseCanceled { pvbMap[pvb] = true continue } allProcessed = false } return allProcessed, nil } return wait.PollUntilContextCancel(ctx, 5*time.Second, true, checkFunc) } func (kb *kubernetesBackupper) backupItem(log logrus.FieldLogger, gr schema.GroupResource, itemBackupper *itemBackupper, unstructured *unstructured.Unstructured, preferredGVR schema.GroupVersionResource, itemBlock *BackupItemBlock) bool { backedUpItem, _, err := itemBackupper.backupItem(log, unstructured, gr, preferredGVR, false, false, itemBlock) if aggregate, ok := err.(kubeerrs.Aggregate); ok { log.WithField("name", unstructured.GetName()).Infof("%d errors encountered backup up item", len(aggregate.Errors())) // log each error separately so we get error location info in the log, and an // accurate count of errors for _, err = range aggregate.Errors() { log.WithError(err).WithField("name", unstructured.GetName()).Error("Error backing up item") } return false } if err != nil { log.WithError(err).WithField("name", unstructured.GetName()).Error("Error backing up item") return false } return backedUpItem } func (kb *kubernetesBackupper) finalizeItem( log logrus.FieldLogger, gr schema.GroupResource, itemBackupper *itemBackupper, unstructured *unstructured.Unstructured, preferredGVR schema.GroupVersionResource, ) (bool, []FileForArchive) { backedUpItem, updateFiles, err := itemBackupper.backupItem(log, unstructured, gr, preferredGVR, true, true, nil) if aggregate, ok := err.(kubeerrs.Aggregate); ok { log.WithField("name", unstructured.GetName()).Infof("%d errors encountered backup up item", len(aggregate.Errors())) // log each error separately so we get error location info in the log, and an // accurate count of errors for _, err = range aggregate.Errors() { log.WithError(err).WithField("name", unstructured.GetName()).Error("Error backing up item") } return false, updateFiles } if err != nil { log.WithError(err).WithField("name", unstructured.GetName()).Error("Error backing up item") return false, updateFiles } return backedUpItem, updateFiles } // backupCRD checks if the resource is a custom resource, and if so, backs up the custom resource definition // associated with it. func (kb *kubernetesBackupper) backupCRD(log logrus.FieldLogger, gr schema.GroupResource, itemBackupper *itemBackupper) { crdGroupResource := kuberesource.CustomResourceDefinitions log.Debugf("Getting server preferred API version for %s", crdGroupResource) gvr, apiResource, err := kb.discoveryHelper.ResourceFor(crdGroupResource.WithVersion("")) if err != nil { log.WithError(errors.WithStack(err)).Errorf("Error getting resolved resource for %s", crdGroupResource) return } log.Debugf("Got server preferred API version %s for %s", gvr.Version, crdGroupResource) log.Debugf("Getting dynamic client for %s", gvr.String()) crdClient, err := kb.dynamicFactory.ClientForGroupVersionResource(gvr.GroupVersion(), apiResource, "") if err != nil { log.WithError(errors.WithStack(err)).Errorf("Error getting dynamic client for %s", crdGroupResource) return } log.Debugf("Got dynamic client for %s", gvr.String()) // try to get a CRD whose name matches the provided GroupResource unstructured, err := crdClient.Get(gr.String(), metav1.GetOptions{}) if apierrors.IsNotFound(err) { // not found: this means the GroupResource provided was not a // custom resource, so there's no CRD to back up. log.Debugf("No CRD found for GroupResource %s", gr.String()) return } if err != nil { log.WithError(errors.WithStack(err)).Errorf("Error getting CRD %s", gr.String()) return } log.Infof("Found associated CRD %s to add to backup", gr.String()) kb.backupItem(log, gvr.GroupResource(), itemBackupper, unstructured, gvr, nil) } func (kb *kubernetesBackupper) writeBackupVersion(tw tarWriter) error { versionFile := filepath.Join(velerov1api.MetadataDir, "version") versionString := fmt.Sprintf("%s\n", BackupFormatVersion) hdr := &tar.Header{ Name: versionFile, Size: int64(len(versionString)), Typeflag: tar.TypeReg, Mode: 0755, ModTime: time.Now(), } if err := tw.WriteHeader(hdr); err != nil { return errors.WithStack(err) } if _, err := tw.Write([]byte(versionString)); err != nil { return errors.WithStack(err) } return nil } func (kb *kubernetesBackupper) FinalizeBackup( log logrus.FieldLogger, backupRequest *Request, inBackupFile io.Reader, outBackupFile io.Writer, backupItemActionResolver framework.BackupItemActionResolverV2, asyncBIAOperations []*itemoperation.BackupOperation, backupStore persistence.BackupStore, ) error { gzw := gzip.NewWriter(outBackupFile) defer gzw.Close() tw := NewTarWriter(tar.NewWriter(gzw)) defer tw.Close() gzr, err := gzip.NewReader(inBackupFile) if err != nil { log.Infof("error creating gzip reader: %v", err) return err } defer gzr.Close() tr := tar.NewReader(gzr) backupRequest.ResolvedActions, err = backupItemActionResolver.ResolveActions(kb.discoveryHelper, log) if err != nil { log.WithError(errors.WithStack(err)).Debugf("Error from backupItemActionResolver.ResolveActions") return err } // set up a temp dir for the itemCollector to use to temporarily // store items as they're scraped from the API. tempDir, err := os.MkdirTemp("", "") if err != nil { return errors.Wrap(err, "error creating temp dir for backup") } defer os.RemoveAll(tempDir) collector := &itemCollector{ log: log, backupRequest: backupRequest, discoveryHelper: kb.discoveryHelper, dynamicFactory: kb.dynamicFactory, cohabitatingResources: cohabitatingResources(), dir: tempDir, pageSize: kb.clientPageSize, } // Get item list from itemoperation.BackupOperation.Spec.PostOperationItems var resourceIDs []velero.ResourceIdentifier for _, operation := range asyncBIAOperations { if len(operation.Spec.PostOperationItems) != 0 { resourceIDs = append(resourceIDs, operation.Spec.PostOperationItems...) } } items := collector.getItemsFromResourceIdentifiers(resourceIDs) log.WithField("progress", "").Infof("Collected %d items from the async BIA operations PostOperationItems list", len(items)) itemBackupper := &itemBackupper{ backupRequest: backupRequest, tarWriter: tw, dynamicFactory: kb.dynamicFactory, kbClient: kb.kbClient, discoveryHelper: kb.discoveryHelper, itemHookHandler: &hook.NoOpItemHookHandler{}, podVolumeSnapshotTracker: podvolume.NewTracker(), hookTracker: hook.NewHookTracker(), kubernetesBackupper: kb, } updateFiles := make(map[string]FileForArchive) backedUpGroupResources := map[schema.GroupResource]bool{} unstructuredDataUploads := make([]unstructured.Unstructured, 0) for i, item := range items { log.WithFields(map[string]any{ "progress": "", "resource": item.groupResource.String(), "namespace": item.namespace, "name": item.name, }).Infof("Processing item") // use an anonymous func so we can defer-close/remove the file // as soon as we're done with it func() { var unstructured unstructured.Unstructured f, err := os.Open(item.path) if err != nil { log.WithError(errors.WithStack(err)).Error("Error opening file containing item") return } defer f.Close() defer os.Remove(f.Name()) if err := json.NewDecoder(f).Decode(&unstructured); err != nil { log.WithError(errors.WithStack(err)).Error("Error decoding JSON from file") return } if item.groupResource == kuberesource.DataUploads { unstructuredDataUploads = append(unstructuredDataUploads, unstructured) } backedUp, itemFiles := kb.finalizeItem(log, item.groupResource, itemBackupper, &unstructured, item.preferredGVR) if backedUp { backedUpGroupResources[item.groupResource] = true for _, itemFile := range itemFiles { updateFiles[itemFile.FilePath] = itemFile } } }() // updated total is computed as "how many items we've backed up so far, plus // how many items we know of that are remaining" backedUpItems := backupRequest.BackedUpItems.Len() totalItems := backedUpItems + (len(items) - (i + 1)) log.WithFields(map[string]any{ "progress": "", "resource": item.groupResource.String(), "namespace": item.namespace, "name": item.name, }).Infof("Updated %d items out of an estimated total of %d (estimate will change throughout the backup finalizer)", backedUpItems, totalItems) } volumeInfos, err := backupStore.GetBackupVolumeInfos(backupRequest.Backup.Name) if err != nil { log.WithError(err).Errorf("fail to get the backup VolumeInfos for backup %s", backupRequest.Name) return err } if err := updateVolumeInfos(volumeInfos, unstructuredDataUploads, asyncBIAOperations, log); err != nil { log.WithError(err).Errorf("fail to update VolumeInfos for backup %s", backupRequest.Name) return err } if err := putVolumeInfos(backupRequest.Name, volumeInfos, backupStore); err != nil { log.WithError(err).Errorf("fail to put the VolumeInfos for backup %s", backupRequest.Name) return err } // write new tar archive replacing files in original with content updateFiles for matches if err := buildFinalTarball(tr, tw, updateFiles); err != nil { log.Errorf("Error building final tarball: %s", err.Error()) return err } log.WithField("progress", "").Infof("Updated a total of %d items", backupRequest.BackedUpItems.Len()) return nil } func buildFinalTarball(tr *tar.Reader, tw tarWriter, updateFiles map[string]FileForArchive) error { tw.Lock() defer tw.Unlock() for { header, err := tr.Next() if err == io.EOF { break } if err != nil { return errors.WithStack(err) } newFile, ok := updateFiles[header.Name] if ok { // add updated file to archive, skip over tr file content if err := tw.WriteHeader(newFile.Header); err != nil { return errors.WithStack(err) } if _, err := tw.Write(newFile.FileBytes); err != nil { return errors.WithStack(err) } delete(updateFiles, header.Name) // skip over file contents from old tarball _, err := io.ReadAll(tr) if err != nil { return errors.WithStack(err) } } else { // Add original content to new tarball, as item wasn't updated oldContents, err := io.ReadAll(tr) if err != nil { return errors.WithStack(err) } if err := tw.WriteHeader(header); err != nil { return errors.WithStack(err) } if _, err := tw.Write(oldContents); err != nil { return errors.WithStack(err) } } } // iterate over any remaining map entries, which represent updated items that // were not in the original backup tarball for _, newFile := range updateFiles { if err := tw.WriteHeader(newFile.Header); err != nil { return errors.WithStack(err) } if _, err := tw.Write(newFile.FileBytes); err != nil { return errors.WithStack(err) } } return nil } type tarWriter struct { *tar.Writer *sync.Mutex } func NewTarWriter(writer *tar.Writer) tarWriter { return tarWriter{ Writer: writer, Mutex: &sync.Mutex{}, } } // updateVolumeInfos update the VolumeInfos according to the AsyncOperations func updateVolumeInfos( volumeInfos []*volume.BackupVolumeInfo, unstructuredItems []unstructured.Unstructured, operations []*itemoperation.BackupOperation, log logrus.FieldLogger, ) error { for _, unstructured := range unstructuredItems { var dataUpload velerov2alpha1.DataUpload err := runtime.DefaultUnstructuredConverter.FromUnstructured(unstructured.UnstructuredContent(), &dataUpload) if err != nil { log.WithError(err).Errorf("fail to convert DataUpload: %s/%s", unstructured.GetNamespace(), unstructured.GetName()) return err } for index := range volumeInfos { if volumeInfos[index].PVCName == dataUpload.Spec.SourcePVC && volumeInfos[index].PVCNamespace == dataUpload.Spec.SourceNamespace && volumeInfos[index].SnapshotDataMovementInfo != nil { if dataUpload.Status.CompletionTimestamp != nil { volumeInfos[index].CompletionTimestamp = dataUpload.Status.CompletionTimestamp } volumeInfos[index].SnapshotDataMovementInfo.SnapshotHandle = dataUpload.Status.SnapshotID volumeInfos[index].SnapshotDataMovementInfo.RetainedSnapshot = dataUpload.Spec.CSISnapshot.VolumeSnapshot volumeInfos[index].SnapshotDataMovementInfo.Size = dataUpload.Status.Progress.TotalBytes volumeInfos[index].SnapshotDataMovementInfo.IncrementalSize = dataUpload.Status.IncrementalBytes volumeInfos[index].SnapshotDataMovementInfo.Phase = dataUpload.Status.Phase if dataUpload.Status.Phase == velerov2alpha1.DataUploadPhaseCompleted { volumeInfos[index].Result = volume.VolumeResultSucceeded } else { volumeInfos[index].Result = volume.VolumeResultFailed } } } } // Update CSI snapshot VolumeInfo's CompletionTimestamp by the operation update time. for volumeIndex := range volumeInfos { if volumeInfos[volumeIndex].BackupMethod == volume.CSISnapshot && volumeInfos[volumeIndex].CSISnapshotInfo != nil { for opIndex := range operations { if volumeInfos[volumeIndex].CSISnapshotInfo.OperationID == operations[opIndex].Spec.OperationID { // The VolumeSnapshot and VolumeSnapshotContent don't have a completion timestamp, // so use the operation.Status.Updated as the alternative. It is not the exact time // when the snapshot turns ready, but the operation controller periodically watch the // VSC and VS status. When the controller finds they reach to the ReadyToUse state, // The operation.Status.Updated is set as the found time. volumeInfos[volumeIndex].CompletionTimestamp = operations[opIndex].Status.Updated // Set Succeeded to true when the operation has no error. if operations[opIndex].Status.Error == "" { volumeInfos[volumeIndex].Result = volume.VolumeResultSucceeded } else { volumeInfos[volumeIndex].Result = volume.VolumeResultFailed } } } } } return nil } func putVolumeInfos( backupName string, volumeInfos []*volume.BackupVolumeInfo, backupStore persistence.BackupStore, ) error { backupVolumeInfoBuf := new(bytes.Buffer) gzw := gzip.NewWriter(backupVolumeInfoBuf) defer gzw.Close() if err := json.NewEncoder(gzw).Encode(volumeInfos); err != nil { return errors.Wrap(err, "error encoding restore results to JSON") } if err := gzw.Close(); err != nil { return errors.Wrap(err, "error closing gzip writer") } return backupStore.PutBackupVolumeInfos(backupName, backupVolumeInfoBuf) } ================================================ FILE: pkg/backup/backup_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backup import ( "archive/tar" "bytes" "compress/gzip" "context" "encoding/json" "fmt" "io" "sort" "strings" "sync" "testing" "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "github.com/vmware-tanzu/velero/internal/resourcepolicies" "github.com/vmware-tanzu/velero/internal/volume" "github.com/vmware-tanzu/velero/pkg/apis/velero/shared" velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/discovery" "github.com/vmware-tanzu/velero/pkg/features" "github.com/vmware-tanzu/velero/pkg/itemoperation" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/persistence" persistencemocks "github.com/vmware-tanzu/velero/pkg/persistence/mocks" "github.com/vmware-tanzu/velero/pkg/plugin/velero" biav2 "github.com/vmware-tanzu/velero/pkg/plugin/velero/backupitemaction/v2" ibav1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/itemblockaction/v1" vsv1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/volumesnapshotter/v1" "github.com/vmware-tanzu/velero/pkg/podvolume" "github.com/vmware-tanzu/velero/pkg/test" kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube" ) func TestBackedUpItemsMatchesTarballContents(t *testing.T) { // TODO: figure out if this can be replaced with the restmapper // (https://github.com/kubernetes/apimachinery/blob/035e418f1ad9b6da47c4e01906a0cfe32f4ee2e7/pkg/api/meta/restmapper.go) gvkToResource := map[string]string{ "v1/Pod": "pods", "apps/v1/Deployment": "deployments.apps", "v1/PersistentVolume": "persistentvolumes", } h := newHarness(t, nil) defer h.itemBlockPool.Stop() req := &Request{ Backup: defaultBackup().Result(), SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), WorkerPool: &h.itemBlockPool, } backupFile := bytes.NewBuffer([]byte{}) apiResources := []*test.APIResource{ test.Pods( builder.ForPod("foo", "bar").Result(), builder.ForPod("zoo", "raz").Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), builder.ForDeployment("zoo", "raz").Result(), ), test.PVs( builder.ForPersistentVolume("bar").ClaimRef("foo", "pvc1").Result(), builder.ForPersistentVolume("baz").ClaimRef("bar", "pvc2").Result(), ), } for _, resource := range apiResources { h.addItems(t, resource) } h.backupper.Backup(h.log, req, backupFile, nil, nil, nil) // go through BackedUpItems after the backup to assemble the list of files we // expect to see in the tarball and compare to see if they match var expectedFiles []string for item := range req.BackedUpItems.CopyItemMap() { file := "resources/" + gvkToResource[item.resource] if item.namespace != "" { file = file + "/namespaces/" + item.namespace } else { file = file + "/cluster" } file = file + "/" + item.name + ".json" expectedFiles = append(expectedFiles, file) fileWithVersion := "resources/" + gvkToResource[item.resource] if item.namespace != "" { fileWithVersion = fileWithVersion + "/v1-preferredversion/" + "namespaces/" + item.namespace } else { fileWithVersion = fileWithVersion + "/v1-preferredversion" + "/cluster" } fileWithVersion = fileWithVersion + "/" + item.name + ".json" expectedFiles = append(expectedFiles, fileWithVersion) } assertTarballContents(t, backupFile, append(expectedFiles, "metadata/version")...) } // TestBackupProgressIsUpdated verifies that after a backup has run, its // status.progress fields are updated to reflect the total number of items // backed up. It validates this by comparing their values to the length of // the request's BackedUpItems field. func TestBackupProgressIsUpdated(t *testing.T) { h := newHarness(t, nil) defer h.itemBlockPool.Stop() req := &Request{ Backup: defaultBackup().Result(), SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), WorkerPool: &h.itemBlockPool, } backupFile := bytes.NewBuffer([]byte{}) apiResources := []*test.APIResource{ test.Pods( builder.ForPod("foo", "bar").Result(), builder.ForPod("zoo", "raz").Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), builder.ForDeployment("zoo", "raz").Result(), ), test.PVs( builder.ForPersistentVolume("bar").Result(), builder.ForPersistentVolume("baz").Result(), ), } for _, resource := range apiResources { h.addItems(t, resource) } h.backupper.Backup(h.log, req, backupFile, nil, nil, nil) require.NotNil(t, req.Status.Progress) assert.Equal(t, req.BackedUpItems.Len(), req.Status.Progress.TotalItems) assert.Equal(t, req.BackedUpItems.Len(), req.Status.Progress.ItemsBackedUp) } // TestBackupOldResourceFiltering runs backups with different combinations // of resource filters (included/excluded resources, included/excluded // namespaces, label selectors, "include cluster resources" flag), and // verifies that the set of items written to the backup tarball are // correct. Validation is done by looking at the names of the files in // the backup tarball; the contents of the files are not checked. func TestBackupOldResourceFiltering(t *testing.T) { tests := []struct { name string backup *velerov1.Backup apiResources []*test.APIResource want []string actions []biav2.BackupItemAction }{ { name: "no filters backs up everything", backup: defaultBackup().Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("foo", "bar").Result(), builder.ForPod("zoo", "raz").Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), builder.ForDeployment("zoo", "raz").Result(), ), }, want: []string{ "resources/pods/namespaces/foo/bar.json", "resources/pods/namespaces/zoo/raz.json", "resources/deployments.apps/namespaces/foo/bar.json", "resources/deployments.apps/namespaces/zoo/raz.json", "resources/pods/v1-preferredversion/namespaces/foo/bar.json", "resources/pods/v1-preferredversion/namespaces/zoo/raz.json", "resources/deployments.apps/v1-preferredversion/namespaces/foo/bar.json", "resources/deployments.apps/v1-preferredversion/namespaces/zoo/raz.json", }, }, { name: "included resources filter only backs up resources of those types", backup: defaultBackup(). IncludedResources("pods"). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("foo", "bar").Result(), builder.ForPod("zoo", "raz").Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), builder.ForDeployment("zoo", "raz").Result(), ), }, want: []string{ "resources/pods/namespaces/foo/bar.json", "resources/pods/namespaces/zoo/raz.json", "resources/pods/v1-preferredversion/namespaces/foo/bar.json", "resources/pods/v1-preferredversion/namespaces/zoo/raz.json", }, }, { name: "excluded resources filter only backs up resources not of those types", backup: defaultBackup(). ExcludedResources("deployments"). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("foo", "bar").Result(), builder.ForPod("zoo", "raz").Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), builder.ForDeployment("zoo", "raz").Result(), ), }, want: []string{ "resources/pods/namespaces/foo/bar.json", "resources/pods/namespaces/zoo/raz.json", "resources/pods/v1-preferredversion/namespaces/foo/bar.json", "resources/pods/v1-preferredversion/namespaces/zoo/raz.json", }, }, { name: "included namespaces filter only backs up resources in those namespaces", backup: defaultBackup(). IncludedNamespaces("foo"). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("foo", "bar").Result(), builder.ForPod("zoo", "raz").Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), builder.ForDeployment("zoo", "raz").Result(), ), }, want: []string{ "resources/pods/namespaces/foo/bar.json", "resources/deployments.apps/namespaces/foo/bar.json", "resources/pods/v1-preferredversion/namespaces/foo/bar.json", "resources/deployments.apps/v1-preferredversion/namespaces/foo/bar.json", }, }, { name: "excluded namespaces filter only backs up resources not in those namespaces", backup: defaultBackup(). ExcludedNamespaces("zoo"). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("foo", "bar").Result(), builder.ForPod("zoo", "raz").Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), builder.ForDeployment("zoo", "raz").Result(), ), }, want: []string{ "resources/pods/namespaces/foo/bar.json", "resources/deployments.apps/namespaces/foo/bar.json", "resources/pods/v1-preferredversion/namespaces/foo/bar.json", "resources/deployments.apps/v1-preferredversion/namespaces/foo/bar.json", }, }, { name: "IncludeClusterResources=false only backs up namespaced resources", backup: defaultBackup(). IncludeClusterResources(false). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("foo", "bar").Result(), builder.ForPod("zoo", "raz").Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), builder.ForDeployment("zoo", "raz").Result(), ), test.PVs( builder.ForPersistentVolume("bar").Result(), builder.ForPersistentVolume("baz").Result(), ), }, want: []string{ "resources/pods/namespaces/foo/bar.json", "resources/pods/namespaces/zoo/raz.json", "resources/deployments.apps/namespaces/foo/bar.json", "resources/deployments.apps/namespaces/zoo/raz.json", "resources/pods/v1-preferredversion/namespaces/foo/bar.json", "resources/pods/v1-preferredversion/namespaces/zoo/raz.json", "resources/deployments.apps/v1-preferredversion/namespaces/foo/bar.json", "resources/deployments.apps/v1-preferredversion/namespaces/zoo/raz.json", }, }, { name: "label selector only backs up matching resources", backup: defaultBackup(). LabelSelector(&metav1.LabelSelector{MatchLabels: map[string]string{"a": "b"}}). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("foo", "bar").ObjectMeta(builder.WithLabels("a", "b")).Result(), builder.ForPod("zoo", "raz").Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), builder.ForDeployment("zoo", "raz").ObjectMeta(builder.WithLabels("a", "b")).Result(), ), test.PVs( builder.ForPersistentVolume("bar").ObjectMeta(builder.WithLabels("a", "b")).Result(), builder.ForPersistentVolume("baz").ObjectMeta(builder.WithLabels("a", "c")).Result(), ), }, want: []string{ "resources/pods/namespaces/foo/bar.json", "resources/deployments.apps/namespaces/zoo/raz.json", "resources/persistentvolumes/cluster/bar.json", "resources/pods/v1-preferredversion/namespaces/foo/bar.json", "resources/deployments.apps/v1-preferredversion/namespaces/zoo/raz.json", "resources/persistentvolumes/v1-preferredversion/cluster/bar.json", }, }, { name: "OrLabelSelector only backs up matching resources", backup: defaultBackup(). OrLabelSelector([]*metav1.LabelSelector{{MatchLabels: map[string]string{"a1": "b1"}}, {MatchLabels: map[string]string{"a2": "b2"}}, {MatchLabels: map[string]string{"a3": "b3"}}, {MatchLabels: map[string]string{"a4": "b4"}}}). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("foo", "bar").ObjectMeta(builder.WithLabels("a1", "b1")).Result(), builder.ForPod("zoo", "raz").Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), builder.ForDeployment("zoo", "raz").ObjectMeta(builder.WithLabels("a2", "b2")).Result(), ), test.PVs( builder.ForPersistentVolume("bar").ObjectMeta(builder.WithLabels("a4", "b4")).Result(), builder.ForPersistentVolume("baz").ObjectMeta(builder.WithLabels("a5", "b5")).Result(), ), }, want: []string{ "resources/pods/namespaces/foo/bar.json", "resources/deployments.apps/namespaces/zoo/raz.json", "resources/persistentvolumes/cluster/bar.json", "resources/pods/v1-preferredversion/namespaces/foo/bar.json", "resources/deployments.apps/v1-preferredversion/namespaces/zoo/raz.json", "resources/persistentvolumes/v1-preferredversion/cluster/bar.json", }, }, { name: "resources with velero.io/exclude-from-backup=true label are not included", backup: defaultBackup(). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("foo", "bar").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "true")).Result(), builder.ForPod("zoo", "raz").Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), builder.ForDeployment("zoo", "raz").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "true")).Result(), ), test.PVs( builder.ForPersistentVolume("bar").ObjectMeta(builder.WithLabels("a", "b")).Result(), builder.ForPersistentVolume("baz").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "true")).Result(), ), }, want: []string{ "resources/pods/namespaces/zoo/raz.json", "resources/deployments.apps/namespaces/foo/bar.json", "resources/persistentvolumes/cluster/bar.json", "resources/pods/v1-preferredversion/namespaces/zoo/raz.json", "resources/deployments.apps/v1-preferredversion/namespaces/foo/bar.json", "resources/persistentvolumes/v1-preferredversion/cluster/bar.json", }, }, { name: "resources with velero.io/exclude-from-backup=true label are not included even if matching label selector", backup: defaultBackup(). LabelSelector(&metav1.LabelSelector{MatchLabels: map[string]string{"a": "b"}}). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("foo", "bar").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "true", "a", "b")).Result(), builder.ForPod("zoo", "raz").ObjectMeta(builder.WithLabels("a", "b")).Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), builder.ForDeployment("zoo", "raz").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "true", "a", "b")).Result(), ), test.PVs( builder.ForPersistentVolume("bar").ObjectMeta(builder.WithLabels("a", "b")).Result(), builder.ForPersistentVolume("baz").ObjectMeta(builder.WithLabels("a", "b", velerov1.ExcludeFromBackupLabel, "true")).Result(), ), }, want: []string{ "resources/pods/namespaces/zoo/raz.json", "resources/persistentvolumes/cluster/bar.json", "resources/pods/v1-preferredversion/namespaces/zoo/raz.json", "resources/persistentvolumes/v1-preferredversion/cluster/bar.json", }, }, { name: "resources with velero.io/exclude-from-backup label specified but not 'true' are included", backup: defaultBackup(). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("foo", "bar").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "false")).Result(), builder.ForPod("zoo", "raz").Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), builder.ForDeployment("zoo", "raz").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "1")).Result(), ), test.PVs( builder.ForPersistentVolume("bar").ObjectMeta(builder.WithLabels("a", "b")).Result(), builder.ForPersistentVolume("baz").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "")).Result(), ), }, want: []string{ "resources/pods/namespaces/foo/bar.json", "resources/pods/namespaces/zoo/raz.json", "resources/deployments.apps/namespaces/foo/bar.json", "resources/deployments.apps/namespaces/zoo/raz.json", "resources/persistentvolumes/cluster/bar.json", "resources/persistentvolumes/cluster/baz.json", "resources/pods/v1-preferredversion/namespaces/foo/bar.json", "resources/pods/v1-preferredversion/namespaces/zoo/raz.json", "resources/deployments.apps/v1-preferredversion/namespaces/foo/bar.json", "resources/deployments.apps/v1-preferredversion/namespaces/zoo/raz.json", "resources/persistentvolumes/v1-preferredversion/cluster/bar.json", "resources/persistentvolumes/v1-preferredversion/cluster/baz.json", }, }, { name: "should include cluster-scoped resources if backing up subset of namespaces and IncludeClusterResources=true", backup: defaultBackup(). IncludedNamespaces("ns-1", "ns-2"). IncludeClusterResources(true). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-1").Result(), builder.ForPod("ns-3", "pod-1").Result(), ), test.PVs( builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result(), ), }, want: []string{ "resources/pods/namespaces/ns-1/pod-1.json", "resources/pods/namespaces/ns-2/pod-1.json", "resources/persistentvolumes/cluster/pv-1.json", "resources/persistentvolumes/cluster/pv-2.json", "resources/pods/v1-preferredversion/namespaces/ns-1/pod-1.json", "resources/pods/v1-preferredversion/namespaces/ns-2/pod-1.json", "resources/persistentvolumes/v1-preferredversion/cluster/pv-1.json", "resources/persistentvolumes/v1-preferredversion/cluster/pv-2.json", }, }, { name: "should not include cluster-scoped resource if backing up subset of namespaces and IncludeClusterResources=false", backup: defaultBackup(). IncludedNamespaces("ns-1", "ns-2"). IncludeClusterResources(false). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-1").Result(), builder.ForPod("ns-3", "pod-1").Result(), ), test.PVs( builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result(), ), }, want: []string{ "resources/pods/namespaces/ns-1/pod-1.json", "resources/pods/namespaces/ns-2/pod-1.json", "resources/pods/v1-preferredversion/namespaces/ns-1/pod-1.json", "resources/pods/v1-preferredversion/namespaces/ns-2/pod-1.json", }, }, { name: "should not include cluster-scoped resource if backing up subset of namespaces and IncludeClusterResources=nil", backup: defaultBackup(). IncludedNamespaces("ns-1", "ns-2"). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-1").Result(), builder.ForPod("ns-3", "pod-1").Result(), ), test.PVs( builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result(), ), }, want: []string{ "resources/pods/namespaces/ns-1/pod-1.json", "resources/pods/namespaces/ns-2/pod-1.json", "resources/pods/v1-preferredversion/namespaces/ns-1/pod-1.json", "resources/pods/v1-preferredversion/namespaces/ns-2/pod-1.json", }, }, { name: "should include cluster-scoped resources if backing up all namespaces and IncludeClusterResources=true", backup: defaultBackup(). IncludeClusterResources(true). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-1").Result(), builder.ForPod("ns-3", "pod-1").Result(), ), test.PVs( builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result(), ), }, want: []string{ "resources/pods/namespaces/ns-1/pod-1.json", "resources/pods/namespaces/ns-2/pod-1.json", "resources/pods/namespaces/ns-3/pod-1.json", "resources/persistentvolumes/cluster/pv-1.json", "resources/persistentvolumes/cluster/pv-2.json", "resources/pods/v1-preferredversion/namespaces/ns-1/pod-1.json", "resources/pods/v1-preferredversion/namespaces/ns-2/pod-1.json", "resources/pods/v1-preferredversion/namespaces/ns-3/pod-1.json", "resources/persistentvolumes/v1-preferredversion/cluster/pv-1.json", "resources/persistentvolumes/v1-preferredversion/cluster/pv-2.json", }, }, { name: "should not include cluster-scoped resources if backing up all namespaces and IncludeClusterResources=false", backup: defaultBackup(). IncludeClusterResources(false). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-1").Result(), builder.ForPod("ns-3", "pod-1").Result(), ), test.PVs( builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result(), ), }, want: []string{ "resources/pods/namespaces/ns-1/pod-1.json", "resources/pods/namespaces/ns-2/pod-1.json", "resources/pods/namespaces/ns-3/pod-1.json", "resources/pods/v1-preferredversion/namespaces/ns-1/pod-1.json", "resources/pods/v1-preferredversion/namespaces/ns-2/pod-1.json", "resources/pods/v1-preferredversion/namespaces/ns-3/pod-1.json", }, }, { name: "should include cluster-scoped resources if backing up all namespaces and IncludeClusterResources=nil", backup: defaultBackup(). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-1").Result(), builder.ForPod("ns-3", "pod-1").Result(), ), test.PVs( builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result(), ), }, want: []string{ "resources/pods/namespaces/ns-1/pod-1.json", "resources/pods/namespaces/ns-2/pod-1.json", "resources/pods/namespaces/ns-3/pod-1.json", "resources/persistentvolumes/cluster/pv-1.json", "resources/persistentvolumes/cluster/pv-2.json", "resources/pods/v1-preferredversion/namespaces/ns-1/pod-1.json", "resources/pods/v1-preferredversion/namespaces/ns-2/pod-1.json", "resources/pods/v1-preferredversion/namespaces/ns-3/pod-1.json", "resources/persistentvolumes/v1-preferredversion/cluster/pv-1.json", "resources/persistentvolumes/v1-preferredversion/cluster/pv-2.json", }, }, { name: "when a wildcard and a specific resource are included, the wildcard takes precedence", backup: defaultBackup(). IncludedResources("*", "pods"). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("foo", "bar").Result(), builder.ForPod("zoo", "raz").Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), builder.ForDeployment("zoo", "raz").Result(), ), }, want: []string{ "resources/pods/namespaces/foo/bar.json", "resources/pods/namespaces/zoo/raz.json", "resources/deployments.apps/namespaces/foo/bar.json", "resources/deployments.apps/namespaces/zoo/raz.json", "resources/pods/v1-preferredversion/namespaces/foo/bar.json", "resources/pods/v1-preferredversion/namespaces/zoo/raz.json", "resources/deployments.apps/v1-preferredversion/namespaces/foo/bar.json", "resources/deployments.apps/v1-preferredversion/namespaces/zoo/raz.json", }, }, { name: "wildcard excludes are ignored", backup: defaultBackup(). ExcludedResources("*"). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("foo", "bar").Result(), builder.ForPod("zoo", "raz").Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), builder.ForDeployment("zoo", "raz").Result(), ), }, want: []string{ "resources/pods/namespaces/foo/bar.json", "resources/pods/namespaces/zoo/raz.json", "resources/deployments.apps/namespaces/foo/bar.json", "resources/deployments.apps/namespaces/zoo/raz.json", "resources/pods/v1-preferredversion/namespaces/foo/bar.json", "resources/pods/v1-preferredversion/namespaces/zoo/raz.json", "resources/deployments.apps/v1-preferredversion/namespaces/foo/bar.json", "resources/deployments.apps/v1-preferredversion/namespaces/zoo/raz.json", }, }, { name: "unresolvable included resources are ignored", backup: defaultBackup(). IncludedResources("pods", "unresolvable"). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("foo", "bar").Result(), builder.ForPod("zoo", "raz").Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), builder.ForDeployment("zoo", "raz").Result(), ), }, want: []string{ "resources/pods/namespaces/foo/bar.json", "resources/pods/namespaces/zoo/raz.json", "resources/pods/v1-preferredversion/namespaces/foo/bar.json", "resources/pods/v1-preferredversion/namespaces/zoo/raz.json", }, }, { name: "when all included resources are unresolvable, nothing is included", backup: defaultBackup(). IncludedResources("unresolvable-1", "unresolvable-2"). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("foo", "bar").Result(), builder.ForPod("zoo", "raz").Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), builder.ForDeployment("zoo", "raz").Result(), ), }, want: []string{}, }, { name: "unresolvable excluded resources are ignored", backup: defaultBackup(). ExcludedResources("deployments", "unresolvable"). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("foo", "bar").Result(), builder.ForPod("zoo", "raz").Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), builder.ForDeployment("zoo", "raz").Result(), ), }, want: []string{ "resources/pods/namespaces/foo/bar.json", "resources/pods/namespaces/zoo/raz.json", "resources/pods/v1-preferredversion/namespaces/foo/bar.json", "resources/pods/v1-preferredversion/namespaces/zoo/raz.json", }, }, { name: "when all excluded resources are unresolvable, nothing is excluded", backup: defaultBackup(). IncludedResources("*"). ExcludedResources("unresolvable-1", "unresolvable-2"). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("foo", "bar").Result(), builder.ForPod("zoo", "raz").Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), builder.ForDeployment("zoo", "raz").Result(), ), }, want: []string{ "resources/pods/namespaces/foo/bar.json", "resources/pods/namespaces/zoo/raz.json", "resources/pods/v1-preferredversion/namespaces/foo/bar.json", "resources/pods/v1-preferredversion/namespaces/zoo/raz.json", "resources/deployments.apps/namespaces/foo/bar.json", "resources/deployments.apps/namespaces/zoo/raz.json", "resources/deployments.apps/v1-preferredversion/namespaces/foo/bar.json", "resources/deployments.apps/v1-preferredversion/namespaces/zoo/raz.json", }, }, { name: "terminating resources are not backed up", backup: defaultBackup().Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").ObjectMeta(builder.WithDeletionTimestamp(time.Now())).Result(), ), }, want: []string{ "resources/pods/namespaces/ns-1/pod-1.json", "resources/pods/v1-preferredversion/namespaces/ns-1/pod-1.json", }, }, { name: "new filters' default value should not impact the old filters' function", backup: defaultBackup().IncludedNamespaces("foo").IncludeClusterResources(true).Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("foo", "bar").Volumes(builder.ForVolume("foo").PersistentVolumeClaimSource("test-1").Result()).Result(), builder.ForPod("zoo", "raz").Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), builder.ForDeployment("zoo", "raz").Result(), ), test.PVCs( builder.ForPersistentVolumeClaim("foo", "test-1").VolumeName("test1").Result(), ), test.PVs( builder.ForPersistentVolume("test1").Result(), builder.ForPersistentVolume("test2").Result(), ), }, want: []string{ "resources/deployments.apps/namespaces/foo/bar.json", "resources/deployments.apps/v1-preferredversion/namespaces/foo/bar.json", "resources/persistentvolumeclaims/namespaces/foo/test-1.json", "resources/persistentvolumeclaims/v1-preferredversion/namespaces/foo/test-1.json", "resources/persistentvolumes/cluster/test1.json", "resources/persistentvolumes/cluster/test2.json", "resources/persistentvolumes/v1-preferredversion/cluster/test1.json", "resources/persistentvolumes/v1-preferredversion/cluster/test2.json", "resources/pods/namespaces/foo/bar.json", "resources/pods/v1-preferredversion/namespaces/foo/bar.json", }, actions: []biav2.BackupItemAction{ &pluggableAction{ selector: velero.ResourceSelector{IncludedResources: []string{"persistentvolumeclaims"}}, executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { additionalItems := []velero.ResourceIdentifier{ {GroupResource: kuberesource.PersistentVolumes, Name: "test1"}, } return item, additionalItems, "", nil, nil }, }, }, }, { name: "Resource's CRD should be included", backup: defaultBackup().IncludedNamespaces("foo").Result(), apiResources: []*test.APIResource{ test.CRDs( builder.ForCustomResourceDefinitionV1Beta1("backups.velero.io").Result(), builder.ForCustomResourceDefinitionV1Beta1("volumesnapshotlocations.velero.io").Result(), builder.ForCustomResourceDefinitionV1Beta1("test.velero.io").Result(), ), test.VSLs( builder.ForVolumeSnapshotLocation("foo", "bar").Result(), ), test.Backups( builder.ForBackup("zoo", "raz").Result(), ), }, want: []string{ "resources/customresourcedefinitions.apiextensions.k8s.io/cluster/volumesnapshotlocations.velero.io.json", "resources/customresourcedefinitions.apiextensions.k8s.io/v1beta1-preferredversion/cluster/volumesnapshotlocations.velero.io.json", "resources/volumesnapshotlocations.velero.io/namespaces/foo/bar.json", "resources/volumesnapshotlocations.velero.io/v1-preferredversion/namespaces/foo/bar.json", }, }, { name: "Resource's CRD is not included, when CRD is excluded.", backup: defaultBackup().IncludedNamespaces("foo").ExcludedResources("customresourcedefinitions.apiextensions.k8s.io").Result(), apiResources: []*test.APIResource{ test.CRDs( builder.ForCustomResourceDefinitionV1Beta1("backups.velero.io").Result(), builder.ForCustomResourceDefinitionV1Beta1("volumesnapshotlocations.velero.io").Result(), builder.ForCustomResourceDefinitionV1Beta1("test.velero.io").Result(), ), test.VSLs( builder.ForVolumeSnapshotLocation("foo", "bar").Result(), ), test.Backups( builder.ForBackup("zoo", "raz").Result(), ), }, want: []string{ "resources/volumesnapshotlocations.velero.io/namespaces/foo/bar.json", "resources/volumesnapshotlocations.velero.io/v1-preferredversion/namespaces/foo/bar.json", }, }, } itemBlockPool := StartItemBlockWorkerPool(t.Context(), 1, logrus.StandardLogger()) defer itemBlockPool.Stop() for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { var ( h = newHarness(t, itemBlockPool) req = &Request{ Backup: tc.backup, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), WorkerPool: itemBlockPool, } backupFile = bytes.NewBuffer([]byte{}) ) for _, resource := range tc.apiResources { h.addItems(t, resource) } h.backupper.Backup(h.log, req, backupFile, tc.actions, nil, nil) assertTarballContents(t, backupFile, append(tc.want, "metadata/version")...) }) } } // TestCRDInclusion tests whether related CRDs are included, based on // backed-up resources and "include cluster resources" flag, and // verifies that the set of items written to the backup tarball are // correct. Validation is done by looking at the names of the files in // the backup tarball; the contents of the files are not checked. func TestCRDInclusion(t *testing.T) { tests := []struct { name string backup *velerov1.Backup apiResources []*test.APIResource want []string }{ { name: "include cluster resources=auto includes all CRDs when running a full-cluster backup", backup: defaultBackup(). Result(), apiResources: []*test.APIResource{ test.CRDs( builder.ForCustomResourceDefinitionV1Beta1("backups.velero.io").Result(), builder.ForCustomResourceDefinitionV1Beta1("volumesnapshotlocations.velero.io").Result(), builder.ForCustomResourceDefinitionV1Beta1("test.velero.io").Result(), ), test.VSLs( builder.ForVolumeSnapshotLocation("foo", "vsl-1").Result(), ), }, want: []string{ "resources/customresourcedefinitions.apiextensions.k8s.io/cluster/backups.velero.io.json", "resources/customresourcedefinitions.apiextensions.k8s.io/cluster/volumesnapshotlocations.velero.io.json", "resources/customresourcedefinitions.apiextensions.k8s.io/cluster/test.velero.io.json", "resources/volumesnapshotlocations.velero.io/namespaces/foo/vsl-1.json", "resources/customresourcedefinitions.apiextensions.k8s.io/v1beta1-preferredversion/cluster/backups.velero.io.json", "resources/customresourcedefinitions.apiextensions.k8s.io/v1beta1-preferredversion/cluster/volumesnapshotlocations.velero.io.json", "resources/customresourcedefinitions.apiextensions.k8s.io/v1beta1-preferredversion/cluster/test.velero.io.json", "resources/volumesnapshotlocations.velero.io/v1-preferredversion/namespaces/foo/vsl-1.json", }, }, { name: "include cluster resources=false excludes all CRDs when backing up all namespaces", backup: defaultBackup(). IncludeClusterResources(false). Result(), apiResources: []*test.APIResource{ test.CRDs( builder.ForCustomResourceDefinitionV1Beta1("backups.velero.io").Result(), builder.ForCustomResourceDefinitionV1Beta1("volumesnapshotlocations.velero.io").Result(), builder.ForCustomResourceDefinitionV1Beta1("test.velero.io").Result(), ), test.VSLs( builder.ForVolumeSnapshotLocation("foo", "vsl-1").Result(), ), }, want: []string{ "resources/volumesnapshotlocations.velero.io/namespaces/foo/vsl-1.json", "resources/volumesnapshotlocations.velero.io/v1-preferredversion/namespaces/foo/vsl-1.json", }, }, { name: "include cluster resources=true includes all CRDs when running a full-cluster backup", backup: defaultBackup(). IncludeClusterResources(true). Result(), apiResources: []*test.APIResource{ test.CRDs( builder.ForCustomResourceDefinitionV1Beta1("backups.velero.io").Result(), builder.ForCustomResourceDefinitionV1Beta1("volumesnapshotlocations.velero.io").Result(), builder.ForCustomResourceDefinitionV1Beta1("test.velero.io").Result(), ), test.VSLs( builder.ForVolumeSnapshotLocation("foo", "vsl-1").Result(), ), }, want: []string{ "resources/customresourcedefinitions.apiextensions.k8s.io/cluster/backups.velero.io.json", "resources/customresourcedefinitions.apiextensions.k8s.io/cluster/volumesnapshotlocations.velero.io.json", "resources/customresourcedefinitions.apiextensions.k8s.io/cluster/test.velero.io.json", "resources/volumesnapshotlocations.velero.io/namespaces/foo/vsl-1.json", "resources/customresourcedefinitions.apiextensions.k8s.io/v1beta1-preferredversion/cluster/backups.velero.io.json", "resources/customresourcedefinitions.apiextensions.k8s.io/v1beta1-preferredversion/cluster/volumesnapshotlocations.velero.io.json", "resources/customresourcedefinitions.apiextensions.k8s.io/v1beta1-preferredversion/cluster/test.velero.io.json", "resources/volumesnapshotlocations.velero.io/v1-preferredversion/namespaces/foo/vsl-1.json", }, }, { name: "include cluster resources=auto includes CRDs with CRs when backing up selected namespaces", backup: defaultBackup(). IncludedNamespaces("foo"). Result(), apiResources: []*test.APIResource{ test.CRDs( builder.ForCustomResourceDefinitionV1Beta1("backups.velero.io").Result(), builder.ForCustomResourceDefinitionV1Beta1("volumesnapshotlocations.velero.io").Result(), builder.ForCustomResourceDefinitionV1Beta1("test.velero.io").Result(), ), test.VSLs( builder.ForVolumeSnapshotLocation("foo", "vsl-1").Result(), ), }, want: []string{ "resources/customresourcedefinitions.apiextensions.k8s.io/cluster/volumesnapshotlocations.velero.io.json", "resources/volumesnapshotlocations.velero.io/namespaces/foo/vsl-1.json", "resources/customresourcedefinitions.apiextensions.k8s.io/v1beta1-preferredversion/cluster/volumesnapshotlocations.velero.io.json", "resources/volumesnapshotlocations.velero.io/v1-preferredversion/namespaces/foo/vsl-1.json", }, }, { name: "include-cluster-resources=false excludes all CRDs when backing up selected namespaces", backup: defaultBackup(). IncludeClusterResources(false). IncludedNamespaces("foo"). Result(), apiResources: []*test.APIResource{ test.CRDs( builder.ForCustomResourceDefinitionV1Beta1("backups.velero.io").Result(), builder.ForCustomResourceDefinitionV1Beta1("volumesnapshotlocations.velero.io").Result(), builder.ForCustomResourceDefinitionV1Beta1("test.velero.io").Result(), ), test.VSLs( builder.ForVolumeSnapshotLocation("foo", "bar").Result(), ), }, want: []string{ "resources/volumesnapshotlocations.velero.io/namespaces/foo/bar.json", "resources/volumesnapshotlocations.velero.io/v1-preferredversion/namespaces/foo/bar.json", }, }, { name: "include cluster resources=true includes all CRDs when backing up selected namespaces", backup: defaultBackup(). IncludeClusterResources(true). IncludedNamespaces("foo"). Result(), apiResources: []*test.APIResource{ test.CRDs( builder.ForCustomResourceDefinitionV1Beta1("backups.velero.io").Result(), builder.ForCustomResourceDefinitionV1Beta1("volumesnapshotlocations.velero.io").Result(), builder.ForCustomResourceDefinitionV1Beta1("test.velero.io").Result(), ), test.VSLs( builder.ForVolumeSnapshotLocation("foo", "vsl-1").Result(), ), }, want: []string{ "resources/customresourcedefinitions.apiextensions.k8s.io/cluster/backups.velero.io.json", "resources/customresourcedefinitions.apiextensions.k8s.io/cluster/volumesnapshotlocations.velero.io.json", "resources/customresourcedefinitions.apiextensions.k8s.io/cluster/test.velero.io.json", "resources/volumesnapshotlocations.velero.io/namespaces/foo/vsl-1.json", "resources/customresourcedefinitions.apiextensions.k8s.io/v1beta1-preferredversion/cluster/backups.velero.io.json", "resources/customresourcedefinitions.apiextensions.k8s.io/v1beta1-preferredversion/cluster/volumesnapshotlocations.velero.io.json", "resources/customresourcedefinitions.apiextensions.k8s.io/v1beta1-preferredversion/cluster/test.velero.io.json", "resources/volumesnapshotlocations.velero.io/v1-preferredversion/namespaces/foo/vsl-1.json", }, }, } itemBlockPool := StartItemBlockWorkerPool(t.Context(), 1, logrus.StandardLogger()) defer itemBlockPool.Stop() for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { var ( h = newHarness(t, itemBlockPool) req = &Request{ Backup: tc.backup, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), WorkerPool: itemBlockPool, } backupFile = bytes.NewBuffer([]byte{}) ) for _, resource := range tc.apiResources { h.addItems(t, resource) } h.backupper.Backup(h.log, req, backupFile, nil, nil, nil) assertTarballContents(t, backupFile, append(tc.want, "metadata/version")...) }) } } // TestBackupResourceCohabitation runs backups for resources that "cohabitate", // meaning they exist in multiple API groups (e.g. deployments.extensions and // deployments.apps), and verifies that only one copy of each resource is backed // up, with preference for the non-"extensions" API group. func TestBackupResourceCohabitation(t *testing.T) { tests := []struct { name string backup *velerov1.Backup apiResources []*test.APIResource want []string }{ { name: "when deployments exist only in extensions, they're backed up", backup: defaultBackup().Result(), apiResources: []*test.APIResource{ test.ExtensionsDeployments( builder.ForDeployment("foo", "bar").Result(), builder.ForDeployment("zoo", "raz").Result(), ), }, want: []string{ "resources/deployments.extensions/namespaces/foo/bar.json", "resources/deployments.extensions/namespaces/zoo/raz.json", "resources/deployments.extensions/v1-preferredversion/namespaces/foo/bar.json", "resources/deployments.extensions/v1-preferredversion/namespaces/zoo/raz.json", }, }, { name: "when deployments exist in both apps and extensions, only apps/deployments are backed up", backup: defaultBackup().Result(), apiResources: []*test.APIResource{ test.ExtensionsDeployments( builder.ForDeployment("foo", "bar").Result(), builder.ForDeployment("zoo", "raz").Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), builder.ForDeployment("zoo", "raz").Result(), ), }, want: []string{ "resources/deployments.apps/namespaces/foo/bar.json", "resources/deployments.apps/namespaces/zoo/raz.json", "resources/deployments.apps/v1-preferredversion/namespaces/foo/bar.json", "resources/deployments.apps/v1-preferredversion/namespaces/zoo/raz.json", }, }, { name: "when deployments exist that are not in the cohabiting groups those are backed up along with apps/deployments", backup: defaultBackup().Result(), apiResources: []*test.APIResource{ test.VeleroDeployments( builder.ForTestCR("Deployment", "foo", "bar").Result(), builder.ForTestCR("Deployment", "zoo", "raz").Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), builder.ForDeployment("zoo", "raz").Result(), ), }, want: []string{ "resources/deployments.apps/namespaces/foo/bar.json", "resources/deployments.apps/namespaces/zoo/raz.json", "resources/deployments.apps/v1-preferredversion/namespaces/foo/bar.json", "resources/deployments.apps/v1-preferredversion/namespaces/zoo/raz.json", "resources/deployments.velero.io/namespaces/foo/bar.json", "resources/deployments.velero.io/namespaces/zoo/raz.json", "resources/deployments.velero.io/v1-preferredversion/namespaces/foo/bar.json", "resources/deployments.velero.io/v1-preferredversion/namespaces/zoo/raz.json", }, }, } itemBlockPool := StartItemBlockWorkerPool(t.Context(), 1, logrus.StandardLogger()) defer itemBlockPool.Stop() for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { var ( h = newHarness(t, itemBlockPool) req = &Request{ Backup: tc.backup, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), WorkerPool: itemBlockPool, } backupFile = bytes.NewBuffer([]byte{}) ) for _, resource := range tc.apiResources { h.addItems(t, resource) } h.backupper.Backup(h.log, req, backupFile, nil, nil, nil) assertTarballContents(t, backupFile, append(tc.want, "metadata/version")...) }) } } // TestBackupUsesNewCohabitatingResourcesForEachBackup ensures that when two backups are // run that each include cohabiting resources, one copy of the relevant resources is // backed up in each backup. Verification is done by looking at the contents of the backup // tarball. This covers a specific issue that was fixed by https://github.com/vmware-tanzu/velero/pull/485. func TestBackupUsesNewCohabitatingResourcesForEachBackup(t *testing.T) { h := newHarness(t, nil) defer h.itemBlockPool.Stop() // run and verify backup 1 backup1 := &Request{ Backup: defaultBackup().Result(), SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), WorkerPool: &h.itemBlockPool, } backup1File := bytes.NewBuffer([]byte{}) h.addItems(t, test.Deployments(builder.ForDeployment("ns-1", "deploy-1").Result())) h.addItems(t, test.ExtensionsDeployments(builder.ForDeployment("ns-1", "deploy-1").Result())) h.backupper.Backup(h.log, backup1, backup1File, nil, nil, nil) assertTarballContents(t, backup1File, "metadata/version", "resources/deployments.apps/namespaces/ns-1/deploy-1.json", "resources/deployments.apps/v1-preferredversion/namespaces/ns-1/deploy-1.json") // run and verify backup 2 backup2 := &Request{ Backup: defaultBackup().Result(), SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), WorkerPool: &h.itemBlockPool, } backup2File := bytes.NewBuffer([]byte{}) h.backupper.Backup(h.log, backup2, backup2File, nil, nil, nil) assertTarballContents(t, backup2File, "metadata/version", "resources/deployments.apps/namespaces/ns-1/deploy-1.json", "resources/deployments.apps/v1-preferredversion/namespaces/ns-1/deploy-1.json") } // TestBackupResourceOrdering runs backups of the core API group and ensures that items are backed // up in the expected order (pods, PVCs, PVs, everything else). Verification is done by looking // at the order of files written to the backup tarball. func TestBackupResourceOrdering(t *testing.T) { tests := []struct { name string backup *velerov1.Backup apiResources []*test.APIResource }{ { name: "core API group: pods come before pvcs, pvcs come before pvs, pvs come before anything else", backup: defaultBackup(). SnapshotVolumes(false). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("foo", "bar").Result(), builder.ForPod("zoo", "raz").Result(), ), test.PVCs( builder.ForPersistentVolumeClaim("foo", "bar").Result(), builder.ForPersistentVolumeClaim("zoo", "raz").Result(), ), test.PVs( builder.ForPersistentVolume("bar").Result(), builder.ForPersistentVolume("baz").Result(), ), test.Secrets( builder.ForSecret("foo", "bar").Result(), builder.ForSecret("zoo", "raz").Result(), ), }, }, } itemBlockPool := StartItemBlockWorkerPool(t.Context(), 1, logrus.StandardLogger()) defer itemBlockPool.Stop() for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { var ( h = newHarness(t, itemBlockPool) req = &Request{ Backup: tc.backup, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), WorkerPool: itemBlockPool, } backupFile = bytes.NewBuffer([]byte{}) ) for _, resource := range tc.apiResources { h.addItems(t, resource) } h.backupper.Backup(h.log, req, backupFile, nil, nil, nil) assertTarballOrdering(t, backupFile, "pods", "persistentvolumeclaims", "persistentvolumes") }) } } // recordResourcesAction is a backup item action that can be configured // to run for specific resources/namespaces and simply records the items // that it is executed for. type recordResourcesAction struct { name string selector velero.ResourceSelector ids []string backups []velerov1.Backup executionErr error additionalItems []velero.ResourceIdentifier operationID string postOperationItems []velero.ResourceIdentifier skippedCSISnapshot bool } func (a *recordResourcesAction) Execute(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { metadata, err := meta.Accessor(item) if err != nil { return item, a.additionalItems, a.operationID, a.postOperationItems, err } a.ids = append(a.ids, kubeutil.NamespaceAndName(metadata)) a.backups = append(a.backups, *backup) if a.skippedCSISnapshot { u := &unstructured.Unstructured{Object: item.UnstructuredContent()} u.SetAnnotations(map[string]string{velerov1.SkippedNoCSIPVAnnotation: "true"}) item = u a.additionalItems = nil } return item, a.additionalItems, a.operationID, a.postOperationItems, a.executionErr } func (a *recordResourcesAction) AppliesTo() (velero.ResourceSelector, error) { return a.selector, nil } func (a *recordResourcesAction) Progress(operationID string, backup *velerov1.Backup) (velero.OperationProgress, error) { return velero.OperationProgress{}, nil } func (a *recordResourcesAction) Cancel(operationID string, backup *velerov1.Backup) error { return nil } func (a *recordResourcesAction) Name() string { return a.name } func (a *recordResourcesAction) ForResource(resource string) *recordResourcesAction { a.selector.IncludedResources = append(a.selector.IncludedResources, resource) return a } func (a *recordResourcesAction) ForNamespace(namespace string) *recordResourcesAction { a.selector.IncludedNamespaces = append(a.selector.IncludedNamespaces, namespace) return a } func (a *recordResourcesAction) ForLabelSelector(selector string) *recordResourcesAction { a.selector.LabelSelector = selector return a } func (a *recordResourcesAction) WithAdditionalItems(items []velero.ResourceIdentifier) *recordResourcesAction { a.additionalItems = items return a } func (a *recordResourcesAction) WithName(name string) *recordResourcesAction { a.name = name return a } func (a *recordResourcesAction) WithExecutionErr(executionErr error) *recordResourcesAction { a.executionErr = executionErr return a } func (a *recordResourcesAction) WithSkippedCSISnapshotFlag(flag bool) *recordResourcesAction { a.skippedCSISnapshot = flag return a } // TestBackupItemActionsForSkippedPV runs backups with backup item actions, and // verifies that the data in SkippedPVTracker is updated as expected. func TestBackupItemActionsForSkippedPV(t *testing.T) { itemBlockPool := StartItemBlockWorkerPool(t.Context(), 1, logrus.StandardLogger()) defer itemBlockPool.Stop() tests := []struct { name string backupReq *Request apiResources []*test.APIResource runtimeResources []runtime.Object actions []*recordResourcesAction resPolicies *resourcepolicies.ResourcePolicies // {pvName:{approach: reason}} expectSkippedPVs map[string]map[string]string expectNotSkippedPVs []string }{ { name: "backup item action returns the 'not a CSI volume' error and the PV should be tracked as skippedPV", backupReq: &Request{ Backup: defaultBackup().SnapshotVolumes(false).Result(), SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), WorkerPool: itemBlockPool, }, resPolicies: &resourcepolicies.ResourcePolicies{ Version: "v1", VolumePolicies: []resourcepolicies.VolumePolicy{ { Action: resourcepolicies.Action{Type: "snapshot"}, Conditions: map[string]any{ "storageClass": []string{"gp2"}, }, }, }, }, apiResources: []*test.APIResource{ test.PVs( builder.ForPersistentVolume("pv-1").StorageClass("gp2").Result(), ), test.PVCs( builder.ForPersistentVolumeClaim("ns-1", "pvc-1").VolumeName("pv-1").StorageClass("gp2").Phase(corev1api.ClaimBound).Result(), ), }, runtimeResources: []runtime.Object{ builder.ForPersistentVolume("pv-1").StorageClass("gp2").Result(), builder.ForPersistentVolumeClaim("ns-1", "pvc-1").VolumeName("pv-1").StorageClass("gp2").Phase(corev1api.ClaimBound).Result(), }, actions: []*recordResourcesAction{ new(recordResourcesAction).WithName(csiBIAPluginName).ForNamespace("ns-1").ForResource("persistentvolumeclaims").WithSkippedCSISnapshotFlag(true), }, expectSkippedPVs: map[string]map[string]string{ "pv-1": { csiSnapshotApproach: "skipped b/c it's not a CSI volume", }, }, }, { name: "backup item action named as CSI plugin executed successfully and the PV will be removed from the skipped PV tracker", backupReq: &Request{ Backup: defaultBackup().Result(), SkippedPVTracker: &skipPVTracker{ RWMutex: &sync.RWMutex{}, pvs: map[string]map[string]string{ "pv-1": { "any": "whatever reason", }, }, includedPVs: map[string]struct{}{}, }, BackedUpItems: NewBackedUpItemsMap(), WorkerPool: itemBlockPool, }, apiResources: []*test.APIResource{ test.PVCs( builder.ForPersistentVolumeClaim("ns-1", "pvc-1").VolumeName("pv-1").Phase(corev1api.ClaimBound).Result(), ), }, runtimeResources: []runtime.Object{ builder.ForPersistentVolumeClaim("ns-1", "pvc-1").VolumeName("pv-1").Phase(corev1api.ClaimBound).Result(), builder.ForPersistentVolume("pv-1").StorageClass("gp2").Result(), }, actions: []*recordResourcesAction{ new(recordResourcesAction).ForNamespace("ns-1").ForResource("persistentvolumeclaims").WithName(csiBIAPluginName), }, expectNotSkippedPVs: []string{"pv-1"}, }, } // Enable CSI feature before running the test, because Velero will check whether // CSI feature is enabled before executing CSI plugin actions. features.NewFeatureFlagSet("EnableCSI") defer func() { features.NewFeatureFlagSet("") }() for _, tc := range tests { t.Run(tc.name, func(tt *testing.T) { var ( h = newHarness(t, itemBlockPool) backupFile = bytes.NewBuffer([]byte{}) fakeClient = test.NewFakeControllerRuntimeClient(t, tc.runtimeResources...) ) h.backupper.kbClient = fakeClient for _, resource := range tc.apiResources { h.addItems(t, resource) } actions := []biav2.BackupItemAction{} for _, action := range tc.actions { actions = append(actions, action) } if tc.resPolicies != nil { tc.backupReq.ResPolicies = new(resourcepolicies.Policies) require.NoError(t, tc.backupReq.ResPolicies.BuildPolicy(tc.resPolicies)) } err := h.backupper.Backup(h.log, tc.backupReq, backupFile, actions, nil, nil) require.NoError(t, err) if tc.expectSkippedPVs != nil { for pvName, reasons := range tc.expectSkippedPVs { v, ok := tc.backupReq.SkippedPVTracker.pvs[pvName] assert.True(tt, ok) for approach, reason := range reasons { assert.Equal(tt, reason, v[approach]) } } } for _, pvName := range tc.expectNotSkippedPVs { _, ok := tc.backupReq.SkippedPVTracker.pvs[pvName] assert.False(tt, ok) } }) } } // TestBackupActionsRunForCorrectItems runs backups with backup item actions, and // verifies that each backup item action is run for the correct set of resources based on its // AppliesTo() resource selector. Verification is done by using the recordResourcesAction struct, // which records which resources it's executed for. func TestBackupActionsRunForCorrectItems(t *testing.T) { tests := []struct { name string backup *velerov1.Backup apiResources []*test.APIResource // actions is a map from a recordResourcesAction (which will record the items it was called for) // to a slice of expected items, formatted as {namespace}/{name}. actions map[*recordResourcesAction][]string }{ { name: "single action with no selector runs for all items", backup: defaultBackup(). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result(), ), test.PVs( builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result(), ), }, actions: map[*recordResourcesAction][]string{ new(recordResourcesAction): {"ns-1/pod-1", "ns-2/pod-2", "pv-1", "pv-2"}, }, }, { name: "single action with a resource selector for namespaced resources runs only for matching resources", backup: defaultBackup(). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result(), ), test.PVs( builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result(), ), }, actions: map[*recordResourcesAction][]string{ new(recordResourcesAction).ForResource("pods"): {"ns-1/pod-1", "ns-2/pod-2"}, }, }, { name: "single action with a resource selector for cluster-scoped resources runs only for matching resources", backup: defaultBackup(). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result(), ), test.PVs( builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result(), ), }, actions: map[*recordResourcesAction][]string{ new(recordResourcesAction).ForResource("persistentvolumes"): {"pv-1", "pv-2"}, }, }, { name: "single action with a namespace selector runs only for resources in that namespace", backup: defaultBackup(). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result(), ), test.PVCs( builder.ForPersistentVolumeClaim("ns-1", "pvc-1").Result(), builder.ForPersistentVolumeClaim("ns-2", "pvc-2").Result(), ), test.PVs( builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result(), ), test.Namespaces( builder.ForNamespace("ns-1").Result(), builder.ForNamespace("ns-2").Result(), ), }, actions: map[*recordResourcesAction][]string{ new(recordResourcesAction).ForNamespace("ns-1"): {"ns-1/pod-1", "ns-1/pvc-1"}, }, }, { name: "single action with a resource and namespace selector runs only for matching resources", backup: defaultBackup(). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result(), ), test.PVs( builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result(), ), }, actions: map[*recordResourcesAction][]string{ new(recordResourcesAction).ForResource("pods").ForNamespace("ns-1"): {"ns-1/pod-1"}, }, }, { name: "multiple actions, each with a different resource selector using short name, run for matching resources", backup: defaultBackup(). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result(), ), test.PVs( builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result(), ), }, actions: map[*recordResourcesAction][]string{ new(recordResourcesAction).ForResource("po"): {"ns-1/pod-1", "ns-2/pod-2"}, new(recordResourcesAction).ForResource("pv"): {"pv-1", "pv-2"}, }, }, { name: "actions with selectors that don't match anything don't run for any resources", backup: defaultBackup(). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").Result(), ), test.PVCs( builder.ForPersistentVolumeClaim("ns-2", "pvc-2").Result(), ), test.PVs( builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result(), ), }, actions: map[*recordResourcesAction][]string{ new(recordResourcesAction).ForNamespace("ns-1").ForResource("persistentvolumeclaims"): nil, new(recordResourcesAction).ForNamespace("ns-2").ForResource("pods"): nil, }, }, { name: "action with a selector that has unresolvable resources doesn't run for any resources", backup: defaultBackup(). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").Result(), ), test.PVCs( builder.ForPersistentVolumeClaim("ns-2", "pvc-2").Result(), ), test.PVs( builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result(), ), }, actions: map[*recordResourcesAction][]string{ new(recordResourcesAction).ForResource("unresolvable"): nil, }, }, } itemBlockPool := StartItemBlockWorkerPool(t.Context(), 1, logrus.StandardLogger()) defer itemBlockPool.Stop() for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { var ( h = newHarness(t, itemBlockPool) req = &Request{ Backup: tc.backup, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), WorkerPool: itemBlockPool, } backupFile = bytes.NewBuffer([]byte{}) ) for _, resource := range tc.apiResources { h.addItems(t, resource) } actions := []biav2.BackupItemAction{} for action := range tc.actions { actions = append(actions, action) } err := h.backupper.Backup(h.log, req, backupFile, actions, nil, nil) require.NoError(t, err) for action, want := range tc.actions { assert.Equal(t, want, action.ids) } }) } } // TestBackupWithInvalidActions runs backups with backup item actions that are invalid // in some way (e.g. an invalid label selector returned from AppliesTo(), an error returned // from AppliesTo()) and verifies that this causes the backupper.Backup(...) method to // return an error. func TestBackupWithInvalidActions(t *testing.T) { // all test cases in this function are expected to cause the method under test // to return an error, so no expected results need to be set up. tests := []struct { name string backup *velerov1.Backup apiResources []*test.APIResource actions []biav2.BackupItemAction }{ { name: "action with invalid label selector results in an error", backup: defaultBackup(). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("foo", "bar").Result(), builder.ForPod("zoo", "raz").Result(), ), test.PVs( builder.ForPersistentVolume("bar").Result(), builder.ForPersistentVolume("baz").Result(), ), }, actions: []biav2.BackupItemAction{ new(recordResourcesAction).ForLabelSelector("=invalid-selector"), }, }, { name: "action returning an error from AppliesTo results in an error", backup: defaultBackup(). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("foo", "bar").Result(), builder.ForPod("zoo", "raz").Result(), ), test.PVs( builder.ForPersistentVolume("bar").Result(), builder.ForPersistentVolume("baz").Result(), ), }, actions: []biav2.BackupItemAction{ &appliesToErrorAction{}, }, }, } itemBlockPool := StartItemBlockWorkerPool(t.Context(), 1, logrus.StandardLogger()) defer itemBlockPool.Stop() for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { var ( h = newHarness(t, itemBlockPool) req = &Request{ Backup: tc.backup, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), WorkerPool: itemBlockPool, } backupFile = bytes.NewBuffer([]byte{}) ) for _, resource := range tc.apiResources { h.addItems(t, resource) } assert.Error(t, h.backupper.Backup(h.log, req, backupFile, tc.actions, nil, nil)) }) } } // appliesToErrorAction is a backup item action that always returns // an error when AppliesTo() is called. type appliesToErrorAction struct{} func (a *appliesToErrorAction) AppliesTo() (velero.ResourceSelector, error) { return velero.ResourceSelector{}, errors.New("error calling AppliesTo") } func (a *appliesToErrorAction) Execute(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { panic("not implemented") } func (a *appliesToErrorAction) GetRelatedItems(item runtime.Unstructured, backup *velerov1.Backup) ([]velero.ResourceIdentifier, error) { panic("not implemented") } func (a *appliesToErrorAction) Progress(operationID string, backup *velerov1.Backup) (velero.OperationProgress, error) { panic("not implemented") } func (a *appliesToErrorAction) Cancel(operationID string, backup *velerov1.Backup) error { panic("not implemented") } func (a *appliesToErrorAction) Name() string { return "" } // TestBackupActionModifications runs backups with backup item actions that make modifications // to items in their Execute(...) methods and verifies that these modifications are // persisted to the backup tarball. Verification is done by inspecting the file contents // of the tarball. func TestBackupActionModifications(t *testing.T) { // modifyingActionGetter is a helper function that returns a *pluggableAction, whose Execute(...) // method modifies the item being passed in by calling the 'modify' function on it. modifyingActionGetter := func(modify func(*unstructured.Unstructured)) *pluggableAction { return &pluggableAction{ executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { obj, ok := item.(*unstructured.Unstructured) if !ok { return nil, nil, "", nil, errors.Errorf("unexpected type %T", item) } res := obj.DeepCopy() modify(res) return res, nil, "", nil, nil }, } } tests := []struct { name string backup *velerov1.Backup apiResources []*test.APIResource actions []biav2.BackupItemAction want map[string]unstructuredObject }{ { name: "action that adds a label to item gets persisted", backup: defaultBackup().Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").Result(), ), }, actions: []biav2.BackupItemAction{ modifyingActionGetter(func(item *unstructured.Unstructured) { item.SetLabels(map[string]string{"updated": "true"}) }), }, want: map[string]unstructuredObject{ "resources/pods/namespaces/ns-1/pod-1.json": toUnstructuredOrFail(t, builder.ForPod("ns-1", "pod-1").ObjectMeta(builder.WithLabels("updated", "true")).Result()), }, }, { name: "action that removes labels from item gets persisted", backup: defaultBackup().Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").ObjectMeta(builder.WithLabels("should-be-removed", "true")).Result(), ), }, actions: []biav2.BackupItemAction{ modifyingActionGetter(func(item *unstructured.Unstructured) { item.SetLabels(nil) }), }, want: map[string]unstructuredObject{ "resources/pods/namespaces/ns-1/pod-1.json": toUnstructuredOrFail(t, builder.ForPod("ns-1", "pod-1").Result()), }, }, { name: "action that sets a spec field on item gets persisted", backup: defaultBackup().Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").Result(), ), }, actions: []biav2.BackupItemAction{ modifyingActionGetter(func(item *unstructured.Unstructured) { item.Object["spec"].(map[string]any)["nodeName"] = "foo" }), }, want: map[string]unstructuredObject{ "resources/pods/namespaces/ns-1/pod-1.json": toUnstructuredOrFail(t, builder.ForPod("ns-1", "pod-1").NodeName("foo").Result()), }, }, { name: "modifications to name and namespace in an action are persisted in JSON and in filename", backup: defaultBackup(). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").Result(), ), }, actions: []biav2.BackupItemAction{ modifyingActionGetter(func(item *unstructured.Unstructured) { item.SetName(item.GetName() + "-updated") item.SetNamespace(item.GetNamespace() + "-updated") }), }, want: map[string]unstructuredObject{ "resources/pods/namespaces/ns-1-updated/pod-1-updated.json": toUnstructuredOrFail(t, builder.ForPod("ns-1-updated", "pod-1-updated").Result()), }, }, } itemBlockPool := StartItemBlockWorkerPool(t.Context(), 1, logrus.StandardLogger()) defer itemBlockPool.Stop() for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { var ( h = newHarness(t, itemBlockPool) req = &Request{ Backup: tc.backup, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), WorkerPool: itemBlockPool, } backupFile = bytes.NewBuffer([]byte{}) ) for _, resource := range tc.apiResources { h.addItems(t, resource) } err := h.backupper.Backup(h.log, req, backupFile, tc.actions, nil, nil) require.NoError(t, err) assertTarballFileContents(t, backupFile, tc.want) }) } } // TestBackupActionAdditionalItems runs backups with backup item actions that return // additional items to be backed up, and verifies that those items are included in the // backup tarball as appropriate. Verification is done by looking at the files that exist // in the backup tarball. func TestBackupActionAdditionalItems(t *testing.T) { tests := []struct { name string backup *velerov1.Backup apiResources []*test.APIResource actions []biav2.BackupItemAction ibActions []ibav1.ItemBlockAction want []string }{ { name: "additional items that are already being backed up are not backed up twice", backup: defaultBackup().Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result(), builder.ForPod("ns-3", "pod-3").Result(), ), }, actions: []biav2.BackupItemAction{ &pluggableAction{ selector: velero.ResourceSelector{IncludedNamespaces: []string{"ns-1"}}, executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { additionalItems := []velero.ResourceIdentifier{ {GroupResource: kuberesource.Pods, Namespace: "ns-2", Name: "pod-2"}, {GroupResource: kuberesource.Pods, Namespace: "ns-3", Name: "pod-3"}, } return item, additionalItems, "", nil, nil }, }, }, want: []string{ "resources/pods/namespaces/ns-1/pod-1.json", "resources/pods/namespaces/ns-2/pod-2.json", "resources/pods/namespaces/ns-3/pod-3.json", "resources/pods/v1-preferredversion/namespaces/ns-1/pod-1.json", "resources/pods/v1-preferredversion/namespaces/ns-2/pod-2.json", "resources/pods/v1-preferredversion/namespaces/ns-3/pod-3.json", }, }, { name: "when using a backup namespace filter, additional items that are in a non-included namespace are not backed up", backup: defaultBackup().IncludedNamespaces("ns-1").Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result(), builder.ForPod("ns-3", "pod-3").Result(), ), }, actions: []biav2.BackupItemAction{ &pluggableAction{ executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { additionalItems := []velero.ResourceIdentifier{ {GroupResource: kuberesource.Pods, Namespace: "ns-2", Name: "pod-2"}, {GroupResource: kuberesource.Pods, Namespace: "ns-3", Name: "pod-3"}, } return item, additionalItems, "", nil, nil }, }, }, want: []string{ "resources/pods/namespaces/ns-1/pod-1.json", "resources/pods/v1-preferredversion/namespaces/ns-1/pod-1.json", }, }, { name: "when using a backup namespace filter, additional items that are cluster-scoped are backed up", backup: defaultBackup().IncludedNamespaces("ns-1").Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result(), ), test.PVs( builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result(), ), }, actions: []biav2.BackupItemAction{ &pluggableAction{ executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { additionalItems := []velero.ResourceIdentifier{ {GroupResource: kuberesource.PersistentVolumes, Name: "pv-1"}, {GroupResource: kuberesource.PersistentVolumes, Name: "pv-2"}, } return item, additionalItems, "", nil, nil }, }, }, want: []string{ "resources/pods/namespaces/ns-1/pod-1.json", "resources/persistentvolumes/cluster/pv-1.json", "resources/persistentvolumes/cluster/pv-2.json", "resources/pods/v1-preferredversion/namespaces/ns-1/pod-1.json", "resources/persistentvolumes/v1-preferredversion/cluster/pv-1.json", "resources/persistentvolumes/v1-preferredversion/cluster/pv-2.json", }, }, { name: "when using a backup resource filter, additional items that are non-included resources are not backed up", backup: defaultBackup().IncludedResources("pods").Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").Result(), ), test.PVs( builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result(), ), }, actions: []biav2.BackupItemAction{ &pluggableAction{ executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { additionalItems := []velero.ResourceIdentifier{ {GroupResource: kuberesource.PersistentVolumes, Name: "pv-1"}, {GroupResource: kuberesource.PersistentVolumes, Name: "pv-2"}, } return item, additionalItems, "", nil, nil }, }, }, want: []string{ "resources/pods/namespaces/ns-1/pod-1.json", "resources/pods/v1-preferredversion/namespaces/ns-1/pod-1.json", }, }, { name: "when IncludeClusterResources=false, additional items that are cluster-scoped are not backed up", backup: defaultBackup().IncludeClusterResources(false).Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result(), ), test.PVs( builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result(), ), }, actions: []biav2.BackupItemAction{ &pluggableAction{ executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { additionalItems := []velero.ResourceIdentifier{ {GroupResource: kuberesource.PersistentVolumes, Name: "pv-1"}, {GroupResource: kuberesource.PersistentVolumes, Name: "pv-2"}, } return item, additionalItems, "", nil, nil }, }, }, want: []string{ "resources/pods/namespaces/ns-1/pod-1.json", "resources/pods/namespaces/ns-2/pod-2.json", "resources/pods/v1-preferredversion/namespaces/ns-1/pod-1.json", "resources/pods/v1-preferredversion/namespaces/ns-2/pod-2.json", }, }, { name: "additional items with the velero.io/exclude-from-backup label are not backed up", backup: defaultBackup().IncludedNamespaces("ns-1").Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").Result(), ), test.PVs( builder.ForPersistentVolume("pv-1").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "true")).Result(), builder.ForPersistentVolume("pv-2").Result(), ), }, actions: []biav2.BackupItemAction{ &pluggableAction{ executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { additionalItems := []velero.ResourceIdentifier{ {GroupResource: kuberesource.PersistentVolumes, Name: "pv-1"}, {GroupResource: kuberesource.PersistentVolumes, Name: "pv-2"}, } return item, additionalItems, "", nil, nil }, }, }, want: []string{ "resources/pods/namespaces/ns-1/pod-1.json", "resources/persistentvolumes/cluster/pv-2.json", "resources/pods/v1-preferredversion/namespaces/ns-1/pod-1.json", "resources/persistentvolumes/v1-preferredversion/cluster/pv-2.json", }, }, { name: "if additional items aren't found in the API, they're skipped and the original item is still backed up", backup: defaultBackup().Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result(), builder.ForPod("ns-3", "pod-3").Result(), ), }, actions: []biav2.BackupItemAction{ &pluggableAction{ selector: velero.ResourceSelector{IncludedNamespaces: []string{"ns-1"}}, executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { additionalItems := []velero.ResourceIdentifier{ {GroupResource: kuberesource.Pods, Namespace: "ns-4", Name: "pod-4"}, {GroupResource: kuberesource.Pods, Namespace: "ns-5", Name: "pod-5"}, } return item, additionalItems, "", nil, nil }, }, }, want: []string{ "resources/pods/namespaces/ns-1/pod-1.json", "resources/pods/namespaces/ns-2/pod-2.json", "resources/pods/namespaces/ns-3/pod-3.json", "resources/pods/v1-preferredversion/namespaces/ns-1/pod-1.json", "resources/pods/v1-preferredversion/namespaces/ns-2/pod-2.json", "resources/pods/v1-preferredversion/namespaces/ns-3/pod-3.json", }, }, } itemBlockPool := StartItemBlockWorkerPool(t.Context(), 1, logrus.StandardLogger()) defer itemBlockPool.Stop() for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { var ( h = newHarness(t, itemBlockPool) req = &Request{ Backup: tc.backup, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), WorkerPool: itemBlockPool, } backupFile = bytes.NewBuffer([]byte{}) ) for _, resource := range tc.apiResources { h.addItems(t, resource) } err := h.backupper.Backup(h.log, req, backupFile, tc.actions, nil, nil) require.NoError(t, err) assertTarballContents(t, backupFile, append(tc.want, "metadata/version")...) }) } } // recordResourcesIBA is an ItemBlock item action that can be configured // to run for specific resources/namespaces and simply records the items // that it is executed for. type recordResourcesIBA struct { name string selector velero.ResourceSelector ids []string backups []velerov1.Backup executionErr error relatedItems []velero.ResourceIdentifier } func (a *recordResourcesIBA) GetRelatedItems(item runtime.Unstructured, backup *velerov1.Backup) ([]velero.ResourceIdentifier, error) { metadata, err := meta.Accessor(item) if err != nil { return a.relatedItems, err } a.ids = append(a.ids, kubeutil.NamespaceAndName(metadata)) a.backups = append(a.backups, *backup) return a.relatedItems, a.executionErr } func (a *recordResourcesIBA) AppliesTo() (velero.ResourceSelector, error) { return a.selector, nil } func (a *recordResourcesIBA) Name() string { return a.name } func (a *recordResourcesIBA) ForResource(resource string) *recordResourcesIBA { a.selector.IncludedResources = append(a.selector.IncludedResources, resource) return a } func (a *recordResourcesIBA) ForNamespace(namespace string) *recordResourcesIBA { a.selector.IncludedNamespaces = append(a.selector.IncludedNamespaces, namespace) return a } func (a *recordResourcesIBA) ForLabelSelector(selector string) *recordResourcesIBA { a.selector.LabelSelector = selector return a } func (a *recordResourcesIBA) WithRelatedItems(items []velero.ResourceIdentifier) *recordResourcesIBA { a.relatedItems = items return a } func (a *recordResourcesIBA) WithName(name string) *recordResourcesIBA { a.name = name return a } func (a *recordResourcesIBA) WithExecutionErr(executionErr error) *recordResourcesIBA { a.executionErr = executionErr return a } // TestItemBlockActionsRunForCorrectItems runs backups with ItemBlock actions, and // verifies that each action is run for the correct set of resources based on its // AppliesTo() resource selector. Verification is done by using the recordResourcesIBA struct, // which records which resources it's executed for. func TestItemBlockActionsRunForCorrectItems(t *testing.T) { tests := []struct { name string backup *velerov1.Backup apiResources []*test.APIResource // actions is a map from a recordResourcesIBA (which will record the items it was called for) // to a slice of expected items, formatted as {namespace}/{name}. actions map[*recordResourcesIBA][]string }{ { name: "single action with no selector runs for all items", backup: defaultBackup(). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result(), ), test.PVs( builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result(), ), }, actions: map[*recordResourcesIBA][]string{ new(recordResourcesIBA): {"ns-1/pod-1", "ns-2/pod-2", "pv-1", "pv-2"}, }, }, { name: "single action with a resource selector for namespaced resources runs only for matching resources", backup: defaultBackup(). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result(), ), test.PVs( builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result(), ), }, actions: map[*recordResourcesIBA][]string{ new(recordResourcesIBA).ForResource("pods"): {"ns-1/pod-1", "ns-2/pod-2"}, }, }, { name: "single action with a resource selector for cluster-scoped resources runs only for matching resources", backup: defaultBackup(). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result(), ), test.PVs( builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result(), ), }, actions: map[*recordResourcesIBA][]string{ new(recordResourcesIBA).ForResource("persistentvolumes"): {"pv-1", "pv-2"}, }, }, { name: "single action with a namespace selector runs only for resources in that namespace", backup: defaultBackup(). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result(), ), test.PVCs( builder.ForPersistentVolumeClaim("ns-1", "pvc-1").Result(), builder.ForPersistentVolumeClaim("ns-2", "pvc-2").Result(), ), test.PVs( builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result(), ), test.Namespaces( builder.ForNamespace("ns-1").Result(), builder.ForNamespace("ns-2").Result(), ), }, actions: map[*recordResourcesIBA][]string{ new(recordResourcesIBA).ForNamespace("ns-1"): {"ns-1/pod-1", "ns-1/pvc-1"}, }, }, { name: "single action with a resource and namespace selector runs only for matching resources", backup: defaultBackup(). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result(), ), test.PVs( builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result(), ), }, actions: map[*recordResourcesIBA][]string{ new(recordResourcesIBA).ForResource("pods").ForNamespace("ns-1"): {"ns-1/pod-1"}, }, }, { name: "multiple actions, each with a different resource selector using short name, run for matching resources", backup: defaultBackup(). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result(), ), test.PVs( builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result(), ), }, actions: map[*recordResourcesIBA][]string{ new(recordResourcesIBA).ForResource("po"): {"ns-1/pod-1", "ns-2/pod-2"}, new(recordResourcesIBA).ForResource("pv"): {"pv-1", "pv-2"}, }, }, { name: "actions with selectors that don't match anything don't run for any resources", backup: defaultBackup(). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").Result(), ), test.PVCs( builder.ForPersistentVolumeClaim("ns-2", "pvc-2").Result(), ), test.PVs( builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result(), ), }, actions: map[*recordResourcesIBA][]string{ new(recordResourcesIBA).ForNamespace("ns-1").ForResource("persistentvolumeclaims"): nil, new(recordResourcesIBA).ForNamespace("ns-2").ForResource("pods"): nil, }, }, { name: "action with a selector that has unresolvable resources doesn't run for any resources", backup: defaultBackup(). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").Result(), ), test.PVCs( builder.ForPersistentVolumeClaim("ns-2", "pvc-2").Result(), ), test.PVs( builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result(), ), }, actions: map[*recordResourcesIBA][]string{ new(recordResourcesIBA).ForResource("unresolvable"): nil, }, }, } itemBlockPool := StartItemBlockWorkerPool(t.Context(), 1, logrus.StandardLogger()) defer itemBlockPool.Stop() for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { var ( h = newHarness(t, itemBlockPool) req = &Request{ Backup: tc.backup, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), WorkerPool: itemBlockPool, } backupFile = bytes.NewBuffer([]byte{}) ) for _, resource := range tc.apiResources { h.addItems(t, resource) } actions := []ibav1.ItemBlockAction{} for action := range tc.actions { actions = append(actions, action) } err := h.backupper.Backup(h.log, req, backupFile, nil, actions, nil) require.NoError(t, err) for action, want := range tc.actions { assert.Equal(t, want, action.ids) } }) } } // TestBackupWithInvalidItemBlockActions runs backups with ItemBlock actions that are invalid // in some way (e.g. an invalid label selector returned from AppliesTo(), an error returned // from AppliesTo()) and verifies that this causes the backupper.Backup(...) method to // return an error. func TestBackupWithInvalidItemBlockActions(t *testing.T) { // all test cases in this function are expected to cause the method under test // to return an error, so no expected results need to be set up. tests := []struct { name string backup *velerov1.Backup apiResources []*test.APIResource actions []ibav1.ItemBlockAction }{ { name: "action with invalid label selector results in an error", backup: defaultBackup(). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("foo", "bar").Result(), builder.ForPod("zoo", "raz").Result(), ), test.PVs( builder.ForPersistentVolume("bar").Result(), builder.ForPersistentVolume("baz").Result(), ), }, actions: []ibav1.ItemBlockAction{ new(recordResourcesIBA).ForLabelSelector("=invalid-selector"), }, }, { name: "action returning an error from AppliesTo results in an error", backup: defaultBackup(). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("foo", "bar").Result(), builder.ForPod("zoo", "raz").Result(), ), test.PVs( builder.ForPersistentVolume("bar").Result(), builder.ForPersistentVolume("baz").Result(), ), }, actions: []ibav1.ItemBlockAction{ &appliesToErrorAction{}, }, }, } itemBlockPool := StartItemBlockWorkerPool(t.Context(), 1, logrus.StandardLogger()) defer itemBlockPool.Stop() for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { var ( h = newHarness(t, itemBlockPool) req = &Request{ Backup: tc.backup, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), WorkerPool: itemBlockPool, } backupFile = bytes.NewBuffer([]byte{}) ) for _, resource := range tc.apiResources { h.addItems(t, resource) } assert.Error(t, h.backupper.Backup(h.log, req, backupFile, nil, tc.actions, nil)) }) } } // TestItemBlockActionRelatedItems runs backups with ItemBlock actions that return // related items, and verifies that those items are included in the // backup tarball as appropriate. Verification is done by looking at the files that exist // in the backup tarball. func TestItemBlockActionRelatedItems(t *testing.T) { tests := []struct { name string backup *velerov1.Backup apiResources []*test.APIResource actions []ibav1.ItemBlockAction want []string }{ { name: "related items that are already being backed up are not backed up twice", backup: defaultBackup().Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result(), builder.ForPod("ns-3", "pod-3").Result(), ), }, actions: []ibav1.ItemBlockAction{ &pluggableIBA{ selector: velero.ResourceSelector{IncludedNamespaces: []string{"ns-1"}}, getRelatedItemsFunc: func(item runtime.Unstructured, backup *velerov1.Backup) ([]velero.ResourceIdentifier, error) { relatedItems := []velero.ResourceIdentifier{ {GroupResource: kuberesource.Pods, Namespace: "ns-2", Name: "pod-2"}, {GroupResource: kuberesource.Pods, Namespace: "ns-3", Name: "pod-3"}, } return relatedItems, nil }, }, }, want: []string{ "resources/pods/namespaces/ns-1/pod-1.json", "resources/pods/namespaces/ns-2/pod-2.json", "resources/pods/namespaces/ns-3/pod-3.json", "resources/pods/v1-preferredversion/namespaces/ns-1/pod-1.json", "resources/pods/v1-preferredversion/namespaces/ns-2/pod-2.json", "resources/pods/v1-preferredversion/namespaces/ns-3/pod-3.json", }, }, { name: "when using a backup namespace filter, related items that are in a non-included namespace are not backed up", backup: defaultBackup().IncludedNamespaces("ns-1").Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result(), builder.ForPod("ns-3", "pod-3").Result(), ), }, actions: []ibav1.ItemBlockAction{ &pluggableIBA{ getRelatedItemsFunc: func(item runtime.Unstructured, backup *velerov1.Backup) ([]velero.ResourceIdentifier, error) { relatedItems := []velero.ResourceIdentifier{ {GroupResource: kuberesource.Pods, Namespace: "ns-2", Name: "pod-2"}, {GroupResource: kuberesource.Pods, Namespace: "ns-3", Name: "pod-3"}, } return relatedItems, nil }, }, }, want: []string{ "resources/pods/namespaces/ns-1/pod-1.json", "resources/pods/v1-preferredversion/namespaces/ns-1/pod-1.json", }, }, { name: "when using a backup namespace filter, related items that are cluster-scoped are backed up", backup: defaultBackup().IncludedNamespaces("ns-1").Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result(), ), test.PVs( builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result(), ), }, actions: []ibav1.ItemBlockAction{ &pluggableIBA{ getRelatedItemsFunc: func(item runtime.Unstructured, backup *velerov1.Backup) ([]velero.ResourceIdentifier, error) { relatedItems := []velero.ResourceIdentifier{ {GroupResource: kuberesource.PersistentVolumes, Name: "pv-1"}, {GroupResource: kuberesource.PersistentVolumes, Name: "pv-2"}, } return relatedItems, nil }, }, }, want: []string{ "resources/pods/namespaces/ns-1/pod-1.json", "resources/persistentvolumes/cluster/pv-1.json", "resources/persistentvolumes/cluster/pv-2.json", "resources/pods/v1-preferredversion/namespaces/ns-1/pod-1.json", "resources/persistentvolumes/v1-preferredversion/cluster/pv-1.json", "resources/persistentvolumes/v1-preferredversion/cluster/pv-2.json", }, }, { name: "when using a backup resource filter, related items that are non-included resources are not backed up", backup: defaultBackup().IncludedResources("pods").Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").Result(), ), test.PVs( builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result(), ), }, actions: []ibav1.ItemBlockAction{ &pluggableIBA{ getRelatedItemsFunc: func(item runtime.Unstructured, backup *velerov1.Backup) ([]velero.ResourceIdentifier, error) { relatedItems := []velero.ResourceIdentifier{ {GroupResource: kuberesource.PersistentVolumes, Name: "pv-1"}, {GroupResource: kuberesource.PersistentVolumes, Name: "pv-2"}, } return relatedItems, nil }, }, }, want: []string{ "resources/pods/namespaces/ns-1/pod-1.json", "resources/pods/v1-preferredversion/namespaces/ns-1/pod-1.json", }, }, { name: "when IncludeClusterResources=false, related items that are cluster-scoped are not backed up", backup: defaultBackup().IncludeClusterResources(false).Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result(), ), test.PVs( builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result(), ), }, actions: []ibav1.ItemBlockAction{ &pluggableIBA{ getRelatedItemsFunc: func(item runtime.Unstructured, backup *velerov1.Backup) ([]velero.ResourceIdentifier, error) { relatedItems := []velero.ResourceIdentifier{ {GroupResource: kuberesource.PersistentVolumes, Name: "pv-1"}, {GroupResource: kuberesource.PersistentVolumes, Name: "pv-2"}, } return relatedItems, nil }, }, }, want: []string{ "resources/pods/namespaces/ns-1/pod-1.json", "resources/pods/namespaces/ns-2/pod-2.json", "resources/pods/v1-preferredversion/namespaces/ns-1/pod-1.json", "resources/pods/v1-preferredversion/namespaces/ns-2/pod-2.json", }, }, { name: "related items with the velero.io/exclude-from-backup label are not backed up", backup: defaultBackup().IncludedNamespaces("ns-1").Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").Result(), ), test.PVs( builder.ForPersistentVolume("pv-1").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "true")).Result(), builder.ForPersistentVolume("pv-2").Result(), ), }, actions: []ibav1.ItemBlockAction{ &pluggableIBA{ getRelatedItemsFunc: func(item runtime.Unstructured, backup *velerov1.Backup) ([]velero.ResourceIdentifier, error) { relatedItems := []velero.ResourceIdentifier{ {GroupResource: kuberesource.PersistentVolumes, Name: "pv-1"}, {GroupResource: kuberesource.PersistentVolumes, Name: "pv-2"}, } return relatedItems, nil }, }, }, want: []string{ "resources/pods/namespaces/ns-1/pod-1.json", "resources/persistentvolumes/cluster/pv-2.json", "resources/pods/v1-preferredversion/namespaces/ns-1/pod-1.json", "resources/persistentvolumes/v1-preferredversion/cluster/pv-2.json", }, }, { name: "if related items aren't found in the API, they're skipped and the original item is still backed up", backup: defaultBackup().Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result(), builder.ForPod("ns-3", "pod-3").Result(), ), }, actions: []ibav1.ItemBlockAction{ &pluggableIBA{ selector: velero.ResourceSelector{IncludedNamespaces: []string{"ns-1"}}, getRelatedItemsFunc: func(item runtime.Unstructured, backup *velerov1.Backup) ([]velero.ResourceIdentifier, error) { relatedItems := []velero.ResourceIdentifier{ {GroupResource: kuberesource.Pods, Namespace: "ns-4", Name: "pod-4"}, {GroupResource: kuberesource.Pods, Namespace: "ns-5", Name: "pod-5"}, } return relatedItems, nil }, }, }, want: []string{ "resources/pods/namespaces/ns-1/pod-1.json", "resources/pods/namespaces/ns-2/pod-2.json", "resources/pods/namespaces/ns-3/pod-3.json", "resources/pods/v1-preferredversion/namespaces/ns-1/pod-1.json", "resources/pods/v1-preferredversion/namespaces/ns-2/pod-2.json", "resources/pods/v1-preferredversion/namespaces/ns-3/pod-3.json", }, }, } itemBlockPool := StartItemBlockWorkerPool(t.Context(), 1, logrus.StandardLogger()) defer itemBlockPool.Stop() for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { var ( h = newHarness(t, itemBlockPool) req = &Request{ Backup: tc.backup, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), WorkerPool: itemBlockPool, } backupFile = bytes.NewBuffer([]byte{}) ) for _, resource := range tc.apiResources { h.addItems(t, resource) } err := h.backupper.Backup(h.log, req, backupFile, nil, tc.actions, nil) require.NoError(t, err) assertTarballContents(t, backupFile, append(tc.want, "metadata/version")...) }) } } // volumeSnapshotterGetter is a simple implementation of the VolumeSnapshotterGetter // interface that returns vsv1.VolumeSnapshotters from a map if they exist. type volumeSnapshotterGetter map[string]vsv1.VolumeSnapshotter func (vsg volumeSnapshotterGetter) GetVolumeSnapshotter(name string) (vsv1.VolumeSnapshotter, error) { snapshotter, ok := vsg[name] if !ok { return nil, errors.New("volume snapshotter not found") } return snapshotter, nil } func int64Ptr(val int) *int64 { i := int64(val) return &i } type volumeIdentifier struct { volumeID string volumeAZ string } type volumeInfo struct { volumeType string iops *int64 snapshotErr bool } // fakeVolumeSnapshotter is a test fake for the vsv1.VolumeSnapshotter interface. type fakeVolumeSnapshotter struct { // PVVolumeNames is a map from PV name to volume ID, used as the basis // for the GetVolumeID method. PVVolumeNames map[string]string // Volumes is a map from volume identifier (volume ID + AZ) to a struct // of volume info, used for the GetVolumeInfo and CreateSnapshot methods. Volumes map[volumeIdentifier]*volumeInfo } // WithVolume is a test helper for registering persistent volumes that the // fakeVolumeSnapshotter should handle. func (vs *fakeVolumeSnapshotter) WithVolume(pvName, id, az, volumeType string, iops int, snapshotErr bool) *fakeVolumeSnapshotter { if vs.PVVolumeNames == nil { vs.PVVolumeNames = make(map[string]string) } vs.PVVolumeNames[pvName] = id if vs.Volumes == nil { vs.Volumes = make(map[volumeIdentifier]*volumeInfo) } identifier := volumeIdentifier{ volumeID: id, volumeAZ: az, } vs.Volumes[identifier] = &volumeInfo{ volumeType: volumeType, iops: int64Ptr(iops), snapshotErr: snapshotErr, } return vs } // Init is a no-op. func (*fakeVolumeSnapshotter) Init(config map[string]string) error { return nil } // GetVolumeID looks up the PV name in the PVVolumeNames map and returns the result // if found, or an error otherwise. func (vs *fakeVolumeSnapshotter) GetVolumeID(pv runtime.Unstructured) (string, error) { obj := pv.(*unstructured.Unstructured) volumeID, ok := vs.PVVolumeNames[obj.GetName()] if !ok { return "", errors.New("unsupported volume type") } return volumeID, nil } // CreateSnapshot looks up the volume in the Volume map. If it's not found, an error is // returned; if snapshotErr is true on the result, an error is returned; otherwise, // a snapshotID of "-snapshot" is returned. func (vs *fakeVolumeSnapshotter) CreateSnapshot(volumeID, volumeAZ string, tags map[string]string) (snapshotID string, err error) { vi, ok := vs.Volumes[volumeIdentifier{volumeID: volumeID, volumeAZ: volumeAZ}] if !ok { return "", errors.New("volume not found") } if vi.snapshotErr { return "", errors.New("error calling CreateSnapshot") } return volumeID + "-snapshot", nil } // GetVolumeInfo returns volume info if it exists in the Volumes map // for the specified volume ID and AZ, or an error otherwise. func (vs *fakeVolumeSnapshotter) GetVolumeInfo(volumeID, volumeAZ string) (string, *int64, error) { vi, ok := vs.Volumes[volumeIdentifier{volumeID: volumeID, volumeAZ: volumeAZ}] if !ok { return "", nil, errors.New("volume not found") } return vi.volumeType, vi.iops, nil } // CreateVolumeFromSnapshot panics because it's not expected to be used for backups. func (*fakeVolumeSnapshotter) CreateVolumeFromSnapshot(snapshotID, volumeType, volumeAZ string, iops *int64) (volumeID string, err error) { panic("CreateVolumeFromSnapshot should not be used for backups") } // SetVolumeID panics because it's not expected to be used for backups. func (*fakeVolumeSnapshotter) SetVolumeID(pv runtime.Unstructured, volumeID string) (runtime.Unstructured, error) { panic("SetVolumeID should not be used for backups") } // DeleteSnapshot panics because it's not expected to be used for backups. func (*fakeVolumeSnapshotter) DeleteSnapshot(snapshotID string) error { panic("DeleteSnapshot should not be used for backups") } // TestBackupWithSnapshots runs backups with volume snapshot locations and volume snapshotters // configured and verifies that snapshots are created as appropriate. Verification is done by // looking at the backup request's VolumeSnapshots field. This test uses the fakeVolumeSnapshotter // struct in place of real volume snapshotters. func TestBackupWithSnapshots(t *testing.T) { // TODO: add more verification for skippedPVTracker itemBlockPool := StartItemBlockWorkerPool(t.Context(), 1, logrus.StandardLogger()) defer itemBlockPool.Stop() tests := []struct { name string req *Request vsls []*velerov1.VolumeSnapshotLocation apiResources []*test.APIResource snapshotterGetter volumeSnapshotterGetter want []*volume.Snapshot }{ { name: "persistent volume with no zone annotation creates a snapshot", req: &Request{ Backup: defaultBackup().Result(), SnapshotLocations: []*velerov1.VolumeSnapshotLocation{ newSnapshotLocation("velero", "default", "default"), }, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), WorkerPool: itemBlockPool, }, apiResources: []*test.APIResource{ test.PVs( builder.ForPersistentVolume("pv-1").Result(), ), }, snapshotterGetter: map[string]vsv1.VolumeSnapshotter{ "default": new(fakeVolumeSnapshotter).WithVolume("pv-1", "vol-1", "", "type-1", 100, false), }, want: []*volume.Snapshot{ { Spec: volume.SnapshotSpec{ BackupName: "backup-1", Location: "default", PersistentVolumeName: "pv-1", ProviderVolumeID: "vol-1", VolumeType: "type-1", VolumeIOPS: int64Ptr(100), }, Status: volume.SnapshotStatus{ Phase: volume.SnapshotPhaseCompleted, ProviderSnapshotID: "vol-1-snapshot", }, }, }, }, { name: "persistent volume with deprecated zone annotation creates a snapshot", req: &Request{ Backup: defaultBackup().Result(), SnapshotLocations: []*velerov1.VolumeSnapshotLocation{ newSnapshotLocation("velero", "default", "default"), }, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), WorkerPool: itemBlockPool, }, apiResources: []*test.APIResource{ test.PVs( builder.ForPersistentVolume("pv-1").ObjectMeta(builder.WithLabels("failure-domain.beta.kubernetes.io/zone", "zone-1")).Result(), ), }, snapshotterGetter: map[string]vsv1.VolumeSnapshotter{ "default": new(fakeVolumeSnapshotter).WithVolume("pv-1", "vol-1", "zone-1", "type-1", 100, false), }, want: []*volume.Snapshot{ { Spec: volume.SnapshotSpec{ BackupName: "backup-1", Location: "default", PersistentVolumeName: "pv-1", ProviderVolumeID: "vol-1", VolumeAZ: "zone-1", VolumeType: "type-1", VolumeIOPS: int64Ptr(100), }, Status: volume.SnapshotStatus{ Phase: volume.SnapshotPhaseCompleted, ProviderSnapshotID: "vol-1-snapshot", }, }, }, }, { name: "persistent volume with GA zone annotation creates a snapshot", req: &Request{ Backup: defaultBackup().Result(), SnapshotLocations: []*velerov1.VolumeSnapshotLocation{ newSnapshotLocation("velero", "default", "default"), }, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), WorkerPool: itemBlockPool, }, apiResources: []*test.APIResource{ test.PVs( builder.ForPersistentVolume("pv-1").ObjectMeta(builder.WithLabels("topology.kubernetes.io/zone", "zone-1")).Result(), ), }, snapshotterGetter: map[string]vsv1.VolumeSnapshotter{ "default": new(fakeVolumeSnapshotter).WithVolume("pv-1", "vol-1", "zone-1", "type-1", 100, false), }, want: []*volume.Snapshot{ { Spec: volume.SnapshotSpec{ BackupName: "backup-1", Location: "default", PersistentVolumeName: "pv-1", ProviderVolumeID: "vol-1", VolumeAZ: "zone-1", VolumeType: "type-1", VolumeIOPS: int64Ptr(100), }, Status: volume.SnapshotStatus{ Phase: volume.SnapshotPhaseCompleted, ProviderSnapshotID: "vol-1-snapshot", }, }, }, }, { name: "persistent volume with both GA and deprecated zone annotation creates a snapshot and should use the GA", req: &Request{ Backup: defaultBackup().Result(), SnapshotLocations: []*velerov1.VolumeSnapshotLocation{ newSnapshotLocation("velero", "default", "default"), }, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), WorkerPool: itemBlockPool, }, apiResources: []*test.APIResource{ test.PVs( builder.ForPersistentVolume("pv-1").ObjectMeta(builder.WithLabelsMap(map[string]string{"failure-domain.beta.kubernetes.io/zone": "zone-1-deprecated", "topology.kubernetes.io/zone": "zone-1-ga"})).Result(), ), }, snapshotterGetter: map[string]vsv1.VolumeSnapshotter{ "default": new(fakeVolumeSnapshotter).WithVolume("pv-1", "vol-1", "zone-1-ga", "type-1", 100, false), }, want: []*volume.Snapshot{ { Spec: volume.SnapshotSpec{ BackupName: "backup-1", Location: "default", PersistentVolumeName: "pv-1", ProviderVolumeID: "vol-1", VolumeAZ: "zone-1-ga", VolumeType: "type-1", VolumeIOPS: int64Ptr(100), }, Status: volume.SnapshotStatus{ Phase: volume.SnapshotPhaseCompleted, ProviderSnapshotID: "vol-1-snapshot", }, }, }, }, { name: "error returned from CreateSnapshot results in a failed snapshot", req: &Request{ Backup: defaultBackup().Result(), SnapshotLocations: []*velerov1.VolumeSnapshotLocation{ newSnapshotLocation("velero", "default", "default"), }, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), WorkerPool: itemBlockPool, }, apiResources: []*test.APIResource{ test.PVs( builder.ForPersistentVolume("pv-1").Result(), ), }, snapshotterGetter: map[string]vsv1.VolumeSnapshotter{ "default": new(fakeVolumeSnapshotter).WithVolume("pv-1", "vol-1", "", "type-1", 100, true), }, want: []*volume.Snapshot{ { Spec: volume.SnapshotSpec{ BackupName: "backup-1", Location: "default", PersistentVolumeName: "pv-1", ProviderVolumeID: "vol-1", VolumeType: "type-1", VolumeIOPS: int64Ptr(100), }, Status: volume.SnapshotStatus{ Phase: volume.SnapshotPhaseFailed, }, }, }, }, { name: "backup with SnapshotVolumes=false does not create any snapshots", req: &Request{ Backup: defaultBackup().SnapshotVolumes(false).Result(), SnapshotLocations: []*velerov1.VolumeSnapshotLocation{ newSnapshotLocation("velero", "default", "default"), }, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), WorkerPool: itemBlockPool, }, apiResources: []*test.APIResource{ test.PVs( builder.ForPersistentVolume("pv-1").Result(), ), }, snapshotterGetter: map[string]vsv1.VolumeSnapshotter{ "default": new(fakeVolumeSnapshotter).WithVolume("pv-1", "vol-1", "", "type-1", 100, false), }, want: nil, }, { name: "backup with no volume snapshot locations does not create any snapshots", req: &Request{ Backup: defaultBackup().Result(), SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), WorkerPool: itemBlockPool, }, apiResources: []*test.APIResource{ test.PVs( builder.ForPersistentVolume("pv-1").Result(), ), }, snapshotterGetter: map[string]vsv1.VolumeSnapshotter{ "default": new(fakeVolumeSnapshotter).WithVolume("pv-1", "vol-1", "", "type-1", 100, false), }, want: nil, }, { name: "backup with no volume snapshotters does not create any snapshots", req: &Request{ Backup: defaultBackup().Result(), SnapshotLocations: []*velerov1.VolumeSnapshotLocation{ newSnapshotLocation("velero", "default", "default"), }, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), WorkerPool: itemBlockPool, }, apiResources: []*test.APIResource{ test.PVs( builder.ForPersistentVolume("pv-1").Result(), ), }, snapshotterGetter: map[string]vsv1.VolumeSnapshotter{}, want: nil, }, { name: "unsupported persistent volume type does not create any snapshots", req: &Request{ Backup: defaultBackup().Result(), SnapshotLocations: []*velerov1.VolumeSnapshotLocation{ newSnapshotLocation("velero", "default", "default"), }, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), WorkerPool: itemBlockPool, }, apiResources: []*test.APIResource{ test.PVs( builder.ForPersistentVolume("pv-1").Result(), ), }, snapshotterGetter: map[string]vsv1.VolumeSnapshotter{ "default": new(fakeVolumeSnapshotter), }, want: nil, }, { name: "when there are multiple volumes, snapshot locations, and snapshotters, volumes are matched to the right snapshotters", req: &Request{ Backup: defaultBackup().Result(), SnapshotLocations: []*velerov1.VolumeSnapshotLocation{ newSnapshotLocation("velero", "default", "default"), newSnapshotLocation("velero", "another", "another"), }, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), WorkerPool: itemBlockPool, }, apiResources: []*test.APIResource{ test.PVs( builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result(), ), }, snapshotterGetter: map[string]vsv1.VolumeSnapshotter{ "default": new(fakeVolumeSnapshotter).WithVolume("pv-1", "vol-1", "", "type-1", 100, false), "another": new(fakeVolumeSnapshotter).WithVolume("pv-2", "vol-2", "", "type-2", 100, false), }, want: []*volume.Snapshot{ { Spec: volume.SnapshotSpec{ BackupName: "backup-1", Location: "default", PersistentVolumeName: "pv-1", ProviderVolumeID: "vol-1", VolumeType: "type-1", VolumeIOPS: int64Ptr(100), }, Status: volume.SnapshotStatus{ Phase: volume.SnapshotPhaseCompleted, ProviderSnapshotID: "vol-1-snapshot", }, }, { Spec: volume.SnapshotSpec{ BackupName: "backup-1", Location: "another", PersistentVolumeName: "pv-2", ProviderVolumeID: "vol-2", VolumeType: "type-2", VolumeIOPS: int64Ptr(100), }, Status: volume.SnapshotStatus{ Phase: volume.SnapshotPhaseCompleted, ProviderSnapshotID: "vol-2-snapshot", }, }, }, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { var ( h = newHarness(t, itemBlockPool) backupFile = bytes.NewBuffer([]byte{}) ) for _, resource := range tc.apiResources { h.addItems(t, resource) } err := h.backupper.Backup(h.log, tc.req, backupFile, nil, nil, tc.snapshotterGetter) require.NoError(t, err) assert.Equal(t, tc.want, tc.req.VolumeSnapshots.Get()) }) } } // TestBackupWithAsyncOperations runs backups which return operationIDs and // verifies that the itemoperations are tracked as appropriate. Verification is done by // looking at the backup request's itemOperationsList field. func TestBackupWithAsyncOperations(t *testing.T) { // completedOperationAction is a *pluggableAction, whose Execute(...) // method returns an operationID which will always be done when calling Progress. completedOperationAction := &pluggableAction{ executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { obj, ok := item.(*unstructured.Unstructured) if !ok { return nil, nil, "", nil, errors.Errorf("unexpected type %T", item) } return obj, nil, obj.GetName() + "-1", nil, nil }, progressFunc: func(operationID string, backup *velerov1.Backup) (velero.OperationProgress, error) { return velero.OperationProgress{ Completed: true, Description: "Done!", }, nil }, } // incompleteOperationAction is a *pluggableAction, whose Execute(...) // method returns an operationID which will never be done when calling Progress. incompleteOperationAction := &pluggableAction{ executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { obj, ok := item.(*unstructured.Unstructured) if !ok { return nil, nil, "", nil, errors.Errorf("unexpected type %T", item) } return obj, nil, obj.GetName() + "-1", nil, nil }, progressFunc: func(operationID string, backup *velerov1.Backup) (velero.OperationProgress, error) { return velero.OperationProgress{ Completed: false, Description: "Working...", }, nil }, } // noOperationAction is a *pluggableAction, whose Execute(...) // method does not return an operationID. noOperationAction := &pluggableAction{ executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { obj, ok := item.(*unstructured.Unstructured) if !ok { return nil, nil, "", nil, errors.Errorf("unexpected type %T", item) } return obj, nil, "", nil, nil }, } itemBlockPool := StartItemBlockWorkerPool(t.Context(), 1, logrus.StandardLogger()) defer itemBlockPool.Stop() tests := []struct { name string req *Request apiResources []*test.APIResource actions []biav2.BackupItemAction want []*itemoperation.BackupOperation }{ { name: "action that starts a short-running process records operation", req: &Request{ Backup: defaultBackup().Result(), SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), WorkerPool: itemBlockPool, }, apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").Result(), ), }, actions: []biav2.BackupItemAction{ completedOperationAction, }, want: []*itemoperation.BackupOperation{ { Spec: itemoperation.BackupOperationSpec{ BackupName: "backup-1", ResourceIdentifier: velero.ResourceIdentifier{ GroupResource: kuberesource.Pods, Namespace: "ns-1", Name: "pod-1"}, OperationID: "pod-1-1", }, Status: itemoperation.OperationStatus{ Phase: "New", }, }, }, }, { name: "action that starts a long-running process records operation", req: &Request{ Backup: defaultBackup().Result(), SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), WorkerPool: itemBlockPool, }, apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-2").Result(), ), }, actions: []biav2.BackupItemAction{ incompleteOperationAction, }, want: []*itemoperation.BackupOperation{ { Spec: itemoperation.BackupOperationSpec{ BackupName: "backup-1", ResourceIdentifier: velero.ResourceIdentifier{ GroupResource: kuberesource.Pods, Namespace: "ns-1", Name: "pod-2"}, OperationID: "pod-2-1", }, Status: itemoperation.OperationStatus{ Phase: "New", }, }, }, }, { name: "action that has no operation doesn't record one", req: &Request{ Backup: defaultBackup().Result(), SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), WorkerPool: itemBlockPool, }, apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-3").Result(), ), }, actions: []biav2.BackupItemAction{ noOperationAction, }, want: []*itemoperation.BackupOperation{}, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { var ( h = newHarness(t, itemBlockPool) backupFile = bytes.NewBuffer([]byte{}) ) for _, resource := range tc.apiResources { h.addItems(t, resource) } err := h.backupper.Backup(h.log, tc.req, backupFile, tc.actions, nil, nil) require.NoError(t, err) resultOper := *tc.req.GetItemOperationsList() // set want Created times so it won't fail the assert.Equal test for i, wantOper := range tc.want { wantOper.Status.Created = resultOper[i].Status.Created } assert.Equal(t, tc.want, *tc.req.GetItemOperationsList()) }) } } // TestBackupWithInvalidHooks runs backups with invalid hook specifications and verifies // that an error is returned. func TestBackupWithInvalidHooks(t *testing.T) { tests := []struct { name string backup *velerov1.Backup apiResources []*test.APIResource want error }{ { name: "hook with invalid label selector causes backup to fail", backup: defaultBackup(). Hooks(velerov1.BackupHooks{ Resources: []velerov1.BackupResourceHookSpec{ { Name: "hook-with-invalid-label-selector", LabelSelector: &metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { Key: "foo", Operator: metav1.LabelSelectorOperator("nonexistent-operator"), Values: []string{"bar"}, }, }, }, }, }, }). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("foo", "bar").Result(), ), }, want: errors.New("\"nonexistent-operator\" is not a valid label selector operator"), }, } itemBlockPool := StartItemBlockWorkerPool(t.Context(), 1, logrus.StandardLogger()) defer itemBlockPool.Stop() for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { var ( h = newHarness(t, itemBlockPool) req = &Request{ Backup: tc.backup, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), WorkerPool: itemBlockPool, } backupFile = bytes.NewBuffer([]byte{}) ) for _, resource := range tc.apiResources { h.addItems(t, resource) } assert.EqualError(t, h.backupper.Backup(h.log, req, backupFile, nil, nil, nil), tc.want.Error()) }) } } // TestBackupWithHooks runs backups with valid hook specifications and verifies that the // hooks are run. It uses a MockPodCommandExecutor since hooks can't actually be executed // in running pods during the unit test. Verification is done by asserting expected method // calls on the mock object. func TestBackupWithHooks(t *testing.T) { type expectedCall struct { podNamespace string podName string hookName string hook *velerov1.ExecHook err error } tests := []struct { name string backup *velerov1.Backup apiResources []*test.APIResource actions []ibav1.ItemBlockAction wantExecutePodCommandCalls []*expectedCall wantBackedUp []string wantHookExecutionLog []test.HookExecutionEntry }{ { name: "pre hook with no resource filters runs for all pods", backup: defaultBackup(). Hooks(velerov1.BackupHooks{ Resources: []velerov1.BackupResourceHookSpec{ { Name: "hook-1", PreHooks: []velerov1.BackupResourceHook{ { Exec: &velerov1.ExecHook{ Command: []string{"ls", "/tmp"}, }, }, }, }, }, }). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result(), ), }, wantExecutePodCommandCalls: []*expectedCall{ { podNamespace: "ns-1", podName: "pod-1", hookName: "hook-1", hook: &velerov1.ExecHook{ Command: []string{"ls", "/tmp"}, }, err: nil, }, { podNamespace: "ns-2", podName: "pod-2", hookName: "hook-1", hook: &velerov1.ExecHook{ Command: []string{"ls", "/tmp"}, }, err: nil, }, }, wantBackedUp: []string{ "resources/pods/namespaces/ns-1/pod-1.json", "resources/pods/namespaces/ns-2/pod-2.json", "resources/pods/v1-preferredversion/namespaces/ns-1/pod-1.json", "resources/pods/v1-preferredversion/namespaces/ns-2/pod-2.json", }, }, { name: "post hook with no resource filters runs for all pods", backup: defaultBackup(). Hooks(velerov1.BackupHooks{ Resources: []velerov1.BackupResourceHookSpec{ { Name: "hook-1", PostHooks: []velerov1.BackupResourceHook{ { Exec: &velerov1.ExecHook{ Command: []string{"ls", "/tmp"}, }, }, }, }, }, }). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result(), ), }, wantExecutePodCommandCalls: []*expectedCall{ { podNamespace: "ns-1", podName: "pod-1", hookName: "hook-1", hook: &velerov1.ExecHook{ Command: []string{"ls", "/tmp"}, }, err: nil, }, { podNamespace: "ns-2", podName: "pod-2", hookName: "hook-1", hook: &velerov1.ExecHook{ Command: []string{"ls", "/tmp"}, }, err: nil, }, }, wantBackedUp: []string{ "resources/pods/namespaces/ns-1/pod-1.json", "resources/pods/namespaces/ns-2/pod-2.json", "resources/pods/v1-preferredversion/namespaces/ns-1/pod-1.json", "resources/pods/v1-preferredversion/namespaces/ns-2/pod-2.json", }, }, { name: "pre and post hooks run for a pod", backup: defaultBackup(). Hooks(velerov1.BackupHooks{ Resources: []velerov1.BackupResourceHookSpec{ { Name: "hook-1", PreHooks: []velerov1.BackupResourceHook{ { Exec: &velerov1.ExecHook{ Command: []string{"pre"}, }, }, }, PostHooks: []velerov1.BackupResourceHook{ { Exec: &velerov1.ExecHook{ Command: []string{"post"}, }, }, }, }, }, }). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").Result(), ), }, wantExecutePodCommandCalls: []*expectedCall{ { podNamespace: "ns-1", podName: "pod-1", hookName: "hook-1", hook: &velerov1.ExecHook{ Command: []string{"pre"}, }, err: nil, }, { podNamespace: "ns-1", podName: "pod-1", hookName: "hook-1", hook: &velerov1.ExecHook{ Command: []string{"post"}, }, err: nil, }, }, wantBackedUp: []string{ "resources/pods/namespaces/ns-1/pod-1.json", "resources/pods/v1-preferredversion/namespaces/ns-1/pod-1.json", }, }, { name: "pre and post hooks run for two pods sequentially by pods in different ItemBlocks", backup: defaultBackup(). Hooks(velerov1.BackupHooks{ Resources: []velerov1.BackupResourceHookSpec{ { Name: "hook-1", PreHooks: []velerov1.BackupResourceHook{ { Exec: &velerov1.ExecHook{ Command: []string{"pre"}, }, }, }, PostHooks: []velerov1.BackupResourceHook{ { Exec: &velerov1.ExecHook{ Command: []string{"post"}, }, }, }, }, }, }). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-1", "pod-2").Result(), ), }, wantExecutePodCommandCalls: []*expectedCall{ { podNamespace: "ns-1", podName: "pod-1", hookName: "hook-1", hook: &velerov1.ExecHook{ Command: []string{"pre"}, }, err: nil, }, { podNamespace: "ns-1", podName: "pod-1", hookName: "hook-1", hook: &velerov1.ExecHook{ Command: []string{"post"}, }, err: nil, }, { podNamespace: "ns-1", podName: "pod-2", hookName: "hook-1", hook: &velerov1.ExecHook{ Command: []string{"pre"}, }, err: nil, }, { podNamespace: "ns-1", podName: "pod-2", hookName: "hook-1", hook: &velerov1.ExecHook{ Command: []string{"post"}, }, err: nil, }, }, wantHookExecutionLog: []test.HookExecutionEntry{ { Namespace: "ns-1", Name: "pod-1", HookName: "hook-1", HookCommand: []string{"pre"}, }, { Namespace: "ns-1", Name: "pod-1", HookName: "hook-1", HookCommand: []string{"post"}, }, { Namespace: "ns-1", Name: "pod-2", HookName: "hook-1", HookCommand: []string{"pre"}, }, { Namespace: "ns-1", Name: "pod-2", HookName: "hook-1", HookCommand: []string{"post"}, }, }, wantBackedUp: []string{ "resources/pods/namespaces/ns-1/pod-1.json", "resources/pods/namespaces/ns-1/pod-2.json", "resources/pods/v1-preferredversion/namespaces/ns-1/pod-1.json", "resources/pods/v1-preferredversion/namespaces/ns-1/pod-2.json", }, }, { name: "both pre hooks run before both post hooks for pods in the same ItemBlock", backup: defaultBackup(). Hooks(velerov1.BackupHooks{ Resources: []velerov1.BackupResourceHookSpec{ { Name: "hook-1", PreHooks: []velerov1.BackupResourceHook{ { Exec: &velerov1.ExecHook{ Command: []string{"pre"}, }, }, }, PostHooks: []velerov1.BackupResourceHook{ { Exec: &velerov1.ExecHook{ Command: []string{"post"}, }, }, }, }, }, }). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-1", "pod-2").Result(), ), }, actions: []ibav1.ItemBlockAction{ &pluggableIBA{ selector: velero.ResourceSelector{IncludedNamespaces: []string{"ns-1"}}, getRelatedItemsFunc: func(item runtime.Unstructured, backup *velerov1.Backup) ([]velero.ResourceIdentifier, error) { relatedItems := []velero.ResourceIdentifier{ {GroupResource: kuberesource.Pods, Namespace: "ns-1", Name: "pod-1"}, {GroupResource: kuberesource.Pods, Namespace: "ns-1", Name: "pod-2"}, } return relatedItems, nil }, }, }, wantExecutePodCommandCalls: []*expectedCall{ { podNamespace: "ns-1", podName: "pod-1", hookName: "hook-1", hook: &velerov1.ExecHook{ Command: []string{"pre"}, }, err: nil, }, { podNamespace: "ns-1", podName: "pod-1", hookName: "hook-1", hook: &velerov1.ExecHook{ Command: []string{"post"}, }, err: nil, }, { podNamespace: "ns-1", podName: "pod-2", hookName: "hook-1", hook: &velerov1.ExecHook{ Command: []string{"pre"}, }, err: nil, }, { podNamespace: "ns-1", podName: "pod-2", hookName: "hook-1", hook: &velerov1.ExecHook{ Command: []string{"post"}, }, err: nil, }, }, wantHookExecutionLog: []test.HookExecutionEntry{ { Namespace: "ns-1", Name: "pod-1", HookName: "hook-1", HookCommand: []string{"pre"}, }, { Namespace: "ns-1", Name: "pod-2", HookName: "hook-1", HookCommand: []string{"pre"}, }, { Namespace: "ns-1", Name: "pod-1", HookName: "hook-1", HookCommand: []string{"post"}, }, { Namespace: "ns-1", Name: "pod-2", HookName: "hook-1", HookCommand: []string{"post"}, }, }, wantBackedUp: []string{ "resources/pods/namespaces/ns-1/pod-1.json", "resources/pods/namespaces/ns-1/pod-2.json", "resources/pods/v1-preferredversion/namespaces/ns-1/pod-1.json", "resources/pods/v1-preferredversion/namespaces/ns-1/pod-2.json", }, }, { name: "item is not backed up if hook returns an error when OnError=Fail", backup: defaultBackup(). Hooks(velerov1.BackupHooks{ Resources: []velerov1.BackupResourceHookSpec{ { Name: "hook-1", PreHooks: []velerov1.BackupResourceHook{ { Exec: &velerov1.ExecHook{ Command: []string{"ls", "/tmp"}, OnError: velerov1.HookErrorModeFail, }, }, }, }, }, }). Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result(), ), }, wantExecutePodCommandCalls: []*expectedCall{ { podNamespace: "ns-1", podName: "pod-1", hookName: "hook-1", hook: &velerov1.ExecHook{ Command: []string{"ls", "/tmp"}, OnError: velerov1.HookErrorModeFail, }, err: errors.New("exec hook error"), }, { podNamespace: "ns-2", podName: "pod-2", hookName: "hook-1", hook: &velerov1.ExecHook{ Command: []string{"ls", "/tmp"}, OnError: velerov1.HookErrorModeFail, }, err: nil, }, }, wantBackedUp: []string{ "resources/pods/namespaces/ns-2/pod-2.json", "resources/pods/v1-preferredversion/namespaces/ns-2/pod-2.json", }, }, } itemBlockPool := StartItemBlockWorkerPool(t.Context(), 1, logrus.StandardLogger()) defer itemBlockPool.Stop() for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { var ( h = newHarness(t, itemBlockPool) req = &Request{ Backup: tc.backup, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), WorkerPool: itemBlockPool, } backupFile = bytes.NewBuffer([]byte{}) podCommandExecutor = new(test.MockPodCommandExecutor) ) h.backupper.podCommandExecutor = podCommandExecutor defer podCommandExecutor.AssertExpectations(t) for _, expect := range tc.wantExecutePodCommandCalls { podCommandExecutor.On("ExecutePodCommand", mock.Anything, mock.Anything, expect.podNamespace, expect.podName, expect.hookName, expect.hook, ).Return(expect.err) } for _, resource := range tc.apiResources { h.addItems(t, resource) } require.NoError(t, h.backupper.Backup(h.log, req, backupFile, nil, tc.actions, nil)) if tc.wantHookExecutionLog != nil { assert.Equal(t, tc.wantHookExecutionLog, podCommandExecutor.HookExecutionLog) } assertTarballContents(t, backupFile, append(tc.wantBackedUp, "metadata/version")...) }) } } type fakePodVolumeBackupperFactory struct{} func (f *fakePodVolumeBackupperFactory) NewBackupper(context.Context, logrus.FieldLogger, *velerov1.Backup, string) (podvolume.Backupper, error) { return &fakePodVolumeBackupper{}, nil } type fakePodVolumeBackupper struct { pvbs []*velerov1.PodVolumeBackup } // BackupPodVolumes returns one pod volume backup per entry in volumes, with namespace "velero" // and name "pvb---". func (b *fakePodVolumeBackupper) BackupPodVolumes(backup *velerov1.Backup, pod *corev1api.Pod, volumes []string, _ *resourcepolicies.Policies, _ logrus.FieldLogger) ([]*velerov1.PodVolumeBackup, *podvolume.PVCBackupSummary, []error) { var res []*velerov1.PodVolumeBackup pvcSummary := podvolume.NewPVCBackupSummary() anno := pod.GetAnnotations() if anno != nil && anno["backup.velero.io/bakupper-skip"] != "" { return res, pvcSummary, nil } for _, vol := range volumes { pvb := builder.ForPodVolumeBackup("velero", fmt.Sprintf("pvb-%s-%s-%s", pod.Namespace, pod.Name, vol)).Volume(vol).Result() res = append(res, pvb) } b.pvbs = res return res, pvcSummary, nil } func (b *fakePodVolumeBackupper) WaitAllPodVolumesProcessed(log logrus.FieldLogger) []*velerov1.PodVolumeBackup { return b.pvbs } func (b *fakePodVolumeBackupper) GetPodVolumeBackupByPodAndVolume(podNamespace, podName, volume string) (*velerov1.PodVolumeBackup, error) { for _, pvb := range b.pvbs { if pvb.Spec.Pod.Namespace == podNamespace && pvb.Spec.Pod.Name == podName && pvb.Spec.Volume == volume { return pvb, nil } } return nil, nil } func (b *fakePodVolumeBackupper) ListPodVolumeBackupsByPod(podNamespace, podName string) ([]*velerov1.PodVolumeBackup, error) { var pvbs []*velerov1.PodVolumeBackup for _, pvb := range b.pvbs { if pvb.Spec.Pod.Namespace == podNamespace && pvb.Spec.Pod.Name == podName { pvbs = append(pvbs, pvb) } } return pvbs, nil } // TestBackupWithPodVolume runs backups of pods that are annotated for PodVolume backup, // and ensures that the pod volume backupper is called, that the returned PodVolumeBackups // are added to the Request object, and that when PVCs are backed up with PodVolume, the // claimed PVs are not also snapshotted using a VolumeSnapshotter. func TestBackupWithPodVolume(t *testing.T) { tests := []struct { name string backup *velerov1.Backup apiResources []*test.APIResource pod *corev1api.Pod vsl *velerov1.VolumeSnapshotLocation snapshotterGetter volumeSnapshotterGetter want []*velerov1.PodVolumeBackup }{ { name: "a pod annotated for pod volume backup should result in pod volume backups being returned", backup: defaultBackup().Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1"). ObjectMeta(builder.WithAnnotations("backup.velero.io/backup-volumes", "foo")). Volumes(&corev1api.Volume{ Name: "foo", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "foo", }, }, }). Result(), ), test.PVCs(), }, want: []*velerov1.PodVolumeBackup{ builder.ForPodVolumeBackup("velero", "pvb-ns-1-pod-1-foo").Volume("foo").Result(), }, }, { name: "when a PVC is used by two pods and annotated for pod volume backup on both, only one should be backed up", backup: defaultBackup().Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1"). ObjectMeta(builder.WithAnnotations("backup.velero.io/backup-volumes", "foo")). Volumes(builder.ForVolume("foo").PersistentVolumeClaimSource("pvc-1").Result()). Result(), ), test.Pods( builder.ForPod("ns-1", "pod-2"). ObjectMeta(builder.WithAnnotations("backup.velero.io/backup-volumes", "bar")). Volumes(builder.ForVolume("bar").PersistentVolumeClaimSource("pvc-1").Result()). Result(), ), test.PVCs(), }, want: []*velerov1.PodVolumeBackup{ builder.ForPodVolumeBackup("velero", "pvb-ns-1-pod-1-foo").Volume("foo").Result(), }, }, { name: "when a PVC is used by two pods and annotated for pod volume backup on both, the backup for pod1 gives up the PVC, the backup for pod2 should include it", backup: defaultBackup().Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1"). ObjectMeta(builder.WithAnnotations("backup.velero.io/backup-volumes", "foo", "backup.velero.io/bakupper-skip", "foo")). Volumes(builder.ForVolume("foo").PersistentVolumeClaimSource("pvc-1").Result()). Result(), ), test.Pods( builder.ForPod("ns-1", "pod-2"). ObjectMeta(builder.WithAnnotations("backup.velero.io/backup-volumes", "bar")). Volumes(builder.ForVolume("bar").PersistentVolumeClaimSource("pvc-1").Result()). Result(), ), test.PVCs(), }, want: []*velerov1.PodVolumeBackup{ builder.ForPodVolumeBackup("velero", "pvb-ns-1-pod-2-bar").Volume("bar").Result(), }, }, { name: "when PVC pod volumes are backed up using pod volume backup, their claimed PVs are not also snapshotted", backup: defaultBackup().Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1"). Volumes( builder.ForVolume("vol-1").PersistentVolumeClaimSource("pvc-1").Result(), builder.ForVolume("vol-2").PersistentVolumeClaimSource("pvc-2").Result(), ). ObjectMeta( builder.WithAnnotations("backup.velero.io/backup-volumes", "vol-1,vol-2"), ). Result(), ), test.PVCs( builder.ForPersistentVolumeClaim("ns-1", "pvc-1").VolumeName("pv-1").Result(), builder.ForPersistentVolumeClaim("ns-1", "pvc-2").VolumeName("pv-2").Result(), ), test.PVs( builder.ForPersistentVolume("pv-1").ClaimRef("ns-1", "pvc-1").Result(), builder.ForPersistentVolume("pv-2").ClaimRef("ns-1", "pvc-2").Result(), ), }, pod: builder.ForPod("ns-1", "pod-1"). Volumes( builder.ForVolume("vol-1").PersistentVolumeClaimSource("pvc-1").Result(), builder.ForVolume("vol-2").PersistentVolumeClaimSource("pvc-2").Result(), ). ObjectMeta( builder.WithAnnotations("backup.velero.io/backup-volumes", "vol-1,vol-2"), ). Result(), vsl: newSnapshotLocation("velero", "default", "default"), snapshotterGetter: map[string]vsv1.VolumeSnapshotter{ "default": new(fakeVolumeSnapshotter). WithVolume("pv-1", "vol-1", "", "type-1", 100, false). WithVolume("pv-2", "vol-2", "", "type-1", 100, false), }, want: []*velerov1.PodVolumeBackup{ builder.ForPodVolumeBackup("velero", "pvb-ns-1-pod-1-vol-1").Volume("vol-1").Result(), builder.ForPodVolumeBackup("velero", "pvb-ns-1-pod-1-vol-2").Volume("vol-2").Result(), }, }, } itemBlockPool := StartItemBlockWorkerPool(t.Context(), 1, logrus.StandardLogger()) defer itemBlockPool.Stop() for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { var ( h = newHarness(t, itemBlockPool) req = &Request{ Backup: tc.backup, SnapshotLocations: []*velerov1.VolumeSnapshotLocation{tc.vsl}, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), WorkerPool: itemBlockPool, } backupFile = bytes.NewBuffer([]byte{}) ) if tc.pod != nil { require.NoError(t, h.backupper.kbClient.Create(t.Context(), tc.pod)) } h.backupper.podVolumeBackupperFactory = new(fakePodVolumeBackupperFactory) for _, resource := range tc.apiResources { h.addItems(t, resource) } require.NoError(t, h.backupper.Backup(h.log, req, backupFile, nil, nil, tc.snapshotterGetter)) assert.Equal(t, tc.want, req.PodVolumeBackups) // this assumes that we don't have any test cases where some PVs should be snapshotted using a VolumeSnapshotter assert.Nil(t, req.VolumeSnapshots.Get()) }) } } // pluggableAction is a backup item action that can be plugged with Execute // and Progress function bodies at runtime. type pluggableAction struct { selector velero.ResourceSelector executeFunc func(runtime.Unstructured, *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) progressFunc func(string, *velerov1.Backup) (velero.OperationProgress, error) } func (a *pluggableAction) Execute(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { if a.executeFunc == nil { return item, nil, "", nil, nil } return a.executeFunc(item, backup) } func (a *pluggableAction) AppliesTo() (velero.ResourceSelector, error) { return a.selector, nil } func (a *pluggableAction) Progress(operationID string, backup *velerov1.Backup) (velero.OperationProgress, error) { if a.progressFunc == nil { return velero.OperationProgress{}, nil } return a.progressFunc(operationID, backup) } func (a *pluggableAction) Cancel(operationID string, backup *velerov1.Backup) error { return nil } func (a *pluggableAction) Name() string { return "" } // pluggableIBA is an ItemBlock action that can be plugged with GetRelatedItems function bodies at runtime. type pluggableIBA struct { selector velero.ResourceSelector getRelatedItemsFunc func(runtime.Unstructured, *velerov1.Backup) ([]velero.ResourceIdentifier, error) } func (a *pluggableIBA) GetRelatedItems(item runtime.Unstructured, backup *velerov1.Backup) ([]velero.ResourceIdentifier, error) { if a.getRelatedItemsFunc == nil { return nil, nil } return a.getRelatedItemsFunc(item, backup) } func (a *pluggableIBA) AppliesTo() (velero.ResourceSelector, error) { return a.selector, nil } func (a *pluggableIBA) Name() string { return "" } type harness struct { *test.APIServer backupper *kubernetesBackupper log logrus.FieldLogger itemBlockPool ItemBlockWorkerPool } func (h *harness) addItems(t *testing.T, resource *test.APIResource) { t.Helper() h.DiscoveryClient.WithAPIResource(resource) require.NoError(t, h.backupper.discoveryHelper.Refresh()) for _, item := range resource.Items { obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(item) require.NoError(t, err) unstructuredObj := &unstructured.Unstructured{Object: obj} if resource.Namespaced { _, err = h.DynamicClient.Resource(resource.GVR()).Namespace(item.GetNamespace()).Create(t.Context(), unstructuredObj, metav1.CreateOptions{}) } else { _, err = h.DynamicClient.Resource(resource.GVR()).Create(t.Context(), unstructuredObj, metav1.CreateOptions{}) } require.NoError(t, err) } } func newHarness(t *testing.T, itemBlockPool *ItemBlockWorkerPool) *harness { t.Helper() apiServer := test.NewAPIServer(t) log := logrus.StandardLogger() discoveryHelper, err := discovery.NewHelper(apiServer.DiscoveryClient, log) require.NoError(t, err) if itemBlockPool == nil { itemBlockPool = StartItemBlockWorkerPool(t.Context(), 1, log) } return &harness{ APIServer: apiServer, backupper: &kubernetesBackupper{ kbClient: test.NewFakeControllerRuntimeClient(t), dynamicFactory: client.NewDynamicFactory(apiServer.DynamicClient), discoveryHelper: discoveryHelper, // unsupported podCommandExecutor: nil, podVolumeBackupperFactory: new(fakePodVolumeBackupperFactory), podVolumeTimeout: 60 * time.Second, }, log: log, itemBlockPool: *itemBlockPool, } } func newSnapshotLocation(ns, name, provider string) *velerov1.VolumeSnapshotLocation { return &velerov1.VolumeSnapshotLocation{ ObjectMeta: metav1.ObjectMeta{ Namespace: ns, Name: name, }, Spec: velerov1.VolumeSnapshotLocationSpec{ Provider: provider, }, } } func defaultBackup() *builder.BackupBuilder { return builder.ForBackup(velerov1.DefaultNamespace, "backup-1").DefaultVolumesToFsBackup(false) } func toUnstructuredOrFail(t *testing.T, obj any) map[string]any { t.Helper() res, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj) require.NoError(t, err) return res } // assertTarballContents verifies that the gzipped tarball stored in the provided // backupFile contains exactly the file names specified. func assertTarballContents(t *testing.T, backupFile io.Reader, items ...string) { t.Helper() gzr, err := gzip.NewReader(backupFile) require.NoError(t, err) r := tar.NewReader(gzr) var files []string for { hdr, err := r.Next() if err == io.EOF { break } require.NoError(t, err) files = append(files, hdr.Name) } sort.Strings(files) sort.Strings(items) assert.Equal(t, items, files) } // unstructuredObject is a type alias to improve readability. type unstructuredObject map[string]any // assertTarballFileContents verifies that the gzipped tarball stored in the provided // backupFile contains the files specified as keys in 'want', and for each of those // files verifies that the content of the file is JSON and is equivalent to the JSON // content stored as values in 'want'. func assertTarballFileContents(t *testing.T, backupFile io.Reader, want map[string]unstructuredObject) { t.Helper() gzr, err := gzip.NewReader(backupFile) require.NoError(t, err) r := tar.NewReader(gzr) items := make(map[string][]byte) for { hdr, err := r.Next() if err == io.EOF { break } require.NoError(t, err) bytes, err := io.ReadAll(r) require.NoError(t, err) items[hdr.Name] = bytes } for name, wantItem := range want { gotData, ok := items[name] assert.True(t, ok, "did not find item %s in tarball", name) if !ok { continue } // json-unmarshal the data from the tarball var got unstructuredObject err := json.Unmarshal(gotData, &got) require.NoError(t, err) if err != nil { continue } assert.Equal(t, wantItem, got) } } // assertTarballOrdering ensures that resources were written to the tarball in the expected // order. Any resources *not* in orderedResources are required to come *after* all resources // in orderedResources, in any order. func assertTarballOrdering(t *testing.T, backupFile io.Reader, orderedResources ...string) { t.Helper() gzr, err := gzip.NewReader(backupFile) require.NoError(t, err) r := tar.NewReader(gzr) // lastSeen tracks the index in 'orderedResources' of the last resource type // we saw in the tarball. Once we've seen a resource in 'orderedResources', // we should never see another instance of a prior resource. lastSeen := 0 for { hdr, err := r.Next() if err == io.EOF { break } require.NoError(t, err) // ignore files like metadata/version if !strings.HasPrefix(hdr.Name, "resources/") { continue } // get the resource name parts := strings.Split(hdr.Name, "/") require.GreaterOrEqual(t, len(parts), 2) resourceName := parts[1] // Find the index in 'orderedResources' of the resource type for // the current tar item, if it exists. This index ('current') *must* // be greater than or equal to 'lastSeen', which was the last resource // we saw, since otherwise the current resource would be out of order. By // initializing current to len(ordered), we're saying that if the resource // is not explicitly in orederedResources, then it must come *after* // all orderedResources. current := len(orderedResources) for i, item := range orderedResources { if item == resourceName { current = i break } } // the index of the current resource must be the same as or greater than the index of // the last resource we saw for the backed-up order to be correct. assert.GreaterOrEqual(t, current, lastSeen, "%s was backed up out of order", resourceName) lastSeen = current } } func TestBackupNewResourceFiltering(t *testing.T) { tests := []struct { name string backup *velerov1.Backup apiResources []*test.APIResource want []string actions []biav2.BackupItemAction }{ { name: "no namespace-scoped resources + some cluster-scoped resources", backup: defaultBackup().IncludedClusterScopedResources("persistentvolumes").ExcludedNamespaceScopedResources("*").Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("foo", "bar").Result(), builder.ForPod("zoo", "raz").Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), builder.ForDeployment("zoo", "raz").Result(), ), test.PVs( builder.ForPersistentVolume("testing").Result(), ), }, want: []string{ "resources/persistentvolumes/cluster/testing.json", "resources/persistentvolumes/v1-preferredversion/cluster/testing.json", }, }, { name: "no namespace-scoped resources + all cluster-scoped resources", backup: defaultBackup().IncludedClusterScopedResources("*").ExcludedNamespaceScopedResources("*").Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("foo", "bar").Result(), builder.ForPod("zoo", "raz").Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), builder.ForDeployment("zoo", "raz").Result(), ), test.PVs( builder.ForPersistentVolume("test1").Result(), builder.ForPersistentVolume("test2").Result(), ), test.CRDs( builder.ForCustomResourceDefinitionV1Beta1("backups.velero.io").Result(), ), }, want: []string{ "resources/customresourcedefinitions.apiextensions.k8s.io/cluster/backups.velero.io.json", "resources/persistentvolumes/cluster/test1.json", "resources/persistentvolumes/cluster/test2.json", "resources/customresourcedefinitions.apiextensions.k8s.io/v1beta1-preferredversion/cluster/backups.velero.io.json", "resources/persistentvolumes/v1-preferredversion/cluster/test1.json", "resources/persistentvolumes/v1-preferredversion/cluster/test2.json", }, }, { name: "some namespace-scoped resources + no cluster-scoped resources 1", backup: defaultBackup().ExcludedClusterScopedResources("*").IncludedNamespaces("foo", "zoo").Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("foo", "bar").Result(), builder.ForPod("zoo", "raz").Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), builder.ForDeployment("zoo", "raz").Result(), ), test.PVs( builder.ForPersistentVolume("test1").Result(), builder.ForPersistentVolume("test2").Result(), ), test.CRDs( builder.ForCustomResourceDefinitionV1Beta1("backups.velero.io").Result(), ), }, want: []string{ "resources/deployments.apps/namespaces/foo/bar.json", "resources/deployments.apps/namespaces/zoo/raz.json", "resources/deployments.apps/v1-preferredversion/namespaces/foo/bar.json", "resources/deployments.apps/v1-preferredversion/namespaces/zoo/raz.json", "resources/pods/namespaces/foo/bar.json", "resources/pods/namespaces/zoo/raz.json", "resources/pods/v1-preferredversion/namespaces/foo/bar.json", "resources/pods/v1-preferredversion/namespaces/zoo/raz.json", }, }, { name: "some namespace-scoped resources + no cluster-scoped resources 2", backup: defaultBackup().ExcludedClusterScopedResources("*").IncludedNamespaceScopedResources("pods", "deployments").Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("foo", "bar").Result(), builder.ForPod("zoo", "raz").Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), builder.ForDeployment("zoo", "raz").Result(), ), test.PVs( builder.ForPersistentVolume("test1").Result(), builder.ForPersistentVolume("test2").Result(), ), test.CRDs( builder.ForCustomResourceDefinitionV1Beta1("backups.velero.io").Result(), ), }, want: []string{ "resources/deployments.apps/namespaces/foo/bar.json", "resources/deployments.apps/namespaces/zoo/raz.json", "resources/deployments.apps/v1-preferredversion/namespaces/foo/bar.json", "resources/deployments.apps/v1-preferredversion/namespaces/zoo/raz.json", "resources/pods/namespaces/foo/bar.json", "resources/pods/namespaces/zoo/raz.json", "resources/pods/v1-preferredversion/namespaces/foo/bar.json", "resources/pods/v1-preferredversion/namespaces/zoo/raz.json", }, }, { name: "some namespace-scoped resources + no cluster-scoped resources 3", backup: defaultBackup().ExcludedClusterScopedResources("*").IncludedNamespaces("foo").IncludedNamespaceScopedResources("pods", "deployments").Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("foo", "bar").Result(), builder.ForPod("zoo", "raz").Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), builder.ForDeployment("zoo", "raz").Result(), ), test.PVs( builder.ForPersistentVolume("test1").Result(), builder.ForPersistentVolume("test2").Result(), ), test.CRDs( builder.ForCustomResourceDefinitionV1Beta1("backups.velero.io").Result(), ), }, want: []string{ "resources/deployments.apps/namespaces/foo/bar.json", "resources/deployments.apps/v1-preferredversion/namespaces/foo/bar.json", "resources/pods/namespaces/foo/bar.json", "resources/pods/v1-preferredversion/namespaces/foo/bar.json", }, }, { name: "some namespace-scoped resources + no cluster-scoped resources 4", backup: defaultBackup().ExcludedClusterScopedResources("*").ExcludedNamespaceScopedResources("pods").Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("foo", "bar").Result(), builder.ForPod("zoo", "raz").Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), builder.ForDeployment("zoo", "raz").Result(), ), test.PVs( builder.ForPersistentVolume("test1").Result(), builder.ForPersistentVolume("test2").Result(), ), test.CRDs( builder.ForCustomResourceDefinitionV1Beta1("backups.velero.io").Result(), ), }, want: []string{ "resources/deployments.apps/namespaces/foo/bar.json", "resources/deployments.apps/namespaces/zoo/raz.json", "resources/deployments.apps/v1-preferredversion/namespaces/foo/bar.json", "resources/deployments.apps/v1-preferredversion/namespaces/zoo/raz.json", }, }, { name: "some namespace-scoped resources + only related cluster-scoped resources 2", backup: defaultBackup().IncludedNamespaces("foo").IncludedNamespaceScopedResources("pods", "persistentvolumeclaims").Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("foo", "bar").Volumes(builder.ForVolume("foo").PersistentVolumeClaimSource("test-1").Result()).Result(), builder.ForPod("zoo", "raz").Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), builder.ForDeployment("zoo", "raz").Result(), ), test.PVCs( builder.ForPersistentVolumeClaim("foo", "test-1").VolumeName("test1").Result(), ), test.PVs( builder.ForPersistentVolume("test1").Result(), builder.ForPersistentVolume("test2").Result(), ), }, want: []string{ "resources/persistentvolumeclaims/namespaces/foo/test-1.json", "resources/persistentvolumeclaims/v1-preferredversion/namespaces/foo/test-1.json", "resources/persistentvolumes/cluster/test1.json", "resources/persistentvolumes/v1-preferredversion/cluster/test1.json", "resources/pods/namespaces/foo/bar.json", "resources/pods/v1-preferredversion/namespaces/foo/bar.json", }, actions: []biav2.BackupItemAction{ &pluggableAction{ selector: velero.ResourceSelector{IncludedResources: []string{"persistentvolumeclaims"}}, executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { additionalItems := []velero.ResourceIdentifier{ {GroupResource: kuberesource.PersistentVolumes, Name: "test1"}, } return item, additionalItems, "", nil, nil }, }, }, }, { name: "some namespace-scoped resources + only related cluster-scoped resources 3", backup: defaultBackup().IncludedNamespaces("foo").ExcludedNamespaceScopedResources("deployments").Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("foo", "bar").Volumes(builder.ForVolume("foo").PersistentVolumeClaimSource("test-1").Result()).Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), builder.ForDeployment("zoo", "raz").Result(), ), test.PVCs( builder.ForPersistentVolumeClaim("foo", "test-1").VolumeName("test1").Result(), ), test.PVs( builder.ForPersistentVolume("test1").Result(), builder.ForPersistentVolume("test2").Result(), ), }, want: []string{ "resources/persistentvolumeclaims/namespaces/foo/test-1.json", "resources/persistentvolumeclaims/v1-preferredversion/namespaces/foo/test-1.json", "resources/persistentvolumes/cluster/test1.json", "resources/persistentvolumes/v1-preferredversion/cluster/test1.json", "resources/pods/namespaces/foo/bar.json", "resources/pods/v1-preferredversion/namespaces/foo/bar.json", }, actions: []biav2.BackupItemAction{ &pluggableAction{ selector: velero.ResourceSelector{IncludedResources: []string{"persistentvolumeclaims"}}, executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { additionalItems := []velero.ResourceIdentifier{ {GroupResource: kuberesource.PersistentVolumes, Name: "test1"}, } return item, additionalItems, "", nil, nil }, }, }, }, { name: "some namespace-scoped resources + some additional cluster-scoped resources 1", backup: defaultBackup().IncludedNamespaces("foo").IncludedClusterScopedResources("customresourcedefinitions").Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("foo", "bar").Result(), builder.ForPod("zoo", "raz").Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), builder.ForDeployment("zoo", "raz").Result(), ), test.PVCs( builder.ForPersistentVolumeClaim("foo", "test-1").VolumeName("test1").Result(), ), test.PVs( builder.ForPersistentVolume("test1").Result(), builder.ForPersistentVolume("test2").Result(), ), test.CRDs( builder.ForCustomResourceDefinitionV1Beta1("backups.velero.io").Result(), ), }, want: []string{ "resources/customresourcedefinitions.apiextensions.k8s.io/cluster/backups.velero.io.json", "resources/customresourcedefinitions.apiextensions.k8s.io/v1beta1-preferredversion/cluster/backups.velero.io.json", "resources/deployments.apps/namespaces/foo/bar.json", "resources/deployments.apps/v1-preferredversion/namespaces/foo/bar.json", "resources/persistentvolumeclaims/namespaces/foo/test-1.json", "resources/persistentvolumeclaims/v1-preferredversion/namespaces/foo/test-1.json", "resources/persistentvolumes/cluster/test1.json", "resources/persistentvolumes/v1-preferredversion/cluster/test1.json", "resources/pods/namespaces/foo/bar.json", "resources/pods/v1-preferredversion/namespaces/foo/bar.json", }, actions: []biav2.BackupItemAction{ &pluggableAction{ selector: velero.ResourceSelector{IncludedResources: []string{"persistentvolumeclaims"}}, executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { additionalItems := []velero.ResourceIdentifier{ {GroupResource: kuberesource.PersistentVolumes, Name: "test1"}, } return item, additionalItems, "", nil, nil }, }, }, }, { name: "some namespace-scoped resources + some additional cluster-scoped resources 2", backup: defaultBackup().IncludedNamespaceScopedResources("persistentvolumeclaims").IncludedClusterScopedResources("customresourcedefinitions").Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("foo", "bar").Result(), builder.ForPod("zoo", "raz").Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), builder.ForDeployment("zoo", "raz").Result(), ), test.PVCs( builder.ForPersistentVolumeClaim("foo", "test-1").VolumeName("test1").Result(), ), test.PVs( builder.ForPersistentVolume("test1").Result(), builder.ForPersistentVolume("test2").Result(), ), test.CRDs( builder.ForCustomResourceDefinitionV1Beta1("backups.velero.io").Result(), ), }, want: []string{ "resources/customresourcedefinitions.apiextensions.k8s.io/cluster/backups.velero.io.json", "resources/customresourcedefinitions.apiextensions.k8s.io/v1beta1-preferredversion/cluster/backups.velero.io.json", "resources/persistentvolumeclaims/namespaces/foo/test-1.json", "resources/persistentvolumeclaims/v1-preferredversion/namespaces/foo/test-1.json", "resources/persistentvolumes/cluster/test1.json", "resources/persistentvolumes/v1-preferredversion/cluster/test1.json", }, actions: []biav2.BackupItemAction{ &pluggableAction{ selector: velero.ResourceSelector{IncludedResources: []string{"persistentvolumeclaims"}}, executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { additionalItems := []velero.ResourceIdentifier{ {GroupResource: kuberesource.PersistentVolumes, Name: "test1"}, } return item, additionalItems, "", nil, nil }, }, }, }, { name: "some namespace-scoped resources + some additional cluster-scoped resources 3", backup: defaultBackup().IncludedNamespaces("foo").IncludedNamespaceScopedResources("pods", "persistentvolumeclaims").IncludedClusterScopedResources("customresourcedefinitions").Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("foo", "bar").Result(), builder.ForPod("zoo", "raz").Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), builder.ForDeployment("zoo", "raz").Result(), ), test.PVCs( builder.ForPersistentVolumeClaim("foo", "test-1").VolumeName("test1").Result(), ), test.PVs( builder.ForPersistentVolume("test1").Result(), builder.ForPersistentVolume("test2").Result(), ), test.CRDs( builder.ForCustomResourceDefinitionV1Beta1("backups.velero.io").Result(), ), }, want: []string{ "resources/customresourcedefinitions.apiextensions.k8s.io/cluster/backups.velero.io.json", "resources/customresourcedefinitions.apiextensions.k8s.io/v1beta1-preferredversion/cluster/backups.velero.io.json", "resources/persistentvolumeclaims/namespaces/foo/test-1.json", "resources/persistentvolumeclaims/v1-preferredversion/namespaces/foo/test-1.json", "resources/persistentvolumes/cluster/test1.json", "resources/persistentvolumes/v1-preferredversion/cluster/test1.json", "resources/pods/namespaces/foo/bar.json", "resources/pods/v1-preferredversion/namespaces/foo/bar.json", }, actions: []biav2.BackupItemAction{ &pluggableAction{ selector: velero.ResourceSelector{IncludedResources: []string{"persistentvolumeclaims"}}, executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { additionalItems := []velero.ResourceIdentifier{ {GroupResource: kuberesource.PersistentVolumes, Name: "test1"}, } return item, additionalItems, "", nil, nil }, }, }, }, { name: "some namespace-scoped resources + some additional cluster-scoped resources 4", backup: defaultBackup().IncludedNamespaces("foo").IncludedNamespaceScopedResources("pods", "persistentvolumeclaims").IncludedClusterScopedResources("*").ExcludedClusterScopedResources("customresourcedefinitions.apiextensions.k8s.io").Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("foo", "bar").Result(), builder.ForPod("zoo", "raz").Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), builder.ForDeployment("zoo", "raz").Result(), ), test.PVCs( builder.ForPersistentVolumeClaim("foo", "test-1").VolumeName("test1").Result(), ), test.PVs( builder.ForPersistentVolume("test1").Result(), builder.ForPersistentVolume("test2").Result(), ), test.CRDs( builder.ForCustomResourceDefinitionV1Beta1("backups.velero.io").Result(), ), }, want: []string{ "resources/persistentvolumeclaims/namespaces/foo/test-1.json", "resources/persistentvolumeclaims/v1-preferredversion/namespaces/foo/test-1.json", "resources/persistentvolumes/cluster/test1.json", "resources/persistentvolumes/cluster/test2.json", "resources/persistentvolumes/v1-preferredversion/cluster/test1.json", "resources/persistentvolumes/v1-preferredversion/cluster/test2.json", "resources/pods/namespaces/foo/bar.json", "resources/pods/v1-preferredversion/namespaces/foo/bar.json", }, actions: []biav2.BackupItemAction{ &pluggableAction{ selector: velero.ResourceSelector{IncludedResources: []string{"persistentvolumeclaims"}}, executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { additionalItems := []velero.ResourceIdentifier{ {GroupResource: kuberesource.PersistentVolumes, Name: "test1"}, } return item, additionalItems, "", nil, nil }, }, }, }, { name: "some namespace-scoped resources + all cluster-scoped resources 1", backup: defaultBackup().IncludedNamespaces("foo").IncludedClusterScopedResources("*").Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("foo", "bar").Result(), builder.ForPod("zoo", "raz").Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), builder.ForDeployment("zoo", "raz").Result(), ), test.PVCs( builder.ForPersistentVolumeClaim("foo", "test-1").VolumeName("test1").Result(), ), test.PVs( builder.ForPersistentVolume("test1").Result(), builder.ForPersistentVolume("test2").Result(), ), }, want: []string{ "resources/deployments.apps/namespaces/foo/bar.json", "resources/deployments.apps/v1-preferredversion/namespaces/foo/bar.json", "resources/persistentvolumeclaims/namespaces/foo/test-1.json", "resources/persistentvolumeclaims/v1-preferredversion/namespaces/foo/test-1.json", "resources/persistentvolumes/cluster/test1.json", "resources/persistentvolumes/v1-preferredversion/cluster/test1.json", "resources/persistentvolumes/cluster/test2.json", "resources/persistentvolumes/v1-preferredversion/cluster/test2.json", "resources/pods/namespaces/foo/bar.json", "resources/pods/v1-preferredversion/namespaces/foo/bar.json", }, }, { name: "some namespace-scoped resources + all cluster-scoped resources 2", backup: defaultBackup().IncludedNamespaceScopedResources("pods").IncludedClusterScopedResources("*").Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("foo", "bar").Result(), builder.ForPod("zoo", "raz").Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), builder.ForDeployment("zoo", "raz").Result(), ), test.PVs( builder.ForPersistentVolume("test1").Result(), builder.ForPersistentVolume("test2").Result(), ), test.CRDs( builder.ForCustomResourceDefinitionV1Beta1("backups.velero.io").Result(), ), }, want: []string{ "resources/customresourcedefinitions.apiextensions.k8s.io/cluster/backups.velero.io.json", "resources/customresourcedefinitions.apiextensions.k8s.io/v1beta1-preferredversion/cluster/backups.velero.io.json", "resources/persistentvolumes/cluster/test1.json", "resources/persistentvolumes/cluster/test2.json", "resources/persistentvolumes/v1-preferredversion/cluster/test1.json", "resources/persistentvolumes/v1-preferredversion/cluster/test2.json", "resources/pods/namespaces/foo/bar.json", "resources/pods/namespaces/zoo/raz.json", "resources/pods/v1-preferredversion/namespaces/foo/bar.json", "resources/pods/v1-preferredversion/namespaces/zoo/raz.json", }, }, { name: "some namespace-scoped resources + all cluster-scoped resources 3", backup: defaultBackup().IncludedNamespaces("foo").IncludedNamespaceScopedResources("pods").IncludedClusterScopedResources("*").Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("foo", "bar").Result(), builder.ForPod("zoo", "raz").Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), builder.ForDeployment("zoo", "raz").Result(), ), test.PVs( builder.ForPersistentVolume("test1").Result(), builder.ForPersistentVolume("test2").Result(), ), test.CRDs( builder.ForCustomResourceDefinitionV1Beta1("backups.velero.io").Result(), ), }, want: []string{ "resources/customresourcedefinitions.apiextensions.k8s.io/cluster/backups.velero.io.json", "resources/customresourcedefinitions.apiextensions.k8s.io/v1beta1-preferredversion/cluster/backups.velero.io.json", "resources/persistentvolumes/cluster/test1.json", "resources/persistentvolumes/cluster/test2.json", "resources/persistentvolumes/v1-preferredversion/cluster/test1.json", "resources/persistentvolumes/v1-preferredversion/cluster/test2.json", "resources/pods/namespaces/foo/bar.json", "resources/pods/v1-preferredversion/namespaces/foo/bar.json", }, }, { name: "all namespace-scoped resources + no cluster-scoped resources", backup: defaultBackup().ExcludedClusterScopedResources("*").Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("foo", "bar").Result(), builder.ForPod("zoo", "raz").Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), builder.ForDeployment("zoo", "raz").Result(), ), test.PVs( builder.ForPersistentVolume("test1").Result(), builder.ForPersistentVolume("test2").Result(), ), test.CRDs( builder.ForCustomResourceDefinitionV1Beta1("backups.velero.io").Result(), ), }, want: []string{ "resources/deployments.apps/namespaces/foo/bar.json", "resources/deployments.apps/namespaces/zoo/raz.json", "resources/deployments.apps/v1-preferredversion/namespaces/foo/bar.json", "resources/deployments.apps/v1-preferredversion/namespaces/zoo/raz.json", "resources/pods/namespaces/foo/bar.json", "resources/pods/namespaces/zoo/raz.json", "resources/pods/v1-preferredversion/namespaces/foo/bar.json", "resources/pods/v1-preferredversion/namespaces/zoo/raz.json", }, }, { name: "all namespace-scoped resources + all cluster-scoped resources", backup: defaultBackup().IncludedClusterScopedResources("*").Result(), apiResources: []*test.APIResource{ test.Pods( builder.ForPod("foo", "bar").Result(), builder.ForPod("zoo", "raz").Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), builder.ForDeployment("zoo", "raz").Result(), ), test.PVs( builder.ForPersistentVolume("test1").Result(), builder.ForPersistentVolume("test2").Result(), ), test.CRDs( builder.ForCustomResourceDefinitionV1Beta1("backups.velero.io").Result(), ), }, want: []string{ "resources/customresourcedefinitions.apiextensions.k8s.io/cluster/backups.velero.io.json", "resources/customresourcedefinitions.apiextensions.k8s.io/v1beta1-preferredversion/cluster/backups.velero.io.json", "resources/deployments.apps/namespaces/foo/bar.json", "resources/deployments.apps/namespaces/zoo/raz.json", "resources/deployments.apps/v1-preferredversion/namespaces/foo/bar.json", "resources/deployments.apps/v1-preferredversion/namespaces/zoo/raz.json", "resources/persistentvolumes/cluster/test1.json", "resources/persistentvolumes/cluster/test2.json", "resources/persistentvolumes/v1-preferredversion/cluster/test1.json", "resources/persistentvolumes/v1-preferredversion/cluster/test2.json", "resources/pods/namespaces/foo/bar.json", "resources/pods/namespaces/zoo/raz.json", "resources/pods/v1-preferredversion/namespaces/foo/bar.json", "resources/pods/v1-preferredversion/namespaces/zoo/raz.json", }, }, { name: "namespace resource should be included even it's not specified in the include list, when IncludedNamespaces has specified value 1", backup: defaultBackup().IncludedNamespaces("foo").IncludedNamespaceScopedResources("Secrets").Result(), apiResources: []*test.APIResource{ test.Secrets( builder.ForSecret("foo", "bar").Result(), builder.ForSecret("zoo", "raz").Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), builder.ForDeployment("zoo", "raz").Result(), ), test.PVs( builder.ForPersistentVolume("foo").Result(), ), test.Namespaces( builder.ForNamespace("foo").Result(), ), }, want: []string{ "resources/namespaces/cluster/foo.json", "resources/namespaces/v1-preferredversion/cluster/foo.json", "resources/secrets/namespaces/foo/bar.json", "resources/secrets/v1-preferredversion/namespaces/foo/bar.json", }, }, { name: "namespace resource should be included even it's not specified in the include list, when IncludedNamespaces has specified value 2", backup: defaultBackup().IncludedNamespaces("foo").IncludedClusterScopedResources("persistentvolumes").Result(), apiResources: []*test.APIResource{ test.Secrets( builder.ForSecret("foo", "bar").Result(), builder.ForSecret("zoo", "raz").Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), builder.ForDeployment("zoo", "raz").Result(), ), test.PVs( builder.ForPersistentVolume("foo").Result(), ), test.Namespaces( builder.ForNamespace("foo").Result(), ), }, want: []string{ "resources/namespaces/cluster/foo.json", "resources/namespaces/v1-preferredversion/cluster/foo.json", "resources/secrets/namespaces/foo/bar.json", "resources/secrets/v1-preferredversion/namespaces/foo/bar.json", "resources/deployments.apps/namespaces/foo/bar.json", "resources/deployments.apps/v1-preferredversion/namespaces/foo/bar.json", "resources/persistentvolumes/cluster/foo.json", "resources/persistentvolumes/v1-preferredversion/cluster/foo.json", }, }, { name: "namespace resource should be included even it's not specified in the include list, when IncludedNamespaces is asterisk.", backup: defaultBackup().IncludedNamespaces("*").IncludedClusterScopedResources("persistentvolumes").Result(), apiResources: []*test.APIResource{ test.Secrets( builder.ForSecret("foo", "bar").Result(), builder.ForSecret("zoo", "raz").Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), builder.ForDeployment("zoo", "raz").Result(), ), test.PVs( builder.ForPersistentVolume("foo").Result(), ), test.Namespaces( builder.ForNamespace("foo").Result(), builder.ForNamespace("zoo").Result(), ), }, want: []string{ "resources/namespaces/cluster/foo.json", "resources/namespaces/v1-preferredversion/cluster/foo.json", "resources/namespaces/cluster/zoo.json", "resources/namespaces/v1-preferredversion/cluster/zoo.json", "resources/secrets/namespaces/foo/bar.json", "resources/secrets/namespaces/zoo/raz.json", "resources/secrets/v1-preferredversion/namespaces/foo/bar.json", "resources/secrets/v1-preferredversion/namespaces/zoo/raz.json", "resources/deployments.apps/namespaces/foo/bar.json", "resources/deployments.apps/namespaces/zoo/raz.json", "resources/deployments.apps/v1-preferredversion/namespaces/foo/bar.json", "resources/deployments.apps/v1-preferredversion/namespaces/zoo/raz.json", "resources/persistentvolumes/cluster/foo.json", "resources/persistentvolumes/v1-preferredversion/cluster/foo.json", }, }, { name: "when all namespace-scoped resources are involved, cluster-scoped resources should be included too", backup: defaultBackup().IncludedNamespaces("*").IncludedNamespaceScopedResources("*").Result(), apiResources: []*test.APIResource{ test.Secrets( builder.ForSecret("foo", "bar").Result(), builder.ForSecret("zoo", "raz").Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), builder.ForDeployment("zoo", "raz").Result(), ), test.PVs( builder.ForPersistentVolume("foo").Result(), builder.ForPersistentVolume("bar").Result(), ), test.Namespaces( builder.ForNamespace("foo").Result(), builder.ForNamespace("zoo").Result(), ), }, want: []string{ "resources/namespaces/cluster/foo.json", "resources/namespaces/v1-preferredversion/cluster/foo.json", "resources/namespaces/cluster/zoo.json", "resources/namespaces/v1-preferredversion/cluster/zoo.json", "resources/secrets/namespaces/foo/bar.json", "resources/secrets/namespaces/zoo/raz.json", "resources/secrets/v1-preferredversion/namespaces/foo/bar.json", "resources/secrets/v1-preferredversion/namespaces/zoo/raz.json", "resources/deployments.apps/namespaces/foo/bar.json", "resources/deployments.apps/namespaces/zoo/raz.json", "resources/deployments.apps/v1-preferredversion/namespaces/foo/bar.json", "resources/deployments.apps/v1-preferredversion/namespaces/zoo/raz.json", "resources/persistentvolumes/cluster/foo.json", "resources/persistentvolumes/v1-preferredversion/cluster/foo.json", "resources/persistentvolumes/cluster/bar.json", "resources/persistentvolumes/v1-preferredversion/cluster/bar.json", }, }, { name: "IncludedNamespaces is asterisk, but not all namespace-scoped resource types are include, additional cluster-scoped resources should not be included.", backup: defaultBackup().IncludedNamespaces("*").IncludedNamespaceScopedResources("secrets").Result(), apiResources: []*test.APIResource{ test.Secrets( builder.ForSecret("foo", "bar").Result(), builder.ForSecret("zoo", "raz").Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), builder.ForDeployment("zoo", "raz").Result(), ), test.PVs( builder.ForPersistentVolume("foo").Result(), builder.ForPersistentVolume("bar").Result(), ), test.Namespaces( builder.ForNamespace("foo").Result(), builder.ForNamespace("zoo").Result(), ), }, want: []string{ "resources/namespaces/cluster/foo.json", "resources/namespaces/v1-preferredversion/cluster/foo.json", "resources/namespaces/cluster/zoo.json", "resources/namespaces/v1-preferredversion/cluster/zoo.json", "resources/secrets/namespaces/foo/bar.json", "resources/secrets/namespaces/zoo/raz.json", "resources/secrets/v1-preferredversion/namespaces/foo/bar.json", "resources/secrets/v1-preferredversion/namespaces/zoo/raz.json", }, }, { name: "Resource's CRD should be included", backup: defaultBackup().IncludedNamespaces("foo").IncludedNamespaceScopedResources("volumesnapshotlocations.velero.io", "backups.velero.io").Result(), apiResources: []*test.APIResource{ test.CRDs( builder.ForCustomResourceDefinitionV1Beta1("backups.velero.io").Result(), builder.ForCustomResourceDefinitionV1Beta1("volumesnapshotlocations.velero.io").Result(), builder.ForCustomResourceDefinitionV1Beta1("test.velero.io").Result(), ), test.VSLs( builder.ForVolumeSnapshotLocation("foo", "bar").Result(), ), test.Backups( builder.ForBackup("zoo", "raz").Result(), ), }, want: []string{ "resources/customresourcedefinitions.apiextensions.k8s.io/cluster/volumesnapshotlocations.velero.io.json", "resources/customresourcedefinitions.apiextensions.k8s.io/v1beta1-preferredversion/cluster/volumesnapshotlocations.velero.io.json", "resources/volumesnapshotlocations.velero.io/namespaces/foo/bar.json", "resources/volumesnapshotlocations.velero.io/v1-preferredversion/namespaces/foo/bar.json", }, }, { name: "Resource's CRD is not included, when CRD is excluded.", backup: defaultBackup().IncludedNamespaces("foo").IncludedNamespaceScopedResources("volumesnapshotlocations.velero.io", "backups.velero.io").ExcludedClusterScopedResources("customresourcedefinitions.apiextensions.k8s.io").Result(), apiResources: []*test.APIResource{ test.CRDs( builder.ForCustomResourceDefinitionV1Beta1("backups.velero.io").Result(), builder.ForCustomResourceDefinitionV1Beta1("volumesnapshotlocations.velero.io").Result(), builder.ForCustomResourceDefinitionV1Beta1("test.velero.io").Result(), ), test.VSLs( builder.ForVolumeSnapshotLocation("foo", "bar").Result(), ), test.Backups( builder.ForBackup("zoo", "raz").Result(), ), }, want: []string{ "resources/volumesnapshotlocations.velero.io/namespaces/foo/bar.json", "resources/volumesnapshotlocations.velero.io/v1-preferredversion/namespaces/foo/bar.json", }, }, } itemBlockPool := StartItemBlockWorkerPool(t.Context(), 1, logrus.StandardLogger()) defer itemBlockPool.Stop() for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { var ( h = newHarness(t, itemBlockPool) req = &Request{ Backup: tc.backup, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), WorkerPool: itemBlockPool, } backupFile = bytes.NewBuffer([]byte{}) ) for _, resource := range tc.apiResources { h.addItems(t, resource) } h.backupper.Backup(h.log, req, backupFile, tc.actions, nil, nil) assertTarballContents(t, backupFile, append(tc.want, "metadata/version")...) }) } } func TestBackupNamespaces(t *testing.T) { tests := []struct { name string backup *velerov1.Backup apiResources []*test.APIResource want []string }{ { name: "LabelSelector test", backup: defaultBackup().LabelSelector(&metav1.LabelSelector{MatchLabels: map[string]string{"a": "b"}}). Result(), apiResources: []*test.APIResource{ test.Namespaces( builder.ForNamespace("ns-1").Phase(corev1api.NamespaceActive).Result(), builder.ForNamespace("ns-2").Phase(corev1api.NamespaceActive).Result(), builder.ForNamespace("ns-3").Phase(corev1api.NamespaceActive).Result(), ), test.Deployments( builder.ForDeployment("ns-1", "deploy-1").ObjectMeta(builder.WithLabels("a", "b")).Result(), ), }, want: []string{ "resources/namespaces/cluster/ns-1.json", "resources/namespaces/v1-preferredversion/cluster/ns-1.json", "resources/deployments.apps/namespaces/ns-1/deploy-1.json", "resources/deployments.apps/v1-preferredversion/namespaces/ns-1/deploy-1.json", }, }, { name: "OrLabelSelector test", backup: defaultBackup().OrLabelSelector([]*metav1.LabelSelector{ {MatchLabels: map[string]string{"a": "b"}}, {MatchLabels: map[string]string{"c": "d"}}, }).Result(), apiResources: []*test.APIResource{ test.Namespaces( builder.ForNamespace("ns-1").Phase(corev1api.NamespaceActive).Result(), builder.ForNamespace("ns-2").Phase(corev1api.NamespaceActive).Result(), builder.ForNamespace("ns-3").Phase(corev1api.NamespaceActive).Result(), ), test.Deployments( builder.ForDeployment("ns-1", "deploy-1").ObjectMeta(builder.WithLabels("a", "b")).Result(), builder.ForDeployment("ns-2", "deploy-2").ObjectMeta(builder.WithLabels("c", "d")).Result(), ), }, want: []string{ "resources/namespaces/cluster/ns-1.json", "resources/namespaces/v1-preferredversion/cluster/ns-1.json", "resources/namespaces/cluster/ns-2.json", "resources/namespaces/v1-preferredversion/cluster/ns-2.json", "resources/deployments.apps/namespaces/ns-1/deploy-1.json", "resources/deployments.apps/v1-preferredversion/namespaces/ns-1/deploy-1.json", "resources/deployments.apps/namespaces/ns-2/deploy-2.json", "resources/deployments.apps/v1-preferredversion/namespaces/ns-2/deploy-2.json", }, }, { name: "LabelSelector and Namespace filtering test", backup: defaultBackup().IncludedNamespaces("ns-1").LabelSelector(&metav1.LabelSelector{MatchLabels: map[string]string{"a": "b"}}). Result(), apiResources: []*test.APIResource{ test.Namespaces( builder.ForNamespace("ns-1").Phase(corev1api.NamespaceActive).Result(), builder.ForNamespace("ns-2").Phase(corev1api.NamespaceActive).Result(), builder.ForNamespace("ns-3").Phase(corev1api.NamespaceActive).Result(), ), test.Deployments( builder.ForDeployment("ns-1", "deploy-1").ObjectMeta(builder.WithLabels("a", "b")).Result(), ), }, want: []string{ "resources/namespaces/cluster/ns-1.json", "resources/namespaces/v1-preferredversion/cluster/ns-1.json", "resources/deployments.apps/namespaces/ns-1/deploy-1.json", "resources/deployments.apps/v1-preferredversion/namespaces/ns-1/deploy-1.json", }, }, { name: "LabelSelector and Namespace exclude filtering test", backup: defaultBackup().ExcludedNamespaces("ns-1", "ns-2").LabelSelector(&metav1.LabelSelector{MatchLabels: map[string]string{"a": "b"}}). Result(), apiResources: []*test.APIResource{ test.Namespaces( builder.ForNamespace("ns-1").ObjectMeta(builder.WithLabels("a", "b")).Phase(corev1api.NamespaceActive).Result(), builder.ForNamespace("ns-2").Phase(corev1api.NamespaceActive).Result(), builder.ForNamespace("ns-3").Phase(corev1api.NamespaceActive).Result(), ), test.Deployments( builder.ForDeployment("ns-1", "deploy-1").ObjectMeta(builder.WithLabels("a", "b")).Result(), ), }, want: []string{ "resources/namespaces/cluster/ns-1.json", "resources/namespaces/v1-preferredversion/cluster/ns-1.json", "resources/namespaces/cluster/ns-3.json", "resources/namespaces/v1-preferredversion/cluster/ns-3.json", }, }, { name: "Empty namespace test", backup: defaultBackup().IncludedNamespaces("invalid*").Result(), apiResources: []*test.APIResource{ test.Namespaces( builder.ForNamespace("ns-1").Phase(corev1api.NamespaceActive).Result(), builder.ForNamespace("ns-2").Phase(corev1api.NamespaceActive).Result(), builder.ForNamespace("ns-3").Phase(corev1api.NamespaceActive).Result(), ), test.Deployments( builder.ForDeployment("ns-1", "deploy-1").ObjectMeta(builder.WithLabels("a", "b")).Result(), ), }, want: []string{}, }, { name: "Default namespace filter test", backup: defaultBackup().Result(), apiResources: []*test.APIResource{ test.Namespaces( builder.ForNamespace("ns-1").Phase(corev1api.NamespaceActive).Result(), builder.ForNamespace("ns-2").Phase(corev1api.NamespaceActive).Result(), builder.ForNamespace("ns-3").Phase(corev1api.NamespaceActive).Result(), ), test.Deployments( builder.ForDeployment("ns-1", "deploy-1").ObjectMeta(builder.WithLabels("a", "b")).Result(), ), }, want: []string{ "resources/namespaces/cluster/ns-1.json", "resources/namespaces/v1-preferredversion/cluster/ns-1.json", "resources/namespaces/cluster/ns-2.json", "resources/namespaces/v1-preferredversion/cluster/ns-2.json", "resources/namespaces/cluster/ns-3.json", "resources/namespaces/v1-preferredversion/cluster/ns-3.json", "resources/deployments.apps/namespaces/ns-1/deploy-1.json", "resources/deployments.apps/v1-preferredversion/namespaces/ns-1/deploy-1.json", }, }, } itemBlockPool := StartItemBlockWorkerPool(t.Context(), 1, logrus.StandardLogger()) defer itemBlockPool.Stop() for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { var ( h = newHarness(t, itemBlockPool) req = &Request{ Backup: tc.backup, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), WorkerPool: itemBlockPool, } backupFile = bytes.NewBuffer([]byte{}) ) for _, resource := range tc.apiResources { h.addItems(t, resource) } h.backupper.Backup(h.log, req, backupFile, nil, nil, nil) assertTarballContents(t, backupFile, append(tc.want, "metadata/version")...) }) } } func TestUpdateVolumeInfos(t *testing.T) { timeExample := time.Date(2014, 6, 5, 11, 56, 45, 0, time.Local) now := metav1.NewTime(timeExample) logger := logrus.StandardLogger() tests := []struct { name string operations []*itemoperation.BackupOperation dataUpload *velerov2alpha1.DataUpload volumeInfos []*volume.BackupVolumeInfo expectedVolumeInfos []*volume.BackupVolumeInfo }{ { name: "CSISnapshot VolumeInfo update with Operation fails", operations: []*itemoperation.BackupOperation{ { Spec: itemoperation.BackupOperationSpec{ OperationID: "test-operation", }, Status: itemoperation.OperationStatus{ Error: "failed", Updated: &now, }, }, }, volumeInfos: []*volume.BackupVolumeInfo{ { BackupMethod: volume.CSISnapshot, CompletionTimestamp: &metav1.Time{}, CSISnapshotInfo: &volume.CSISnapshotInfo{ OperationID: "test-operation", }, }, }, expectedVolumeInfos: []*volume.BackupVolumeInfo{ { BackupMethod: volume.CSISnapshot, CompletionTimestamp: &now, Result: volume.VolumeResultFailed, CSISnapshotInfo: &volume.CSISnapshotInfo{ OperationID: "test-operation", }, }, }, }, { name: "CSISnapshot VolumeInfo update", operations: []*itemoperation.BackupOperation{ { Spec: itemoperation.BackupOperationSpec{ OperationID: "test-operation", }, Status: itemoperation.OperationStatus{ Updated: &now, }, }, }, volumeInfos: []*volume.BackupVolumeInfo{ { BackupMethod: volume.CSISnapshot, CompletionTimestamp: &metav1.Time{}, CSISnapshotInfo: &volume.CSISnapshotInfo{ OperationID: "test-operation", }, }, }, expectedVolumeInfos: []*volume.BackupVolumeInfo{ { BackupMethod: volume.CSISnapshot, CompletionTimestamp: &now, Result: volume.VolumeResultSucceeded, CSISnapshotInfo: &volume.CSISnapshotInfo{ OperationID: "test-operation", }, }, }, }, { name: "DataUpload VolumeInfo update with fail phase", operations: []*itemoperation.BackupOperation{}, dataUpload: builder.ForDataUpload("velero", "du-1"). CompletionTimestamp(&now). CSISnapshot(&velerov2alpha1.CSISnapshotSpec{VolumeSnapshot: "vs-1"}). SnapshotID("snapshot-id"). Progress(shared.DataMoveOperationProgress{TotalBytes: 1000}). IncrementalBytes(500). Phase(velerov2alpha1.DataUploadPhaseFailed). SourceNamespace("ns-1"). SourcePVC("pvc-1"). Result(), volumeInfos: []*volume.BackupVolumeInfo{ { PVCName: "pvc-1", PVCNamespace: "ns-1", CompletionTimestamp: &metav1.Time{}, SnapshotDataMovementInfo: &volume.SnapshotDataMovementInfo{ DataMover: "velero", }, }, }, expectedVolumeInfos: []*volume.BackupVolumeInfo{ { PVCName: "pvc-1", PVCNamespace: "ns-1", CompletionTimestamp: &now, Result: volume.VolumeResultFailed, SnapshotDataMovementInfo: &volume.SnapshotDataMovementInfo{ DataMover: "velero", RetainedSnapshot: "vs-1", SnapshotHandle: "snapshot-id", Size: 1000, IncrementalSize: 500, Phase: velerov2alpha1.DataUploadPhaseFailed, }, }, }, }, { name: "DataUpload VolumeInfo update", operations: []*itemoperation.BackupOperation{}, dataUpload: builder.ForDataUpload("velero", "du-1"). CompletionTimestamp(&now). CSISnapshot(&velerov2alpha1.CSISnapshotSpec{VolumeSnapshot: "vs-1"}). SnapshotID("snapshot-id"). Progress(shared.DataMoveOperationProgress{TotalBytes: 1000}). IncrementalBytes(500). Phase(velerov2alpha1.DataUploadPhaseCompleted). SourceNamespace("ns-1"). SourcePVC("pvc-1"). Result(), volumeInfos: []*volume.BackupVolumeInfo{ { PVCName: "pvc-1", PVCNamespace: "ns-1", CompletionTimestamp: &metav1.Time{}, SnapshotDataMovementInfo: &volume.SnapshotDataMovementInfo{ DataMover: "velero", }, }, }, expectedVolumeInfos: []*volume.BackupVolumeInfo{ { PVCName: "pvc-1", PVCNamespace: "ns-1", CompletionTimestamp: &now, Result: volume.VolumeResultSucceeded, SnapshotDataMovementInfo: &volume.SnapshotDataMovementInfo{ DataMover: "velero", RetainedSnapshot: "vs-1", SnapshotHandle: "snapshot-id", Size: 1000, IncrementalSize: 500, Phase: velerov2alpha1.DataUploadPhaseCompleted, }, }, }, }, { // This is an error case. No crash happen here is good enough. name: "VolumeInfo doesn't have SnapshotDataMovementInfo when there is a matching DataUpload", operations: []*itemoperation.BackupOperation{}, dataUpload: builder.ForDataUpload("velero", "du-1"). CompletionTimestamp(&now). CSISnapshot(&velerov2alpha1.CSISnapshotSpec{VolumeSnapshot: "vs-1"}). SnapshotID("snapshot-id"). Progress(shared.DataMoveOperationProgress{TotalBytes: 1000}). IncrementalBytes(500). Phase(velerov2alpha1.DataUploadPhaseCompleted). SourceNamespace("ns-1"). SourcePVC("pvc-1"). Result(), volumeInfos: []*volume.BackupVolumeInfo{ { PVCName: "pvc-1", PVCNamespace: "ns-1", CompletionTimestamp: &metav1.Time{}, SnapshotDataMovementInfo: nil, }, }, expectedVolumeInfos: []*volume.BackupVolumeInfo{ { PVCName: "pvc-1", PVCNamespace: "ns-1", CompletionTimestamp: &metav1.Time{}, SnapshotDataMovementInfo: nil, }, }, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { unstructures := []unstructured.Unstructured{} if tc.dataUpload != nil { duMap, error := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.dataUpload) require.NoError(t, error) unstructures = append(unstructures, unstructured.Unstructured{ Object: duMap, }, ) } require.NoError(t, updateVolumeInfos(tc.volumeInfos, unstructures, tc.operations, logger)) if len(tc.expectedVolumeInfos) > 0 { require.Equal(t, tc.expectedVolumeInfos[0].CompletionTimestamp, tc.volumeInfos[0].CompletionTimestamp) require.Equal(t, tc.expectedVolumeInfos[0].SnapshotDataMovementInfo, tc.volumeInfos[0].SnapshotDataMovementInfo) } }) } } func TestPutVolumeInfos(t *testing.T) { backupName := "backup-01" backupStore := new(persistencemocks.BackupStore) backupStore.On("PutBackupVolumeInfos", mock.Anything, mock.Anything).Return(nil) require.NoError(t, putVolumeInfos(backupName, []*volume.BackupVolumeInfo{}, backupStore)) } type fakeSingleObjectBackupStoreGetter struct { store persistence.BackupStore } func (f *fakeSingleObjectBackupStoreGetter) Get(*velerov1.BackupStorageLocation, persistence.ObjectStoreGetter, logrus.FieldLogger) (persistence.BackupStore, error) { return f.store, nil } // NewFakeSingleObjectBackupStoreGetter returns an ObjectBackupStoreGetter // that will return only the given BackupStore. func NewFakeSingleObjectBackupStoreGetter(store persistence.BackupStore) persistence.ObjectBackupStoreGetter { return &fakeSingleObjectBackupStoreGetter{store: store} } ================================================ FILE: pkg/backup/delete_helpers.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backup import ( "fmt" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/label" ) // NewDeleteBackupRequest creates a DeleteBackupRequest for the backup identified by name and uid. func NewDeleteBackupRequest(name string, uid string) *velerov1api.DeleteBackupRequest { return &velerov1api.DeleteBackupRequest{ ObjectMeta: metav1.ObjectMeta{ GenerateName: name + "-", Labels: map[string]string{ velerov1api.BackupNameLabel: label.GetValidName(name), velerov1api.BackupUIDLabel: uid, }, }, Spec: velerov1api.DeleteBackupRequestSpec{ BackupName: name, }, } } // NewDeleteBackupRequestListOptions creates a ListOptions with a label selector configured to // find DeleteBackupRequests for the backup identified by name and uid. func NewDeleteBackupRequestListOptions(name, uid string) metav1.ListOptions { return metav1.ListOptions{ LabelSelector: fmt.Sprintf("%s=%s,%s=%s", velerov1api.BackupNameLabel, label.GetValidName(name), velerov1api.BackupUIDLabel, uid), } } ================================================ FILE: pkg/backup/item_backupper.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backup import ( "archive/tar" "context" "encoding/json" "fmt" "strings" "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" kubeerrs "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" kbClient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/internal/hook" "github.com/vmware-tanzu/velero/internal/resourcepolicies" "github.com/vmware-tanzu/velero/internal/volume" "github.com/vmware-tanzu/velero/internal/volumehelper" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/archive" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/discovery" "github.com/vmware-tanzu/velero/pkg/features" "github.com/vmware-tanzu/velero/pkg/itemblock" "github.com/vmware-tanzu/velero/pkg/itemoperation" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/plugin/velero" vsv1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/volumesnapshotter/v1" "github.com/vmware-tanzu/velero/pkg/podvolume" "github.com/vmware-tanzu/velero/pkg/util/boolptr" csiutil "github.com/vmware-tanzu/velero/pkg/util/csi" ) const ( csiBIAPluginName = "velero.io/csi-pvc-backupper" vsphereBIAPluginName = "velero.io/vsphere-pvc-backupper" ) // itemBackupper can back up individual items to a tar writer. type itemBackupper struct { backupRequest *Request tarWriter tarWriter dynamicFactory client.DynamicFactory kbClient kbClient.Client discoveryHelper discovery.Helper podVolumeBackupper podvolume.Backupper podVolumeContext context.Context podVolumeSnapshotTracker *podvolume.Tracker kubernetesBackupper *kubernetesBackupper volumeSnapshotterCache *VolumeSnapshotterCache itemHookHandler hook.ItemHookHandler hookTracker *hook.HookTracker volumeHelperImpl volumehelper.VolumeHelper } type FileForArchive struct { FilePath string Header *tar.Header FileBytes []byte } // backupItem backs up an individual item to tarWriter. The item may be excluded based on the // namespaces IncludesExcludes list. // If finalize is true, then it returns the bytes instead of writing them to the tarWriter // In addition to the error return, backupItem also returns a bool indicating whether the item // was actually backed up. func (ib *itemBackupper) backupItem(logger logrus.FieldLogger, obj runtime.Unstructured, groupResource schema.GroupResource, preferredGVR schema.GroupVersionResource, mustInclude, finalize bool, itemBlock *BackupItemBlock) (bool, []FileForArchive, error) { selectedForBackup, files, err := ib.backupItemInternal(logger, obj, groupResource, preferredGVR, mustInclude, finalize, itemBlock) // return if not selected, an error occurred, there are no files to add, or for finalize if !selectedForBackup || err != nil || len(files) == 0 || finalize { return selectedForBackup, files, err } ib.tarWriter.Lock() defer ib.tarWriter.Unlock() for _, file := range files { if err := ib.tarWriter.WriteHeader(file.Header); err != nil { return false, []FileForArchive{}, errors.WithStack(err) } if _, err := ib.tarWriter.Write(file.FileBytes); err != nil { return false, []FileForArchive{}, errors.WithStack(err) } } return true, []FileForArchive{}, nil } func (ib *itemBackupper) itemInclusionChecks(log logrus.FieldLogger, mustInclude bool, metadata metav1.Object, obj runtime.Unstructured, groupResource schema.GroupResource) bool { if mustInclude { log.Infof("Skipping the exclusion checks for this resource") } else { if metadata.GetLabels()[velerov1api.ExcludeFromBackupLabel] == "true" { log.Infof("Excluding item because it has label %s=true", velerov1api.ExcludeFromBackupLabel) ib.trackSkippedPV(obj, groupResource, "", fmt.Sprintf("item has label %s=true", velerov1api.ExcludeFromBackupLabel), log) return false } // NOTE: we have to re-check namespace & resource includes/excludes because it's possible that // backupItem can be invoked by a custom action. namespace := metadata.GetNamespace() if namespace != "" && !ib.backupRequest.NamespaceIncludesExcludes.ShouldInclude(namespace) { log.Info("Excluding item because namespace is excluded") return false } // NOTE: we specifically allow namespaces to be backed up even if it's excluded. // This check is more permissive for cluster resources to let those passed in by // plugins' additional items to get involved. // Only expel cluster resource when it's specifically listed in the excluded list here. if namespace == "" && groupResource != kuberesource.Namespaces && ib.backupRequest.ResourceIncludesExcludes.ShouldExclude(groupResource.String()) { log.Info("Excluding item because resource is cluster-scoped and is excluded by cluster filter.") return false } // Only check namespace-scoped resource to avoid expelling cluster resources // are not specified in included list. if namespace != "" && !ib.backupRequest.ResourceIncludesExcludes.ShouldInclude(groupResource.String()) { log.Info("Excluding item because resource is excluded") return false } } if metadata.GetDeletionTimestamp() != nil { log.Info("Skipping item because it's being deleted.") return false } return true } func (ib *itemBackupper) backupItemInternal(logger logrus.FieldLogger, obj runtime.Unstructured, groupResource schema.GroupResource, preferredGVR schema.GroupVersionResource, mustInclude, finalize bool, itemBlock *BackupItemBlock) (bool, []FileForArchive, error) { var itemFiles []FileForArchive metadata, err := meta.Accessor(obj) if err != nil { return false, itemFiles, err } namespace := metadata.GetNamespace() name := metadata.GetName() log := logger.WithFields(map[string]any{ "name": name, "resource": groupResource.String(), "namespace": namespace, }) if !ib.itemInclusionChecks(log, mustInclude, metadata, obj, groupResource) { return false, itemFiles, nil } key := itemKey{ resource: resourceKey(obj), namespace: namespace, name: name, } if ib.backupRequest.BackedUpItems.Has(key) { log.Info("Skipping item because it's already been backed up.") // returning true since this item *is* in the backup, even though we're not backing it up here return true, itemFiles, nil } ib.backupRequest.BackedUpItems.AddItem(key) log.Info("Backing up item") var ( backupErrs []error pod *corev1api.Pod pvbVolumes []string ) if optedOut, podName := ib.podVolumeSnapshotTracker.OptedoutByPod(namespace, name); optedOut { ib.trackSkippedPV(obj, groupResource, podVolumeApproach, fmt.Sprintf("opted out due to annotation in pod %s", podName), log) } if groupResource == kuberesource.Pods { // pod needs to be initialized for the unstructured converter pod = new(corev1api.Pod) if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), pod); err != nil { backupErrs = append(backupErrs, errors.WithStack(err)) // nil it on error since it's not valid pod = nil } else { // Get the list of volumes to back up using pod volume backup from the pod's annotations // or volume policy approach. Remove from this list any volumes that use a PVC that we've // already backed up (this would be in a read-write-many scenario, // where it's been backed up from another pod), since we don't need >1 backup per PVC. for _, volume := range pod.Spec.Volumes { shouldDoFSBackup, err := ib.volumeHelperImpl.ShouldPerformFSBackup(volume, *pod) if err != nil { backupErrs = append(backupErrs, errors.WithStack(err)) } if shouldDoFSBackup { // track the volumes backing up by PVB , so that when we backup PVCs/PVs // via an item action in the next step, we don't snapshot PVs that will have their data backed up // with pod volume backup. ib.podVolumeSnapshotTracker.Track(pod, volume.Name) if found, pvcName := ib.podVolumeSnapshotTracker.TakenForPodVolume(pod, volume.Name); found { log.WithFields(map[string]any{ "podVolume": volume, "pvcName": pvcName, }).Info("Pod volume uses a persistent volume claim which has already been backed up from another pod, skipping.") continue } pvbVolumes = append(pvbVolumes, volume.Name) } else { ib.podVolumeSnapshotTracker.Optout(pod, volume.Name) } } } } // capture the version of the object before invoking plugin actions as the plugin may update // the group version of the object. versionPath := resourceVersion(obj) updatedObj, additionalItemFiles, err := ib.executeActions(log, obj, groupResource, name, namespace, metadata, finalize, itemBlock) if err != nil { backupErrs = append(backupErrs, err) return false, itemFiles, kubeerrs.NewAggregate(backupErrs) } // If err is nil and updatedObj is nil, it means the item is skipped by plugin action, // we should return here to avoid backing up the item, and avoid potential NPE in the following code. if updatedObj == nil { log.Infof("Remove item from the backup's backupItems list and totalItems list because it's skipped by plugin action.") ib.backupRequest.BackedUpItems.DeleteItem(key) return false, itemFiles, nil } itemFiles = append(itemFiles, additionalItemFiles...) obj = updatedObj if metadata, err = meta.Accessor(obj); err != nil { return false, itemFiles, errors.WithStack(err) } // update name and namespace in case they were modified in an action name = metadata.GetName() namespace = metadata.GetNamespace() if groupResource == kuberesource.PersistentVolumes { if err := ib.addVolumeInfo(obj, log); err != nil { backupErrs = append(backupErrs, err) } if err := ib.takePVSnapshot(obj, log); err != nil { backupErrs = append(backupErrs, err) } } if groupResource == kuberesource.Pods && pod != nil { // this function will return partial results, so process podVolumeBackups // even if there are errors. podVolumeBackups, podVolumePVCBackupSummary, errs := ib.backupPodVolumes(log, pod, pvbVolumes) backupErrs = append(backupErrs, errs...) // Mark the volumes that has been processed by pod volume backup as Taken in the tracker. for _, pvb := range podVolumeBackups { ib.podVolumeSnapshotTracker.Take(pod, pvb.Spec.Volume) } // Track/Untrack the volumes based on podVolumePVCBackupSummary if podVolumePVCBackupSummary != nil { for _, skippedPVC := range podVolumePVCBackupSummary.Skipped { if obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(skippedPVC.PVC); err != nil { backupErrs = append(backupErrs, errors.WithStack(err)) } else { ib.trackSkippedPV(&unstructured.Unstructured{Object: obj}, kuberesource.PersistentVolumeClaims, podVolumeApproach, skippedPVC.Reason, log) } } for _, pvc := range podVolumePVCBackupSummary.Backedup { if obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(pvc); err != nil { backupErrs = append(backupErrs, errors.WithStack(err)) } else { ib.unTrackSkippedPV(&unstructured.Unstructured{Object: obj}, kuberesource.PersistentVolumeClaims, log) } } } } if len(backupErrs) != 0 { return false, itemFiles, kubeerrs.NewAggregate(backupErrs) } itemBytes, err := json.Marshal(obj.UnstructuredContent()) if err != nil { return false, itemFiles, errors.WithStack(err) } if versionPath == preferredGVR.Version { // backing up preferred version backup without API Group version - for backward compatibility log.Debugf("Resource %s/%s, version= %s, preferredVersion=%s", groupResource.String(), name, versionPath, preferredGVR.Version) itemFiles = append(itemFiles, getFileForArchive(namespace, name, groupResource.String(), "", itemBytes)) versionPath = versionPath + velerov1api.PreferredVersionDir } itemFiles = append(itemFiles, getFileForArchive(namespace, name, groupResource.String(), versionPath, itemBytes)) return true, itemFiles, nil } func getFileForArchive(namespace, name, groupResource, versionPath string, itemBytes []byte) FileForArchive { filePath := archive.GetVersionedItemFilePath("", groupResource, namespace, name, versionPath) hdr := &tar.Header{ Name: filePath, Size: int64(len(itemBytes)), Typeflag: tar.TypeReg, Mode: 0755, ModTime: time.Now(), } return FileForArchive{FilePath: filePath, Header: hdr, FileBytes: itemBytes} } // backupPodVolumes triggers pod volume backups of the specified pod volumes, and returns a list of PodVolumeBackups // for volumes that were successfully backed up, and a slice of any errors that were encountered. func (ib *itemBackupper) backupPodVolumes(log logrus.FieldLogger, pod *corev1api.Pod, volumes []string) ([]*velerov1api.PodVolumeBackup, *podvolume.PVCBackupSummary, []error) { if len(volumes) == 0 { return nil, nil, nil } if ib.podVolumeBackupper == nil { log.Warn("No pod volume backupper, not backing up pod's volumes") return nil, nil, nil } return ib.podVolumeBackupper.BackupPodVolumes(ib.backupRequest.Backup, pod, volumes, ib.backupRequest.ResPolicies, log) } func (ib *itemBackupper) executeActions( log logrus.FieldLogger, obj runtime.Unstructured, groupResource schema.GroupResource, name, namespace string, metadata metav1.Object, finalize bool, itemBlock *BackupItemBlock, ) (runtime.Unstructured, []FileForArchive, error) { var itemFiles []FileForArchive for _, action := range ib.backupRequest.ResolvedActions { if !action.ShouldUse(groupResource, namespace, metadata, log) { continue } log.Info("Executing custom action") actionName := action.Name() if act, err := ib.getMatchAction(obj, groupResource, actionName); err != nil { return nil, itemFiles, errors.WithStack(err) } else if act != nil && act.Type == resourcepolicies.Skip { log.Infof("Skip executing Backup Item Action: %s of resource %s: %s/%s for the matched resource policies", actionName, groupResource, namespace, name) ib.trackSkippedPV(obj, groupResource, "", "skipped due to resource policy ", log) continue } // If the EnableCSI feature is not enabled, but the executing action is from CSI plugin, skip the action. if csiutil.ShouldSkipAction(actionName) { log.Infof("Skip action %s for resource %s:%s/%s, because the CSI feature is not enabled. Feature setting is %s.", actionName, groupResource.String(), metadata.GetNamespace(), metadata.GetName(), features.Serialize()) continue } if groupResource == kuberesource.PersistentVolumeClaims && actionName == csiBIAPluginName { snapshotVolume, err := ib.volumeHelperImpl.ShouldPerformSnapshot(obj, kuberesource.PersistentVolumeClaims) if err != nil { return nil, itemFiles, errors.WithStack(err) } if !snapshotVolume { ib.trackSkippedPV( obj, kuberesource.PersistentVolumeClaims, volumeSnapshotApproach, "not satisfy the criteria for VolumePolicy or the legacy snapshot way", log, ) continue } } updatedItem, additionalItemIdentifiers, operationID, postOperationItems, err := action.Execute(obj, ib.backupRequest.Backup) if err != nil { return nil, itemFiles, errors.Wrapf(err, "error executing custom action (groupResource=%s, namespace=%s, name=%s)", groupResource.String(), namespace, name) } u := &unstructured.Unstructured{Object: updatedItem.UnstructuredContent()} if _, ok := u.GetAnnotations()[velerov1api.SkipFromBackupAnnotation]; ok { log.Infof("Resource (groupResource=%s, namespace=%s, name=%s) is skipped from backup by action %s.", groupResource.String(), namespace, name, actionName) return nil, itemFiles, nil } if actionName == csiBIAPluginName { if additionalItemIdentifiers == nil && u.GetAnnotations()[velerov1api.SkippedNoCSIPVAnnotation] == "true" { // snapshot was skipped by CSI plugin log.Infof("skip CSI snapshot for PVC %s as it's not a CSI compatible volume", namespace+"/"+name) ib.trackSkippedPV(obj, groupResource, csiSnapshotApproach, "skipped b/c it's not a CSI volume", log) delete(u.GetAnnotations(), velerov1api.SkippedNoCSIPVAnnotation) } else { // the snapshot has been taken by the BIA plugin log.Infof("Untrack the PVC %s, because it's backed up by CSI BIA.", namespace+"/"+name) ib.unTrackSkippedPV(obj, kuberesource.PersistentVolumeClaims, log) } } mustInclude := u.GetAnnotations()[velerov1api.MustIncludeAdditionalItemAnnotation] == "true" || finalize // remove the annotation as it's for communication between BIA and velero server, // we don't want the resource be restored with this annotation. delete(u.GetAnnotations(), velerov1api.MustIncludeAdditionalItemAnnotation) obj = u // If async plugin started async operation, add it to the ItemOperations list // ignore during finalize phase if operationID != "" { if finalize { return nil, itemFiles, fmt.Errorf("backup Item Action created operation during finalize (groupResource=%s, namespace=%s, name=%s)", groupResource.String(), namespace, name) } resourceIdentifier := velero.ResourceIdentifier{ GroupResource: groupResource, Namespace: namespace, Name: name, } now := metav1.Now() newOperation := itemoperation.BackupOperation{ Spec: itemoperation.BackupOperationSpec{ BackupName: ib.backupRequest.Backup.Name, BackupUID: string(ib.backupRequest.Backup.UID), BackupItemAction: action.Name(), ResourceIdentifier: resourceIdentifier, OperationID: operationID, }, Status: itemoperation.OperationStatus{ Phase: itemoperation.OperationPhaseNew, Created: &now, }, } newOperation.Spec.PostOperationItems = postOperationItems itemOperList := ib.backupRequest.GetItemOperationsList() *itemOperList = append(*itemOperList, &newOperation) } for _, additionalItem := range additionalItemIdentifiers { var itemList []itemblock.ItemBlockItem // get item content from itemBlock if it's there to avoid the additional APIServer call // We could have multiple versions to back up if EnableAPIGroupVersions is set if itemBlock != nil { itemList = itemBlock.FindItem(additionalItem.GroupResource, additionalItem.Namespace, additionalItem.Name) } // if item is not in itemblock, pull it from the cluster if len(itemList) == 0 { log.Infof("Additional Item %s %s/%s not found in ItemBlock, getting from cluster", additionalItem.GroupResource, additionalItem.Namespace, additionalItem.Name) gvr, resource, err := ib.discoveryHelper.ResourceFor(additionalItem.GroupResource.WithVersion("")) if err != nil { return nil, itemFiles, err } client, err := ib.dynamicFactory.ClientForGroupVersionResource(gvr.GroupVersion(), resource, additionalItem.Namespace) if err != nil { return nil, itemFiles, err } item, err := client.Get(additionalItem.Name, metav1.GetOptions{}) if apierrors.IsNotFound(err) { log.WithFields(logrus.Fields{ "groupResource": additionalItem.GroupResource, "namespace": additionalItem.Namespace, "name": additionalItem.Name, }).Warnf("Additional item was not found in Kubernetes API, can't back it up") continue } if err != nil { return nil, itemFiles, errors.WithStack(err) } itemList = append(itemList, itemblock.ItemBlockItem{ Gr: additionalItem.GroupResource, Item: item, PreferredGVR: gvr, }) } for _, item := range itemList { _, additionalItemFiles, err := ib.backupItem(log, item.Item, additionalItem.GroupResource, item.PreferredGVR, mustInclude, finalize, itemBlock) if err != nil { return nil, itemFiles, err } itemFiles = append(itemFiles, additionalItemFiles...) } } } return obj, itemFiles, nil } // zoneLabelDeprecated is the label that stores availability-zone info // on PVs this is deprecated on Kubernetes >= 1.17.0 // zoneLabel is the label that stores availability-zone info // on PVs const ( zoneLabelDeprecated = "failure-domain.beta.kubernetes.io/zone" // this is reused for nodeAffinity requirements zoneLabel = "topology.kubernetes.io/zone" awsEbsCsiZoneKey = "topology.ebs.csi.aws.com/zone" azureCsiZoneKey = "topology.disk.csi.azure.com/zone" gkeCsiZoneKey = "topology.gke.io/zone" gkeZoneSeparator = "__" // OpenStack CSI drivers topology keys cinderCsiZoneKey = "topology.manila.csi.openstack.org/zone" manilaCsiZoneKey = "topology.cinder.csi.openstack.org/zone" ) // takePVSnapshot triggers a snapshot for the volume/disk underlying a PersistentVolume if the provided // backup has volume snapshots enabled and the PV is of a compatible type. Also records cloud // disk type and IOPS (if applicable) to be able to restore to current state later. func (ib *itemBackupper) takePVSnapshot(obj runtime.Unstructured, log logrus.FieldLogger) error { log.Info("Executing takePVSnapshot") pv := new(corev1api.PersistentVolume) if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), pv); err != nil { return errors.WithStack(err) } log = log.WithField("persistentVolume", pv.Name) snapshotVolume, err := ib.volumeHelperImpl.ShouldPerformSnapshot(obj, kuberesource.PersistentVolumes) if err != nil { return err } if !snapshotVolume { ib.trackSkippedPV( obj, kuberesource.PersistentVolumes, volumeSnapshotApproach, "not satisfy the criteria for VolumePolicy or the legacy snapshot way", log, ) return nil } // #4758 Do not take snapshot for CSI PV to avoid duplicated snapshotting, when CSI feature is enabled. if features.IsEnabled(velerov1api.CSIFeatureFlag) && pv.Spec.CSI != nil { log.Infof("Skipping snapshot of persistent volume %s, because it's handled by CSI plugin.", pv.Name) return nil } // TODO: Snapshot data mover is only supported for CSI plugin scenario by now. // Need to add a mechanism to choose running which plugin for resources. // After that, this warning can be removed. if boolptr.IsSetToTrue(ib.backupRequest.Spec.SnapshotMoveData) { log.Warnf("VolumeSnapshotter plugin doesn't support data movement.") if features.IsEnabled(velerov1api.CSIFeatureFlag) && pv.Spec.CSI == nil { log.Warn("Cannot use CSI data mover to handle PV, because PV doesn't contain CSI in spec.", " Fall back to Velero native snapshot.") } } if ib.backupRequest.ResPolicies != nil { pvc := new(corev1api.PersistentVolumeClaim) if pv.Spec.ClaimRef != nil { err = ib.kbClient.Get(context.Background(), kbClient.ObjectKey{ Namespace: pv.Spec.ClaimRef.Namespace, Name: pv.Spec.ClaimRef.Name}, pvc) if err != nil { return err } } vfd := resourcepolicies.NewVolumeFilterData(pv, nil, pvc) if action, err := ib.backupRequest.ResPolicies.GetMatchAction(vfd); err != nil { log.WithError(err).Errorf("Error getting matched resource policies for pv %s", pv.Name) return nil } else if action != nil && action.Type == resourcepolicies.Skip { log.Infof("skip snapshot of pv %s for the matched resource policies", pv.Name) // at this point we are sure this object is PV therefore we'll call the tracker directly ib.backupRequest.SkippedPVTracker.Track(pv.Name, volumeSnapshotApproach, "matched action is 'skip' in chosen resource policies") return nil } } // TODO: -- once failure-domain.beta.kubernetes.io/zone is no longer // supported in any velero-supported version of Kubernetes, remove fallback checking of it pvFailureDomainZone, labelFound := pv.Labels[zoneLabel] if !labelFound { log.Infof("label %q is not present on PersistentVolume, checking deprecated label...", zoneLabel) pvFailureDomainZone, labelFound = pv.Labels[zoneLabelDeprecated] if !labelFound { var k string log.Infof("label %q is not present on PersistentVolume", zoneLabelDeprecated) k, pvFailureDomainZone = zoneFromPVNodeAffinity(pv, awsEbsCsiZoneKey, azureCsiZoneKey, gkeCsiZoneKey, cinderCsiZoneKey, manilaCsiZoneKey, zoneLabel, zoneLabelDeprecated) if pvFailureDomainZone != "" { log.Infof("zone info from nodeAffinity requirements: %s, key: %s", pvFailureDomainZone, k) } else { log.Infof("zone info not available in nodeAffinity requirements") } } } var ( volumeID, location string volumeSnapshotter vsv1.VolumeSnapshotter ) for _, snapshotLocation := range ib.backupRequest.SnapshotLocations { log := log.WithField("volumeSnapshotLocation", snapshotLocation.Name) bs, err := ib.volumeSnapshotterCache.SetNX(snapshotLocation) if err != nil { log.WithError(err).Error("Error getting volume snapshotter for volume snapshot location") continue } if volumeID, err = bs.GetVolumeID(obj); err != nil { log.WithError(err).Errorf("Error attempting to get volume ID for persistent volume") continue } if volumeID == "" { log.Warn("No volume ID returned by volume snapshotter for persistent volume") continue } log.Infof("Got volume ID for persistent volume") volumeSnapshotter = bs location = snapshotLocation.Name break } if volumeSnapshotter == nil { // the PV may still has change to be snapshotted by CSI plugin's `PVCBackupItemAction` in PVC backup logic log.Info("Persistent volume is not a supported volume type for Velero-native volumeSnapshotter snapshot, skipping.") ib.backupRequest.SkippedPVTracker.Track(pv.Name, volumeSnapshotApproach, "no applicable volumesnapshotter found") return nil } log = log.WithField("volumeID", volumeID) // create tags from the backup's labels tags := map[string]string{} for k, v := range ib.backupRequest.GetLabels() { tags[k] = v } tags["velero.io/backup"] = ib.backupRequest.Name tags["velero.io/pv"] = pv.Name log.Info("Getting volume information") volumeType, iops, err := volumeSnapshotter.GetVolumeInfo(volumeID, pvFailureDomainZone) if err != nil { return errors.WithMessage(err, "error getting volume info") } log.Info("Snapshotting persistent volume") snapshot := volumeSnapshot(ib.backupRequest.Backup, pv.Name, volumeID, volumeType, pvFailureDomainZone, location, iops) var errs []error log.Info("Untrack the PV %s from the skipped volumes, because it's backed by Velero native snapshot.", pv.Name) ib.backupRequest.SkippedPVTracker.Untrack(pv.Name) snapshotID, err := volumeSnapshotter.CreateSnapshot(snapshot.Spec.ProviderVolumeID, snapshot.Spec.VolumeAZ, tags) if err != nil { errs = append(errs, errors.Wrap(err, "error taking snapshot of volume")) snapshot.Status.Phase = volume.SnapshotPhaseFailed } else { snapshot.Status.Phase = volume.SnapshotPhaseCompleted snapshot.Status.ProviderSnapshotID = snapshotID } ib.backupRequest.VolumeSnapshots.Add(snapshot) // nil errors are automatically removed return kubeerrs.NewAggregate(errs) } func (ib *itemBackupper) getMatchAction(obj runtime.Unstructured, groupResource schema.GroupResource, backupItemActionName string) (*resourcepolicies.Action, error) { if ib.backupRequest.ResPolicies != nil && groupResource == kuberesource.PersistentVolumeClaims && (backupItemActionName == csiBIAPluginName || backupItemActionName == vsphereBIAPluginName) { pvc := &corev1api.PersistentVolumeClaim{} if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), pvc); err != nil { return nil, errors.WithStack(err) } var pv *corev1api.PersistentVolume if pvName := pvc.Spec.VolumeName; pvName != "" { pv = &corev1api.PersistentVolume{} if err := ib.kbClient.Get(context.Background(), kbClient.ObjectKey{Name: pvName}, pv); err != nil { return nil, errors.WithStack(err) } } // If pv is nil for unbound PVCs - policy matching will use PVC-only conditions vfd := resourcepolicies.NewVolumeFilterData(pv, nil, pvc) return ib.backupRequest.ResPolicies.GetMatchAction(vfd) } return nil, nil } // trackSkippedPV tracks the skipped PV based on the object and the given approach and reason // this function will be called throughout the process of backup, it needs to handle any object func (ib *itemBackupper) trackSkippedPV(obj runtime.Unstructured, groupResource schema.GroupResource, approach string, reason string, log logrus.FieldLogger) { if name, err := getPVName(obj, groupResource); len(name) > 0 && err == nil { ib.backupRequest.SkippedPVTracker.Track(name, approach, reason) } else if err != nil { // Log at info level for tracking purposes. This is not an error because // it's expected for some resources (e.g., PVCs in Pending or Lost phase) // to not have a PV name. This occurs when volume policy skips unbound PVCs. log.WithError(err).Infof("unable to get PV name, skip tracking.") } } // unTrackSkippedPV removes skipped PV based on the object from the tracker // this function will be called throughout the process of backup, it needs to handle any object func (ib *itemBackupper) unTrackSkippedPV(obj runtime.Unstructured, groupResource schema.GroupResource, log logrus.FieldLogger) { if name, err := getPVName(obj, groupResource); len(name) > 0 && err == nil { ib.backupRequest.SkippedPVTracker.Untrack(name) } else if err != nil { // For PVCs in Pending or Lost phase, it's expected that there's no PV name. // Log at debug level instead of warning to reduce noise. if groupResource == kuberesource.PersistentVolumeClaims { pvc := new(corev1api.PersistentVolumeClaim) if convErr := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), pvc); convErr == nil { if pvc.Status.Phase == corev1api.ClaimPending || pvc.Status.Phase == corev1api.ClaimLost { log.WithError(err).Debugf("unable to get PV name for %s PVC, skip untracking.", pvc.Status.Phase) return } } } log.WithError(err).Warnf("unable to get PV name, skip untracking.") } } func (ib *itemBackupper) addVolumeInfo(obj runtime.Unstructured, log logrus.FieldLogger) error { pv := new(corev1api.PersistentVolume) err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), pv) if err != nil { log.WithError(err).Warnf("Fail to convert PV") return err } pvcName := "" pvcNamespace := "" if pv.Spec.ClaimRef != nil { pvcName = pv.Spec.ClaimRef.Name pvcNamespace = pv.Spec.ClaimRef.Namespace } ib.backupRequest.VolumesInformation.InsertPVMap(*pv, pvcName, pvcNamespace) return nil } // convert the input object to PV/PVC and get the PV name func getPVName(obj runtime.Unstructured, groupResource schema.GroupResource) (string, error) { if groupResource == kuberesource.PersistentVolumes { pv := new(corev1api.PersistentVolume) if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), pv); err != nil { return "", fmt.Errorf("failed to convert object to PV: %w", err) } return pv.Name, nil } if groupResource == kuberesource.PersistentVolumeClaims { pvc := new(corev1api.PersistentVolumeClaim) if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), pvc); err != nil { return "", fmt.Errorf("failed to convert object to PVC: %w", err) } if pvc.Spec.VolumeName == "" { return "", fmt.Errorf("PV name is not set in PVC") } return pvc.Spec.VolumeName, nil } return "", nil } func volumeSnapshot(backup *velerov1api.Backup, volumeName, volumeID, volumeType, az, location string, iops *int64) *volume.Snapshot { return &volume.Snapshot{ Spec: volume.SnapshotSpec{ BackupName: backup.Name, BackupUID: string(backup.UID), Location: location, PersistentVolumeName: volumeName, ProviderVolumeID: volumeID, VolumeType: volumeType, VolumeAZ: az, VolumeIOPS: iops, }, Status: volume.SnapshotStatus{ Phase: volume.SnapshotPhaseNew, }, } } // resourceKey returns a string representing the object's GroupVersionKind (e.g. // apps/v1/Deployment). func resourceKey(obj runtime.Unstructured) string { gvk := obj.GetObjectKind().GroupVersionKind() return fmt.Sprintf("%s/%s", gvk.GroupVersion().String(), gvk.Kind) } // resourceVersion returns a string representing the object's API Version (e.g. // v1 if item belongs to apps/v1 func resourceVersion(obj runtime.Unstructured) string { gvk := obj.GetObjectKind().GroupVersionKind() return gvk.Version } // zoneFromPVNodeAffinity iterates the node affinity requirement of a PV to // get its availability zone, it returns the key merely for logging. func zoneFromPVNodeAffinity(res *corev1api.PersistentVolume, topologyKeys ...string) (string, string) { nodeAffinity := res.Spec.NodeAffinity if nodeAffinity == nil { return "", "" } keySet := sets.NewString(topologyKeys...) providerGke := false zones := make([]string, 0) for _, term := range nodeAffinity.Required.NodeSelectorTerms { if term.MatchExpressions == nil { continue } for _, exp := range term.MatchExpressions { if keySet.Has(exp.Key) && exp.Operator == "In" && len(exp.Values) > 0 { if exp.Key == gkeCsiZoneKey { providerGke = true zones = append(zones, exp.Values[0]) } else { return exp.Key, exp.Values[0] } } } } if providerGke { return gkeCsiZoneKey, strings.Join(zones, gkeZoneSeparator) } return "", "" } ================================================ FILE: pkg/backup/item_backupper_test.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backup import ( "bytes" "testing" "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/runtime/schema" ctrlfake "sigs.k8s.io/controller-runtime/pkg/client/fake" "github.com/vmware-tanzu/velero/internal/resourcepolicies" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/stretchr/testify/assert" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "github.com/vmware-tanzu/velero/pkg/builder" ) func Test_resourceKey(t *testing.T) { tests := []struct { resource metav1.Object want string }{ {resource: builder.ForPod("default", "test").Result(), want: "v1/Pod"}, {resource: builder.ForDeployment("default", "test").Result(), want: "apps/v1/Deployment"}, {resource: builder.ForPersistentVolume("test").Result(), want: "v1/PersistentVolume"}, {resource: builder.ForRole("default", "test").Result(), want: "rbac.authorization.k8s.io/v1/Role"}, } for _, tt := range tests { t.Run(tt.want, func(t *testing.T) { content, _ := runtime.DefaultUnstructuredConverter.ToUnstructured(tt.resource) unstructured := &unstructured.Unstructured{Object: content} assert.Equal(t, tt.want, resourceKey(unstructured)) }) } } func Test_zoneFromPVNodeAffinity(t *testing.T) { keys := []string{ awsEbsCsiZoneKey, azureCsiZoneKey, gkeCsiZoneKey, zoneLabel, zoneLabelDeprecated, } tests := []struct { name string pv *corev1api.PersistentVolume wantKey string wantValue string }{ { name: "AWS CSI Volume", pv: builder.ForPersistentVolume("awscsi").NodeAffinityRequired( builder.ForNodeSelector( *builder.NewNodeSelectorTermBuilder().WithMatchExpression("topology.ebs.csi.aws.com/zone", "In", "us-east-2c").Result(), ).Result(), ).Result(), wantKey: "topology.ebs.csi.aws.com/zone", wantValue: "us-east-2c", }, { name: "Azure CSI Volume", pv: builder.ForPersistentVolume("azurecsi").NodeAffinityRequired( builder.ForNodeSelector( *builder.NewNodeSelectorTermBuilder().WithMatchExpression("topology.disk.csi.azure.com/zone", "In", "us-central").Result(), ).Result(), ).Result(), wantKey: "topology.disk.csi.azure.com/zone", wantValue: "us-central", }, { name: "GCP CSI Volume", pv: builder.ForPersistentVolume("gcpcsi").NodeAffinityRequired( builder.ForNodeSelector( *builder.NewNodeSelectorTermBuilder().WithMatchExpression("topology.gke.io/zone", "In", "us-west1-a").Result(), ).Result(), ).Result(), wantKey: "topology.gke.io/zone", wantValue: "us-west1-a", }, { name: "AWS CSI Volume with multiple zone value, returns the first", pv: builder.ForPersistentVolume("awscsi").NodeAffinityRequired( builder.ForNodeSelector( *builder.NewNodeSelectorTermBuilder().WithMatchExpression("topology.ebs.csi.aws.com/zone", "In", "us-east-2c", "us-west").Result(), ).Result(), ).Result(), wantKey: "topology.ebs.csi.aws.com/zone", wantValue: "us-east-2c", }, { name: "Volume with no matching key", pv: builder.ForPersistentVolume("no-matching-pv").NodeAffinityRequired( builder.ForNodeSelector( *builder.NewNodeSelectorTermBuilder().WithMatchExpression("some-key", "In", "us-west").Result(), ).Result(), ).Result(), wantKey: "", wantValue: "", }, { name: "Volume with multiple valid keys, returns the first match", // it should never happen pv: builder.ForPersistentVolume("multi-matching-pv").NodeAffinityRequired( builder.ForNodeSelector( *builder.NewNodeSelectorTermBuilder().WithMatchExpression("topology.disk.csi.azure.com/zone", "In", "us-central").Result(), *builder.NewNodeSelectorTermBuilder().WithMatchExpression("topology.ebs.csi.aws.com/zone", "In", "us-east-2c", "us-west").Result(), *builder.NewNodeSelectorTermBuilder().WithMatchExpression("topology.ebs.csi.aws.com/zone", "In", "unknown").Result(), ).Result(), ).Result(), wantKey: "topology.disk.csi.azure.com/zone", wantValue: "us-central", }, { /* an valid example of node affinity in a GKE's regional PV nodeAffinity: required: nodeSelectorTerms: - matchExpressions: - key: topology.gke.io/zone operator: In values: - us-central1-a - matchExpressions: - key: topology.gke.io/zone operator: In values: - us-central1-c */ name: "Volume with multiple valid keys, and provider is gke, returns all valid entries's first zone value", pv: builder.ForPersistentVolume("multi-matching-pv").NodeAffinityRequired( builder.ForNodeSelector( *builder.NewNodeSelectorTermBuilder().WithMatchExpression("topology.gke.io/zone", "In", "us-central1-c").Result(), *builder.NewNodeSelectorTermBuilder().WithMatchExpression("topology.gke.io/zone", "In", "us-east-2c", "us-east-2b").Result(), *builder.NewNodeSelectorTermBuilder().WithMatchExpression("topology.gke.io/zone", "In", "europe-north1-a").Result(), ).Result(), ).Result(), wantKey: "topology.gke.io/zone", wantValue: "us-central1-c__us-east-2c__europe-north1-a", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { k, v := zoneFromPVNodeAffinity(tt.pv, keys...) assert.Equal(t, tt.wantKey, k) assert.Equal(t, tt.wantValue, v) }) } } func TestGetPVName(t *testing.T) { testcases := []struct { name string obj metav1.Object groupResource schema.GroupResource pvName string hasErr bool }{ { name: "pv should return pv name", obj: builder.ForPersistentVolume("test-pv").Result(), groupResource: kuberesource.PersistentVolumes, pvName: "test-pv", hasErr: false, }, { name: "pvc without volumeName should return error", obj: builder.ForPersistentVolumeClaim("ns", "pvc-1").Result(), groupResource: kuberesource.PersistentVolumeClaims, pvName: "", hasErr: true, }, { name: "pvc with volumeName should return pv name", obj: builder.ForPersistentVolumeClaim("ns", "pvc-1").VolumeName("test-pv-2").Result(), groupResource: kuberesource.PersistentVolumeClaims, pvName: "test-pv-2", hasErr: false, }, { name: "unsupported group resource should return empty pv name", obj: builder.ForPod("ns", "pod1").Result(), groupResource: kuberesource.Pods, pvName: "", hasErr: false, }, } for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { o := &unstructured.Unstructured{Object: nil} if tc.obj != nil { data, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.obj) o = &unstructured.Unstructured{Object: data} require.NoError(t, err) } name, err2 := getPVName(o, tc.groupResource) assert.Equal(t, tc.pvName, name) assert.Equal(t, tc.hasErr, err2 != nil) }) } } func TestRandom(t *testing.T) { pv := new(corev1api.PersistentVolume) pvc := new(corev1api.PersistentVolumeClaim) obj := builder.ForPod("ns1", "pod1").ServiceAccount("sa").Result() o, _ := runtime.DefaultUnstructuredConverter.ToUnstructured(obj) err1 := runtime.DefaultUnstructuredConverter.FromUnstructured(o, pv) err2 := runtime.DefaultUnstructuredConverter.FromUnstructured(o, pvc) t.Logf("err1: %v, err2: %v", err1, err2) } func TestAddVolumeInfo(t *testing.T) { tests := []struct { name string pv *corev1api.PersistentVolume }{ { name: "PV has ClaimRef", pv: builder.ForPersistentVolume("testPV").ClaimRef("testNS", "testPVC").Result(), }, { name: "PV has no ClaimRef", pv: builder.ForPersistentVolume("testPV").Result(), }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { ib := itemBackupper{} ib.backupRequest = new(Request) ib.backupRequest.VolumesInformation.Init() pvObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.pv) require.NoError(t, err) logger := logrus.StandardLogger() err = ib.addVolumeInfo(&unstructured.Unstructured{Object: pvObj}, logger) require.NoError(t, err) }) } } func TestGetMatchAction_PendingLostPVC(t *testing.T) { scheme := runtime.NewScheme() require.NoError(t, corev1api.AddToScheme(scheme)) // Create resource policies that skip Pending/Lost PVCs resPolicies := &resourcepolicies.ResourcePolicies{ Version: "v1", VolumePolicies: []resourcepolicies.VolumePolicy{ { Conditions: map[string]any{ "pvcPhase": []string{"Pending", "Lost"}, }, Action: resourcepolicies.Action{ Type: resourcepolicies.Skip, }, }, }, } policies := &resourcepolicies.Policies{} err := policies.BuildPolicy(resPolicies) require.NoError(t, err) testCases := []struct { name string pvc *corev1api.PersistentVolumeClaim pv *corev1api.PersistentVolume expectedAction *resourcepolicies.Action expectError bool }{ { name: "Pending PVC with no VolumeName should match pvcPhase policy", pvc: builder.ForPersistentVolumeClaim("ns", "pending-pvc"). StorageClass("test-sc"). Phase(corev1api.ClaimPending). Result(), pv: nil, expectedAction: &resourcepolicies.Action{Type: resourcepolicies.Skip}, expectError: false, }, { name: "Lost PVC with no VolumeName should match pvcPhase policy", pvc: builder.ForPersistentVolumeClaim("ns", "lost-pvc"). StorageClass("test-sc"). Phase(corev1api.ClaimLost). Result(), pv: nil, expectedAction: &resourcepolicies.Action{Type: resourcepolicies.Skip}, expectError: false, }, { name: "Bound PVC with VolumeName and matching PV should not match pvcPhase policy", pvc: builder.ForPersistentVolumeClaim("ns", "bound-pvc"). StorageClass("test-sc"). VolumeName("test-pv"). Phase(corev1api.ClaimBound). Result(), pv: builder.ForPersistentVolume("test-pv").StorageClass("test-sc").Result(), expectedAction: nil, expectError: false, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { // Build fake client with PV if present clientBuilder := ctrlfake.NewClientBuilder().WithScheme(scheme) if tc.pv != nil { clientBuilder = clientBuilder.WithObjects(tc.pv) } fakeClient := clientBuilder.Build() ib := &itemBackupper{ kbClient: fakeClient, backupRequest: &Request{ ResPolicies: policies, }, } // Convert PVC to unstructured pvcData, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.pvc) require.NoError(t, err) obj := &unstructured.Unstructured{Object: pvcData} action, err := ib.getMatchAction(obj, kuberesource.PersistentVolumeClaims, csiBIAPluginName) if tc.expectError { require.Error(t, err) } else { require.NoError(t, err) } if tc.expectedAction == nil { assert.Nil(t, action) } else { require.NotNil(t, action) assert.Equal(t, tc.expectedAction.Type, action.Type) } }) } } func TestTrackSkippedPV_PendingLostPVC(t *testing.T) { testCases := []struct { name string pvc *corev1api.PersistentVolumeClaim }{ { name: "Pending PVC should log at info level", pvc: builder.ForPersistentVolumeClaim("ns", "pending-pvc"). Phase(corev1api.ClaimPending). Result(), }, { name: "Lost PVC should log at info level", pvc: builder.ForPersistentVolumeClaim("ns", "lost-pvc"). Phase(corev1api.ClaimLost). Result(), }, { name: "Bound PVC without VolumeName should log at info level", pvc: builder.ForPersistentVolumeClaim("ns", "bound-pvc"). Phase(corev1api.ClaimBound). Result(), }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { ib := &itemBackupper{ backupRequest: &Request{ SkippedPVTracker: NewSkipPVTracker(), }, } // Set up log capture logOutput := &bytes.Buffer{} logger := logrus.New() logger.SetOutput(logOutput) logger.SetLevel(logrus.DebugLevel) // Convert PVC to unstructured pvcData, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.pvc) require.NoError(t, err) obj := &unstructured.Unstructured{Object: pvcData} ib.trackSkippedPV(obj, kuberesource.PersistentVolumeClaims, "", "test reason", logger) logStr := logOutput.String() assert.Contains(t, logStr, "level=info") assert.Contains(t, logStr, "unable to get PV name, skip tracking.") }) } } func TestUnTrackSkippedPV_PendingLostPVC(t *testing.T) { testCases := []struct { name string pvc *corev1api.PersistentVolumeClaim expectWarningLog bool expectDebugMessage string }{ { name: "Pending PVC should log at debug level, not warning", pvc: builder.ForPersistentVolumeClaim("ns", "pending-pvc"). Phase(corev1api.ClaimPending). Result(), expectWarningLog: false, expectDebugMessage: "unable to get PV name for Pending PVC, skip untracking.", }, { name: "Lost PVC should log at debug level, not warning", pvc: builder.ForPersistentVolumeClaim("ns", "lost-pvc"). Phase(corev1api.ClaimLost). Result(), expectWarningLog: false, expectDebugMessage: "unable to get PV name for Lost PVC, skip untracking.", }, { name: "Bound PVC without VolumeName should log warning", pvc: builder.ForPersistentVolumeClaim("ns", "bound-pvc"). Phase(corev1api.ClaimBound). Result(), expectWarningLog: true, expectDebugMessage: "", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { ib := &itemBackupper{ backupRequest: &Request{ SkippedPVTracker: NewSkipPVTracker(), }, } // Set up log capture logOutput := &bytes.Buffer{} logger := logrus.New() logger.SetOutput(logOutput) logger.SetLevel(logrus.DebugLevel) // Convert PVC to unstructured pvcData, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.pvc) require.NoError(t, err) obj := &unstructured.Unstructured{Object: pvcData} ib.unTrackSkippedPV(obj, kuberesource.PersistentVolumeClaims, logger) logStr := logOutput.String() if tc.expectWarningLog { assert.Contains(t, logStr, "level=warning") assert.Contains(t, logStr, "unable to get PV name, skip untracking.") } else { assert.NotContains(t, logStr, "level=warning") if tc.expectDebugMessage != "" { assert.Contains(t, logStr, "level=debug") assert.Contains(t, logStr, tc.expectDebugMessage) } } }) } } ================================================ FILE: pkg/backup/item_block_worker_pool.go ================================================ /* Copyright the Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backup import ( "context" "sync" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/runtime/schema" ) type ItemBlockWorkerPool struct { inputChannel chan ItemBlockInput wg *sync.WaitGroup logger logrus.FieldLogger cancelFunc context.CancelFunc } type ItemBlockInput struct { itemBlock *BackupItemBlock returnChan chan ItemBlockReturn } type ItemBlockReturn struct { itemBlock *BackupItemBlock resources []schema.GroupResource err error } func (p *ItemBlockWorkerPool) GetInputChannel() chan ItemBlockInput { return p.inputChannel } func StartItemBlockWorkerPool(ctx context.Context, workers int, log logrus.FieldLogger) *ItemBlockWorkerPool { // Buffer will hold up to 10 ItemBlocks waiting for processing inputChannel := make(chan ItemBlockInput, max(workers, 10)) ctx, cancelFunc := context.WithCancel(ctx) wg := &sync.WaitGroup{} for i := 0; i < workers; i++ { logger := log.WithField("worker", i) wg.Add(1) go processItemBlockWorker(ctx, inputChannel, logger, wg) } pool := &ItemBlockWorkerPool{ inputChannel: inputChannel, cancelFunc: cancelFunc, logger: log, wg: wg, } return pool } func (p *ItemBlockWorkerPool) Stop() { p.cancelFunc() p.logger.Info("ItemBlock worker stopping") p.wg.Wait() p.logger.Info("ItemBlock worker stopped") } func processItemBlockWorker(ctx context.Context, inputChannel chan ItemBlockInput, logger logrus.FieldLogger, wg *sync.WaitGroup) { for { select { case m := <-inputChannel: logger.Infof("processing ItemBlock for backup %v", m.itemBlock.itemBackupper.backupRequest.Name) grList := m.itemBlock.itemBackupper.kubernetesBackupper.backupItemBlock(m.itemBlock) logger.Infof("finished processing ItemBlock for backup %v", m.itemBlock.itemBackupper.backupRequest.Name) m.returnChan <- ItemBlockReturn{ itemBlock: m.itemBlock, resources: grList, err: nil, } case <-ctx.Done(): logger.Info("stopping ItemBlock worker") wg.Done() return } } } ================================================ FILE: pkg/backup/item_collector.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backup import ( "context" "encoding/json" "fmt" "os" "sort" "strings" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/tools/pager" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/discovery" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/plugin/velero" "github.com/vmware-tanzu/velero/pkg/util/collections" ) // itemCollector collects items from the Kubernetes API according to // the backup spec and writes them to files inside dir. type itemCollector struct { log logrus.FieldLogger backupRequest *Request discoveryHelper discovery.Helper dynamicFactory client.DynamicFactory cohabitatingResources map[string]*cohabitatingResource dir string pageSize int nsTracker nsTracker } // nsTracker is used to integrate several namespace filters together. // 1. Backup's namespace Include/Exclude filters; // 2. Backup's (Or)LabelSelector selected namespace; // 3. Backup's (Or)LabelSelector selected resources' namespaces. // // Rules: // // a. When backup namespace Include/Exclude filters get everything, // The namespaces, which do not have backup including resources, // are not collected. // // b. If the namespace I/E filters and the (Or)LabelSelectors selected // namespaces are different. The tracker takes the union of them. type nsTracker struct { singleLabelSelector labels.Selector orLabelSelector []labels.Selector namespaceFilter *collections.NamespaceIncludesExcludes logger logrus.FieldLogger namespaceMap map[string]bool } // track add the namespace into the namespaceMap. func (nt *nsTracker) track(ns string) { if nt.namespaceMap == nil { nt.namespaceMap = make(map[string]bool) } if _, ok := nt.namespaceMap[ns]; !ok { nt.namespaceMap[ns] = true } } // isTracked check whether the namespace's name exists in // namespaceMap. func (nt *nsTracker) isTracked(ns string) bool { if nt.namespaceMap != nil { return nt.namespaceMap[ns] } return false } // init initialize the namespaceMap, and add elements according to // namespace include/exclude filters and the backup label selectors. func (nt *nsTracker) init( unstructuredNSs []unstructured.Unstructured, singleLabelSelector labels.Selector, orLabelSelector []labels.Selector, namespaceFilter *collections.NamespaceIncludesExcludes, logger logrus.FieldLogger, ) { if nt.namespaceMap == nil { nt.namespaceMap = make(map[string]bool) } nt.singleLabelSelector = singleLabelSelector nt.orLabelSelector = orLabelSelector nt.namespaceFilter = namespaceFilter nt.logger = logger for _, namespace := range unstructuredNSs { ns := new(corev1api.Namespace) if err := runtime.DefaultUnstructuredConverter.FromUnstructured(namespace.UnstructuredContent(), ns); err != nil { nt.logger.WithError(err).Errorf("Fail to convert unstructured into namespace %s", namespace.GetName()) continue } if ns.Status.Phase != corev1api.NamespaceActive { nt.logger.Infof("Skip namespace %s because it's not in Active phase.", namespace.GetName()) continue } if nt.singleLabelSelector != nil && nt.singleLabelSelector.Matches(labels.Set(namespace.GetLabels())) { nt.logger.Debugf("Track namespace %s, because its labels match backup LabelSelector.", namespace.GetName(), ) nt.track(namespace.GetName()) continue } if len(nt.orLabelSelector) > 0 { for _, selector := range nt.orLabelSelector { if selector.Matches(labels.Set(namespace.GetLabels())) { nt.logger.Debugf("Track namespace %s, because its labels match the backup OrLabelSelector.", namespace.GetName(), ) nt.track(namespace.GetName()) continue } } } // Skip the backup when the backup's namespace filter has // default value, and the namespace doesn't match backup // LabelSelector and OrLabelSelector. // https://github.com/vmware-tanzu/velero/issues/7105 if nt.namespaceFilter.IncludeEverything() && (nt.singleLabelSelector != nil || len(nt.orLabelSelector) > 0) { continue } if nt.namespaceFilter.ShouldInclude(namespace.GetName()) { nt.logger.Debugf("Track namespace %s, because its name match the backup namespace filter.", namespace.GetName(), ) nt.track(namespace.GetName()) } } } // filterNamespaces filters the input resource list to remove the // namespaces not tracked by the nsTracker. func (nt *nsTracker) filterNamespaces( resources []*kubernetesResource, ) []*kubernetesResource { result := make([]*kubernetesResource, 0) for _, resource := range resources { if resource.groupResource != kuberesource.Namespaces || nt.isTracked(resource.name) { result = append(result, resource) } } return result } type kubernetesResource struct { groupResource schema.GroupResource preferredGVR schema.GroupVersionResource namespace, name, path string orderedResource bool // set to true during backup processing when added to an ItemBlock // or if the item is excluded from backup. inItemBlockOrExcluded bool // Kind is added to facilitate creating an itemKey for progress tracking kind string } // getItemsFromResourceIdentifiers get the kubernetesResources // specified by the input parameter resourceIDs. func (r *itemCollector) getItemsFromResourceIdentifiers( resourceIDs []velero.ResourceIdentifier, ) []*kubernetesResource { grResourceIDsMap := make(map[schema.GroupResource][]velero.ResourceIdentifier) for _, resourceID := range resourceIDs { grResourceIDsMap[resourceID.GroupResource] = append( grResourceIDsMap[resourceID.GroupResource], resourceID) } return r.getItems(grResourceIDsMap) } // getAllItems gets all backup-relevant items from all API groups. func (r *itemCollector) getAllItems() []*kubernetesResource { resources := r.getItems(nil) return r.nsTracker.filterNamespaces(resources) } // getItems gets all backup-relevant items from all API groups, // // If resourceIDsMap is nil, then all items from the cluster are // pulled for each API group, subject to include/exclude rules, // except the namespace, because the namespace filtering depends on // all namespaced-scoped resources. // // If resourceIDsMap is supplied, then only those resources are // returned, with the appropriate APIGroup information filled in. In // this case, include/exclude rules are not invoked, since we already // have the list of items, we just need the item collector/discovery // helper to fill in the missing GVR, etc. context. func (r *itemCollector) getItems( resourceIDsMap map[schema.GroupResource][]velero.ResourceIdentifier, ) []*kubernetesResource { var resources []*kubernetesResource for _, group := range r.discoveryHelper.Resources() { groupItems, err := r.getGroupItems(r.log, group, resourceIDsMap) if err != nil { r.log.WithError(err).WithField("apiGroup", group.String()). Error("Error collecting resources from API group") continue } resources = append(resources, groupItems...) } return resources } // getGroupItems collects all relevant items from a single API group. // If resourceIDsMap is supplied, then only those items are returned, // with GVR/APIResource metadata supplied. func (r *itemCollector) getGroupItems( log logrus.FieldLogger, group *metav1.APIResourceList, resourceIDsMap map[schema.GroupResource][]velero.ResourceIdentifier, ) ([]*kubernetesResource, error) { log = log.WithField("group", group.GroupVersion) log.Infof("Getting items for group") // Parse so we can check if this is the core group gv, err := schema.ParseGroupVersion(group.GroupVersion) if err != nil { return nil, errors.Wrapf(err, "error parsing GroupVersion %q", group.GroupVersion) } if gv.Group == "" { // This is the core group, so make sure we process in the following order: // pods, pvcs, pvs, everything else. sortCoreGroup(group) } var items []*kubernetesResource for _, resource := range group.APIResources { resourceItems, err := r.getResourceItems(log, gv, resource, resourceIDsMap) if err != nil { log.WithError(err).WithField("resource", resource.String()). Error("Error getting items for resource") continue } items = append(items, resourceItems...) } return items, nil } // sortResourcesByOrder sorts items by the names specified in "order". // Items are not in order will be put at the end in original order. func sortResourcesByOrder( log logrus.FieldLogger, items []*kubernetesResource, order []string, ) []*kubernetesResource { if len(order) == 0 { return items } log.Debugf("Sorting resources using the following order %v...", order) itemMap := make(map[string]*kubernetesResource) for _, item := range items { var fullname string if item.namespace != "" { fullname = fmt.Sprintf("%s/%s", item.namespace, item.name) } else { fullname = item.name } itemMap[fullname] = item } var sortedItems []*kubernetesResource // First select items from the order for _, name := range order { if item, ok := itemMap[name]; ok { item.orderedResource = true sortedItems = append(sortedItems, item) log.Debugf("%s added to sorted resource list.", item.name) delete(itemMap, name) } else { log.Warnf("Cannot find resource '%s'.", name) } } // Now append the rest in sortedGroupItems, maintain the original order for _, item := range items { var fullname string if item.namespace != "" { fullname = fmt.Sprintf("%s/%s", item.namespace, item.name) } else { fullname = item.name } if _, ok := itemMap[fullname]; !ok { //This item has been inserted in the result continue } sortedItems = append(sortedItems, item) log.Debugf("%s added to sorted resource list.", item.name) } return sortedItems } // getOrderedResourcesForType gets order of resourceType from orderResources. func getOrderedResourcesForType( orderedResources map[string]string, resourceType string, ) []string { if orderedResources == nil { return nil } orderStr, ok := orderedResources[resourceType] if !ok || len(orderStr) == 0 { return nil } orders := strings.Split(orderStr, ",") return orders } // getResourceItems collects all relevant items for a given group-version-resource. // If resourceIDsMap is supplied, the items will be pulled from here // rather than from the cluster and applying include/exclude rules. func (r *itemCollector) getResourceItems( log logrus.FieldLogger, gv schema.GroupVersion, resource metav1.APIResource, resourceIDsMap map[schema.GroupResource][]velero.ResourceIdentifier, ) ([]*kubernetesResource, error) { log = log.WithField("resource", resource.Name) log.Info("Getting items for resource") var ( gvr = gv.WithResource(resource.Name) gr = gvr.GroupResource() ) orders := getOrderedResourcesForType( r.backupRequest.Backup.Spec.OrderedResources, resource.Name, ) // Getting the preferred group version of this resource preferredGVR, _, err := r.discoveryHelper.ResourceFor(gr.WithVersion("")) if err != nil { return nil, errors.WithStack(err) } // If we have a resourceIDs map, then only return items listed in it if resourceIDsMap != nil { resourceIDs, ok := resourceIDsMap[gr] if !ok { log.Info("Skipping resource because no items found in supplied ResourceIdentifier list") return nil, nil } var items []*kubernetesResource for _, resourceID := range resourceIDs { log.WithFields( logrus.Fields{ "namespace": resourceID.Namespace, "name": resourceID.Name, }, ).Infof("Getting item") resourceClient, err := r.dynamicFactory.ClientForGroupVersionResource( gv, resource, resourceID.Namespace, ) if err != nil { log.WithError(errors.WithStack(err)).Error("Error getting client for resource") continue } unstructured, err := resourceClient.Get(resourceID.Name, metav1.GetOptions{}) if err != nil { log.WithError(errors.WithStack(err)).Error("Error getting item") continue } path, err := r.writeToFile(unstructured) if err != nil { log.WithError(err).Error("Error writing item to file") continue } items = append(items, &kubernetesResource{ groupResource: gr, preferredGVR: preferredGVR, namespace: resourceID.Namespace, name: resourceID.Name, path: path, kind: resource.Kind, }) } return items, nil } if !r.backupRequest.ResourceIncludesExcludes.ShouldInclude(gr.String()) { log.Infof("Skipping resource because it's excluded") return nil, nil } if cohabitator, found := r.cohabitatingResources[resource.Name]; found { if gv.Group == cohabitator.groupResource1.Group || gv.Group == cohabitator.groupResource2.Group { if cohabitator.seen { log.WithFields( logrus.Fields{ "cohabitatingResource1": cohabitator.groupResource1.String(), "cohabitatingResource2": cohabitator.groupResource2.String(), }, ).Infof("Skipping resource because it cohabitates and we've already processed it") return nil, nil } cohabitator.seen = true } } // Handle namespace resource here. // Namespace are filtered by namespace include/exclude filters, // backup LabelSelectors and OrLabelSelectors are checked too. if gr == kuberesource.Namespaces { return r.collectNamespaces( resource, gv, gr, preferredGVR, log, ) } clusterScoped := !resource.Namespaced namespacesToList := getNamespacesToList(r.backupRequest.NamespaceIncludesExcludes) // If we get here, we're backing up something other than namespaces if clusterScoped { namespacesToList = []string{""} } var items []*kubernetesResource for _, namespace := range namespacesToList { unstructuredItems, err := r.listResourceByLabelsPerNamespace( namespace, gr, gv, resource, log) if err != nil { continue } // Collect items in included Namespaces for i := range unstructuredItems { item := &unstructuredItems[i] path, err := r.writeToFile(item) if err != nil { log.WithError(err).Error("Error writing item to file") continue } items = append(items, &kubernetesResource{ groupResource: gr, preferredGVR: preferredGVR, namespace: item.GetNamespace(), name: item.GetName(), path: path, kind: resource.Kind, }) if item.GetNamespace() != "" { log.Debugf("Track namespace %s in nsTracker", item.GetNamespace()) r.nsTracker.track(item.GetNamespace()) } } } if len(orders) > 0 { items = sortResourcesByOrder(r.log, items, orders) } return items, nil } func (r *itemCollector) listResourceByLabelsPerNamespace( namespace string, gr schema.GroupResource, gv schema.GroupVersion, resource metav1.APIResource, logger logrus.FieldLogger, ) ([]unstructured.Unstructured, error) { // List items from Kubernetes API logger = logger.WithField("namespace", namespace) resourceClient, err := r.dynamicFactory.ClientForGroupVersionResource(gv, resource, namespace) if err != nil { logger.WithError(err).Error("Error getting dynamic client") return nil, err } var orLabelSelectors []string if r.backupRequest.Spec.OrLabelSelectors != nil { for _, s := range r.backupRequest.Spec.OrLabelSelectors { orLabelSelectors = append(orLabelSelectors, metav1.FormatLabelSelector(s)) } } else { orLabelSelectors = []string{} } logger.Info("Listing items") unstructuredItems := make([]unstructured.Unstructured, 0) // Listing items for orLabelSelectors errListingForNS := false for _, label := range orLabelSelectors { unstructuredItems, err = r.listItemsForLabel(unstructuredItems, gr, label, resourceClient) if err != nil { errListingForNS = true } } if errListingForNS { logger.WithError(err).Error("Error listing items") return nil, err } var labelSelector string if selector := r.backupRequest.Spec.LabelSelector; selector != nil { labelSelector = metav1.FormatLabelSelector(selector) } // Listing items for labelSelector (singular) if len(orLabelSelectors) == 0 { unstructuredItems, err = r.listItemsForLabel( unstructuredItems, gr, labelSelector, resourceClient, ) if err != nil { logger.WithError(err).Error("Error listing items") return nil, err } } logger.Infof("Retrieved %d items", len(unstructuredItems)) return unstructuredItems, nil } func (r *itemCollector) writeToFile(item *unstructured.Unstructured) (string, error) { f, err := os.CreateTemp(r.dir, "") if err != nil { return "", errors.Wrap(err, "error creating temp file") } defer f.Close() jsonBytes, err := json.Marshal(item) if err != nil { return "", errors.Wrap(err, "error converting item to JSON") } if _, err := f.Write(jsonBytes); err != nil { return "", errors.Wrap(err, "error writing JSON to file") } if err := f.Close(); err != nil { return "", errors.Wrap(err, "error closing file") } return f.Name(), nil } // sortCoreGroup sorts the core API group. func sortCoreGroup(group *metav1.APIResourceList) { sort.SliceStable(group.APIResources, func(i, j int) bool { return coreGroupResourcePriority(group.APIResources[i].Name) < coreGroupResourcePriority(group.APIResources[j].Name) }) } // These constants represent the relative priorities for resources in the core API group. We want to // ensure that we process pods, then pvcs, then pvs, then anything else. This ensures that when a // pod is backed up, we can perform a pre hook, then process pvcs and pvs (including taking a // snapshot), then perform a post hook on the pod. const ( pod = iota pvc pv other ) // coreGroupResourcePriority returns the relative priority of the resource, in the following order: // pods, pvcs, pvs, everything else. func coreGroupResourcePriority(resource string) int { switch strings.ToLower(resource) { case "pods": return pod case "persistentvolumeclaims": return pvc case "persistentvolumes": return pv } return other } // getNamespacesToList examines ie and resolves the includes and excludes to a full list of // namespaces to list. If ie is nil or it includes *, the result is just "" (list across all // namespaces). Otherwise, the result is a list of every included namespace minus all excluded ones. func getNamespacesToList(ie *collections.NamespaceIncludesExcludes) []string { if ie == nil { return []string{""} } if ie.ShouldInclude("*") { // "" means all namespaces return []string{""} } var list []string for _, i := range ie.GetIncludes() { if ie.ShouldInclude(i) { list = append(list, i) } } return list } type cohabitatingResource struct { resource string groupResource1 schema.GroupResource groupResource2 schema.GroupResource seen bool } func newCohabitatingResource(resource, group1, group2 string) *cohabitatingResource { return &cohabitatingResource{ resource: resource, groupResource1: schema.GroupResource{Group: group1, Resource: resource}, groupResource2: schema.GroupResource{Group: group2, Resource: resource}, seen: false, } } // function to process pager client calls when the pageSize is specified func (r *itemCollector) processPagerClientCalls( gr schema.GroupResource, label string, resourceClient client.Dynamic, ) (runtime.Object, error) { // If limit is positive, use a pager to split list over multiple requests // Use Velero's dynamic list function instead of the default listPager := pager.New(pager.SimplePageFunc(func(opts metav1.ListOptions) (runtime.Object, error) { return resourceClient.List(opts) })) // Use the page size defined in the server config // TODO allow configuration of page buffer size listPager.PageSize = int64(r.pageSize) // Add each item to temporary slice list, paginated, err := listPager.List(context.Background(), metav1.ListOptions{LabelSelector: label}) if err != nil { r.log.WithError(errors.WithStack(err)).Error("Error listing resources") return list, err } if !paginated { r.log.Infof("list for groupResource %s was not paginated", gr) } return list, nil } func (r *itemCollector) listItemsForLabel( unstructuredItems []unstructured.Unstructured, gr schema.GroupResource, label string, resourceClient client.Dynamic, ) ([]unstructured.Unstructured, error) { if r.pageSize > 0 { // process pager client calls list, err := r.processPagerClientCalls(gr, label, resourceClient) if err != nil { return unstructuredItems, err } err = meta.EachListItem(list, func(object runtime.Object) error { u, ok := object.(*unstructured.Unstructured) if !ok { r.log.WithError(errors.WithStack(fmt.Errorf("expected *unstructured.Unstructured but got %T", u))). Error("unable to understand entry in the list") return fmt.Errorf("expected *unstructured.Unstructured but got %T", u) } unstructuredItems = append(unstructuredItems, *u) return nil }) if err != nil { r.log.WithError(errors.WithStack(err)).Error("unable to understand paginated list") return unstructuredItems, err } } else { unstructuredList, err := resourceClient.List(metav1.ListOptions{LabelSelector: label}) if err != nil { r.log.WithError(errors.WithStack(err)).Error("Error listing items") return unstructuredItems, err } unstructuredItems = append(unstructuredItems, unstructuredList.Items...) } return unstructuredItems, nil } // collectNamespaces process namespace resource according to namespace filters. func (r *itemCollector) collectNamespaces( resource metav1.APIResource, gv schema.GroupVersion, gr schema.GroupResource, preferredGVR schema.GroupVersionResource, log logrus.FieldLogger, ) ([]*kubernetesResource, error) { resourceClient, err := r.dynamicFactory.ClientForGroupVersionResource(gv, resource, "") if err != nil { log.WithError(err).Error("Error getting dynamic client") return nil, errors.WithStack(err) } unstructuredList, err := resourceClient.List(metav1.ListOptions{}) activeNamespacesHashSet := make(map[string]bool) for _, namespace := range unstructuredList.Items { activeNamespacesHashSet[namespace.GetName()] = true } if err != nil { log.WithError(errors.WithStack(err)).Error("error list namespaces") return nil, errors.WithStack(err) } // Change to look at the struct includes/excludes // In case wildcards are expanded, we need to look at the struct includes/excludes for _, includedNSName := range r.backupRequest.NamespaceIncludesExcludes.GetIncludes() { nsExists := false // Skip checking the namespace existing when it's "*". if includedNSName == "*" { continue } if _, ok := activeNamespacesHashSet[includedNSName]; ok { nsExists = true } if !nsExists { log.Errorf("fail to get the namespace %s specified in backup.Spec.IncludedNamespaces", includedNSName) } } var singleSelector labels.Selector var orSelectors []labels.Selector if r.backupRequest.Backup.Spec.LabelSelector != nil { var err error singleSelector, err = metav1.LabelSelectorAsSelector( r.backupRequest.Backup.Spec.LabelSelector) if err != nil { log.WithError(err).Errorf("Fail to convert backup LabelSelector %s into selector.", metav1.FormatLabelSelector(r.backupRequest.Backup.Spec.LabelSelector)) } } if r.backupRequest.Backup.Spec.OrLabelSelectors != nil { for _, ls := range r.backupRequest.Backup.Spec.OrLabelSelectors { orSelector, err := metav1.LabelSelectorAsSelector(ls) if err != nil { log.WithError(err).Errorf("Fail to convert backup OrLabelSelector %s into selector.", metav1.FormatLabelSelector(ls)) } orSelectors = append(orSelectors, orSelector) } } r.nsTracker.init( unstructuredList.Items, singleSelector, orSelectors, r.backupRequest.NamespaceIncludesExcludes, log, ) var items []*kubernetesResource for index := range unstructuredList.Items { nsName := unstructuredList.Items[index].GetName() path, err := r.writeToFile(&unstructuredList.Items[index]) if err != nil { log.WithError(err).Errorf("Error writing item %s to file", nsName) continue } items = append(items, &kubernetesResource{ groupResource: gr, preferredGVR: preferredGVR, name: nsName, path: path, kind: resource.Kind, }) } return items, nil } ================================================ FILE: pkg/backup/item_collector_test.go ================================================ /* Copyright 2017, 2019, 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backup import ( "testing" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/test" "github.com/vmware-tanzu/velero/pkg/util/collections" ) func TestSortCoreGroup(t *testing.T) { group := &metav1.APIResourceList{ GroupVersion: "v1", APIResources: []metav1.APIResource{ {Name: "persistentvolumes"}, {Name: "configmaps"}, {Name: "antelopes"}, {Name: "persistentvolumeclaims"}, {Name: "pods"}, }, } sortCoreGroup(group) expected := []string{ "pods", "persistentvolumeclaims", "persistentvolumes", "configmaps", "antelopes", } for i, r := range group.APIResources { assert.Equal(t, expected[i], r.Name) } } func TestSortOrderedResource(t *testing.T) { log := logrus.StandardLogger() podResources := []*kubernetesResource{ {namespace: "ns1", name: "pod3"}, {namespace: "ns1", name: "pod1"}, {namespace: "ns1", name: "pod2"}, } order := []string{"ns1/pod2", "ns1/pod1"} expectedResources := []*kubernetesResource{ {namespace: "ns1", name: "pod2", orderedResource: true}, {namespace: "ns1", name: "pod1", orderedResource: true}, {namespace: "ns1", name: "pod3"}, } sortedResources := sortResourcesByOrder(log, podResources, order) assert.Equal(t, expectedResources, sortedResources) // Test cluster resources pvResources := []*kubernetesResource{ {name: "pv1"}, {name: "pv2"}, {name: "pv3"}, } pvOrder := []string{"pv5", "pv2", "pv1"} expectedPvResources := []*kubernetesResource{ {name: "pv2", orderedResource: true}, {name: "pv1", orderedResource: true}, {name: "pv3"}, } sortedPvResources := sortResourcesByOrder(log, pvResources, pvOrder) assert.Equal(t, expectedPvResources, sortedPvResources) } func TestFilterNamespaces(t *testing.T) { tests := []struct { name string resources []*kubernetesResource needToTrack string expectedResources []*kubernetesResource }{ { name: "Namespace include by the filter but not in namespacesContainResource", resources: []*kubernetesResource{ { groupResource: kuberesource.Namespaces, preferredGVR: kuberesource.Namespaces.WithVersion("v1"), name: "ns1", }, { groupResource: kuberesource.Namespaces, preferredGVR: kuberesource.Namespaces.WithVersion("v1"), name: "ns2", }, { groupResource: kuberesource.Pods, preferredGVR: kuberesource.Namespaces.WithVersion("v1"), name: "pod1", }, }, needToTrack: "ns1", expectedResources: []*kubernetesResource{ { groupResource: kuberesource.Namespaces, preferredGVR: kuberesource.Namespaces.WithVersion("v1"), name: "ns1", }, { groupResource: kuberesource.Pods, preferredGVR: kuberesource.Namespaces.WithVersion("v1"), name: "pod1", }, }, }, } for _, tc := range tests { t.Run(tc.name, func(*testing.T) { r := itemCollector{ backupRequest: &Request{}, } if tc.needToTrack != "" { r.nsTracker.track(tc.needToTrack) } require.Equal(t, tc.expectedResources, r.nsTracker.filterNamespaces(tc.resources)) }) } } func TestItemCollectorBackupNamespaces(t *testing.T) { tests := []struct { name string ie *collections.NamespaceIncludesExcludes namespaces []*corev1api.Namespace backup *velerov1api.Backup expectedTrackedNS []string converter runtime.UnstructuredConverter }{ { name: "ns filter by namespace IE filter", backup: builder.ForBackup("velero", "backup").Result(), ie: collections.NewNamespaceIncludesExcludes().Includes("ns1"), namespaces: []*corev1api.Namespace{ builder.ForNamespace("ns1").Phase(corev1api.NamespaceActive).Result(), builder.ForNamespace("ns2").Phase(corev1api.NamespaceActive).Result(), }, expectedTrackedNS: []string{"ns1"}, }, { name: "ns filter by backup labelSelector", backup: builder.ForBackup("velero", "backup").LabelSelector(&metav1.LabelSelector{ MatchLabels: map[string]string{"name": "ns1"}, }).Result(), ie: collections.NewNamespaceIncludesExcludes().Includes("*"), namespaces: []*corev1api.Namespace{ builder.ForNamespace("ns1").ObjectMeta(builder.WithLabels("name", "ns1")).Phase(corev1api.NamespaceActive).Result(), builder.ForNamespace("ns2").Phase(corev1api.NamespaceActive).Result(), }, expectedTrackedNS: []string{"ns1"}, }, { name: "ns filter by backup orLabelSelector", backup: builder.ForBackup("velero", "backup").OrLabelSelector([]*metav1.LabelSelector{ {MatchLabels: map[string]string{"name": "ns1"}}, }).Result(), ie: collections.NewNamespaceIncludesExcludes().Includes("*"), namespaces: []*corev1api.Namespace{ builder.ForNamespace("ns1").ObjectMeta(builder.WithLabels("name", "ns1")).Phase(corev1api.NamespaceActive).Result(), builder.ForNamespace("ns2").Phase(corev1api.NamespaceActive).Result(), }, expectedTrackedNS: []string{"ns1"}, }, { name: "ns not included by IE filter, but included by labelSelector", backup: builder.ForBackup("velero", "backup").LabelSelector(&metav1.LabelSelector{ MatchLabels: map[string]string{"name": "ns1"}, }).Result(), ie: collections.NewNamespaceIncludesExcludes().Excludes("ns1"), namespaces: []*corev1api.Namespace{ builder.ForNamespace("ns1").ObjectMeta(builder.WithLabels("name", "ns1")).Phase(corev1api.NamespaceActive).Result(), builder.ForNamespace("ns2").Phase(corev1api.NamespaceActive).Result(), }, expectedTrackedNS: []string{"ns1"}, }, { name: "ns not included by IE filter, but included by orLabelSelector", backup: builder.ForBackup("velero", "backup").OrLabelSelector([]*metav1.LabelSelector{ {MatchLabels: map[string]string{"name": "ns1"}}, }).Result(), ie: collections.NewNamespaceIncludesExcludes().Excludes("ns1", "ns2"), namespaces: []*corev1api.Namespace{ builder.ForNamespace("ns1").ObjectMeta(builder.WithLabels("name", "ns1")).Phase(corev1api.NamespaceActive).Result(), builder.ForNamespace("ns2").Phase(corev1api.NamespaceActive).Result(), builder.ForNamespace("ns3").Phase(corev1api.NamespaceActive).Result(), }, expectedTrackedNS: []string{"ns1", "ns3"}, }, { name: "No ns filters", backup: builder.ForBackup("velero", "backup").Result(), ie: collections.NewNamespaceIncludesExcludes().Includes("*"), namespaces: []*corev1api.Namespace{ builder.ForNamespace("ns1").ObjectMeta(builder.WithLabels("name", "ns1")).Phase(corev1api.NamespaceActive).Result(), builder.ForNamespace("ns2").Phase(corev1api.NamespaceActive).Result(), }, expectedTrackedNS: []string{"ns1", "ns2"}, }, { name: "ns specified by the IncludeNamespaces cannot be found", backup: builder.ForBackup("velero", "backup").IncludedNamespaces("ns1", "invalid", "*").Result(), ie: collections.NewNamespaceIncludesExcludes().Includes("ns1", "invalid", "*"), namespaces: []*corev1api.Namespace{ builder.ForNamespace("ns1").ObjectMeta(builder.WithLabels("name", "ns1")).Phase(corev1api.NamespaceActive).Result(), builder.ForNamespace("ns2").Phase(corev1api.NamespaceActive).Result(), builder.ForNamespace("ns3").Phase(corev1api.NamespaceActive).Result(), }, expectedTrackedNS: []string{"ns1"}, }, { name: "terminating ns should not tracked", backup: builder.ForBackup("velero", "backup").Result(), ie: collections.NewNamespaceIncludesExcludes().Includes("ns1", "ns2"), namespaces: []*corev1api.Namespace{ builder.ForNamespace("ns1").Phase(corev1api.NamespaceTerminating).Result(), builder.ForNamespace("ns2").Phase(corev1api.NamespaceActive).Result(), }, expectedTrackedNS: []string{"ns2"}, }, } for _, tc := range tests { t.Run(tc.name, func(*testing.T) { tempDir := t.TempDir() var unstructuredNSList unstructured.UnstructuredList for _, ns := range tc.namespaces { unstructuredNS, err := runtime.DefaultUnstructuredConverter.ToUnstructured(ns) require.NoError(t, err) unstructuredNSList.Items = append(unstructuredNSList.Items, unstructured.Unstructured{Object: unstructuredNS}) } dc := &test.FakeDynamicClient{} dc.On("List", mock.Anything).Return(&unstructuredNSList, nil) factory := &test.FakeDynamicFactory{} factory.On( "ClientForGroupVersionResource", mock.Anything, mock.Anything, mock.Anything, ).Return(dc, nil) r := itemCollector{ backupRequest: &Request{ Backup: tc.backup, NamespaceIncludesExcludes: tc.ie, }, dynamicFactory: factory, dir: tempDir, } if tc.converter == nil { tc.converter = runtime.DefaultUnstructuredConverter } r.collectNamespaces( metav1.APIResource{ Name: "Namespace", Kind: "Namespace", Namespaced: false, }, kuberesource.Namespaces.WithVersion("").GroupVersion(), kuberesource.Namespaces, kuberesource.Namespaces.WithVersion(""), logrus.StandardLogger(), ) for _, ns := range tc.expectedTrackedNS { require.True(t, r.nsTracker.isTracked(ns)) } }) } } ================================================ FILE: pkg/backup/itemblock.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backup import ( "encoding/json" "os" "github.com/pkg/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "github.com/vmware-tanzu/velero/pkg/itemblock" ) type BackupItemBlock struct { itemblock.ItemBlock // This is a reference to the shared itemBackupper for the backup itemBackupper *itemBackupper } func NewBackupItemBlock(log logrus.FieldLogger, itemBackupper *itemBackupper) *BackupItemBlock { return &BackupItemBlock{ ItemBlock: itemblock.ItemBlock{Log: log}, itemBackupper: itemBackupper, } } func (b *BackupItemBlock) addKubernetesResource(item *kubernetesResource, log logrus.FieldLogger) *unstructured.Unstructured { // no-op if item has already been processed (in a block or previously excluded) if item.inItemBlockOrExcluded { return nil } var unstructured unstructured.Unstructured item.inItemBlockOrExcluded = true f, err := os.Open(item.path) if err != nil { log.WithError(errors.WithStack(err)).Error("Error opening file containing item") return nil } defer f.Close() defer os.Remove(f.Name()) if err := json.NewDecoder(f).Decode(&unstructured); err != nil { log.WithError(errors.WithStack(err)).Error("Error decoding JSON from file") return nil } metadata, err := meta.Accessor(&unstructured) if err != nil { log.WithError(errors.WithStack(err)).Warn("Error accessing item metadata") return nil } // Don't add to ItemBlock if item is excluded // itemInclusionChecks logs the reason if !b.itemBackupper.itemInclusionChecks(log, false, metadata, &unstructured, item.groupResource) { return nil } log.Infof("adding %s %s/%s to ItemBlock", item.groupResource, item.namespace, item.name) b.AddUnstructured(item.groupResource, &unstructured, item.preferredGVR) return &unstructured } ================================================ FILE: pkg/backup/pv_skip_tracker.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backup import ( "sort" "sync" ) type SkippedPV struct { Name string `json:"name"` Reasons []PVSkipReason `json:"reasons"` } func (s *SkippedPV) SerializeSkipReasons() string { ret := "" for _, reason := range s.Reasons { ret = ret + reason.Approach + ": " + reason.Reason + ";" } return ret } type PVSkipReason struct { Approach string `json:"approach"` Reason string `json:"reason"` } // skipPVTracker keeps track of persistent volumes that have been skipped and the reason why they are skipped. type skipPVTracker struct { *sync.RWMutex // pvs is a map of name of the pv to the list of reasons why it is skipped. // The reasons are stored in a map each key of the map is the backup approach, each approach can have one reason pvs map[string]map[string]string // includedPVs is a set of pv to be included in the backup, the element in this set should not be in the "pvs" map includedPVs map[string]struct{} } const ( podVolumeApproach = "podvolume" csiSnapshotApproach = "csiSnapshot" volumeSnapshotApproach = "volumeSnapshot" vsphereSnapshotApproach = "vsphereSnapshot" anyApproach = "any" ) func NewSkipPVTracker() *skipPVTracker { return &skipPVTracker{ RWMutex: &sync.RWMutex{}, pvs: make(map[string]map[string]string), includedPVs: make(map[string]struct{}), } } // Track tracks the pv with the specified name and the reason why it is skipped func (pt *skipPVTracker) Track(name, approach, reason string) { pt.Lock() defer pt.Unlock() if name == "" || reason == "" { return } if _, ok := pt.includedPVs[name]; ok { return } skipReasons := pt.pvs[name] if skipReasons == nil { skipReasons = make(map[string]string) pt.pvs[name] = skipReasons } if approach == "" { approach = anyApproach } skipReasons[approach] = reason } // Untrack removes the pvc with the specified namespace and name. // This func should be called when the PV is taken for snapshot, regardless native snapshot, CSI snapshot or fsb backup // therefore, in one backup processed if a PV is Untracked once, it will not be tracked again. func (pt *skipPVTracker) Untrack(name string) { pt.Lock() defer pt.Unlock() pt.includedPVs[name] = struct{}{} delete(pt.pvs, name) } // Summary returns the summary of the tracked pvcs. func (pt *skipPVTracker) Summary() []SkippedPV { pt.RLock() defer pt.RUnlock() keys := make([]string, 0, len(pt.pvs)) for key := range pt.pvs { keys = append(keys, key) } sort.Strings(keys) res := make([]SkippedPV, 0, len(keys)) for _, key := range keys { if skipReasons := pt.pvs[key]; len(skipReasons) > 0 { entry := SkippedPV{ Name: key, Reasons: make([]PVSkipReason, 0, len(skipReasons)), } approaches := make([]string, 0, len(skipReasons)) for a := range skipReasons { approaches = append(approaches, a) } sort.Strings(approaches) for _, a := range approaches { entry.Reasons = append(entry.Reasons, PVSkipReason{ Approach: a, Reason: skipReasons[a], }) } res = append(res, entry) } } return res } ================================================ FILE: pkg/backup/pv_skip_tracker_test.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backup import ( "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestSummary(t *testing.T) { tracker := NewSkipPVTracker() tracker.Track("pv5", "", "skipped due to policy") tracker.Track("pv3", podVolumeApproach, "it's set to opt-out") tracker.Track("pv3", csiSnapshotApproach, "not applicable for CSI ") // shouldn't be added tracker.Track("", podVolumeApproach, "pvc3 is set to be skipped") tracker.Track("pv10", volumeSnapshotApproach, "added by mistake") tracker.Untrack("pv10") expected := []SkippedPV{ { Name: "pv3", Reasons: []PVSkipReason{ { Approach: csiSnapshotApproach, Reason: "not applicable for CSI ", }, { Approach: podVolumeApproach, Reason: "it's set to opt-out", }, }, }, { Name: "pv5", Reasons: []PVSkipReason{ { Approach: anyApproach, Reason: "skipped due to policy", }, }, }, } assert.Equal(t, expected, tracker.Summary()) } func TestSerializeSkipReasons(t *testing.T) { tracker := NewSkipPVTracker() //tracker.Track("pv5", "", "skipped due to policy") tracker.Track("pv3", podVolumeApproach, "it's set to opt-out") tracker.Track("pv3", csiSnapshotApproach, "not applicable for CSI ") for _, skippedPV := range tracker.Summary() { require.Equal(t, "csiSnapshot: not applicable for CSI ;podvolume: it's set to opt-out;", skippedPV.SerializeSkipReasons()) } } func TestTrackUntrack(t *testing.T) { // If a pv is untracked explicitly it can't be Tracked again, b/c the pv is considered backed up already. tracker := NewSkipPVTracker() tracker.Track("pv3", podVolumeApproach, "it's set to opt-out") tracker.Untrack("pv3") tracker.Track("pv3", csiSnapshotApproach, "not applicable for CSI ") assert.Empty(t, tracker.Summary()) } ================================================ FILE: pkg/backup/request.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backup import ( "sync" "github.com/vmware-tanzu/velero/internal/hook" "github.com/vmware-tanzu/velero/internal/resourcepolicies" "github.com/vmware-tanzu/velero/internal/volume" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/itemoperation" "github.com/vmware-tanzu/velero/pkg/plugin/framework" "github.com/vmware-tanzu/velero/pkg/util/collections" ) type itemKey struct { resource string namespace string name string } type SynchronizedVSList struct { sync.Mutex VolumeSnapshotList []*volume.Snapshot } func (s *SynchronizedVSList) Add(vs *volume.Snapshot) { s.Lock() defer s.Unlock() s.VolumeSnapshotList = append(s.VolumeSnapshotList, vs) } func (s *SynchronizedVSList) Get() []*volume.Snapshot { s.Lock() defer s.Unlock() return s.VolumeSnapshotList } // Request is a request for a backup, with all references to other objects // materialized (e.g. backup/snapshot locations, includes/excludes, etc.) type Request struct { *velerov1api.Backup StorageLocation *velerov1api.BackupStorageLocation SnapshotLocations []*velerov1api.VolumeSnapshotLocation NamespaceIncludesExcludes *collections.NamespaceIncludesExcludes ResourceIncludesExcludes collections.IncludesExcludesInterface ResourceHooks []hook.ResourceHook ResolvedActions []framework.BackupItemResolvedActionV2 ResolvedItemBlockActions []framework.ItemBlockResolvedAction VolumeSnapshots SynchronizedVSList PodVolumeBackups []*velerov1api.PodVolumeBackup BackedUpItems *backedUpItemsMap itemOperationsList *[]*itemoperation.BackupOperation ResPolicies *resourcepolicies.Policies SkippedPVTracker *skipPVTracker VolumesInformation volume.BackupVolumesInformation WorkerPool *ItemBlockWorkerPool } // BackupVolumesInformation contains the information needs by generating // the backup BackupVolumeInfo array. // GetItemOperationsList returns ItemOperationsList, initializing it if necessary func (r *Request) GetItemOperationsList() *[]*itemoperation.BackupOperation { if r.itemOperationsList == nil { list := []*itemoperation.BackupOperation{} r.itemOperationsList = &list } return r.itemOperationsList } // BackupResourceList returns the list of backed up resources grouped by the API // Version and Kind func (r *Request) BackupResourceList() map[string][]string { return r.BackedUpItems.ResourceMap() } func (r *Request) FillVolumesInformation() { skippedPVMap := make(map[string]string) for _, skippedPV := range r.SkippedPVTracker.Summary() { skippedPVMap[skippedPV.Name] = skippedPV.SerializeSkipReasons() } r.VolumesInformation.SkippedPVs = skippedPVMap r.VolumesInformation.NativeSnapshots = r.VolumeSnapshots.Get() r.VolumesInformation.PodVolumeBackups = r.PodVolumeBackups r.VolumesInformation.BackupOperations = *r.GetItemOperationsList() r.VolumesInformation.BackupName = r.Backup.Name } func (r *Request) StopWorkerPool() { r.WorkerPool.Stop() } ================================================ FILE: pkg/backup/request_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backup import ( "testing" "github.com/stretchr/testify/assert" ) func TestRequest_BackupResourceList(t *testing.T) { items := []itemKey{ { resource: "apps/v1/Deployment", name: "my-deploy", namespace: "default", }, { resource: "v1/Pod", name: "pod1", namespace: "ns1", }, { resource: "v1/Pod", name: "pod2", namespace: "ns2", }, { resource: "v1/PersistentVolume", name: "my-pv", }, } backedUpItems := NewBackedUpItemsMap() for _, it := range items { backedUpItems.AddItem(it) } req := Request{BackedUpItems: backedUpItems} assert.Equal(t, map[string][]string{ "apps/v1/Deployment": {"default/my-deploy"}, "v1/Pod": {"ns1/pod1", "ns2/pod2"}, "v1/PersistentVolume": {"my-pv"}, }, req.BackupResourceList()) } func TestRequest_BackupResourceListEntriesSorted(t *testing.T) { items := []itemKey{ { resource: "v1/Pod", name: "pod2", namespace: "ns2", }, { resource: "v1/Pod", name: "pod1", namespace: "ns1", }, } backedUpItems := NewBackedUpItemsMap() for _, it := range items { backedUpItems.AddItem(it) } req := Request{BackedUpItems: backedUpItems} assert.Equal(t, map[string][]string{ "v1/Pod": {"ns1/pod1", "ns2/pod2"}, }, req.BackupResourceList()) } ================================================ FILE: pkg/backup/snapshots.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backup import ( "context" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/util/sets" kbclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/features" "github.com/vmware-tanzu/velero/pkg/label" "github.com/vmware-tanzu/velero/pkg/util/boolptr" ) // GetBackupCSIResources is used to get CSI snapshot related resources. // Returns VolumeSnapshot, VolumeSnapshotContent, VolumeSnapshotClasses referenced func GetBackupCSIResources( client kbclient.Client, globalCRClient kbclient.Client, backup *velerov1api.Backup, backupLog logrus.FieldLogger, ) ( volumeSnapshots []snapshotv1api.VolumeSnapshot, volumeSnapshotContents []snapshotv1api.VolumeSnapshotContent, volumeSnapshotClasses []snapshotv1api.VolumeSnapshotClass, ) { if boolptr.IsSetToTrue(backup.Spec.SnapshotMoveData) { backupLog.Info("backup SnapshotMoveData is set to true, skip VolumeSnapshot resource persistence.") } else if features.IsEnabled(velerov1api.CSIFeatureFlag) { selector := label.NewSelectorForBackup(backup.Name) vscList := &snapshotv1api.VolumeSnapshotContentList{} vsList := new(snapshotv1api.VolumeSnapshotList) err := globalCRClient.List(context.TODO(), vsList, &kbclient.ListOptions{ LabelSelector: label.NewSelectorForBackup(backup.Name), }) if err != nil { backupLog.Error(err) } volumeSnapshots = append(volumeSnapshots, vsList.Items...) if err := client.List(context.Background(), vscList, &kbclient.ListOptions{LabelSelector: selector}); err != nil { backupLog.Error(err) } if len(vscList.Items) >= 0 { volumeSnapshotContents = vscList.Items } vsClassSet := sets.NewString() for index := range volumeSnapshotContents { // persist the volumesnapshotclasses referenced by vsc if volumeSnapshotContents[index].Spec.VolumeSnapshotClassName != nil && !vsClassSet.Has(*volumeSnapshotContents[index].Spec.VolumeSnapshotClassName) { vsClass := &snapshotv1api.VolumeSnapshotClass{} if err := client.Get(context.TODO(), kbclient.ObjectKey{Name: *volumeSnapshotContents[index].Spec.VolumeSnapshotClassName}, vsClass); err != nil { backupLog.Error(err) } else { vsClassSet.Insert(*volumeSnapshotContents[index].Spec.VolumeSnapshotClassName) volumeSnapshotClasses = append(volumeSnapshotClasses, *vsClass) } } } backup.Status.CSIVolumeSnapshotsAttempted = len(volumeSnapshots) } return volumeSnapshots, volumeSnapshotContents, volumeSnapshotClasses } ================================================ FILE: pkg/backup/volume_snapshotter_cache.go ================================================ package backup import ( "sync" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" vsv1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/volumesnapshotter/v1" ) type VolumeSnapshotterCache struct { cache map[string]vsv1.VolumeSnapshotter mutex sync.Mutex getter VolumeSnapshotterGetter } func NewVolumeSnapshotterCache(getter VolumeSnapshotterGetter) *VolumeSnapshotterCache { return &VolumeSnapshotterCache{ cache: make(map[string]vsv1.VolumeSnapshotter), getter: getter, } } func (c *VolumeSnapshotterCache) SetNX(location *velerov1api.VolumeSnapshotLocation) (vsv1.VolumeSnapshotter, error) { c.mutex.Lock() defer c.mutex.Unlock() if snapshotter, exists := c.cache[location.Name]; exists { return snapshotter, nil } snapshotter, err := c.getter.GetVolumeSnapshotter(location.Spec.Provider) if err != nil { return nil, err } if err := snapshotter.Init(location.Spec.Config); err != nil { return nil, err } c.cache[location.Name] = snapshotter return snapshotter, nil } ================================================ FILE: pkg/builder/backup_builder.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package builder import ( "fmt" "time" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/vmware-tanzu/velero/internal/resourcepolicies" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/sirupsen/logrus" "github.com/vmware-tanzu/velero/pkg/util/boolptr" "github.com/vmware-tanzu/velero/pkg/util/logging" ) /* Example usage: var backup = builder.ForBackup("velero", "backup-1"). ObjectMeta( builder.WithLabels("foo", "bar"), builder.WithClusterName("cluster-1"), ). SnapshotVolumes(true). Result() */ // BackupBuilder builds Backup objects. type BackupBuilder struct { object *velerov1api.Backup } // ForBackup is the constructor for a BackupBuilder. func ForBackup(ns, name string) *BackupBuilder { return &BackupBuilder{ object: &velerov1api.Backup{ TypeMeta: metav1.TypeMeta{ APIVersion: velerov1api.SchemeGroupVersion.String(), Kind: "Backup", }, ObjectMeta: metav1.ObjectMeta{ Namespace: ns, Name: name, }, }, } } // Result returns the built Backup. func (b *BackupBuilder) Result() *velerov1api.Backup { return b.object } // ObjectMeta applies functional options to the Backup's ObjectMeta. func (b *BackupBuilder) ObjectMeta(opts ...ObjectMetaOpt) *BackupBuilder { for _, opt := range opts { opt(b.object) } return b } // FromSchedule sets the Backup's spec and labels from the Schedule template func (b *BackupBuilder) FromSchedule(schedule *velerov1api.Schedule) *BackupBuilder { var labels map[string]string // Check if there's explicit Labels defined in the Schedule object template // and if present then copy it to the backup object. if schedule.Spec.Template.Metadata.Labels != nil { logger := logging.DefaultLogger(logging.LogLevelFlag(logrus.InfoLevel).Parse(), logging.NewFormatFlag().Parse()) labels = schedule.Spec.Template.Metadata.Labels logger.WithFields(logrus.Fields{ "backup": fmt.Sprintf("%s/%s", b.object.GetNamespace(), b.object.GetName()), "labels": schedule.Spec.Template.Metadata.Labels, }).Info("Schedule.template.metadata.labels set - using those labels instead of schedule.labels for backup object") } else { labels = schedule.Labels logrus.WithFields(logrus.Fields{ "backup": fmt.Sprintf("%s/%s", b.object.GetNamespace(), b.object.GetName()), "labels": schedule.Labels, }).Info("No Schedule.template.metadata.labels set - using Schedule.labels for backup object") } if labels == nil { labels = make(map[string]string) } labels[velerov1api.ScheduleNameLabel] = schedule.Name b.object.Spec = schedule.Spec.Template b.ObjectMeta(WithLabelsMap(labels)) if schedule.Annotations != nil { b.ObjectMeta(WithAnnotationsMap(schedule.Annotations)) } if boolptr.IsSetToTrue(schedule.Spec.UseOwnerReferencesInBackup) { b.object.SetOwnerReferences([]metav1.OwnerReference{ { APIVersion: velerov1api.SchemeGroupVersion.String(), Kind: "Schedule", Name: schedule.Name, UID: schedule.UID, Controller: boolptr.True(), }, }) } if schedule.Spec.Template.ResourcePolicy != nil { b.ResourcePolicies(schedule.Spec.Template.ResourcePolicy.Name) } return b } // IncludedNamespaces sets the Backup's included namespaces. func (b *BackupBuilder) IncludedNamespaces(namespaces ...string) *BackupBuilder { b.object.Spec.IncludedNamespaces = namespaces return b } // ExcludedNamespaces sets the Backup's excluded namespaces. func (b *BackupBuilder) ExcludedNamespaces(namespaces ...string) *BackupBuilder { b.object.Spec.ExcludedNamespaces = namespaces return b } // IncludedResources sets the Backup's included resources. func (b *BackupBuilder) IncludedResources(resources ...string) *BackupBuilder { b.object.Spec.IncludedResources = resources return b } // ExcludedResources sets the Backup's excluded resources. func (b *BackupBuilder) ExcludedResources(resources ...string) *BackupBuilder { b.object.Spec.ExcludedResources = resources return b } // IncludedClusterScopedResources sets the Backup's included cluster resources. func (b *BackupBuilder) IncludedClusterScopedResources(resources ...string) *BackupBuilder { b.object.Spec.IncludedClusterScopedResources = resources return b } // ExcludedClusterScopedResources sets the Backup's excluded cluster resources. func (b *BackupBuilder) ExcludedClusterScopedResources(resources ...string) *BackupBuilder { b.object.Spec.ExcludedClusterScopedResources = resources return b } // IncludedNamespaceScopedResources sets the Backup's included namespaced resources. func (b *BackupBuilder) IncludedNamespaceScopedResources(resources ...string) *BackupBuilder { b.object.Spec.IncludedNamespaceScopedResources = resources return b } // ExcludedNamespaceScopedResources sets the Backup's excluded namespaced resources. func (b *BackupBuilder) ExcludedNamespaceScopedResources(resources ...string) *BackupBuilder { b.object.Spec.ExcludedNamespaceScopedResources = resources return b } // IncludeClusterResources sets the Backup's "include cluster resources" flag. func (b *BackupBuilder) IncludeClusterResources(val bool) *BackupBuilder { b.object.Spec.IncludeClusterResources = &val return b } // LabelSelector sets the Backup's label selector. func (b *BackupBuilder) LabelSelector(selector *metav1.LabelSelector) *BackupBuilder { b.object.Spec.LabelSelector = selector return b } // OrLabelSelector sets the Backup's orLabelSelector set. func (b *BackupBuilder) OrLabelSelector(orSelectors []*metav1.LabelSelector) *BackupBuilder { b.object.Spec.OrLabelSelectors = orSelectors return b } // SnapshotVolumes sets the Backup's "snapshot volumes" flag. func (b *BackupBuilder) SnapshotVolumes(val bool) *BackupBuilder { b.object.Spec.SnapshotVolumes = &val return b } // DefaultVolumesToFsBackup sets the Backup's "DefaultVolumesToFsBackup" flag. func (b *BackupBuilder) DefaultVolumesToFsBackup(val bool) *BackupBuilder { b.object.Spec.DefaultVolumesToFsBackup = &val return b } // DefaultVolumesToRestic sets the Backup's "DefaultVolumesToRestic" flag. func (b *BackupBuilder) DefaultVolumesToRestic(val bool) *BackupBuilder { b.object.Spec.DefaultVolumesToRestic = &val return b } // Phase sets the Backup's phase. func (b *BackupBuilder) Phase(phase velerov1api.BackupPhase) *BackupBuilder { b.object.Status.Phase = phase return b } // Phase sets the Backup's queue position. func (b *BackupBuilder) QueuePosition(queuePos int) *BackupBuilder { b.object.Status.QueuePosition = queuePos return b } // StorageLocation sets the Backup's storage location. func (b *BackupBuilder) StorageLocation(location string) *BackupBuilder { b.object.Spec.StorageLocation = location return b } // VolumeSnapshotLocations sets the Backup's volume snapshot locations. func (b *BackupBuilder) VolumeSnapshotLocations(locations ...string) *BackupBuilder { b.object.Spec.VolumeSnapshotLocations = locations return b } // TTL sets the Backup's TTL. func (b *BackupBuilder) TTL(ttl time.Duration) *BackupBuilder { b.object.Spec.TTL.Duration = ttl return b } // VolumeGroupSnapshotLabelKey sets the label key to group PVCs for VolumeGroupSnapshot. func (b *BackupBuilder) VolumeGroupSnapshotLabelKey(labelKey string) *BackupBuilder { b.object.Spec.VolumeGroupSnapshotLabelKey = labelKey return b } // Expiration sets the Backup's expiration. func (b *BackupBuilder) Expiration(val time.Time) *BackupBuilder { b.object.Status.Expiration = &metav1.Time{Time: val} return b } // StartTimestamp sets the Backup's start timestamp. func (b *BackupBuilder) StartTimestamp(val time.Time) *BackupBuilder { b.object.Status.StartTimestamp = &metav1.Time{Time: val} return b } // CompletionTimestamp sets the Backup's completion timestamp. func (b *BackupBuilder) CompletionTimestamp(val time.Time) *BackupBuilder { b.object.Status.CompletionTimestamp = &metav1.Time{Time: val} return b } // Hooks sets the Backup's hooks. func (b *BackupBuilder) Hooks(hooks velerov1api.BackupHooks) *BackupBuilder { b.object.Spec.Hooks = hooks return b } // OrderedResources sets the Backup's OrderedResources func (b *BackupBuilder) OrderedResources(orders map[string]string) *BackupBuilder { b.object.Spec.OrderedResources = orders return b } // CSISnapshotTimeout sets the Backup's CSISnapshotTimeout func (b *BackupBuilder) CSISnapshotTimeout(timeout time.Duration) *BackupBuilder { b.object.Spec.CSISnapshotTimeout.Duration = timeout return b } // ItemOperationTimeout sets the Backup's ItemOperationTimeout func (b *BackupBuilder) ItemOperationTimeout(timeout time.Duration) *BackupBuilder { b.object.Spec.ItemOperationTimeout.Duration = timeout return b } // ResourcePolicies sets the Backup's resource polices. func (b *BackupBuilder) ResourcePolicies(name string) *BackupBuilder { b.object.Spec.ResourcePolicy = &corev1api.TypedLocalObjectReference{Kind: resourcepolicies.ConfigmapRefType, Name: name} return b } // SnapshotMoveData sets the Backup's "snapshot move data" flag. func (b *BackupBuilder) SnapshotMoveData(val bool) *BackupBuilder { b.object.Spec.SnapshotMoveData = &val return b } // DataMover sets the Backup's data mover func (b *BackupBuilder) DataMover(name string) *BackupBuilder { b.object.Spec.DataMover = name return b } // ParallelFilesUpload sets the Backup's uploader parallel uploads func (b *BackupBuilder) ParallelFilesUpload(parallel int) *BackupBuilder { if b.object.Spec.UploaderConfig == nil { b.object.Spec.UploaderConfig = &velerov1api.UploaderConfigForBackup{} } b.object.Spec.UploaderConfig.ParallelFilesUpload = parallel return b } // WithStatus sets the Backup's status. func (b *BackupBuilder) WithStatus(status velerov1api.BackupStatus) *BackupBuilder { b.object.Status = status return b } ================================================ FILE: pkg/builder/backup_storage_location_builder.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package builder import ( "time" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) // BackupStorageLocationBuilder builds BackupStorageLocation objects. type BackupStorageLocationBuilder struct { object *velerov1api.BackupStorageLocation } // ForBackupStorageLocation is the constructor for a BackupStorageLocationBuilder. func ForBackupStorageLocation(ns, name string) *BackupStorageLocationBuilder { return &BackupStorageLocationBuilder{ object: &velerov1api.BackupStorageLocation{ TypeMeta: metav1.TypeMeta{ APIVersion: velerov1api.SchemeGroupVersion.String(), Kind: "BackupStorageLocation", }, ObjectMeta: metav1.ObjectMeta{ Namespace: ns, Name: name, }, }, } } // Result returns the built BackupStorageLocation. func (b *BackupStorageLocationBuilder) Result() *velerov1api.BackupStorageLocation { return b.object } // ObjectMeta applies functional options to the BackupStorageLocation's ObjectMeta. func (b *BackupStorageLocationBuilder) ObjectMeta(opts ...ObjectMetaOpt) *BackupStorageLocationBuilder { for _, opt := range opts { opt(b.object) } return b } // Provider sets the BackupStorageLocation's provider. func (b *BackupStorageLocationBuilder) Provider(name string) *BackupStorageLocationBuilder { b.object.Spec.Provider = name return b } // Bucket sets the BackupStorageLocation's object storage bucket. func (b *BackupStorageLocationBuilder) Bucket(val string) *BackupStorageLocationBuilder { if b.object.Spec.StorageType.ObjectStorage == nil { b.object.Spec.StorageType.ObjectStorage = new(velerov1api.ObjectStorageLocation) } b.object.Spec.ObjectStorage.Bucket = val return b } // Prefix sets the BackupStorageLocation's object storage prefix. func (b *BackupStorageLocationBuilder) Prefix(val string) *BackupStorageLocationBuilder { if b.object.Spec.StorageType.ObjectStorage == nil { b.object.Spec.StorageType.ObjectStorage = new(velerov1api.ObjectStorageLocation) } b.object.Spec.ObjectStorage.Prefix = val return b } // CACert sets the BackupStorageLocation's object storage CACert. func (b *BackupStorageLocationBuilder) CACert(val []byte) *BackupStorageLocationBuilder { if b.object.Spec.StorageType.ObjectStorage == nil { b.object.Spec.StorageType.ObjectStorage = new(velerov1api.ObjectStorageLocation) } b.object.Spec.ObjectStorage.CACert = val return b } // CACertRef sets the BackupStorageLocation's object storage CACertRef (Secret reference). func (b *BackupStorageLocationBuilder) CACertRef(selector *corev1api.SecretKeySelector) *BackupStorageLocationBuilder { if b.object.Spec.StorageType.ObjectStorage == nil { b.object.Spec.StorageType.ObjectStorage = new(velerov1api.ObjectStorageLocation) } b.object.Spec.ObjectStorage.CACertRef = selector return b } // Default sets the BackupStorageLocation's is default or not func (b *BackupStorageLocationBuilder) Default(isDefault bool) *BackupStorageLocationBuilder { b.object.Spec.Default = isDefault return b } // AccessMode sets the BackupStorageLocation's access mode. func (b *BackupStorageLocationBuilder) AccessMode(accessMode velerov1api.BackupStorageLocationAccessMode) *BackupStorageLocationBuilder { b.object.Spec.AccessMode = accessMode return b } // ValidationFrequency sets the BackupStorageLocation's validation frequency. func (b *BackupStorageLocationBuilder) ValidationFrequency(frequency time.Duration) *BackupStorageLocationBuilder { b.object.Spec.ValidationFrequency = &metav1.Duration{Duration: frequency} return b } // LastValidationTime sets the BackupStorageLocation's last validated time. func (b *BackupStorageLocationBuilder) LastValidationTime(lastValidated time.Time) *BackupStorageLocationBuilder { b.object.Status.LastValidationTime = &metav1.Time{Time: lastValidated} return b } // Phase sets the BackupStorageLocation's status phase. func (b *BackupStorageLocationBuilder) Phase(phase velerov1api.BackupStorageLocationPhase) *BackupStorageLocationBuilder { b.object.Status.Phase = phase return b } // Credential sets the BackupStorageLocation's credential selector. func (b *BackupStorageLocationBuilder) Credential(selector *corev1api.SecretKeySelector) *BackupStorageLocationBuilder { b.object.Spec.Credential = selector return b } ================================================ FILE: pkg/builder/config_map_builder.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package builder import ( corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // ConfigMapBuilder builds ConfigMap objects. type ConfigMapBuilder struct { object *corev1api.ConfigMap } // ForConfigMap is the constructor for a ConfigMapBuilder. func ForConfigMap(ns, name string) *ConfigMapBuilder { return &ConfigMapBuilder{ object: &corev1api.ConfigMap{ TypeMeta: metav1.TypeMeta{ APIVersion: corev1api.SchemeGroupVersion.String(), Kind: "ConfigMap", }, ObjectMeta: metav1.ObjectMeta{ Namespace: ns, Name: name, }, }, } } // Result returns the built ConfigMap. func (b *ConfigMapBuilder) Result() *corev1api.ConfigMap { return b.object } // ObjectMeta applies functional options to the ConfigMap's ObjectMeta. func (b *ConfigMapBuilder) ObjectMeta(opts ...ObjectMetaOpt) *ConfigMapBuilder { for _, opt := range opts { opt(b.object) } return b } // Data set's the ConfigMap's data. func (b *ConfigMapBuilder) Data(vals ...string) *ConfigMapBuilder { b.object.Data = setMapEntries(b.object.Data, vals...) return b } ================================================ FILE: pkg/builder/container_builder.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package builder import ( "encoding/json" "strings" corev1api "k8s.io/api/core/v1" apimachineryRuntime "k8s.io/apimachinery/pkg/runtime" "github.com/vmware-tanzu/velero/pkg/label" ) // ContainerBuilder builds Container objects type ContainerBuilder struct { object *corev1api.Container } // ForContainer is the constructor for ContainerBuilder. func ForContainer(name, image string) *ContainerBuilder { return &ContainerBuilder{ object: &corev1api.Container{ Name: name, Image: image, }, } } // ForPluginContainer is a helper builder specifically for plugin init containers func ForPluginContainer(image string, pullPolicy corev1api.PullPolicy) *ContainerBuilder { volumeMount := ForVolumeMount("plugins", "/target").Result() return ForContainer(getName(image), image).PullPolicy(pullPolicy).VolumeMounts(volumeMount) } // getName returns the 'name' component of a docker image that includes the entire string // except the registry name, and transforms the combined string into a DNS-1123 compatible name // that fits within the 63-character limit for Kubernetes container names. func getName(image string) string { slashIndex := strings.Index(image, "/") slashCount := 0 if slashIndex >= 0 { slashCount = strings.Count(image[slashIndex:], "/") } start := 0 if slashCount > 1 || slashIndex == 0 { // always start after the first slash when there is a registry name // or if the string starts with a slash. start = slashIndex + 1 } // If the image spec is by digest, remove the digest. // If it is by tag, remove the tag. // Otherwise (implicit :latest) leave it alone. end := len(image) atIndex := strings.LastIndex(image, "@") if atIndex > 0 { end = atIndex } else { colonIndex := strings.LastIndex(image, ":") if colonIndex > 0 { end = colonIndex } } // https://github.com/distribution/distribution/blob/main/docs/spec/api.md#overview // valid repository names match the regex [a-z0-9]+(?:[._-][a-z0-9]+)* // image repository names can container [._] but [._] are not allowed in RFC-1123 labels. // replace '/', '_' and '.' with '-' re := strings.NewReplacer("/", "-", "_", "-", ".", "-") name := re.Replace(image[start:end]) // Ensure the name doesn't exceed Kubernetes container name length limit return label.GetValidName(name) } // Result returns the built Container. func (b *ContainerBuilder) Result() *corev1api.Container { return b.object } // ResultRawExtension returns the Container as runtime.RawExtension. func (b *ContainerBuilder) ResultRawExtension() apimachineryRuntime.RawExtension { result, err := json.Marshal(b.object) if err != nil { return apimachineryRuntime.RawExtension{} } return apimachineryRuntime.RawExtension{ Raw: result, } } // Args sets the container's Args. func (b *ContainerBuilder) Args(args ...string) *ContainerBuilder { b.object.Args = append(b.object.Args, args...) return b } // VolumeMounts sets the container's VolumeMounts. func (b *ContainerBuilder) VolumeMounts(volumeMounts ...*corev1api.VolumeMount) *ContainerBuilder { for _, v := range volumeMounts { b.object.VolumeMounts = append(b.object.VolumeMounts, *v) } return b } // Resources sets the container's Resources. func (b *ContainerBuilder) Resources(resources *corev1api.ResourceRequirements) *ContainerBuilder { b.object.Resources = *resources return b } // SecurityContext sets the container's SecurityContext. func (b *ContainerBuilder) SecurityContext(securityContext *corev1api.SecurityContext) *ContainerBuilder { b.object.SecurityContext = securityContext return b } func (b *ContainerBuilder) Env(vars ...*corev1api.EnvVar) *ContainerBuilder { for _, v := range vars { b.object.Env = append(b.object.Env, *v) } return b } func (b *ContainerBuilder) PullPolicy(pullPolicy corev1api.PullPolicy) *ContainerBuilder { b.object.ImagePullPolicy = pullPolicy return b } func (b *ContainerBuilder) Command(command []string) *ContainerBuilder { if b.object.Command == nil { b.object.Command = []string{} } b.object.Command = append(b.object.Command, command...) return b } ================================================ FILE: pkg/builder/container_builder_test.go ================================================ /* Copyright 2018, 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package builder import ( "testing" "github.com/stretchr/testify/assert" ) func TestGetName(t *testing.T) { tests := []struct { name string image string expected string }{ { name: "image name with registry hostname and tag", image: "gcr.io/my-repo/my-image:latest", expected: "my-repo-my-image", }, { name: "image name with registry hostname, without tag", image: "gcr.io/my-repo/my-image", expected: "my-repo-my-image", }, { name: "image name without registry hostname, with tag", image: "my-repo/my-image:latest", expected: "my-repo-my-image", }, { name: "image name without registry hostname, without tag", image: "my-repo/my-image", expected: "my-repo-my-image", }, { name: "image name with registry hostname and port, and tag", image: "mycustomregistry.io:8080/my-repo/my-image:latest", expected: "my-repo-my-image", }, { name: "image name with no / in it", image: "my-image", expected: "my-image", }, { name: "image name starting with / in it", image: "/my-image", expected: "my-image", }, { name: "image name with repo starting with a / as first char", image: "/my-repo/my-image", expected: "my-repo-my-image", }, { name: "image name with registry hostname, etoomany slashes, without tag", image: "gcr.io/my-repo/mystery/another/my-image", expected: "my-repo-mystery-another-my-image", }, { name: "image name with registry hostname starting with a / will include the registry name ¯\\_(ツ)_/¯", image: "/gcr.io/my-repo/mystery/another/my-image", expected: "gcr-io-my-repo-mystery-another-my-image", }, { name: "image repository names containing _ ", image: "projects.registry.vmware.com/tanzu_migrator/route-2-httpproxy:myTag", expected: "tanzu-migrator-route-2-httpproxy", }, { name: "image repository names containing . ", image: "projects.registry.vmware.com/tanzu.migrator/route-2-httpproxy:myTag", expected: "tanzu-migrator-route-2-httpproxy", }, { name: "pull by digest", image: "quay.io/vmware-tanzu/velero@sha256:a75f9e8c3ced3943515f249597be389f8233e1258d289b11184796edceaa7dab", expected: "vmware-tanzu-velero", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { assert.Equal(t, test.expected, getName(test.image)) }) } } func TestGetNameWithLongPaths(t *testing.T) { tests := []struct { name string image string validate func(t *testing.T, result string) }{ { name: "plugin with deeply nested repository path exceeding 63 characters", image: "arohcpsvcdev.azurecr.io/redhat-user-workloads/ocp-art-tenant/oadp-hypershift-oadp-plugin-main@sha256:adb840bf3890b4904a8cdda1a74c82cf8d96c52eba9944ac10e795335d6fd450", validate: func(t *testing.T, result string) { t.Helper() // Should not exceed DNS-1123 label limit of 63 characters assert.LessOrEqual(t, len(result), 63, "Container name must satisfy DNS-1123 label constraints (max 63 chars)") // Should be exactly 63 characters (truncated with hash) assert.Len(t, result, 63) // Should be deterministic result2 := getName("arohcpsvcdev.azurecr.io/redhat-user-workloads/ocp-art-tenant/oadp-hypershift-oadp-plugin-main@sha256:adb840bf3890b4904a8cdda1a74c82cf8d96c52eba9944ac10e795335d6fd450") assert.Equal(t, result, result2) }, }, { name: "plugin with normal path length (should remain unchanged)", image: "arohcpsvcdev.azurecr.io/konveyor/velero-plugin-for-microsoft-azure@sha256:b2db5f09da514e817a74c992dcca5f90b77c2ab0b2797eba947d224271d6070e", validate: func(t *testing.T, result string) { t.Helper() assert.Equal(t, "konveyor-velero-plugin-for-microsoft-azure", result) assert.LessOrEqual(t, len(result), 63) }, }, { name: "very long nested path", image: "registry.example.com/org/team/project/subproject/component/service/application-name-with-many-words:v1.2.3", validate: func(t *testing.T, result string) { t.Helper() assert.LessOrEqual(t, len(result), 63) }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { result := getName(test.image) test.validate(t, result) }) } } ================================================ FILE: pkg/builder/customresourcedefinition_v1beta1_builder.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package builder import ( apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // CustomResourceDefinitionV1Beta1Builder builds v1beta1 CustomResourceDefinition objects. type CustomResourceDefinitionV1Beta1Builder struct { object *apiextv1beta1.CustomResourceDefinition } // ForCustomResourceDefinitionV1Beta1 is the constructor for a CustomResourceDefinitionV1Beta1Builder. func ForCustomResourceDefinitionV1Beta1(name string) *CustomResourceDefinitionV1Beta1Builder { return &CustomResourceDefinitionV1Beta1Builder{ object: &apiextv1beta1.CustomResourceDefinition{ TypeMeta: metav1.TypeMeta{ APIVersion: apiextv1beta1.SchemeGroupVersion.String(), Kind: "CustomResourceDefinition", }, ObjectMeta: metav1.ObjectMeta{ Name: name, }, }, } } // Condition adds a CustomResourceDefinitionCondition objects to a CustomResourceDefinitionV1Beta1Builder. func (c *CustomResourceDefinitionV1Beta1Builder) Condition(cond apiextv1beta1.CustomResourceDefinitionCondition) *CustomResourceDefinitionV1Beta1Builder { c.object.Status.Conditions = append(c.object.Status.Conditions, cond) return c } // Result returns the built CustomResourceDefinition. func (c *CustomResourceDefinitionV1Beta1Builder) Result() *apiextv1beta1.CustomResourceDefinition { return c.object } // ObjectMeta applies functional options to the CustomResourceDefinition's ObjectMeta. func (c *CustomResourceDefinitionV1Beta1Builder) ObjectMeta(opts ...ObjectMetaOpt) *CustomResourceDefinitionV1Beta1Builder { for _, opt := range opts { opt(c.object) } return c } // CustomResourceDefinitionV1Beta1ConditionBuilder builds CustomResourceDefinitionV1Beta1Condition objects. type CustomResourceDefinitionV1Beta1ConditionBuilder struct { object apiextv1beta1.CustomResourceDefinitionCondition } // ForCustomResourceDefinitionV1Beta1Condition is the construction for a CustomResourceDefinitionV1Beta1ConditionBuilder. func ForCustomResourceDefinitionV1Beta1Condition() *CustomResourceDefinitionV1Beta1ConditionBuilder { return &CustomResourceDefinitionV1Beta1ConditionBuilder{ object: apiextv1beta1.CustomResourceDefinitionCondition{}, } } // Type sets the Condition's type. func (c *CustomResourceDefinitionV1Beta1ConditionBuilder) Type(t apiextv1beta1.CustomResourceDefinitionConditionType) *CustomResourceDefinitionV1Beta1ConditionBuilder { c.object.Type = t return c } // Status sets the Condition's status. func (c *CustomResourceDefinitionV1Beta1ConditionBuilder) Status(cs apiextv1beta1.ConditionStatus) *CustomResourceDefinitionV1Beta1ConditionBuilder { c.object.Status = cs return c } // Result returns the built v1beta1 CustomResourceDefinitionCondition. func (c *CustomResourceDefinitionV1Beta1ConditionBuilder) Result() apiextv1beta1.CustomResourceDefinitionCondition { return c.object } ================================================ FILE: pkg/builder/data_download_builder.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package builder import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/vmware-tanzu/velero/pkg/apis/velero/shared" velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" ) // DataDownloadBuilder builds DataDownload objects. type DataDownloadBuilder struct { object *velerov2alpha1api.DataDownload } // ForDataDownload is the constructor of DataDownloadBuilder func ForDataDownload(namespace, name string) *DataDownloadBuilder { return &DataDownloadBuilder{ object: &velerov2alpha1api.DataDownload{ TypeMeta: metav1.TypeMeta{ Kind: "DataDownload", APIVersion: velerov2alpha1api.SchemeGroupVersion.String(), }, ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: namespace, }, }, } } // Result returns the built DataDownload. func (d *DataDownloadBuilder) Result() *velerov2alpha1api.DataDownload { return d.object } // BackupStorageLocation sets the DataDownload's backup storage location. func (d *DataDownloadBuilder) BackupStorageLocation(name string) *DataDownloadBuilder { d.object.Spec.BackupStorageLocation = name return d } // Phase sets the DataDownload's phase. func (d *DataDownloadBuilder) Phase(phase velerov2alpha1api.DataDownloadPhase) *DataDownloadBuilder { d.object.Status.Phase = phase return d } // SnapshotID sets the DataDownload's SnapshotID. func (d *DataDownloadBuilder) SnapshotID(id string) *DataDownloadBuilder { d.object.Spec.SnapshotID = id return d } // DataMover sets the DataDownload's DataMover. func (d *DataDownloadBuilder) DataMover(dataMover string) *DataDownloadBuilder { d.object.Spec.DataMover = dataMover return d } // SourceNamespace sets the DataDownload's SourceNamespace. func (d *DataDownloadBuilder) SourceNamespace(sourceNamespace string) *DataDownloadBuilder { d.object.Spec.SourceNamespace = sourceNamespace return d } // TargetVolume sets the DataDownload's TargetVolume. func (d *DataDownloadBuilder) TargetVolume(targetVolume velerov2alpha1api.TargetVolumeSpec) *DataDownloadBuilder { d.object.Spec.TargetVolume = targetVolume return d } // Cancel sets the DataDownload's Cancel. func (d *DataDownloadBuilder) Cancel(cancel bool) *DataDownloadBuilder { d.object.Spec.Cancel = cancel return d } // OperationTimeout sets the DataDownload's OperationTimeout. func (d *DataDownloadBuilder) OperationTimeout(timeout metav1.Duration) *DataDownloadBuilder { d.object.Spec.OperationTimeout = timeout return d } // DataMoverConfig sets the DataDownload's DataMoverConfig. func (d *DataDownloadBuilder) DataMoverConfig(config *map[string]string) *DataDownloadBuilder { d.object.Spec.DataMoverConfig = *config return d } // ObjectMeta applies functional options to the DataDownload's ObjectMeta. func (d *DataDownloadBuilder) ObjectMeta(opts ...ObjectMetaOpt) *DataDownloadBuilder { for _, opt := range opts { opt(d.object) } return d } // Labels sets the DataDownload's Labels. func (d *DataDownloadBuilder) Labels(labels map[string]string) *DataDownloadBuilder { d.object.Labels = labels return d } // Annotations sets the DataDownload's Annotations. func (d *DataDownloadBuilder) Annotations(annotations map[string]string) *DataDownloadBuilder { d.object.Annotations = annotations return d } // StartTimestamp sets the DataDownload's StartTimestamp. func (d *DataDownloadBuilder) StartTimestamp(startTime *metav1.Time) *DataDownloadBuilder { d.object.Status.StartTimestamp = startTime return d } // CompletionTimestamp sets the DataDownload's StartTimestamp. func (d *DataDownloadBuilder) CompletionTimestamp(completionTimestamp *metav1.Time) *DataDownloadBuilder { d.object.Status.CompletionTimestamp = completionTimestamp return d } // Progress sets the DataDownload's Progress. func (d *DataDownloadBuilder) Progress(progress shared.DataMoveOperationProgress) *DataDownloadBuilder { d.object.Status.Progress = progress return d } // Node sets the DataDownload's Node. func (d *DataDownloadBuilder) Node(node string) *DataDownloadBuilder { d.object.Status.Node = node return d } // NodeOS sets the DataDownload's Node OS. func (d *DataDownloadBuilder) NodeOS(nodeOS velerov2alpha1api.NodeOS) *DataDownloadBuilder { d.object.Spec.NodeOS = nodeOS return d } // AcceptedByNode sets the DataDownload's AcceptedByNode. func (d *DataDownloadBuilder) AcceptedByNode(node string) *DataDownloadBuilder { d.object.Status.AcceptedByNode = node return d } // AcceptedTimestamp sets the DataDownload's AcceptedTimestamp. func (d *DataDownloadBuilder) AcceptedTimestamp(acceptedTimestamp *metav1.Time) *DataDownloadBuilder { d.object.Status.AcceptedTimestamp = acceptedTimestamp return d } // Finalizers sets the DataDownload's Finalizers. func (d *DataDownloadBuilder) Finalizers(finalizers []string) *DataDownloadBuilder { d.object.Finalizers = finalizers return d } // Message sets the DataDownload's Message. func (d *DataDownloadBuilder) Message(msg string) *DataDownloadBuilder { d.object.Status.Message = msg return d } ================================================ FILE: pkg/builder/data_upload_builder.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package builder import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/vmware-tanzu/velero/pkg/apis/velero/shared" velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" ) // DataUploadBuilder builds DataUpload objects type DataUploadBuilder struct { object *velerov2alpha1api.DataUpload } // ForDataUpload is the constructor for a DataUploadBuilder. func ForDataUpload(ns, name string) *DataUploadBuilder { return &DataUploadBuilder{ object: &velerov2alpha1api.DataUpload{ TypeMeta: metav1.TypeMeta{ APIVersion: velerov2alpha1api.SchemeGroupVersion.String(), Kind: "DataUpload", }, ObjectMeta: metav1.ObjectMeta{ Namespace: ns, Name: name, }, }, } } // Result returns the built DataUpload. func (d *DataUploadBuilder) Result() *velerov2alpha1api.DataUpload { return d.object } // BackupStorageLocation sets the DataUpload's backup storage location. func (d *DataUploadBuilder) BackupStorageLocation(name string) *DataUploadBuilder { d.object.Spec.BackupStorageLocation = name return d } // Phase sets the DataUpload's phase. func (d *DataUploadBuilder) Phase(phase velerov2alpha1api.DataUploadPhase) *DataUploadBuilder { d.object.Status.Phase = phase return d } // SnapshotID sets the DataUpload's SnapshotID. func (d *DataUploadBuilder) SnapshotID(id string) *DataUploadBuilder { d.object.Status.SnapshotID = id return d } // DataMover sets the DataUpload's DataMover. func (d *DataUploadBuilder) DataMover(dataMover string) *DataUploadBuilder { d.object.Spec.DataMover = dataMover return d } // SourceNamespace sets the DataUpload's SourceNamespace. func (d *DataUploadBuilder) SourceNamespace(sourceNamespace string) *DataUploadBuilder { d.object.Spec.SourceNamespace = sourceNamespace return d } // SourcePVC sets the DataUpload's SourcePVC. func (d *DataUploadBuilder) SourcePVC(sourcePVC string) *DataUploadBuilder { d.object.Spec.SourcePVC = sourcePVC return d } // SnapshotType sets the DataUpload's SnapshotType. func (d *DataUploadBuilder) SnapshotType(SnapshotType velerov2alpha1api.SnapshotType) *DataUploadBuilder { d.object.Spec.SnapshotType = SnapshotType return d } // Cancel sets the DataUpload's Cancel. func (d *DataUploadBuilder) Cancel(cancel bool) *DataUploadBuilder { d.object.Spec.Cancel = cancel return d } // OperationTimeout sets the DataUpload's OperationTimeout. func (d *DataUploadBuilder) OperationTimeout(timeout metav1.Duration) *DataUploadBuilder { d.object.Spec.OperationTimeout = timeout return d } // DataMoverConfig sets the DataUpload's DataMoverConfig. func (d *DataUploadBuilder) DataMoverConfig(config map[string]string) *DataUploadBuilder { d.object.Spec.DataMoverConfig = config return d } // CSISnapshot sets the DataUpload's CSISnapshot. func (d *DataUploadBuilder) CSISnapshot(cSISnapshot *velerov2alpha1api.CSISnapshotSpec) *DataUploadBuilder { d.object.Spec.CSISnapshot = cSISnapshot return d } // StartTimestamp sets the DataUpload's StartTimestamp. func (d *DataUploadBuilder) StartTimestamp(startTimestamp *metav1.Time) *DataUploadBuilder { d.object.Status.StartTimestamp = startTimestamp return d } // CompletionTimestamp sets the DataUpload's StartTimestamp. func (d *DataUploadBuilder) CompletionTimestamp(completionTimestamp *metav1.Time) *DataUploadBuilder { d.object.Status.CompletionTimestamp = completionTimestamp return d } // Labels sets the DataUpload's Labels. func (d *DataUploadBuilder) Labels(labels map[string]string) *DataUploadBuilder { d.object.Labels = labels return d } // Annotations sets the DataUpload's Annotations. func (d *DataUploadBuilder) Annotations(annotations map[string]string) *DataUploadBuilder { d.object.Annotations = annotations return d } // Progress sets the DataUpload's Progress. func (d *DataUploadBuilder) Progress(progress shared.DataMoveOperationProgress) *DataUploadBuilder { d.object.Status.Progress = progress return d } // IncrementalBytes sets the DataUpload's IncrementalBytes. func (d *DataUploadBuilder) IncrementalBytes(incrementalBytes int64) *DataUploadBuilder { d.object.Status.IncrementalBytes = incrementalBytes return d } // Node sets the DataUpload's Node. func (d *DataUploadBuilder) Node(node string) *DataUploadBuilder { d.object.Status.Node = node return d } // NodeOS sets the DataUpload's Node OS. func (d *DataUploadBuilder) NodeOS(nodeOS velerov2alpha1api.NodeOS) *DataUploadBuilder { d.object.Status.NodeOS = nodeOS return d } // AcceptedByNode sets the DataUpload's AcceptedByNode. func (d *DataUploadBuilder) AcceptedByNode(node string) *DataUploadBuilder { d.object.Status.AcceptedByNode = node return d } // AcceptedTimestamp sets the DataUpload's AcceptedTimestamp. func (d *DataUploadBuilder) AcceptedTimestamp(acceptedTimestamp *metav1.Time) *DataUploadBuilder { d.object.Status.AcceptedTimestamp = acceptedTimestamp return d } // Finalizers sets the DataUpload's Finalizers. func (d *DataUploadBuilder) Finalizers(finalizers []string) *DataUploadBuilder { d.object.Finalizers = finalizers return d } // Message sets the DataUpload's Message. func (d *DataUploadBuilder) Message(msg string) *DataUploadBuilder { d.object.Status.Message = msg return d } // TotalBytes sets the DataUpload's TotalBytes. func (d *DataUploadBuilder) TotalBytes(size int64) *DataUploadBuilder { d.object.Status.Progress.TotalBytes = size return d } ================================================ FILE: pkg/builder/delete_backup_request_builder.go ================================================ /* Copyright 2023 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package builder import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) // DeleteBackupRequestBuilder builds DeleteBackupRequest objects type DeleteBackupRequestBuilder struct { object *velerov1api.DeleteBackupRequest } // ForDeleteBackupRequest is the constructor for a DeleteBackupRequestBuilder. func ForDeleteBackupRequest(ns, name string) *DeleteBackupRequestBuilder { return &DeleteBackupRequestBuilder{ object: &velerov1api.DeleteBackupRequest{ TypeMeta: metav1.TypeMeta{ APIVersion: velerov1api.SchemeGroupVersion.String(), Kind: "DeleteBackupRequest", }, ObjectMeta: metav1.ObjectMeta{ Namespace: ns, Name: name, }, }, } } // Result returns the built DeleteBackupRequest. func (b *DeleteBackupRequestBuilder) Result() *velerov1api.DeleteBackupRequest { return b.object } // ObjectMeta applies functional options to the DeleteBackupRequest's ObjectMeta. func (b *DeleteBackupRequestBuilder) ObjectMeta(opts ...ObjectMetaOpt) *DeleteBackupRequestBuilder { for _, opt := range opts { opt(b.object) } return b } // BackupName sets the DeleteBackupRequest's backup name. func (b *DeleteBackupRequestBuilder) BackupName(name string) *DeleteBackupRequestBuilder { b.object.Spec.BackupName = name return b } // Phase sets the DeleteBackupRequest's phase. func (b *DeleteBackupRequestBuilder) Phase(phase velerov1api.DeleteBackupRequestPhase) *DeleteBackupRequestBuilder { b.object.Status.Phase = phase return b } // Errors sets the DeleteBackupRequest's errors. func (b *DeleteBackupRequestBuilder) Errors(errors ...string) *DeleteBackupRequestBuilder { b.object.Status.Errors = errors return b } ================================================ FILE: pkg/builder/deployment_builder.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package builder import ( appsv1api "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // DeploymentBuilder builds Deployment objects. type DeploymentBuilder struct { object *appsv1api.Deployment } // ForDeployment is the constructor for a DeploymentBuilder. func ForDeployment(ns, name string) *DeploymentBuilder { return &DeploymentBuilder{ object: &appsv1api.Deployment{ TypeMeta: metav1.TypeMeta{ APIVersion: appsv1api.SchemeGroupVersion.String(), Kind: "Deployment", }, ObjectMeta: metav1.ObjectMeta{ Namespace: ns, Name: name, }, }, } } // Result returns the built Deployment. func (b *DeploymentBuilder) Result() *appsv1api.Deployment { return b.object } // ObjectMeta applies functional options to the Deployment's ObjectMeta. func (b *DeploymentBuilder) ObjectMeta(opts ...ObjectMetaOpt) *DeploymentBuilder { for _, opt := range opts { opt(b.object) } return b } ================================================ FILE: pkg/builder/download_request_builder.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package builder import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) // DownloadRequestBuilder builds DownloadRequest objects. type DownloadRequestBuilder struct { object *velerov1api.DownloadRequest } // ForDownloadRequest is the constructor for a DownloadRequestBuilder. func ForDownloadRequest(ns, name string) *DownloadRequestBuilder { return &DownloadRequestBuilder{ object: &velerov1api.DownloadRequest{ TypeMeta: metav1.TypeMeta{ APIVersion: velerov1api.SchemeGroupVersion.String(), Kind: "DownloadRequest", }, ObjectMeta: metav1.ObjectMeta{ Namespace: ns, Name: name, }, }, } } // Result returns the built DownloadRequest. func (b *DownloadRequestBuilder) Result() *velerov1api.DownloadRequest { return b.object } // Phase sets the DownloadRequest's status phase. func (b *DownloadRequestBuilder) Phase(phase velerov1api.DownloadRequestPhase) *DownloadRequestBuilder { b.object.Status.Phase = phase return b } // Target sets the DownloadRequest's target kind and target name. func (b *DownloadRequestBuilder) Target(targetKind velerov1api.DownloadTargetKind, targetName string) *DownloadRequestBuilder { b.object.Spec.Target.Kind = targetKind b.object.Spec.Target.Name = targetName return b } ================================================ FILE: pkg/builder/item_operation_builder.go ================================================ /* Copyright 2023 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package builder import ( "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "github.com/vmware-tanzu/velero/pkg/itemoperation" "github.com/vmware-tanzu/velero/pkg/plugin/velero" ) // OperationStatusBuilder builds OperationStatus objects type OperationStatusBuilder struct { object *itemoperation.OperationStatus } // ForOperationStatus is the constructor for a OperationStatusBuilder. func ForOperationStatus() *OperationStatusBuilder { return &OperationStatusBuilder{ object: &itemoperation.OperationStatus{}, } } // Result returns the built OperationStatus. func (osb *OperationStatusBuilder) Result() *itemoperation.OperationStatus { return osb.object } // Phase sets the OperationStatus's phase. func (osb *OperationStatusBuilder) Phase(phase itemoperation.OperationPhase) *OperationStatusBuilder { osb.object.Phase = phase return osb } // Error sets the OperationStatus's error. func (osb *OperationStatusBuilder) Error(err string) *OperationStatusBuilder { osb.object.Error = err return osb } // Progress sets the OperationStatus's progress. func (osb *OperationStatusBuilder) Progress(nComplete int64, nTotal int64, operationUnits string) *OperationStatusBuilder { osb.object.NCompleted = nComplete osb.object.NTotal = nTotal osb.object.OperationUnits = operationUnits return osb } // Description sets the OperationStatus's description. func (osb *OperationStatusBuilder) Description(desc string) *OperationStatusBuilder { osb.object.Description = desc return osb } // Created sets the OperationStatus's creation timestamp. func (osb *OperationStatusBuilder) Created(t time.Time) *OperationStatusBuilder { osb.object.Created = &metav1.Time{Time: t} return osb } // Updated sets the OperationStatus's last update timestamp. func (osb *OperationStatusBuilder) Updated(t time.Time) *OperationStatusBuilder { osb.object.Updated = &metav1.Time{Time: t} return osb } // Started sets the OperationStatus's start timestamp. func (osb *OperationStatusBuilder) Started(t time.Time) *OperationStatusBuilder { osb.object.Started = &metav1.Time{Time: t} return osb } // BackupOperationBuilder builds BackupOperation objects type BackupOperationBuilder struct { object *itemoperation.BackupOperation } // ForBackupOperation is the constructor for a BackupOperationBuilder. func ForBackupOperation() *BackupOperationBuilder { return &BackupOperationBuilder{ object: &itemoperation.BackupOperation{}, } } // Result returns the built BackupOperation. func (bb *BackupOperationBuilder) Result() *itemoperation.BackupOperation { return bb.object } // BackupName sets the BackupOperation's backup name. func (bb *BackupOperationBuilder) BackupName(name string) *BackupOperationBuilder { bb.object.Spec.BackupName = name return bb } // OperationID sets the BackupOperation's operation ID. func (bb *BackupOperationBuilder) OperationID(id string) *BackupOperationBuilder { bb.object.Spec.OperationID = id return bb } // Status sets the BackupOperation's status. func (bb *BackupOperationBuilder) Status(status itemoperation.OperationStatus) *BackupOperationBuilder { bb.object.Status = status return bb } // ResourceIdentifier sets the BackupOperation's resource identifier. func (bb *BackupOperationBuilder) ResourceIdentifier(group, resource, ns, name string) *BackupOperationBuilder { bb.object.Spec.ResourceIdentifier = velero.ResourceIdentifier{ GroupResource: schema.GroupResource{ Group: group, Resource: resource, }, Namespace: ns, Name: name, } return bb } // BackupItemAction sets the BackupOperation's backup item action. func (bb *BackupOperationBuilder) BackupItemAction(bia string) *BackupOperationBuilder { bb.object.Spec.BackupItemAction = bia return bb } // PostOperationItem adds a post-operation item to the BackupOperation's list of post-operation items. func (bb *BackupOperationBuilder) PostOperationItem(group, resource, ns, name string) *BackupOperationBuilder { bb.object.Spec.PostOperationItems = append(bb.object.Spec.PostOperationItems, velero.ResourceIdentifier{ GroupResource: schema.GroupResource{ Group: group, Resource: resource, }, Namespace: ns, Name: name, }) return bb } // RestoreOperationBuilder builds RestoreOperation objects type RestoreOperationBuilder struct { object *itemoperation.RestoreOperation } // ForRestoreOperation is the constructor for a RestoreOperationBuilder. func ForRestoreOperation() *RestoreOperationBuilder { return &RestoreOperationBuilder{ object: &itemoperation.RestoreOperation{}, } } // Result returns the built RestoreOperation. func (rb *RestoreOperationBuilder) Result() *itemoperation.RestoreOperation { return rb.object } // RestoreName sets the RestoreOperation's restore name. func (rb *RestoreOperationBuilder) RestoreName(name string) *RestoreOperationBuilder { rb.object.Spec.RestoreName = name return rb } // OperationID sets the RestoreOperation's operation ID. func (rb *RestoreOperationBuilder) OperationID(id string) *RestoreOperationBuilder { rb.object.Spec.OperationID = id return rb } // RestoreItemAction sets the RestoreOperation's restore item action. func (rb *RestoreOperationBuilder) RestoreItemAction(ria string) *RestoreOperationBuilder { rb.object.Spec.RestoreItemAction = ria return rb } // Status sets the RestoreOperation's status. func (rb *RestoreOperationBuilder) Status(status itemoperation.OperationStatus) *RestoreOperationBuilder { rb.object.Status = status return rb } // ResourceIdentifier sets the RestoreOperation's resource identifier. func (rb *RestoreOperationBuilder) ResourceIdentifier(group, resource, ns, name string) *RestoreOperationBuilder { rb.object.Spec.ResourceIdentifier = velero.ResourceIdentifier{ GroupResource: schema.GroupResource{ Group: group, Resource: resource, }, Namespace: ns, Name: name, } return rb } ================================================ FILE: pkg/builder/job_builder.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package builder import ( batchv1api "k8s.io/api/batch/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) type JobBuilder struct { object *batchv1api.Job } // ForJob is the constructor for a JobBuilder. func ForJob(ns, name string) *JobBuilder { return &JobBuilder{ object: &batchv1api.Job{ TypeMeta: metav1.TypeMeta{ APIVersion: batchv1api.SchemeGroupVersion.String(), Kind: "Job", }, ObjectMeta: metav1.ObjectMeta{ Namespace: ns, Name: name, }, }, } } // Result returns the built Job. func (j *JobBuilder) Result() *batchv1api.Job { return j.object } // ObjectMeta applies functional options to the Job's ObjectMeta. func (j *JobBuilder) ObjectMeta(opts ...ObjectMetaOpt) *JobBuilder { for _, opt := range opts { opt(j.object) } return j } // Succeeded sets Succeeded on the Job's Status. func (j *JobBuilder) Succeeded(succeeded int) *JobBuilder { j.object.Status.Succeeded = int32(succeeded) return j } ================================================ FILE: pkg/builder/json_schema_props_builder.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package builder import ( apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" ) // JSONSchemaPropsBuilder builds JSONSchemaProps objects. type JSONSchemaPropsBuilder struct { object *apiextv1.JSONSchemaProps } // ForJSONSchemaPropsBuilder is the constructor for a JSONSchemaPropsBuilder. func ForJSONSchemaPropsBuilder() *JSONSchemaPropsBuilder { return &JSONSchemaPropsBuilder{ object: &apiextv1.JSONSchemaProps{}, } } // Maximum sets the Maximum field on a JSONSchemaPropsBuilder object. func (b *JSONSchemaPropsBuilder) Maximum(f float64) *JSONSchemaPropsBuilder { b.object.Maximum = &f return b } // Result returns the built object. func (b *JSONSchemaPropsBuilder) Result() *apiextv1.JSONSchemaProps { return b.object } ================================================ FILE: pkg/builder/namespace_builder.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package builder import ( corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // NamespaceBuilder builds Namespace objects. type NamespaceBuilder struct { object *corev1api.Namespace } // ForNamespace is the constructor for a NamespaceBuilder. func ForNamespace(name string) *NamespaceBuilder { return &NamespaceBuilder{ object: &corev1api.Namespace{ TypeMeta: metav1.TypeMeta{ APIVersion: corev1api.SchemeGroupVersion.String(), Kind: "Namespace", }, ObjectMeta: metav1.ObjectMeta{ Name: name, }, }, } } // Result returns the built Namespace. func (b *NamespaceBuilder) Result() *corev1api.Namespace { return b.object } // ObjectMeta applies functional options to the Namespace's ObjectMeta. func (b *NamespaceBuilder) ObjectMeta(opts ...ObjectMetaOpt) *NamespaceBuilder { for _, opt := range opts { opt(b.object) } return b } // Phase sets the namespace's phase func (b *NamespaceBuilder) Phase(val corev1api.NamespacePhase) *NamespaceBuilder { b.object.Status.Phase = val return b } ================================================ FILE: pkg/builder/node_builder.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package builder import ( corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // NodeBuilder builds Node objects. type NodeBuilder struct { object *corev1api.Node } // ForNode is the constructor for a NodeBuilder. func ForNode(name string) *NodeBuilder { return &NodeBuilder{ object: &corev1api.Node{ TypeMeta: metav1.TypeMeta{ APIVersion: corev1api.SchemeGroupVersion.String(), Kind: "Node", }, ObjectMeta: metav1.ObjectMeta{ Name: name, }, }, } } func (b *NodeBuilder) Labels(labels map[string]string) *NodeBuilder { b.object.Labels = labels return b } // Result returns the built Node. func (b *NodeBuilder) Result() *corev1api.Node { return b.object } ================================================ FILE: pkg/builder/node_selector_builder.go ================================================ /* Copyright 2023 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package builder import corev1api "k8s.io/api/core/v1" // NodeSelectorBuilder builds NodeSelector objects type NodeSelectorBuilder struct { object *corev1api.NodeSelector } // ForNodeSelector returns the NodeSelectorBuilder instance with given terms func ForNodeSelector(term ...corev1api.NodeSelectorTerm) *NodeSelectorBuilder { return &NodeSelectorBuilder{ object: &corev1api.NodeSelector{ NodeSelectorTerms: term, }, } } // Result returns the built NodeSelector func (b *NodeSelectorBuilder) Result() *corev1api.NodeSelector { return b.object } // NodeSelectorTermBuilder builds NodeSelectorTerm objects. type NodeSelectorTermBuilder struct { object *corev1api.NodeSelectorTerm } // NewNodeSelectorTermBuilder initializes an instance of NodeSelectorTermBuilder func NewNodeSelectorTermBuilder() *NodeSelectorTermBuilder { return &NodeSelectorTermBuilder{ object: &corev1api.NodeSelectorTerm{ MatchExpressions: make([]corev1api.NodeSelectorRequirement, 0), MatchFields: make([]corev1api.NodeSelectorRequirement, 0), }, } } // WithMatchExpression appends the MatchExpression to the NodeSelectorTerm func (ntb *NodeSelectorTermBuilder) WithMatchExpression(key string, op string, values ...string) *NodeSelectorTermBuilder { req := corev1api.NodeSelectorRequirement{ Key: key, Operator: corev1api.NodeSelectorOperator(op), Values: values, } ntb.object.MatchExpressions = append(ntb.object.MatchExpressions, req) return ntb } // WithMatchField appends the MatchField to the NodeSelectorTerm func (ntb *NodeSelectorTermBuilder) WithMatchField(key string, op string, values ...string) *NodeSelectorTermBuilder { req := corev1api.NodeSelectorRequirement{ Key: key, Operator: corev1api.NodeSelectorOperator(op), Values: values, } ntb.object.MatchFields = append(ntb.object.MatchFields, req) return ntb } // Result returns the built NodeSelectorTerm func (ntb *NodeSelectorTermBuilder) Result() *corev1api.NodeSelectorTerm { return ntb.object } ================================================ FILE: pkg/builder/object_meta.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package builder import ( "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" ) // ObjectMetaOpt is a functional option for ObjectMeta. type ObjectMetaOpt func(metav1.Object) // WithName is a functional option that applies the specified // name to an object. func WithName(val string) func(obj metav1.Object) { return func(obj metav1.Object) { obj.SetName(val) } } // WithResourceVersion is a functional option that applies the specified // resourceVersion to an object func WithResourceVersion(val string) func(obj metav1.Object) { return func(obj metav1.Object) { obj.SetResourceVersion(val) } } // WithLabels is a functional option that applies the specified // label keys/values to an object. func WithLabels(vals ...string) func(obj metav1.Object) { return func(obj metav1.Object) { obj.SetLabels(setMapEntries(obj.GetLabels(), vals...)) } } // WithLabelsMap is a functional option that applies the specified labels map to // an object. func WithLabelsMap(labels map[string]string) func(obj metav1.Object) { return func(obj metav1.Object) { objLabels := obj.GetLabels() if objLabels == nil { objLabels = make(map[string]string) } // If the label already exists in the object, it will be overwritten for k, v := range labels { objLabels[k] = v } obj.SetLabels(objLabels) } } // WithAnnotations is a functional option that applies the specified // annotation keys/values to an object. func WithAnnotations(vals ...string) func(obj metav1.Object) { return func(obj metav1.Object) { obj.SetAnnotations(setMapEntries(obj.GetAnnotations(), vals...)) } } // WithAnnotationsMap is a functional option that applies the specified annotations map to // an object. func WithAnnotationsMap(annotations map[string]string) func(obj metav1.Object) { return func(obj metav1.Object) { objAnnotations := obj.GetAnnotations() if objAnnotations == nil { objAnnotations = make(map[string]string) } // If the label already exists in the object, it will be overwritten for k, v := range annotations { objAnnotations[k] = v } obj.SetAnnotations(objAnnotations) } } func setMapEntries(m map[string]string, vals ...string) map[string]string { if m == nil { m = make(map[string]string) } // if we don't have a value for every key, add an empty // string at the end to serve as the value for the last // key. if len(vals)%2 != 0 { vals = append(vals, "") } for i := 0; i < len(vals); i += 2 { key := vals[i] val := vals[i+1] // If the label already exists in the object, it will be overwritten m[key] = val } return m } // WithFinalizers is a functional option that applies the specified // finalizers to an object. func WithFinalizers(vals ...string) func(obj metav1.Object) { return func(obj metav1.Object) { obj.SetFinalizers(vals) } } // WithDeletionTimestamp is a functional option that applies the specified // deletion timestamp to an object. func WithDeletionTimestamp(val time.Time) func(obj metav1.Object) { return func(obj metav1.Object) { obj.SetDeletionTimestamp(&metav1.Time{Time: val}) } } // WithUID is a functional option that applies the specified UID to an object. func WithUID(val string) func(obj metav1.Object) { return func(obj metav1.Object) { obj.SetUID(types.UID(val)) } } // WithGenerateName is a functional option that applies the specified generate name to an object. func WithGenerateName(val string) func(obj metav1.Object) { return func(obj metav1.Object) { obj.SetGenerateName(val) } } // WithManagedFields is a functional option that applies the specified managed fields to an object. func WithManagedFields(val []metav1.ManagedFieldsEntry) func(obj metav1.Object) { return func(obj metav1.Object) { obj.SetManagedFields(val) } } // WithCreationTimestamp is a functional option that applies the specified creationTimestamp func WithCreationTimestamp(t time.Time) func(obj metav1.Object) { return func(obj metav1.Object) { obj.SetCreationTimestamp(metav1.Time{Time: t}) } } // WithOwnerReference is a functional option that applies the specified OwnerReference to an object. func WithOwnerReference(val []metav1.OwnerReference) func(obj metav1.Object) { return func(obj metav1.Object) { obj.SetOwnerReferences(val) } } ================================================ FILE: pkg/builder/persistent_volume_builder.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package builder import ( corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // PersistentVolumeBuilder builds PersistentVolume objects. type PersistentVolumeBuilder struct { object *corev1api.PersistentVolume } // ForPersistentVolume is the constructor for a PersistentVolumeBuilder. func ForPersistentVolume(name string) *PersistentVolumeBuilder { return &PersistentVolumeBuilder{ object: &corev1api.PersistentVolume{ TypeMeta: metav1.TypeMeta{ APIVersion: corev1api.SchemeGroupVersion.String(), Kind: "PersistentVolume", }, ObjectMeta: metav1.ObjectMeta{ Name: name, }, }, } } // Result returns the built PersistentVolume. func (b *PersistentVolumeBuilder) Result() *corev1api.PersistentVolume { return b.object } // ObjectMeta applies functional options to the PersistentVolume's ObjectMeta. func (b *PersistentVolumeBuilder) ObjectMeta(opts ...ObjectMetaOpt) *PersistentVolumeBuilder { for _, opt := range opts { opt(b.object) } return b } // ReclaimPolicy sets the PersistentVolume's reclaim policy. func (b *PersistentVolumeBuilder) ReclaimPolicy(policy corev1api.PersistentVolumeReclaimPolicy) *PersistentVolumeBuilder { b.object.Spec.PersistentVolumeReclaimPolicy = policy return b } // ClaimRef sets the PersistentVolume's claim ref. func (b *PersistentVolumeBuilder) ClaimRef(ns, name string) *PersistentVolumeBuilder { b.object.Spec.ClaimRef = &corev1api.ObjectReference{ Namespace: ns, Name: name, } return b } // AWSEBSVolumeID sets the PersistentVolume's AWSElasticBlockStore volume ID. func (b *PersistentVolumeBuilder) AWSEBSVolumeID(volumeID string) *PersistentVolumeBuilder { if b.object.Spec.AWSElasticBlockStore == nil { b.object.Spec.AWSElasticBlockStore = new(corev1api.AWSElasticBlockStoreVolumeSource) } b.object.Spec.AWSElasticBlockStore.VolumeID = volumeID return b } // CSI sets the PersistentVolume's CSI. func (b *PersistentVolumeBuilder) CSI(driver, volumeHandle string) *PersistentVolumeBuilder { if b.object.Spec.CSI == nil { b.object.Spec.CSI = new(corev1api.CSIPersistentVolumeSource) } b.object.Spec.CSI.Driver = driver b.object.Spec.CSI.VolumeHandle = volumeHandle return b } // StorageClass sets the PersistentVolume's storage class name. func (b *PersistentVolumeBuilder) StorageClass(name string) *PersistentVolumeBuilder { b.object.Spec.StorageClassName = name return b } // VolumeMode sets the PersistentVolume's volume mode. func (b *PersistentVolumeBuilder) VolumeMode(volMode corev1api.PersistentVolumeMode) *PersistentVolumeBuilder { b.object.Spec.VolumeMode = &volMode return b } // NodeAffinityRequired sets the PersistentVolume's NodeAffinity Requirement. func (b *PersistentVolumeBuilder) NodeAffinityRequired(req *corev1api.NodeSelector) *PersistentVolumeBuilder { b.object.Spec.NodeAffinity = &corev1api.VolumeNodeAffinity{ Required: req, } return b } // Phase sets the PersistentVolume's phase. func (b *PersistentVolumeBuilder) Phase(phase corev1api.PersistentVolumePhase) *PersistentVolumeBuilder { b.object.Status.Phase = phase return b } ================================================ FILE: pkg/builder/persistent_volume_claim_builder.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package builder import ( corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // PersistentVolumeClaimBuilder builds PersistentVolumeClaim objects. type PersistentVolumeClaimBuilder struct { object *corev1api.PersistentVolumeClaim } // ForPersistentVolumeClaim is the constructor for a PersistentVolumeClaimBuilder. func ForPersistentVolumeClaim(ns, name string) *PersistentVolumeClaimBuilder { return &PersistentVolumeClaimBuilder{ object: &corev1api.PersistentVolumeClaim{ TypeMeta: metav1.TypeMeta{ APIVersion: corev1api.SchemeGroupVersion.String(), Kind: "PersistentVolumeClaim", }, ObjectMeta: metav1.ObjectMeta{ Namespace: ns, Name: name, }, }, } } // Result returns the built PersistentVolumeClaim. func (b *PersistentVolumeClaimBuilder) Result() *corev1api.PersistentVolumeClaim { return b.object } // ObjectMeta applies functional options to the PersistentVolumeClaim's ObjectMeta. func (b *PersistentVolumeClaimBuilder) ObjectMeta(opts ...ObjectMetaOpt) *PersistentVolumeClaimBuilder { for _, opt := range opts { opt(b.object) } return b } // VolumeName sets the PersistentVolumeClaim's volume name. func (b *PersistentVolumeClaimBuilder) VolumeName(name string) *PersistentVolumeClaimBuilder { b.object.Spec.VolumeName = name return b } // StorageClass sets the PersistentVolumeClaim's storage class name. func (b *PersistentVolumeClaimBuilder) StorageClass(name string) *PersistentVolumeClaimBuilder { b.object.Spec.StorageClassName = &name return b } // Phase sets the PersistentVolumeClaim's status Phase. func (b *PersistentVolumeClaimBuilder) Phase(phase corev1api.PersistentVolumeClaimPhase) *PersistentVolumeClaimBuilder { b.object.Status.Phase = phase return b } // RequestResource sets the PersistentVolumeClaim's spec.Resources.Requests. func (b *PersistentVolumeClaimBuilder) RequestResource(requests corev1api.ResourceList) *PersistentVolumeClaimBuilder { if b.object.Spec.Resources.Requests == nil { b.object.Spec.Resources.Requests = make(map[corev1api.ResourceName]resource.Quantity) } b.object.Spec.Resources.Requests = requests return b } // LimitResource sets the PersistentVolumeClaim's spec.Resources.Limits. func (b *PersistentVolumeClaimBuilder) LimitResource(limits corev1api.ResourceList) *PersistentVolumeClaimBuilder { if b.object.Spec.Resources.Limits == nil { b.object.Spec.Resources.Limits = make(map[corev1api.ResourceName]resource.Quantity) } b.object.Spec.Resources.Limits = limits return b } // DataSource sets the PersistentVolumeClaim's spec.DataSource. func (b *PersistentVolumeClaimBuilder) DataSource(dataSource *corev1api.TypedLocalObjectReference) *PersistentVolumeClaimBuilder { b.object.Spec.DataSource = dataSource return b } // DataSourceRef sets the PersistentVolumeClaim's spec.DataSourceRef. func (b *PersistentVolumeClaimBuilder) DataSourceRef(dataSourceRef *corev1api.TypedObjectReference) *PersistentVolumeClaimBuilder { b.object.Spec.DataSourceRef = dataSourceRef return b } // Selector sets the PersistentVolumeClaim's spec.Selector. func (b *PersistentVolumeClaimBuilder) Selector(labelSelector *metav1.LabelSelector) *PersistentVolumeClaimBuilder { b.object.Spec.Selector = labelSelector return b } ================================================ FILE: pkg/builder/pod_builder.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package builder import ( corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // PodBuilder builds Pod objects. type PodBuilder struct { object *corev1api.Pod } // ForPod is the constructor for a PodBuilder. func ForPod(ns, name string) *PodBuilder { return &PodBuilder{ object: &corev1api.Pod{ TypeMeta: metav1.TypeMeta{ APIVersion: corev1api.SchemeGroupVersion.String(), Kind: "Pod", }, ObjectMeta: metav1.ObjectMeta{ Namespace: ns, Name: name, }, }, } } // Result returns the built Pod. func (b *PodBuilder) Result() *corev1api.Pod { return b.object } // ObjectMeta applies functional options to the Pod's ObjectMeta. func (b *PodBuilder) ObjectMeta(opts ...ObjectMetaOpt) *PodBuilder { for _, opt := range opts { opt(b.object) } return b } // ServiceAccount sets serviceAccounts on pod. func (b *PodBuilder) ServiceAccount(sa string) *PodBuilder { b.object.Spec.ServiceAccountName = sa return b } // Volumes appends to the pod's volumes func (b *PodBuilder) Volumes(volumes ...*corev1api.Volume) *PodBuilder { for _, v := range volumes { b.object.Spec.Volumes = append(b.object.Spec.Volumes, *v) } return b } // NodeName sets the pod's node name func (b *PodBuilder) NodeName(val string) *PodBuilder { b.object.Spec.NodeName = val return b } func (b *PodBuilder) Labels(labels map[string]string) *PodBuilder { b.object.Labels = labels return b } func (b *PodBuilder) InitContainers(containers ...*corev1api.Container) *PodBuilder { for _, c := range containers { b.object.Spec.InitContainers = append(b.object.Spec.InitContainers, *c) } return b } func (b *PodBuilder) InitContainerState(state corev1api.ContainerState) *PodBuilder { b.object.Status.InitContainerStatuses = append(b.object.Status.InitContainerStatuses, corev1api.ContainerStatus{State: state}) return b } func (b *PodBuilder) Containers(containers ...*corev1api.Container) *PodBuilder { for _, c := range containers { b.object.Spec.Containers = append(b.object.Spec.Containers, *c) } return b } func (b *PodBuilder) ContainerStatuses(containerStatuses ...*corev1api.ContainerStatus) *PodBuilder { for _, c := range containerStatuses { b.object.Status.ContainerStatuses = append(b.object.Status.ContainerStatuses, *c) } return b } func (b *PodBuilder) Phase(phase corev1api.PodPhase) *PodBuilder { b.object.Status.Phase = phase return b } func (b *PodBuilder) Status(status corev1api.PodStatus) *PodBuilder { b.object.Status = status return b } ================================================ FILE: pkg/builder/pod_volume_backup_builder.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package builder import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) // PodVolumeBackupBuilder builds PodVolumeBackup objects type PodVolumeBackupBuilder struct { object *velerov1api.PodVolumeBackup } // ForPodVolumeBackup is the constructor for a PodVolumeBackupBuilder. func ForPodVolumeBackup(ns, name string) *PodVolumeBackupBuilder { return &PodVolumeBackupBuilder{ object: &velerov1api.PodVolumeBackup{ TypeMeta: metav1.TypeMeta{ APIVersion: velerov1api.SchemeGroupVersion.String(), Kind: "PodVolumeBackup", }, ObjectMeta: metav1.ObjectMeta{ Namespace: ns, Name: name, }, }, } } // Result returns the built PodVolumeBackup. func (b *PodVolumeBackupBuilder) Result() *velerov1api.PodVolumeBackup { return b.object } // ObjectMeta applies functional options to the PodVolumeBackup's ObjectMeta. func (b *PodVolumeBackupBuilder) ObjectMeta(opts ...ObjectMetaOpt) *PodVolumeBackupBuilder { for _, opt := range opts { opt(b.object) } return b } // Phase sets the PodVolumeBackup's phase. func (b *PodVolumeBackupBuilder) Phase(phase velerov1api.PodVolumeBackupPhase) *PodVolumeBackupBuilder { b.object.Status.Phase = phase return b } // Node sets the PodVolumeBackup's node name. func (b *PodVolumeBackupBuilder) Node(name string) *PodVolumeBackupBuilder { b.object.Spec.Node = name return b } // BackupStorageLocation sets the PodVolumeBackup's backup storage location. func (b *PodVolumeBackupBuilder) BackupStorageLocation(name string) *PodVolumeBackupBuilder { b.object.Spec.BackupStorageLocation = name return b } // SnapshotID sets the PodVolumeBackup's snapshot ID. func (b *PodVolumeBackupBuilder) SnapshotID(snapshotID string) *PodVolumeBackupBuilder { b.object.Status.SnapshotID = snapshotID return b } func (b *PodVolumeBackupBuilder) StartTimestamp(startTimestamp *metav1.Time) *PodVolumeBackupBuilder { b.object.Status.StartTimestamp = startTimestamp return b } func (b *PodVolumeBackupBuilder) CompletionTimestamp(completionTimestamp *metav1.Time) *PodVolumeBackupBuilder { b.object.Status.CompletionTimestamp = completionTimestamp return b } // PodName sets the name of the pod associated with this PodVolumeBackup. func (b *PodVolumeBackupBuilder) PodName(name string) *PodVolumeBackupBuilder { b.object.Spec.Pod.Name = name return b } // PodNamespace sets the name of the pod associated with this PodVolumeBackup. func (b *PodVolumeBackupBuilder) PodNamespace(ns string) *PodVolumeBackupBuilder { b.object.Spec.Pod.Namespace = ns return b } // Volume sets the name of the volume associated with this PodVolumeBackup. func (b *PodVolumeBackupBuilder) Volume(volume string) *PodVolumeBackupBuilder { b.object.Spec.Volume = volume return b } // UploaderType sets the type of uploader to use for this PodVolumeBackup. func (b *PodVolumeBackupBuilder) UploaderType(uploaderType string) *PodVolumeBackupBuilder { b.object.Spec.UploaderType = uploaderType return b } // Annotations sets the PodVolumeBackup's Annotations. func (b *PodVolumeBackupBuilder) Annotations(annotations map[string]string) *PodVolumeBackupBuilder { b.object.Annotations = annotations return b } // Cancel sets the PodVolumeBackup's Cancel. func (b *PodVolumeBackupBuilder) Cancel(cancel bool) *PodVolumeBackupBuilder { b.object.Spec.Cancel = cancel return b } // AcceptedTimestamp sets the PodVolumeBackup's AcceptedTimestamp. func (b *PodVolumeBackupBuilder) AcceptedTimestamp(acceptedTimestamp *metav1.Time) *PodVolumeBackupBuilder { b.object.Status.AcceptedTimestamp = acceptedTimestamp return b } // Finalizers sets the PodVolumeBackup's Finalizers. func (b *PodVolumeBackupBuilder) Finalizers(finalizers []string) *PodVolumeBackupBuilder { b.object.Finalizers = finalizers return b } // Message sets the PodVolumeBackup's Message. func (b *PodVolumeBackupBuilder) Message(msg string) *PodVolumeBackupBuilder { b.object.Status.Message = msg return b } // OwnerReference sets the PodVolumeBackup's OwnerReference. func (b *PodVolumeBackupBuilder) OwnerReference(ref metav1.OwnerReference) *PodVolumeBackupBuilder { b.object.OwnerReferences = append(b.object.OwnerReferences, ref) return b } // Labels sets the PodVolumeBackup's Labels. func (b *PodVolumeBackupBuilder) Labels(label map[string]string) *PodVolumeBackupBuilder { b.object.Labels = label return b } ================================================ FILE: pkg/builder/pod_volume_restore_builder.go ================================================ /* Copyright 2023 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package builder import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) // PodVolumeRestoreBuilder builds PodVolumeRestore objects. type PodVolumeRestoreBuilder struct { object *velerov1api.PodVolumeRestore } // ForPodVolumeRestore is the constructor for a PodVolumeRestoreBuilder. func ForPodVolumeRestore(ns, name string) *PodVolumeRestoreBuilder { return &PodVolumeRestoreBuilder{ object: &velerov1api.PodVolumeRestore{ TypeMeta: metav1.TypeMeta{ APIVersion: velerov1api.SchemeGroupVersion.String(), Kind: "PodVolumeRestore", }, ObjectMeta: metav1.ObjectMeta{ Namespace: ns, Name: name, }, }, } } // Result returns the built PodVolumeRestore. func (b *PodVolumeRestoreBuilder) Result() *velerov1api.PodVolumeRestore { return b.object } // ObjectMeta applies functional options to the PodVolumeRestore's ObjectMeta. func (b *PodVolumeRestoreBuilder) ObjectMeta(opts ...ObjectMetaOpt) *PodVolumeRestoreBuilder { for _, opt := range opts { opt(b.object) } return b } // Phase sets the PodVolumeRestore's phase. func (b *PodVolumeRestoreBuilder) Phase(phase velerov1api.PodVolumeRestorePhase) *PodVolumeRestoreBuilder { b.object.Status.Phase = phase return b } // BackupStorageLocation sets the PodVolumeRestore's backup storage location. func (b *PodVolumeRestoreBuilder) BackupStorageLocation(name string) *PodVolumeRestoreBuilder { b.object.Spec.BackupStorageLocation = name return b } // SnapshotID sets the PodVolumeRestore's snapshot ID. func (b *PodVolumeRestoreBuilder) SnapshotID(snapshotID string) *PodVolumeRestoreBuilder { b.object.Spec.SnapshotID = snapshotID return b } // PodName sets the name of the pod associated with this PodVolumeRestore. func (b *PodVolumeRestoreBuilder) PodName(name string) *PodVolumeRestoreBuilder { b.object.Spec.Pod.Name = name return b } // PodNamespace sets the name of the pod associated with this PodVolumeRestore. func (b *PodVolumeRestoreBuilder) PodNamespace(ns string) *PodVolumeRestoreBuilder { b.object.Spec.Pod.Namespace = ns return b } // Volume sets the name of the volume associated with this PodVolumeRestore. func (b *PodVolumeRestoreBuilder) Volume(volume string) *PodVolumeRestoreBuilder { b.object.Spec.Volume = volume return b } // UploaderType sets the type of uploader to use for this PodVolumeRestore. func (b *PodVolumeRestoreBuilder) UploaderType(uploaderType string) *PodVolumeRestoreBuilder { b.object.Spec.UploaderType = uploaderType return b } // OwnerReference sets the OwnerReference for this PodVolumeRestore. func (b *PodVolumeRestoreBuilder) OwnerReference(ownerRef []metav1.OwnerReference) *PodVolumeRestoreBuilder { b.object.OwnerReferences = ownerRef return b } // Cancel sets the DataDownload's Cancel. func (b *PodVolumeRestoreBuilder) Cancel(cancel bool) *PodVolumeRestoreBuilder { b.object.Spec.Cancel = cancel return b } // AcceptedTimestamp sets the PodVolumeRestore's AcceptedTimestamp. func (b *PodVolumeRestoreBuilder) AcceptedTimestamp(acceptedTimestamp *metav1.Time) *PodVolumeRestoreBuilder { b.object.Status.AcceptedTimestamp = acceptedTimestamp return b } // Finalizers sets the PodVolumeRestore's Finalizers. func (b *PodVolumeRestoreBuilder) Finalizers(finalizers []string) *PodVolumeRestoreBuilder { b.object.Finalizers = finalizers return b } // Message sets the PodVolumeRestore's Message. func (b *PodVolumeRestoreBuilder) Message(msg string) *PodVolumeRestoreBuilder { b.object.Status.Message = msg return b } // Message sets the PodVolumeRestore's Node. func (b *PodVolumeRestoreBuilder) Node(node string) *PodVolumeRestoreBuilder { b.object.Status.Node = node return b } // Labels sets the PodVolumeRestoreBuilder's Labels. func (b *PodVolumeRestoreBuilder) Labels(label map[string]string) *PodVolumeRestoreBuilder { b.object.Labels = label return b } ================================================ FILE: pkg/builder/priority_class_builder.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package builder import ( corev1api "k8s.io/api/core/v1" schedulingv1api "k8s.io/api/scheduling/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) type PriorityClassBuilder struct { object *schedulingv1api.PriorityClass } func ForPriorityClass(name string) *PriorityClassBuilder { return &PriorityClassBuilder{ object: &schedulingv1api.PriorityClass{ ObjectMeta: metav1.ObjectMeta{ Name: name, }, }, } } func (p *PriorityClassBuilder) Value(value int) *PriorityClassBuilder { p.object.Value = int32(value) return p } func (p *PriorityClassBuilder) PreemptionPolicy(policy string) *PriorityClassBuilder { preemptionPolicy := corev1api.PreemptionPolicy(policy) p.object.PreemptionPolicy = &preemptionPolicy return p } func (p *PriorityClassBuilder) Result() *schedulingv1api.PriorityClass { return p.object } ================================================ FILE: pkg/builder/restore_builder.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package builder import ( "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) // RestoreBuilder builds Restore objects. type RestoreBuilder struct { object *velerov1api.Restore } // ForRestore is the constructor for a RestoreBuilder. func ForRestore(ns, name string) *RestoreBuilder { return &RestoreBuilder{ object: &velerov1api.Restore{ TypeMeta: metav1.TypeMeta{ APIVersion: velerov1api.SchemeGroupVersion.String(), Kind: "Restore", }, ObjectMeta: metav1.ObjectMeta{ Namespace: ns, Name: name, }, }, } } // Result returns the built Restore. func (b *RestoreBuilder) Result() *velerov1api.Restore { return b.object } // ObjectMeta applies functional options to the Restore's ObjectMeta. func (b *RestoreBuilder) ObjectMeta(opts ...ObjectMetaOpt) *RestoreBuilder { for _, opt := range opts { opt(b.object) } return b } // Backup sets the Restore's backup name. func (b *RestoreBuilder) Backup(name string) *RestoreBuilder { b.object.Spec.BackupName = name return b } // Schedule sets the Restore's schedule name. func (b *RestoreBuilder) Schedule(name string) *RestoreBuilder { b.object.Spec.ScheduleName = name return b } // IncludedNamespaces appends to the Restore's included namespaces. func (b *RestoreBuilder) IncludedNamespaces(namespaces ...string) *RestoreBuilder { b.object.Spec.IncludedNamespaces = append(b.object.Spec.IncludedNamespaces, namespaces...) return b } // ExcludedNamespaces appends to the Restore's excluded namespaces. func (b *RestoreBuilder) ExcludedNamespaces(namespaces ...string) *RestoreBuilder { b.object.Spec.ExcludedNamespaces = append(b.object.Spec.ExcludedNamespaces, namespaces...) return b } // IncludedResources appends to the Restore's included resources. func (b *RestoreBuilder) IncludedResources(resources ...string) *RestoreBuilder { b.object.Spec.IncludedResources = append(b.object.Spec.IncludedResources, resources...) return b } // ExcludedResources appends to the Restore's excluded resources. func (b *RestoreBuilder) ExcludedResources(resources ...string) *RestoreBuilder { b.object.Spec.ExcludedResources = append(b.object.Spec.ExcludedResources, resources...) return b } // ExistingResourcePolicy sets the Restore's resource policy. func (b *RestoreBuilder) ExistingResourcePolicy(policy string) *RestoreBuilder { b.object.Spec.ExistingResourcePolicy = velerov1api.PolicyType(policy) return b } // IncludeClusterResources sets the Restore's "include cluster resources" flag. func (b *RestoreBuilder) IncludeClusterResources(val bool) *RestoreBuilder { b.object.Spec.IncludeClusterResources = &val return b } // LabelSelector sets the Restore's label selector. func (b *RestoreBuilder) LabelSelector(selector *metav1.LabelSelector) *RestoreBuilder { b.object.Spec.LabelSelector = selector return b } // OrLabelSelector sets the Restore's orLabelSelector set. func (b *RestoreBuilder) OrLabelSelector(orSelectors []*metav1.LabelSelector) *RestoreBuilder { b.object.Spec.OrLabelSelectors = orSelectors return b } // NamespaceMappings sets the Restore's namespace mappings. func (b *RestoreBuilder) NamespaceMappings(mapping ...string) *RestoreBuilder { if b.object.Spec.NamespaceMapping == nil { b.object.Spec.NamespaceMapping = make(map[string]string) } if len(mapping)%2 != 0 { panic("mapping must contain an even number of values") } for i := 0; i < len(mapping); i += 2 { b.object.Spec.NamespaceMapping[mapping[i]] = mapping[i+1] } return b } // Phase sets the Restore's phase. func (b *RestoreBuilder) Phase(phase velerov1api.RestorePhase) *RestoreBuilder { b.object.Status.Phase = phase return b } // RestorePVs sets the Restore's restore PVs. func (b *RestoreBuilder) RestorePVs(val bool) *RestoreBuilder { b.object.Spec.RestorePVs = &val return b } // PreserveNodePorts sets the Restore's preserved NodePorts. func (b *RestoreBuilder) PreserveNodePorts(val bool) *RestoreBuilder { b.object.Spec.PreserveNodePorts = &val return b } // StartTimestamp sets the Restore's start timestamp. func (b *RestoreBuilder) StartTimestamp(val time.Time) *RestoreBuilder { b.object.Status.StartTimestamp = &metav1.Time{Time: val} return b } // CompletionTimestamp sets the Restore's completion timestamp. func (b *RestoreBuilder) CompletionTimestamp(val time.Time) *RestoreBuilder { b.object.Status.CompletionTimestamp = &metav1.Time{Time: val} return b } // ItemOperationTimeout sets the Restore's ItemOperationTimeout func (b *RestoreBuilder) ItemOperationTimeout(timeout time.Duration) *RestoreBuilder { b.object.Spec.ItemOperationTimeout.Duration = timeout return b } ================================================ FILE: pkg/builder/role_builder.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package builder import ( rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // RoleBuilder builds Role objects. type RoleBuilder struct { object *rbacv1.Role } // ForRole is the constructor for a RoleBuilder. func ForRole(ns, name string) *RoleBuilder { return &RoleBuilder{ object: &rbacv1.Role{ TypeMeta: metav1.TypeMeta{ APIVersion: rbacv1.SchemeGroupVersion.String(), Kind: "Role", }, ObjectMeta: metav1.ObjectMeta{ Namespace: ns, Name: name, }, }, } } // Result returns the built Role. func (b *RoleBuilder) Result() *rbacv1.Role { return b.object } // ObjectMeta applies functional options to the Role's ObjectMeta. func (b *RoleBuilder) ObjectMeta(opts ...ObjectMetaOpt) *RoleBuilder { for _, opt := range opts { opt(b.object) } return b } ================================================ FILE: pkg/builder/schedule_builder.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package builder import ( "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) // ScheduleBuilder builds Schedule objects. type ScheduleBuilder struct { object *velerov1api.Schedule } // ForSchedule is the constructor for a ScheduleBuilder. func ForSchedule(ns, name string) *ScheduleBuilder { return &ScheduleBuilder{ object: &velerov1api.Schedule{ TypeMeta: metav1.TypeMeta{ APIVersion: velerov1api.SchemeGroupVersion.String(), Kind: "Schedule", }, ObjectMeta: metav1.ObjectMeta{ Namespace: ns, Name: name, }, }, } } // Result returns the built Schedule. func (b *ScheduleBuilder) Result() *velerov1api.Schedule { return b.object } // ObjectMeta applies functional options to the Schedule's ObjectMeta. func (b *ScheduleBuilder) ObjectMeta(opts ...ObjectMetaOpt) *ScheduleBuilder { for _, opt := range opts { opt(b.object) } return b } // Phase sets the Schedule's phase. func (b *ScheduleBuilder) Phase(phase velerov1api.SchedulePhase) *ScheduleBuilder { b.object.Status.Phase = phase return b } // ValidationError appends to the Schedule's validation errors. func (b *ScheduleBuilder) ValidationError(err string) *ScheduleBuilder { b.object.Status.ValidationErrors = append(b.object.Status.ValidationErrors, err) return b } // CronSchedule sets the Schedule's cron schedule. func (b *ScheduleBuilder) CronSchedule(expression string) *ScheduleBuilder { b.object.Spec.Schedule = expression return b } // LastBackupTime sets the Schedule's last backup time. func (b *ScheduleBuilder) LastBackupTime(val string) *ScheduleBuilder { t, _ := time.Parse("2006-01-02 15:04:05", val) b.object.Status.LastBackup = &metav1.Time{Time: t} return b } // Template sets the Schedule's template. func (b *ScheduleBuilder) Template(spec velerov1api.BackupSpec) *ScheduleBuilder { b.object.Spec.Template = spec return b } // SkipImmediately sets the Schedule's SkipImmediately. func (b *ScheduleBuilder) SkipImmediately(skip *bool) *ScheduleBuilder { b.object.Spec.SkipImmediately = skip return b } ================================================ FILE: pkg/builder/secret_builder.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package builder import ( corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // SecretBuilder builds Secret objects. type SecretBuilder struct { object *corev1api.Secret } // ForSecret is the constructor for a SecretBuilder. func ForSecret(ns, name string) *SecretBuilder { return &SecretBuilder{ object: &corev1api.Secret{ TypeMeta: metav1.TypeMeta{ APIVersion: corev1api.SchemeGroupVersion.String(), Kind: "Secret", }, ObjectMeta: metav1.ObjectMeta{ Namespace: ns, Name: name, }, }, } } // Result returns the built Secret. func (b *SecretBuilder) Result() *corev1api.Secret { return b.object } // ObjectMeta applies functional options to the Secret's ObjectMeta. func (b *SecretBuilder) ObjectMeta(opts ...ObjectMetaOpt) *SecretBuilder { for _, opt := range opts { opt(b.object) } return b } // Data sets the Secret data. func (b *SecretBuilder) Data(data map[string][]byte) *SecretBuilder { b.object.Data = data return b } ================================================ FILE: pkg/builder/secret_key_selector_builder.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package builder import ( corev1api "k8s.io/api/core/v1" ) // SecretKeySelectorBuilder builds SecretKeySelector objects. type SecretKeySelectorBuilder struct { object *corev1api.SecretKeySelector } // ForSecretKeySelector is the constructor for a SecretKeySelectorBuilder. func ForSecretKeySelector(name string, key string) *SecretKeySelectorBuilder { return &SecretKeySelectorBuilder{ object: &corev1api.SecretKeySelector{ LocalObjectReference: corev1api.LocalObjectReference{ Name: name, }, Key: key, }, } } // Result returns the built SecretKeySelector. func (b *SecretKeySelectorBuilder) Result() *corev1api.SecretKeySelector { return b.object } ================================================ FILE: pkg/builder/server_status_request_builder.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package builder import ( "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) // ServerStatusRequestBuilder builds ServerStatusRequest objects. type ServerStatusRequestBuilder struct { object *velerov1api.ServerStatusRequest } // ForServerStatusRequest is the constructor for a ServerStatusRequestBuilder. func ForServerStatusRequest(ns, name, resourceVersion string) *ServerStatusRequestBuilder { return &ServerStatusRequestBuilder{ object: &velerov1api.ServerStatusRequest{ TypeMeta: metav1.TypeMeta{ APIVersion: velerov1api.SchemeGroupVersion.String(), Kind: "ServerStatusRequest", }, ObjectMeta: metav1.ObjectMeta{ Namespace: ns, Name: name, ResourceVersion: resourceVersion, }, }, } } // Result returns the built ServerStatusRequest. func (b *ServerStatusRequestBuilder) Result() *velerov1api.ServerStatusRequest { return b.object } // ObjectMeta applies functional options to the ServerStatusRequest's ObjectMeta. func (b *ServerStatusRequestBuilder) ObjectMeta(opts ...ObjectMetaOpt) *ServerStatusRequestBuilder { for _, opt := range opts { opt(b.object) } return b } // Phase sets the ServerStatusRequest's phase. func (b *ServerStatusRequestBuilder) Phase(phase velerov1api.ServerStatusRequestPhase) *ServerStatusRequestBuilder { b.object.Status.Phase = phase return b } // ProcessedTimestamp sets the ServerStatusRequest's processed timestamp. func (b *ServerStatusRequestBuilder) ProcessedTimestamp(time time.Time) *ServerStatusRequestBuilder { b.object.Status.ProcessedTimestamp = &metav1.Time{Time: time} return b } // ServerVersion sets the ServerStatusRequest's server version. func (b *ServerStatusRequestBuilder) ServerVersion(version string) *ServerStatusRequestBuilder { b.object.Status.ServerVersion = version return b } // Plugins sets the ServerStatusRequest's plugins. func (b *ServerStatusRequestBuilder) Plugins(plugins []velerov1api.PluginInfo) *ServerStatusRequestBuilder { b.object.Status.Plugins = plugins return b } ================================================ FILE: pkg/builder/service_account_builder.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package builder import ( corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // ServiceAccountBuilder builds ServiceAccount objects. type ServiceAccountBuilder struct { object *corev1api.ServiceAccount } // ForServiceAccount is the constructor for a ServiceAccountBuilder. func ForServiceAccount(ns, name string) *ServiceAccountBuilder { return &ServiceAccountBuilder{ object: &corev1api.ServiceAccount{ TypeMeta: metav1.TypeMeta{ APIVersion: corev1api.SchemeGroupVersion.String(), Kind: "ServiceAccount", }, ObjectMeta: metav1.ObjectMeta{ Namespace: ns, Name: name, }, }, } } // Result returns the built ServiceAccount. func (b *ServiceAccountBuilder) Result() *corev1api.ServiceAccount { return b.object } // ObjectMeta applies functional options to the ServiceAccount's ObjectMeta. func (b *ServiceAccountBuilder) ObjectMeta(opts ...ObjectMetaOpt) *ServiceAccountBuilder { for _, opt := range opts { opt(b.object) } return b } ================================================ FILE: pkg/builder/service_builder.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package builder import ( corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // ServiceBuilder builds Service objects. type ServiceBuilder struct { object *corev1api.Service } // ForService is the constructor for a ServiceBuilder. func ForService(ns, name string) *ServiceBuilder { return &ServiceBuilder{ object: &corev1api.Service{ TypeMeta: metav1.TypeMeta{ APIVersion: corev1api.SchemeGroupVersion.String(), Kind: "Service", }, ObjectMeta: metav1.ObjectMeta{ Namespace: ns, Name: name, }, }, } } // Result returns the built Service. func (s *ServiceBuilder) Result() *corev1api.Service { return s.object } // ObjectMeta applies functional options to the Service's ObjectMeta. func (s *ServiceBuilder) ObjectMeta(opts ...ObjectMetaOpt) *ServiceBuilder { for _, opt := range opts { opt(s.object) } return s } ================================================ FILE: pkg/builder/statefulset_builder.go ================================================ /* Copyright 2021 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package builder import ( appsv1api "k8s.io/api/apps/v1" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // StatefulSetBuilder builds StatefulSet objects. type StatefulSetBuilder struct { object *appsv1api.StatefulSet } // ForStatefulSet is the constructor for a StatefulSetBuilder. func ForStatefulSet(ns, name string) *StatefulSetBuilder { return &StatefulSetBuilder{ object: &appsv1api.StatefulSet{ TypeMeta: metav1.TypeMeta{ APIVersion: appsv1api.SchemeGroupVersion.String(), Kind: "StatefulSet", }, ObjectMeta: metav1.ObjectMeta{ Namespace: ns, Name: name, }, Spec: appsv1api.StatefulSetSpec{ VolumeClaimTemplates: []corev1api.PersistentVolumeClaim{}, }, }, } } // Result returns the built StatefulSet. func (b *StatefulSetBuilder) Result() *appsv1api.StatefulSet { return b.object } // StorageClass sets the StatefulSet's VolumeClaimTemplates storage class name. func (b *StatefulSetBuilder) StorageClass(names ...string) *StatefulSetBuilder { for _, name := range names { nameTmp := name b.object.Spec.VolumeClaimTemplates = append(b.object.Spec.VolumeClaimTemplates, corev1api.PersistentVolumeClaim{Spec: corev1api.PersistentVolumeClaimSpec{StorageClassName: &nameTmp}}) } return b } ================================================ FILE: pkg/builder/storage_class_builder.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package builder import ( corev1api "k8s.io/api/core/v1" storagev1api "k8s.io/api/storage/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // StorageClassBuilder builds StorageClass objects. type StorageClassBuilder struct { object *storagev1api.StorageClass objectSlice []*storagev1api.StorageClass } // ForStorageClass is the constructor for a StorageClassBuilder. func ForStorageClass(name string) *StorageClassBuilder { return &StorageClassBuilder{ object: &storagev1api.StorageClass{ TypeMeta: metav1.TypeMeta{ APIVersion: storagev1api.SchemeGroupVersion.String(), Kind: "StorageClass", }, ObjectMeta: metav1.ObjectMeta{ Name: name, }, }, } } // Result returns the built StorageClass. func (b *StorageClassBuilder) Result() *storagev1api.StorageClass { return b.object } // ObjectMeta applies functional options to the StorageClass's ObjectMeta. func (b *StorageClassBuilder) ObjectMeta(opts ...ObjectMetaOpt) *StorageClassBuilder { for _, opt := range opts { opt(b.object) } return b } // ForStorageClassSlice is the constructor for a storageClassSlice in StorageClassBuilder. func ForStorageClassSlice(names ...string) *StorageClassBuilder { var storageClassSlice []*storagev1api.StorageClass for _, name := range names { storageClass := &storagev1api.StorageClass{ TypeMeta: metav1.TypeMeta{ APIVersion: storagev1api.SchemeGroupVersion.String(), Kind: "StorageClass", }, ObjectMeta: metav1.ObjectMeta{ Name: name, }, } storageClassSlice = append(storageClassSlice, storageClass) } return &StorageClassBuilder{ objectSlice: storageClassSlice, } } // SliceResult returns the built StorageClass slice. func (b *StorageClassBuilder) SliceResult() []*storagev1api.StorageClass { return b.objectSlice } // Provisioner sets StorageClass's provisioner. func (b *StorageClassBuilder) Provisioner(provisioner string) *StorageClassBuilder { b.object.Provisioner = provisioner return b } // ReclaimPolicy sets StorageClass's reclaimPolicy. func (b *StorageClassBuilder) ReclaimPolicy(policy corev1api.PersistentVolumeReclaimPolicy) *StorageClassBuilder { b.object.ReclaimPolicy = &policy return b } ================================================ FILE: pkg/builder/testcr_builder.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package builder import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) // CustomResourceBuilder builds objects based on velero APIVersion CRDs. type TestCRBuilder struct { object *TestCR } // ForTestCR is the constructor for a TestCRBuilder. func ForTestCR(crdKind, ns, name string) *TestCRBuilder { return &TestCRBuilder{ object: &TestCR{ TypeMeta: metav1.TypeMeta{ APIVersion: velerov1api.SchemeGroupVersion.String(), Kind: crdKind, }, ObjectMeta: metav1.ObjectMeta{ Namespace: ns, Name: name, }, }, } } // Result returns the built TestCR. func (b *TestCRBuilder) Result() *TestCR { return b.object } // ObjectMeta applies functional options to the TestCR's ObjectMeta. func (b *TestCRBuilder) ObjectMeta(opts ...ObjectMetaOpt) *TestCRBuilder { for _, opt := range opts { opt(b.object) } return b } type TestCR struct { metav1.TypeMeta `json:",inline"` // +optional metav1.ObjectMeta `json:"metadata,omitempty"` // +optional Spec TestCRSpec `json:"spec,omitempty"` // +optional Status TestCRStatus `json:"status,omitempty"` } type TestCRSpec struct { } type TestCRStatus struct { } ================================================ FILE: pkg/builder/v1_customresourcedefinition_builder.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package builder import ( apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // V1CustomResourceDefinitionBuilder builds CustomResourceDefinition objects. type V1CustomResourceDefinitionBuilder struct { object *apiextv1.CustomResourceDefinition } // ForV1CustomResourceDefinition is the constructor for a V1CustomResourceDefinitionBuilder. func ForV1CustomResourceDefinition(name string) *V1CustomResourceDefinitionBuilder { return &V1CustomResourceDefinitionBuilder{ object: &apiextv1.CustomResourceDefinition{ TypeMeta: metav1.TypeMeta{ APIVersion: apiextv1.SchemeGroupVersion.String(), Kind: "CustomResourceDefinition", }, ObjectMeta: metav1.ObjectMeta{ Name: name, }, }, } } // Condition adds a CustomResourceDefinitionCondition objects to a V1CustomResourceDefinitionBuilder. func (b *V1CustomResourceDefinitionBuilder) Condition(cond apiextv1.CustomResourceDefinitionCondition) *V1CustomResourceDefinitionBuilder { b.object.Status.Conditions = append(b.object.Status.Conditions, cond) return b } // Version adds a CustomResourceDefinitionVersion object to a V1CustomResourceDefinitionBuilder. func (b *V1CustomResourceDefinitionBuilder) Version(ver apiextv1.CustomResourceDefinitionVersion) *V1CustomResourceDefinitionBuilder { b.object.Spec.Versions = append(b.object.Spec.Versions, ver) return b } // PreserveUnknownFields sets PreserveUnknownFields on a CustomResourceDefinition. func (b *V1CustomResourceDefinitionBuilder) PreserveUnknownFields(preserve bool) *V1CustomResourceDefinitionBuilder { b.object.Spec.PreserveUnknownFields = preserve return b } // Result returns the built CustomResourceDefinition. func (b *V1CustomResourceDefinitionBuilder) Result() *apiextv1.CustomResourceDefinition { return b.object } // ObjectMeta applies functional options to the CustomResourceDefinition's ObjectMeta. func (b *V1CustomResourceDefinitionBuilder) ObjectMeta(opts ...ObjectMetaOpt) *V1CustomResourceDefinitionBuilder { for _, opt := range opts { opt(b.object) } return b } // V1CustomResourceDefinitionConditionBuilder builds CustomResourceDefinitionCondition objects. type V1CustomResourceDefinitionConditionBuilder struct { object apiextv1.CustomResourceDefinitionCondition } // ForV1CustomResourceDefinitionCondition is the constructor for a V1CustomResourceDefinitionConditionBuilder. func ForV1CustomResourceDefinitionCondition() *V1CustomResourceDefinitionConditionBuilder { return &V1CustomResourceDefinitionConditionBuilder{ object: apiextv1.CustomResourceDefinitionCondition{}, } } // Type sets the Condition's type. func (c *V1CustomResourceDefinitionConditionBuilder) Type(t apiextv1.CustomResourceDefinitionConditionType) *V1CustomResourceDefinitionConditionBuilder { c.object.Type = t return c } // Status sets the Condition's status. func (c *V1CustomResourceDefinitionConditionBuilder) Status(cs apiextv1.ConditionStatus) *V1CustomResourceDefinitionConditionBuilder { c.object.Status = cs return c } // Result returns the built CustomResourceDefinitionCondition. func (c *V1CustomResourceDefinitionConditionBuilder) Result() apiextv1.CustomResourceDefinitionCondition { return c.object } // V1CustomResourceDefinitionVersionBuilder builds CustomResourceDefinitionVersion objects. type V1CustomResourceDefinitionVersionBuilder struct { object apiextv1.CustomResourceDefinitionVersion } // ForV1CustomResourceDefinitionVersion is the constructor for a V1CustomResourceDefinitionVersionBuilder. func ForV1CustomResourceDefinitionVersion(name string) *V1CustomResourceDefinitionVersionBuilder { return &V1CustomResourceDefinitionVersionBuilder{ object: apiextv1.CustomResourceDefinitionVersion{Name: name}, } } // Served sets the Served field on a CustomResourceDefinitionVersion. func (c *V1CustomResourceDefinitionVersionBuilder) Served(s bool) *V1CustomResourceDefinitionVersionBuilder { c.object.Served = s return c } // Storage sets the Storage field on a CustomResourceDefinitionVersion. func (c *V1CustomResourceDefinitionVersionBuilder) Storage(s bool) *V1CustomResourceDefinitionVersionBuilder { c.object.Storage = s return c } func (c *V1CustomResourceDefinitionVersionBuilder) Schema(s *apiextv1.JSONSchemaProps) *V1CustomResourceDefinitionVersionBuilder { if c.object.Schema == nil { c.object.Schema = new(apiextv1.CustomResourceValidation) } c.object.Schema.OpenAPIV3Schema = s return c } // Result returns the built CustomResourceDefinitionVersion. func (c *V1CustomResourceDefinitionVersionBuilder) Result() apiextv1.CustomResourceDefinitionVersion { return c.object } ================================================ FILE: pkg/builder/volume_builder.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package builder import ( corev1api "k8s.io/api/core/v1" ) // VolumeBuilder builds Volume objects. type VolumeBuilder struct { object *corev1api.Volume } // ForVolume is the constructor for a VolumeBuilder. func ForVolume(name string) *VolumeBuilder { return &VolumeBuilder{ object: &corev1api.Volume{ Name: name, }, } } // Result returns the built Volume. func (b *VolumeBuilder) Result() *corev1api.Volume { return b.object } // PersistentVolumeClaimSource sets the Volume's persistent volume claim source. func (b *VolumeBuilder) PersistentVolumeClaimSource(claimName string) *VolumeBuilder { b.object.PersistentVolumeClaim = &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: claimName, } return b } // CSISource sets the Volume's CSI source. func (b *VolumeBuilder) CSISource(driver string) *VolumeBuilder { b.object.CSI = &corev1api.CSIVolumeSource{ Driver: driver, } return b } ================================================ FILE: pkg/builder/volume_mount_builder.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package builder import ( corev1api "k8s.io/api/core/v1" ) // VolumeMountBuilder builds VolumeMount objects. type VolumeMountBuilder struct { object *corev1api.VolumeMount } // ForVolumeMount is the constructor for a VolumeMountBuilder. func ForVolumeMount(name, mountPath string) *VolumeMountBuilder { return &VolumeMountBuilder{ object: &corev1api.VolumeMount{ Name: name, MountPath: mountPath, }, } } // Result returns the built VolumeMount. func (b *VolumeMountBuilder) Result() *corev1api.VolumeMount { return b.object } ================================================ FILE: pkg/builder/volume_snapshot_builder.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package builder import ( snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // VolumeSnapshotBuilder builds VolumeSnapshot objects. type VolumeSnapshotBuilder struct { object *snapshotv1api.VolumeSnapshot } // ForVolumeSnapshot is the constructor for VolumeSnapshotBuilder. func ForVolumeSnapshot(ns, name string) *VolumeSnapshotBuilder { return &VolumeSnapshotBuilder{ object: &snapshotv1api.VolumeSnapshot{ TypeMeta: metav1.TypeMeta{ APIVersion: snapshotv1api.SchemeGroupVersion.String(), Kind: "VolumeSnapshot", }, ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: ns, }, }, } } // ObjectMeta applies functional options to the VolumeSnapshot's ObjectMeta. func (v *VolumeSnapshotBuilder) ObjectMeta(opts ...ObjectMetaOpt) *VolumeSnapshotBuilder { for _, opt := range opts { opt(v.object) } return v } // Result return the built VolumeSnapshot. func (v *VolumeSnapshotBuilder) Result() *snapshotv1api.VolumeSnapshot { return v.object } // Status init the built VolumeSnapshot's status. func (v *VolumeSnapshotBuilder) Status() *VolumeSnapshotBuilder { v.object.Status = &snapshotv1api.VolumeSnapshotStatus{} return v } // BoundVolumeSnapshotContentName set built VolumeSnapshot's status BoundVolumeSnapshotContentName field. func (v *VolumeSnapshotBuilder) BoundVolumeSnapshotContentName(vscName string) *VolumeSnapshotBuilder { v.object.Status.BoundVolumeSnapshotContentName = &vscName return v } // SourcePVC set the built VolumeSnapshot's spec.Source.PersistentVolumeClaimName. func (v *VolumeSnapshotBuilder) SourcePVC(name string) *VolumeSnapshotBuilder { v.object.Spec.Source.PersistentVolumeClaimName = &name return v } // SourceVolumeSnapshotContentName set the built VolumeSnapshot's spec.Source.VolumeSnapshotContentName func (v *VolumeSnapshotBuilder) SourceVolumeSnapshotContentName(name string) *VolumeSnapshotBuilder { v.object.Spec.Source.VolumeSnapshotContentName = &name return v } // RestoreSize set the built VolumeSnapshot's status.RestoreSize. func (v *VolumeSnapshotBuilder) RestoreSize(size string) *VolumeSnapshotBuilder { resourceSize := resource.MustParse(size) v.object.Status.RestoreSize = &resourceSize return v } // VolumeSnapshotClass set the built VolumeSnapshot's spec.VolumeSnapshotClassName value. func (v *VolumeSnapshotBuilder) VolumeSnapshotClass(name string) *VolumeSnapshotBuilder { v.object.Spec.VolumeSnapshotClassName = &name return v } // StatusError set the built VolumeSnapshot's status.Error value. func (v *VolumeSnapshotBuilder) StatusError(snapshotError snapshotv1api.VolumeSnapshotError) *VolumeSnapshotBuilder { v.object.Status.Error = &snapshotError return v } // ReadyToUse set the built VolumeSnapshot's status.ReadyToUse value. func (v *VolumeSnapshotBuilder) ReadyToUse(readyToUse bool) *VolumeSnapshotBuilder { v.object.Status.ReadyToUse = &readyToUse return v } ================================================ FILE: pkg/builder/volume_snapshot_class_builder.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package builder import ( snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // VolumeSnapshotClassBuilder builds VolumeSnapshotClass objects. type VolumeSnapshotClassBuilder struct { object *snapshotv1api.VolumeSnapshotClass } // ForVolumeSnapshotClass is the constructor of VolumeSnapshotClassBuilder. func ForVolumeSnapshotClass(name string) *VolumeSnapshotClassBuilder { return &VolumeSnapshotClassBuilder{ object: &snapshotv1api.VolumeSnapshotClass{ TypeMeta: metav1.TypeMeta{ Kind: "VolumeSnapshotClass", APIVersion: snapshotv1api.SchemeGroupVersion.String(), }, ObjectMeta: metav1.ObjectMeta{ Name: name, }, }, } } // Result returns the built VolumeSnapshotClass. func (b *VolumeSnapshotClassBuilder) Result() *snapshotv1api.VolumeSnapshotClass { return b.object } // Driver sets the driver of built VolumeSnapshotClass. func (b *VolumeSnapshotClassBuilder) Driver(driver string) *VolumeSnapshotClassBuilder { b.object.Driver = driver return b } // ObjectMeta applies functional options to the VolumeSnapshotClass's ObjectMeta. func (b *VolumeSnapshotClassBuilder) ObjectMeta(opts ...ObjectMetaOpt) *VolumeSnapshotClassBuilder { for _, opt := range opts { opt(b.object) } return b } ================================================ FILE: pkg/builder/volume_snapshot_content_builder.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package builder import ( snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" ) // VolumeSnapshotContentBuilder builds VolumeSnapshotContent object. type VolumeSnapshotContentBuilder struct { object *snapshotv1api.VolumeSnapshotContent } // ForVolumeSnapshotContent is the constructor of VolumeSnapshotContentBuilder. func ForVolumeSnapshotContent(name string) *VolumeSnapshotContentBuilder { return &VolumeSnapshotContentBuilder{ object: &snapshotv1api.VolumeSnapshotContent{ TypeMeta: metav1.TypeMeta{ APIVersion: snapshotv1api.SchemeGroupVersion.String(), Kind: "VolumeSnapshotContent", }, ObjectMeta: metav1.ObjectMeta{ Name: name, }, }, } } // Result returns the built VolumeSnapshotContent. func (v *VolumeSnapshotContentBuilder) Result() *snapshotv1api.VolumeSnapshotContent { return v.object } // Status initiates VolumeSnapshotContent's status. func (v *VolumeSnapshotContentBuilder) Status(status *snapshotv1api.VolumeSnapshotContentStatus) *VolumeSnapshotContentBuilder { v.object.Status = status return v } // DeletionPolicy sets built VolumeSnapshotContent's spec.DeletionPolicy value. func (v *VolumeSnapshotContentBuilder) DeletionPolicy(policy snapshotv1api.DeletionPolicy) *VolumeSnapshotContentBuilder { v.object.Spec.DeletionPolicy = policy return v } // VolumeSnapshotRef sets the built VolumeSnapshotContent's spec.VolumeSnapshotRef value. func (v *VolumeSnapshotContentBuilder) VolumeSnapshotRef(namespace, name, uid string) *VolumeSnapshotContentBuilder { v.object.Spec.VolumeSnapshotRef = corev1api.ObjectReference{ APIVersion: "snapshot.storage.k8s.io/v1", Kind: "VolumeSnapshot", Namespace: namespace, Name: name, UID: types.UID(uid), } return v } // VolumeSnapshotClassName sets the built VolumeSnapshotContent's spec.VolumeSnapshotClassName value. func (v *VolumeSnapshotContentBuilder) VolumeSnapshotClassName(name string) *VolumeSnapshotContentBuilder { v.object.Spec.VolumeSnapshotClassName = &name return v } // ObjectMeta applies functional options to the VolumeSnapshotContent's ObjectMeta. func (v *VolumeSnapshotContentBuilder) ObjectMeta(opts ...ObjectMetaOpt) *VolumeSnapshotContentBuilder { for _, opt := range opts { opt(v.object) } return v } func (v *VolumeSnapshotContentBuilder) Driver(driver string) *VolumeSnapshotContentBuilder { v.object.Spec.Driver = driver return v } func (v *VolumeSnapshotContentBuilder) Source(source snapshotv1api.VolumeSnapshotContentSource) *VolumeSnapshotContentBuilder { v.object.Spec.Source = source return v } ================================================ FILE: pkg/builder/volume_snapshot_location_builder.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package builder import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" corev1api "k8s.io/api/core/v1" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) // VolumeSnapshotLocationBuilder builds VolumeSnapshotLocation objects. type VolumeSnapshotLocationBuilder struct { object *velerov1api.VolumeSnapshotLocation } // ForVolumeSnapshotLocation is the constructor for a VolumeSnapshotLocationBuilder. func ForVolumeSnapshotLocation(ns, name string) *VolumeSnapshotLocationBuilder { return &VolumeSnapshotLocationBuilder{ object: &velerov1api.VolumeSnapshotLocation{ TypeMeta: metav1.TypeMeta{ APIVersion: velerov1api.SchemeGroupVersion.String(), Kind: "VolumeSnapshotLocation", }, ObjectMeta: metav1.ObjectMeta{ Namespace: ns, Name: name, }, }, } } // Result returns the built VolumeSnapshotLocation. func (b *VolumeSnapshotLocationBuilder) Result() *velerov1api.VolumeSnapshotLocation { return b.object } // ObjectMeta applies functional options to the VolumeSnapshotLocation's ObjectMeta. func (b *VolumeSnapshotLocationBuilder) ObjectMeta(opts ...ObjectMetaOpt) *VolumeSnapshotLocationBuilder { for _, opt := range opts { opt(b.object) } return b } // Provider sets the VolumeSnapshotLocation's provider. func (b *VolumeSnapshotLocationBuilder) Provider(name string) *VolumeSnapshotLocationBuilder { b.object.Spec.Provider = name return b } // Credential sets the VolumeSnapshotLocation's credential selector. func (b *VolumeSnapshotLocationBuilder) Credential(selector *corev1api.SecretKeySelector) *VolumeSnapshotLocationBuilder { b.object.Spec.Credential = selector return b } ================================================ FILE: pkg/buildinfo/buildinfo.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ // Package buildinfo holds build-time information like the velero version. // This is a separate package so that other packages can import it without // worrying about introducing circular dependencies. package buildinfo import "fmt" var ( // Version is the current version of Velero, set by the go linker's -X flag at build time. Version string // GitSHA is the actual commit that is being built, set by the go linker's -X flag at build time. GitSHA string // GitTreeState indicates if the git tree is clean or dirty, set by the go linker's -X flag at build // time. GitTreeState string // ImageRegistry is the image registry that this build of Velero should use by default to pull the // Velero and Restore Helper images from. ImageRegistry string ) // FormattedGitSHA renders the Git SHA with an indicator of the tree state. func FormattedGitSHA() string { if GitTreeState != "clean" { return fmt.Sprintf("%s-%s", GitSHA, GitTreeState) } return GitSHA } ================================================ FILE: pkg/buildinfo/buildinfo_test.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package buildinfo import ( "testing" "github.com/stretchr/testify/assert" ) func TestFormattedGitSHA(t *testing.T) { tests := []struct { name string sha string state string expected string }{ { "Clean git state has no suffix", "abc123", "clean", "abc123", }, { "Dirty git status includes suffix", "abc123", "dirty", "abc123-dirty", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { GitSHA = test.sha GitTreeState = test.state assert.Equal(t, test.expected, FormattedGitSHA()) }) } } ================================================ FILE: pkg/client/auth_providers.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package client import ( // Make sure we import the client-go auth provider plugins. _ "k8s.io/client-go/plugin/pkg/client/auth/azure" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" _ "k8s.io/client-go/plugin/pkg/client/auth/oidc" ) ================================================ FILE: pkg/client/client.go ================================================ /* Copyright 2017, 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package client import ( "fmt" "runtime" "github.com/pkg/errors" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" "github.com/vmware-tanzu/velero/pkg/buildinfo" ) // Config returns a *rest.Config, using either the kubeconfig (if specified) or an in-cluster // configuration. func Config(kubeconfig, kubecontext, baseName string, qps float32, burst int) (*rest.Config, error) { loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() loadingRules.ExplicitPath = kubeconfig configOverrides := &clientcmd.ConfigOverrides{CurrentContext: kubecontext} kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides) clientConfig, err := kubeConfig.ClientConfig() if err != nil { return nil, errors.Wrap(err, "error finding Kubernetes API server config in --kubeconfig, $KUBECONFIG, or in-cluster configuration") } if qps > 0.0 { clientConfig.QPS = qps } if burst > 0 { clientConfig.Burst = burst } clientConfig.UserAgent = buildUserAgent( baseName, buildinfo.Version, buildinfo.FormattedGitSHA(), runtime.GOOS, runtime.GOARCH, ) return clientConfig, nil } // buildUserAgent builds a User-Agent string from given args. func buildUserAgent(command, version, formattedSha, os, arch string) string { return fmt.Sprintf( "%s/%s (%s/%s) %s", command, version, os, arch, formattedSha) } ================================================ FILE: pkg/client/client_test.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package client import ( "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestBuildUserAgent(t *testing.T) { tests := []struct { name string command string os string arch string gitSha string version string expected string }{ { name: "Test general interpolation in correct order", command: "velero", os: "darwin", arch: "amd64", gitSha: "abc123", version: "v0.1.1", expected: "velero/v0.1.1 (darwin/amd64) abc123", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { resp := buildUserAgent(test.command, test.version, test.gitSha, test.os, test.arch) assert.Equal(t, test.expected, resp) }) } } func TestConfig(t *testing.T) { tests := []struct { name string kubeconfig string kubecontext string QPS float32 burst int expectedHost string }{ { name: "Test using the right cluster as context indexed", kubeconfig: "kubeconfig", kubecontext: "federal-context", QPS: 1.0, burst: 1, expectedHost: "https://horse.org:4443", }, { name: "Test using the right cluster as context indexed", kubeconfig: "kubeconfig", kubecontext: "queen-anne-context", QPS: 200.0, burst: 20, expectedHost: "https://pig.org:443", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { client, err := Config(test.kubeconfig, test.kubecontext, "velero", test.QPS, test.burst) require.NoError(t, err) assert.Equal(t, test.expectedHost, client.Host) assert.Equal(t, test.QPS, client.QPS) assert.Equal(t, test.burst, client.Burst) }) } } ================================================ FILE: pkg/client/config.go ================================================ /* Copyright 2021 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package client import ( "encoding/json" "os" "path/filepath" "strconv" "strings" "github.com/pkg/errors" ) const ( ConfigKeyNamespace = "namespace" ConfigKeyFeatures = "features" ConfigKeyCACert = "cacert" ConfigKeyColorized = "colorized" ) // VeleroConfig is a map of strings to any for deserializing Velero client config options. // The alias is a way to attach type-asserting convenience methods. type VeleroConfig map[string]any // LoadConfig loads the Velero client configuration file and returns it as a VeleroConfig. If the // file does not exist, an empty map is returned. func LoadConfig() (VeleroConfig, error) { fileName := configFileName() _, err := os.Stat(fileName) if os.IsNotExist(err) { // If the file isn't there, just return an empty map return VeleroConfig{}, nil } if err != nil { // For any other Stat() error, return it return nil, errors.WithStack(err) } configFile, err := os.Open(fileName) if err != nil { return nil, errors.WithStack(err) } defer configFile.Close() var config VeleroConfig if err := json.NewDecoder(configFile).Decode(&config); err != nil { return nil, errors.WithStack(err) } return config, nil } // SaveConfig saves the passed in config map to the Velero client configuration file. func SaveConfig(config VeleroConfig) error { fileName := configFileName() // Try to make the directory in case it doesn't exist dir := filepath.Dir(fileName) if err := os.MkdirAll(dir, 0700); err != nil { return errors.WithStack(err) } configFile, err := os.OpenFile(fileName, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600) if err != nil { return errors.WithStack(err) } defer configFile.Close() return json.NewEncoder(configFile).Encode(&config) } func (c VeleroConfig) Namespace() string { val, ok := c[ConfigKeyNamespace] if !ok { return "" } ns, ok := val.(string) if !ok { return "" } return ns } func (c VeleroConfig) Features() []string { val, ok := c[ConfigKeyFeatures] if !ok { return []string{} } features, ok := val.(string) if !ok { return []string{} } return strings.Split(features, ",") } func (c VeleroConfig) Colorized() bool { val, ok := c[ConfigKeyColorized] if !ok { return true } valString, ok := val.(string) if !ok { return true } colorized, err := strconv.ParseBool(valString) if err != nil { return true } return colorized } func (c VeleroConfig) CACertFile() string { val, ok := c[ConfigKeyCACert] if !ok { return "" } caCertFile, ok := val.(string) if !ok { return "" } return caCertFile } func configFileName() string { return filepath.Join(os.Getenv("HOME"), ".config", "velero", "config.json") } ================================================ FILE: pkg/client/config_test.go ================================================ /* Copyright 2021 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package client import ( "os" "reflect" "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestVeleroConfig(t *testing.T) { c := VeleroConfig{ "namespace": "foo", "features": "feature1,feature2", } assert.Equal(t, "foo", c.Namespace()) assert.Equal(t, []string{"feature1", "feature2"}, c.Features()) assert.True(t, c.Colorized()) } func removeConfigfileName() error { // Remove config file if it exist configFile := configFileName() e := os.Remove(configFile) if e != nil { if !os.IsNotExist(e) { return e } } return nil } func TestConfigOperations(t *testing.T) { preHomeEnv := "" prevEnv := os.Environ() for _, entry := range prevEnv { parts := strings.SplitN(entry, "=", 2) if len(parts) == 2 && parts[0] == "HOME" { preHomeEnv = parts[1] break } } os.Unsetenv("HOME") os.Setenv("HOME", ".") // Remove config file if it exists err := removeConfigfileName() require.NoError(t, err) // Test LoadConfig: expect an empty velero config expectedConfig := VeleroConfig{} config, err := LoadConfig() require.NoError(t, err) assert.True(t, reflect.DeepEqual(expectedConfig, config)) // Test savedConfig expectedFeature := "EnableCSI" expectedColorized := true expectedNamespace := "ns-velero" expectedCACert := "ca-cert" config[ConfigKeyFeatures] = expectedFeature config[ConfigKeyColorized] = expectedColorized config[ConfigKeyNamespace] = expectedNamespace config[ConfigKeyCACert] = expectedCACert err = SaveConfig(config) require.NoError(t, err) savedConfig, err := LoadConfig() require.NoError(t, err) // Test Features feature := savedConfig.Features() assert.Len(t, feature, 1) assert.Equal(t, expectedFeature, feature[0]) // Test Colorized colorized := savedConfig.Colorized() assert.Equal(t, expectedColorized, colorized) // Test Namespace namespace := savedConfig.Namespace() assert.Equal(t, expectedNamespace, namespace) // Test Features caCertFile := savedConfig.CACertFile() assert.Equal(t, expectedCACert, caCertFile) t.Cleanup(func() { err = removeConfigfileName() require.NoError(t, err) os.Unsetenv("HOME") os.Setenv("HOME", preHomeEnv) }) } ================================================ FILE: pkg/client/dynamic.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package client import ( "context" "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/dynamic" "k8s.io/client-go/dynamic/dynamicinformer" ) // DynamicFactory contains methods for retrieving dynamic clients for GroupVersionResources and // GroupVersionKinds. type DynamicFactory interface { // ClientForGroupVersionResource returns a Dynamic client for the given group/version // and resource for the given namespace. ClientForGroupVersionResource(gv schema.GroupVersion, resource metav1.APIResource, namespace string) (Dynamic, error) // DynamicSharedInformerFactory returns a DynamicSharedInformerFactory. DynamicSharedInformerFactory() dynamicinformer.DynamicSharedInformerFactory } // dynamicFactory implements DynamicFactory. type dynamicFactory struct { dynamicClient dynamic.Interface } // NewDynamicFactory returns a new ClientPool-based dynamic factory. func NewDynamicFactory(dynamicClient dynamic.Interface) DynamicFactory { return &dynamicFactory{dynamicClient: dynamicClient} } func (f *dynamicFactory) ClientForGroupVersionResource(gv schema.GroupVersion, resource metav1.APIResource, namespace string) (Dynamic, error) { return &dynamicResourceClient{ resourceClient: f.dynamicClient.Resource(gv.WithResource(resource.Name)).Namespace(namespace), }, nil } func (f *dynamicFactory) DynamicSharedInformerFactory() dynamicinformer.DynamicSharedInformerFactory { return dynamicinformer.NewDynamicSharedInformerFactory(f.dynamicClient, time.Minute) } // Creator creates an object. type Creator interface { // Create creates an object. Create(obj *unstructured.Unstructured) (*unstructured.Unstructured, error) } // Lister lists objects. type Lister interface { // List lists all the objects of a given resource. List(metav1.ListOptions) (*unstructured.UnstructuredList, error) } // Watcher watches objects. type Watcher interface { // Watch watches for changes to objects of a given resource. Watch(metav1.ListOptions) (watch.Interface, error) } // Getter gets an object. type Getter interface { // Get fetches an object by name. Get(name string, opts metav1.GetOptions) (*unstructured.Unstructured, error) } // Patcher patches an object. type Patcher interface { //Patch patches the named object using the provided patch bytes, which are expected to be in JSON merge patch format. The patched object is returned. Patch(name string, data []byte) (*unstructured.Unstructured, error) } // Deletor deletes an object. type Deletor interface { //Patch patches the named object using the provided patch bytes, which are expected to be in JSON merge patch format. The patched object is returned. Delete(name string, opts metav1.DeleteOptions) error } // StatusUpdater updates status field of a object type StatusUpdater interface { UpdateStatus(obj *unstructured.Unstructured, opts metav1.UpdateOptions) (*unstructured.Unstructured, error) } // Applier applies changes to an object using server-side apply type Applier interface { Apply(name string, obj *unstructured.Unstructured, opts metav1.ApplyOptions) (*unstructured.Unstructured, error) } // Dynamic contains client methods that Velero needs for backing up and restoring resources. type Dynamic interface { Creator Lister Watcher Getter Patcher Deletor StatusUpdater Applier } // dynamicResourceClient implements Dynamic. type dynamicResourceClient struct { resourceClient dynamic.ResourceInterface } var _ Dynamic = &dynamicResourceClient{} func (d *dynamicResourceClient) Create(obj *unstructured.Unstructured) (*unstructured.Unstructured, error) { return d.resourceClient.Create(context.TODO(), obj, metav1.CreateOptions{}) } func (d *dynamicResourceClient) List(options metav1.ListOptions) (*unstructured.UnstructuredList, error) { return d.resourceClient.List(context.TODO(), options) } func (d *dynamicResourceClient) Watch(options metav1.ListOptions) (watch.Interface, error) { return d.resourceClient.Watch(context.TODO(), options) } func (d *dynamicResourceClient) Get(name string, opts metav1.GetOptions) (*unstructured.Unstructured, error) { return d.resourceClient.Get(context.TODO(), name, opts) } func (d *dynamicResourceClient) Apply(name string, obj *unstructured.Unstructured, opts metav1.ApplyOptions) (*unstructured.Unstructured, error) { return d.resourceClient.Apply(context.TODO(), name, obj, opts) } func (d *dynamicResourceClient) Patch(name string, data []byte) (*unstructured.Unstructured, error) { return d.resourceClient.Patch(context.TODO(), name, types.MergePatchType, data, metav1.PatchOptions{}) } func (d *dynamicResourceClient) Delete(name string, opts metav1.DeleteOptions) error { return d.resourceClient.Delete(context.TODO(), name, opts) } func (d *dynamicResourceClient) UpdateStatus(obj *unstructured.Unstructured, opts metav1.UpdateOptions) (*unstructured.Unstructured, error) { return d.resourceClient.UpdateStatus(context.TODO(), obj, opts) } ================================================ FILE: pkg/client/factory.go ================================================ /* Copyright 2017, 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package client import ( "os" volumegroupsnapshotv1beta1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1beta1" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" "k8s.io/client-go/discovery" k8scheme "k8s.io/client-go/kubernetes/scheme" kbclient "sigs.k8s.io/controller-runtime/pkg/client" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" "github.com/pkg/errors" "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" ) //go:generate mockery --name Factory // Factory knows how to create a VeleroClient and Kubernetes client. type Factory interface { // BindFlags binds common flags (--kubeconfig, --namespace) to the passed-in FlagSet. BindFlags(flags *pflag.FlagSet) // KubeClient returns a Kubernetes client. It uses the following priority to specify the cluster // configuration: --kubeconfig flag, KUBECONFIG environment variable, in-cluster configuration. KubeClient() (kubernetes.Interface, error) // DynamicClient returns a Kubernetes dynamic client. It uses the following priority to specify the cluster // configuration: --kubeconfig flag, KUBECONFIG environment variable, in-cluster configuration. DynamicClient() (dynamic.Interface, error) // KubebuilderClient returns a client for the controller runtime framework. It adds Kubernetes and Velero // types to its scheme. It uses the following priority to specify the cluster // configuration: --kubeconfig flag, KUBECONFIG environment variable, in-cluster configuration. KubebuilderClient() (kbclient.Client, error) // KubebuilderWatchClient returns a client with watcher for the controller runtime framework. // It adds Kubernetes and Velero types to its scheme. It uses the following priority to specify the cluster // configuration: --kubeconfig flag, KUBECONFIG environment variable, in-cluster configuration. KubebuilderWatchClient() (kbclient.WithWatch, error) // DiscoveryClient returns a Kubernetes discovery client. It uses the following priority to specify the cluster // configuration: --kubeconfig flag, KUBECONFIG environment variable, in-cluster configuration. DiscoveryClient() (discovery.AggregatedDiscoveryInterface, error) // SetBasename changes the basename for an already-constructed client. // This is useful for generating clients that require a different user-agent string below the root `velero` // command, such as the server subcommand. SetBasename(string) // SetClientQPS sets the Queries Per Second for a client. SetClientQPS(float32) // SetClientBurst sets the Burst for a client. SetClientBurst(int) // ClientConfig returns a rest.Config struct used for client-go clients. ClientConfig() (*rest.Config, error) // Namespace returns the namespace which the Factory will create clients for. Namespace() string } type factory struct { flags *pflag.FlagSet kubeconfig string kubecontext string baseName string namespace string clientQPS float32 clientBurst int } // NewFactory returns a Factory. func NewFactory(baseName string, config VeleroConfig) Factory { f := &factory{ flags: pflag.NewFlagSet("", pflag.ContinueOnError), baseName: baseName, } f.namespace = os.Getenv("VELERO_NAMESPACE") if config.Namespace() != "" { f.namespace = config.Namespace() } // We didn't get the namespace via env var or config file, so use the default. // Command line flags will override when BindFlags is called. if f.namespace == "" { f.namespace = velerov1api.DefaultNamespace } f.flags.StringVar(&f.kubeconfig, "kubeconfig", "", "Path to the kubeconfig file to use to talk to the Kubernetes apiserver. If unset, try the environment variable KUBECONFIG, as well as in-cluster configuration") f.flags.StringVarP(&f.namespace, "namespace", "n", f.namespace, "The namespace in which Velero should operate") f.flags.StringVar(&f.kubecontext, "kubecontext", "", "The context to use to talk to the Kubernetes apiserver. If unset defaults to whatever your current-context is (kubectl config current-context)") return f } func (f *factory) BindFlags(flags *pflag.FlagSet) { flags.AddFlagSet(f.flags) } func (f *factory) ClientConfig() (*rest.Config, error) { return Config(f.kubeconfig, f.kubecontext, f.baseName, f.clientQPS, f.clientBurst) } func (f *factory) KubeClient() (kubernetes.Interface, error) { clientConfig, err := f.ClientConfig() if err != nil { return nil, err } kubeClient, err := kubernetes.NewForConfig(clientConfig) if err != nil { return nil, errors.WithStack(err) } return kubeClient, nil } func (f *factory) DynamicClient() (dynamic.Interface, error) { clientConfig, err := f.ClientConfig() if err != nil { return nil, err } dynamicClient, err := dynamic.NewForConfig(clientConfig) if err != nil { return nil, errors.WithStack(err) } return dynamicClient, nil } func (f *factory) KubebuilderClient() (kbclient.Client, error) { clientConfig, err := f.ClientConfig() if err != nil { return nil, err } scheme := runtime.NewScheme() if err := velerov1api.AddToScheme(scheme); err != nil { return nil, err } if err := velerov2alpha1api.AddToScheme(scheme); err != nil { return nil, err } if err := k8scheme.AddToScheme(scheme); err != nil { return nil, err } if err := apiextv1beta1.AddToScheme(scheme); err != nil { return nil, err } if err := apiextv1.AddToScheme(scheme); err != nil { return nil, err } if err := snapshotv1api.AddToScheme(scheme); err != nil { return nil, err } if err := volumegroupsnapshotv1beta1.AddToScheme(scheme); err != nil { return nil, err } kubebuilderClient, err := kbclient.New(clientConfig, kbclient.Options{ Scheme: scheme, }) if err != nil { return nil, err } return kubebuilderClient, nil } func (f *factory) KubebuilderWatchClient() (kbclient.WithWatch, error) { clientConfig, err := f.ClientConfig() if err != nil { return nil, err } scheme := runtime.NewScheme() if err := velerov1api.AddToScheme(scheme); err != nil { return nil, err } if err := velerov2alpha1api.AddToScheme(scheme); err != nil { return nil, err } if err := k8scheme.AddToScheme(scheme); err != nil { return nil, err } if err := apiextv1beta1.AddToScheme(scheme); err != nil { return nil, err } if err := apiextv1.AddToScheme(scheme); err != nil { return nil, err } if err := snapshotv1api.AddToScheme(scheme); err != nil { return nil, err } if err := volumegroupsnapshotv1beta1.AddToScheme(scheme); err != nil { return nil, err } kubebuilderWatchClient, err := kbclient.NewWithWatch(clientConfig, kbclient.Options{ Scheme: scheme, }) if err != nil { return nil, err } return kubebuilderWatchClient, nil } func (f *factory) DiscoveryClient() (discovery.AggregatedDiscoveryInterface, error) { clientConfig, err := f.ClientConfig() if err != nil { return nil, err } return discovery.NewDiscoveryClientForConfig(clientConfig) } func (f *factory) SetBasename(name string) { f.baseName = name } func (f *factory) SetClientQPS(qps float32) { f.clientQPS = qps } func (f *factory) SetClientBurst(burst int) { f.clientBurst = burst } func (f *factory) Namespace() string { return f.namespace } ================================================ FILE: pkg/client/factory_test.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package client import ( "fmt" "os" "strings" "testing" flag "github.com/spf13/pflag" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" ) // TestFactory tests the client.Factory interface. func TestFactory(t *testing.T) { // Velero client configuration is currently omitted due to requiring a // test filesystem in pkg/test. This causes an import cycle as pkg/test // uses pkg/client's interfaces to implement fakes // Env variable should set the namespace if no config or argument are used os.Setenv("VELERO_NAMESPACE", "env-velero") f := NewFactory("velero", make(map[string]any)) assert.Equal(t, "env-velero", f.Namespace()) os.Unsetenv("VELERO_NAMESPACE") // Argument should change the namespace f = NewFactory("velero", make(map[string]any)) s := "flag-velero" flags := new(flag.FlagSet) f.BindFlags(flags) flags.Parse([]string{"--namespace", s}) assert.Equal(t, s, f.Namespace()) // An argument overrides the env variable if both are set. os.Setenv("VELERO_NAMESPACE", "env-velero") f = NewFactory("velero", make(map[string]any)) flags = new(flag.FlagSet) f.BindFlags(flags) flags.Parse([]string{"--namespace", s}) assert.Equal(t, s, f.Namespace()) os.Unsetenv("VELERO_NAMESPACE") tests := []struct { name string kubeconfig string kubecontext string QPS float32 burst int baseName string expectedHost string }{ { name: "Test flag setting in factory ClientConfig (test data #1)", kubeconfig: "kubeconfig", kubecontext: "federal-context", QPS: 1.0, burst: 1, baseName: "bn-velero-1", expectedHost: "https://horse.org:4443", }, { name: "Test flag setting in factory ClientConfig (test data #2)", kubeconfig: "kubeconfig", kubecontext: "queen-anne-context", QPS: 200.0, burst: 20, baseName: "bn-velero-2", expectedHost: "https://pig.org:443", }, } baseName := "velero-bn" config, err := LoadConfig() require.NoError(t, err) for _, test := range tests { t.Run(test.name, func(t *testing.T) { f = NewFactory(baseName, config) f.SetClientBurst(test.burst) f.SetClientQPS(test.QPS) f.SetBasename(test.baseName) flags = new(flag.FlagSet) f.BindFlags(flags) flags.Parse([]string{"--kubeconfig", test.kubeconfig, "--kubecontext", test.kubecontext}) clientConfig, _ := f.ClientConfig() assert.Equal(t, test.expectedHost, clientConfig.Host) assert.Equal(t, test.QPS, clientConfig.QPS) assert.Equal(t, test.burst, clientConfig.Burst) strings.Contains(clientConfig.UserAgent, test.baseName) kubeClient, _ := f.KubeClient() group := kubeClient.NodeV1().RESTClient().APIVersion().Group assert.NotNil(t, kubeClient) assert.Equal(t, "node.k8s.io", group) namespace := "ns1" dynamicClient, _ := f.DynamicClient() resource := &schema.GroupVersionResource{ Group: "group_test", Version: "verion_test", } list, e := dynamicClient.Resource(*resource).Namespace(namespace).List( t.Context(), metav1.ListOptions{ LabelSelector: "none", }, ) require.ErrorContains(t, e, fmt.Sprintf("Get \"%s/apis/%s/%s/namespaces/%s", test.expectedHost, resource.Group, resource.Version, namespace)) assert.Nil(t, list) assert.NotNil(t, dynamicClient) kubebuilderClient, e := f.KubebuilderClient() require.NoError(t, e) assert.NotNil(t, kubebuilderClient) kbClientWithWatch, e := f.KubebuilderWatchClient() require.NoError(t, e) assert.NotNil(t, kbClientWithWatch) }) } } ================================================ FILE: pkg/client/kubeconfig ================================================ current-context: federal-context apiVersion: v1 clusters: - cluster: api-version: v1 server: http://cow.org:8080 name: cow-cluster - cluster: server: https://horse.org:4443 name: horse-cluster - cluster: insecure-skip-tls-verify: true server: https://pig.org:443 name: pig-cluster contexts: - context: cluster: horse-cluster namespace: chisel-ns user: green-user name: federal-context - context: cluster: pig-cluster namespace: saw-ns user: black-user name: queen-anne-context kind: Config preferences: colors: true users: - name: blue-user user: token: blue-token - name: green-user user: ================================================ FILE: pkg/client/mocks/Factory.go ================================================ // Code generated by mockery v2.28.1. DO NOT EDIT. package mocks import ( discovery "k8s.io/client-go/discovery" dynamic "k8s.io/client-go/dynamic" kubernetes "k8s.io/client-go/kubernetes" mock "github.com/stretchr/testify/mock" pflag "github.com/spf13/pflag" pkgclient "sigs.k8s.io/controller-runtime/pkg/client" rest "k8s.io/client-go/rest" ) // Factory is an autogenerated mock type for the Factory type type Factory struct { mock.Mock } // BindFlags provides a mock function with given fields: flags func (_m *Factory) BindFlags(flags *pflag.FlagSet) { _m.Called(flags) } // ClientConfig provides a mock function with given fields: func (_m *Factory) ClientConfig() (*rest.Config, error) { ret := _m.Called() var r0 *rest.Config var r1 error if rf, ok := ret.Get(0).(func() (*rest.Config, error)); ok { return rf() } if rf, ok := ret.Get(0).(func() *rest.Config); ok { r0 = rf() } else { if ret.Get(0) != nil { r0 = ret.Get(0).(*rest.Config) } } if rf, ok := ret.Get(1).(func() error); ok { r1 = rf() } else { r1 = ret.Error(1) } return r0, r1 } // DiscoveryClient provides a mock function with given fields: func (_m *Factory) DiscoveryClient() (discovery.AggregatedDiscoveryInterface, error) { ret := _m.Called() var r0 discovery.AggregatedDiscoveryInterface var r1 error if rf, ok := ret.Get(0).(func() (discovery.AggregatedDiscoveryInterface, error)); ok { return rf() } if rf, ok := ret.Get(0).(func() discovery.AggregatedDiscoveryInterface); ok { r0 = rf() } else { if ret.Get(0) != nil { r0 = ret.Get(0).(discovery.AggregatedDiscoveryInterface) } } if rf, ok := ret.Get(1).(func() error); ok { r1 = rf() } else { r1 = ret.Error(1) } return r0, r1 } // DynamicClient provides a mock function with given fields: func (_m *Factory) DynamicClient() (dynamic.Interface, error) { ret := _m.Called() var r0 dynamic.Interface var r1 error if rf, ok := ret.Get(0).(func() (dynamic.Interface, error)); ok { return rf() } if rf, ok := ret.Get(0).(func() dynamic.Interface); ok { r0 = rf() } else { if ret.Get(0) != nil { r0 = ret.Get(0).(dynamic.Interface) } } if rf, ok := ret.Get(1).(func() error); ok { r1 = rf() } else { r1 = ret.Error(1) } return r0, r1 } // KubeClient provides a mock function with given fields: func (_m *Factory) KubeClient() (kubernetes.Interface, error) { ret := _m.Called() var r0 kubernetes.Interface var r1 error if rf, ok := ret.Get(0).(func() (kubernetes.Interface, error)); ok { return rf() } if rf, ok := ret.Get(0).(func() kubernetes.Interface); ok { r0 = rf() } else { if ret.Get(0) != nil { r0 = ret.Get(0).(kubernetes.Interface) } } if rf, ok := ret.Get(1).(func() error); ok { r1 = rf() } else { r1 = ret.Error(1) } return r0, r1 } // KubebuilderClient provides a mock function with given fields: func (_m *Factory) KubebuilderClient() (pkgclient.Client, error) { ret := _m.Called() var r0 pkgclient.Client var r1 error if rf, ok := ret.Get(0).(func() (pkgclient.Client, error)); ok { return rf() } if rf, ok := ret.Get(0).(func() pkgclient.Client); ok { r0 = rf() } else { if ret.Get(0) != nil { r0 = ret.Get(0).(pkgclient.Client) } } if rf, ok := ret.Get(1).(func() error); ok { r1 = rf() } else { r1 = ret.Error(1) } return r0, r1 } // KubebuilderWatchClient provides a mock function with given fields: func (_m *Factory) KubebuilderWatchClient() (pkgclient.WithWatch, error) { ret := _m.Called() var r0 pkgclient.WithWatch var r1 error if rf, ok := ret.Get(0).(func() (pkgclient.WithWatch, error)); ok { return rf() } if rf, ok := ret.Get(0).(func() pkgclient.WithWatch); ok { r0 = rf() } else { if ret.Get(0) != nil { r0 = ret.Get(0).(pkgclient.WithWatch) } } if rf, ok := ret.Get(1).(func() error); ok { r1 = rf() } else { r1 = ret.Error(1) } return r0, r1 } // Namespace provides a mock function with given fields: func (_m *Factory) Namespace() string { ret := _m.Called() var r0 string if rf, ok := ret.Get(0).(func() string); ok { r0 = rf() } else { r0 = ret.Get(0).(string) } return r0 } // SetBasename provides a mock function with given fields: _a0 func (_m *Factory) SetBasename(_a0 string) { _m.Called(_a0) } // SetClientBurst provides a mock function with given fields: _a0 func (_m *Factory) SetClientBurst(_a0 int) { _m.Called(_a0) } // SetClientQPS provides a mock function with given fields: _a0 func (_m *Factory) SetClientQPS(_a0 float32) { _m.Called(_a0) } type mockConstructorTestingTNewFactory interface { mock.TestingT Cleanup(func()) } // NewFactory creates a new instance of Factory. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. func NewFactory(t mockConstructorTestingTNewFactory) *Factory { mock := &Factory{} mock.Mock.Test(t) t.Cleanup(func() { mock.AssertExpectations(t) }) return mock } ================================================ FILE: pkg/client/retry.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package client import ( "context" "math" "time" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/util/retry" kbclient "sigs.k8s.io/controller-runtime/pkg/client" ) func CreateRetryGenerateName(client kbclient.Client, ctx context.Context, obj kbclient.Object) error { retryCreateFn := func() error { // needed to ensure that the name from the failed create isn't left on the object between retries obj.SetName("") return client.Create(ctx, obj, &kbclient.CreateOptions{}) } if obj.GetGenerateName() != "" && obj.GetName() == "" { return retry.OnError(retry.DefaultRetry, apierrors.IsAlreadyExists, retryCreateFn) } else { return client.Create(ctx, obj, &kbclient.CreateOptions{}) } } // CapBackoff provides a backoff with a set backoff cap func CapBackoff(cap time.Duration) wait.Backoff { if cap < 0 { cap = 0 } return wait.Backoff{ Steps: math.MaxInt, Duration: 10 * time.Millisecond, Cap: cap, Factor: retry.DefaultBackoff.Factor, Jitter: retry.DefaultBackoff.Jitter, } } // RetryOnRetriableMaxBackOff accepts a patch function param, retrying when the provided retriable function returns true. func RetryOnRetriableMaxBackOff(maxDuration time.Duration, fn func() error, retriable func(error) bool) error { return retry.OnError(CapBackoff(maxDuration), func(err error) bool { return retriable(err) }, fn) } // RetryOnErrorMaxBackOff accepts a patch function param, retrying when the error is not nil. func RetryOnErrorMaxBackOff(maxDuration time.Duration, fn func() error) error { return RetryOnRetriableMaxBackOff(maxDuration, fn, func(err error) bool { return err != nil }) } ================================================ FILE: pkg/cmd/cli/backup/backup.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backup import ( "github.com/spf13/cobra" "github.com/vmware-tanzu/velero/pkg/client" ) func NewCommand(f client.Factory) *cobra.Command { c := &cobra.Command{ Use: "backup", Short: "Work with backups", Long: "Work with backups", } c.AddCommand( NewCreateCommand(f, "create"), NewGetCommand(f, "get"), NewLogsCommand(f), NewDescribeCommand(f, "describe"), NewDownloadCommand(f), NewDeleteCommand(f, "delete"), ) return c } ================================================ FILE: pkg/cmd/cli/backup/backup_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backup import ( "testing" "github.com/stretchr/testify/assert" factorymocks "github.com/vmware-tanzu/velero/pkg/client/mocks" ) func TestNewBackupCommand(t *testing.T) { // create a factory f := &factorymocks.Factory{} // create command cmd := NewCommand(f) assert.Equal(t, "Work with backups", cmd.Short) } ================================================ FILE: pkg/cmd/cli/backup/create.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backup import ( "context" "fmt" "strings" "time" "github.com/spf13/cobra" "github.com/spf13/pflag" kubeerrs "k8s.io/apimachinery/pkg/util/errors" "k8s.io/client-go/tools/cache" kbclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" "github.com/vmware-tanzu/velero/pkg/cmd/util/flag" "github.com/vmware-tanzu/velero/pkg/cmd/util/output" "github.com/vmware-tanzu/velero/pkg/util/collections" "github.com/vmware-tanzu/velero/pkg/util/kube" ) func NewCreateCommand(f client.Factory, use string) *cobra.Command { o := NewCreateOptions() c := &cobra.Command{ Use: use + " NAME", Short: "Create a backup", Args: cobra.MaximumNArgs(1), Run: func(c *cobra.Command, args []string) { cmd.CheckError(o.Complete(args, f)) cmd.CheckError(o.Validate(c, args, f)) cmd.CheckError(o.Run(c, f)) }, Example: ` # Create a backup containing all resources. velero backup create backup1 # Create a backup including only the nginx namespace. velero backup create nginx-backup --include-namespaces nginx # Create a backup excluding the velero and default namespaces. velero backup create backup2 --exclude-namespaces velero,default # Create a backup based on a schedule named daily-backup. velero backup create --from-schedule daily-backup # View the YAML for a backup that doesn't snapshot volumes, without sending it to the server. velero backup create backup3 --snapshot-volumes=false -o yaml # Wait for a backup to complete before returning from the command. velero backup create backup4 --wait`, } o.BindFlags(c.Flags()) o.BindWait(c.Flags()) o.BindFromSchedule(c.Flags()) output.BindFlags(c.Flags()) output.ClearOutputFlagDefault(c) return c } type CreateOptions struct { Name string TTL time.Duration SnapshotVolumes flag.OptionalBool SnapshotMoveData flag.OptionalBool DataMover string DefaultVolumesToFsBackup flag.OptionalBool IncludeNamespaces flag.StringArray ExcludeNamespaces flag.StringArray IncludeResources flag.StringArray ExcludeResources flag.StringArray IncludeClusterScopedResources flag.StringArray ExcludeClusterScopedResources flag.StringArray IncludeNamespaceScopedResources flag.StringArray ExcludeNamespaceScopedResources flag.StringArray Labels flag.Map Annotations flag.Map Selector flag.LabelSelector OrSelector flag.OrLabelSelector IncludeClusterResources flag.OptionalBool Wait bool StorageLocation string SnapshotLocations []string FromSchedule string OrderedResources string CSISnapshotTimeout time.Duration ItemOperationTimeout time.Duration ResPoliciesConfigmap string client kbclient.WithWatch ParallelFilesUpload int } func NewCreateOptions() *CreateOptions { return &CreateOptions{ IncludeNamespaces: flag.NewStringArray("*"), Labels: flag.NewMap(), Annotations: flag.NewMap(), SnapshotVolumes: flag.NewOptionalBool(nil), IncludeClusterResources: flag.NewOptionalBool(nil), } } func (o *CreateOptions) BindFlags(flags *pflag.FlagSet) { flags.DurationVar(&o.TTL, "ttl", o.TTL, "How long before the backup can be garbage collected.") flags.Var(&o.IncludeNamespaces, "include-namespaces", "Namespaces to include in the backup (use '*' for all namespaces).") flags.Var(&o.ExcludeNamespaces, "exclude-namespaces", "Namespaces to exclude from the backup.") flags.Var(&o.IncludeResources, "include-resources", "Resources to include in the backup, formatted as resource.group, such as storageclasses.storage.k8s.io (use '*' for all resources). Cannot work with include-cluster-scoped-resources, exclude-cluster-scoped-resources, include-namespace-scoped-resources and exclude-namespace-scoped-resources.") flags.Var(&o.ExcludeResources, "exclude-resources", "Resources to exclude from the backup, formatted as resource.group, such as storageclasses.storage.k8s.io. Cannot work with include-cluster-scoped-resources, exclude-cluster-scoped-resources, include-namespace-scoped-resources and exclude-namespace-scoped-resources.") flags.Var(&o.IncludeClusterScopedResources, "include-cluster-scoped-resources", "Cluster-scoped resources to include in the backup, formatted as resource.group, such as storageclasses.storage.k8s.io(use '*' for all resources). Cannot work with include-resources, exclude-resources and include-cluster-resources.") flags.Var(&o.ExcludeClusterScopedResources, "exclude-cluster-scoped-resources", "Cluster-scoped resources to exclude from the backup, formatted as resource.group, such as storageclasses.storage.k8s.io(use '*' for all resources). Cannot work with include-resources, exclude-resources and include-cluster-resources.") flags.Var(&o.IncludeNamespaceScopedResources, "include-namespace-scoped-resources", "Namespaced resources to include in the backup, formatted as resource.group, such as deployments.apps(use '*' for all resources). Cannot work with include-resources, exclude-resources and include-cluster-resources.") flags.Var(&o.ExcludeNamespaceScopedResources, "exclude-namespace-scoped-resources", "Namespaced resources to exclude from the backup, formatted as resource.group, such as deployments.apps(use '*' for all resources). Cannot work with include-resources, exclude-resources and include-cluster-resources.") flags.Var(&o.Labels, "labels", "Labels to apply to the backup.") flags.Var(&o.Annotations, "annotations", "Annotations to apply to the backup.") flags.StringVar(&o.StorageLocation, "storage-location", "", "Location in which to store the backup.") flags.StringSliceVar(&o.SnapshotLocations, "volume-snapshot-locations", o.SnapshotLocations, "List of locations (at most one per provider) where volume snapshots should be stored.") flags.VarP(&o.Selector, "selector", "l", "Only back up resources matching this label selector.") flags.Var(&o.OrSelector, "or-selector", "Backup resources matching at least one of the label selector from the list. Label selectors should be separated by ' or '. For example, foo=bar or app=nginx") flags.StringVar(&o.OrderedResources, "ordered-resources", "", "Mapping Kinds to an ordered list of specific resources of that Kind. Resource names are separated by commas and their names are in format 'namespace/resourcename'. For cluster scope resource, simply use resource name. Key-value pairs in the mapping are separated by semi-colon. Example: 'pods=ns1/pod1,ns1/pod2;persistentvolumeclaims=ns1/pvc4,ns1/pvc8'. Optional.") flags.DurationVar(&o.CSISnapshotTimeout, "csi-snapshot-timeout", o.CSISnapshotTimeout, "How long to wait for CSI snapshot creation before timeout.") flags.DurationVar(&o.ItemOperationTimeout, "item-operation-timeout", o.ItemOperationTimeout, "How long to wait for async plugin operations before timeout.") f := flags.VarPF(&o.SnapshotVolumes, "snapshot-volumes", "", "Take snapshots of PersistentVolumes as part of the backup. If the parameter is not set, it is treated as setting to 'true'.") // this allows the user to just specify "--snapshot-volumes" as shorthand for "--snapshot-volumes=true" // like a normal bool flag f.NoOptDefVal = cmd.TRUE f = flags.VarPF(&o.SnapshotMoveData, "snapshot-move-data", "", "Specify whether snapshot data should be moved") f.NoOptDefVal = cmd.TRUE f = flags.VarPF(&o.IncludeClusterResources, "include-cluster-resources", "", "Include cluster-scoped resources in the backup. Cannot work with include-cluster-scoped-resources, exclude-cluster-scoped-resources, include-namespace-scoped-resources and exclude-namespace-scoped-resources.") f.NoOptDefVal = cmd.TRUE f = flags.VarPF(&o.DefaultVolumesToFsBackup, "default-volumes-to-fs-backup", "", "Use pod volume file system backup by default for volumes") f.NoOptDefVal = cmd.TRUE flags.StringVar(&o.ResPoliciesConfigmap, "resource-policies-configmap", "", "Reference to the resource policies configmap that backup should use") flags.StringVar(&o.DataMover, "data-mover", "", "Specify the data mover to be used by the backup. If the parameter is not set or set as 'velero', the built-in data mover will be used") flags.IntVar(&o.ParallelFilesUpload, "parallel-files-upload", 0, "Number of files uploads simultaneously when running a backup. This is only applicable for the kopia uploader") } // BindWait binds the wait flag separately so it is not called by other create // commands that reuse CreateOptions's BindFlags method. func (o *CreateOptions) BindWait(flags *pflag.FlagSet) { flags.BoolVarP(&o.Wait, "wait", "w", o.Wait, "Wait for the operation to complete.") } // BindFromSchedule binds the from-schedule flag separately so it is not called // by other create commands that reuse CreateOptions's BindFlags method. func (o *CreateOptions) BindFromSchedule(flags *pflag.FlagSet) { flags.StringVar(&o.FromSchedule, "from-schedule", "", "Create a backup from the template of an existing schedule. Cannot be used with any other filters. Backup name is optional if used.") } func (o *CreateOptions) Validate(c *cobra.Command, args []string, f client.Factory) error { if err := output.ValidateFlags(c); err != nil { return err } if o.Selector.LabelSelector != nil && o.OrSelector.OrLabelSelectors != nil { return fmt.Errorf("either a 'selector' or an 'or-selector' can be specified, but not both") } // Ensure if FromSchedule is set, it has a non-empty value if err := o.validateFromScheduleFlag(c); err != nil { return err } // Ensure that unless FromSchedule is set, args contains a backup name if o.FromSchedule == "" && len(args) != 1 { return fmt.Errorf("a backup name is required, unless you are creating based on a schedule") } errs := collections.ValidateNamespaceIncludesExcludes(o.IncludeNamespaces, o.ExcludeNamespaces) if len(errs) > 0 { return kubeerrs.NewAggregate(errs) } if o.oldAndNewFilterParametersUsedTogether() { return fmt.Errorf("include-resources, exclude-resources and include-cluster-resources are old filter parameters.\n" + "include-cluster-scoped-resources, exclude-cluster-scoped-resources, include-namespace-scoped-resources and exclude-namespace-scoped-resources are new filter parameters.\n" + "They cannot be used together") } if o.StorageLocation != "" { location := &velerov1api.BackupStorageLocation{} if err := o.client.Get(context.Background(), kbclient.ObjectKey{ Namespace: f.Namespace(), Name: o.StorageLocation, }, location); err != nil { return err } } for _, loc := range o.SnapshotLocations { snapshotLocation := new(velerov1api.VolumeSnapshotLocation) if err := o.client.Get(context.Background(), kbclient.ObjectKey{Namespace: f.Namespace(), Name: loc}, snapshotLocation); err != nil { return err } } return nil } func (o *CreateOptions) validateFromScheduleFlag(c *cobra.Command) error { trimmed := strings.TrimSpace(o.FromSchedule) if c.Flags().Changed("from-schedule") && trimmed == "" { return fmt.Errorf("flag must have a non-empty value: --from-schedule") } // Assign the trimmed value back o.FromSchedule = trimmed return nil } func (o *CreateOptions) Complete(args []string, f client.Factory) error { // If an explicit name is specified, use that name if len(args) > 0 { o.Name = args[0] } client, err := f.KubebuilderWatchClient() if err != nil { return err } o.client = client return nil } func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error { backup, err := o.BuildBackup(f.Namespace()) if err != nil { return err } if printed, err := output.PrintWithFormat(c, backup); printed || err != nil { return err } if o.FromSchedule != "" { fmt.Println("Creating backup from schedule, all other filters are ignored.") } var updates chan *velerov1api.Backup if o.Wait { stop := make(chan struct{}) defer close(stop) updates = make(chan *velerov1api.Backup) lw := kube.InternalLW{ Client: o.client, Namespace: f.Namespace(), ObjectList: new(velerov1api.BackupList), } backupInformer := cache.NewSharedInformer(&lw, &velerov1api.Backup{}, time.Second) _, _ = backupInformer.AddEventHandler( cache.FilteringResourceEventHandler{ FilterFunc: func(obj any) bool { backup, ok := obj.(*velerov1api.Backup) if !ok { return false } return backup.Name == o.Name }, Handler: cache.ResourceEventHandlerFuncs{ UpdateFunc: func(_, obj any) { backup, ok := obj.(*velerov1api.Backup) if !ok { return } updates <- backup }, DeleteFunc: func(obj any) { backup, ok := obj.(*velerov1api.Backup) if !ok { return } updates <- backup }, }, }, ) go backupInformer.Run(stop) } err = o.client.Create(context.TODO(), backup, &kbclient.CreateOptions{}) if err != nil { return err } fmt.Printf("Backup request %q submitted successfully.\n", backup.Name) if o.Wait { fmt.Println("Waiting for backup to complete. You may safely press ctrl-c to stop waiting - your backup will continue in the background.") ticker := time.NewTicker(time.Second) defer ticker.Stop() for { select { case <-ticker.C: fmt.Print(".") case backup, ok := <-updates: if !ok { fmt.Println("\nError waiting: unable to watch backups.") return nil } if backup.Status.Phase == velerov1api.BackupPhaseFailedValidation || backup.Status.Phase == velerov1api.BackupPhaseCompleted || backup.Status.Phase == velerov1api.BackupPhasePartiallyFailed || backup.Status.Phase == velerov1api.BackupPhaseFailed { fmt.Printf("\nBackup completed with status: %s. You may check for more information using the commands `velero backup describe %s` and `velero backup logs %s`.\n", backup.Status.Phase, backup.Name, backup.Name) return nil } } } } // Not waiting fmt.Printf("Run `velero backup describe %s` or `velero backup logs %s` for more details.\n", backup.Name, backup.Name) return nil } // ParseOrderedResources converts to map of Kinds to an ordered list of specific resources of that Kind. // Resource names in the list are in format 'namespace/resourcename' and separated by commas. // Key-value pairs in the mapping are separated by semi-colon. // Ex: 'pods=ns1/pod1,ns1/pod2;persistentvolumeclaims=ns1/pvc4,ns1/pvc8'. func ParseOrderedResources(orderMapStr string) (map[string]string, error) { entries := strings.Split(orderMapStr, ";") if len(entries) == 0 { return nil, fmt.Errorf("invalid OrderedResources '%s'", orderMapStr) } orderedResources := make(map[string]string) for _, entry := range entries { kv := strings.Split(entry, "=") if len(kv) != 2 { return nil, fmt.Errorf("invalid OrderedResources '%s'", entry) } kind := strings.TrimSpace(kv[0]) order := strings.TrimSpace(kv[1]) orderedResources[kind] = order } return orderedResources, nil } func (o *CreateOptions) BuildBackup(namespace string) (*velerov1api.Backup, error) { var backupBuilder *builder.BackupBuilder if o.FromSchedule != "" { schedule := new(velerov1api.Schedule) err := o.client.Get(context.TODO(), kbclient.ObjectKey{Namespace: namespace, Name: o.FromSchedule}, schedule) if err != nil { return nil, err } if o.Name == "" { o.Name = schedule.TimestampedName(time.Now().UTC()) } backupBuilder = builder.ForBackup(namespace, o.Name). FromSchedule(schedule) } else { backupBuilder = builder.ForBackup(namespace, o.Name). IncludedNamespaces(o.IncludeNamespaces...). ExcludedNamespaces(o.ExcludeNamespaces...). IncludedResources(o.IncludeResources...). ExcludedResources(o.ExcludeResources...). IncludedClusterScopedResources(o.IncludeClusterScopedResources...). ExcludedClusterScopedResources(o.ExcludeClusterScopedResources...). IncludedNamespaceScopedResources(o.IncludeNamespaceScopedResources...). ExcludedNamespaceScopedResources(o.ExcludeNamespaceScopedResources...). LabelSelector(o.Selector.LabelSelector). OrLabelSelector(o.OrSelector.OrLabelSelectors). TTL(o.TTL). StorageLocation(o.StorageLocation). VolumeSnapshotLocations(o.SnapshotLocations...). CSISnapshotTimeout(o.CSISnapshotTimeout). ItemOperationTimeout(o.ItemOperationTimeout). DataMover(o.DataMover) if len(o.OrderedResources) > 0 { orders, err := ParseOrderedResources(o.OrderedResources) if err != nil { return nil, err } backupBuilder.OrderedResources(orders) } if o.SnapshotVolumes.Value != nil { backupBuilder.SnapshotVolumes(*o.SnapshotVolumes.Value) } if o.SnapshotMoveData.Value != nil { backupBuilder.SnapshotMoveData(*o.SnapshotMoveData.Value) } if o.IncludeClusterResources.Value != nil { backupBuilder.IncludeClusterResources(*o.IncludeClusterResources.Value) } if o.DefaultVolumesToFsBackup.Value != nil { backupBuilder.DefaultVolumesToFsBackup(*o.DefaultVolumesToFsBackup.Value) } if o.ResPoliciesConfigmap != "" { backupBuilder.ResourcePolicies(o.ResPoliciesConfigmap) } if o.ParallelFilesUpload > 0 { backupBuilder.ParallelFilesUpload(o.ParallelFilesUpload) } } backup := backupBuilder.ObjectMeta(builder.WithLabelsMap(o.Labels.Data()), builder.WithAnnotationsMap(o.Annotations.Data())).Result() return backup, nil } func (o *CreateOptions) oldAndNewFilterParametersUsedTogether() bool { haveOldResourceFilterParameters := len(o.IncludeResources) > 0 || len(o.ExcludeResources) > 0 || o.IncludeClusterResources.Value != nil haveNewResourceFilterParameters := len(o.IncludeClusterScopedResources) > 0 || (len(o.ExcludeClusterScopedResources) > 0) || (len(o.IncludeNamespaceScopedResources) > 0) || (len(o.ExcludeNamespaceScopedResources) > 0) return haveOldResourceFilterParameters && haveNewResourceFilterParameters } ================================================ FILE: pkg/cmd/cli/backup/create_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backup import ( "fmt" "strconv" "strings" "testing" "time" "github.com/spf13/cobra" flag "github.com/spf13/pflag" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" controllerclient "sigs.k8s.io/controller-runtime/pkg/client" kbclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" factorymocks "github.com/vmware-tanzu/velero/pkg/client/mocks" cmdtest "github.com/vmware-tanzu/velero/pkg/cmd/test" "github.com/vmware-tanzu/velero/pkg/test" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func TestCreateOptions_BuildBackup(t *testing.T) { o := NewCreateOptions() o.Labels.Set("velero.io/test=true") o.Annotations.Set("velero.io/annTest=true") o.OrderedResources = "pods=p1,p2;persistentvolumeclaims=pvc1,pvc2" orders, err := ParseOrderedResources(o.OrderedResources) o.CSISnapshotTimeout = 20 * time.Minute o.ItemOperationTimeout = 20 * time.Minute orLabelSelectors := []*metav1.LabelSelector{ { MatchLabels: map[string]string{"k1": "v1", "k2": "v2"}, }, { MatchLabels: map[string]string{"a1": "b1", "a2": "b2"}, }, } o.OrSelector.OrLabelSelectors = orLabelSelectors require.NoError(t, err) backup, err := o.BuildBackup(cmdtest.VeleroNameSpace) require.NoError(t, err) assert.Equal(t, velerov1api.BackupSpec{ TTL: metav1.Duration{Duration: o.TTL}, IncludedNamespaces: []string(o.IncludeNamespaces), SnapshotVolumes: o.SnapshotVolumes.Value, IncludeClusterResources: o.IncludeClusterResources.Value, OrderedResources: orders, OrLabelSelectors: orLabelSelectors, CSISnapshotTimeout: metav1.Duration{Duration: o.CSISnapshotTimeout}, ItemOperationTimeout: metav1.Duration{Duration: o.ItemOperationTimeout}, }, backup.Spec) assert.Equal(t, map[string]string{ "velero.io/test": "true", }, backup.GetLabels()) assert.Equal(t, map[string]string{ "velero.io/annTest": "true", }, backup.GetAnnotations()) assert.Equal(t, map[string]string{ "pods": "p1,p2", "persistentvolumeclaims": "pvc1,pvc2", }, backup.Spec.OrderedResources) } func TestCreateOptions_ValidateFromScheduleFlag(t *testing.T) { cmd := &cobra.Command{} o := NewCreateOptions() o.BindFromSchedule(cmd.Flags()) t.Run("from-schedule with empty or no value", func(t *testing.T) { cmd.Flags().Set("from-schedule", "") err := o.validateFromScheduleFlag(cmd) require.True(t, cmd.Flags().Changed("from-schedule")) require.Error(t, err) require.Equal(t, "flag must have a non-empty value: --from-schedule", err.Error()) }) t.Run("from-schedule with spaces only", func(t *testing.T) { cmd.Flags().Set("from-schedule", " ") err := o.validateFromScheduleFlag(cmd) require.True(t, cmd.Flags().Changed("from-schedule")) require.Error(t, err) require.Equal(t, "flag must have a non-empty value: --from-schedule", err.Error()) }) t.Run("from-schedule with valid value", func(t *testing.T) { cmd.Flags().Set("from-schedule", "daily") err := o.validateFromScheduleFlag(cmd) require.NoError(t, err) require.Equal(t, "daily", o.FromSchedule) }) t.Run("from-schedule with leading and trailing spaces", func(t *testing.T) { cmd.Flags().Set("from-schedule", " daily ") err := o.validateFromScheduleFlag(cmd) require.NoError(t, err) require.Equal(t, "daily", o.FromSchedule) }) } func TestCreateOptions_BuildBackupFromSchedule(t *testing.T) { o := NewCreateOptions() o.FromSchedule = "test" scheme := runtime.NewScheme() err := velerov1api.AddToScheme(scheme) require.NoError(t, err) o.client = velerotest.NewFakeControllerRuntimeClient(t).(controllerclient.WithWatch) t.Run("inexistent schedule", func(t *testing.T) { _, err := o.BuildBackup(cmdtest.VeleroNameSpace) require.Error(t, err) }) expectedBackupSpec := builder.ForBackup("test", cmdtest.VeleroNameSpace).IncludedNamespaces("test").Result().Spec schedule := builder.ForSchedule(cmdtest.VeleroNameSpace, "test").Template(expectedBackupSpec).ObjectMeta(builder.WithLabels("velero.io/test", "true"), builder.WithAnnotations("velero.io/test", "true")).Result() o.client.Create(t.Context(), schedule, &kbclient.CreateOptions{}) t.Run("existing schedule", func(t *testing.T) { backup, err := o.BuildBackup(cmdtest.VeleroNameSpace) require.NoError(t, err) require.Equal(t, expectedBackupSpec, backup.Spec) require.Equal(t, map[string]string{ "velero.io/test": "true", velerov1api.ScheduleNameLabel: "test", }, backup.GetLabels()) require.Equal(t, map[string]string{ "velero.io/test": "true", }, backup.GetAnnotations()) }) t.Run("command line labels and annotations take precedence over scheduled ones", func(t *testing.T) { o.Labels.Set("velero.io/test=yes,custom-label=true") o.Annotations.Set("velero.io/test=yes,custom-annotation=true") backup, err := o.BuildBackup(cmdtest.VeleroNameSpace) require.NoError(t, err) assert.Equal(t, expectedBackupSpec, backup.Spec) assert.Equal(t, map[string]string{ "velero.io/test": "yes", velerov1api.ScheduleNameLabel: "test", "custom-label": "true", }, backup.GetLabels()) assert.Equal(t, map[string]string{ "velero.io/test": "yes", "custom-annotation": "true", }, backup.GetAnnotations()) }) } func TestCreateOptions_OrderedResources(t *testing.T) { _, err := ParseOrderedResources("pods= ns1/p1; ns1/p2; persistentvolumeclaims=ns2/pvc1, ns2/pvc2") require.Error(t, err) orderedResources, err := ParseOrderedResources("pods= ns1/p1,ns1/p2 ; persistentvolumeclaims=ns2/pvc1,ns2/pvc2") require.NoError(t, err) expectedResources := map[string]string{ "pods": "ns1/p1,ns1/p2", "persistentvolumeclaims": "ns2/pvc1,ns2/pvc2", } assert.Equal(t, expectedResources, orderedResources) orderedResources, err = ParseOrderedResources("pods= ns1/p1,ns1/p2 ; persistentvolumes=pv1,pv2") require.NoError(t, err) expectedMixedResources := map[string]string{ "pods": "ns1/p1,ns1/p2", "persistentvolumes": "pv1,pv2", } assert.Equal(t, expectedMixedResources, orderedResources) } func TestCreateCommand(t *testing.T) { name := "nameToBeCreated" args := []string{name} t.Run("create a backup create command with full options except fromSchedule and wait, then run by create option", func(t *testing.T) { // create a factory f := &factorymocks.Factory{} // create command cmd := NewCreateCommand(f, "") assert.Equal(t, "Create a backup", cmd.Short) includeNamespaces := "app1,app2" excludeNamespaces := "pod1,pod2,pod3" includeResources := "sc,sts" excludeResources := "job" includeClusterScopedResources := "pv,ComponentStatus" excludeClusterScopedResources := "MutatingWebhookConfiguration,APIService" includeNamespaceScopedResources := "Endpoints,Event,PodTemplate" excludeNamespaceScopedResources := "Secret,MultiClusterIngress" labels := "c=foo" annotations := "ann=foo" storageLocation := "bsl-name-1" snapshotLocations := "region=minio" selector := "a=pod" orderedResources := "pod=pod1,pod2,pod3" csiSnapshotTimeout := "8m30s" itemOperationTimeout := "99h1m6s" snapshotVolumes := "false" snapshotMoveData := "true" includeClusterResources := "true" defaultVolumesToFsBackup := "true" resPoliciesConfigmap := "cm-name-2" dataMover := "velero" parallelFilesUpload := 10 flags := new(flag.FlagSet) o := NewCreateOptions() o.BindFlags(flags) o.BindWait(flags) o.BindFromSchedule(flags) flags.Parse([]string{"--include-namespaces", includeNamespaces}) flags.Parse([]string{"--exclude-namespaces", excludeNamespaces}) flags.Parse([]string{"--include-resources", includeResources}) flags.Parse([]string{"--exclude-resources", excludeResources}) flags.Parse([]string{"--include-cluster-scoped-resources", includeClusterScopedResources}) flags.Parse([]string{"--exclude-cluster-scoped-resources", excludeClusterScopedResources}) flags.Parse([]string{"--include-namespace-scoped-resources", includeNamespaceScopedResources}) flags.Parse([]string{"--exclude-namespace-scoped-resources", excludeNamespaceScopedResources}) flags.Parse([]string{"--labels", labels}) flags.Parse([]string{"--annotations", annotations}) flags.Parse([]string{"--storage-location", storageLocation}) flags.Parse([]string{"--volume-snapshot-locations", snapshotLocations}) flags.Parse([]string{"--selector", selector}) flags.Parse([]string{"--ordered-resources", orderedResources}) flags.Parse([]string{"--csi-snapshot-timeout", csiSnapshotTimeout}) flags.Parse([]string{"--item-operation-timeout", itemOperationTimeout}) flags.Parse([]string{fmt.Sprintf("--snapshot-volumes=%s", snapshotVolumes)}) flags.Parse([]string{fmt.Sprintf("--snapshot-move-data=%s", snapshotMoveData)}) flags.Parse([]string{"--include-cluster-resources", includeClusterResources}) flags.Parse([]string{"--default-volumes-to-fs-backup", defaultVolumesToFsBackup}) flags.Parse([]string{"--resource-policies-configmap", resPoliciesConfigmap}) flags.Parse([]string{"--data-mover", dataMover}) flags.Parse([]string{"--parallel-files-upload", strconv.Itoa(parallelFilesUpload)}) //flags.Parse([]string{"--wait"}) client := velerotest.NewFakeControllerRuntimeClient(t).(kbclient.WithWatch) f.On("Namespace").Return(mock.Anything) f.On("KubebuilderWatchClient").Return(client, nil) //Complete e := o.Complete(args, f) require.NoError(t, e) //Validate e = o.Validate(cmd, args, f) require.ErrorContains(t, e, "include-resources, exclude-resources and include-cluster-resources are old filter parameters") require.ErrorContains(t, e, "include-cluster-scoped-resources, exclude-cluster-scoped-resources, include-namespace-scoped-resources and exclude-namespace-scoped-resources are new filter parameters.\nThey cannot be used together") //cmd e = o.Run(cmd, f) require.NoError(t, e) //Execute cmd.SetArgs([]string{"bk-name-exe"}) e = cmd.Execute() require.NoError(t, e) // verify all options are set as expected require.Equal(t, name, o.Name) require.Equal(t, includeNamespaces, o.IncludeNamespaces.String()) require.Equal(t, excludeNamespaces, o.ExcludeNamespaces.String()) require.Equal(t, includeResources, o.IncludeResources.String()) require.Equal(t, excludeResources, o.ExcludeResources.String()) require.Equal(t, includeClusterScopedResources, o.IncludeClusterScopedResources.String()) require.Equal(t, excludeClusterScopedResources, o.ExcludeClusterScopedResources.String()) require.Equal(t, includeNamespaceScopedResources, o.IncludeNamespaceScopedResources.String()) require.Equal(t, excludeNamespaceScopedResources, o.ExcludeNamespaceScopedResources.String()) require.True(t, test.CompareSlice(strings.Split(labels, ","), strings.Split(o.Labels.String(), ","))) require.True(t, test.CompareSlice(strings.Split(annotations, ","), strings.Split(o.Annotations.String(), ","))) require.Equal(t, storageLocation, o.StorageLocation) require.Equal(t, snapshotLocations, strings.Split(o.SnapshotLocations[0], ",")[0]) require.Equal(t, selector, o.Selector.String()) require.Equal(t, orderedResources, o.OrderedResources) require.Equal(t, csiSnapshotTimeout, o.CSISnapshotTimeout.String()) require.Equal(t, itemOperationTimeout, o.ItemOperationTimeout.String()) require.Equal(t, snapshotVolumes, o.SnapshotVolumes.String()) require.Equal(t, snapshotMoveData, o.SnapshotMoveData.String()) require.Equal(t, includeClusterResources, o.IncludeClusterResources.String()) require.Equal(t, defaultVolumesToFsBackup, o.DefaultVolumesToFsBackup.String()) require.Equal(t, resPoliciesConfigmap, o.ResPoliciesConfigmap) require.Equal(t, dataMover, o.DataMover) require.Equal(t, parallelFilesUpload, o.ParallelFilesUpload) //assert.Equal(t, true, o.Wait) // verify oldAndNewFilterParametersUsedTogether mix := o.oldAndNewFilterParametersUsedTogether() require.True(t, mix) }) t.Run("create a backup create command with specific storage-location setting", func(t *testing.T) { bsl := "bsl-1" // create a factory f := &factorymocks.Factory{} cmd := NewCreateCommand(f, "") kbclient := velerotest.NewFakeControllerRuntimeClient(t).(kbclient.WithWatch) f.On("Namespace").Return(mock.Anything) f.On("KubebuilderWatchClient").Return(kbclient, nil) flags := new(flag.FlagSet) o := NewCreateOptions() o.BindFlags(flags) o.BindWait(flags) o.BindFromSchedule(flags) flags.Parse([]string{"--include-namespaces", "ns-1"}) flags.Parse([]string{"--storage-location", bsl}) // Complete e := o.Complete(args, f) require.NoError(t, e) // Validate e = o.Validate(cmd, args, f) assert.ErrorContains(t, e, fmt.Sprintf("backupstoragelocations.velero.io \"%s\" not found", bsl)) }) t.Run("create a backup create command with specific volume-snapshot-locations setting", func(t *testing.T) { vslName := "vsl-1" // create a factory f := &factorymocks.Factory{} cmd := NewCreateCommand(f, "") kbclient := velerotest.NewFakeControllerRuntimeClient(t).(kbclient.WithWatch) vsl := builder.ForVolumeSnapshotLocation(cmdtest.VeleroNameSpace, vslName).Result() kbclient.Create(cmd.Context(), vsl, &controllerclient.CreateOptions{}) f.On("Namespace").Return(cmdtest.VeleroNameSpace) f.On("KubebuilderWatchClient").Return(kbclient, nil) flags := new(flag.FlagSet) o := NewCreateOptions() o.BindFlags(flags) o.BindWait(flags) o.BindFromSchedule(flags) flags.Parse([]string{"--include-namespaces", "ns-1"}) flags.Parse([]string{"--volume-snapshot-locations", vslName}) // Complete e := o.Complete(args, f) require.NoError(t, e) // Validate e = o.Validate(cmd, args, f) assert.NoError(t, e) }) t.Run("create the other create command with fromSchedule option for Run() other branches", func(t *testing.T) { f := &factorymocks.Factory{} c := NewCreateCommand(f, "") assert.Equal(t, "Create a backup", c.Short) flags := new(flag.FlagSet) o := NewCreateOptions() o.BindFlags(flags) o.BindWait(flags) o.BindFromSchedule(flags) fromSchedule := "schedule-name-1" flags.Parse([]string{"--from-schedule", fromSchedule}) kbclient := velerotest.NewFakeControllerRuntimeClient(t).(kbclient.WithWatch) schedule := builder.ForSchedule(cmdtest.VeleroNameSpace, fromSchedule).Result() kbclient.Create(t.Context(), schedule, &controllerclient.CreateOptions{}) f.On("Namespace").Return(cmdtest.VeleroNameSpace) f.On("KubebuilderWatchClient").Return(kbclient, nil) e := o.Complete(args, f) require.NoError(t, e) e = o.Run(c, f) require.NoError(t, e) c.SetArgs([]string{"bk-1"}) e = c.Execute() assert.NoError(t, e) }) } ================================================ FILE: pkg/cmd/cli/backup/delete.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backup import ( "context" "fmt" "github.com/pkg/errors" "github.com/spf13/cobra" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" kubeerrs "k8s.io/apimachinery/pkg/util/errors" controllerclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" "github.com/vmware-tanzu/velero/pkg/cmd/cli" "github.com/vmware-tanzu/velero/pkg/cmd/util/confirm" "github.com/vmware-tanzu/velero/pkg/label" ) // NewDeleteCommand creates a new command that deletes a backup. func NewDeleteCommand(f client.Factory, use string) *cobra.Command { o := cli.NewDeleteOptions("backup") c := &cobra.Command{ Use: fmt.Sprintf("%s [NAMES]", use), Short: "Delete backups", Example: ` # Delete a backup named "backup-1". velero backup delete backup-1 # Delete a backup named "backup-1" without prompting for confirmation. velero backup delete backup-1 --confirm # Delete backups named "backup-1" and "backup-2". velero backup delete backup-1 backup-2 # Delete all backups triggered by schedule "schedule-1". velero backup delete --selector velero.io/schedule-name=schedule-1 # Delete all backups. velero backup delete --all`, Run: func(c *cobra.Command, args []string) { cmd.CheckError(o.Complete(f, args)) cmd.CheckError(o.Validate(c, f, args)) cmd.CheckError(Run(o)) }, } o.BindFlags(c.Flags()) return c } // Run performs the delete backup operation. func Run(o *cli.DeleteOptions) error { if !o.Confirm && !confirm.GetConfirmation() { // Don't do anything unless we get confirmation return nil } var ( backups []*velerov1api.Backup errs []error ) // get the list of backups to delete switch { case len(o.Names) > 0: for _, name := range o.Names { backup := new(velerov1api.Backup) err := o.Client.Get(context.TODO(), controllerclient.ObjectKey{Namespace: o.Namespace, Name: name}, backup) if err != nil { errs = append(errs, errors.WithStack(err)) continue } backups = append(backups, backup) } default: selector := labels.Everything() if o.Selector.LabelSelector != nil { convertedSelector, err := metav1.LabelSelectorAsSelector(o.Selector.LabelSelector) if err != nil { return errors.WithStack(err) } selector = convertedSelector } backupList := new(velerov1api.BackupList) err := o.Client.List(context.TODO(), backupList, &controllerclient.ListOptions{LabelSelector: selector, Namespace: o.Namespace}) if err != nil { return errors.WithStack(err) } for i := range backupList.Items { backups = append(backups, &backupList.Items[i]) } } if len(backups) == 0 { fmt.Println("No backups found") return nil } // create a backup deletion request for each for _, b := range backups { deleteRequest := builder.ForDeleteBackupRequest(o.Namespace, "").BackupName(b.Name). ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, label.GetValidName(b.Name), velerov1api.BackupUIDLabel, string(b.UID)), builder.WithGenerateName(b.Name+"-")).Result() if err := client.CreateRetryGenerateName(o.Client, context.TODO(), deleteRequest); err != nil { errs = append(errs, err) continue } fmt.Printf("Request to delete backup %q submitted successfully.\nThe backup will be fully deleted after all associated data (disk snapshots, backup files, restores) are removed.\n", b.Name) } return kubeerrs.NewAggregate(errs) } ================================================ FILE: pkg/cmd/cli/backup/delete_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backup import ( "fmt" "os" "os/exec" "testing" flag "github.com/spf13/pflag" "github.com/stretchr/testify/require" controllerclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/pkg/builder" factorymocks "github.com/vmware-tanzu/velero/pkg/client/mocks" "github.com/vmware-tanzu/velero/pkg/cmd/cli" cmdtest "github.com/vmware-tanzu/velero/pkg/cmd/test" velerotest "github.com/vmware-tanzu/velero/pkg/test" veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" ) func TestDeleteCommand(t *testing.T) { backup1 := "backup-name-1" backup2 := "backup-name-2" // create a factory f := &factorymocks.Factory{} client := velerotest.NewFakeControllerRuntimeClient(t) client.Create(t.Context(), builder.ForBackup(cmdtest.VeleroNameSpace, backup1).Result(), &controllerclient.CreateOptions{}) client.Create(t.Context(), builder.ForBackup("default", backup2).Result(), &controllerclient.CreateOptions{}) f.On("KubebuilderClient").Return(client, nil) f.On("Namespace").Return(cmdtest.VeleroNameSpace) // create command c := NewDeleteCommand(f, "velero backup delete") c.SetArgs([]string{backup1, backup2}) require.Equal(t, "Delete backups", c.Short) o := cli.NewDeleteOptions("backup") flags := new(flag.FlagSet) o.BindFlags(flags) flags.Parse([]string{"--confirm"}) args := []string{backup1, backup2} e := o.Complete(f, args) require.NoError(t, e) e = o.Validate(c, f, args) require.NoError(t, e) Run(o) e = c.Execute() require.NoError(t, e) if os.Getenv(cmdtest.CaptureFlag) == "1" { return } cmd := exec.CommandContext(t.Context(), os.Args[0], []string{"-test.run=TestDeleteCommand"}...) cmd.Env = append(os.Environ(), fmt.Sprintf("%s=1", cmdtest.CaptureFlag)) stdout, _, err := veleroexec.RunCommand(cmd) if err != nil { require.Contains(t, stdout, fmt.Sprintf("backups.velero.io \"%s\" not found.", backup2)) } } ================================================ FILE: pkg/cmd/cli/backup/describe.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backup import ( "context" "fmt" "os" "github.com/spf13/cobra" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" controllerclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" "github.com/vmware-tanzu/velero/pkg/cmd/util/output" "github.com/vmware-tanzu/velero/pkg/label" ) func NewDescribeCommand(f client.Factory, use string) *cobra.Command { var ( listOptions metav1.ListOptions details bool insecureSkipTLSVerify bool outputFormat = "plaintext" ) config, err := client.LoadConfig() if err != nil { fmt.Fprintf(os.Stderr, "WARNING: Error reading config file: %v\n", err) } caCertFile := config.CACertFile() c := &cobra.Command{ Use: use + " [NAME1] [NAME2] [NAME...]", Short: "Describe backups", Run: func(c *cobra.Command, args []string) { kbClient, err := f.KubebuilderClient() cmd.CheckError(err) if outputFormat != "plaintext" && outputFormat != "json" { cmd.CheckError(fmt.Errorf("invalid output format '%s'. valid value are 'plaintext, json'", outputFormat)) } backups := new(velerov1api.BackupList) if len(args) > 0 { for _, name := range args { backup := new(velerov1api.Backup) err := kbClient.Get(context.Background(), controllerclient.ObjectKey{Namespace: f.Namespace(), Name: name}, backup) cmd.CheckError(err) backups.Items = append(backups.Items, *backup) } } else { parsedSelector, err := labels.Parse(listOptions.LabelSelector) cmd.CheckError(err) err = kbClient.List(context.Background(), backups, &controllerclient.ListOptions{LabelSelector: parsedSelector, Namespace: f.Namespace()}) cmd.CheckError(err) } first := true for i, backup := range backups.Items { deleteRequestList := new(velerov1api.DeleteBackupRequestList) err := kbClient.List(context.Background(), deleteRequestList, &controllerclient.ListOptions{ Namespace: f.Namespace(), LabelSelector: labels.SelectorFromSet(map[string]string{velerov1api.BackupNameLabel: label.GetValidName(backup.Name), velerov1api.BackupUIDLabel: string(backup.UID)}), }) if err != nil { fmt.Fprintf(os.Stderr, "error getting DeleteBackupRequests for backup %s: %v\n", backup.Name, err) } podVolumeBackupList := new(velerov1api.PodVolumeBackupList) err = kbClient.List(context.Background(), podVolumeBackupList, &controllerclient.ListOptions{ Namespace: f.Namespace(), LabelSelector: labels.SelectorFromSet(map[string]string{velerov1api.BackupNameLabel: label.GetValidName(backup.Name)}), }) if err != nil { fmt.Fprintf(os.Stderr, "error getting PodVolumeBackups for backup %s: %v\n", backup.Name, err) } // structured output only applies to a single backup in case of OOM // To describe the list of backups in structured format, users could iterate over the list and describe backup one after another. if len(backups.Items) == 1 && outputFormat != "plaintext" { s := output.DescribeBackupInSF(context.Background(), kbClient, &backups.Items[i], deleteRequestList.Items, podVolumeBackupList.Items, details, insecureSkipTLSVerify, caCertFile, outputFormat) fmt.Print(s) } else { s := output.DescribeBackup(context.Background(), kbClient, &backups.Items[i], deleteRequestList.Items, podVolumeBackupList.Items, details, insecureSkipTLSVerify, caCertFile) if first { first = false fmt.Print(s) } else { fmt.Printf("\n\n%s", s) } } } cmd.CheckError(err) }, } c.Flags().StringVarP(&listOptions.LabelSelector, "selector", "l", listOptions.LabelSelector, "Only show items matching this label selector.") c.Flags().BoolVar(&details, "details", details, "Display additional detail in the command output.") c.Flags().BoolVar(&insecureSkipTLSVerify, "insecure-skip-tls-verify", insecureSkipTLSVerify, "If true, the object store's TLS certificate will not be checked for validity. This is insecure and susceptible to man-in-the-middle attacks. Not recommended for production.") c.Flags().StringVar(&caCertFile, "cacert", caCertFile, "Path to a certificate bundle to use when verifying TLS connections. If not specified, the CA certificate from the BackupStorageLocation will be used if available.") c.Flags().StringVarP(&outputFormat, "output", "o", outputFormat, "Output display format. Valid formats are 'plaintext, json'. 'json' only applies to a single backup") return c } ================================================ FILE: pkg/cmd/cli/backup/describe_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backup import ( "fmt" "os" "os/exec" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/client-go/rest" controllerclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/pkg/builder" factorymocks "github.com/vmware-tanzu/velero/pkg/client/mocks" cmdtest "github.com/vmware-tanzu/velero/pkg/cmd/test" "github.com/vmware-tanzu/velero/pkg/features" "github.com/vmware-tanzu/velero/pkg/test" veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" ) func TestNewDescribeCommand(t *testing.T) { // create a factory f := &factorymocks.Factory{} backupName := "bk-describe-1" testBackup := builder.ForBackup(cmdtest.VeleroNameSpace, backupName).SnapshotVolumes(false).Result() clientConfig := rest.Config{} kbClient := test.NewFakeControllerRuntimeClient(t) kbClient.Create(t.Context(), testBackup, &controllerclient.CreateOptions{}) f.On("ClientConfig").Return(&clientConfig, nil) f.On("Namespace").Return(cmdtest.VeleroNameSpace) f.On("KubebuilderClient").Return(kbClient, nil) // create command c := NewDescribeCommand(f, "velero backup describe") assert.Equal(t, "Describe backups", c.Short) features.NewFeatureFlagSet("EnableCSI") defer features.NewFeatureFlagSet() c.SetArgs([]string{backupName}) e := c.Execute() require.NoError(t, e) if os.Getenv(cmdtest.CaptureFlag) == "1" { return } cmd := exec.CommandContext(t.Context(), os.Args[0], []string{"-test.run=TestNewDescribeCommand"}...) cmd.Env = append(os.Environ(), fmt.Sprintf("%s=1", cmdtest.CaptureFlag)) stdout, _, err := veleroexec.RunCommand(cmd) if err == nil { assert.Contains(t, stdout, "Backup Volumes:") assert.Contains(t, stdout, "Or label selector: ") assert.Contains(t, stdout, fmt.Sprintf("Name: %s", backupName)) return } t.Fatalf("process ran with err %v, want backups by get()", err) } ================================================ FILE: pkg/cmd/cli/backup/download.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backup import ( "context" "fmt" "os" "path/filepath" "time" "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" controllerclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" "github.com/vmware-tanzu/velero/pkg/cmd/util/cacert" "github.com/vmware-tanzu/velero/pkg/cmd/util/downloadrequest" ) func NewDownloadCommand(f client.Factory) *cobra.Command { config, err := client.LoadConfig() if err != nil { fmt.Fprintf(os.Stderr, "WARNING: Error reading config file: %v\n", err) } o := NewDownloadOptions() o.caCertFile = config.CACertFile() c := &cobra.Command{ Use: "download NAME", Short: "Download all Kubernetes manifests for a backup", Long: "Download all Kubernetes manifests for a backup. Contents of persistent volume snapshots are not included.", Args: cobra.ExactArgs(1), Run: func(c *cobra.Command, args []string) { cmd.CheckError(o.Complete(args)) cmd.CheckError(o.Validate(c, args, f)) cmd.CheckError(o.Run(c, f)) }, } o.BindFlags(c.Flags()) return c } type DownloadOptions struct { Name string Output string Force bool Timeout time.Duration InsecureSkipTLSVerify bool writeOptions int caCertFile string } func NewDownloadOptions() *DownloadOptions { return &DownloadOptions{ Timeout: time.Minute, } } func (o *DownloadOptions) BindFlags(flags *pflag.FlagSet) { flags.StringVarP(&o.Output, "output", "o", o.Output, "Path to output file. Defaults to -data.tar.gz in the current directory.") flags.BoolVar(&o.Force, "force", o.Force, "Forces the download and will overwrite file if it exists already.") flags.DurationVar(&o.Timeout, "timeout", o.Timeout, "Maximum time to wait to process download request.") flags.BoolVar(&o.InsecureSkipTLSVerify, "insecure-skip-tls-verify", o.InsecureSkipTLSVerify, "If true, the object store's TLS certificate will not be checked for validity. This is insecure and susceptible to man-in-the-middle attacks. Not recommended for production.") flags.StringVar(&o.caCertFile, "cacert", o.caCertFile, "Path to a certificate bundle to use when verifying TLS connections. If not specified, the CA certificate from the BackupStorageLocation will be used if available.") } func (o *DownloadOptions) Validate(c *cobra.Command, args []string, f client.Factory) error { kbClient, err := f.KubebuilderClient() cmd.CheckError(err) backup := new(velerov1api.Backup) if err := kbClient.Get(context.Background(), controllerclient.ObjectKey{Namespace: f.Namespace(), Name: o.Name}, backup); err != nil { return err } return nil } func (o *DownloadOptions) Complete(args []string) error { o.Name = args[0] o.writeOptions = os.O_RDWR | os.O_CREATE | os.O_EXCL if o.Force { o.writeOptions = os.O_RDWR | os.O_CREATE | os.O_TRUNC } if o.Output == "" { path, err := os.Getwd() if err != nil { return errors.Wrapf(err, "error getting current directory") } o.Output = filepath.Join(path, fmt.Sprintf("%s-data.tar.gz", o.Name)) } return nil } func (o *DownloadOptions) Run(c *cobra.Command, f client.Factory) error { kbClient, err := f.KubebuilderClient() cmd.CheckError(err) // Get the backup to fetch BSL cacert backup := new(velerov1api.Backup) if err := kbClient.Get(context.Background(), controllerclient.ObjectKey{Namespace: f.Namespace(), Name: o.Name}, backup); err != nil { return err } // Get BSL cacert if available bslCACert, err := cacert.GetCACertFromBackup(context.Background(), kbClient, f.Namespace(), backup) if err != nil { // Log the error but don't fail - we can still try to download without the BSL cacert fmt.Fprintf(os.Stderr, "WARNING: Error getting cacert from BSL: %v\n", err) bslCACert = "" } backupDest, err := os.OpenFile(o.Output, o.writeOptions, 0600) if err != nil { return err } defer backupDest.Close() err = downloadrequest.StreamWithBSLCACert(context.Background(), kbClient, f.Namespace(), o.Name, velerov1api.DownloadTargetKindBackupContents, backupDest, o.Timeout, o.InsecureSkipTLSVerify, o.caCertFile, bslCACert) if err != nil { os.Remove(o.Output) cmd.CheckError(err) } fmt.Printf("Backup %s has been successfully downloaded to %s\n", o.Name, backupDest.Name()) return nil } ================================================ FILE: pkg/cmd/cli/backup/download_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backup import ( "fmt" "os" "os/exec" "strconv" "testing" flag "github.com/spf13/pflag" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/vmware-tanzu/velero/pkg/builder" factorymocks "github.com/vmware-tanzu/velero/pkg/client/mocks" cmdtest "github.com/vmware-tanzu/velero/pkg/cmd/test" velerotest "github.com/vmware-tanzu/velero/pkg/test" veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" ) func TestNewDownloadCommand(t *testing.T) { // create a factory f := &factorymocks.Factory{} backupName := "backup-1" kbclient := velerotest.NewFakeControllerRuntimeClient(t) err := kbclient.Create(t.Context(), builder.ForBackup(cmdtest.VeleroNameSpace, backupName).Result()) require.NoError(t, err) err = kbclient.Create(t.Context(), builder.ForBackup(cmdtest.VeleroNameSpace, "bk-to-be-download").Result()) require.NoError(t, err) f.On("Namespace").Return(cmdtest.VeleroNameSpace) f.On("KubebuilderClient").Return(kbclient, nil) // create command c := NewDownloadCommand(f) c.SetArgs([]string{"bk-to-be-download"}) assert.Equal(t, "Download all Kubernetes manifests for a backup", c.Short) // create a DownloadOptions with full options set and then run this backup command output := "path/to/download/bk.json" force := true timeout := "1m30s" insecureSkipTlsVerify := false cacert := "secret=YHJKKS" flags := new(flag.FlagSet) o := NewDownloadOptions() o.BindFlags(flags) flags.Parse([]string{"--output", output}) flags.Parse([]string{"--force"}) flags.Parse([]string{"--timeout", timeout}) flags.Parse([]string{fmt.Sprintf("--insecure-skip-tls-verify=%s", strconv.FormatBool(insecureSkipTlsVerify))}) flags.Parse([]string{"--cacert", cacert}) args := []string{backupName, "arg2"} e := o.Complete(args) require.NoError(t, e) e = o.Validate(c, args, f) require.NoError(t, e) // verify all options are set as expected assert.Equal(t, output, o.Output) assert.Equal(t, force, o.Force) assert.Equal(t, timeout, o.Timeout.String()) assert.Equal(t, insecureSkipTlsVerify, o.InsecureSkipTLSVerify) assert.Equal(t, cacert, o.caCertFile) if os.Getenv(cmdtest.CaptureFlag) == "1" { e = c.Execute() defer os.Remove("bk-to-be-download-data.tar.gz") assert.NoError(t, e) return } cmd := exec.CommandContext(t.Context(), os.Args[0], []string{"-test.run=TestNewDownloadCommand"}...) cmd.Env = append(os.Environ(), fmt.Sprintf("%s=1", cmdtest.CaptureFlag)) _, stderr, err := veleroexec.RunCommand(cmd) if err != nil { require.Contains(t, stderr, "download request download url timeout") return } t.Fatalf("process ran with err %v, want backup delete successfully", err) } ================================================ FILE: pkg/cmd/cli/backup/get.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backup import ( "context" "github.com/spf13/cobra" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" kbclient "sigs.k8s.io/controller-runtime/pkg/client" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" "github.com/vmware-tanzu/velero/pkg/cmd/util/output" ) func NewGetCommand(f client.Factory, use string) *cobra.Command { var listOptions metav1.ListOptions c := &cobra.Command{ Use: use, Short: "Get backups", Run: func(c *cobra.Command, args []string) { err := output.ValidateFlags(c) cmd.CheckError(err) kbClient, err := f.KubebuilderClient() cmd.CheckError(err) backups := new(api.BackupList) if len(args) > 0 { for _, name := range args { backup := new(api.Backup) err := kbClient.Get(context.TODO(), kbclient.ObjectKey{Namespace: f.Namespace(), Name: name}, backup) cmd.CheckError(err) backups.Items = append(backups.Items, *backup) } } else { parsedSelector, err := labels.Parse(listOptions.LabelSelector) cmd.CheckError(err) err = kbClient.List(context.TODO(), backups, &kbclient.ListOptions{ LabelSelector: parsedSelector, Namespace: f.Namespace(), }) cmd.CheckError(err) } _, err = output.PrintWithFormat(c, backups) cmd.CheckError(err) }, } c.Flags().StringVarP(&listOptions.LabelSelector, "selector", "l", listOptions.LabelSelector, "Only show items matching this label selector") output.BindFlags(c.Flags()) return c } ================================================ FILE: pkg/cmd/cli/backup/get_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backup import ( "fmt" "os" "os/exec" "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" kbclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/pkg/builder" factorymocks "github.com/vmware-tanzu/velero/pkg/client/mocks" cmdtest "github.com/vmware-tanzu/velero/pkg/cmd/test" velerotest "github.com/vmware-tanzu/velero/pkg/test" veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" ) func TestNewGetCommand(t *testing.T) { args := []string{"b1", "b2", "b3"} // create a factory f := &factorymocks.Factory{} client := velerotest.NewFakeControllerRuntimeClient(t) for _, backupName := range args { backup := builder.ForBackup(cmdtest.VeleroNameSpace, backupName).ObjectMeta(builder.WithLabels("abc", "abc")).Result() err := client.Create(t.Context(), backup, &kbclient.CreateOptions{}) require.NoError(t, err) } f.On("KubebuilderClient").Return(client, nil) f.On("Namespace").Return(cmdtest.VeleroNameSpace) // create command c := NewGetCommand(f, "velero backup get") assert.Equal(t, "Get backups", c.Short) c.SetArgs(args) e := c.Execute() require.NoError(t, e) if os.Getenv(cmdtest.CaptureFlag) == "1" { return } cmd := exec.CommandContext(t.Context(), os.Args[0], []string{"-test.run=TestNewGetCommand"}...) cmd.Env = append(os.Environ(), fmt.Sprintf("%s=1", cmdtest.CaptureFlag)) stdout, _, err := veleroexec.RunCommand(cmd) require.NoError(t, err) if err == nil { output := strings.Split(stdout, "\n") i := 0 for _, line := range output { if strings.Contains(line, "New") { i++ } } assert.Len(t, args, i) } d := NewGetCommand(f, "velero backup get") c.SetArgs([]string{"-l", "abc=abc"}) e = d.Execute() require.NoError(t, e) cmd = exec.CommandContext(t.Context(), os.Args[0], []string{"-test.run=TestNewGetCommand"}...) cmd.Env = append(os.Environ(), fmt.Sprintf("%s=1", cmdtest.CaptureFlag)) stdout, _, err = veleroexec.RunCommand(cmd) require.NoError(t, err) if err == nil { output := strings.Split(stdout, "\n") i := 0 for _, line := range output { if strings.Contains(line, "New") { i++ } } assert.Len(t, args, i) } } ================================================ FILE: pkg/cmd/cli/backup/logs.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backup import ( "context" "fmt" "os" "time" "github.com/spf13/cobra" "github.com/spf13/pflag" apierrors "k8s.io/apimachinery/pkg/api/errors" kbclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" "github.com/vmware-tanzu/velero/pkg/cmd/util/cacert" "github.com/vmware-tanzu/velero/pkg/cmd/util/downloadrequest" ) type LogsOptions struct { Timeout time.Duration InsecureSkipTLSVerify bool CaCertFile string Client kbclient.Client BackupName string } func NewLogsOptions() LogsOptions { config, err := client.LoadConfig() if err != nil { fmt.Fprintf(os.Stderr, "WARNING: Error reading config file: %v\n", err) } return LogsOptions{ Timeout: time.Minute, InsecureSkipTLSVerify: false, CaCertFile: config.CACertFile(), } } func (l *LogsOptions) BindFlags(flags *pflag.FlagSet) { flags.DurationVar(&l.Timeout, "timeout", l.Timeout, "How long to wait to receive logs.") flags.BoolVar(&l.InsecureSkipTLSVerify, "insecure-skip-tls-verify", l.InsecureSkipTLSVerify, "If true, the object store's TLS certificate will not be checked for validity. This is insecure and susceptible to man-in-the-middle attacks. Not recommended for production.") flags.StringVar(&l.CaCertFile, "cacert", l.CaCertFile, "Path to a certificate bundle to use when verifying TLS connections.") } func (l *LogsOptions) Run(c *cobra.Command, f client.Factory) error { backup := new(velerov1api.Backup) err := l.Client.Get(context.Background(), kbclient.ObjectKey{Namespace: f.Namespace(), Name: l.BackupName}, backup) if apierrors.IsNotFound(err) { return fmt.Errorf("backup %q does not exist", l.BackupName) } else if err != nil { return fmt.Errorf("error checking for backup %q: %v", l.BackupName, err) } switch backup.Status.Phase { case velerov1api.BackupPhaseCompleted, velerov1api.BackupPhasePartiallyFailed, velerov1api.BackupPhaseFailed, velerov1api.BackupPhaseWaitingForPluginOperations, velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed: // terminal and waiting for plugin operations phases, do nothing. default: return fmt.Errorf("logs for backup %q are not available until it's finished processing, please wait "+ "until the backup has a phase of Completed or Failed and try again", l.BackupName) } // Get BSL cacert if available bslCACert, err := cacert.GetCACertFromBackup(context.Background(), l.Client, f.Namespace(), backup) if err != nil { // Log the error but don't fail - we can still try to download without the BSL cacert fmt.Fprintf(os.Stderr, "WARNING: Error getting cacert from BSL: %v\n", err) bslCACert = "" } err = downloadrequest.StreamWithBSLCACert(context.Background(), l.Client, f.Namespace(), l.BackupName, velerov1api.DownloadTargetKindBackupLog, os.Stdout, l.Timeout, l.InsecureSkipTLSVerify, l.CaCertFile, bslCACert) return err } func (l *LogsOptions) Complete(args []string, f client.Factory) error { if len(args) > 0 { l.BackupName = args[0] } kbClient, err := f.KubebuilderClient() if err != nil { return err } l.Client = kbClient return nil } func NewLogsCommand(f client.Factory) *cobra.Command { l := NewLogsOptions() c := &cobra.Command{ Use: "logs BACKUP", Short: "Get backup logs", Args: cobra.ExactArgs(1), Run: func(c *cobra.Command, args []string) { err := l.Complete(args, f) cmd.CheckError(err) err = l.Run(c, f) cmd.CheckError(err) }, } l.BindFlags(c.Flags()) return c } ================================================ FILE: pkg/cmd/cli/backup/logs_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backup import ( "bytes" "compress/gzip" "fmt" "io" "net/http" "net/http/httptest" "os" "strconv" "testing" "time" flag "github.com/spf13/pflag" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" kbclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" factorymocks "github.com/vmware-tanzu/velero/pkg/client/mocks" cmdtest "github.com/vmware-tanzu/velero/pkg/cmd/test" "github.com/vmware-tanzu/velero/pkg/cmd/util/cacert" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func TestNewLogsCommand(t *testing.T) { t.Run("Flag test", func(t *testing.T) { l := NewLogsOptions() flags := new(flag.FlagSet) l.BindFlags(flags) timeout := "1m0s" insecureSkipTLSVerify := "true" caCertFile := "testing" flags.Parse([]string{"--timeout", timeout}) flags.Parse([]string{"--insecure-skip-tls-verify", insecureSkipTLSVerify}) flags.Parse([]string{"--cacert", caCertFile}) require.Equal(t, timeout, l.Timeout.String()) require.Equal(t, insecureSkipTLSVerify, strconv.FormatBool(l.InsecureSkipTLSVerify)) require.Equal(t, caCertFile, l.CaCertFile) }) t.Run("Backup not complete test", func(t *testing.T) { backupName := "bk-logs-1" // create a factory f := &factorymocks.Factory{} kbClient := velerotest.NewFakeControllerRuntimeClient(t) backup := builder.ForBackup(cmdtest.VeleroNameSpace, backupName).Result() err := kbClient.Create(t.Context(), backup, &kbclient.CreateOptions{}) require.NoError(t, err) f.On("Namespace").Return(cmdtest.VeleroNameSpace) f.On("KubebuilderClient").Return(kbClient, nil) c := NewLogsCommand(f) assert.Equal(t, "Get backup logs", c.Short) l := NewLogsOptions() flags := new(flag.FlagSet) l.BindFlags(flags) err = l.Complete([]string{backupName}, f) require.NoError(t, err) err = l.Run(c, f) require.Error(t, err) require.ErrorContains(t, err, fmt.Sprintf("logs for backup \"%s\" are not available until it's finished processing", backupName)) }) t.Run("Backup not exist test", func(t *testing.T) { backupName := "not-exist" // create a factory f := &factorymocks.Factory{} kbClient := velerotest.NewFakeControllerRuntimeClient(t) f.On("Namespace").Return(cmdtest.VeleroNameSpace) f.On("KubebuilderClient").Return(kbClient, nil) c := NewLogsCommand(f) assert.Equal(t, "Get backup logs", c.Short) l := NewLogsOptions() flags := new(flag.FlagSet) l.BindFlags(flags) err := l.Complete([]string{backupName}, f) require.NoError(t, err) err = l.Run(c, f) require.Error(t, err) require.Equal(t, fmt.Sprintf("backup \"%s\" does not exist", backupName), err.Error()) c.Execute() }) t.Run("Normal backup log test", func(t *testing.T) { backupName := "bk-logs-1" // create a factory f := &factorymocks.Factory{} kbClient := velerotest.NewFakeControllerRuntimeClient(t) backup := builder.ForBackup(cmdtest.VeleroNameSpace, backupName).Phase(velerov1api.BackupPhaseCompleted).Result() err := kbClient.Create(t.Context(), backup, &kbclient.CreateOptions{}) require.NoError(t, err) f.On("Namespace").Return(cmdtest.VeleroNameSpace) f.On("KubebuilderClient").Return(kbClient, nil) c := NewLogsCommand(f) assert.Equal(t, "Get backup logs", c.Short) l := NewLogsOptions() flags := new(flag.FlagSet) l.BindFlags(flags) err = l.Complete([]string{backupName}, f) require.NoError(t, err) timeout := time.After(3 * time.Second) done := make(chan bool) go func() { err = l.Run(c, f) require.Error(t, err) }() select { case <-timeout: t.Skip("Test didn't finish in time, because BSL is not in Available state.") case <-done: } }) t.Run("Invalid client test", func(t *testing.T) { // create a factory f := &factorymocks.Factory{} kbClient := velerotest.NewFakeControllerRuntimeClient(t) f.On("Namespace").Return(cmdtest.VeleroNameSpace) c := NewLogsCommand(f) assert.Equal(t, "Get backup logs", c.Short) l := NewLogsOptions() flags := new(flag.FlagSet) l.BindFlags(flags) f.On("KubebuilderClient").Return(kbClient, fmt.Errorf("test error")) err := l.Complete([]string{""}, f) require.Equal(t, "test error", err.Error()) }) t.Run("Backup with BSL cacert test", func(t *testing.T) { backupName := "bk-logs-with-cacert" bslName := "test-bsl" expectedCACert := "test-cacert-content" expectedLogContent := "test backup log content" // create a factory f := &factorymocks.Factory{} kbClient := velerotest.NewFakeControllerRuntimeClient(t) // Create BSL with cacert bsl := builder.ForBackupStorageLocation(cmdtest.VeleroNameSpace, bslName). Provider("aws"). Bucket("test-bucket"). CACert([]byte(expectedCACert)). Result() err := kbClient.Create(t.Context(), bsl, &kbclient.CreateOptions{}) require.NoError(t, err) // Create backup referencing the BSL backup := builder.ForBackup(cmdtest.VeleroNameSpace, backupName). Phase(velerov1api.BackupPhaseCompleted). StorageLocation(bslName). Result() err = kbClient.Create(t.Context(), backup, &kbclient.CreateOptions{}) require.NoError(t, err) f.On("Namespace").Return(cmdtest.VeleroNameSpace) f.On("KubebuilderClient").Return(kbClient, nil) c := NewLogsCommand(f) assert.Equal(t, "Get backup logs", c.Short) l := NewLogsOptions() flags := new(flag.FlagSet) l.BindFlags(flags) err = l.Complete([]string{backupName}, f) require.NoError(t, err) // Verify that the BSL cacert can be fetched correctly before running the command fetchedBackup := &velerov1api.Backup{} err = kbClient.Get(t.Context(), kbclient.ObjectKey{Namespace: cmdtest.VeleroNameSpace, Name: backupName}, fetchedBackup) require.NoError(t, err) // Test the cacert fetching logic directly cacertValue, err := cacert.GetCACertFromBackup(t.Context(), kbClient, cmdtest.VeleroNameSpace, fetchedBackup) require.NoError(t, err) assert.Equal(t, expectedCACert, cacertValue, "BSL cacert should be retrieved correctly") // Create a mock HTTP server to serve the log content mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // For logs, we need to gzip the content gzipWriter := gzip.NewWriter(w) defer gzipWriter.Close() gzipWriter.Write([]byte(expectedLogContent)) })) defer mockServer.Close() // Mock the download request controller by updating DownloadRequests go func() { time.Sleep(50 * time.Millisecond) // Wait a bit for the request to be created // List all DownloadRequests downloadRequestList := &velerov1api.DownloadRequestList{} if err := kbClient.List(t.Context(), downloadRequestList, &kbclient.ListOptions{ Namespace: cmdtest.VeleroNameSpace, }); err == nil { // Update each download request with the mock server URL for _, dr := range downloadRequestList.Items { if dr.Spec.Target.Kind == velerov1api.DownloadTargetKindBackupLog && dr.Spec.Target.Name == backupName { dr.Status.DownloadURL = mockServer.URL dr.Status.Phase = velerov1api.DownloadRequestPhaseProcessed kbClient.Update(t.Context(), &dr) } } } }() // Capture the output var logOutput bytes.Buffer // Temporarily redirect stdout oldStdout := os.Stdout r, w, _ := os.Pipe() os.Stdout = w // Run the logs command - it should now succeed l.Timeout = 5 * time.Second err = l.Run(c, f) // Restore stdout and read the output w.Close() os.Stdout = oldStdout io.Copy(&logOutput, r) // Verify the command succeeded and output is correct require.NoError(t, err) assert.Equal(t, expectedLogContent, logOutput.String()) }) } func TestBSLCACertBehavior(t *testing.T) { t.Run("Backup with BSL without cacert test", func(t *testing.T) { backupName := "bk-logs-without-cacert" bslName := "test-bsl-no-cacert" // create a factory f := &factorymocks.Factory{} kbClient := velerotest.NewFakeControllerRuntimeClient(t) // Create BSL without cacert bsl := builder.ForBackupStorageLocation(cmdtest.VeleroNameSpace, bslName). Provider("aws"). Bucket("test-bucket"). // No CACert() call - BSL will have no cacert Result() err := kbClient.Create(t.Context(), bsl, &kbclient.CreateOptions{}) require.NoError(t, err) // Create backup referencing the BSL backup := builder.ForBackup(cmdtest.VeleroNameSpace, backupName). Phase(velerov1api.BackupPhaseCompleted). StorageLocation(bslName). Result() err = kbClient.Create(t.Context(), backup, &kbclient.CreateOptions{}) require.NoError(t, err) f.On("Namespace").Return(cmdtest.VeleroNameSpace) f.On("KubebuilderClient").Return(kbClient, nil) c := NewLogsCommand(f) assert.Equal(t, "Get backup logs", c.Short) l := NewLogsOptions() flags := new(flag.FlagSet) l.BindFlags(flags) err = l.Complete([]string{backupName}, f) require.NoError(t, err) // Verify that the BSL cacert returns empty string when not present fetchedBackup := &velerov1api.Backup{} err = kbClient.Get(t.Context(), kbclient.ObjectKey{Namespace: cmdtest.VeleroNameSpace, Name: backupName}, fetchedBackup) require.NoError(t, err) // Test the cacert fetching logic directly cacertValue, err := cacert.GetCACertFromBackup(t.Context(), kbClient, cmdtest.VeleroNameSpace, fetchedBackup) require.NoError(t, err) assert.Empty(t, cacertValue, "BSL cacert should be empty when not configured") // The command should still work without cacert l.Timeout = 100 * time.Millisecond err = l.Run(c, f) require.Error(t, err) // The error should be about download request timeout, not about cacert fetching assert.Contains(t, err.Error(), "download") }) t.Run("Backup with nonexistent BSL test", func(t *testing.T) { backupName := "bk-logs-with-missing-bsl" bslName := "nonexistent-bsl" // create a factory f := &factorymocks.Factory{} kbClient := velerotest.NewFakeControllerRuntimeClient(t) // Create backup referencing a BSL that doesn't exist backup := builder.ForBackup(cmdtest.VeleroNameSpace, backupName). Phase(velerov1api.BackupPhaseCompleted). StorageLocation(bslName). Result() err := kbClient.Create(t.Context(), backup, &kbclient.CreateOptions{}) require.NoError(t, err) f.On("Namespace").Return(cmdtest.VeleroNameSpace) f.On("KubebuilderClient").Return(kbClient, nil) c := NewLogsCommand(f) l := NewLogsOptions() flags := new(flag.FlagSet) l.BindFlags(flags) err = l.Complete([]string{backupName}, f) require.NoError(t, err) // Verify that the BSL cacert returns empty string when BSL doesn't exist fetchedBackup := &velerov1api.Backup{} err = kbClient.Get(t.Context(), kbclient.ObjectKey{Namespace: cmdtest.VeleroNameSpace, Name: backupName}, fetchedBackup) require.NoError(t, err) // Test the cacert fetching logic directly - should not error when BSL is missing cacertValue, err := cacert.GetCACertFromBackup(t.Context(), kbClient, cmdtest.VeleroNameSpace, fetchedBackup) require.NoError(t, err) assert.Empty(t, cacertValue, "BSL cacert should be empty when BSL doesn't exist") // The command should still try to run even without BSL l.Timeout = 100 * time.Millisecond err = l.Run(c, f) require.Error(t, err) assert.Contains(t, err.Error(), "download") }) } ================================================ FILE: pkg/cmd/cli/backuplocation/backup_location.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backuplocation import ( "github.com/spf13/cobra" "github.com/vmware-tanzu/velero/pkg/client" ) func NewCommand(f client.Factory) *cobra.Command { c := &cobra.Command{ Use: "backup-location", Short: "Work with backup storage locations", Long: "Work with backup storage locations", } c.AddCommand( NewCreateCommand(f, "create"), NewDeleteCommand(f, "delete"), NewGetCommand(f, "get"), NewSetCommand(f, "set"), ) return c } ================================================ FILE: pkg/cmd/cli/backuplocation/backup_location_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backuplocation import ( "testing" "github.com/stretchr/testify/assert" factorymocks "github.com/vmware-tanzu/velero/pkg/client/mocks" ) func TestNewCommand(t *testing.T) { // create a factory f := &factorymocks.Factory{} // create command c := NewCommand(f) assert.Equal(t, "Work with backup storage locations", c.Short) } ================================================ FILE: pkg/cmd/cli/backuplocation/create.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backuplocation import ( "context" "fmt" "os" "path/filepath" "strings" "time" "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kbclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/internal/storage" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" "github.com/vmware-tanzu/velero/pkg/cmd/util/flag" "github.com/vmware-tanzu/velero/pkg/cmd/util/output" ) func NewCreateCommand(f client.Factory, use string) *cobra.Command { o := NewCreateOptions() c := &cobra.Command{ Use: use + " NAME", Short: "Create a backup storage location", Args: cobra.ExactArgs(1), Run: func(c *cobra.Command, args []string) { cmd.CheckError(o.Complete(args, f)) cmd.CheckError(o.Validate(c, args, f)) cmd.CheckError(o.Run(c, f)) }, } o.BindFlags(c.Flags()) output.BindFlags(c.Flags()) output.ClearOutputFlagDefault(c) return c } type CreateOptions struct { Name string Provider string Bucket string Credential flag.Map DefaultBackupStorageLocation bool Prefix string BackupSyncPeriod, ValidationFrequency time.Duration Config flag.Map Labels flag.Map CACertFile string AccessMode *flag.Enum } func NewCreateOptions() *CreateOptions { return &CreateOptions{ Credential: flag.NewMap(), Config: flag.NewMap(), Labels: flag.NewMap(), AccessMode: flag.NewEnum( string(velerov1api.BackupStorageLocationAccessModeReadWrite), string(velerov1api.BackupStorageLocationAccessModeReadWrite), string(velerov1api.BackupStorageLocationAccessModeReadOnly), ), } } func (o *CreateOptions) BindFlags(flags *pflag.FlagSet) { flags.StringVar(&o.Provider, "provider", o.Provider, "Name of the backup storage provider (e.g. aws, azure, gcp).") flags.StringVar(&o.Bucket, "bucket", o.Bucket, "Name of the object storage bucket where backups should be stored.") flags.Var(&o.Credential, "credential", "The credential to be used by this location as a key-value pair, where the key is the Kubernetes Secret name, and the value is the data key name within the Secret. Optional, one value only.") flags.BoolVar(&o.DefaultBackupStorageLocation, "default", o.DefaultBackupStorageLocation, "Sets this new location to be the new default backup storage location. Optional.") flags.StringVar(&o.Prefix, "prefix", o.Prefix, "Prefix under which all Velero data should be stored within the bucket. Optional.") flags.DurationVar(&o.BackupSyncPeriod, "backup-sync-period", o.BackupSyncPeriod, "How often to ensure all Velero backups in object storage exist as Backup API objects in the cluster. Optional. Set this to `0s` to disable sync. Default: 1 minute.") flags.DurationVar(&o.ValidationFrequency, "validation-frequency", o.ValidationFrequency, "How often to verify if the backup storage location is valid. Optional. Set this to `0s` to disable sync. Default 1 minute.") flags.Var(&o.Config, "config", "Configuration key-value pairs.") flags.Var(&o.Labels, "labels", "Labels to apply to the backup storage location.") flags.StringVar(&o.CACertFile, "cacert", o.CACertFile, "File containing a certificate bundle to use when verifying TLS connections to the object store. Optional.") flags.Var( o.AccessMode, "access-mode", fmt.Sprintf("Access mode for the backup storage location. Valid values are %s", strings.Join(o.AccessMode.AllowedValues(), ",")), ) } func (o *CreateOptions) Validate(c *cobra.Command, args []string, f client.Factory) error { if err := output.ValidateFlags(c); err != nil { return err } if o.Provider == "" { return errors.New("--provider is required") } if o.Bucket == "" { return errors.New("--bucket is required") } if o.BackupSyncPeriod < 0 { return errors.New("--backup-sync-period must be non-negative") } if len(o.Credential.Data()) > 1 { return errors.New("--credential can only contain 1 key/value pair") } return nil } func (o *CreateOptions) Complete(args []string, f client.Factory) error { o.Name = args[0] return nil } func (o *CreateOptions) BuildBackupStorageLocation(namespace string, setBackupSyncPeriod, setValidationFrequency bool) (*velerov1api.BackupStorageLocation, error) { var caCertData []byte if o.CACertFile != "" { realPath, err := filepath.Abs(o.CACertFile) if err != nil { return nil, err } caCertData, err = os.ReadFile(realPath) if err != nil { return nil, err } } backupStorageLocation := &velerov1api.BackupStorageLocation{ ObjectMeta: metav1.ObjectMeta{ Namespace: namespace, Name: o.Name, Labels: o.Labels.Data(), }, Spec: velerov1api.BackupStorageLocationSpec{ Provider: o.Provider, StorageType: velerov1api.StorageType{ ObjectStorage: &velerov1api.ObjectStorageLocation{ Bucket: o.Bucket, Prefix: o.Prefix, CACert: caCertData, }, }, Config: o.Config.Data(), Default: o.DefaultBackupStorageLocation, AccessMode: velerov1api.BackupStorageLocationAccessMode(o.AccessMode.String()), }, } if setBackupSyncPeriod { backupStorageLocation.Spec.BackupSyncPeriod = &metav1.Duration{Duration: o.BackupSyncPeriod} } if setValidationFrequency { backupStorageLocation.Spec.ValidationFrequency = &metav1.Duration{Duration: o.ValidationFrequency} } for secretName, secretKey := range o.Credential.Data() { backupStorageLocation.Spec.Credential = builder.ForSecretKeySelector(secretName, secretKey).Result() break } return backupStorageLocation, nil } func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error { setBackupSyncPeriod := c.Flags().Changed("backup-sync-period") setValidationFrequency := c.Flags().Changed("validation-frequency") backupStorageLocation, err := o.BuildBackupStorageLocation(f.Namespace(), setBackupSyncPeriod, setValidationFrequency) if err != nil { return err } if printed, err := output.PrintWithFormat(c, backupStorageLocation); printed || err != nil { return err } kbClient, err := f.KubebuilderClient() if err != nil { return err } if o.DefaultBackupStorageLocation { // There is one and only one default backup storage location. // Disable any existing default backup storage location first. defalutBSLs, err := storage.GetDefaultBackupStorageLocations(context.Background(), kbClient, f.Namespace()) if err != nil { return errors.WithStack(err) } if len(defalutBSLs.Items) > 0 { return errors.New("there is already exist default backup storage location, please unset it first or do not set --default flag") } } if err := kbClient.Create(context.Background(), backupStorageLocation, &kbclient.CreateOptions{}); err != nil { return errors.WithStack(err) } fmt.Printf("Backup storage location %q configured successfully.\n", backupStorageLocation.Name) return nil } ================================================ FILE: pkg/cmd/cli/backuplocation/create_test.go ================================================ /* Copyright the Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backuplocation import ( "fmt" "reflect" "strings" "testing" "time" flag "github.com/spf13/pflag" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" factorymocks "github.com/vmware-tanzu/velero/pkg/client/mocks" veleroflag "github.com/vmware-tanzu/velero/pkg/cmd/util/flag" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func TestBuildBackupStorageLocationSetsNamespace(t *testing.T) { o := NewCreateOptions() bsl, err := o.BuildBackupStorageLocation("velero-test-ns", false, false) require.NoError(t, err) assert.Equal(t, "velero-test-ns", bsl.Namespace) } func TestBuildBackupStorageLocationSetsSyncPeriod(t *testing.T) { o := NewCreateOptions() o.BackupSyncPeriod = 2 * time.Minute bsl, err := o.BuildBackupStorageLocation("velero-test-ns", false, false) require.NoError(t, err) assert.Nil(t, bsl.Spec.BackupSyncPeriod) bsl, err = o.BuildBackupStorageLocation("velero-test-ns", true, false) require.NoError(t, err) assert.Equal(t, &metav1.Duration{Duration: 2 * time.Minute}, bsl.Spec.BackupSyncPeriod) } func TestBuildBackupStorageLocationSetsValidationFrequency(t *testing.T) { o := NewCreateOptions() o.ValidationFrequency = 2 * time.Minute bsl, err := o.BuildBackupStorageLocation("velero-test-ns", false, false) require.NoError(t, err) assert.Nil(t, bsl.Spec.ValidationFrequency) bsl, err = o.BuildBackupStorageLocation("velero-test-ns", false, true) require.NoError(t, err) assert.Equal(t, &metav1.Duration{Duration: 2 * time.Minute}, bsl.Spec.ValidationFrequency) } func TestBuildBackupStorageLocationSetsCredential(t *testing.T) { o := NewCreateOptions() bsl, err := o.BuildBackupStorageLocation("velero-test-ns", false, false) require.NoError(t, err) assert.Nil(t, bsl.Spec.Credential) setErr := o.Credential.Set("my-secret=key-from-secret") require.NoError(t, setErr) bsl, err = o.BuildBackupStorageLocation("velero-test-ns", false, true) require.NoError(t, err) assert.Equal(t, &corev1api.SecretKeySelector{ LocalObjectReference: corev1api.LocalObjectReference{Name: "my-secret"}, Key: "key-from-secret", }, bsl.Spec.Credential) } func TestBuildBackupStorageLocationSetsLabels(t *testing.T) { o := NewCreateOptions() err := o.Labels.Set("key=value") require.NoError(t, err) bsl, err := o.BuildBackupStorageLocation("velero-test-ns", false, false) require.NoError(t, err) assert.Equal(t, map[string]string{"key": "value"}, bsl.Labels) } func TestCreateCommand_Run(t *testing.T) { // create a factory f := &factorymocks.Factory{} // create command c := NewCreateCommand(f, "") assert.Equal(t, "Create a backup storage location", c.Short) // create a CreateOptions with full options set and then run this backup command name := "bsl-name-to-be-created" provider := "aws" bucket := "velero123456" credential := veleroflag.NewMap() credential.Set("secret=a") defaultBackupStorageLocation := true prefix := "builds" backupSyncPeriod := "1m30s" validationFrequency := "128h1m6s" bslConfig := veleroflag.NewMap() bslConfigStr := "region=minio" bslConfig.Set(bslConfigStr) labels := "a=too,b=woo" caCertFile := "bsl-name-1" accessMode := "ReadWrite" flags := new(flag.FlagSet) o := NewCreateOptions() o.BindFlags(flags) flags.Parse([]string{"--provider", provider}) flags.Parse([]string{"--bucket", bucket}) flags.Parse([]string{"--credential", credential.String()}) flags.Parse([]string{"--default"}) flags.Parse([]string{"--prefix", prefix}) flags.Parse([]string{"--backup-sync-period", backupSyncPeriod}) flags.Parse([]string{"--validation-frequency", validationFrequency}) flags.Parse([]string{"--config", bslConfigStr}) flags.Parse([]string{"--labels", labels}) flags.Parse([]string{"--cacert", caCertFile}) flags.Parse([]string{"--access-mode", accessMode}) args := []string{name, "arg2"} kbclient := velerotest.NewFakeControllerRuntimeClient(t) f.On("Namespace").Return(mock.Anything) f.On("KubebuilderClient").Return(kbclient, nil) o.Complete(args, f) e := o.Validate(c, args, f) require.NoError(t, e) e = o.Run(c, f) require.ErrorContains(t, e, fmt.Sprintf("%s: no such file or directory", caCertFile)) // verify all options are set as expected assert.Equal(t, name, o.Name) assert.Equal(t, provider, o.Provider) assert.Equal(t, bucket, o.Bucket) assert.True(t, reflect.DeepEqual(credential, o.Credential)) assert.Equal(t, defaultBackupStorageLocation, o.DefaultBackupStorageLocation) assert.Equal(t, prefix, o.Prefix) assert.Equal(t, backupSyncPeriod, o.BackupSyncPeriod.String()) assert.Equal(t, validationFrequency, o.ValidationFrequency.String()) assert.True(t, reflect.DeepEqual(bslConfig, o.Config)) assert.True(t, velerotest.CompareSlice(strings.Split(labels, ","), strings.Split(o.Labels.String(), ","))) assert.Equal(t, caCertFile, o.CACertFile) assert.Equal(t, accessMode, o.AccessMode.String()) // create the other create command without fromSchedule option for Run() other branches c = NewCreateCommand(f, "velero backup-location create") assert.Equal(t, "Create a backup storage location", c.Short) o = NewCreateOptions() o.Labels.Set("velero.io/test=true") args = []string{"backup-name-2", "arg2"} o.Complete(args, f) e = o.Run(c, f) require.NoError(t, e) c.SetArgs([]string{"bsl-1", "--provider=aws", "--bucket=bk1", "--default"}) e = c.Execute() assert.NoError(t, e) } ================================================ FILE: pkg/cmd/cli/backuplocation/delete.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backuplocation import ( "context" "fmt" "github.com/pkg/errors" "github.com/spf13/cobra" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" kubeerrs "k8s.io/apimachinery/pkg/util/errors" kbclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" "github.com/vmware-tanzu/velero/pkg/cmd/cli" "github.com/vmware-tanzu/velero/pkg/cmd/util/confirm" ) // NewDeleteCommand creates and returns a new cobra command for deleting backup-locations. func NewDeleteCommand(f client.Factory, use string) *cobra.Command { o := cli.NewDeleteOptions("backup-location") c := &cobra.Command{ Use: fmt.Sprintf("%s [NAMES]", use), Short: "Delete backup storage locations", Example: ` # Delete a backup storage location named "backup-location-1". velero backup-location delete backup-location-1 # Delete a backup storage location named "backup-location-1" without prompting for confirmation. velero backup-location delete backup-location-1 --confirm # Delete backup storage locations named "backup-location-1" and "backup-location-2". velero backup-location delete backup-location-1 backup-location-2 # Delete all backup storage locations labeled with "foo=bar". velero backup-location delete --selector foo=bar # Delete all backup storage locations. velero backup-location delete --all`, Run: func(c *cobra.Command, args []string) { cmd.CheckError(o.Complete(f, args)) cmd.CheckError(o.Validate(c, f, args)) cmd.CheckError(Run(f, o)) }, } o.BindFlags(c.Flags()) return c } // Run performs the delete backup-location operation. func Run(f client.Factory, o *cli.DeleteOptions) error { if !o.Confirm && !confirm.GetConfirmation() { // Don't do anything unless we get confirmation return nil } kbClient, err := f.KubebuilderClient() cmd.CheckError(err) locations := new(velerov1api.BackupStorageLocationList) var errs []error switch { case len(o.Names) > 0: for _, name := range o.Names { location := &velerov1api.BackupStorageLocation{} err = kbClient.Get(context.Background(), kbclient.ObjectKey{ Namespace: f.Namespace(), Name: name, }, location) if err != nil { errs = append(errs, errors.WithStack(err)) continue } locations.Items = append(locations.Items, *location) } default: selector := labels.Everything().String() if o.Selector.LabelSelector != nil { selector = o.Selector.String() } err := kbClient.List(context.Background(), locations, &kbclient.ListOptions{ Namespace: f.Namespace(), Raw: &metav1.ListOptions{LabelSelector: selector}, }) if err != nil { return errors.WithStack(err) } } if len(locations.Items) == 0 { fmt.Println("No backup-locations found") return nil } // Create a backup-location deletion request for each for i, location := range locations.Items { if err := kbClient.Delete(context.Background(), &locations.Items[i], &kbclient.DeleteOptions{}); err != nil { errs = append(errs, errors.WithStack(err)) continue } fmt.Printf("Backup storage location %q deleted successfully.\n", location.Name) // Delete backups associated with the deleted BSL. backupList, err := findAssociatedBackups(kbClient, location.Name, f.Namespace()) if err != nil { errs = append(errs, fmt.Errorf("find backups associated with BSL %q: %w", location.Name, err)) } else if deleteErrs := deleteBackups(kbClient, backupList); deleteErrs != nil { errs = append(errs, deleteErrs...) } // Delete backup repositories associated with the deleted BSL. backupRepoList, err := findAssociatedBackupRepos(kbClient, location.Name, f.Namespace()) if err != nil { errs = append(errs, fmt.Errorf("find backup repositories associated with BSL %q: %w", location.Name, err)) } else if deleteErrs := deleteBackupRepos(kbClient, backupRepoList); deleteErrs != nil { errs = append(errs, deleteErrs...) } } return kubeerrs.NewAggregate(errs) } func findAssociatedBackups(client kbclient.Client, bslName, ns string) (velerov1api.BackupList, error) { var backups velerov1api.BackupList err := client.List(context.Background(), &backups, &kbclient.ListOptions{ Namespace: ns, Raw: &metav1.ListOptions{LabelSelector: velerov1api.StorageLocationLabel + "=" + bslName}, }) return backups, err } func findAssociatedBackupRepos(client kbclient.Client, bslName, ns string) (velerov1api.BackupRepositoryList, error) { var repos velerov1api.BackupRepositoryList err := client.List(context.Background(), &repos, &kbclient.ListOptions{ Namespace: ns, Raw: &metav1.ListOptions{LabelSelector: velerov1api.StorageLocationLabel + "=" + bslName}, }) return repos, err } func deleteBackups(client kbclient.Client, backups velerov1api.BackupList) []error { var errs []error for i, backup := range backups.Items { if err := client.Delete(context.Background(), &backups.Items[i], &kbclient.DeleteOptions{}); err != nil { errs = append(errs, errors.WithStack(fmt.Errorf("delete backup %q associated with deleted BSL: %w", backup.Name, err))) continue } fmt.Printf("Backup associated with deleted BSL(s) %q deleted successfully.\n", backup.Name) } return errs } func deleteBackupRepos(client kbclient.Client, repos velerov1api.BackupRepositoryList) []error { var errs []error for i, repo := range repos.Items { if err := client.Delete(context.Background(), &repos.Items[i], &kbclient.DeleteOptions{}); err != nil { errs = append(errs, errors.WithStack(fmt.Errorf("delete backup repository %q associated with deleted BSL: %w", repo.Name, err))) continue } fmt.Printf("Backup repository associated with deleted BSL(s) %q deleted successfully.\n", repo.Name) } return errs } ================================================ FILE: pkg/cmd/cli/backuplocation/delete_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backuplocation import ( "fmt" "os" "os/exec" "testing" flag "github.com/spf13/pflag" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" factorymocks "github.com/vmware-tanzu/velero/pkg/client/mocks" "github.com/vmware-tanzu/velero/pkg/cmd/cli" cmdtest "github.com/vmware-tanzu/velero/pkg/cmd/test" velerotest "github.com/vmware-tanzu/velero/pkg/test" veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" ) func TestNewDeleteCommand(t *testing.T) { // create a factory f := &factorymocks.Factory{} kbclient := velerotest.NewFakeControllerRuntimeClient(t) f.On("Namespace").Return(mock.Anything) f.On("KubebuilderClient").Return(kbclient, nil) // create command c := NewDeleteCommand(f, "velero backup-location delete") assert.Equal(t, "Delete backup storage locations", c.Short) o := cli.NewDeleteOptions("backup") flags := new(flag.FlagSet) o.BindFlags(flags) flags.Parse([]string{"--confirm"}) args := []string{"bk-loc-1", "bk-loc-2"} e := o.Complete(f, args) require.NoError(t, e) e = o.Validate(c, f, args) require.NoError(t, e) c.SetArgs([]string{"bk-1", "--confirm"}) e = c.Execute() require.NoError(t, e) e = Run(f, o) require.NoError(t, e) if os.Getenv(cmdtest.CaptureFlag) == "1" { return } cmd := exec.CommandContext(t.Context(), os.Args[0], []string{"-test.run=TestNewDeleteCommand"}...) cmd.Env = append(os.Environ(), fmt.Sprintf("%s=1", cmdtest.CaptureFlag)) stdout, _, err := veleroexec.RunCommand(cmd) if err == nil { assert.Contains(t, stdout, "No backup-locations found") return } t.Fatalf("process ran with err %v, want backups by get()", err) } func TestDeleteFunctions(t *testing.T) { //t.Run("create the other create command with fromSchedule option for Run() other branches", func(t *testing.T) { // create a factory f := &factorymocks.Factory{} kbclient := velerotest.NewFakeControllerRuntimeClient(t) f.On("Namespace").Return(mock.Anything) f.On("KubebuilderClient").Return(kbclient, nil) bkList := velerov1api.BackupList{} bkrepoList := velerov1api.BackupRepositoryList{} t.Run("findAssociatedBackups", func(t *testing.T) { bkList, e := findAssociatedBackups(kbclient, "bk-loc-1", "ns1") assert.Empty(t, bkList.Items) assert.NoError(t, e) }) t.Run("findAssociatedBackupRepos", func(t *testing.T) { bkrepoList, e := findAssociatedBackupRepos(kbclient, "bk-loc-1", "ns1") assert.Empty(t, bkrepoList.Items) assert.NoError(t, e) }) t.Run("deleteBackups", func(t *testing.T) { bk := velerov1api.Backup{} bk.Name = "bk-name-last" bkList.Items = append(bkList.Items, bk) errList := deleteBackups(kbclient, bkList) assert.Len(t, errList, 1) assert.ErrorContains(t, errList[0], fmt.Sprintf("delete backup \"%s\" associated with deleted BSL: backups.velero.io \"%s\" not found", bk.Name, bk.Name)) }) t.Run("deleteBackupRepos", func(t *testing.T) { bkrepo := velerov1api.BackupRepository{} bkrepo.Name = "bk-repo-name-last" bkrepoList.Items = append(bkrepoList.Items, bkrepo) errList := deleteBackupRepos(kbclient, bkrepoList) assert.Len(t, errList, 1) assert.ErrorContains(t, errList[0], fmt.Sprintf("delete backup repository \"%s\" associated with deleted BSL: backuprepositories.velero.io \"%s\" not found", bkrepo.Name, bkrepo.Name)) }) } ================================================ FILE: pkg/cmd/cli/backuplocation/get.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backuplocation import ( "context" "github.com/spf13/cobra" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kbclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" "github.com/vmware-tanzu/velero/pkg/cmd/util/output" ) func NewGetCommand(f client.Factory, use string) *cobra.Command { var listOptions metav1.ListOptions var showDefaultOnly bool c := &cobra.Command{ Use: use, Short: "Get backup storage locations", Run: func(c *cobra.Command, args []string) { err := output.ValidateFlags(c) cmd.CheckError(err) kbClient, err := f.KubebuilderClient() cmd.CheckError(err) locations := new(velerov1api.BackupStorageLocationList) if len(args) > 0 { for _, name := range args { location := &velerov1api.BackupStorageLocation{} err = kbClient.Get(context.Background(), kbclient.ObjectKey{ Namespace: f.Namespace(), Name: name, }, location) cmd.CheckError(err) if showDefaultOnly { if location.Spec.Default { locations.Items = append(locations.Items, *location) break } } else { locations.Items = append(locations.Items, *location) } } } else { err := kbClient.List(context.Background(), locations, &kbclient.ListOptions{ Namespace: f.Namespace(), Raw: &listOptions, }) cmd.CheckError(err) if showDefaultOnly { for i := 0; i < len(locations.Items); i++ { if locations.Items[i].Spec.Default { continue } if i != len(locations.Items)-1 { copy(locations.Items[i:], locations.Items[i+1:]) i = i - 1 } locations.Items = locations.Items[:len(locations.Items)-1] } } } _, err = output.PrintWithFormat(c, locations) cmd.CheckError(err) }, } c.Flags().BoolVar(&showDefaultOnly, "default", false, "Displays the current default backup storage location.") c.Flags().StringVarP(&listOptions.LabelSelector, "selector", "l", listOptions.LabelSelector, "Only show items matching this label selector.") output.BindFlags(c.Flags()) return c } ================================================ FILE: pkg/cmd/cli/backuplocation/get_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backuplocation import ( "fmt" "os" "os/exec" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" factorymocks "github.com/vmware-tanzu/velero/pkg/client/mocks" cmdtest "github.com/vmware-tanzu/velero/pkg/cmd/test" velerotest "github.com/vmware-tanzu/velero/pkg/test" veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" ) func TestNewGetCommand(t *testing.T) { bkList := []string{"b1", "b2"} f := &factorymocks.Factory{} kbclient := velerotest.NewFakeControllerRuntimeClient(t) f.On("Namespace").Return(mock.Anything) f.On("KubebuilderClient").Return(kbclient, nil) // get command c := NewGetCommand(f, "velero backup-location get") assert.Equal(t, "Get backup storage locations", c.Short) c.Execute() if os.Getenv(cmdtest.CaptureFlag) == "1" { c.SetArgs([]string{"b1", "b2", "--default"}) c.Execute() return } cmd := exec.CommandContext(t.Context(), os.Args[0], []string{"-test.run=TestNewGetCommand"}...) cmd.Env = append(os.Environ(), fmt.Sprintf("%s=1", cmdtest.CaptureFlag)) _, stderr, err := veleroexec.RunCommand(cmd) if err != nil { assert.Contains(t, stderr, fmt.Sprintf("backupstoragelocations.velero.io \"%s\" not found", bkList[0])) return } t.Fatalf("process ran with err %v, want backup delete successfully", err) } ================================================ FILE: pkg/cmd/cli/backuplocation/set.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backuplocation import ( "context" "fmt" "os" "path/filepath" "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" kbclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/internal/storage" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" "github.com/vmware-tanzu/velero/pkg/cmd/util/flag" "github.com/vmware-tanzu/velero/pkg/util/boolptr" ) func NewSetCommand(f client.Factory, use string) *cobra.Command { o := NewSetOptions() c := &cobra.Command{ Use: use + " NAME", Short: "Set specific features for a backup storage location", Args: cobra.ExactArgs(1), Run: func(c *cobra.Command, args []string) { cmd.CheckError(o.Complete(args, f)) cmd.CheckError(o.Validate(c, args, f)) cmd.CheckError(o.Run(c, f)) }, } o.BindFlags(c.Flags()) return c } type SetOptions struct { Name string CACertFile string Credential flag.Map DefaultBackupStorageLocation flag.OptionalBool } func NewSetOptions() *SetOptions { return &SetOptions{ Credential: flag.NewMap(), } } func (o *SetOptions) BindFlags(flags *pflag.FlagSet) { flags.StringVar(&o.CACertFile, "cacert", o.CACertFile, "File containing a certificate bundle to use when verifying TLS connections to the object store. Optional.") flags.Var(&o.Credential, "credential", "Sets the credential to be used by this location as a key-value pair, where the key is the Kubernetes Secret name, and the value is the data key name within the Secret. Optional, one value only.") f := flags.VarPF(&o.DefaultBackupStorageLocation, "default", "", "Sets this new location to be the new default backup storage location. Optional.") f.NoOptDefVal = cmd.TRUE } func (o *SetOptions) Validate(c *cobra.Command, args []string, f client.Factory) error { if len(o.Credential.Data()) > 1 { return errors.New("--credential can only contain 1 key/value pair") } return nil } func (o *SetOptions) Complete(args []string, f client.Factory) error { o.Name = args[0] return nil } func (o *SetOptions) Run(c *cobra.Command, f client.Factory) error { kbClient, err := f.KubebuilderClient() if err != nil { return err } var caCertData []byte if o.CACertFile != "" { realPath, err := filepath.Abs(o.CACertFile) if err != nil { return err } caCertData, err = os.ReadFile(realPath) if err != nil { return err } } location := &velerov1api.BackupStorageLocation{} err = kbClient.Get(context.Background(), kbclient.ObjectKey{ Namespace: f.Namespace(), Name: o.Name, }, location) if err != nil { return errors.WithStack(err) } defaultOpt := o.DefaultBackupStorageLocation.Value if defaultOpt != nil { if *defaultOpt { // set default backup storage location // need first check if there is already a default backup storage location defalutBSLs, err := storage.GetDefaultBackupStorageLocations(context.Background(), kbClient, f.Namespace()) if err != nil { return errors.WithStack(err) } if len(defalutBSLs.Items) > 0 { if len(defalutBSLs.Items) == 1 && defalutBSLs.Items[0].Name == o.Name { // the default backup storage location is the one we want to set // so we do not need to do anything fmt.Printf("Backup storage location %q is already the default backup storage location.\n", o.Name) return nil } return errors.New("there are already exist default backup storage locations, please unset them first") } } } else { // user do not specify default option // we should keep the original default option o.DefaultBackupStorageLocation = flag.OptionalBool{Value: &location.Spec.Default} } location.Spec.Default = boolptr.IsSetToTrue(o.DefaultBackupStorageLocation.Value) location.Spec.StorageType.ObjectStorage.CACert = caCertData for name, key := range o.Credential.Data() { location.Spec.Credential = builder.ForSecretKeySelector(name, key).Result() break } if err := kbClient.Update(context.Background(), location, &kbclient.UpdateOptions{}); err != nil { return errors.WithStack(err) } fmt.Printf("Backup storage location %q configured successfully.\n", o.Name) return nil } ================================================ FILE: pkg/cmd/cli/backuplocation/set_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backuplocation import ( "fmt" "os" "os/exec" "reflect" "testing" flag "github.com/spf13/pflag" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" factorymocks "github.com/vmware-tanzu/velero/pkg/client/mocks" cmdtest "github.com/vmware-tanzu/velero/pkg/cmd/test" veleroflag "github.com/vmware-tanzu/velero/pkg/cmd/util/flag" velerotest "github.com/vmware-tanzu/velero/pkg/test" "github.com/vmware-tanzu/velero/pkg/util/boolptr" veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" ) func TestNewSetCommand(t *testing.T) { backupName := "arg2" // create a config for factory f := &factorymocks.Factory{} kbclient := velerotest.NewFakeControllerRuntimeClient(t) f.On("Namespace").Return(mock.Anything) f.On("KubebuilderClient").Return(kbclient, nil) // create command c := NewSetCommand(f, "") assert.Equal(t, "Set specific features for a backup storage location", c.Short) // create a SetOptions with full options set and then run this backup command cacert := "a/b/c/ut-cert.ca" defaultBackupStorageLocation := true credential := veleroflag.NewMap() credential.Set("secret=a") flags := new(flag.FlagSet) o := NewSetOptions() o.BindFlags(flags) flags.Parse([]string{"--cacert", cacert}) flags.Parse([]string{"--credential", credential.String()}) flags.Parse([]string{"--default"}) args := []string{backupName} o.Complete(args, f) e := o.Validate(c, args, f) require.NoError(t, e) e = o.Run(c, f) require.ErrorContains(t, e, fmt.Sprintf("%s: no such file or directory", cacert)) // verify all options are set as expected assert.Equal(t, backupName, o.Name) assert.Equal(t, cacert, o.CACertFile) assert.Equal(t, defaultBackupStorageLocation, boolptr.IsSetToTrue(o.DefaultBackupStorageLocation.Value)) assert.True(t, reflect.DeepEqual(credential, o.Credential)) assert.ErrorContains(t, e, fmt.Sprintf("%s: no such file or directory", cacert)) } func TestSetCommand_Execute(t *testing.T) { bsl := "bsl-1" if os.Getenv(cmdtest.CaptureFlag) == "1" { // create a config for factory f := &factorymocks.Factory{} kbclient := velerotest.NewFakeControllerRuntimeClient(t) f.On("Namespace").Return(mock.Anything) f.On("KubebuilderClient").Return(kbclient, nil) // create command c := NewSetCommand(f, "velero backup-location set") c.SetArgs([]string{bsl}) c.Execute() return } cmd := exec.CommandContext(t.Context(), os.Args[0], []string{"-test.run=TestSetCommand_Execute"}...) cmd.Env = append(os.Environ(), fmt.Sprintf("%s=1", cmdtest.CaptureFlag)) _, stderr, err := veleroexec.RunCommand(cmd) if err != nil { assert.Contains(t, stderr, "backupstoragelocations.velero.io \"bsl-1\" not found") return } t.Fatalf("process ran with err %v, want backup delete successfully", err) } ================================================ FILE: pkg/cmd/cli/bug/bug.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package bug import ( "bytes" "context" "errors" "fmt" "net/url" "os" "os/exec" "runtime" "strings" "text/template" "time" "github.com/spf13/cobra" "github.com/vmware-tanzu/velero/pkg/buildinfo" "github.com/vmware-tanzu/velero/pkg/cmd" "github.com/vmware-tanzu/velero/pkg/features" ) const ( // kubectlTimeout is how long we wait in seconds for `kubectl version` // before killing the process kubectlTimeout = 5 * time.Second issueURL = "https://github.com/vmware-tanzu/velero/issues/new" // IssueTemplate is used to generate .github/ISSUE_TEMPLATE/bug_report.md // as well as the initial text that's place in a new Github issue as // the result of running `velero bug`. IssueTemplate = `--- name: Bug report about: Tell us about a problem you are experiencing --- **What steps did you take and what happened:** **What did you expect to happen:** **The following information will help us better understand what's going on**: _If you are using velero v1.7.0+:_ Please use ` + "`velero debug --backup --restore ` " + `to generate the support bundle, and attach to this issue, more options please refer to ` + "`velero debug --help` " + ` _If you are using earlier versions:_ Please provide the output of the following commands (Pasting long output into a [GitHub gist](https://gist.github.com) or other pastebin is fine.) - ` + "`kubectl logs deployment/velero -n velero`" + ` - ` + "`velero backup describe ` or `kubectl get backup/ -n velero -o yaml`" + ` - ` + "`velero backup logs `" + ` - ` + "`velero restore describe ` or `kubectl get restore/ -n velero -o yaml`" + ` - ` + "`velero restore logs `" + ` **Anything else you would like to add:** **Environment:** - Velero version (use ` + "`velero version`" + `):{{.VeleroVersion}} {{.GitCommit}} - Velero features (use ` + "`velero client config get features`" + `): {{.Features}} - Kubernetes version (use ` + "`kubectl version`" + `): {{- if .KubectlVersion}} ` + "```" + ` {{.KubectlVersion}} ` + "```" + ` {{end}} - Kubernetes installer & version: - Cloud provider or hardware configuration: - OS (e.g. from ` + "`/etc/os-release`" + `): {{- if .RuntimeOS}} - RuntimeOS: {{.RuntimeOS}}{{end}} {{- if .RuntimeArch}} - RuntimeArch: {{.RuntimeArch}}{{end}} **Vote on this issue!** This is an invitation to the Velero community to vote on issues, you can see the project's [top voted issues listed here](https://github.com/vmware-tanzu/velero/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc). Use the "reaction smiley face" up to the right of this comment to vote. - :+1: for "I would like to see this bug fixed as soon as possible" - :-1: for "There are more important bugs to focus on right now" ` ) func NewCommand() *cobra.Command { c := &cobra.Command{ Use: "bug", Short: "Report a Velero bug", Long: "Open a browser window to report a Velero bug", Run: func(c *cobra.Command, args []string) { kubectlVersion, err := getKubectlVersion() if err != nil { // we don't want to prevent the user from submitting a bug // if we can't get the kubectl version, so just display a warning fmt.Fprintf(os.Stderr, "WARNING: can't get kubectl version: %v\n", err) } body, err := renderToString(newBugInfo(kubectlVersion)) cmd.CheckError(err) cmd.CheckError(showIssueInBrowser(body)) }, } return c } type VeleroBugInfo struct { VeleroVersion string GitCommit string RuntimeOS string RuntimeArch string KubectlVersion string Features string } // cmdExistsOnPath checks to see if an executable is available on the current PATH func cmdExistsOnPath(name string) bool { if _, err := exec.LookPath(name); err != nil { return false } return true } // getKubectlVersion makes a best-effort to run `kubectl version` // and return it's output. This func will timeout and return an empty // string after kubectlTimeout if we're not connected to a cluster. func getKubectlVersion() (string, error) { if !cmdExistsOnPath("kubectl") { return "", errors.New("kubectl not found on PATH") } kubectlCmd := exec.CommandContext(context.Background(), "kubectl", "version") var outbuf bytes.Buffer kubectlCmd.Stdout = &outbuf if err := kubectlCmd.Start(); err != nil { return "", errors.New("can't start kubectl") } done := make(chan error, 1) go func() { done <- kubectlCmd.Wait() }() select { case <-time.After(kubectlTimeout): // we don't care about the possible error returned from Kill() here, // just return an empty string if err := kubectlCmd.Process.Kill(); err != nil { return "", fmt.Errorf("error killing kubectl process: %w", err) } return "", errors.New("timeout waiting for kubectl version") case err := <-done: if err != nil { return "", errors.New("error waiting for kubectl process") } } versionOut := outbuf.String() kubectlVersion := strings.TrimSpace(versionOut) return kubectlVersion, nil } func newBugInfo(kubectlVersion string) *VeleroBugInfo { return &VeleroBugInfo{ VeleroVersion: buildinfo.Version, GitCommit: buildinfo.FormattedGitSHA(), RuntimeOS: runtime.GOOS, RuntimeArch: runtime.GOARCH, KubectlVersion: kubectlVersion, Features: features.Serialize(), } } // renderToString renders IssueTemplate to a string using the // supplied *VeleroBugInfo func renderToString(bugInfo *VeleroBugInfo) (string, error) { outputTemplate, err := template.New("ghissue").Parse(IssueTemplate) if err != nil { return "", err } var buf bytes.Buffer err = outputTemplate.Execute(&buf, bugInfo) if err != nil { return "", err } return buf.String(), nil } // showIssueInBrowser opens a browser window to submit a Github issue using // a platform specific binary. func showIssueInBrowser(body string) error { url := issueURL + "?body=" + url.QueryEscape(body) ctx := context.Background() switch runtime.GOOS { case "darwin": return exec.CommandContext(ctx, "open", url).Start() case "linux": if cmdExistsOnPath("xdg-open") { return exec.CommandContext(ctx, "xdg-open", url).Start() } return fmt.Errorf("velero can't open a browser window using the command '%s'", "xdg-open") case "windows": return exec.CommandContext(ctx, "rundll32", "url.dll,FileProtocolHandler", url).Start() default: return fmt.Errorf("velero can't open a browser window on platform %s", runtime.GOOS) } } ================================================ FILE: pkg/cmd/cli/client/client.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package client import ( "github.com/spf13/cobra" "github.com/vmware-tanzu/velero/pkg/cmd/cli/client/config" ) func NewCommand() *cobra.Command { c := &cobra.Command{ Use: "client", Short: "Velero client related commands", } c.AddCommand( config.NewCommand(), ) return c } ================================================ FILE: pkg/cmd/cli/client/config/config.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package config import ( "github.com/spf13/cobra" ) func NewCommand() *cobra.Command { c := &cobra.Command{ Use: "config", Short: "Get and set client configuration file values", } c.AddCommand( NewGetCommand(), NewSetCommand(), ) return c } ================================================ FILE: pkg/cmd/cli/client/config/get.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package config import ( "fmt" "sort" "github.com/spf13/cobra" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" ) func NewGetCommand() *cobra.Command { c := &cobra.Command{ Use: "get [KEY 1] [KEY 2] [...]", Short: "Get client configuration file values", Run: func(c *cobra.Command, args []string) { config, err := client.LoadConfig() cmd.CheckError(err) if len(args) == 0 { keys := make([]string, 0, len(config)) for key := range config { keys = append(keys, key) } sort.Strings(keys) for _, key := range keys { fmt.Printf("%s: %s\n", key, config[key]) } } else { for _, key := range args { value, found := config[key] if !found { value = "" } fmt.Printf("%s: %s\n", key, value) } } }, } return c } ================================================ FILE: pkg/cmd/cli/client/config/set.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package config import ( "fmt" "os" "strings" "github.com/spf13/cobra" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" ) func NewSetCommand() *cobra.Command { c := &cobra.Command{ Use: "set KEY=VALUE [KEY=VALUE]...", Short: "Set client configuration file values", Args: cobra.MinimumNArgs(1), Run: func(c *cobra.Command, args []string) { config, err := client.LoadConfig() cmd.CheckError(err) for _, arg := range args { pair := strings.Split(arg, "=") if len(pair) != 2 { fmt.Fprintf(os.Stderr, "WARNING: invalid KEY=VALUE: %q\n", arg) continue } key, value := pair[0], pair[1] if value == "" { delete(config, key) } else { config[key] = value } } cmd.CheckError(client.SaveConfig(config)) }, } return c } ================================================ FILE: pkg/cmd/cli/completion/completion.go ================================================ /* Copyright 2021 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package completion import ( "fmt" "os" "github.com/spf13/cobra" ) func NewCommand() *cobra.Command { c := &cobra.Command{ Use: "completion [bash|zsh|fish]", Short: "Generate completion script", Long: `To load completions: Bash: $ source <(velero completion bash) # To load completions for each session, execute once: Linux: $ velero completion bash > /etc/bash_completion.d/velero MacOS: $ velero completion bash > /usr/local/etc/bash_completion.d/velero Zsh: # If shell completion is not already enabled in your environment you will need # to enable it. You can execute the following once: $ echo "autoload -U compinit; compinit" >> ~/.zshrc # To load completions for each session, execute once: $ velero completion zsh > "${fpath[1]}/_velero" # You will need to start a new shell for this setup to take effect. Fish: $ velero completion fish | source # To load completions for each session, execute once: $ velero completion fish > ~/.config/fish/completions/velero.fish `, Args: cobra.ExactArgs(1), ValidArgs: []string{"bash", "zsh", "fish"}, Run: func(cmd *cobra.Command, args []string) { shell := args[0] switch shell { case "bash": if err := cmd.Root().GenBashCompletion(os.Stdout); err != nil { fmt.Println("fail to generate bash completion script", err) os.Exit(1) } case "zsh": // # fix #4912 // cobra does not support zsh completion ouptput used by source command // according to https://github.com/spf13/cobra/issues/1529 // Need to append compdef manually to do that. zshHead := "#compdef velero\ncompdef _velero velero\n" out := os.Stdout if _, err := out.Write([]byte(zshHead)); err != nil { fmt.Println("fail to append compdef command into zsh completion script: ", err) os.Exit(1) } if err := cmd.Root().GenZshCompletion(out); err != nil { fmt.Println("fail to generate zsh completion script: ", err) os.Exit(1) } case "fish": if err := cmd.Root().GenFishCompletion(os.Stdout, true); err != nil { fmt.Println("fail to generate fish completion script: ", err) os.Exit(1) } default: fmt.Println("Invalid shell specified, specify bash, zsh, or fish") os.Exit(1) } }, } return c } ================================================ FILE: pkg/cmd/cli/create/create.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package create import ( "github.com/spf13/cobra" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd/cli/backup" "github.com/vmware-tanzu/velero/pkg/cmd/cli/backuplocation" "github.com/vmware-tanzu/velero/pkg/cmd/cli/restore" "github.com/vmware-tanzu/velero/pkg/cmd/cli/schedule" "github.com/vmware-tanzu/velero/pkg/cmd/cli/snapshotlocation" ) func NewCommand(f client.Factory) *cobra.Command { c := &cobra.Command{ Use: "create", Short: "Create velero resources", Long: "Create velero resources", } c.AddCommand( backup.NewCreateCommand(f, "backup"), schedule.NewCreateCommand(f, "schedule"), restore.NewCreateCommand(f, "restore"), backuplocation.NewCreateCommand(f, "backup-location"), snapshotlocation.NewCreateCommand(f, "snapshot-location"), ) return c } ================================================ FILE: pkg/cmd/cli/datamover/backup.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package datamover import ( "context" "fmt" "os" "strings" "time" "github.com/bombsimon/logrusr/v3" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" "github.com/vmware-tanzu/velero/internal/credentials" "github.com/vmware-tanzu/velero/pkg/buildinfo" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd/util/signals" "github.com/vmware-tanzu/velero/pkg/datamover" "github.com/vmware-tanzu/velero/pkg/datapath" "github.com/vmware-tanzu/velero/pkg/repository" "github.com/vmware-tanzu/velero/pkg/uploader" "github.com/vmware-tanzu/velero/pkg/util/filesystem" "github.com/vmware-tanzu/velero/pkg/util/kube" "github.com/vmware-tanzu/velero/pkg/util/logging" ctrl "sigs.k8s.io/controller-runtime" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" ctlcache "sigs.k8s.io/controller-runtime/pkg/cache" ctlclient "sigs.k8s.io/controller-runtime/pkg/client" ) type dataMoverBackupConfig struct { volumePath string volumeMode string duName string resourceTimeout time.Duration } func NewBackupCommand(f client.Factory) *cobra.Command { config := dataMoverBackupConfig{} logLevelFlag := logging.LogLevelFlag(logrus.InfoLevel) formatFlag := logging.NewFormatFlag() command := &cobra.Command{ Use: "backup", Short: "Run the velero data-mover backup", Long: "Run the velero data-mover backup", Hidden: true, Run: func(c *cobra.Command, args []string) { logLevel := logLevelFlag.Parse() logrus.Infof("Setting log-level to %s", strings.ToUpper(logLevel.String())) logger := logging.DefaultLogger(logLevel, formatFlag.Parse()) logger.Infof("Starting Velero data-mover backup %s (%s)", buildinfo.Version, buildinfo.FormattedGitSHA()) f.SetBasename(fmt.Sprintf("%s-%s", c.Parent().Name(), c.Name())) s, err := newdataMoverBackup(logger, f, config) if err != nil { kube.ExitPodWithMessage(logger, false, "Failed to create data mover backup, %v", err) } s.run() }, } command.Flags().Var(logLevelFlag, "log-level", fmt.Sprintf("The level at which to log. Valid values are %s.", strings.Join(logLevelFlag.AllowedValues(), ", "))) command.Flags().Var(formatFlag, "log-format", fmt.Sprintf("The format for log output. Valid values are %s.", strings.Join(formatFlag.AllowedValues(), ", "))) command.Flags().StringVar(&config.volumePath, "volume-path", config.volumePath, "The full path of the volume to be backed up") command.Flags().StringVar(&config.volumeMode, "volume-mode", config.volumeMode, "The mode of the volume to be backed up") command.Flags().StringVar(&config.duName, "data-upload", config.duName, "The data upload name") command.Flags().DurationVar(&config.resourceTimeout, "resource-timeout", config.resourceTimeout, "How long to wait for resource processes which are not covered by other specific timeout parameters.") _ = command.MarkFlagRequired("volume-path") _ = command.MarkFlagRequired("volume-mode") _ = command.MarkFlagRequired("data-upload") _ = command.MarkFlagRequired("resource-timeout") return command } type dataMoverBackup struct { logger logrus.FieldLogger ctx context.Context cancelFunc context.CancelFunc client ctlclient.Client cache ctlcache.Cache namespace string nodeName string config dataMoverBackupConfig kubeClient kubernetes.Interface dataPathMgr *datapath.Manager } func newdataMoverBackup(logger logrus.FieldLogger, factory client.Factory, config dataMoverBackupConfig) (*dataMoverBackup, error) { ctx, cancelFunc := context.WithCancel(context.Background()) clientConfig, err := factory.ClientConfig() if err != nil { cancelFunc() return nil, errors.Wrap(err, "error to create client config") } ctrl.SetLogger(logrusr.New(logger)) klog.SetLogger(logrusr.New(logger)) // klog.Logger is used by k8s.io/client-go scheme := runtime.NewScheme() if err := velerov1api.AddToScheme(scheme); err != nil { cancelFunc() return nil, errors.Wrap(err, "error to add velero v1 scheme") } if err := velerov2alpha1api.AddToScheme(scheme); err != nil { cancelFunc() return nil, errors.Wrap(err, "error to add velero v2alpha1 scheme") } if err := corev1api.AddToScheme(scheme); err != nil { cancelFunc() return nil, errors.Wrap(err, "error to add core v1 scheme") } nodeName := os.Getenv("NODE_NAME") // use a field selector to filter to only pods scheduled on this node. cacheOption := ctlcache.Options{ Scheme: scheme, ByObject: map[ctlclient.Object]ctlcache.ByObject{ &corev1api.Pod{}: { Field: fields.Set{"spec.nodeName": nodeName}.AsSelector(), }, &velerov2alpha1api.DataUpload{}: { Field: fields.Set{"metadata.namespace": factory.Namespace()}.AsSelector(), }, }, } cli, err := ctlclient.New(clientConfig, ctlclient.Options{ Scheme: scheme, }) if err != nil { cancelFunc() return nil, errors.Wrap(err, "error to create client") } var cache ctlcache.Cache retry := 10 for { cache, err = ctlcache.New(clientConfig, cacheOption) if err == nil { break } retry-- if retry == 0 { break } logger.WithError(err).Warn("Failed to create client cache, need retry") time.Sleep(time.Second) } if err != nil { cancelFunc() return nil, errors.Wrap(err, "error to create client cache") } s := &dataMoverBackup{ logger: logger, ctx: ctx, cancelFunc: cancelFunc, client: cli, cache: cache, config: config, namespace: factory.Namespace(), nodeName: nodeName, } s.kubeClient, err = factory.KubeClient() if err != nil { cancelFunc() return nil, errors.Wrap(err, "error to create kube client") } s.dataPathMgr = datapath.NewManager(1) return s, nil } var funcExitWithMessage = kube.ExitPodWithMessage var funcCreateDataPathService = (*dataMoverBackup).createDataPathService func (s *dataMoverBackup) run() { signals.CancelOnShutdown(s.cancelFunc, s.logger) go func() { if err := s.cache.Start(s.ctx); err != nil { s.logger.WithError(err).Warn("error starting cache") } }() s.runDataPath() } func (s *dataMoverBackup) runDataPath() { s.logger.Infof("Starting micro service in node %s for du %s", s.nodeName, s.config.duName) dpService, err := funcCreateDataPathService(s) if err != nil { s.cancelFunc() funcExitWithMessage(s.logger, false, "Failed to create data path service for DataUpload %s: %v", s.config.duName, err) return } s.logger.Infof("Starting data path service %s", s.config.duName) err = dpService.Init() if err != nil { dpService.Shutdown() s.cancelFunc() funcExitWithMessage(s.logger, false, "Failed to init data path service for DataUpload %s: %v", s.config.duName, err) return } s.logger.Infof("Running data path service %s", s.config.duName) result, err := dpService.RunCancelableDataPath(s.ctx) if err != nil { dpService.Shutdown() s.cancelFunc() funcExitWithMessage(s.logger, false, "Failed to run data path service for DataUpload %s: %v", s.config.duName, err) return } s.logger.WithField("du", s.config.duName).Info("Data path service completed") dpService.Shutdown() s.logger.WithField("du", s.config.duName).Info("Data path service is shut down") s.cancelFunc() funcExitWithMessage(s.logger, true, result) } var funcNewCredentialFileStore = credentials.NewNamespacedFileStore var funcNewCredentialSecretStore = credentials.NewNamespacedSecretStore func (s *dataMoverBackup) createDataPathService() (dataPathService, error) { credentialFileStore, err := funcNewCredentialFileStore( s.client, s.namespace, credentials.DefaultStoreDirectory(), filesystem.NewFileSystem(), ) if err != nil { return nil, errors.Wrapf(err, "error to create credential file store") } credSecretStore, err := funcNewCredentialSecretStore(s.client, s.namespace) if err != nil { return nil, errors.Wrapf(err, "error to create credential secret store") } credGetter := &credentials.CredentialGetter{FromFile: credentialFileStore, FromSecret: credSecretStore} duInformer, err := s.cache.GetInformer(s.ctx, &velerov2alpha1api.DataUpload{}) if err != nil { return nil, errors.Wrap(err, "error to get controller-runtime informer from manager") } repoEnsurer := repository.NewEnsurer(s.client, s.logger, s.config.resourceTimeout) return datamover.NewBackupMicroService(s.ctx, s.client, s.kubeClient, s.config.duName, s.namespace, s.nodeName, datapath.AccessPoint{ ByPath: s.config.volumePath, VolMode: uploader.PersistentVolumeMode(s.config.volumeMode), }, s.dataPathMgr, repoEnsurer, credGetter, duInformer, s.logger), nil } ================================================ FILE: pkg/cmd/cli/datamover/backup_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package datamover import ( "context" "errors" "fmt" "testing" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" ctlclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/internal/credentials" cacheMock "github.com/vmware-tanzu/velero/pkg/cmd/cli/datamover/mocks" velerotest "github.com/vmware-tanzu/velero/pkg/test" "github.com/vmware-tanzu/velero/pkg/util/filesystem" ) func fakeCreateDataPathServiceWithErr(_ *dataMoverBackup) (dataPathService, error) { return nil, errors.New("fake-create-data-path-error") } var frHelper *fakeRunHelper func fakeCreateDataPathService(_ *dataMoverBackup) (dataPathService, error) { return frHelper, nil } type fakeRunHelper struct { initErr error runCancelableDataPathErr error runCancelableDataPathResult string exitMessage string succeed bool } func (fr *fakeRunHelper) Init() error { return fr.initErr } func (fr *fakeRunHelper) RunCancelableDataPath(_ context.Context) (string, error) { if fr.runCancelableDataPathErr != nil { return "", fr.runCancelableDataPathErr } else { return fr.runCancelableDataPathResult, nil } } func (fr *fakeRunHelper) Shutdown() { } func (fr *fakeRunHelper) ExitWithMessage(logger logrus.FieldLogger, succeed bool, message string, a ...any) { fr.succeed = succeed fr.exitMessage = fmt.Sprintf(message, a...) } func TestRunDataPath(t *testing.T) { tests := []struct { name string duName string createDataPathFail bool initDataPathErr error runCancelableDataPathErr error runCancelableDataPathResult string expectedMessage string expectedSucceed bool }{ { name: "create data path failed", duName: "fake-name", createDataPathFail: true, expectedMessage: "Failed to create data path service for DataUpload fake-name: fake-create-data-path-error", }, { name: "init data path failed", duName: "fake-name", initDataPathErr: errors.New("fake-init-data-path-error"), expectedMessage: "Failed to init data path service for DataUpload fake-name: fake-init-data-path-error", }, { name: "run data path failed", duName: "fake-name", runCancelableDataPathErr: errors.New("fake-run-data-path-error"), expectedMessage: "Failed to run data path service for DataUpload fake-name: fake-run-data-path-error", }, { name: "succeed", duName: "fake-name", runCancelableDataPathResult: "fake-run-data-path-result", expectedMessage: "fake-run-data-path-result", expectedSucceed: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { frHelper = &fakeRunHelper{ initErr: test.initDataPathErr, runCancelableDataPathErr: test.runCancelableDataPathErr, runCancelableDataPathResult: test.runCancelableDataPathResult, } if test.createDataPathFail { funcCreateDataPathService = fakeCreateDataPathServiceWithErr } else { funcCreateDataPathService = fakeCreateDataPathService } funcExitWithMessage = frHelper.ExitWithMessage s := &dataMoverBackup{ logger: velerotest.NewLogger(), cancelFunc: func() {}, config: dataMoverBackupConfig{ duName: test.duName, }, } s.runDataPath() assert.Equal(t, test.expectedMessage, frHelper.exitMessage) assert.Equal(t, test.expectedSucceed, frHelper.succeed) }) } } type fakeCreateDataPathServiceHelper struct { fileStoreErr error secretStoreErr error } func (fc *fakeCreateDataPathServiceHelper) NewNamespacedFileStore(_ ctlclient.Client, _ string, _ string, _ filesystem.Interface) (credentials.FileStore, error) { return nil, fc.fileStoreErr } func (fc *fakeCreateDataPathServiceHelper) NewNamespacedSecretStore(_ ctlclient.Client, _ string) (credentials.SecretStore, error) { return nil, fc.secretStoreErr } func TestCreateDataPathService(t *testing.T) { tests := []struct { name string fileStoreErr error secretStoreErr error mockGetInformer bool getInformerErr error expectedError string }{ { name: "create credential file store error", fileStoreErr: errors.New("fake-file-store-error"), expectedError: "error to create credential file store: fake-file-store-error", }, { name: "create credential secret store", secretStoreErr: errors.New("fake-secret-store-error"), expectedError: "error to create credential secret store: fake-secret-store-error", }, { name: "get informer error", mockGetInformer: true, getInformerErr: errors.New("fake-get-informer-error"), expectedError: "error to get controller-runtime informer from manager: fake-get-informer-error", }, { name: "succeed", mockGetInformer: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fcHelper := &fakeCreateDataPathServiceHelper{ fileStoreErr: test.fileStoreErr, secretStoreErr: test.secretStoreErr, } funcNewCredentialFileStore = fcHelper.NewNamespacedFileStore funcNewCredentialSecretStore = fcHelper.NewNamespacedSecretStore cache := cacheMock.NewCache(t) if test.mockGetInformer { cache.On("GetInformer", mock.Anything, mock.Anything).Return(nil, test.getInformerErr) } funcExitWithMessage = frHelper.ExitWithMessage s := &dataMoverBackup{ cache: cache, } _, err := s.createDataPathService() if test.expectedError != "" { assert.EqualError(t, err, test.expectedError) } else { assert.NoError(t, err) } }) } } ================================================ FILE: pkg/cmd/cli/datamover/data_mover.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package datamover import ( "context" "github.com/spf13/cobra" "github.com/vmware-tanzu/velero/pkg/client" ) func NewCommand(f client.Factory) *cobra.Command { command := &cobra.Command{ Use: "data-mover", Short: "Run the velero data-mover", Long: "Run the velero data-mover", Hidden: true, } command.AddCommand( NewBackupCommand(f), NewRestoreCommand(f), ) return command } type dataPathService interface { Init() error RunCancelableDataPath(context.Context) (string, error) Shutdown() } ================================================ FILE: pkg/cmd/cli/datamover/mocks/Cache.go ================================================ // Code generated by mockery v2.39.1. DO NOT EDIT. package mocks import ( cache "sigs.k8s.io/controller-runtime/pkg/cache" client "sigs.k8s.io/controller-runtime/pkg/client" context "context" mock "github.com/stretchr/testify/mock" schema "k8s.io/apimachinery/pkg/runtime/schema" types "k8s.io/apimachinery/pkg/types" ) // Cache is an autogenerated mock type for the Cache type type Cache struct { mock.Mock } // Get provides a mock function with given fields: ctx, key, obj, opts func (_m *Cache) Get(ctx context.Context, key types.NamespacedName, obj client.Object, opts ...client.GetOption) error { _va := make([]interface{}, len(opts)) for _i := range opts { _va[_i] = opts[_i] } var _ca []interface{} _ca = append(_ca, ctx, key, obj) _ca = append(_ca, _va...) ret := _m.Called(_ca...) if len(ret) == 0 { panic("no return value specified for Get") } var r0 error if rf, ok := ret.Get(0).(func(context.Context, types.NamespacedName, client.Object, ...client.GetOption) error); ok { r0 = rf(ctx, key, obj, opts...) } else { r0 = ret.Error(0) } return r0 } // GetInformer provides a mock function with given fields: ctx, obj, opts func (_m *Cache) GetInformer(ctx context.Context, obj client.Object, opts ...cache.InformerGetOption) (cache.Informer, error) { _va := make([]interface{}, len(opts)) for _i := range opts { _va[_i] = opts[_i] } var _ca []interface{} _ca = append(_ca, ctx, obj) _ca = append(_ca, _va...) ret := _m.Called(_ca...) if len(ret) == 0 { panic("no return value specified for GetInformer") } var r0 cache.Informer var r1 error if rf, ok := ret.Get(0).(func(context.Context, client.Object, ...cache.InformerGetOption) (cache.Informer, error)); ok { return rf(ctx, obj, opts...) } if rf, ok := ret.Get(0).(func(context.Context, client.Object, ...cache.InformerGetOption) cache.Informer); ok { r0 = rf(ctx, obj, opts...) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(cache.Informer) } } if rf, ok := ret.Get(1).(func(context.Context, client.Object, ...cache.InformerGetOption) error); ok { r1 = rf(ctx, obj, opts...) } else { r1 = ret.Error(1) } return r0, r1 } // GetInformerForKind provides a mock function with given fields: ctx, gvk, opts func (_m *Cache) GetInformerForKind(ctx context.Context, gvk schema.GroupVersionKind, opts ...cache.InformerGetOption) (cache.Informer, error) { _va := make([]interface{}, len(opts)) for _i := range opts { _va[_i] = opts[_i] } var _ca []interface{} _ca = append(_ca, ctx, gvk) _ca = append(_ca, _va...) ret := _m.Called(_ca...) if len(ret) == 0 { panic("no return value specified for GetInformerForKind") } var r0 cache.Informer var r1 error if rf, ok := ret.Get(0).(func(context.Context, schema.GroupVersionKind, ...cache.InformerGetOption) (cache.Informer, error)); ok { return rf(ctx, gvk, opts...) } if rf, ok := ret.Get(0).(func(context.Context, schema.GroupVersionKind, ...cache.InformerGetOption) cache.Informer); ok { r0 = rf(ctx, gvk, opts...) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(cache.Informer) } } if rf, ok := ret.Get(1).(func(context.Context, schema.GroupVersionKind, ...cache.InformerGetOption) error); ok { r1 = rf(ctx, gvk, opts...) } else { r1 = ret.Error(1) } return r0, r1 } // IndexField provides a mock function with given fields: ctx, obj, field, extractValue func (_m *Cache) IndexField(ctx context.Context, obj client.Object, field string, extractValue client.IndexerFunc) error { ret := _m.Called(ctx, obj, field, extractValue) if len(ret) == 0 { panic("no return value specified for IndexField") } var r0 error if rf, ok := ret.Get(0).(func(context.Context, client.Object, string, client.IndexerFunc) error); ok { r0 = rf(ctx, obj, field, extractValue) } else { r0 = ret.Error(0) } return r0 } // List provides a mock function with given fields: ctx, list, opts func (_m *Cache) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { _va := make([]interface{}, len(opts)) for _i := range opts { _va[_i] = opts[_i] } var _ca []interface{} _ca = append(_ca, ctx, list) _ca = append(_ca, _va...) ret := _m.Called(_ca...) if len(ret) == 0 { panic("no return value specified for List") } var r0 error if rf, ok := ret.Get(0).(func(context.Context, client.ObjectList, ...client.ListOption) error); ok { r0 = rf(ctx, list, opts...) } else { r0 = ret.Error(0) } return r0 } // RemoveInformer provides a mock function with given fields: ctx, obj func (_m *Cache) RemoveInformer(ctx context.Context, obj client.Object) error { ret := _m.Called(ctx, obj) if len(ret) == 0 { panic("no return value specified for RemoveInformer") } var r0 error if rf, ok := ret.Get(0).(func(context.Context, client.Object) error); ok { r0 = rf(ctx, obj) } else { r0 = ret.Error(0) } return r0 } // Start provides a mock function with given fields: ctx func (_m *Cache) Start(ctx context.Context) error { ret := _m.Called(ctx) if len(ret) == 0 { panic("no return value specified for Start") } var r0 error if rf, ok := ret.Get(0).(func(context.Context) error); ok { r0 = rf(ctx) } else { r0 = ret.Error(0) } return r0 } // WaitForCacheSync provides a mock function with given fields: ctx func (_m *Cache) WaitForCacheSync(ctx context.Context) bool { ret := _m.Called(ctx) if len(ret) == 0 { panic("no return value specified for WaitForCacheSync") } var r0 bool if rf, ok := ret.Get(0).(func(context.Context) bool); ok { r0 = rf(ctx) } else { r0 = ret.Get(0).(bool) } return r0 } // NewCache creates a new instance of Cache. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. func NewCache(t interface { mock.TestingT Cleanup(func()) }) *Cache { mock := &Cache{} mock.Mock.Test(t) t.Cleanup(func() { mock.AssertExpectations(t) }) return mock } ================================================ FILE: pkg/cmd/cli/datamover/restore.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package datamover import ( "context" "fmt" "os" "strings" "time" "github.com/bombsimon/logrusr/v3" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" ctrl "sigs.k8s.io/controller-runtime" "github.com/vmware-tanzu/velero/internal/credentials" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" "github.com/vmware-tanzu/velero/pkg/buildinfo" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd/util/signals" "github.com/vmware-tanzu/velero/pkg/datamover" "github.com/vmware-tanzu/velero/pkg/datapath" "github.com/vmware-tanzu/velero/pkg/repository" "github.com/vmware-tanzu/velero/pkg/uploader" "github.com/vmware-tanzu/velero/pkg/util/filesystem" "github.com/vmware-tanzu/velero/pkg/util/kube" "github.com/vmware-tanzu/velero/pkg/util/logging" ctlcache "sigs.k8s.io/controller-runtime/pkg/cache" ctlclient "sigs.k8s.io/controller-runtime/pkg/client" ) type dataMoverRestoreConfig struct { volumePath string volumeMode string ddName string cacheDir string resourceTimeout time.Duration } func NewRestoreCommand(f client.Factory) *cobra.Command { logLevelFlag := logging.LogLevelFlag(logrus.InfoLevel) formatFlag := logging.NewFormatFlag() config := dataMoverRestoreConfig{} command := &cobra.Command{ Use: "restore", Short: "Run the velero data-mover restore", Long: "Run the velero data-mover restore", Hidden: true, Run: func(c *cobra.Command, args []string) { logLevel := logLevelFlag.Parse() logrus.Infof("Setting log-level to %s", strings.ToUpper(logLevel.String())) logger := logging.DefaultLogger(logLevel, formatFlag.Parse()) logger.Infof("Starting Velero data-mover restore %s (%s)", buildinfo.Version, buildinfo.FormattedGitSHA()) f.SetBasename(fmt.Sprintf("%s-%s", c.Parent().Name(), c.Name())) s, err := newdataMoverRestore(logger, f, config) if err != nil { kube.ExitPodWithMessage(logger, false, "Failed to create data mover restore, %v", err) } s.run() }, } command.Flags().Var(logLevelFlag, "log-level", fmt.Sprintf("The level at which to log. Valid values are %s.", strings.Join(logLevelFlag.AllowedValues(), ", "))) command.Flags().Var(formatFlag, "log-format", fmt.Sprintf("The format for log output. Valid values are %s.", strings.Join(formatFlag.AllowedValues(), ", "))) command.Flags().StringVar(&config.volumePath, "volume-path", config.volumePath, "The full path of the volume to be restored") command.Flags().StringVar(&config.volumeMode, "volume-mode", config.volumeMode, "The mode of the volume to be restored") command.Flags().StringVar(&config.ddName, "data-download", config.ddName, "The data download name") command.Flags().StringVar(&config.cacheDir, "cache-volume-path", config.cacheDir, "The full path of the cache volume") command.Flags().DurationVar(&config.resourceTimeout, "resource-timeout", config.resourceTimeout, "How long to wait for resource processes which are not covered by other specific timeout parameters.") _ = command.MarkFlagRequired("volume-path") _ = command.MarkFlagRequired("volume-mode") _ = command.MarkFlagRequired("data-download") _ = command.MarkFlagRequired("resource-timeout") return command } type dataMoverRestore struct { logger logrus.FieldLogger ctx context.Context cancelFunc context.CancelFunc client ctlclient.Client cache ctlcache.Cache namespace string nodeName string config dataMoverRestoreConfig kubeClient kubernetes.Interface dataPathMgr *datapath.Manager } func newdataMoverRestore(logger logrus.FieldLogger, factory client.Factory, config dataMoverRestoreConfig) (*dataMoverRestore, error) { ctx, cancelFunc := context.WithCancel(context.Background()) clientConfig, err := factory.ClientConfig() if err != nil { cancelFunc() return nil, errors.Wrap(err, "error to create client config") } ctrl.SetLogger(logrusr.New(logger)) klog.SetLogger(logrusr.New(logger)) // klog.Logger is used by k8s.io/client-go scheme := runtime.NewScheme() if err := velerov1api.AddToScheme(scheme); err != nil { cancelFunc() return nil, errors.Wrap(err, "error to add velero v1 scheme") } if err := velerov2alpha1api.AddToScheme(scheme); err != nil { cancelFunc() return nil, errors.Wrap(err, "error to add velero v2alpha1 scheme") } if err := corev1api.AddToScheme(scheme); err != nil { cancelFunc() return nil, errors.Wrap(err, "error to add core v1 scheme") } nodeName := os.Getenv("NODE_NAME") // use a field selector to filter to only pods scheduled on this node. cacheOption := ctlcache.Options{ Scheme: scheme, ByObject: map[ctlclient.Object]ctlcache.ByObject{ &corev1api.Pod{}: { Field: fields.Set{"spec.nodeName": nodeName}.AsSelector(), }, &velerov2alpha1api.DataDownload{}: { Field: fields.Set{"metadata.namespace": factory.Namespace()}.AsSelector(), }, }, } cli, err := ctlclient.New(clientConfig, ctlclient.Options{ Scheme: scheme, }) if err != nil { cancelFunc() return nil, errors.Wrap(err, "error to create client") } var cache ctlcache.Cache retry := 10 for { cache, err = ctlcache.New(clientConfig, cacheOption) if err == nil { break } retry-- if retry == 0 { break } logger.WithError(err).Warn("Failed to create client cache, need retry") time.Sleep(time.Second) } if err != nil { cancelFunc() return nil, errors.Wrap(err, "error to create client cache") } s := &dataMoverRestore{ logger: logger, ctx: ctx, cancelFunc: cancelFunc, client: cli, cache: cache, config: config, namespace: factory.Namespace(), nodeName: nodeName, } s.kubeClient, err = factory.KubeClient() if err != nil { cancelFunc() return nil, errors.Wrap(err, "error to create kube client") } s.dataPathMgr = datapath.NewManager(1) return s, nil } var funcCreateDataPathRestore = (*dataMoverRestore).createDataPathService func (s *dataMoverRestore) run() { signals.CancelOnShutdown(s.cancelFunc, s.logger) go func() { if err := s.cache.Start(s.ctx); err != nil { s.logger.WithError(err).Warn("error starting cache") } }() s.runDataPath() } func (s *dataMoverRestore) runDataPath() { s.logger.Infof("Starting micro service in node %s for dd %s", s.nodeName, s.config.ddName) dpService, err := funcCreateDataPathRestore(s) if err != nil { s.cancelFunc() funcExitWithMessage(s.logger, false, "Failed to create data path service for DataDownload %s: %v", s.config.ddName, err) return } s.logger.Infof("Starting data path service %s", s.config.ddName) err = dpService.Init() if err != nil { dpService.Shutdown() s.cancelFunc() funcExitWithMessage(s.logger, false, "Failed to init data path service for DataDownload %s: %v", s.config.ddName, err) return } result, err := dpService.RunCancelableDataPath(s.ctx) if err != nil { dpService.Shutdown() s.cancelFunc() funcExitWithMessage(s.logger, false, "Failed to run data path service for DataDownload %s: %v", s.config.ddName, err) return } s.logger.WithField("dd", s.config.ddName).Info("Data path service completed") dpService.Shutdown() s.logger.WithField("dd", s.config.ddName).Info("Data path service is shut down") s.cancelFunc() funcExitWithMessage(s.logger, true, result) } func (s *dataMoverRestore) createDataPathService() (dataPathService, error) { credentialFileStore, err := funcNewCredentialFileStore( s.client, s.namespace, credentials.DefaultStoreDirectory(), filesystem.NewFileSystem(), ) if err != nil { return nil, errors.Wrapf(err, "error to create credential file store") } credSecretStore, err := funcNewCredentialSecretStore(s.client, s.namespace) if err != nil { return nil, errors.Wrapf(err, "error to create credential secret store") } credGetter := &credentials.CredentialGetter{FromFile: credentialFileStore, FromSecret: credSecretStore} duInformer, err := s.cache.GetInformer(s.ctx, &velerov2alpha1api.DataDownload{}) if err != nil { return nil, errors.Wrap(err, "error to get controller-runtime informer from manager") } repoEnsurer := repository.NewEnsurer(s.client, s.logger, s.config.resourceTimeout) return datamover.NewRestoreMicroService(s.ctx, s.client, s.kubeClient, s.config.ddName, s.namespace, s.nodeName, datapath.AccessPoint{ ByPath: s.config.volumePath, VolMode: uploader.PersistentVolumeMode(s.config.volumeMode), }, s.dataPathMgr, repoEnsurer, credGetter, duInformer, s.config.cacheDir, s.logger), nil } ================================================ FILE: pkg/cmd/cli/datamover/restore_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package datamover import ( "errors" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" cacheMock "github.com/vmware-tanzu/velero/pkg/cmd/cli/datamover/mocks" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func fakeCreateDataPathRestoreWithErr(_ *dataMoverRestore) (dataPathService, error) { return nil, errors.New("fake-create-data-path-error") } func fakeCreateDataPathRestore(_ *dataMoverRestore) (dataPathService, error) { return frHelper, nil } func TestRunDataPathRestore(t *testing.T) { tests := []struct { name string ddName string createDataPathFail bool initDataPathErr error runCancelableDataPathErr error runCancelableDataPathResult string expectedMessage string expectedSucceed bool }{ { name: "create data path failed", ddName: "fake-name", createDataPathFail: true, expectedMessage: "Failed to create data path service for DataDownload fake-name: fake-create-data-path-error", }, { name: "init data path failed", ddName: "fake-name", initDataPathErr: errors.New("fake-init-data-path-error"), expectedMessage: "Failed to init data path service for DataDownload fake-name: fake-init-data-path-error", }, { name: "run data path failed", ddName: "fake-name", runCancelableDataPathErr: errors.New("fake-run-data-path-error"), expectedMessage: "Failed to run data path service for DataDownload fake-name: fake-run-data-path-error", }, { name: "succeed", ddName: "fake-name", runCancelableDataPathResult: "fake-run-data-path-result", expectedMessage: "fake-run-data-path-result", expectedSucceed: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { frHelper = &fakeRunHelper{ initErr: test.initDataPathErr, runCancelableDataPathErr: test.runCancelableDataPathErr, runCancelableDataPathResult: test.runCancelableDataPathResult, } if test.createDataPathFail { funcCreateDataPathRestore = fakeCreateDataPathRestoreWithErr } else { funcCreateDataPathRestore = fakeCreateDataPathRestore } funcExitWithMessage = frHelper.ExitWithMessage s := &dataMoverRestore{ logger: velerotest.NewLogger(), cancelFunc: func() {}, config: dataMoverRestoreConfig{ ddName: test.ddName, }, } s.runDataPath() assert.Equal(t, test.expectedMessage, frHelper.exitMessage) assert.Equal(t, test.expectedSucceed, frHelper.succeed) }) } } func TestCreateDataPathRestore(t *testing.T) { tests := []struct { name string fileStoreErr error secretStoreErr error mockGetInformer bool getInformerErr error expectedError string }{ { name: "create credential file store error", fileStoreErr: errors.New("fake-file-store-error"), expectedError: "error to create credential file store: fake-file-store-error", }, { name: "create credential secret store", secretStoreErr: errors.New("fake-secret-store-error"), expectedError: "error to create credential secret store: fake-secret-store-error", }, { name: "get informer error", mockGetInformer: true, getInformerErr: errors.New("fake-get-informer-error"), expectedError: "error to get controller-runtime informer from manager: fake-get-informer-error", }, { name: "succeed", mockGetInformer: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fcHelper := &fakeCreateDataPathServiceHelper{ fileStoreErr: test.fileStoreErr, secretStoreErr: test.secretStoreErr, } funcNewCredentialFileStore = fcHelper.NewNamespacedFileStore funcNewCredentialSecretStore = fcHelper.NewNamespacedSecretStore cache := cacheMock.NewCache(t) if test.mockGetInformer { cache.On("GetInformer", mock.Anything, mock.Anything).Return(nil, test.getInformerErr) } funcExitWithMessage = frHelper.ExitWithMessage s := &dataMoverRestore{ cache: cache, } _, err := s.createDataPathService() if test.expectedError != "" { assert.EqualError(t, err, test.expectedError) } else { assert.NoError(t, err) } }) } } ================================================ FILE: pkg/cmd/cli/debug/cshd-scripts/velero.cshd ================================================ def capture_backup_logs(cmd, namespace): if args.backup: log("Collecting log and information for backup: {}".format(args.backup)) backupDescCmd = "{} --namespace={} backup describe {} --details".format(cmd, namespace, args.backup) capture_local(cmd=backupDescCmd, file_name="backup_describe_{}.txt".format(args.backup)) backupLogsCmd = "{} --namespace={} backup logs {}".format(cmd, namespace, args.backup) capture_local(cmd=backupLogsCmd, file_name="backup_{}.log".format(args.backup)) def capture_restore_logs(cmd, namespace): if args.restore: log("Collecting log and information for restore: {}".format(args.restore)) restoreDescCmd = "{} --namespace={} restore describe {} --details".format(cmd, namespace, args.restore) capture_local(cmd=restoreDescCmd, file_name="restore_describe_{}.txt".format(args.restore)) restoreLogsCmd = "{} --namespace={} restore logs {}".format(cmd, namespace, args.restore) capture_local(cmd=restoreLogsCmd, file_name="restore_{}.log".format(args.restore)) ns = args.namespace if args.namespace else "velero" output = args.output if args.output else "bundle.tar.gz" cmd = args.cmd if args.cmd else "velero" # Working dir for writing during script execution crshd = crashd_config(workdir="./velero-bundle") set_defaults(kube_config(path=args.kubeconfig, cluster_context=args.kubecontext)) log("Collecting velero resources in namespace: {}". format(ns)) kube_capture(what="objects", namespaces=[ns], groups=['velero.io']) capture_local(cmd="{} version -n {}".format(cmd, ns), file_name="version.txt") log("Collecting velero deployment logs in namespace: {}". format(ns)) kube_capture(what="logs", namespaces=[ns]) capture_backup_logs(cmd, ns) capture_restore_logs(cmd, ns) archive(output_file=output, source_paths=[crshd.workdir]) log("Generated debug information bundle: {}".format(output)) ================================================ FILE: pkg/cmd/cli/debug/debug.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package debug import ( "bytes" "context" _ "embed" "fmt" "os" "path/filepath" "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/pflag" "github.com/vmware-tanzu/crash-diagnostics/exec" appsv1api "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/tools/clientcmd" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" ) //go:embed cshd-scripts/velero.cshd var scriptBytes []byte type option struct { // currCmd the velero command currCmd string // workdir for crashd will be $baseDir/velero-debug baseDir string // the namespace where velero server is installed namespace string // the absolute path for the log bundle to be generated outputPath string // the absolute path for the kubeconfig file that will be read by crashd for calling K8S API kubeconfigPath string // the kubecontext to be used for calling K8S API kubeContext string // optional, the name of the backup resource whose log will be packaged into the debug bundle backup string // optional, the name of the restore resource whose log will be packaged into the debug bundle restore string // optional, it controls whether to print the debug log messages when calling crashd verbose bool } func (o *option) bindFlags(flags *pflag.FlagSet) { flags.StringVar(&o.outputPath, "output", "", "The path of the bundle tarball, by default it's ./bundle---
---.tar.gz. Optional") flags.StringVar(&o.backup, "backup", "", "The name of the backup resource whose log will be collected, no backup logs will be collected if it's not set. Optional") flags.StringVar(&o.restore, "restore", "", "The name of the restore resource whose log will be collected, no restore logs will be collected if it's not set. Optional") flags.BoolVar(&o.verbose, "verbose", false, "When it's set to true the debug messages by crashd will be printed during execution. Default value is false.") } func (o *option) asCrashdArgMap() exec.ArgMap { return exec.ArgMap{ "cmd": o.currCmd, "output": o.outputPath, "namespace": o.namespace, "basedir": o.baseDir, "backup": o.backup, "restore": o.restore, "kubeconfig": o.kubeconfigPath, "kubecontext": o.kubeContext, } } func (o *option) complete(f client.Factory, fs *pflag.FlagSet) error { if len(o.outputPath) == 0 { o.outputPath = fmt.Sprintf("./bundle-%s.tar.gz", time.Now().Format("2006-01-02-15-04-05")) } absOutputPath, err := filepath.Abs(o.outputPath) if err != nil { return fmt.Errorf("invalid output path: %v", err) } o.outputPath = absOutputPath tmpDir, err := os.MkdirTemp("", "crashd") if err != nil { return err } o.baseDir = tmpDir o.namespace = f.Namespace() kp, kc := kubeconfigAndContext(fs) o.currCmd, err = os.Executable() if err != nil { return err } o.kubeconfigPath, err = filepath.Abs(kp) if err != nil { return fmt.Errorf("invalid kubeconfig path: %s, %v", kp, err) } o.kubeContext = kc return nil } func (o *option) validate(f client.Factory) error { crClient, err := f.KubebuilderClient() if err != nil { return err } deploymentList := new(appsv1api.DeploymentList) selector, err := labels.Parse("component=velero") cmd.CheckError(err) err = crClient.List(context.TODO(), deploymentList, &ctrlclient.ListOptions{ Namespace: o.namespace, LabelSelector: selector, }) if err != nil { return errors.Wrap(err, "failed to check velero deployment") } if len(deploymentList.Items) == 0 { return fmt.Errorf("velero deployment does not exist in namespace: %s", o.namespace) } if len(o.backup) > 0 { backup := new(velerov1api.Backup) if err := crClient.Get(context.TODO(), ctrlclient.ObjectKey{Namespace: o.namespace, Name: o.backup}, backup); err != nil { return err } } if len(o.restore) > 0 { restore := new(velerov1api.Restore) if err := crClient.Get(context.TODO(), ctrlclient.ObjectKey{Namespace: o.namespace, Name: o.restore}, restore); err != nil { return err } } return nil } // NewCommand creates a cobra command. func NewCommand(f client.Factory) *cobra.Command { o := &option{} c := &cobra.Command{ Use: "debug", Short: "Generate debug bundle", Long: `Generate a tarball containing the logs of velero deployment, plugin logs, node-agent DaemonSet, specs of resources created by velero server, and optionally the logs of backup and restore.`, Run: func(c *cobra.Command, args []string) { flags := c.Flags() err := o.complete(f, flags) cmd.CheckError(err) defer func(opt *option) { if len(opt.baseDir) > 0 { if err := os.RemoveAll(opt.baseDir); err != nil { fmt.Fprintf(os.Stderr, "Failed to remove temp dir: %s: %v\n", opt.baseDir, err) } } }(o) err = o.validate(f) cmd.CheckError(err) err = runCrashd(o) cmd.CheckError(err) }, } o.bindFlags(c.Flags()) return c } func runCrashd(o *option) error { pwd, err := os.Getwd() if err != nil { return err } defer func() { if err := os.Chdir(pwd); err != nil { fmt.Fprintf(os.Stderr, "Failed to go back to workdir: %v", err) } }() if err := os.Chdir(o.baseDir); err != nil { return err } logrus.SetOutput(os.Stdout) if o.verbose { logrus.SetLevel(logrus.DebugLevel) } return exec.Execute("velero-debug-collector", bytes.NewReader(scriptBytes), o.asCrashdArgMap()) } func kubeconfigAndContext(fs *pflag.FlagSet) (string, string) { pathOpt := clientcmd.NewDefaultPathOptions() kubeconfig, _ := fs.GetString("kubeconfig") if len(kubeconfig) > 0 { pathOpt.LoadingRules.ExplicitPath = kubeconfig } kubecontext, _ := fs.GetString("kubecontext") return pathOpt.GetDefaultFilename(), kubecontext } ================================================ FILE: pkg/cmd/cli/delete/delete.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package delete import ( "github.com/spf13/cobra" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd/cli/backup" "github.com/vmware-tanzu/velero/pkg/cmd/cli/backuplocation" "github.com/vmware-tanzu/velero/pkg/cmd/cli/restore" "github.com/vmware-tanzu/velero/pkg/cmd/cli/schedule" ) func NewCommand(f client.Factory) *cobra.Command { c := &cobra.Command{ Use: "delete", Short: "Delete velero resources", Long: "Delete velero resources", } backupCommand := backup.NewDeleteCommand(f, "backup") backupCommand.Aliases = []string{"backups"} backuplocationCommand := backuplocation.NewDeleteCommand(f, "backup-location") backuplocationCommand.Aliases = []string{"backup-locations"} restoreCommand := restore.NewDeleteCommand(f, "restore") restoreCommand.Aliases = []string{"restores"} scheduleCommand := schedule.NewDeleteCommand(f, "schedule") scheduleCommand.Aliases = []string{"schedules"} c.AddCommand( backupCommand, backuplocationCommand, restoreCommand, scheduleCommand, ) return c } ================================================ FILE: pkg/cmd/cli/delete_options.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cli import ( "errors" "github.com/spf13/cobra" "github.com/spf13/pflag" controllerclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd/util/confirm" ) // DeleteOptions contains parameters used for deleting a restore. type DeleteOptions struct { *SelectOptions confirm.ConfirmOptions Client controllerclient.Client Namespace string } func NewDeleteOptions(singularTypeName string) *DeleteOptions { o := &DeleteOptions{} o.ConfirmOptions = *confirm.NewConfirmOptionsWithDescription("Confirm deletion") o.SelectOptions = NewSelectOptions("delete", singularTypeName) return o } // Complete fills in the correct values for all the options. func (o *DeleteOptions) Complete(f client.Factory, args []string) error { o.Namespace = f.Namespace() client, err := f.KubebuilderClient() if err != nil { return err } o.Client = client return o.SelectOptions.Complete(args) } // Validate validates the fields of the DeleteOptions struct. func (o *DeleteOptions) Validate(c *cobra.Command, f client.Factory, args []string) error { if o.Client == nil { return errors.New("velero client is not set; unable to proceed") } return o.SelectOptions.Validate() } // BindFlags binds options for this command to flags. func (o *DeleteOptions) BindFlags(flags *pflag.FlagSet) { o.ConfirmOptions.BindFlags(flags) o.SelectOptions.BindFlags(flags) } // Xor returns true if exactly one of the provided values is true, // or false otherwise. func xor(val bool, vals ...bool) bool { res := val for _, v := range vals { if res && v { return false } res = res || v } return res } ================================================ FILE: pkg/cmd/cli/describe/describe.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package describe import ( "github.com/spf13/cobra" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd/cli/backup" "github.com/vmware-tanzu/velero/pkg/cmd/cli/restore" "github.com/vmware-tanzu/velero/pkg/cmd/cli/schedule" ) func NewCommand(f client.Factory) *cobra.Command { c := &cobra.Command{ Use: "describe", Short: "Describe velero resources", Long: "Describe velero resources", } backupCommand := backup.NewDescribeCommand(f, "backups") backupCommand.Aliases = []string{"backup"} scheduleCommand := schedule.NewDescribeCommand(f, "schedules") scheduleCommand.Aliases = []string{"schedule"} restoreCommand := restore.NewDescribeCommand(f, "restores") restoreCommand.Aliases = []string{"restore"} c.AddCommand( backupCommand, scheduleCommand, restoreCommand, ) return c } ================================================ FILE: pkg/cmd/cli/get/get.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package get import ( "github.com/spf13/cobra" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd/cli/backup" "github.com/vmware-tanzu/velero/pkg/cmd/cli/backuplocation" "github.com/vmware-tanzu/velero/pkg/cmd/cli/plugin" "github.com/vmware-tanzu/velero/pkg/cmd/cli/restore" "github.com/vmware-tanzu/velero/pkg/cmd/cli/schedule" "github.com/vmware-tanzu/velero/pkg/cmd/cli/snapshotlocation" ) func NewCommand(f client.Factory) *cobra.Command { c := &cobra.Command{ Use: "get", Short: "Get velero resources", Long: "Get velero resources", } backupCommand := backup.NewGetCommand(f, "backups") backupCommand.Aliases = []string{"backup"} scheduleCommand := schedule.NewGetCommand(f, "schedules") scheduleCommand.Aliases = []string{"schedule"} restoreCommand := restore.NewGetCommand(f, "restores") restoreCommand.Aliases = []string{"restore"} backupLocationCommand := backuplocation.NewGetCommand(f, "backup-locations") backupLocationCommand.Aliases = []string{"backup-location"} snapshotLocationCommand := snapshotlocation.NewGetCommand(f, "snapshot-locations") snapshotLocationCommand.Aliases = []string{"snapshot-location"} pluginCommand := plugin.NewGetCommand(f, "plugins") pluginCommand.Aliases = []string{"plugin"} c.AddCommand( backupCommand, scheduleCommand, restoreCommand, backupLocationCommand, snapshotLocationCommand, pluginCommand, ) return c } ================================================ FILE: pkg/cmd/cli/install/install.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package install import ( "fmt" "os" "path/filepath" "strings" "time" "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "github.com/vmware-tanzu/velero/internal/velero" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" "github.com/vmware-tanzu/velero/pkg/cmd/util/flag" "github.com/vmware-tanzu/velero/pkg/cmd/util/output" "github.com/vmware-tanzu/velero/pkg/install" velerotypes "github.com/vmware-tanzu/velero/pkg/types" "github.com/vmware-tanzu/velero/pkg/uploader" kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube" ) // Options collects all the options for installing Velero into a Kubernetes cluster. type Options struct { Namespace string Image string BucketName string Prefix string ProviderName string PodAnnotations flag.Map PodLabels flag.Map ServiceAccountAnnotations flag.Map ServiceAccountName string VeleroPodCPURequest string VeleroPodMemRequest string VeleroPodCPULimit string VeleroPodMemLimit string NodeAgentPodCPURequest string NodeAgentPodMemRequest string NodeAgentPodCPULimit string NodeAgentPodMemLimit string RestoreOnly bool SecretFile string NoSecret bool DryRun bool BackupStorageConfig flag.Map VolumeSnapshotConfig flag.Map UseNodeAgent bool UseNodeAgentWindows bool PrivilegedNodeAgent bool Wait bool UseVolumeSnapshots bool DefaultRepoMaintenanceFrequency time.Duration GarbageCollectionFrequency time.Duration PodVolumeOperationTimeout time.Duration Plugins flag.StringArray NoDefaultBackupLocation bool CRDsOnly bool CACertFile string Features string DefaultVolumesToFsBackup bool UploaderType string DefaultSnapshotMoveData bool DisableInformerCache bool ScheduleSkipImmediately bool PodResources kubeutil.PodResources KeepLatestMaintenanceJobs int BackupRepoConfigMap string RepoMaintenanceJobConfigMap string NodeAgentConfigMap string ItemBlockWorkerCount int ConcurrentBackups int NodeAgentDisableHostPath bool kubeletRootDir string Apply bool ServerPriorityClassName string NodeAgentPriorityClassName string } // BindFlags adds command line values to the options struct. func (o *Options) BindFlags(flags *pflag.FlagSet) { flags.StringVar(&o.ProviderName, "provider", o.ProviderName, "Provider name for backup and volume storage") flags.StringVar(&o.BucketName, "bucket", o.BucketName, "Name of the object storage bucket where backups should be stored") flags.StringVar(&o.SecretFile, "secret-file", o.SecretFile, "File containing credentials for backup and volume provider. If not specified, --no-secret must be used for confirmation. Optional.") flags.BoolVar(&o.NoSecret, "no-secret", o.NoSecret, "Flag indicating if a secret should be created. Must be used as confirmation if --secret-file is not provided. Optional.") flags.BoolVar(&o.Apply, "apply", o.Apply, "Flag indicating if resources should be applied instead of created. This can be used for updating existing resources.") flags.BoolVar(&o.NoDefaultBackupLocation, "no-default-backup-location", o.NoDefaultBackupLocation, "Flag indicating if a default backup location should be created. Must be used as confirmation if --bucket or --provider are not provided. Optional.") flags.StringVar(&o.Image, "image", o.Image, "Image to use for the Velero and node agent pods. Optional.") flags.StringVar(&o.Prefix, "prefix", o.Prefix, "Prefix under which all Velero data should be stored within the bucket. Optional.") flags.Var(&o.PodAnnotations, "pod-annotations", "Annotations to add to the Velero and node agent pods. Optional. Format is key1=value1,key2=value2") flags.Var(&o.PodLabels, "pod-labels", "Labels to add to the Velero and node agent pods. Optional. Format is key1=value1,key2=value2") flags.Var(&o.ServiceAccountAnnotations, "sa-annotations", "Annotations to add to the Velero ServiceAccount. Add iam.gke.io/gcp-service-account=[GSA_NAME]@[PROJECT_NAME].iam.gserviceaccount.com for workload identity. Optional. Format is key1=value1,key2=value2") flags.StringVar(&o.ServiceAccountName, "service-account-name", o.ServiceAccountName, "ServiceAccountName to be set to the Velero and node agent pods, it should be created before the installation, and the user also needs to create the rolebinding for it."+ " Optional, if this attribute is set, the default service account 'velero' will not be created, and the flag --sa-annotations will be disregarded.") flags.StringVar(&o.VeleroPodCPURequest, "velero-pod-cpu-request", o.VeleroPodCPURequest, `CPU request for Velero pod. A value of "0" is treated as unbounded. Optional.`) flags.StringVar(&o.VeleroPodMemRequest, "velero-pod-mem-request", o.VeleroPodMemRequest, `Memory request for Velero pod. A value of "0" is treated as unbounded. Optional.`) flags.StringVar(&o.VeleroPodCPULimit, "velero-pod-cpu-limit", o.VeleroPodCPULimit, `CPU limit for Velero pod. A value of "0" is treated as unbounded. Optional.`) flags.StringVar(&o.VeleroPodMemLimit, "velero-pod-mem-limit", o.VeleroPodMemLimit, `Memory limit for Velero pod. A value of "0" is treated as unbounded. Optional.`) flags.StringVar(&o.NodeAgentPodCPURequest, "node-agent-pod-cpu-request", o.NodeAgentPodCPURequest, `CPU request for node-agent pod. A value of "0" is treated as unbounded. Optional.`) flags.StringVar(&o.NodeAgentPodMemRequest, "node-agent-pod-mem-request", o.NodeAgentPodMemRequest, `Memory request for node-agent pod. A value of "0" is treated as unbounded. Optional.`) flags.StringVar(&o.NodeAgentPodCPULimit, "node-agent-pod-cpu-limit", o.NodeAgentPodCPULimit, `CPU limit for node-agent pod. A value of "0" is treated as unbounded. Optional.`) flags.StringVar(&o.NodeAgentPodMemLimit, "node-agent-pod-mem-limit", o.NodeAgentPodMemLimit, `Memory limit for node-agent pod. A value of "0" is treated as unbounded. Optional.`) flags.StringVar(&o.kubeletRootDir, "kubelet-root-dir", o.kubeletRootDir, `Kubelet root directory for the node agent. Optional.`) flags.Var(&o.BackupStorageConfig, "backup-location-config", "Configuration to use for the backup storage location. Format is key1=value1,key2=value2") flags.Var(&o.VolumeSnapshotConfig, "snapshot-location-config", "Configuration to use for the volume snapshot location. Format is key1=value1,key2=value2") flags.BoolVar(&o.UseVolumeSnapshots, "use-volume-snapshots", o.UseVolumeSnapshots, "Whether or not to create snapshot location automatically. Set to false if you do not plan to create volume snapshots via a storage provider.") flags.BoolVar(&o.RestoreOnly, "restore-only", o.RestoreOnly, "Run the server in restore-only mode. Optional.") flags.BoolVar(&o.DryRun, "dry-run", o.DryRun, "Generate resources, but don't send them to the cluster. Use with -o. Optional.") flags.BoolVar(&o.UseNodeAgent, "use-node-agent", o.UseNodeAgent, "Create Velero node-agent daemonset. Optional. Velero node-agent hosts and associates Velero modules that need to run in one or more Linux nodes.") flags.BoolVar(&o.UseNodeAgentWindows, "use-node-agent-windows", o.UseNodeAgentWindows, "Create Velero node-agent-windows daemonset. Optional. Velero node-agent-windows hosts and associates Velero modules that need to run in one or more Windows nodes.") flags.BoolVar(&o.PrivilegedNodeAgent, "privileged-node-agent", o.PrivilegedNodeAgent, "Use privileged mode for the node agent. Optional. Required to backup block devices.") flags.BoolVar(&o.Wait, "wait", o.Wait, "Wait for Velero deployment to be ready. Optional.") flags.DurationVar(&o.DefaultRepoMaintenanceFrequency, "default-repo-maintain-frequency", o.DefaultRepoMaintenanceFrequency, "How often 'maintain' is run for backup repositories by default. Optional.") flags.DurationVar(&o.GarbageCollectionFrequency, "garbage-collection-frequency", o.GarbageCollectionFrequency, "How often the garbage collection runs for expired backups.(default 1h)") flags.DurationVar(&o.PodVolumeOperationTimeout, "pod-volume-operation-timeout", o.PodVolumeOperationTimeout, "How long to wait for pod volume operations to complete before timing out(default 4h). Optional.") flags.Var(&o.Plugins, "plugins", "Plugin container images to install into the Velero Deployment") flags.BoolVar(&o.CRDsOnly, "crds-only", o.CRDsOnly, "Only generate CustomResourceDefinition resources. Useful for updating CRDs for an existing Velero install.") flags.StringVar(&o.CACertFile, "cacert", o.CACertFile, "File containing a certificate bundle to use when verifying TLS connections to the object store. Optional.") flags.StringVar(&o.Features, "features", o.Features, "Comma separated list of Velero feature flags to be set on the Velero deployment and the node-agent daemonset, if node-agent is enabled") flags.BoolVar(&o.DefaultVolumesToFsBackup, "default-volumes-to-fs-backup", o.DefaultVolumesToFsBackup, "Bool flag to configure Velero server to use pod volume file system backup by default for all volumes on all backups. Optional.") flags.StringVar(&o.UploaderType, "uploader-type", o.UploaderType, fmt.Sprintf("The type of uploader to transfer the data of pod volumes, supported value: '%s'", uploader.KopiaType)) flags.BoolVar(&o.DefaultSnapshotMoveData, "default-snapshot-move-data", o.DefaultSnapshotMoveData, "Bool flag to configure Velero server to move data by default for all snapshots supporting data movement. Optional.") flags.BoolVar(&o.DisableInformerCache, "disable-informer-cache", o.DisableInformerCache, "Disable informer cache for Get calls on restore. With this enabled, it will speed up restore in cases where there are backup resources which already exist in the cluster, but for very large clusters this will increase velero memory usage. Default is false (don't disable). Optional.") flags.BoolVar(&o.ScheduleSkipImmediately, "schedule-skip-immediately", o.ScheduleSkipImmediately, "Skip the first scheduled backup immediately after creating a schedule. Default is false (don't skip).") flags.BoolVar(&o.NodeAgentDisableHostPath, "node-agent-disable-host-path", o.NodeAgentDisableHostPath, "Don't mount the pod volume host path to node-agent. Optional. Pod volume host path mount is required by fs-backup but could be disabled for other backup methods.") flags.IntVar( &o.KeepLatestMaintenanceJobs, "keep-latest-maintenance-jobs", o.KeepLatestMaintenanceJobs, "Number of latest maintenance jobs to keep each repository. Optional.", ) flags.StringVar( &o.PodResources.CPURequest, "maintenance-job-cpu-request", o.PodResources.CPURequest, "CPU request for maintenance jobs. Default is no limit.", ) flags.StringVar( &o.PodResources.MemoryRequest, "maintenance-job-mem-request", o.PodResources.MemoryRequest, "Memory request for maintenance jobs. Default is no limit.", ) flags.StringVar( &o.PodResources.CPULimit, "maintenance-job-cpu-limit", o.PodResources.CPULimit, "CPU limit for maintenance jobs. Default is no limit.", ) flags.StringVar( &o.PodResources.MemoryLimit, "maintenance-job-mem-limit", o.PodResources.MemoryLimit, "Memory limit for maintenance jobs. Default is no limit.", ) flags.StringVar( &o.BackupRepoConfigMap, "backup-repository-configmap", o.BackupRepoConfigMap, "The name of configMap containing backup repository configurations.", ) flags.StringVar( &o.RepoMaintenanceJobConfigMap, "repo-maintenance-job-configmap", o.RepoMaintenanceJobConfigMap, "The name of ConfigMap containing repository maintenance Job configurations.", ) flags.StringVar( &o.NodeAgentConfigMap, "node-agent-configmap", o.NodeAgentConfigMap, "The name of ConfigMap containing node-agent configurations.", ) flags.IntVar( &o.ItemBlockWorkerCount, "item-block-worker-count", o.ItemBlockWorkerCount, "Number of worker threads to process ItemBlocks. Default is one. Optional.", ) flags.IntVar( &o.ConcurrentBackups, "concurrent-backups", o.ConcurrentBackups, "Number of backups to process concurrently. Default is one. Optional.", ) flags.StringVar( &o.ServerPriorityClassName, "server-priority-class-name", o.ServerPriorityClassName, "Priority class name for the Velero server deployment. Optional.", ) flags.StringVar( &o.NodeAgentPriorityClassName, "node-agent-priority-class-name", o.NodeAgentPriorityClassName, "Priority class name for the node agent daemonset. Optional.", ) } // NewInstallOptions instantiates a new, default InstallOptions struct. func NewInstallOptions() *Options { return &Options{ Namespace: velerov1api.DefaultNamespace, Image: velero.DefaultVeleroImage(), BackupStorageConfig: flag.NewMap(), VolumeSnapshotConfig: flag.NewMap(), PodAnnotations: flag.NewMap(), PodLabels: flag.NewMap(), ServiceAccountAnnotations: flag.NewMap(), VeleroPodCPURequest: install.DefaultVeleroPodCPURequest, VeleroPodMemRequest: install.DefaultVeleroPodMemRequest, VeleroPodCPULimit: install.DefaultVeleroPodCPULimit, VeleroPodMemLimit: install.DefaultVeleroPodMemLimit, NodeAgentPodCPURequest: install.DefaultNodeAgentPodCPURequest, NodeAgentPodMemRequest: install.DefaultNodeAgentPodMemRequest, NodeAgentPodCPULimit: install.DefaultNodeAgentPodCPULimit, NodeAgentPodMemLimit: install.DefaultNodeAgentPodMemLimit, // Default to creating a VSL unless we're told otherwise UseVolumeSnapshots: true, NoDefaultBackupLocation: false, CRDsOnly: false, DefaultVolumesToFsBackup: false, UploaderType: uploader.KopiaType, DefaultSnapshotMoveData: false, DisableInformerCache: false, ScheduleSkipImmediately: false, kubeletRootDir: install.DefaultKubeletRootDir, NodeAgentDisableHostPath: false, } } // AsVeleroOptions translates the values provided at the command line into values used to instantiate Kubernetes resources func (o *Options) AsVeleroOptions() (*install.VeleroOptions, error) { var secretData []byte if o.SecretFile != "" && !o.NoSecret { realPath, err := filepath.Abs(o.SecretFile) if err != nil { return nil, err } secretData, err = os.ReadFile(realPath) if err != nil { return nil, err } } var caCertData []byte if o.CACertFile != "" { realPath, err := filepath.Abs(o.CACertFile) if err != nil { return nil, err } caCertData, err = os.ReadFile(realPath) if err != nil { return nil, err } } veleroPodResources, err := kubeutil.ParseCPUAndMemoryResources( o.VeleroPodCPURequest, o.VeleroPodMemRequest, o.VeleroPodCPULimit, o.VeleroPodMemLimit, ) if err != nil { return nil, err } nodeAgentPodResources, err := kubeutil.ParseCPUAndMemoryResources( o.NodeAgentPodCPURequest, o.NodeAgentPodMemRequest, o.NodeAgentPodCPULimit, o.NodeAgentPodMemLimit, ) if err != nil { return nil, err } return &install.VeleroOptions{ Namespace: o.Namespace, Image: o.Image, ProviderName: o.ProviderName, Bucket: o.BucketName, Prefix: o.Prefix, PodAnnotations: o.PodAnnotations.Data(), PodLabels: o.PodLabels.Data(), ServiceAccountAnnotations: o.ServiceAccountAnnotations.Data(), ServiceAccountName: o.ServiceAccountName, VeleroPodResources: veleroPodResources, NodeAgentPodResources: nodeAgentPodResources, SecretData: secretData, RestoreOnly: o.RestoreOnly, UseNodeAgent: o.UseNodeAgent, UseNodeAgentWindows: o.UseNodeAgentWindows, PrivilegedNodeAgent: o.PrivilegedNodeAgent, UseVolumeSnapshots: o.UseVolumeSnapshots, BSLConfig: o.BackupStorageConfig.Data(), VSLConfig: o.VolumeSnapshotConfig.Data(), DefaultRepoMaintenanceFrequency: o.DefaultRepoMaintenanceFrequency, GarbageCollectionFrequency: o.GarbageCollectionFrequency, PodVolumeOperationTimeout: o.PodVolumeOperationTimeout, Plugins: o.Plugins, NoDefaultBackupLocation: o.NoDefaultBackupLocation, CACertData: caCertData, Features: strings.Split(o.Features, ","), DefaultVolumesToFsBackup: o.DefaultVolumesToFsBackup, UploaderType: o.UploaderType, DefaultSnapshotMoveData: o.DefaultSnapshotMoveData, DisableInformerCache: o.DisableInformerCache, ScheduleSkipImmediately: o.ScheduleSkipImmediately, PodResources: o.PodResources, KeepLatestMaintenanceJobs: o.KeepLatestMaintenanceJobs, BackupRepoConfigMap: o.BackupRepoConfigMap, RepoMaintenanceJobConfigMap: o.RepoMaintenanceJobConfigMap, NodeAgentConfigMap: o.NodeAgentConfigMap, ItemBlockWorkerCount: o.ItemBlockWorkerCount, ConcurrentBackups: o.ConcurrentBackups, KubeletRootDir: o.kubeletRootDir, NodeAgentDisableHostPath: o.NodeAgentDisableHostPath, ServerPriorityClassName: o.ServerPriorityClassName, NodeAgentPriorityClassName: o.NodeAgentPriorityClassName, }, nil } // NewCommand creates a cobra command. func NewCommand(f client.Factory) *cobra.Command { o := NewInstallOptions() c := &cobra.Command{ Use: "install", Short: "Install Velero", Long: `Install Velero onto a Kubernetes cluster using the supplied provider information, such as the provider's name, a bucket name, and a file containing the credentials to access that bucket. A prefix within the bucket and configuration for the backup store location may also be supplied. Additionally, volume snapshot information for the same provider may be supplied. All required CustomResourceDefinitions will be installed to the server, as well as the Velero Deployment and associated node-agent DaemonSet. The provided secret data will be created in a Secret named 'cloud-credentials'. All namespaced resources will be placed in the 'velero' namespace by default. The '--namespace' flag can be used to specify a different namespace to install into. Use '--wait' to wait for the Velero Deployment to be ready before proceeding. Use '-o yaml' or '-o json' with '--dry-run' to output all generated resources as text instead of sending the resources to the server. This is useful as a starting point for more customized installations. `, Example: ` # velero install --provider gcp --plugins velero/velero-plugin-for-gcp:v1.0.0 --bucket mybucket --secret-file ./gcp-service-account.json # velero install --provider aws --plugins velero/velero-plugin-for-aws:v1.0.0 --bucket backups --secret-file ./aws-iam-creds --backup-location-config region=us-east-2 --snapshot-location-config region=us-east-2 # velero install --provider aws --plugins velero/velero-plugin-for-aws:v1.0.0 --bucket backups --secret-file ./aws-iam-creds --backup-location-config region=us-east-2 --snapshot-location-config region=us-east-2 --use-node-agent # velero install --provider gcp --plugins velero/velero-plugin-for-gcp:v1.0.0 --bucket gcp-backups --secret-file ./gcp-creds.json --wait # velero install --provider aws --plugins velero/velero-plugin-for-aws:v1.0.0 --bucket backups --backup-location-config region=us-west-2 --snapshot-location-config region=us-west-2 --no-secret --pod-annotations iam.amazonaws.com/role=arn:aws:iam:::role/ # velero install --provider gcp --plugins velero/velero-plugin-for-gcp:v1.0.0 --bucket gcp-backups --secret-file ./gcp-creds.json --velero-pod-cpu-request=1000m --velero-pod-cpu-limit=5000m --velero-pod-mem-request=512Mi --velero-pod-mem-limit=1024Mi # velero install --provider gcp --plugins velero/velero-plugin-for-gcp:v1.0.0 --bucket gcp-backups --secret-file ./gcp-creds.json --node-agent-pod-cpu-request=1000m --node-agent-pod-cpu-limit=5000m --node-agent-pod-mem-request=512Mi --node-agent-pod-mem-limit=1024Mi # velero install --provider azure --plugins velero/velero-plugin-for-microsoft-azure:v1.0.0 --bucket $BLOB_CONTAINER --secret-file ./credentials-velero --backup-location-config resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,storageAccount=$AZURE_STORAGE_ACCOUNT_ID[,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID] --snapshot-location-config apiTimeout=[,resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID]`, Run: func(c *cobra.Command, args []string) { cmd.CheckError(o.Validate(c, args, f)) cmd.CheckError(o.Complete(args, f)) cmd.CheckError(o.Run(c, f)) }, } o.BindFlags(c.Flags()) output.BindFlags(c.Flags()) output.ClearOutputFlagDefault(c) return c } // Run executes a command in the context of the provided arguments. func (o *Options) Run(c *cobra.Command, f client.Factory) error { var resources *unstructured.UnstructuredList if o.CRDsOnly { resources = install.AllCRDs() } else { vo, err := o.AsVeleroOptions() if err != nil { return err } resources = install.AllResources(vo) } if _, err := output.PrintWithFormat(c, resources); err != nil { return err } if o.DryRun { return nil } dynamicClient, err := f.DynamicClient() if err != nil { return err } dynamicFactory := client.NewDynamicFactory(dynamicClient) kbClient, err := f.KubebuilderClient() if err != nil { return err } errorMsg := fmt.Sprintf("\n\nError installing Velero. Use `kubectl logs deploy/velero -n %s` to check the deploy logs", o.Namespace) err = install.Install(dynamicFactory, kbClient, resources, os.Stdout, o.Apply) if err != nil { return errors.Wrap(err, errorMsg) } if o.Wait { fmt.Println("Waiting for Velero deployment to be ready.") if _, err = install.DeploymentIsReady(dynamicFactory, o.Namespace); err != nil { return errors.Wrap(err, errorMsg) } if o.UseNodeAgent { fmt.Println("Waiting for node-agent daemonset to be ready.") if _, err = install.NodeAgentIsReady(dynamicFactory, o.Namespace); err != nil { return errors.Wrap(err, errorMsg) } } if o.UseNodeAgentWindows { fmt.Println("Waiting for node-agent-windows daemonset to be ready.") if _, err = install.NodeAgentWindowsIsReady(dynamicFactory, o.Namespace); err != nil { return errors.Wrap(err, errorMsg) } } } if o.SecretFile == "" { fmt.Printf("\nNo secret file was specified, no Secret created.\n\n") } if o.NoDefaultBackupLocation { fmt.Printf("\nNo bucket and provider were specified, no default backup storage location created.\n\n") } fmt.Printf("Velero is installed! ⛵ Use 'kubectl logs deployment/velero -n %s' to view the status.\n", o.Namespace) return nil } // Complete completes options for a command. func (o *Options) Complete(args []string, f client.Factory) error { o.Namespace = f.Namespace() return nil } // Validate validates options provided to a command. func (o *Options) Validate(c *cobra.Command, args []string, f client.Factory) error { if err := output.ValidateFlags(c); err != nil { return err } // If we're only installing CRDs, we can skip the rest of the validation. if o.CRDsOnly { return nil } if msg, err := uploader.ValidateUploaderType(o.UploaderType); err != nil { return err } else if msg != "" { fmt.Printf("⚠️ %s\n", msg) } // Our main 3 providers don't support bucket names starting with a dash, and a bucket name starting with one // can indicate that an environment variable was left blank. // This case will help catch that error if strings.HasPrefix(o.BucketName, "-") { return errors.Errorf("Bucket names cannot begin with a dash. Bucket name was: %s", o.BucketName) } if o.NoDefaultBackupLocation { if o.BucketName != "" { return errors.New("Cannot use both --bucket and --no-default-backup-location at the same time") } if o.Prefix != "" { return errors.New("Cannot use both --prefix and --no-default-backup-location at the same time") } if o.BackupStorageConfig.String() != "" { return errors.New("Cannot use both --backup-location-config and --no-default-backup-location at the same time") } } else { if o.ProviderName == "" { return errors.New("--provider is required") } if o.BucketName == "" { return errors.New("--bucket is required") } } if o.UseVolumeSnapshots { if o.ProviderName == "" { return errors.New("--provider is required when --use-volume-snapshots is set to true") } } else { if o.VolumeSnapshotConfig.String() != "" { return errors.New("--snapshot-location-config must be empty when --use-volume-snapshots=false") } } if o.NoDefaultBackupLocation && !o.UseVolumeSnapshots { if o.ProviderName != "" { return errors.New("--provider must be empty when using --no-default-backup-location and --use-volume-snapshots=false") } } else { if len(o.Plugins) == 0 { return errors.New("--plugins flag is required") } } if o.DefaultVolumesToFsBackup && !o.UseNodeAgent { return errors.New("--use-node-agent is required when using --default-volumes-to-fs-backup") } switch { case o.SecretFile == "" && !o.NoSecret: return errors.New("One of --secret-file or --no-secret is required") case o.SecretFile != "" && o.NoSecret: return errors.New("Cannot use both --secret-file and --no-secret") } if o.DefaultRepoMaintenanceFrequency < 0 { return errors.New("--default-repo-maintain-frequency must be non-negative") } if o.GarbageCollectionFrequency < 0 { return errors.New("--garbage-collection-frequency must be non-negative") } if o.PodVolumeOperationTimeout < 0 { return errors.New("--pod-volume-operation-timeout must be non-negative") } crClient, err := f.KubebuilderClient() if err != nil { return fmt.Errorf("fail to create go-client %w", err) } if len(o.NodeAgentConfigMap) > 0 { if err := kubeutil.VerifyJSONConfigs(c.Context(), o.Namespace, crClient, o.NodeAgentConfigMap, &velerotypes.NodeAgentConfigs{}); err != nil { return fmt.Errorf("--node-agent-configmap specified ConfigMap %s is invalid: %w", o.NodeAgentConfigMap, err) } } if len(o.RepoMaintenanceJobConfigMap) > 0 { if err := kubeutil.VerifyJSONConfigs(c.Context(), o.Namespace, crClient, o.RepoMaintenanceJobConfigMap, &velerotypes.JobConfigs{}); err != nil { return fmt.Errorf("--repo-maintenance-job-configmap specified ConfigMap %s is invalid: %w", o.RepoMaintenanceJobConfigMap, err) } } if len(o.BackupRepoConfigMap) > 0 { config := make(map[string]any) if err := kubeutil.VerifyJSONConfigs(c.Context(), o.Namespace, crClient, o.BackupRepoConfigMap, &config); err != nil { return fmt.Errorf("--backup-repository-configmap specified ConfigMap %s is invalid: %w", o.BackupRepoConfigMap, err) } } return nil } ================================================ FILE: pkg/cmd/cli/install/install_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package install import ( "testing" "github.com/spf13/pflag" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestPriorityClassNameFlag(t *testing.T) { // Test that the flag is properly defined o := NewInstallOptions() flags := pflag.NewFlagSet("test", pflag.ContinueOnError) o.BindFlags(flags) // Verify the server priority class flag exists serverFlag := flags.Lookup("server-priority-class-name") assert.NotNil(t, serverFlag, "server-priority-class-name flag should exist") assert.Equal(t, "Priority class name for the Velero server deployment. Optional.", serverFlag.Usage) // Verify the node agent priority class flag exists nodeAgentFlag := flags.Lookup("node-agent-priority-class-name") assert.NotNil(t, nodeAgentFlag, "node-agent-priority-class-name flag should exist") assert.Equal(t, "Priority class name for the node agent daemonset. Optional.", nodeAgentFlag.Usage) // Test with values for both server and node agent testCases := []struct { name string serverPriorityClassName string nodeAgentPriorityClassName string expectedServerValue string expectedNodeAgentValue string }{ { name: "with both priority class names", serverPriorityClassName: "high-priority", nodeAgentPriorityClassName: "medium-priority", expectedServerValue: "high-priority", expectedNodeAgentValue: "medium-priority", }, { name: "with only server priority class name", serverPriorityClassName: "high-priority", nodeAgentPriorityClassName: "", expectedServerValue: "high-priority", expectedNodeAgentValue: "", }, { name: "with only node agent priority class name", serverPriorityClassName: "", nodeAgentPriorityClassName: "medium-priority", expectedServerValue: "", expectedNodeAgentValue: "medium-priority", }, { name: "without priority class names", serverPriorityClassName: "", nodeAgentPriorityClassName: "", expectedServerValue: "", expectedNodeAgentValue: "", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { o := NewInstallOptions() o.ServerPriorityClassName = tc.serverPriorityClassName o.NodeAgentPriorityClassName = tc.nodeAgentPriorityClassName veleroOptions, err := o.AsVeleroOptions() require.NoError(t, err) assert.Equal(t, tc.expectedServerValue, veleroOptions.ServerPriorityClassName) assert.Equal(t, tc.expectedNodeAgentValue, veleroOptions.NodeAgentPriorityClassName) }) } } ================================================ FILE: pkg/cmd/cli/nodeagent/node_agent.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package nodeagent import ( "github.com/spf13/cobra" "github.com/vmware-tanzu/velero/pkg/client" ) func NewCommand(f client.Factory) *cobra.Command { c := &cobra.Command{ Use: "node-agent", Short: "Work with node-agent", Long: "Work with node-agent", } c.AddCommand( NewServerCommand(f), ) return c } ================================================ FILE: pkg/cmd/cli/nodeagent/server.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package nodeagent import ( "context" "fmt" "math" "net/http" "os" "strings" "time" "github.com/bombsimon/logrusr/v3" snapshotv1client "github.com/kubernetes-csi/external-snapshotter/client/v8/clientset/versioned" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/sirupsen/logrus" "github.com/spf13/cobra" corev1api "k8s.io/api/core/v1" storagev1api "k8s.io/api/storage/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/kubernetes" cacheutil "k8s.io/client-go/tools/cache" "k8s.io/klog/v2" "k8s.io/utils/clock" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" "github.com/vmware-tanzu/velero/pkg/buildinfo" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" "github.com/vmware-tanzu/velero/pkg/cmd/util/signals" "github.com/vmware-tanzu/velero/pkg/constant" "github.com/vmware-tanzu/velero/pkg/controller" "github.com/vmware-tanzu/velero/pkg/datapath" "github.com/vmware-tanzu/velero/pkg/exposer" "github.com/vmware-tanzu/velero/pkg/metrics" "github.com/vmware-tanzu/velero/pkg/nodeagent" repository "github.com/vmware-tanzu/velero/pkg/repository/manager" velerotypes "github.com/vmware-tanzu/velero/pkg/types" "github.com/vmware-tanzu/velero/pkg/util/filesystem" "github.com/vmware-tanzu/velero/pkg/util/kube" "github.com/vmware-tanzu/velero/pkg/util/logging" ) var ( scheme = runtime.NewScheme() ) const ( // the port where prometheus metrics are exposed defaultMetricsAddress = ":8085" defaultResourceTimeout = 10 * time.Minute defaultDataMoverPrepareTimeout = 30 * time.Minute defaultDataPathConcurrentNum = 1 ) type nodeAgentServerConfig struct { metricsAddress string resourceTimeout time.Duration dataMoverPrepareTimeout time.Duration nodeAgentConfig string backupRepoConfig string } func NewServerCommand(f client.Factory) *cobra.Command { logLevelFlag := logging.LogLevelFlag(logrus.InfoLevel) formatFlag := logging.NewFormatFlag() config := nodeAgentServerConfig{ metricsAddress: defaultMetricsAddress, resourceTimeout: defaultResourceTimeout, dataMoverPrepareTimeout: defaultDataMoverPrepareTimeout, } command := &cobra.Command{ Use: "server", Short: "Run the velero node-agent server", Long: "Run the velero node-agent server", Hidden: true, Run: func(c *cobra.Command, args []string) { logLevel := logLevelFlag.Parse() logrus.Infof("Setting log-level to %s", strings.ToUpper(logLevel.String())) logger := logging.DefaultMergeLogger(logLevel, formatFlag.Parse()) logger.Infof("Starting Velero node-agent server %s (%s)", buildinfo.Version, buildinfo.FormattedGitSHA()) f.SetBasename(fmt.Sprintf("%s-%s", c.Parent().Name(), c.Name())) s, err := newNodeAgentServer(logger, f, config) cmd.CheckError(err) s.run() }, } command.Flags().Var(logLevelFlag, "log-level", fmt.Sprintf("The level at which to log. Valid values are %s.", strings.Join(logLevelFlag.AllowedValues(), ", "))) command.Flags().Var(formatFlag, "log-format", fmt.Sprintf("The format for log output. Valid values are %s.", strings.Join(formatFlag.AllowedValues(), ", "))) command.Flags().DurationVar(&config.resourceTimeout, "resource-timeout", config.resourceTimeout, "How long to wait for resource processes which are not covered by other specific timeout parameters. Default is 10 minutes.") command.Flags().DurationVar(&config.dataMoverPrepareTimeout, "data-mover-prepare-timeout", config.dataMoverPrepareTimeout, "How long to wait for preparing a DataUpload/DataDownload. Default is 30 minutes.") command.Flags().StringVar(&config.metricsAddress, "metrics-address", config.metricsAddress, "The address to expose prometheus metrics") command.Flags().StringVar(&config.nodeAgentConfig, "node-agent-configmap", config.nodeAgentConfig, "The name of ConfigMap containing node-agent configurations.") command.Flags().StringVar(&config.backupRepoConfig, "backup-repository-configmap", config.backupRepoConfig, "The name of ConfigMap containing backup repository configurations.") return command } type nodeAgentServer struct { logger logrus.FieldLogger ctx context.Context cancelFunc context.CancelFunc fileSystem filesystem.Interface mgr manager.Manager metrics *metrics.ServerMetrics metricsAddress string namespace string nodeName string config nodeAgentServerConfig kubeClient kubernetes.Interface csiSnapshotClient *snapshotv1client.Clientset dataPathMgr *datapath.Manager dataPathConfigs *velerotypes.NodeAgentConfigs backupRepoConfigs map[string]string vgdpCounter *exposer.VgdpCounter repoConfigMgr repository.ConfigManager } func newNodeAgentServer(logger logrus.FieldLogger, factory client.Factory, config nodeAgentServerConfig) (*nodeAgentServer, error) { ctx, cancelFunc := context.WithCancel(context.Background()) clientConfig, err := factory.ClientConfig() if err != nil { cancelFunc() return nil, err } ctrl.SetLogger(logrusr.New(logger)) klog.SetLogger(logrusr.New(logger)) // klog.Logger is used by k8s.io/client-go if err := velerov1api.AddToScheme(scheme); err != nil { cancelFunc() return nil, err } if err := velerov2alpha1api.AddToScheme(scheme); err != nil { cancelFunc() return nil, err } if err := corev1api.AddToScheme(scheme); err != nil { cancelFunc() return nil, err } if err := storagev1api.AddToScheme(scheme); err != nil { cancelFunc() return nil, err } nodeName := os.Getenv("NODE_NAME") // use a field selector to filter to only pods scheduled on this node. cacheOption := cache.Options{ ByObject: map[ctrlclient.Object]cache.ByObject{ &corev1api.Pod{}: { Field: fields.Set{"spec.nodeName": nodeName}.AsSelector(), }, &velerov1api.PodVolumeBackup{}: { Field: fields.Set{"metadata.namespace": factory.Namespace()}.AsSelector(), }, &velerov1api.PodVolumeRestore{}: { Field: fields.Set{"metadata.namespace": factory.Namespace()}.AsSelector(), }, &velerov2alpha1api.DataUpload{}: { Field: fields.Set{"metadata.namespace": factory.Namespace()}.AsSelector(), }, &velerov2alpha1api.DataDownload{}: { Field: fields.Set{"metadata.namespace": factory.Namespace()}.AsSelector(), }, &corev1api.Event{}: { Field: fields.Set{"metadata.namespace": factory.Namespace()}.AsSelector(), }, }, } var mgr manager.Manager retry := 10 for { mgr, err = ctrl.NewManager(clientConfig, ctrl.Options{ Scheme: scheme, Cache: cacheOption, }) if err == nil { break } retry-- if retry == 0 { break } logger.WithError(err).Warn("Failed to create controller manager, need retry") time.Sleep(time.Second) } if err != nil { cancelFunc() return nil, errors.Wrap(err, "error creating controller manager") } s := &nodeAgentServer{ logger: logger, ctx: ctx, cancelFunc: cancelFunc, fileSystem: filesystem.NewFileSystem(), mgr: mgr, config: config, namespace: factory.Namespace(), nodeName: nodeName, metricsAddress: config.metricsAddress, repoConfigMgr: repository.NewConfigManager(logger), } // the cache isn't initialized yet when "validatePodVolumesHostPath" is called, the client returned by the manager cannot // be used, so we need the kube client here s.kubeClient, err = factory.KubeClient() if err != nil { return nil, err } if err := s.validatePodVolumesHostPath(s.kubeClient); err != nil { return nil, err } s.csiSnapshotClient, err = snapshotv1client.NewForConfig(clientConfig) if err != nil { return nil, err } if err := s.getDataPathConfigs(); err != nil { return nil, err } if err := s.getBackupRepoConfigs(); err != nil { return nil, err } s.dataPathMgr = datapath.NewManager(s.getDataPathConcurrentNum(defaultDataPathConcurrentNum)) return s, nil } func (s *nodeAgentServer) run() { signals.CancelOnShutdown(s.cancelFunc, s.logger) go func() { metricsMux := http.NewServeMux() metricsMux.Handle("/metrics", promhttp.Handler()) s.logger.Infof("Starting metric server for node agent at address [%s]", s.metricsAddress) server := &http.Server{ Addr: s.metricsAddress, Handler: metricsMux, ReadHeaderTimeout: 3 * time.Second, } if err := server.ListenAndServe(); err != nil { s.logger.Fatalf("Failed to start metric server for node agent at [%s]: %v", s.metricsAddress, err) } }() s.metrics = metrics.NewNodeMetrics() s.metrics.RegisterAllMetrics() s.metrics.InitMetricsForNode(s.nodeName) s.logger.Info("Starting controllers") // Get priority class from dataPathConfigs if available dataMovePriorityClass := "" if s.dataPathConfigs != nil && s.dataPathConfigs.PriorityClassName != "" { priorityClass := s.dataPathConfigs.PriorityClassName // Validate the priority class exists in the cluster ctx, cancel := context.WithTimeout(s.ctx, time.Second*30) defer cancel() if kube.ValidatePriorityClass(ctx, s.kubeClient, priorityClass, s.logger.WithField("component", "data-mover")) { dataMovePriorityClass = priorityClass s.logger.WithField("priorityClassName", priorityClass).Info("Using priority class for data mover pods") } else { s.logger.WithField("priorityClassName", priorityClass).Warn("Priority class not found in cluster, data mover pods will use default priority") } } var loadAffinity []*kube.LoadAffinity if s.dataPathConfigs != nil && len(s.dataPathConfigs.LoadAffinity) > 0 { loadAffinity = s.dataPathConfigs.LoadAffinity s.logger.Infof("Using customized loadAffinity %v", loadAffinity) } var backupPVCConfig map[string]velerotypes.BackupPVC if s.dataPathConfigs != nil && s.dataPathConfigs.BackupPVCConfig != nil { backupPVCConfig = s.dataPathConfigs.BackupPVCConfig s.logger.Infof("Using customized backupPVC config %v", backupPVCConfig) } privilegedFsBackup := s.dataPathConfigs != nil && s.dataPathConfigs.PrivilegedFsBackup podResources := corev1api.ResourceRequirements{} if s.dataPathConfigs != nil && s.dataPathConfigs.PodResources != nil { // To make the PodResources ConfigMap without ephemeral storage request/limit backward compatible, // need to avoid set value as empty, because empty string will cause parsing error. ephemeralStorageRequest := constant.DefaultEphemeralStorageRequest if s.dataPathConfigs.PodResources.EphemeralStorageRequest != "" { ephemeralStorageRequest = s.dataPathConfigs.PodResources.EphemeralStorageRequest } ephemeralStorageLimit := constant.DefaultEphemeralStorageLimit if s.dataPathConfigs.PodResources.EphemeralStorageLimit != "" { ephemeralStorageLimit = s.dataPathConfigs.PodResources.EphemeralStorageLimit } if res, err := kube.ParseResourceRequirements( s.dataPathConfigs.PodResources.CPURequest, s.dataPathConfigs.PodResources.MemoryRequest, ephemeralStorageRequest, s.dataPathConfigs.PodResources.CPULimit, s.dataPathConfigs.PodResources.MemoryLimit, ephemeralStorageLimit, ); err != nil { s.logger.WithError(err).Warn("Pod resource requirements are invalid, ignore") } else { podResources = res s.logger.Infof("Using customized pod resource requirements %v", s.dataPathConfigs.PodResources) } } if s.dataPathConfigs != nil && s.dataPathConfigs.LoadConcurrency != nil && s.dataPathConfigs.LoadConcurrency.PrepareQueueLength > 0 { if counter, err := exposer.StartVgdpCounter(s.ctx, s.mgr, s.dataPathConfigs.LoadConcurrency.PrepareQueueLength); err != nil { s.logger.WithError(err).Warnf("Failed to start VGDP counter, VDGP loads are not constrained") } else { s.vgdpCounter = counter s.logger.Infof("VGDP loads are constrained with %d", s.dataPathConfigs.LoadConcurrency.PrepareQueueLength) } } var cachePVCConfig *velerotypes.CachePVC if s.dataPathConfigs != nil && s.dataPathConfigs.CachePVCConfig != nil { if err := s.validateCachePVCConfig(*s.dataPathConfigs.CachePVCConfig); err != nil { s.logger.WithError(err).Warnf("Ignore cache config %v", s.dataPathConfigs.CachePVCConfig) } else { cachePVCConfig = s.dataPathConfigs.CachePVCConfig s.logger.Infof("Using cache volume configs %v", s.dataPathConfigs.CachePVCConfig) } } var podLabels map[string]string if s.dataPathConfigs != nil && len(s.dataPathConfigs.PodLabels) > 0 { podLabels = s.dataPathConfigs.PodLabels s.logger.Infof("Using customized pod labels %+v", podLabels) } var podAnnotations map[string]string if s.dataPathConfigs != nil && len(s.dataPathConfigs.PodAnnotations) > 0 { podAnnotations = s.dataPathConfigs.PodAnnotations s.logger.Infof("Using customized pod annotations %+v", podAnnotations) } if s.backupRepoConfigs != nil { s.logger.Infof("Using backup repo config %v", s.backupRepoConfigs) } else if cachePVCConfig != nil { s.logger.Info("Backup repo config is not provided, using default values for cache volume configs") } pvbReconciler := controller.NewPodVolumeBackupReconciler( s.mgr.GetClient(), s.mgr, s.kubeClient, s.dataPathMgr, s.vgdpCounter, s.nodeName, s.config.dataMoverPrepareTimeout, s.config.resourceTimeout, podResources, s.metrics, s.logger, dataMovePriorityClass, privilegedFsBackup, podLabels, podAnnotations, ) if err := pvbReconciler.SetupWithManager(s.mgr); err != nil { s.logger.Fatal(err, "unable to create controller", "controller", constant.ControllerPodVolumeBackup) } pvrReconciler := controller.NewPodVolumeRestoreReconciler( s.mgr.GetClient(), s.mgr, s.kubeClient, s.dataPathMgr, s.vgdpCounter, s.nodeName, s.config.dataMoverPrepareTimeout, s.config.resourceTimeout, s.backupRepoConfigs, cachePVCConfig, podResources, s.logger, dataMovePriorityClass, privilegedFsBackup, s.repoConfigMgr, podLabels, podAnnotations, ) if err := pvrReconciler.SetupWithManager(s.mgr); err != nil { s.logger.WithError(err).Fatal("Unable to create the pod volume restore controller") } if err := controller.InitLegacyPodVolumeRestoreReconciler(s.mgr.GetClient(), s.mgr, s.kubeClient, s.dataPathMgr, s.namespace, s.config.resourceTimeout, s.logger); err != nil { s.logger.WithError(err).Fatal("Unable to create the legacy pod volume restore controller") } dataUploadReconciler := controller.NewDataUploadReconciler( s.mgr.GetClient(), s.mgr, s.kubeClient, s.csiSnapshotClient.SnapshotV1(), s.dataPathMgr, s.vgdpCounter, loadAffinity, backupPVCConfig, podResources, clock.RealClock{}, s.nodeName, s.config.dataMoverPrepareTimeout, s.logger, s.metrics, dataMovePriorityClass, podLabels, podAnnotations, ) if err := dataUploadReconciler.SetupWithManager(s.mgr); err != nil { s.logger.WithError(err).Fatal("Unable to create the data upload controller") } var restorePVCConfig velerotypes.RestorePVC if s.dataPathConfigs != nil && s.dataPathConfigs.RestorePVCConfig != nil { restorePVCConfig = *s.dataPathConfigs.RestorePVCConfig s.logger.Infof("Using customized restorePVC config %v", restorePVCConfig) } dataDownloadReconciler := controller.NewDataDownloadReconciler( s.mgr.GetClient(), s.mgr, s.kubeClient, s.dataPathMgr, s.vgdpCounter, loadAffinity, restorePVCConfig, s.backupRepoConfigs, cachePVCConfig, podResources, s.nodeName, s.config.dataMoverPrepareTimeout, s.logger, s.metrics, dataMovePriorityClass, s.repoConfigMgr, podLabels, podAnnotations, ) if err := dataDownloadReconciler.SetupWithManager(s.mgr); err != nil { s.logger.WithError(err).Fatal("Unable to create the data download controller") } go func() { if err := s.waitCacheForResume(); err != nil { s.logger.WithError(err).Error("Failed to wait cache for resume, will not resume DU/DD") return } if err := dataUploadReconciler.AttemptDataUploadResume(s.ctx, s.logger.WithField("node", s.nodeName), s.namespace); err != nil { s.logger.WithError(errors.WithStack(err)).Error("Failed to attempt data upload resume") } if err := dataDownloadReconciler.AttemptDataDownloadResume(s.ctx, s.logger.WithField("node", s.nodeName), s.namespace); err != nil { s.logger.WithError(errors.WithStack(err)).Error("Failed to attempt data download resume") } if err := pvbReconciler.AttemptPVBResume(s.ctx, s.logger.WithField("node", s.nodeName), s.namespace); err != nil { s.logger.WithError(errors.WithStack(err)).Error("Failed to attempt PVB resume") } if err := pvrReconciler.AttemptPVRResume(s.ctx, s.logger.WithField("node", s.nodeName), s.namespace); err != nil { s.logger.WithError(errors.WithStack(err)).Error("Failed to attempt PVR resume") } s.markLegacyPVRsFailed(s.mgr.GetClient()) }() s.logger.Info("Controllers starting...") if err := s.mgr.Start(ctrl.SetupSignalHandler()); err != nil { s.logger.Fatal("Problem starting manager", err) } } func (s *nodeAgentServer) waitCacheForResume() error { podInformer, err := s.mgr.GetCache().GetInformer(s.ctx, &corev1api.Pod{}) if err != nil { return errors.Wrap(err, "error getting pod informer") } duInformer, err := s.mgr.GetCache().GetInformer(s.ctx, &velerov2alpha1api.DataUpload{}) if err != nil { return errors.Wrap(err, "error getting du informer") } ddInformer, err := s.mgr.GetCache().GetInformer(s.ctx, &velerov2alpha1api.DataDownload{}) if err != nil { return errors.Wrap(err, "error getting dd informer") } pvbInformer, err := s.mgr.GetCache().GetInformer(s.ctx, &velerov1api.PodVolumeBackup{}) if err != nil { return errors.Wrap(err, "error getting PVB informer") } pvrInformer, err := s.mgr.GetCache().GetInformer(s.ctx, &velerov1api.PodVolumeRestore{}) if err != nil { return errors.Wrap(err, "error getting PVR informer") } if !cacheutil.WaitForCacheSync(s.ctx.Done(), podInformer.HasSynced, duInformer.HasSynced, ddInformer.HasSynced, pvbInformer.HasSynced, pvrInformer.HasSynced) { return errors.New("error waiting informer synced") } return nil } // validatePodVolumesHostPath validates that the pod volumes path contains a // directory for each Pod running on this node func (s *nodeAgentServer) validatePodVolumesHostPath(client kubernetes.Interface) error { files, err := s.fileSystem.ReadDir(nodeagent.HostPodVolumeMountPath()) if err != nil { if errors.Is(err, os.ErrNotExist) { s.logger.Warnf("Pod volumes host path [%s] doesn't exist, fs-backup is disabled", nodeagent.HostPodVolumeMountPath()) return nil } return errors.Wrap(err, "could not read pod volumes host path") } // create a map of directory names inside the pod volumes path dirs := sets.NewString() for _, f := range files { if f.IsDir() { dirs.Insert(f.Name()) } } pods, err := client.CoreV1().Pods("").List(s.ctx, metav1.ListOptions{FieldSelector: fmt.Sprintf("spec.nodeName=%s,status.phase=Running", s.nodeName)}) if err != nil { return errors.WithStack(err) } valid := true for _, pod := range pods.Items { dirName := string(pod.GetUID()) // if the pod is a mirror pod, the directory name is the hash value of the // mirror pod annotation if hash, ok := pod.GetAnnotations()[corev1api.MirrorPodAnnotationKey]; ok { dirName = hash } if !dirs.Has(dirName) { valid = false s.logger.WithFields(logrus.Fields{ "pod": fmt.Sprintf("%s/%s", pod.GetNamespace(), pod.GetName()), "path": nodeagent.HostPodVolumeMountPath() + "/" + dirName, }).Debug("could not find volumes for pod in host path") } } if !valid { return errors.New("unexpected directory structure for host-pods volume, ensure that the host-pods volume corresponds to the pods subdirectory of the kubelet root directory") } return nil } func (s *nodeAgentServer) markLegacyPVRsFailed(client ctrlclient.Client) { pvrs := &velerov1api.PodVolumeRestoreList{} if err := client.List(s.ctx, pvrs, &ctrlclient.ListOptions{Namespace: s.namespace}); err != nil { s.logger.WithError(errors.WithStack(err)).Error("failed to list podvolumerestores") return } for i, pvr := range pvrs.Items { if !controller.IsLegacyPVR(&pvr) { continue } if pvr.Status.Phase != velerov1api.PodVolumeRestorePhaseInProgress { s.logger.Debugf("the status of podvolumerestore %q is %q, skip", pvr.GetName(), pvr.Status.Phase) continue } pod := &corev1api.Pod{} if err := client.Get(s.ctx, types.NamespacedName{ Namespace: pvr.Spec.Pod.Namespace, Name: pvr.Spec.Pod.Name, }, pod); err != nil { s.logger.WithError(errors.WithStack(err)).Errorf("failed to get pod \"%s/%s\" of podvolumerestore %q", pvr.Spec.Pod.Namespace, pvr.Spec.Pod.Name, pvr.GetName()) continue } if pod.Spec.NodeName != s.nodeName { s.logger.Debugf("the node of pod referenced by podvolumerestore %q is %q, not %q, skip", pvr.GetName(), pod.Spec.NodeName, s.nodeName) continue } if err := controller.UpdatePVRStatusToFailed(s.ctx, client, &pvrs.Items[i], errors.New("cannot survive from node-agent restart"), fmt.Sprintf("get a legacy podvolumerestore with status %q during the server starting, mark it as %q", velerov1api.PodVolumeRestorePhaseInProgress, velerov1api.PodVolumeRestorePhaseFailed), time.Now(), s.logger); err != nil { s.logger.WithError(errors.WithStack(err)).Errorf("failed to patch podvolumerestore %q", pvr.GetName()) continue } s.logger.WithField("podvolumerestore", pvr.GetName()).Warn(pvr.Status.Message) } } var getConfigsFunc = nodeagent.GetConfigs func (s *nodeAgentServer) getDataPathConfigs() error { if s.config.nodeAgentConfig == "" { s.logger.Info("No node-agent configMap is specified") return nil } configs, err := getConfigsFunc(s.ctx, s.namespace, s.kubeClient, s.config.nodeAgentConfig) if err != nil { return errors.Wrapf(err, "error getting node agent configs from configMap %s", s.config.nodeAgentConfig) } s.dataPathConfigs = configs return nil } func (s *nodeAgentServer) getBackupRepoConfigs() error { if s.config.backupRepoConfig == "" { s.logger.Info("No backup repo configMap is specified") return nil } cm, err := s.kubeClient.CoreV1().ConfigMaps(s.namespace).Get(s.ctx, s.config.backupRepoConfig, metav1.GetOptions{}) if err != nil { return errors.Wrapf(err, "error getting backup repo configs from configMap %s", s.config.backupRepoConfig) } if cm.Data == nil { return errors.Errorf("no data is in the backup repo configMap %s", s.config.backupRepoConfig) } s.backupRepoConfigs = cm.Data return nil } func (s *nodeAgentServer) getDataPathConcurrentNum(defaultNum int) int { configs := s.dataPathConfigs if configs == nil || configs.LoadConcurrency == nil { s.logger.Infof("Concurrency configs are not found, use the default number %v", defaultNum) return defaultNum } globalNum := configs.LoadConcurrency.GlobalConfig if globalNum <= 0 { s.logger.Warnf("Global number %v is invalid, use the default value %v", globalNum, defaultNum) globalNum = defaultNum } if len(configs.LoadConcurrency.PerNodeConfig) == 0 { return globalNum } curNode, err := s.kubeClient.CoreV1().Nodes().Get(s.ctx, s.nodeName, metav1.GetOptions{}) if err != nil { s.logger.WithError(err).Warnf("Failed to get node info for %s, use the global number %v", s.nodeName, globalNum) return globalNum } concurrentNum := math.MaxInt32 for _, rule := range configs.LoadConcurrency.PerNodeConfig { selector, err := metav1.LabelSelectorAsSelector(&(rule.NodeSelector)) if err != nil { s.logger.WithError(err).Warnf("Failed to parse rule with label selector %s, skip it", rule.NodeSelector.String()) continue } if rule.Number <= 0 { s.logger.Warnf("Rule with label selector %s is with an invalid number %v, skip it", rule.NodeSelector.String(), rule.Number) continue } if selector.Matches(labels.Set(curNode.GetLabels())) { if concurrentNum > rule.Number { concurrentNum = rule.Number } } } if concurrentNum == math.MaxInt32 { s.logger.Infof("Per node number for node %s is not found, use the global number %v", s.nodeName, globalNum) concurrentNum = globalNum } else { s.logger.Infof("Use the per node number %v over global number %v for node %s", concurrentNum, globalNum, s.nodeName) } return concurrentNum } func (s *nodeAgentServer) validateCachePVCConfig(config velerotypes.CachePVC) error { if config.StorageClass == "" { return errors.New("storage class is absent") } sc, err := s.kubeClient.StorageV1().StorageClasses().Get(s.ctx, config.StorageClass, metav1.GetOptions{}) if err != nil { return errors.Wrapf(err, "error getting storage class %s", config.StorageClass) } if sc.ReclaimPolicy != nil && *sc.ReclaimPolicy != corev1api.PersistentVolumeReclaimDelete { return errors.Errorf("unexpected storage class reclaim policy %v", *sc.ReclaimPolicy) } return nil } ================================================ FILE: pkg/cmd/cli/nodeagent/server_test.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package nodeagent import ( "context" "errors" "fmt" "os" "path/filepath" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/nodeagent" testutil "github.com/vmware-tanzu/velero/pkg/test" velerotypes "github.com/vmware-tanzu/velero/pkg/types" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) func Test_validatePodVolumesHostPath(t *testing.T) { tests := []struct { name string pods []*corev1api.Pod dirs []string createDir bool wantErr bool }{ { name: "no error when pod volumes are present", pods: []*corev1api.Pod{ builder.ForPod("foo", "bar").ObjectMeta(builder.WithUID("foo")).Result(), builder.ForPod("zoo", "raz").ObjectMeta(builder.WithUID("zoo")).Result(), }, dirs: []string{"foo", "zoo"}, createDir: true, wantErr: false, }, { name: "no error when pod volumes are present and there are mirror pods", pods: []*corev1api.Pod{ builder.ForPod("foo", "bar").ObjectMeta(builder.WithUID("foo")).Result(), builder.ForPod("zoo", "raz").ObjectMeta(builder.WithUID("zoo"), builder.WithAnnotations(corev1api.MirrorPodAnnotationKey, "baz")).Result(), }, dirs: []string{"foo", "baz"}, createDir: true, wantErr: false, }, { name: "error when all pod volumes missing", pods: []*corev1api.Pod{ builder.ForPod("foo", "bar").ObjectMeta(builder.WithUID("foo")).Result(), builder.ForPod("zoo", "raz").ObjectMeta(builder.WithUID("zoo")).Result(), }, dirs: []string{"unexpected-dir"}, createDir: true, wantErr: true, }, { name: "error when some pod volumes missing", pods: []*corev1api.Pod{ builder.ForPod("foo", "bar").ObjectMeta(builder.WithUID("foo")).Result(), builder.ForPod("zoo", "raz").ObjectMeta(builder.WithUID("zoo")).Result(), }, dirs: []string{"foo"}, createDir: true, wantErr: true, }, { name: "no error when pod volumes are not present", pods: []*corev1api.Pod{ builder.ForPod("foo", "bar").ObjectMeta(builder.WithUID("foo")).Result(), }, dirs: []string{"foo"}, createDir: false, wantErr: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { fs := testutil.NewFakeFileSystem() for _, dir := range tt.dirs { if tt.createDir { err := fs.MkdirAll(filepath.Join(nodeagent.HostPodVolumeMountPath(), dir), os.ModePerm) if err != nil { t.Error(err) } } } kubeClient := fake.NewSimpleClientset() for _, pod := range tt.pods { _, err := kubeClient.CoreV1().Pods(pod.GetNamespace()).Create(t.Context(), pod, metav1.CreateOptions{}) if err != nil { t.Error(err) } } s := &nodeAgentServer{ logger: testutil.NewLogger(), fileSystem: fs, } err := s.validatePodVolumesHostPath(kubeClient) if tt.wantErr { assert.Error(t, err) } else { assert.NoError(t, err) } }) } } func Test_getDataPathConfigs(t *testing.T) { configs := &velerotypes.NodeAgentConfigs{ LoadConcurrency: &velerotypes.LoadConcurrency{ GlobalConfig: -1, }, } tests := []struct { name string getFunc func(context.Context, string, kubernetes.Interface, string) (*velerotypes.NodeAgentConfigs, error) configMapName string expectConfigs *velerotypes.NodeAgentConfigs expectedErr string }{ { name: "no config specified", }, { name: "failed to get configs", configMapName: "node-agent-config", getFunc: func(context.Context, string, kubernetes.Interface, string) (*velerotypes.NodeAgentConfigs, error) { return nil, errors.New("fake-get-error") }, expectedErr: "error getting node agent configs from configMap node-agent-config: fake-get-error", }, { name: "configs cm not found", configMapName: "node-agent-config", getFunc: func(context.Context, string, kubernetes.Interface, string) (*velerotypes.NodeAgentConfigs, error) { return nil, errors.New("fake-not-found-error") }, expectedErr: "error getting node agent configs from configMap node-agent-config: fake-not-found-error", }, { name: "succeed", configMapName: "node-agent-config", getFunc: func(context.Context, string, kubernetes.Interface, string) (*velerotypes.NodeAgentConfigs, error) { return configs, nil }, expectConfigs: configs, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { s := &nodeAgentServer{ config: nodeAgentServerConfig{ nodeAgentConfig: test.configMapName, }, logger: testutil.NewLogger(), } getConfigsFunc = test.getFunc err := s.getDataPathConfigs() if test.expectedErr == "" { require.NoError(t, err) assert.Equal(t, test.expectConfigs, s.dataPathConfigs) } else { require.EqualError(t, err, test.expectedErr) } }) } } func Test_getDataPathConcurrentNum(t *testing.T) { defaultNum := 100001 globalNum := 6 nodeName := "node-agent-node" node1 := builder.ForNode("node-agent-node").Result() node2 := builder.ForNode("node-agent-node").Labels(map[string]string{ "host-name": "node-1", "xxxx": "yyyyy", }).Result() invalidLabelSelector := metav1.LabelSelector{ MatchLabels: map[string]string{ "inva/lid": "inva/lid", }, } validLabelSelector1 := metav1.LabelSelector{ MatchLabels: map[string]string{ "host-name": "node-1", }, } validLabelSelector2 := metav1.LabelSelector{ MatchLabels: map[string]string{ "xxxx": "yyyyy", }, } tests := []struct { name string configs velerotypes.NodeAgentConfigs setKubeClient bool kubeClientObj []runtime.Object expectNum int expectLog string }{ { name: "configs cm's data path concurrency is nil", expectLog: fmt.Sprintf("Concurrency configs are not found, use the default number %v", defaultNum), expectNum: defaultNum, }, { name: "global number is invalid", configs: velerotypes.NodeAgentConfigs{ LoadConcurrency: &velerotypes.LoadConcurrency{ GlobalConfig: -1, }, }, expectLog: fmt.Sprintf("Global number %v is invalid, use the default value %v", -1, defaultNum), expectNum: defaultNum, }, { name: "global number is valid", configs: velerotypes.NodeAgentConfigs{ LoadConcurrency: &velerotypes.LoadConcurrency{ GlobalConfig: globalNum, }, }, expectNum: globalNum, }, { name: "node is not found", configs: velerotypes.NodeAgentConfigs{ LoadConcurrency: &velerotypes.LoadConcurrency{ GlobalConfig: globalNum, PerNodeConfig: []velerotypes.RuledConfigs{ { Number: 100, }, }, }, }, setKubeClient: true, expectLog: fmt.Sprintf("Failed to get node info for %s, use the global number %v", nodeName, globalNum), expectNum: globalNum, }, { name: "failed to get selector", configs: velerotypes.NodeAgentConfigs{ LoadConcurrency: &velerotypes.LoadConcurrency{ GlobalConfig: globalNum, PerNodeConfig: []velerotypes.RuledConfigs{ { NodeSelector: invalidLabelSelector, Number: 100, }, }, }, }, setKubeClient: true, kubeClientObj: []runtime.Object{node1}, expectLog: fmt.Sprintf("Failed to parse rule with label selector %s, skip it", invalidLabelSelector.String()), expectNum: globalNum, }, { name: "rule number is invalid", configs: velerotypes.NodeAgentConfigs{ LoadConcurrency: &velerotypes.LoadConcurrency{ GlobalConfig: globalNum, PerNodeConfig: []velerotypes.RuledConfigs{ { NodeSelector: validLabelSelector1, Number: -1, }, }, }, }, setKubeClient: true, kubeClientObj: []runtime.Object{node1}, expectLog: fmt.Sprintf("Rule with label selector %s is with an invalid number %v, skip it", validLabelSelector1.String(), -1), expectNum: globalNum, }, { name: "label doesn't match", configs: velerotypes.NodeAgentConfigs{ LoadConcurrency: &velerotypes.LoadConcurrency{ GlobalConfig: globalNum, PerNodeConfig: []velerotypes.RuledConfigs{ { NodeSelector: validLabelSelector1, Number: -1, }, }, }, }, setKubeClient: true, kubeClientObj: []runtime.Object{node1}, expectLog: fmt.Sprintf("Per node number for node %s is not found, use the global number %v", nodeName, globalNum), expectNum: globalNum, }, { name: "match one rule", configs: velerotypes.NodeAgentConfigs{ LoadConcurrency: &velerotypes.LoadConcurrency{ GlobalConfig: globalNum, PerNodeConfig: []velerotypes.RuledConfigs{ { NodeSelector: validLabelSelector1, Number: 66, }, }, }, }, setKubeClient: true, kubeClientObj: []runtime.Object{node2}, expectLog: fmt.Sprintf("Use the per node number %v over global number %v for node %s", 66, globalNum, nodeName), expectNum: 66, }, { name: "match multiple rules", configs: velerotypes.NodeAgentConfigs{ LoadConcurrency: &velerotypes.LoadConcurrency{ GlobalConfig: globalNum, PerNodeConfig: []velerotypes.RuledConfigs{ { NodeSelector: validLabelSelector1, Number: 66, }, { NodeSelector: validLabelSelector2, Number: 36, }, }, }, }, setKubeClient: true, kubeClientObj: []runtime.Object{node2}, expectLog: fmt.Sprintf("Use the per node number %v over global number %v for node %s", 36, globalNum, nodeName), expectNum: 36, }, { name: "match multiple rules 2", configs: velerotypes.NodeAgentConfigs{ LoadConcurrency: &velerotypes.LoadConcurrency{ GlobalConfig: globalNum, PerNodeConfig: []velerotypes.RuledConfigs{ { NodeSelector: validLabelSelector1, Number: 36, }, { NodeSelector: validLabelSelector2, Number: 66, }, }, }, }, setKubeClient: true, kubeClientObj: []runtime.Object{node2}, expectLog: fmt.Sprintf("Use the per node number %v over global number %v for node %s", 36, globalNum, nodeName), expectNum: 36, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...) logBuffer := "" s := &nodeAgentServer{ nodeName: nodeName, dataPathConfigs: &test.configs, logger: testutil.NewSingleLogger(&logBuffer), } if test.setKubeClient { s.kubeClient = fakeKubeClient } num := s.getDataPathConcurrentNum(defaultNum) assert.Equal(t, test.expectNum, num) if test.expectLog == "" { assert.Empty(t, logBuffer) } else { assert.Contains(t, logBuffer, test.expectLog) } }) } } func TestGetBackupRepoConfigs(t *testing.T) { cmNoData := builder.ForConfigMap(velerov1api.DefaultNamespace, "backup-repo-config").Result() cmWithData := builder.ForConfigMap(velerov1api.DefaultNamespace, "backup-repo-config").Data("cacheLimit", "100").Result() tests := []struct { name string configMapName string kubeClientObj []runtime.Object expectConfigs map[string]string expectedErr string }{ { name: "no config specified", }, { name: "failed to get configs", configMapName: "backup-repo-config", expectedErr: "error getting backup repo configs from configMap backup-repo-config: configmaps \"backup-repo-config\" not found", }, { name: "configs data not found", kubeClientObj: []runtime.Object{cmNoData}, configMapName: "backup-repo-config", expectedErr: "no data is in the backup repo configMap backup-repo-config", }, { name: "succeed", configMapName: "backup-repo-config", kubeClientObj: []runtime.Object{cmWithData}, expectConfigs: map[string]string{"cacheLimit": "100"}, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...) s := &nodeAgentServer{ namespace: velerov1api.DefaultNamespace, kubeClient: fakeKubeClient, config: nodeAgentServerConfig{ backupRepoConfig: test.configMapName, }, logger: testutil.NewLogger(), } err := s.getBackupRepoConfigs() if test.expectedErr == "" { require.NoError(t, err) require.Equal(t, test.expectConfigs, s.backupRepoConfigs) } else { require.EqualError(t, err, test.expectedErr) } }) } } func TestValidateCachePVCConfig(t *testing.T) { scWithRetainPolicy := builder.ForStorageClass("fake-storage-class").ReclaimPolicy(corev1api.PersistentVolumeReclaimRetain).Result() scWithDeletePolicy := builder.ForStorageClass("fake-storage-class").ReclaimPolicy(corev1api.PersistentVolumeReclaimDelete).Result() scWithNoPolicy := builder.ForStorageClass("fake-storage-class").Result() tests := []struct { name string config velerotypes.CachePVC kubeClientObj []runtime.Object expectedErr string }{ { name: "no storage class", expectedErr: "storage class is absent", }, { name: "failed to get storage class", config: velerotypes.CachePVC{StorageClass: "fake-storage-class"}, expectedErr: "error getting storage class fake-storage-class: storageclasses.storage.k8s.io \"fake-storage-class\" not found", }, { name: "storage class reclaim policy is not expected", config: velerotypes.CachePVC{StorageClass: "fake-storage-class"}, kubeClientObj: []runtime.Object{scWithRetainPolicy}, expectedErr: "unexpected storage class reclaim policy Retain", }, { name: "storage class reclaim policy is delete", config: velerotypes.CachePVC{StorageClass: "fake-storage-class"}, kubeClientObj: []runtime.Object{scWithDeletePolicy}, }, { name: "storage class with no reclaim policy", config: velerotypes.CachePVC{StorageClass: "fake-storage-class"}, kubeClientObj: []runtime.Object{scWithNoPolicy}, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...) s := &nodeAgentServer{ kubeClient: fakeKubeClient, } err := s.validateCachePVCConfig(test.config) if test.expectedErr == "" { require.NoError(t, err) } else { require.EqualError(t, err, test.expectedErr) } }) } } ================================================ FILE: pkg/cmd/cli/plugin/add.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package plugin import ( "context" "encoding/json" "fmt" "strings" jsonpatch "github.com/evanphx/json-patch/v5" "github.com/pkg/errors" "github.com/spf13/cobra" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" "github.com/vmware-tanzu/velero/pkg/cmd/util/confirm" "github.com/vmware-tanzu/velero/pkg/cmd/util/flag" ) const ( pluginsVolumeName = "plugins" veleroContainer = "velero" ) func NewAddCommand(f client.Factory) *cobra.Command { var ( imagePullPolicies = []string{string(corev1api.PullAlways), string(corev1api.PullIfNotPresent), string(corev1api.PullNever)} imagePullPolicyFlag = flag.NewEnum(string(corev1api.PullIfNotPresent), imagePullPolicies...) ) o := confirm.NewConfirmOptionsWithDescription("Confirm add?, may cause the Velero server pod restart which will fail all ongoing jobs") c := &cobra.Command{ Use: "add IMAGE", Short: "Add a plugin", Args: cobra.ExactArgs(1), Run: func(c *cobra.Command, args []string) { if !o.Confirm && !confirm.GetConfirmation("velero plugin add may cause the Velero server pod restart, so it is a dangerous operation", "once Velero server restarts, all the ongoing jobs will fail.") { // Don't do anything unless we get confirmation return } kubeClient, err := f.KubeClient() if err != nil { cmd.CheckError(err) } veleroDeploy, err := veleroDeployment(context.TODO(), kubeClient, f.Namespace()) if err != nil { cmd.CheckError(err) } original, err := json.Marshal(veleroDeploy) cmd.CheckError(err) // ensure the plugins volume & mount exist volumeExists := false for _, volume := range veleroDeploy.Spec.Template.Spec.Volumes { if volume.Name == pluginsVolumeName { volumeExists = true break } } if !volumeExists { volume := corev1api.Volume{ Name: pluginsVolumeName, VolumeSource: corev1api.VolumeSource{ EmptyDir: &corev1api.EmptyDirVolumeSource{}, }, } volumeMount := corev1api.VolumeMount{ Name: pluginsVolumeName, MountPath: "/plugins", } veleroDeploy.Spec.Template.Spec.Volumes = append(veleroDeploy.Spec.Template.Spec.Volumes, volume) containers := veleroDeploy.Spec.Template.Spec.Containers containerIndex := -1 for x, container := range containers { if container.Name == veleroContainer { containerIndex = x break } } if containerIndex < 0 { cmd.CheckError(errors.New("velero container not found in velero deployment")) } containers[containerIndex].VolumeMounts = append(containers[containerIndex].VolumeMounts, volumeMount) } // add the plugin as an init container plugin := *builder.ForPluginContainer(args[0], corev1api.PullPolicy(imagePullPolicyFlag.String())).Result() veleroDeploy.Spec.Template.Spec.InitContainers = append(veleroDeploy.Spec.Template.Spec.InitContainers, plugin) // create & apply the patch updated, err := json.Marshal(veleroDeploy) cmd.CheckError(err) patchBytes, err := jsonpatch.CreateMergePatch(original, updated) cmd.CheckError(err) _, err = kubeClient.AppsV1().Deployments(veleroDeploy.Namespace).Patch(context.TODO(), veleroDeploy.Name, types.MergePatchType, patchBytes, metav1.PatchOptions{}) cmd.CheckError(err) }, } c.Flags().Var(imagePullPolicyFlag, "image-pull-policy", fmt.Sprintf("The imagePullPolicy for the plugin container. Valid values are %s.", strings.Join(imagePullPolicies, ", "))) o.BindFlags(c.Flags()) return c } ================================================ FILE: pkg/cmd/cli/plugin/get.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package plugin import ( "context" "fmt" "os" "time" "github.com/spf13/cobra" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" "github.com/vmware-tanzu/velero/pkg/cmd/cli/serverstatus" "github.com/vmware-tanzu/velero/pkg/cmd/util/output" ) func NewGetCommand(f client.Factory, use string) *cobra.Command { timeout := 5 * time.Second c := &cobra.Command{ Use: use, Short: "Get information for all plugins on the velero server", Run: func(c *cobra.Command, args []string) { err := output.ValidateFlags(c) cmd.CheckError(err) kbClient, err := f.KubebuilderClient() cmd.CheckError(err) ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() serverStatusGetter := &serverstatus.DefaultServerStatusGetter{ Namespace: f.Namespace(), Context: ctx, } serverStatus, err := serverStatusGetter.GetServerStatus(kbClient) if err != nil { fmt.Fprintf(os.Stdout, "\n", err) return } _, err = output.PrintWithFormat(c, serverStatus) cmd.CheckError(err) }, } c.Flags().DurationVar(&timeout, "timeout", timeout, "Maximum time to wait for plugin information to be reported. Default is 5 seconds.") output.BindFlagsSimple(c.Flags()) return c } ================================================ FILE: pkg/cmd/cli/plugin/helpers.go ================================================ /* Copyright The Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package plugin import ( "context" "github.com/pkg/errors" appsv1api "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/kubernetes" "github.com/vmware-tanzu/velero/pkg/install" ) // veleroDeployment returns a Velero deployment object, selected with label and container name, // refer to https://github.com/vmware-tanzu/velero/issues/3961 for more information func veleroDeployment(ctx context.Context, kubeClient kubernetes.Interface, namespace string) (*appsv1api.Deployment, error) { veleroLabels := labels.FormatLabels(install.Labels()) deployList, err := kubeClient. AppsV1(). Deployments(namespace). List(ctx, metav1.ListOptions{ LabelSelector: veleroLabels, }) if err != nil { return nil, err } for _, deploy := range deployList.Items { for _, container := range deploy.Spec.Template.Spec.Containers { if container.Name == "velero" { return &deploy, nil } } } return nil, errors.New("Velero deployment not found") } ================================================ FILE: pkg/cmd/cli/plugin/plugin.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package plugin import ( "github.com/spf13/cobra" "github.com/vmware-tanzu/velero/pkg/client" ) func NewCommand(f client.Factory) *cobra.Command { c := &cobra.Command{ Use: "plugin", Short: "Work with plugins", Long: "Work with plugins", } c.AddCommand( NewAddCommand(f), NewRemoveCommand(f), NewGetCommand(f, "get"), ) return c } ================================================ FILE: pkg/cmd/cli/plugin/remove.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package plugin import ( "context" "encoding/json" jsonpatch "github.com/evanphx/json-patch/v5" "github.com/pkg/errors" "github.com/spf13/cobra" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" ) func NewRemoveCommand(f client.Factory) *cobra.Command { c := &cobra.Command{ Use: "remove [NAME | IMAGE]", Short: "Remove a plugin", Args: cobra.ExactArgs(1), Run: func(c *cobra.Command, args []string) { kubeClient, err := f.KubeClient() if err != nil { cmd.CheckError(err) } veleroDeploy, err := veleroDeployment(context.TODO(), kubeClient, f.Namespace()) if err != nil { cmd.CheckError(err) } original, err := json.Marshal(veleroDeploy) cmd.CheckError(err) var ( initContainers = veleroDeploy.Spec.Template.Spec.InitContainers index = -1 ) for x, container := range initContainers { if container.Name == args[0] || container.Image == args[0] { index = x break } } if index == -1 { cmd.CheckError(errors.Errorf("init container %s not found in Velero server deployment", args[0])) } veleroDeploy.Spec.Template.Spec.InitContainers = append(initContainers[0:index], initContainers[index+1:]...) updated, err := json.Marshal(veleroDeploy) cmd.CheckError(err) patchBytes, err := jsonpatch.CreateMergePatch(original, updated) cmd.CheckError(err) _, err = kubeClient.AppsV1().Deployments(veleroDeploy.Namespace).Patch(context.TODO(), veleroDeploy.Name, types.MergePatchType, patchBytes, metav1.PatchOptions{}) cmd.CheckError(err) }, } return c } ================================================ FILE: pkg/cmd/cli/podvolume/backup.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package podvolume import ( "context" "fmt" "os" "strings" "time" "github.com/bombsimon/logrusr/v3" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" "github.com/vmware-tanzu/velero/internal/credentials" "github.com/vmware-tanzu/velero/pkg/buildinfo" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd/util/signals" "github.com/vmware-tanzu/velero/pkg/datapath" "github.com/vmware-tanzu/velero/pkg/podvolume" "github.com/vmware-tanzu/velero/pkg/repository" "github.com/vmware-tanzu/velero/pkg/uploader" "github.com/vmware-tanzu/velero/pkg/util/filesystem" "github.com/vmware-tanzu/velero/pkg/util/kube" "github.com/vmware-tanzu/velero/pkg/util/logging" ctrl "sigs.k8s.io/controller-runtime" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ctlcache "sigs.k8s.io/controller-runtime/pkg/cache" ctlclient "sigs.k8s.io/controller-runtime/pkg/client" ) type podVolumeBackupConfig struct { volumePath string pvbName string resourceTimeout time.Duration } func NewBackupCommand(f client.Factory) *cobra.Command { config := podVolumeBackupConfig{} logLevelFlag := logging.LogLevelFlag(logrus.InfoLevel) formatFlag := logging.NewFormatFlag() command := &cobra.Command{ Use: "backup", Short: "Run the velero pod volume backup", Long: "Run the velero pod volume backup", Hidden: true, Run: func(c *cobra.Command, args []string) { logLevel := logLevelFlag.Parse() logrus.Infof("Setting log-level to %s", strings.ToUpper(logLevel.String())) logger := logging.DefaultLogger(logLevel, formatFlag.Parse()) logger.Infof("Starting Velero pod volume backup %s (%s)", buildinfo.Version, buildinfo.FormattedGitSHA()) f.SetBasename(fmt.Sprintf("%s-%s", c.Parent().Name(), c.Name())) s, err := newPodVolumeBackup(logger, f, config) if err != nil { kube.ExitPodWithMessage(logger, false, "Failed to create pod volume backup, %v", err) } s.run() }, } command.Flags().Var(logLevelFlag, "log-level", fmt.Sprintf("The level at which to log. Valid values are %s.", strings.Join(logLevelFlag.AllowedValues(), ", "))) command.Flags().Var(formatFlag, "log-format", fmt.Sprintf("The format for log output. Valid values are %s.", strings.Join(formatFlag.AllowedValues(), ", "))) command.Flags().StringVar(&config.volumePath, "volume-path", config.volumePath, "The full path of the volume to be backed up") command.Flags().StringVar(&config.pvbName, "pod-volume-backup", config.pvbName, "The PVB name") command.Flags().DurationVar(&config.resourceTimeout, "resource-timeout", config.resourceTimeout, "How long to wait for resource processes which are not covered by other specific timeout parameters.") _ = command.MarkFlagRequired("volume-path") _ = command.MarkFlagRequired("pod-volume-backup") _ = command.MarkFlagRequired("resource-timeout") return command } type podVolumeBackup struct { logger logrus.FieldLogger ctx context.Context cancelFunc context.CancelFunc client ctlclient.Client cache ctlcache.Cache namespace string nodeName string config podVolumeBackupConfig kubeClient kubernetes.Interface dataPathMgr *datapath.Manager } func newPodVolumeBackup(logger logrus.FieldLogger, factory client.Factory, config podVolumeBackupConfig) (*podVolumeBackup, error) { ctx, cancelFunc := context.WithCancel(context.Background()) clientConfig, err := factory.ClientConfig() if err != nil { cancelFunc() return nil, errors.Wrap(err, "error to create client config") } ctrl.SetLogger(logrusr.New(logger)) klog.SetLogger(logrusr.New(logger)) // klog.Logger is used by k8s.io/client-go scheme := runtime.NewScheme() if err := velerov1api.AddToScheme(scheme); err != nil { cancelFunc() return nil, errors.Wrap(err, "error to add velero v1 scheme") } if err := corev1api.AddToScheme(scheme); err != nil { cancelFunc() return nil, errors.Wrap(err, "error to add core v1 scheme") } nodeName := os.Getenv("NODE_NAME") // use a field selector to filter to only pods scheduled on this node. cacheOption := ctlcache.Options{ Scheme: scheme, ByObject: map[ctlclient.Object]ctlcache.ByObject{ &corev1api.Pod{}: { Field: fields.Set{"spec.nodeName": nodeName}.AsSelector(), }, &velerov1api.PodVolumeBackup{}: { Field: fields.Set{"metadata.namespace": factory.Namespace()}.AsSelector(), }, }, } cli, err := ctlclient.New(clientConfig, ctlclient.Options{ Scheme: scheme, }) if err != nil { cancelFunc() return nil, errors.Wrap(err, "error to create client") } var cache ctlcache.Cache retry := 10 for { cache, err = ctlcache.New(clientConfig, cacheOption) if err == nil { break } retry-- if retry == 0 { break } logger.WithError(err).Warn("Failed to create client cache, need retry") time.Sleep(time.Second) } if err != nil { cancelFunc() return nil, errors.Wrap(err, "error to create client cache") } s := &podVolumeBackup{ logger: logger, ctx: ctx, cancelFunc: cancelFunc, client: cli, cache: cache, config: config, namespace: factory.Namespace(), nodeName: nodeName, } s.kubeClient, err = factory.KubeClient() if err != nil { cancelFunc() return nil, errors.Wrap(err, "error to create kube client") } s.dataPathMgr = datapath.NewManager(1) return s, nil } var funcExitWithMessage = kube.ExitPodWithMessage var funcCreateDataPathService = (*podVolumeBackup).createDataPathService func (s *podVolumeBackup) run() { signals.CancelOnShutdown(s.cancelFunc, s.logger) go func() { if err := s.cache.Start(s.ctx); err != nil { s.logger.WithError(err).Warn("error starting cache") } }() s.runDataPath() } func (s *podVolumeBackup) runDataPath() { s.logger.Infof("Starting micro service in node %s for PVB %s", s.nodeName, s.config.pvbName) dpService, err := funcCreateDataPathService(s) if err != nil { s.cancelFunc() funcExitWithMessage(s.logger, false, "Failed to create data path service for PVB %s: %v", s.config.pvbName, err) return } s.logger.Infof("Starting data path service %s", s.config.pvbName) err = dpService.Init() if err != nil { dpService.Shutdown() s.cancelFunc() funcExitWithMessage(s.logger, false, "Failed to init data path service for PVB %s: %v", s.config.pvbName, err) return } s.logger.Infof("Running data path service %s", s.config.pvbName) result, err := dpService.RunCancelableDataPath(s.ctx) if err != nil { dpService.Shutdown() s.cancelFunc() funcExitWithMessage(s.logger, false, "Failed to run data path service for PVB %s: %v", s.config.pvbName, err) return } s.logger.WithField("PVB", s.config.pvbName).Info("Data path service completed") dpService.Shutdown() s.logger.WithField("PVB", s.config.pvbName).Info("Data path service is shut down") s.cancelFunc() funcExitWithMessage(s.logger, true, result) } var funcNewCredentialFileStore = credentials.NewNamespacedFileStore var funcNewCredentialSecretStore = credentials.NewNamespacedSecretStore func (s *podVolumeBackup) createDataPathService() (dataPathService, error) { credentialFileStore, err := funcNewCredentialFileStore( s.client, s.namespace, credentials.DefaultStoreDirectory(), filesystem.NewFileSystem(), ) if err != nil { return nil, errors.Wrapf(err, "error to create credential file store") } credSecretStore, err := funcNewCredentialSecretStore(s.client, s.namespace) if err != nil { return nil, errors.Wrapf(err, "error to create credential secret store") } credGetter := &credentials.CredentialGetter{FromFile: credentialFileStore, FromSecret: credSecretStore} pvbInformer, err := s.cache.GetInformer(s.ctx, &velerov1api.PodVolumeBackup{}) if err != nil { return nil, errors.Wrap(err, "error to get controller-runtime informer from manager") } repoEnsurer := repository.NewEnsurer(s.client, s.logger, s.config.resourceTimeout) return podvolume.NewBackupMicroService(s.ctx, s.client, s.kubeClient, s.config.pvbName, s.namespace, s.nodeName, datapath.AccessPoint{ ByPath: s.config.volumePath, VolMode: uploader.PersistentVolumeFilesystem, }, s.dataPathMgr, repoEnsurer, credGetter, pvbInformer, s.logger), nil } ================================================ FILE: pkg/cmd/cli/podvolume/backup_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package podvolume import ( "context" "errors" "fmt" "testing" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" ctlclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/internal/credentials" cacheMock "github.com/vmware-tanzu/velero/pkg/cmd/cli/datamover/mocks" velerotest "github.com/vmware-tanzu/velero/pkg/test" "github.com/vmware-tanzu/velero/pkg/util/filesystem" ) func fakeCreateDataPathServiceWithErr(_ *podVolumeBackup) (dataPathService, error) { return nil, errors.New("fake-create-data-path-error") } var frHelper *fakeRunHelper func fakeCreateDataPathService(_ *podVolumeBackup) (dataPathService, error) { return frHelper, nil } type fakeRunHelper struct { initErr error runCancelableDataPathErr error runCancelableDataPathResult string exitMessage string succeed bool } func (fr *fakeRunHelper) Init() error { return fr.initErr } func (fr *fakeRunHelper) RunCancelableDataPath(_ context.Context) (string, error) { if fr.runCancelableDataPathErr != nil { return "", fr.runCancelableDataPathErr } else { return fr.runCancelableDataPathResult, nil } } func (fr *fakeRunHelper) Shutdown() { } func (fr *fakeRunHelper) ExitWithMessage(logger logrus.FieldLogger, succeed bool, message string, a ...any) { fr.succeed = succeed fr.exitMessage = fmt.Sprintf(message, a...) } func TestRunDataPath(t *testing.T) { tests := []struct { name string pvbName string createDataPathFail bool initDataPathErr error runCancelableDataPathErr error runCancelableDataPathResult string expectedMessage string expectedSucceed bool }{ { name: "create data path failed", pvbName: "fake-name", createDataPathFail: true, expectedMessage: "Failed to create data path service for PVB fake-name: fake-create-data-path-error", }, { name: "init data path failed", pvbName: "fake-name", initDataPathErr: errors.New("fake-init-data-path-error"), expectedMessage: "Failed to init data path service for PVB fake-name: fake-init-data-path-error", }, { name: "run data path failed", pvbName: "fake-name", runCancelableDataPathErr: errors.New("fake-run-data-path-error"), expectedMessage: "Failed to run data path service for PVB fake-name: fake-run-data-path-error", }, { name: "succeed", pvbName: "fake-name", runCancelableDataPathResult: "fake-run-data-path-result", expectedMessage: "fake-run-data-path-result", expectedSucceed: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { frHelper = &fakeRunHelper{ initErr: test.initDataPathErr, runCancelableDataPathErr: test.runCancelableDataPathErr, runCancelableDataPathResult: test.runCancelableDataPathResult, } if test.createDataPathFail { funcCreateDataPathService = fakeCreateDataPathServiceWithErr } else { funcCreateDataPathService = fakeCreateDataPathService } funcExitWithMessage = frHelper.ExitWithMessage s := &podVolumeBackup{ logger: velerotest.NewLogger(), cancelFunc: func() {}, config: podVolumeBackupConfig{ pvbName: test.pvbName, }, } s.runDataPath() assert.Equal(t, test.expectedMessage, frHelper.exitMessage) assert.Equal(t, test.expectedSucceed, frHelper.succeed) }) } } type fakeCreateDataPathServiceHelper struct { fileStoreErr error secretStoreErr error } func (fc *fakeCreateDataPathServiceHelper) NewNamespacedFileStore(_ ctlclient.Client, _ string, _ string, _ filesystem.Interface) (credentials.FileStore, error) { return nil, fc.fileStoreErr } func (fc *fakeCreateDataPathServiceHelper) NewNamespacedSecretStore(_ ctlclient.Client, _ string) (credentials.SecretStore, error) { return nil, fc.secretStoreErr } func TestCreateDataPathService(t *testing.T) { tests := []struct { name string fileStoreErr error secretStoreErr error mockGetInformer bool getInformerErr error expectedError string }{ { name: "create credential file store error", fileStoreErr: errors.New("fake-file-store-error"), expectedError: "error to create credential file store: fake-file-store-error", }, { name: "create credential secret store", secretStoreErr: errors.New("fake-secret-store-error"), expectedError: "error to create credential secret store: fake-secret-store-error", }, { name: "get informer error", mockGetInformer: true, getInformerErr: errors.New("fake-get-informer-error"), expectedError: "error to get controller-runtime informer from manager: fake-get-informer-error", }, { name: "succeed", mockGetInformer: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fcHelper := &fakeCreateDataPathServiceHelper{ fileStoreErr: test.fileStoreErr, secretStoreErr: test.secretStoreErr, } funcNewCredentialFileStore = fcHelper.NewNamespacedFileStore funcNewCredentialSecretStore = fcHelper.NewNamespacedSecretStore cache := cacheMock.NewCache(t) if test.mockGetInformer { cache.On("GetInformer", mock.Anything, mock.Anything).Return(nil, test.getInformerErr) } funcExitWithMessage = frHelper.ExitWithMessage s := &podVolumeBackup{ cache: cache, } _, err := s.createDataPathService() if test.expectedError != "" { assert.EqualError(t, err, test.expectedError) } else { assert.NoError(t, err) } }) } } ================================================ FILE: pkg/cmd/cli/podvolume/podvolume.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package podvolume import ( "context" "github.com/spf13/cobra" "github.com/vmware-tanzu/velero/pkg/client" ) func NewCommand(f client.Factory) *cobra.Command { command := &cobra.Command{ Use: "pod-volume", Short: "Run the velero pod volume backup/restore", Long: "Run the velero pod volume backup/restore", Hidden: true, } command.AddCommand( NewBackupCommand(f), NewRestoreCommand(f), ) return command } type dataPathService interface { Init() error RunCancelableDataPath(context.Context) (string, error) Shutdown() } ================================================ FILE: pkg/cmd/cli/podvolume/restore.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package podvolume import ( "context" "fmt" "os" "strings" "time" "github.com/bombsimon/logrusr/v3" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" ctrl "sigs.k8s.io/controller-runtime" "github.com/vmware-tanzu/velero/internal/credentials" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/buildinfo" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd/util/signals" "github.com/vmware-tanzu/velero/pkg/datapath" "github.com/vmware-tanzu/velero/pkg/podvolume" "github.com/vmware-tanzu/velero/pkg/repository" "github.com/vmware-tanzu/velero/pkg/uploader" "github.com/vmware-tanzu/velero/pkg/util/filesystem" "github.com/vmware-tanzu/velero/pkg/util/kube" "github.com/vmware-tanzu/velero/pkg/util/logging" ctlcache "sigs.k8s.io/controller-runtime/pkg/cache" ctlclient "sigs.k8s.io/controller-runtime/pkg/client" ) type podVolumeRestoreConfig struct { volumePath string pvrName string cacheDir string resourceTimeout time.Duration } func NewRestoreCommand(f client.Factory) *cobra.Command { logLevelFlag := logging.LogLevelFlag(logrus.InfoLevel) formatFlag := logging.NewFormatFlag() config := podVolumeRestoreConfig{} command := &cobra.Command{ Use: "restore", Short: "Run the velero pod volume restore", Long: "Run the velero pod volume restore", Hidden: true, Run: func(c *cobra.Command, args []string) { logLevel := logLevelFlag.Parse() logrus.Infof("Setting log-level to %s", strings.ToUpper(logLevel.String())) logger := logging.DefaultLogger(logLevel, formatFlag.Parse()) logger.Infof("Starting Velero pod volume restore %s (%s)", buildinfo.Version, buildinfo.FormattedGitSHA()) f.SetBasename(fmt.Sprintf("%s-%s", c.Parent().Name(), c.Name())) s, err := newPodVolumeRestore(logger, f, config) if err != nil { kube.ExitPodWithMessage(logger, false, "Failed to create pod volume restore, %v", err) } s.run() }, } command.Flags().Var(logLevelFlag, "log-level", fmt.Sprintf("The level at which to log. Valid values are %s.", strings.Join(logLevelFlag.AllowedValues(), ", "))) command.Flags().Var(formatFlag, "log-format", fmt.Sprintf("The format for log output. Valid values are %s.", strings.Join(formatFlag.AllowedValues(), ", "))) command.Flags().StringVar(&config.volumePath, "volume-path", config.volumePath, "The full path of the volume to be restored") command.Flags().StringVar(&config.pvrName, "pod-volume-restore", config.pvrName, "The PVR name") command.Flags().StringVar(&config.cacheDir, "cache-volume-path", config.cacheDir, "The full path of the cache volume") command.Flags().DurationVar(&config.resourceTimeout, "resource-timeout", config.resourceTimeout, "How long to wait for resource processes which are not covered by other specific timeout parameters.") _ = command.MarkFlagRequired("volume-path") _ = command.MarkFlagRequired("pod-volume-restore") _ = command.MarkFlagRequired("resource-timeout") command.PreRunE = func(cmd *cobra.Command, args []string) error { if config.resourceTimeout <= 0 { return errors.New("resource-timeout must be greater than 0") } if config.volumePath == "" { return errors.New("volume-path cannot be empty") } if config.pvrName == "" { return errors.New("pod-volume-restore name cannot be empty") } return nil } return command } type podVolumeRestore struct { logger logrus.FieldLogger ctx context.Context cancelFunc context.CancelFunc client ctlclient.Client cache ctlcache.Cache namespace string nodeName string config podVolumeRestoreConfig kubeClient kubernetes.Interface dataPathMgr *datapath.Manager } func newPodVolumeRestore(logger logrus.FieldLogger, factory client.Factory, config podVolumeRestoreConfig) (*podVolumeRestore, error) { ctx, cancelFunc := context.WithCancel(context.Background()) clientConfig, err := factory.ClientConfig() if err != nil { cancelFunc() return nil, errors.Wrap(err, "error to create client config") } ctrl.SetLogger(logrusr.New(logger)) klog.SetLogger(logrusr.New(logger)) // klog.Logger is used by k8s.io/client-go scheme := runtime.NewScheme() if err := velerov1api.AddToScheme(scheme); err != nil { cancelFunc() return nil, errors.Wrap(err, "error to add velero v1 scheme") } if err := corev1api.AddToScheme(scheme); err != nil { cancelFunc() return nil, errors.Wrap(err, "error to add core v1 scheme") } nodeName := os.Getenv("NODE_NAME") // use a field selector to filter to only pods scheduled on this node. cacheOption := ctlcache.Options{ Scheme: scheme, ByObject: map[ctlclient.Object]ctlcache.ByObject{ &corev1api.Pod{}: { Field: fields.Set{"spec.nodeName": nodeName}.AsSelector(), }, &velerov1api.PodVolumeRestore{}: { Field: fields.Set{"metadata.namespace": factory.Namespace()}.AsSelector(), }, }, } cli, err := ctlclient.New(clientConfig, ctlclient.Options{ Scheme: scheme, }) if err != nil { cancelFunc() return nil, errors.Wrap(err, "error to create client") } var cache ctlcache.Cache retry := 10 for { cache, err = ctlcache.New(clientConfig, cacheOption) if err == nil { break } retry-- if retry == 0 { break } logger.WithError(err).Warn("Failed to create client cache, need retry") time.Sleep(time.Second) } if err != nil { cancelFunc() return nil, errors.Wrap(err, "error to create client cache") } s := &podVolumeRestore{ logger: logger, ctx: ctx, cancelFunc: cancelFunc, client: cli, cache: cache, config: config, namespace: factory.Namespace(), nodeName: nodeName, } s.kubeClient, err = factory.KubeClient() if err != nil { cancelFunc() return nil, errors.Wrap(err, "error to create kube client") } s.dataPathMgr = datapath.NewManager(1) return s, nil } var funcCreateDataPathRestore = (*podVolumeRestore).createDataPathService func (s *podVolumeRestore) run() { signals.CancelOnShutdown(s.cancelFunc, s.logger) go func() { if err := s.cache.Start(s.ctx); err != nil { s.logger.WithError(err).Warn("error starting cache") } }() s.runDataPath() } func (s *podVolumeRestore) runDataPath() { s.logger.Infof("Starting micro service in node %s for PVR %s", s.nodeName, s.config.pvrName) dpService, err := funcCreateDataPathRestore(s) if err != nil { s.cancelFunc() funcExitWithMessage(s.logger, false, "Failed to create data path service for PVR %s: %v", s.config.pvrName, err) return } s.logger.Infof("Starting data path service %s", s.config.pvrName) err = dpService.Init() if err != nil { dpService.Shutdown() s.cancelFunc() funcExitWithMessage(s.logger, false, "Failed to init data path service for PVR %s: %v", s.config.pvrName, err) return } s.logger.Infof("Running data path service %s", s.config.pvrName) result, err := dpService.RunCancelableDataPath(s.ctx) if err != nil { dpService.Shutdown() s.cancelFunc() funcExitWithMessage(s.logger, false, "Failed to run data path service for PVR %s: %v", s.config.pvrName, err) return } s.logger.WithField("PVR", s.config.pvrName).Info("Data path service completed") dpService.Shutdown() s.logger.WithField("PVR", s.config.pvrName).Info("Data path service is shut down") s.cancelFunc() funcExitWithMessage(s.logger, true, result) } func (s *podVolumeRestore) createDataPathService() (dataPathService, error) { credentialFileStore, err := funcNewCredentialFileStore( s.client, s.namespace, credentials.DefaultStoreDirectory(), filesystem.NewFileSystem(), ) if err != nil { return nil, errors.Wrapf(err, "error to create credential file store") } credSecretStore, err := funcNewCredentialSecretStore(s.client, s.namespace) if err != nil { return nil, errors.Wrapf(err, "error to create credential secret store") } credGetter := &credentials.CredentialGetter{FromFile: credentialFileStore, FromSecret: credSecretStore} pvrInformer, err := s.cache.GetInformer(s.ctx, &velerov1api.PodVolumeRestore{}) if err != nil { return nil, errors.Wrap(err, "error to get controller-runtime informer from manager") } repoEnsurer := repository.NewEnsurer(s.client, s.logger, s.config.resourceTimeout) return podvolume.NewRestoreMicroService(s.ctx, s.client, s.kubeClient, s.config.pvrName, s.namespace, s.nodeName, datapath.AccessPoint{ ByPath: s.config.volumePath, VolMode: uploader.PersistentVolumeFilesystem, }, s.dataPathMgr, repoEnsurer, credGetter, pvrInformer, s.config.cacheDir, s.logger), nil } ================================================ FILE: pkg/cmd/cli/podvolume/restore_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package podvolume import ( "errors" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" cacheMock "github.com/vmware-tanzu/velero/pkg/cmd/cli/datamover/mocks" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func fakeCreateRestoreDataPathServiceWithErr(_ *podVolumeRestore) (dataPathService, error) { return nil, errors.New("fake-create-data-path-error") } func fakeCreateRestoreDataPathService(_ *podVolumeRestore) (dataPathService, error) { return frHelper, nil } func TestRunRestoreDataPath(t *testing.T) { tests := []struct { name string pvrName string createDataPathFail bool initDataPathErr error runCancelableDataPathErr error runCancelableDataPathResult string expectedMessage string expectedSucceed bool }{ { name: "create data path failed", pvrName: "fake-name", createDataPathFail: true, expectedMessage: "Failed to create data path service for PVR fake-name: fake-create-data-path-error", }, { name: "init data path failed", pvrName: "fake-name", initDataPathErr: errors.New("fake-init-data-path-error"), expectedMessage: "Failed to init data path service for PVR fake-name: fake-init-data-path-error", }, { name: "run data path failed", pvrName: "fake-name", runCancelableDataPathErr: errors.New("fake-run-data-path-error"), expectedMessage: "Failed to run data path service for PVR fake-name: fake-run-data-path-error", }, { name: "succeed", pvrName: "fake-name", runCancelableDataPathResult: "fake-run-data-path-result", expectedMessage: "fake-run-data-path-result", expectedSucceed: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { frHelper = &fakeRunHelper{ initErr: test.initDataPathErr, runCancelableDataPathErr: test.runCancelableDataPathErr, runCancelableDataPathResult: test.runCancelableDataPathResult, } if test.createDataPathFail { funcCreateDataPathRestore = fakeCreateRestoreDataPathServiceWithErr } else { funcCreateDataPathRestore = fakeCreateRestoreDataPathService } funcExitWithMessage = frHelper.ExitWithMessage s := &podVolumeRestore{ logger: velerotest.NewLogger(), cancelFunc: func() {}, config: podVolumeRestoreConfig{ pvrName: test.pvrName, }, } s.runDataPath() assert.Equal(t, test.expectedMessage, frHelper.exitMessage) assert.Equal(t, test.expectedSucceed, frHelper.succeed) }) } } func TestCreateRestoreDataPathService(t *testing.T) { tests := []struct { name string fileStoreErr error secretStoreErr error mockGetInformer bool getInformerErr error expectedError string }{ { name: "create credential file store error", fileStoreErr: errors.New("fake-file-store-error"), expectedError: "error to create credential file store: fake-file-store-error", }, { name: "create credential secret store", secretStoreErr: errors.New("fake-secret-store-error"), expectedError: "error to create credential secret store: fake-secret-store-error", }, { name: "get informer error", mockGetInformer: true, getInformerErr: errors.New("fake-get-informer-error"), expectedError: "error to get controller-runtime informer from manager: fake-get-informer-error", }, { name: "succeed", mockGetInformer: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fcHelper := &fakeCreateDataPathServiceHelper{ fileStoreErr: test.fileStoreErr, secretStoreErr: test.secretStoreErr, } funcNewCredentialFileStore = fcHelper.NewNamespacedFileStore funcNewCredentialSecretStore = fcHelper.NewNamespacedSecretStore cache := cacheMock.NewCache(t) if test.mockGetInformer { cache.On("GetInformer", mock.Anything, mock.Anything).Return(nil, test.getInformerErr) } funcExitWithMessage = frHelper.ExitWithMessage s := &podVolumeRestore{ cache: cache, } _, err := s.createDataPathService() if test.expectedError != "" { assert.EqualError(t, err, test.expectedError) } else { assert.NoError(t, err) } }) } } ================================================ FILE: pkg/cmd/cli/repo/get.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package repo import ( "context" "github.com/spf13/cobra" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" "github.com/vmware-tanzu/velero/pkg/cmd/util/output" ) func NewGetCommand(f client.Factory, use string) *cobra.Command { var listOptions metav1.ListOptions c := &cobra.Command{ Use: use, Short: "Get repositories", Run: func(c *cobra.Command, args []string) { err := output.ValidateFlags(c) cmd.CheckError(err) crClient, err := f.KubebuilderClient() cmd.CheckError(err) repos := new(api.BackupRepositoryList) if len(args) > 0 { for _, name := range args { repo := new(api.BackupRepository) err := crClient.Get(context.TODO(), ctrlclient.ObjectKey{Namespace: f.Namespace(), Name: name}, repo) cmd.CheckError(err) repos.Items = append(repos.Items, *repo) } } else { selector := labels.NewSelector() if listOptions.LabelSelector != "" { selector, err = labels.Parse(listOptions.LabelSelector) cmd.CheckError(err) } err = crClient.List(context.TODO(), repos, &ctrlclient.ListOptions{LabelSelector: selector}) cmd.CheckError(err) } _, err = output.PrintWithFormat(c, repos) cmd.CheckError(err) }, } c.Flags().StringVarP(&listOptions.LabelSelector, "selector", "l", listOptions.LabelSelector, "Only show items matching this label selector.") output.BindFlags(c.Flags()) return c } ================================================ FILE: pkg/cmd/cli/repo/repo.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package repo import ( "github.com/spf13/cobra" "github.com/vmware-tanzu/velero/pkg/client" ) func NewCommand(f client.Factory) *cobra.Command { c := &cobra.Command{ Use: "repo", Short: "Work with repositories", Long: "Work with repositories", } c.AddCommand( NewGetCommand(f, "get"), ) return c } ================================================ FILE: pkg/cmd/cli/repomantenance/maintenance.go ================================================ package repomantenance import ( "context" "fmt" "os" "strings" "time" "github.com/bombsimon/logrusr/v3" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/pflag" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/internal/credentials" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerocli "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/repository" "github.com/vmware-tanzu/velero/pkg/util/filesystem" "github.com/vmware-tanzu/velero/pkg/util/logging" repokey "github.com/vmware-tanzu/velero/pkg/repository/keys" "github.com/vmware-tanzu/velero/pkg/repository/maintenance" repomanager "github.com/vmware-tanzu/velero/pkg/repository/manager" ) type Options struct { RepoName string BackupStorageLocation string RepoType string ResourceTimeout time.Duration LogLevelFlag *logging.LevelFlag FormatFlag *logging.FormatFlag } func (o *Options) BindFlags(flags *pflag.FlagSet) { flags.StringVar(&o.RepoName, "repo-name", "", "namespace of the pod/volume that the snapshot is for") flags.StringVar(&o.BackupStorageLocation, "backup-storage-location", "", "backup's storage location name") flags.StringVar(&o.RepoType, "repo-type", velerov1api.BackupRepositoryTypeKopia, "type of the repository where the snapshot is stored") flags.Var(o.LogLevelFlag, "log-level", fmt.Sprintf("The level at which to log. Valid values are %s.", strings.Join(o.LogLevelFlag.AllowedValues(), ", "))) flags.Var(o.FormatFlag, "log-format", fmt.Sprintf("The format for log output. Valid values are %s.", strings.Join(o.FormatFlag.AllowedValues(), ", "))) } func NewCommand(f velerocli.Factory) *cobra.Command { o := &Options{ LogLevelFlag: logging.LogLevelFlag(logrus.InfoLevel), FormatFlag: logging.NewFormatFlag(), } cmd := &cobra.Command{ Use: "repo-maintenance", Hidden: true, Short: "VELERO INTERNAL COMMAND ONLY - not intended to be run directly by users", Run: func(c *cobra.Command, args []string) { o.Run(f) }, } o.BindFlags(cmd.Flags()) return cmd } func (o *Options) Run(f velerocli.Factory) { logger := logging.DefaultLogger(o.LogLevelFlag.Parse(), o.FormatFlag.Parse()) logger.SetOutput(os.Stdout) ctrl.SetLogger(logrusr.New(logger)) pruneError := o.runRepoPrune(f, f.Namespace(), logger) defer func() { if pruneError != nil { os.Exit(1) } }() if pruneError != nil { os.Stdout.WriteString(fmt.Sprintf("%s%v", maintenance.TerminationLogIndicator, pruneError)) } } func (o *Options) initClient(f velerocli.Factory) (client.Client, error) { scheme := runtime.NewScheme() err := velerov1api.AddToScheme(scheme) if err != nil { return nil, errors.Wrap(err, "failed to add velero scheme") } err = corev1api.AddToScheme(scheme) if err != nil { return nil, errors.Wrap(err, "failed to add api core scheme") } config, err := f.ClientConfig() if err != nil { return nil, errors.Wrap(err, "failed to get client config") } cli, err := client.New(config, client.Options{ Scheme: scheme, }) if err != nil { return nil, errors.Wrap(err, "failed to create client") } return cli, nil } func initRepoManager(namespace string, cli client.Client, kubeClient kubernetes.Interface, logger logrus.FieldLogger) (repomanager.Manager, error) { // ensure the repo key secret is set up if err := repokey.EnsureCommonRepositoryKey(kubeClient.CoreV1(), namespace); err != nil { return nil, errors.Wrap(err, "failed to ensure repository key") } repoLocker := repository.NewRepoLocker() credentialFileStore, err := credentials.NewNamespacedFileStore( cli, namespace, credentials.DefaultStoreDirectory(), filesystem.NewFileSystem(), ) if err != nil { return nil, errors.Wrap(err, "failed to create namespaced file store") } credentialSecretStore, err := credentials.NewNamespacedSecretStore(cli, namespace) if err != nil { return nil, errors.Wrap(err, "failed to create namespaced secret store") } return repomanager.NewManager( namespace, cli, repoLocker, credentialFileStore, credentialSecretStore, logger, ), nil } func (o *Options) runRepoPrune(f velerocli.Factory, namespace string, logger logrus.FieldLogger) error { cli, err := o.initClient(f) if err != nil { return err } kubeClient, err := f.KubeClient() if err != nil { return err } var repo *velerov1api.BackupRepository retry := 10 for { repo, err = repository.GetBackupRepository(context.Background(), cli, namespace, repository.BackupRepositoryKey{ VolumeNamespace: o.RepoName, BackupLocation: o.BackupStorageLocation, RepositoryType: o.RepoType, }, true) if err == nil { break } retry-- if retry == 0 { break } logger.WithError(err).Warn("Failed to retrieve backup repo, need retry") time.Sleep(time.Second) } if err != nil { return errors.Wrap(err, "failed to get backup repository") } manager, err := initRepoManager(namespace, cli, kubeClient, logger) if err != nil { return err } err = manager.PruneRepo(repo) if err != nil { return errors.Wrap(err, "failed to prune repo") } return nil } ================================================ FILE: pkg/cmd/cli/restore/create.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package restore import ( "context" "fmt" "sort" "time" "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/tools/cache" kbclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/internal/resourcemodifiers" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" "github.com/vmware-tanzu/velero/pkg/cmd/util/flag" "github.com/vmware-tanzu/velero/pkg/cmd/util/output" "github.com/vmware-tanzu/velero/pkg/util/boolptr" "github.com/vmware-tanzu/velero/pkg/util/kube" "github.com/vmware-tanzu/velero/pkg/util/velero/restore" ) func NewCreateCommand(f client.Factory, use string) *cobra.Command { o := NewCreateOptions() c := &cobra.Command{ Use: use + " [RESTORE_NAME] [--from-backup BACKUP_NAME | --from-schedule SCHEDULE_NAME]", Short: "Create a restore", Example: ` # Create a restore named "restore-1" from backup "backup-1". velero restore create restore-1 --from-backup backup-1 # Create a restore with a default name ("backup-1-") from backup "backup-1". velero restore create --from-backup backup-1 # Create a restore from the latest successful backup triggered by schedule "schedule-1". velero restore create --from-schedule schedule-1 # Create a restore from the latest successful OR partially-failed backup triggered by schedule "schedule-1". velero restore create --from-schedule schedule-1 --allow-partially-failed # Create a restore for only persistentvolumeclaims and persistentvolumes within a backup. velero restore create --from-backup backup-2 --include-resources persistentvolumeclaims,persistentvolumes`, Args: cobra.MaximumNArgs(1), Run: func(c *cobra.Command, args []string) { cmd.CheckError(o.Complete(args, f)) cmd.CheckError(o.Validate(c, args, f)) cmd.CheckError(o.Run(c, f)) }, } o.BindFlags(c.Flags()) output.BindFlags(c.Flags()) output.ClearOutputFlagDefault(c) return c } type CreateOptions struct { BackupName string ScheduleName string RestoreName string RestoreVolumes flag.OptionalBool PreserveNodePorts flag.OptionalBool Labels flag.Map Annotations flag.Map IncludeNamespaces flag.StringArray ExcludeNamespaces flag.StringArray ExistingResourcePolicy string IncludeResources flag.StringArray ExcludeResources flag.StringArray StatusIncludeResources flag.StringArray StatusExcludeResources flag.StringArray NamespaceMappings flag.Map Selector flag.LabelSelector OrSelector flag.OrLabelSelector IncludeClusterResources flag.OptionalBool Wait bool AllowPartiallyFailed flag.OptionalBool ItemOperationTimeout time.Duration ResourceModifierConfigMap string WriteSparseFiles flag.OptionalBool ParallelFilesDownload int client kbclient.WithWatch } func NewCreateOptions() *CreateOptions { return &CreateOptions{ Labels: flag.NewMap(), Annotations: flag.NewMap(), IncludeNamespaces: flag.NewStringArray("*"), NamespaceMappings: flag.NewMap().WithEntryDelimiter(',').WithKeyValueDelimiter(':'), RestoreVolumes: flag.NewOptionalBool(nil), PreserveNodePorts: flag.NewOptionalBool(nil), IncludeClusterResources: flag.NewOptionalBool(nil), WriteSparseFiles: flag.NewOptionalBool(nil), } } func (o *CreateOptions) BindFlags(flags *pflag.FlagSet) { flags.StringVar(&o.BackupName, "from-backup", "", "Backup to restore from") flags.StringVar(&o.ScheduleName, "from-schedule", "", "Schedule to restore from") flags.Var(&o.IncludeNamespaces, "include-namespaces", "Namespaces to include in the restore (use '*' for all namespaces)") flags.Var(&o.ExcludeNamespaces, "exclude-namespaces", "Namespaces to exclude from the restore.") flags.Var(&o.NamespaceMappings, "namespace-mappings", "Namespace mappings from name in the backup to desired restored name in the form src1:dst1,src2:dst2,...") flags.Var(&o.Labels, "labels", "Labels to apply to the restore.") flags.Var(&o.Annotations, "annotations", "Annotations to apply to the restore.") flags.Var(&o.IncludeResources, "include-resources", "Resources to include in the restore, formatted as resource.group, such as storageclasses.storage.k8s.io (use '*' for all resources).") flags.Var(&o.ExcludeResources, "exclude-resources", "Resources to exclude from the restore, formatted as resource.group, such as storageclasses.storage.k8s.io.") flags.StringVar(&o.ExistingResourcePolicy, "existing-resource-policy", "", "Restore Policy to be used during the restore workflow, can be - none or update") flags.Var(&o.StatusIncludeResources, "status-include-resources", "Resources to include in the restore status, formatted as resource.group, such as storageclasses.storage.k8s.io.") flags.Var(&o.StatusExcludeResources, "status-exclude-resources", "Resources to exclude from the restore status, formatted as resource.group, such as storageclasses.storage.k8s.io.") flags.VarP(&o.Selector, "selector", "l", "Only restore resources matching this label selector.") flags.Var(&o.OrSelector, "or-selector", "Restore resources matching at least one of the label selector from the list. Label selectors should be separated by ' or '. For example, foo=bar or app=nginx") flags.DurationVar(&o.ItemOperationTimeout, "item-operation-timeout", o.ItemOperationTimeout, "How long to wait for async plugin operations before timeout.") f := flags.VarPF(&o.RestoreVolumes, "restore-volumes", "", "Whether to restore volumes from snapshots.") // this allows the user to just specify "--restore-volumes" as shorthand for "--restore-volumes=true" // like a normal bool flag f.NoOptDefVal = cmd.TRUE f = flags.VarPF(&o.PreserveNodePorts, "preserve-nodeports", "", "Whether to preserve nodeports of Services when restoring.") // this allows the user to just specify "--preserve-nodeports" as shorthand for "--preserve-nodeports=true" // like a normal bool flag f.NoOptDefVal = cmd.TRUE f = flags.VarPF(&o.IncludeClusterResources, "include-cluster-resources", "", "Include cluster-scoped resources in the restore.") f.NoOptDefVal = cmd.TRUE f = flags.VarPF(&o.AllowPartiallyFailed, "allow-partially-failed", "", "If using --from-schedule, whether to consider PartiallyFailed backups when looking for the most recent one. This flag has no effect if not using --from-schedule.") f.NoOptDefVal = cmd.TRUE flags.BoolVarP(&o.Wait, "wait", "w", o.Wait, "Wait for the operation to complete.") flags.StringVar(&o.ResourceModifierConfigMap, "resource-modifier-configmap", "", "Reference to the resource modifier configmap that restore will use") f = flags.VarPF(&o.WriteSparseFiles, "write-sparse-files", "", "Whether to write sparse files during restoring volumes") f.NoOptDefVal = cmd.TRUE flags.IntVar(&o.ParallelFilesDownload, "parallel-files-download", 0, "The number of restore operations to run in parallel. If set to 0, the default parallelism will be the number of CPUs for the node that node agent pod is running.") } func (o *CreateOptions) Complete(args []string, f client.Factory) error { if len(args) == 1 { o.RestoreName = args[0] } else { sourceName := o.BackupName if o.ScheduleName != "" { sourceName = o.ScheduleName } o.RestoreName = fmt.Sprintf("%s-%s", sourceName, time.Now().Format("20060102150405")) } client, err := f.KubebuilderWatchClient() if err != nil { return err } o.client = client return nil } func (o *CreateOptions) Validate(c *cobra.Command, args []string, f client.Factory) error { if o.BackupName != "" && o.ScheduleName != "" { return errors.New("either a backup or schedule must be specified, but not both") } if o.BackupName == "" && o.ScheduleName == "" { return errors.New("either a backup or schedule must be specified, but not both") } if err := output.ValidateFlags(c); err != nil { return err } if o.client == nil { // This should never happen return errors.New("Velero client is not set; unable to proceed") } if o.Selector.LabelSelector != nil && o.OrSelector.OrLabelSelectors != nil { return errors.New("either a 'selector' or an 'or-selector' can be specified, but not both") } if len(o.ExistingResourcePolicy) > 0 && !restore.IsResourcePolicyValid(o.ExistingResourcePolicy) { return errors.New("existing-resource-policy has invalid value, it accepts only none, update as value") } if o.ParallelFilesDownload < 0 { return errors.New("parallel-files-download cannot be negative") } switch { case o.BackupName != "": backup := new(api.Backup) if err := o.client.Get(context.TODO(), kbclient.ObjectKey{Namespace: f.Namespace(), Name: o.BackupName}, backup); err != nil { return err } case o.ScheduleName != "": backupList := new(api.BackupList) err := o.client.List(context.TODO(), backupList, &kbclient.ListOptions{ LabelSelector: labels.SelectorFromSet(map[string]string{api.ScheduleNameLabel: o.ScheduleName}), Namespace: f.Namespace(), }) if err != nil { return err } if len(backupList.Items) == 0 { return errors.Errorf("No backups found for the schedule %s", o.ScheduleName) } } return nil } // mostRecentBackup returns the backup with the most recent start timestamp that has a phase that's // in the provided list of allowed phases. func mostRecentBackup(backups []api.Backup, allowedPhases ...api.BackupPhase) *api.Backup { // sort the backups in descending order of start time (i.e. most recent to least recent) sort.Slice(backups, func(i, j int) bool { // Use .After() because we want descending sort. var iStartTime, jStartTime time.Time if backups[i].Status.StartTimestamp != nil { iStartTime = backups[i].Status.StartTimestamp.Time } if backups[j].Status.StartTimestamp != nil { jStartTime = backups[j].Status.StartTimestamp.Time } return iStartTime.After(jStartTime) }) // create a map of the allowed phases for easy lookup below phases := map[api.BackupPhase]struct{}{} for _, phase := range allowedPhases { phases[phase] = struct{}{} } var res *api.Backup for i, backup := range backups { // if the backup's phase is one of the allowable ones, record // the backup and break the loop so we can return it if _, ok := phases[backup.Status.Phase]; ok { res = &backups[i] break } } return res } func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error { if o.client == nil { // This should never happen return errors.New("Velero client is not set; unable to proceed") } // if --allow-partially-failed was specified, look up the most recent Completed or // PartiallyFailed backup for the provided schedule, and use that specific backup // to restore from. if o.ScheduleName != "" && boolptr.IsSetToTrue(o.AllowPartiallyFailed.Value) { backupList := new(api.BackupList) err := o.client.List(context.TODO(), backupList, &kbclient.ListOptions{ LabelSelector: labels.SelectorFromSet(map[string]string{api.ScheduleNameLabel: o.ScheduleName}), Namespace: f.Namespace(), }) if err != nil { return err } // if we find a Completed or PartiallyFailed backup for the schedule, restore specifically from that backup. If we don't // find one, proceed as-is -- the Velero server will handle validation. if backup := mostRecentBackup(backupList.Items, api.BackupPhaseCompleted, api.BackupPhasePartiallyFailed); backup != nil { // TODO(sk): this is kind of a hack -- we should revisit this and probably // move this logic to the server side or otherwise solve this problem. o.BackupName = backup.Name o.ScheduleName = "" } } var resModifiers *corev1api.TypedLocalObjectReference if o.ResourceModifierConfigMap != "" { resModifiers = &corev1api.TypedLocalObjectReference{ // Group for core API is "" APIGroup: &corev1api.SchemeGroupVersion.Group, Kind: resourcemodifiers.ConfigmapRefType, Name: o.ResourceModifierConfigMap, } } restore := &api.Restore{ ObjectMeta: metav1.ObjectMeta{ Namespace: f.Namespace(), Name: o.RestoreName, Labels: o.Labels.Data(), Annotations: o.Annotations.Data(), }, Spec: api.RestoreSpec{ BackupName: o.BackupName, ScheduleName: o.ScheduleName, IncludedNamespaces: o.IncludeNamespaces, ExcludedNamespaces: o.ExcludeNamespaces, IncludedResources: o.IncludeResources, ExcludedResources: o.ExcludeResources, ExistingResourcePolicy: api.PolicyType(o.ExistingResourcePolicy), NamespaceMapping: o.NamespaceMappings.Data(), LabelSelector: o.Selector.LabelSelector, OrLabelSelectors: o.OrSelector.OrLabelSelectors, RestorePVs: o.RestoreVolumes.Value, PreserveNodePorts: o.PreserveNodePorts.Value, IncludeClusterResources: o.IncludeClusterResources.Value, ResourceModifier: resModifiers, ItemOperationTimeout: metav1.Duration{ Duration: o.ItemOperationTimeout, }, UploaderConfig: &api.UploaderConfigForRestore{ WriteSparseFiles: o.WriteSparseFiles.Value, ParallelFilesDownload: o.ParallelFilesDownload, }, }, } if len([]string(o.StatusIncludeResources)) > 0 { restore.Spec.RestoreStatus = &api.RestoreStatusSpec{ IncludedResources: o.StatusIncludeResources, ExcludedResources: o.StatusExcludeResources, } } if printed, err := output.PrintWithFormat(c, restore); printed || err != nil { return err } var updates chan *api.Restore if o.Wait { stop := make(chan struct{}) defer close(stop) updates = make(chan *api.Restore) lw := kube.InternalLW{ Client: o.client, Namespace: f.Namespace(), ObjectList: new(api.RestoreList), } restoreInformer := cache.NewSharedInformer(&lw, &api.Restore{}, time.Second) _, _ = restoreInformer.AddEventHandler( cache.FilteringResourceEventHandler{ FilterFunc: func(obj any) bool { restore, ok := obj.(*api.Restore) if !ok { return false } return restore.Name == o.RestoreName }, Handler: cache.ResourceEventHandlerFuncs{ UpdateFunc: func(_, obj any) { restore, ok := obj.(*api.Restore) if !ok { return } updates <- restore }, DeleteFunc: func(obj any) { restore, ok := obj.(*api.Restore) if !ok { return } updates <- restore }, }, }, ) go restoreInformer.Run(stop) } err := o.client.Create(context.TODO(), restore, &kbclient.CreateOptions{}) if err != nil { return err } fmt.Printf("Restore request %q submitted successfully.\n", restore.Name) if o.Wait { fmt.Println("Waiting for restore to complete. You may safely press ctrl-c to stop waiting - your restore will continue in the background.") ticker := time.NewTicker(time.Second) defer ticker.Stop() for { select { case <-ticker.C: fmt.Print(".") case restore, ok := <-updates: if !ok { fmt.Println("\nError waiting: unable to watch restores.") return nil } if restore.Status.Phase == api.RestorePhaseFailedValidation || restore.Status.Phase == api.RestorePhaseCompleted || restore.Status.Phase == api.RestorePhasePartiallyFailed || restore.Status.Phase == api.RestorePhaseFailed { fmt.Printf("\nRestore completed with status: %s. You may check for more information using the commands `velero restore describe %s` and `velero restore logs %s`.\n", restore.Status.Phase, restore.Name, restore.Name) return nil } } } } // Not waiting fmt.Printf("Run `velero restore describe %s` or `velero restore logs %s` for more details.\n", restore.Name, restore.Name) return nil } ================================================ FILE: pkg/cmd/cli/restore/create_test.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package restore import ( "testing" "time" "github.com/spf13/pflag" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" controllerclient "sigs.k8s.io/controller-runtime/pkg/client" kbclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" factorymocks "github.com/vmware-tanzu/velero/pkg/client/mocks" cmdtest "github.com/vmware-tanzu/velero/pkg/cmd/test" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func TestMostRecentBackup(t *testing.T) { backups := []velerov1api.Backup{ *builder.ForBackup(cmdtest.VeleroNameSpace, "backup0").StartTimestamp(time.Now().Add(3 * time.Second)).Phase(velerov1api.BackupPhaseDeleting).Result(), *builder.ForBackup(cmdtest.VeleroNameSpace, "backup1").StartTimestamp(time.Now().Add(time.Second)).Phase(velerov1api.BackupPhaseCompleted).Result(), *builder.ForBackup(cmdtest.VeleroNameSpace, "backup2").StartTimestamp(time.Now().Add(2 * time.Second)).Phase(velerov1api.BackupPhasePartiallyFailed).Result(), } expectedBackup := builder.ForBackup(cmdtest.VeleroNameSpace, "backup2").StartTimestamp(time.Now().Add(2 * time.Second)).Phase(velerov1api.BackupPhasePartiallyFailed).Result() resultBackup := mostRecentBackup(backups, velerov1api.BackupPhaseCompleted, velerov1api.BackupPhasePartiallyFailed) require.Equal(t, expectedBackup.Name, resultBackup.Name) } func TestCreateCommand(t *testing.T) { name := "nameToBeCreated" args := []string{name} t.Run("create a backup create command with full options except fromSchedule and wait, then run by create option", func(t *testing.T) { // create a factory f := &factorymocks.Factory{} // create command cmd := NewCreateCommand(f, "") require.Equal(t, "Create a restore", cmd.Short) backupName := "backup1" scheduleName := "schedule1" restoreVolumes := "true" preserveNodePorts := "true" labels := "c=foo" annotations := "ann=foo" includeNamespaces := "app1,app2" excludeNamespaces := "pod1,pod2,pod3" existingResourcePolicy := "none" includeResources := "sc,sts" excludeResources := "job" statusIncludeResources := "sc,sts" statusExcludeResources := "job" namespaceMappings := "a:b" selector := "foo=bar" includeClusterResources := "true" allowPartiallyFailed := "true" itemOperationTimeout := "10m0s" writeSparseFiles := "true" parallel := 2 flags := new(pflag.FlagSet) o := NewCreateOptions() o.BindFlags(flags) flags.Parse([]string{"--from-backup", backupName}) flags.Parse([]string{"--from-schedule", scheduleName}) flags.Parse([]string{"--restore-volumes", restoreVolumes}) flags.Parse([]string{"--preserve-nodeports", preserveNodePorts}) flags.Parse([]string{"--labels", labels}) flags.Parse([]string{"--annotations", annotations}) flags.Parse([]string{"--existing-resource-policy", existingResourcePolicy}) flags.Parse([]string{"--include-namespaces", includeNamespaces}) flags.Parse([]string{"--exclude-namespaces", excludeNamespaces}) flags.Parse([]string{"--include-resources", includeResources}) flags.Parse([]string{"--exclude-resources", excludeResources}) flags.Parse([]string{"--status-include-resources", statusIncludeResources}) flags.Parse([]string{"--status-exclude-resources", statusExcludeResources}) flags.Parse([]string{"--namespace-mappings", namespaceMappings}) flags.Parse([]string{"--selector", selector}) flags.Parse([]string{"--include-cluster-resources", includeClusterResources}) flags.Parse([]string{"--allow-partially-failed", allowPartiallyFailed}) flags.Parse([]string{"--item-operation-timeout", itemOperationTimeout}) flags.Parse([]string{"--write-sparse-files", writeSparseFiles}) flags.Parse([]string{"--parallel-files-download", "2"}) client := velerotest.NewFakeControllerRuntimeClient(t).(kbclient.WithWatch) f.On("Namespace").Return(mock.Anything) f.On("KubebuilderWatchClient").Return(client, nil) //Complete e := o.Complete(args, f) require.NoError(t, e) //Validate e = o.Validate(cmd, args, f) require.ErrorContains(t, e, "either a backup or schedule must be specified, but not both") //cmd e = o.Run(cmd, f) require.NoError(t, e) require.Equal(t, backupName, o.BackupName) require.Equal(t, scheduleName, o.ScheduleName) require.Equal(t, restoreVolumes, o.RestoreVolumes.String()) require.Equal(t, preserveNodePorts, o.PreserveNodePorts.String()) require.Equal(t, labels, o.Labels.String()) require.Equal(t, annotations, o.Annotations.String()) require.Equal(t, includeNamespaces, o.IncludeNamespaces.String()) require.Equal(t, excludeNamespaces, o.ExcludeNamespaces.String()) require.Equal(t, existingResourcePolicy, o.ExistingResourcePolicy) require.Equal(t, includeResources, o.IncludeResources.String()) require.Equal(t, excludeResources, o.ExcludeResources.String()) require.Equal(t, statusIncludeResources, o.StatusIncludeResources.String()) require.Equal(t, statusExcludeResources, o.StatusExcludeResources.String()) require.Equal(t, namespaceMappings, o.NamespaceMappings.String()) require.Equal(t, selector, o.Selector.String()) require.Equal(t, includeClusterResources, o.IncludeClusterResources.String()) require.Equal(t, allowPartiallyFailed, o.AllowPartiallyFailed.String()) require.Equal(t, itemOperationTimeout, o.ItemOperationTimeout.String()) require.Equal(t, writeSparseFiles, o.WriteSparseFiles.String()) require.Equal(t, parallel, o.ParallelFilesDownload) }) t.Run("create a restore from schedule", func(t *testing.T) { f := &factorymocks.Factory{} c := NewCreateCommand(f, "") require.Equal(t, "Create a restore", c.Short) flags := new(pflag.FlagSet) o := NewCreateOptions() o.BindFlags(flags) fromSchedule := "schedule-name-1" flags.Parse([]string{"--from-schedule", fromSchedule}) kbclient := velerotest.NewFakeControllerRuntimeClient(t).(kbclient.WithWatch) schedule := builder.ForSchedule(cmdtest.VeleroNameSpace, fromSchedule).Result() require.NoError(t, kbclient.Create(t.Context(), schedule, &controllerclient.CreateOptions{})) backup := builder.ForBackup(cmdtest.VeleroNameSpace, "test-backup").FromSchedule(schedule).Phase(velerov1api.BackupPhaseCompleted).Result() require.NoError(t, kbclient.Create(t.Context(), backup, &controllerclient.CreateOptions{})) f.On("Namespace").Return(cmdtest.VeleroNameSpace) f.On("KubebuilderWatchClient").Return(kbclient, nil) require.NoError(t, o.Complete(args, f)) require.NoError(t, o.Validate(c, []string{}, f)) require.NoError(t, o.Run(c, f)) }) t.Run("create a restore from not-existed backup", func(t *testing.T) { f := &factorymocks.Factory{} c := NewCreateCommand(f, "") require.Equal(t, "Create a restore", c.Short) flags := new(pflag.FlagSet) o := NewCreateOptions() o.BindFlags(flags) nonExistedBackupName := "not-exist" flags.Parse([]string{"--wait", "true"}) flags.Parse([]string{"--from-backup", nonExistedBackupName}) kbclient := velerotest.NewFakeControllerRuntimeClient(t).(kbclient.WithWatch) f.On("Namespace").Return(cmdtest.VeleroNameSpace) f.On("KubebuilderWatchClient").Return(kbclient, nil) require.NoError(t, o.Complete(nil, f)) err := o.Validate(c, []string{}, f) require.Equal(t, "backups.velero.io \"not-exist\" not found", err.Error()) }) } ================================================ FILE: pkg/cmd/cli/restore/delete.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package restore import ( "context" "fmt" "github.com/pkg/errors" "github.com/spf13/cobra" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" kubeerrs "k8s.io/apimachinery/pkg/util/errors" controllerclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" "github.com/vmware-tanzu/velero/pkg/cmd/cli" "github.com/vmware-tanzu/velero/pkg/cmd/util/confirm" ) // NewDeleteCommand creates and returns a new cobra command for deleting restores. func NewDeleteCommand(f client.Factory, use string) *cobra.Command { o := cli.NewDeleteOptions("restore") c := &cobra.Command{ Use: fmt.Sprintf("%s [NAMES]", use), Short: "Delete restores", Example: ` # Delete a restore named "restore-1". velero restore delete restore-1 # Delete a restore named "restore-1" without prompting for confirmation. velero restore delete restore-1 --confirm # Delete restores named "restore-1" and "restore-2". velero restore delete restore-1 restore-2 # Delete all restores labeled with "foo=bar". velero restore delete --selector foo=bar # Delete all restores. velero restore delete --all`, Run: func(c *cobra.Command, args []string) { cmd.CheckError(o.Complete(f, args)) cmd.CheckError(o.Validate(c, f, args)) cmd.CheckError(Run(o)) }, } o.BindFlags(c.Flags()) return c } // Run performs the deletion of restore(s). func Run(o *cli.DeleteOptions) error { if !o.Confirm && !confirm.GetConfirmation() { return nil } var ( restores []*velerov1api.Restore errs []error ) switch { case len(o.Names) > 0: for _, name := range o.Names { restore := new(velerov1api.Restore) err := o.Client.Get(context.TODO(), controllerclient.ObjectKey{Namespace: o.Namespace, Name: name}, restore) if err != nil { errs = append(errs, errors.WithStack(err)) continue } restores = append(restores, restore) } default: selector := labels.Everything() if o.Selector.LabelSelector != nil { convertedSelector, err := metav1.LabelSelectorAsSelector(o.Selector.LabelSelector) if err != nil { return errors.WithStack(err) } selector = convertedSelector } restoreList := new(velerov1api.RestoreList) err := o.Client.List(context.TODO(), restoreList, &controllerclient.ListOptions{ Namespace: o.Namespace, LabelSelector: selector, }) if err != nil { errs = append(errs, errors.WithStack(err)) } for i := range restoreList.Items { restores = append(restores, &restoreList.Items[i]) } } if len(errs) > 0 { fmt.Println("errs: ", errs) return kubeerrs.NewAggregate(errs) } if len(restores) == 0 { fmt.Println("No restores found") return nil } for _, r := range restores { err := o.Client.Delete(context.TODO(), r, &controllerclient.DeleteOptions{}) if err != nil { errs = append(errs, errors.WithStack(err)) continue } fmt.Printf("Request to delete restore %q submitted successfully.\nThe restore will be fully deleted after all associated data (restore files in object storage) are removed.\n", r.Name) } return kubeerrs.NewAggregate(errs) } ================================================ FILE: pkg/cmd/cli/restore/delete_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package restore import ( "fmt" "os" "os/exec" "testing" flag "github.com/spf13/pflag" "github.com/stretchr/testify/require" controllerclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/pkg/builder" factorymocks "github.com/vmware-tanzu/velero/pkg/client/mocks" "github.com/vmware-tanzu/velero/pkg/cmd/cli" cmdtest "github.com/vmware-tanzu/velero/pkg/cmd/test" velerotest "github.com/vmware-tanzu/velero/pkg/test" veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" ) func TestDeleteCommand(t *testing.T) { restore1 := "restore-name-1" restore2 := "restore-name-2" // create a factory f := &factorymocks.Factory{} client := velerotest.NewFakeControllerRuntimeClient(t) client.Create(t.Context(), builder.ForRestore(cmdtest.VeleroNameSpace, restore1).Result(), &controllerclient.CreateOptions{}) client.Create(t.Context(), builder.ForRestore("default", restore2).Result(), &controllerclient.CreateOptions{}) f.On("KubebuilderClient").Return(client, nil) f.On("Namespace").Return(cmdtest.VeleroNameSpace) // create command c := NewDeleteCommand(f, "velero restore delete") c.SetArgs([]string{restore1, restore2}) require.Equal(t, "Delete restores", c.Short) o := cli.NewDeleteOptions("restore") flags := new(flag.FlagSet) o.BindFlags(flags) flags.Parse([]string{"--confirm"}) args := []string{restore1, restore2} e := o.Complete(f, args) require.NoError(t, e) e = o.Validate(c, f, args) require.NoError(t, e) Run(o) e = c.Execute() require.NoError(t, e) if os.Getenv(cmdtest.CaptureFlag) == "1" { return } cmd := exec.CommandContext(t.Context(), os.Args[0], []string{"-test.run=TestDeleteCommand"}...) cmd.Env = append(os.Environ(), fmt.Sprintf("%s=1", cmdtest.CaptureFlag)) stdout, _, err := veleroexec.RunCommand(cmd) if err != nil { require.Contains(t, stdout, fmt.Sprintf("restores.velero.io \"%s\" not found.", restore2)) } } ================================================ FILE: pkg/cmd/cli/restore/describe.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package restore import ( "context" "fmt" "os" "github.com/spf13/cobra" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" controllerclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" "github.com/vmware-tanzu/velero/pkg/cmd/util/output" "github.com/vmware-tanzu/velero/pkg/label" ) func NewDescribeCommand(f client.Factory, use string) *cobra.Command { var ( listOptions metav1.ListOptions details bool insecureSkipTLSVerify bool ) config, err := client.LoadConfig() if err != nil { fmt.Fprintf(os.Stderr, "WARNING: Error reading config file: %v\n", err) } caCertFile := config.CACertFile() c := &cobra.Command{ Use: use + " [NAME1] [NAME2] [NAME...]", Short: "Describe restores", Run: func(c *cobra.Command, args []string) { kbClient, err := f.KubebuilderClient() cmd.CheckError(err) restoreList := new(velerov1api.RestoreList) if len(args) > 0 { for _, name := range args { restore := new(velerov1api.Restore) err := kbClient.Get(context.TODO(), controllerclient.ObjectKey{Namespace: f.Namespace(), Name: name}, restore) cmd.CheckError(err) restoreList.Items = append(restoreList.Items, *restore) } } else { parsedSelector, err := labels.Parse(listOptions.LabelSelector) cmd.CheckError(err) err = kbClient.List(context.TODO(), restoreList, &controllerclient.ListOptions{LabelSelector: parsedSelector, Namespace: f.Namespace()}) cmd.CheckError(err) } first := true for i, restore := range restoreList.Items { podVolumeRestoreList := new(velerov1api.PodVolumeRestoreList) err = kbClient.List(context.TODO(), podVolumeRestoreList, &controllerclient.ListOptions{ Namespace: f.Namespace(), LabelSelector: labels.SelectorFromSet(map[string]string{velerov1api.RestoreNameLabel: label.GetValidName(restore.Name)}), }) if err != nil { fmt.Fprintf(os.Stderr, "error getting PodVolumeRestores for restore %s: %v\n", restore.Name, err) } s := output.DescribeRestore(context.Background(), kbClient, &restoreList.Items[i], podVolumeRestoreList.Items, details, insecureSkipTLSVerify, caCertFile) if first { first = false fmt.Print(s) } else { fmt.Printf("\n\n%s", s) } } cmd.CheckError(err) }, } c.Flags().StringVarP(&listOptions.LabelSelector, "selector", "l", listOptions.LabelSelector, "Only show items matching this label selector.") c.Flags().BoolVar(&details, "details", details, "Display additional detail in the command output.") c.Flags().BoolVar(&insecureSkipTLSVerify, "insecure-skip-tls-verify", insecureSkipTLSVerify, "If true, the object store's TLS certificate will not be checked for validity. This is insecure and susceptible to man-in-the-middle attacks. Not recommended for production.") c.Flags().StringVar(&caCertFile, "cacert", caCertFile, "Path to a certificate bundle to use when verifying TLS connections.") return c } ================================================ FILE: pkg/cmd/cli/restore/describe_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package restore import ( "fmt" "os" "os/exec" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/client-go/rest" controllerclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/pkg/builder" factorymocks "github.com/vmware-tanzu/velero/pkg/client/mocks" cmdtest "github.com/vmware-tanzu/velero/pkg/cmd/test" "github.com/vmware-tanzu/velero/pkg/features" "github.com/vmware-tanzu/velero/pkg/test" veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" ) func TestNewDescribeCommand(t *testing.T) { // create a factory f := &factorymocks.Factory{} restoreName := "restore-describe-1" testRestore := builder.ForRestore(cmdtest.VeleroNameSpace, restoreName).Result() clientConfig := rest.Config{} kbClient := test.NewFakeControllerRuntimeClient(t) kbClient.Create(t.Context(), testRestore, &controllerclient.CreateOptions{}) f.On("ClientConfig").Return(&clientConfig, nil) f.On("Namespace").Return(cmdtest.VeleroNameSpace) f.On("KubebuilderClient").Return(kbClient, nil) // create command c := NewDescribeCommand(f, "velero restore describe") assert.Equal(t, "Describe restores", c.Short) features.NewFeatureFlagSet("EnableCSI") defer features.NewFeatureFlagSet() c.SetArgs([]string{restoreName}) e := c.Execute() require.NoError(t, e) if os.Getenv(cmdtest.CaptureFlag) == "1" { return } cmd := exec.CommandContext(t.Context(), os.Args[0], []string{"-test.run=TestNewDescribeCommand"}...) cmd.Env = append(os.Environ(), fmt.Sprintf("%s=1", cmdtest.CaptureFlag)) stdout, _, err := veleroexec.RunCommand(cmd) if err == nil { assert.Contains(t, stdout, fmt.Sprintf("Name: %s", restoreName)) return } t.Fatalf("process ran with err %v, want backups by get()", err) } ================================================ FILE: pkg/cmd/cli/restore/get.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package restore import ( "context" "github.com/spf13/cobra" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" controllerclient "sigs.k8s.io/controller-runtime/pkg/client" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" "github.com/vmware-tanzu/velero/pkg/cmd/util/output" ) func NewGetCommand(f client.Factory, use string) *cobra.Command { var listOptions metav1.ListOptions c := &cobra.Command{ Use: use, Short: "Get restores", Run: func(c *cobra.Command, args []string) { err := output.ValidateFlags(c) cmd.CheckError(err) kbClient, err := f.KubebuilderClient() cmd.CheckError(err) restores := new(api.RestoreList) if len(args) > 0 { for _, name := range args { restore := new(api.Restore) err := kbClient.Get(context.TODO(), controllerclient.ObjectKey{Namespace: f.Namespace(), Name: name}, restore) cmd.CheckError(err) restores.Items = append(restores.Items, *restore) } } else { parsedSelector, err := labels.Parse(listOptions.LabelSelector) cmd.CheckError(err) err = kbClient.List(context.TODO(), restores, &controllerclient.ListOptions{LabelSelector: parsedSelector, Namespace: f.Namespace()}) cmd.CheckError(err) } // Append "(Deleting)" to phase if deletionTimestamp is marked. for i := range restores.Items { if !restores.Items[i].DeletionTimestamp.IsZero() { restores.Items[i].Status.Phase += " (Deleting)" } } if printed, err := output.PrintWithFormat(c, restores); printed || err != nil { cmd.CheckError(err) return } _, err = output.PrintWithFormat(c, restores) cmd.CheckError(err) }, } c.Flags().StringVarP(&listOptions.LabelSelector, "selector", "l", listOptions.LabelSelector, "Only show items matching this label selector.") output.BindFlags(c.Flags()) return c } ================================================ FILE: pkg/cmd/cli/restore/get_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package restore import ( "fmt" "os" "os/exec" "strings" "testing" "github.com/stretchr/testify/require" kbclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/pkg/builder" factorymocks "github.com/vmware-tanzu/velero/pkg/client/mocks" cmdtest "github.com/vmware-tanzu/velero/pkg/cmd/test" velerotest "github.com/vmware-tanzu/velero/pkg/test" veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" ) func TestNewGetCommand(t *testing.T) { args := []string{"b1", "b2", "b3"} // create a factory f := &factorymocks.Factory{} client := velerotest.NewFakeControllerRuntimeClient(t) for _, restoreName := range args { restore := builder.ForRestore(cmdtest.VeleroNameSpace, restoreName).ObjectMeta(builder.WithLabels("abc", "abc")).Result() err := client.Create(t.Context(), restore, &kbclient.CreateOptions{}) require.NoError(t, err) } f.On("KubebuilderClient").Return(client, nil) f.On("Namespace").Return(cmdtest.VeleroNameSpace) // create command c := NewGetCommand(f, "velero restore get") require.Equal(t, "Get restores", c.Short) c.SetArgs(args) e := c.Execute() require.NoError(t, e) if os.Getenv(cmdtest.CaptureFlag) == "1" { return } cmd := exec.CommandContext(t.Context(), os.Args[0], []string{"-test.run=TestNewGetCommand"}...) cmd.Env = append(os.Environ(), fmt.Sprintf("%s=1", cmdtest.CaptureFlag)) stdout, _, err := veleroexec.RunCommand(cmd) require.NoError(t, err) if err == nil { output := strings.Split(stdout, "\n") i := 0 for _, line := range output { if strings.Contains(line, "New") { i++ } } require.Len(t, args, i) } } ================================================ FILE: pkg/cmd/cli/restore/logs.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package restore import ( "context" "fmt" "os" "time" "github.com/spf13/cobra" apierrors "k8s.io/apimachinery/pkg/api/errors" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" "github.com/vmware-tanzu/velero/pkg/cmd/util/cacert" "github.com/vmware-tanzu/velero/pkg/cmd/util/downloadrequest" ) func NewLogsCommand(f client.Factory) *cobra.Command { config, err := client.LoadConfig() if err != nil { fmt.Fprintf(os.Stderr, "WARNING: Error reading config file: %v\n", err) } timeout := time.Minute insecureSkipTLSVerify := false caCertFile := config.CACertFile() c := &cobra.Command{ Use: "logs RESTORE", Short: "Get restore logs", Args: cobra.ExactArgs(1), Run: func(c *cobra.Command, args []string) { restoreName := args[0] kbClient, err := f.KubebuilderClient() cmd.CheckError(err) restore := new(velerov1api.Restore) err = kbClient.Get(context.Background(), ctrlclient.ObjectKey{Namespace: f.Namespace(), Name: restoreName}, restore) if apierrors.IsNotFound(err) { cmd.Exit("Restore %q does not exist.", restoreName) } else if err != nil { cmd.Exit("Error checking for restore %q: %v", restoreName, err) } switch restore.Status.Phase { case velerov1api.RestorePhaseCompleted, velerov1api.RestorePhaseFailed, velerov1api.RestorePhasePartiallyFailed, velerov1api.RestorePhaseWaitingForPluginOperations, velerov1api.RestorePhaseWaitingForPluginOperationsPartiallyFailed: // terminal and waiting for plugin operations phases, don't exit. default: cmd.Exit("Logs for restore %q are not available until it's finished processing. Please wait "+ "until the restore has a phase of Completed or Failed and try again.", restoreName) } // Get BSL cacert if available bslCACert, err := cacert.GetCACertFromRestore(context.Background(), kbClient, f.Namespace(), restore) if err != nil { // Log the error but don't fail - we can still try to download without the BSL cacert fmt.Fprintf(os.Stderr, "WARNING: Error getting cacert from BSL: %v\n", err) bslCACert = "" } err = downloadrequest.StreamWithBSLCACert(context.Background(), kbClient, f.Namespace(), restoreName, velerov1api.DownloadTargetKindRestoreLog, os.Stdout, timeout, insecureSkipTLSVerify, caCertFile, bslCACert) cmd.CheckError(err) }, } c.Flags().DurationVar(&timeout, "timeout", timeout, "How long to wait to receive logs.") c.Flags().BoolVar(&insecureSkipTLSVerify, "insecure-skip-tls-verify", insecureSkipTLSVerify, "If true, the object store's TLS certificate will not be checked for validity. This is insecure and susceptible to man-in-the-middle attacks. Not recommended for production.") c.Flags().StringVar(&caCertFile, "cacert", caCertFile, "Path to a certificate bundle to use when verifying TLS connections. If not specified, the CA certificate from the BackupStorageLocation will be used if available.") return c } ================================================ FILE: pkg/cmd/cli/restore/logs_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package restore import ( "os" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" kbclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" factorymocks "github.com/vmware-tanzu/velero/pkg/client/mocks" cmdtest "github.com/vmware-tanzu/velero/pkg/cmd/test" "github.com/vmware-tanzu/velero/pkg/cmd/util/cacert" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func TestNewLogsCommand(t *testing.T) { t.Run("Flag test", func(t *testing.T) { // create a factory f := &factorymocks.Factory{} c := NewLogsCommand(f) require.Equal(t, "Get restore logs", c.Short) // Test flag parsing timeout := "1s" insecureSkipTLSVerify := "true" caCertFile := "testing" c.Flags().Set("timeout", timeout) c.Flags().Set("insecure-skip-tls-verify", insecureSkipTLSVerify) c.Flags().Set("cacert", caCertFile) timeoutFlag, _ := c.Flags().GetDuration("timeout") require.Equal(t, 1*time.Second, timeoutFlag) insecureFlag, _ := c.Flags().GetBool("insecure-skip-tls-verify") require.True(t, insecureFlag) caCertFlag, _ := c.Flags().GetString("cacert") require.Equal(t, caCertFile, caCertFlag) }) t.Run("Restore not complete test", func(t *testing.T) { restoreName := "rs-logs-1" // create a factory f := &factorymocks.Factory{} kbClient := velerotest.NewFakeControllerRuntimeClient(t) restore := builder.ForRestore(cmdtest.VeleroNameSpace, restoreName).Result() err := kbClient.Create(t.Context(), restore, &kbclient.CreateOptions{}) require.NoError(t, err) f.On("Namespace").Return(cmdtest.VeleroNameSpace) f.On("KubebuilderClient").Return(kbClient, nil) c := NewLogsCommand(f) assert.Equal(t, "Get restore logs", c.Short) // The restore command exits with an error message when restore is not complete // We can't easily test this since it calls cmd.Exit, which exits the process // So we'll skip this test case t.Skip("Cannot test restore not complete case due to cmd.Exit() call") }) t.Run("Restore not exist test", func(t *testing.T) { // create a factory f := &factorymocks.Factory{} kbClient := velerotest.NewFakeControllerRuntimeClient(t) f.On("Namespace").Return(cmdtest.VeleroNameSpace) f.On("KubebuilderClient").Return(kbClient, nil) c := NewLogsCommand(f) assert.Equal(t, "Get restore logs", c.Short) // The restore command exits with an error message when restore doesn't exist // We can't easily test this since it calls cmd.Exit, which exits the process // So we'll skip this test case t.Skip("Cannot test restore not exist case due to cmd.Exit() call") }) t.Run("Restore with BSL cacert test", func(t *testing.T) { restoreName := "rs-logs-with-cacert" backupName := "bk-for-restore" bslName := "test-bsl" // create a factory f := &factorymocks.Factory{} kbClient := velerotest.NewFakeControllerRuntimeClient(t) // Create BSL with cacert bsl := builder.ForBackupStorageLocation(cmdtest.VeleroNameSpace, bslName). Provider("aws"). Bucket("test-bucket"). CACert([]byte("test-cacert-content")). Result() err := kbClient.Create(t.Context(), bsl, &kbclient.CreateOptions{}) require.NoError(t, err) // Create backup referencing the BSL backup := builder.ForBackup(cmdtest.VeleroNameSpace, backupName). StorageLocation(bslName). Result() err = kbClient.Create(t.Context(), backup, &kbclient.CreateOptions{}) require.NoError(t, err) // Create restore referencing the backup restore := builder.ForRestore(cmdtest.VeleroNameSpace, restoreName). Phase(velerov1api.RestorePhaseCompleted). Backup(backupName). Result() err = kbClient.Create(t.Context(), restore, &kbclient.CreateOptions{}) require.NoError(t, err) f.On("Namespace").Return(cmdtest.VeleroNameSpace) f.On("KubebuilderClient").Return(kbClient, nil) c := NewLogsCommand(f) assert.Equal(t, "Get restore logs", c.Short) // We can verify that BSL cacert fetching logic is in place // The actual command will call downloadrequest which requires a controller // to be running, so we'll just verify the command structure require.NotNil(t, c.Run) // Verify the BSL cacert can be fetched cacertValue, err := cacert.GetCACertFromRestore(t.Context(), kbClient, f.Namespace(), restore) require.NoError(t, err) require.Equal(t, "test-cacert-content", cacertValue) }) t.Run("CLI execution test", func(t *testing.T) { // create a factory f := &factorymocks.Factory{} c := NewLogsCommand(f) require.Equal(t, "Get restore logs", c.Short) if os.Getenv(cmdtest.CaptureFlag) == "1" { c.SetArgs([]string{"test"}) e := c.Execute() assert.NoError(t, e) return } }) } ================================================ FILE: pkg/cmd/cli/restore/restore.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package restore import ( "github.com/spf13/cobra" "github.com/vmware-tanzu/velero/pkg/client" ) func NewCommand(f client.Factory) *cobra.Command { c := &cobra.Command{ Use: "restore", Short: "Work with restores", Long: "Work with restores", } c.AddCommand( NewCreateCommand(f, "create"), NewGetCommand(f, "get"), NewLogsCommand(f), NewDescribeCommand(f, "describe"), NewDeleteCommand(f, "delete"), ) return c } ================================================ FILE: pkg/cmd/cli/restore/restore_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package restore import ( "testing" "github.com/stretchr/testify/assert" factorymocks "github.com/vmware-tanzu/velero/pkg/client/mocks" ) func TestNewRestoreCommand(t *testing.T) { // create a factory f := &factorymocks.Factory{} // create command cmd := NewCommand(f) assert.Equal(t, "Work with restores", cmd.Short) } ================================================ FILE: pkg/cmd/cli/schedule/create.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package schedule import ( "context" "fmt" "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/vmware-tanzu/velero/internal/resourcepolicies" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" "github.com/vmware-tanzu/velero/pkg/cmd/cli/backup" "github.com/vmware-tanzu/velero/pkg/cmd/util/output" ) func NewCreateCommand(f client.Factory, use string) *cobra.Command { o := NewCreateOptions() c := &cobra.Command{ Use: use + " NAME --schedule", Short: "Create a schedule", Long: `The --schedule flag is required, in cron notation, using UTC time: | Character Position | Character Period | Acceptable Values | | -------------------|:----------------:| -----------------:| | 1 | Minute | 0-59,* | | 2 | Hour | 0-23,* | | 3 | Day of Month | 1-31,* | | 4 | Month | 1-12,* | | 5 | Day of Week | 0-6,* | The schedule can also be expressed using "@every " syntax. The duration can be specified using a combination of seconds (s), minutes (m), and hours (h), for example: "@every 2h30m".`, Example: ` # Create a backup every 6 hours. velero create schedule NAME --schedule="0 */6 * * *" # Create a backup every 6 hours with the @every notation. velero create schedule NAME --schedule="@every 6h" # Create a daily backup of the web namespace. velero create schedule NAME --schedule="@every 24h" --include-namespaces web # Create a weekly backup, each living for 90 days (2160 hours). velero create schedule NAME --schedule="@every 168h" --ttl 2160h0m0s`, Args: cobra.ExactArgs(1), Run: func(c *cobra.Command, args []string) { cmd.CheckError(o.Complete(args, f)) cmd.CheckError(o.Validate(c, args, f)) cmd.CheckError(o.Run(c, f)) }, } o.BindFlags(c.Flags()) output.BindFlags(c.Flags()) output.ClearOutputFlagDefault(c) return c } type CreateOptions struct { BackupOptions *backup.CreateOptions SkipOptions *SkipOptions Schedule string UseOwnerReferencesInBackup bool Paused bool } func NewCreateOptions() *CreateOptions { return &CreateOptions{ BackupOptions: backup.NewCreateOptions(), SkipOptions: NewSkipOptions(), } } func (o *CreateOptions) BindFlags(flags *pflag.FlagSet) { o.BackupOptions.BindFlags(flags) o.SkipOptions.BindFlags(flags) flags.StringVar(&o.Schedule, "schedule", o.Schedule, "A cron expression specifying a recurring schedule for this backup to run") flags.BoolVar(&o.UseOwnerReferencesInBackup, "use-owner-references-in-backup", o.UseOwnerReferencesInBackup, "Specifies whether to use OwnerReferences on backups created by this Schedule. Notice: if set to true, when schedule is deleted, backups will be deleted too.") flags.BoolVar(&o.Paused, "paused", o.Paused, "Specifies whether the newly created schedule is paused or not.") } func (o *CreateOptions) Validate(c *cobra.Command, args []string, f client.Factory) error { if len(o.Schedule) == 0 { return errors.New("--schedule is required") } return o.BackupOptions.Validate(c, args, f) } func (o *CreateOptions) Complete(args []string, f client.Factory) error { return o.BackupOptions.Complete(args, f) } func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error { var orders map[string]string crClient, err := f.KubebuilderClient() if err != nil { return err } if len(o.BackupOptions.OrderedResources) > 0 { orders, err = backup.ParseOrderedResources(o.BackupOptions.OrderedResources) if err != nil { return err } } schedule := &api.Schedule{ ObjectMeta: metav1.ObjectMeta{ Namespace: f.Namespace(), Name: o.BackupOptions.Name, Labels: o.BackupOptions.Labels.Data(), }, Spec: api.ScheduleSpec{ Template: api.BackupSpec{ IncludedNamespaces: o.BackupOptions.IncludeNamespaces, ExcludedNamespaces: o.BackupOptions.ExcludeNamespaces, IncludedResources: o.BackupOptions.IncludeResources, ExcludedResources: o.BackupOptions.ExcludeResources, IncludedClusterScopedResources: o.BackupOptions.IncludeClusterScopedResources, ExcludedClusterScopedResources: o.BackupOptions.ExcludeClusterScopedResources, IncludedNamespaceScopedResources: o.BackupOptions.IncludeNamespaceScopedResources, ExcludedNamespaceScopedResources: o.BackupOptions.ExcludeNamespaceScopedResources, IncludeClusterResources: o.BackupOptions.IncludeClusterResources.Value, LabelSelector: o.BackupOptions.Selector.LabelSelector, OrLabelSelectors: o.BackupOptions.OrSelector.OrLabelSelectors, SnapshotVolumes: o.BackupOptions.SnapshotVolumes.Value, TTL: metav1.Duration{Duration: o.BackupOptions.TTL}, StorageLocation: o.BackupOptions.StorageLocation, VolumeSnapshotLocations: o.BackupOptions.SnapshotLocations, DefaultVolumesToFsBackup: o.BackupOptions.DefaultVolumesToFsBackup.Value, OrderedResources: orders, CSISnapshotTimeout: metav1.Duration{Duration: o.BackupOptions.CSISnapshotTimeout}, ItemOperationTimeout: metav1.Duration{Duration: o.BackupOptions.ItemOperationTimeout}, DataMover: o.BackupOptions.DataMover, SnapshotMoveData: o.BackupOptions.SnapshotMoveData.Value, }, Schedule: o.Schedule, UseOwnerReferencesInBackup: &o.UseOwnerReferencesInBackup, Paused: o.Paused, SkipImmediately: o.SkipOptions.SkipImmediately.Value, }, } if o.BackupOptions.ResPoliciesConfigmap != "" { schedule.Spec.Template.ResourcePolicy = &corev1api.TypedLocalObjectReference{Kind: resourcepolicies.ConfigmapRefType, Name: o.BackupOptions.ResPoliciesConfigmap} } if o.BackupOptions.ParallelFilesUpload > 0 { schedule.Spec.Template.UploaderConfig = &api.UploaderConfigForBackup{ ParallelFilesUpload: o.BackupOptions.ParallelFilesUpload, } } if printed, err := output.PrintWithFormat(c, schedule); printed || err != nil { return err } err = crClient.Create(context.TODO(), schedule) if err != nil { return err } fmt.Printf("Schedule %q created successfully.\n", schedule.Name) return nil } ================================================ FILE: pkg/cmd/cli/schedule/delete.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package schedule import ( "context" "fmt" "github.com/pkg/errors" "github.com/spf13/cobra" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" kubeerrs "k8s.io/apimachinery/pkg/util/errors" controllerclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" "github.com/vmware-tanzu/velero/pkg/cmd/cli" "github.com/vmware-tanzu/velero/pkg/cmd/util/confirm" ) // NewDeleteCommand creates and returns a new cobra command for deleting schedules. func NewDeleteCommand(f client.Factory, use string) *cobra.Command { o := cli.NewDeleteOptions("schedule") c := &cobra.Command{ Use: fmt.Sprintf("%s [NAMES]", use), Short: "Delete schedules", Example: ` # Delete a schedule named "schedule-1". velero schedule delete schedule-1 # Delete a schedule named "schedule-1" without prompting for confirmation. velero schedule delete schedule-1 --confirm # Delete schedules named "schedule-1" and "schedule-2". velero schedule delete schedule-1 schedule-2 # Delete all schedules labeled with "foo=bar". velero schedule delete --selector foo=bar # Delete all schedules. velero schedule delete --all`, Run: func(c *cobra.Command, args []string) { cmd.CheckError(o.Complete(f, args)) cmd.CheckError(o.Validate(c, f, args)) cmd.CheckError(Run(o)) }, } o.BindFlags(c.Flags()) return c } // Run performs the deletion of schedules. func Run(o *cli.DeleteOptions) error { if !o.Confirm && !confirm.GetConfirmation() { return nil } var ( schedules []*velerov1api.Schedule errs []error ) switch { case len(o.Names) > 0: for _, name := range o.Names { schedule := new(velerov1api.Schedule) err := o.Client.Get(context.TODO(), controllerclient.ObjectKey{Namespace: o.Namespace, Name: name}, schedule) if err != nil { errs = append(errs, errors.WithStack(err)) continue } schedules = append(schedules, schedule) } default: selector := labels.Everything() if o.Selector.LabelSelector != nil { convertedSelector, err := metav1.LabelSelectorAsSelector(o.Selector.LabelSelector) if err != nil { return errors.WithStack(err) } selector = convertedSelector } scheduleList := new(velerov1api.ScheduleList) err := o.Client.List(context.TODO(), scheduleList, &controllerclient.ListOptions{ Namespace: o.Namespace, LabelSelector: selector, }) if err != nil { errs = append(errs, errors.WithStack(err)) } for i := range scheduleList.Items { schedules = append(schedules, &scheduleList.Items[i]) } } if len(schedules) == 0 { fmt.Println("No schedules found") return nil } for _, s := range schedules { err := o.Client.Delete(context.TODO(), s, &controllerclient.DeleteOptions{}) if err != nil { errs = append(errs, errors.WithStack(err)) continue } fmt.Printf("Schedule deleted: %v\n", s.Name) } return kubeerrs.NewAggregate(errs) } ================================================ FILE: pkg/cmd/cli/schedule/describe.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package schedule import ( "context" "fmt" "github.com/spf13/cobra" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" "github.com/vmware-tanzu/velero/pkg/cmd/util/output" ) func NewDescribeCommand(f client.Factory, use string) *cobra.Command { var listOptions metav1.ListOptions c := &cobra.Command{ Use: use + " [NAME1] [NAME2] [NAME...]", Short: "Describe schedules", Run: func(c *cobra.Command, args []string) { crClient, err := f.KubebuilderClient() cmd.CheckError(err) schedules := new(v1.ScheduleList) if len(args) > 0 { for _, name := range args { schedule := new(v1.Schedule) err := crClient.Get(context.TODO(), ctrlclient.ObjectKey{Namespace: f.Namespace(), Name: name}, schedule) cmd.CheckError(err) schedules.Items = append(schedules.Items, *schedule) } } else { selector := labels.NewSelector() if listOptions.LabelSelector != "" { selector, err = labels.Parse(listOptions.LabelSelector) cmd.CheckError(err) } err = crClient.List(context.TODO(), schedules, &ctrlclient.ListOptions{LabelSelector: selector}) cmd.CheckError(err) } first := true for i := range schedules.Items { s := output.DescribeSchedule(&schedules.Items[i]) if first { first = false fmt.Print(s) } else { fmt.Printf("\n\n%s", s) } } cmd.CheckError(err) }, } c.Flags().StringVarP(&listOptions.LabelSelector, "selector", "l", listOptions.LabelSelector, "Only show items matching this label selector.") return c } ================================================ FILE: pkg/cmd/cli/schedule/get.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package schedule import ( "context" "github.com/spf13/cobra" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" "github.com/vmware-tanzu/velero/pkg/cmd/util/output" ) func NewGetCommand(f client.Factory, use string) *cobra.Command { var listOptions metav1.ListOptions c := &cobra.Command{ Use: use, Short: "Get schedules", Run: func(c *cobra.Command, args []string) { err := output.ValidateFlags(c) cmd.CheckError(err) crClient, err := f.KubebuilderClient() cmd.CheckError(err) schedules := new(api.ScheduleList) if len(args) > 0 { for _, name := range args { schedule := new(api.Schedule) err := crClient.Get(context.TODO(), ctrlclient.ObjectKey{Name: name, Namespace: f.Namespace()}, schedule) cmd.CheckError(err) schedules.Items = append(schedules.Items, *schedule) } } else { selector := labels.NewSelector() if listOptions.LabelSelector != "" { selector, err = labels.Parse(listOptions.LabelSelector) cmd.CheckError(err) } err := crClient.List(context.TODO(), schedules, &ctrlclient.ListOptions{LabelSelector: selector}) cmd.CheckError(err) } if printed, err := output.PrintWithFormat(c, schedules); printed || err != nil { cmd.CheckError(err) return } _, err = output.PrintWithFormat(c, schedules) cmd.CheckError(err) }, } c.Flags().StringVarP(&listOptions.LabelSelector, "selector", "l", listOptions.LabelSelector, "Only show items matching this label selector.") output.BindFlags(c.Flags()) return c } ================================================ FILE: pkg/cmd/cli/schedule/pause.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package schedule import ( "context" "fmt" "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" kubeerrs "k8s.io/apimachinery/pkg/util/errors" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" "github.com/vmware-tanzu/velero/pkg/cmd/cli" ) // NewPauseCommand creates the command for pause func NewPauseCommand(f client.Factory, use string) *cobra.Command { o := cli.NewSelectOptions("pause", "schedule") pauseOpts := NewPauseOptions() c := &cobra.Command{ Use: use, Short: "Pause schedules", Example: ` # Pause a schedule named "schedule-1". velero schedule pause schedule-1 # Pause schedules named "schedule-1" and "schedule-2". velero schedule pause schedule-1 schedule-2 # Pause all schedules labeled with "foo=bar". velero schedule pause --selector foo=bar # Pause all schedules. velero schedule pause --all`, Run: func(c *cobra.Command, args []string) { cmd.CheckError(o.Complete(args)) cmd.CheckError(o.Validate()) cmd.CheckError(runPause(f, o, true, pauseOpts.SkipOptions.SkipImmediately.Value)) }, } o.BindFlags(c.Flags()) pauseOpts.BindFlags(c.Flags()) return c } type PauseOptions struct { SkipOptions *SkipOptions } func NewPauseOptions() *PauseOptions { return &PauseOptions{ SkipOptions: NewSkipOptions(), } } func (o *PauseOptions) BindFlags(flags *pflag.FlagSet) { o.SkipOptions.BindFlags(flags) } func runPause(f client.Factory, o *cli.SelectOptions, paused bool, skipImmediately *bool) error { crClient, err := f.KubebuilderClient() if err != nil { return err } var ( schedules []*velerov1api.Schedule errs []error ) switch { case len(o.Names) > 0: for _, name := range o.Names { schedule := new(velerov1api.Schedule) err := crClient.Get(context.TODO(), ctrlclient.ObjectKey{Name: name, Namespace: f.Namespace()}, schedule) if err != nil { errs = append(errs, errors.WithStack(err)) continue } schedules = append(schedules, schedule) } default: selector := labels.Everything() if o.Selector.LabelSelector != nil { convertedSelector, err := metav1.LabelSelectorAsSelector(o.Selector.LabelSelector) if err != nil { return errors.WithStack(err) } selector = convertedSelector } res := new(velerov1api.ScheduleList) err := crClient.List(context.TODO(), res, &ctrlclient.ListOptions{ LabelSelector: selector, }) if err != nil { errs = append(errs, errors.WithStack(err)) } for i := range res.Items { schedules = append(schedules, &res.Items[i]) } } if len(schedules) == 0 { fmt.Println("No schedules found") return nil } msg := "paused" if !paused { msg = "unpaused" } for _, schedule := range schedules { if schedule.Spec.Paused == paused { fmt.Printf("Schedule %s is already %s, skip\n", schedule.Name, msg) continue } schedule.Spec.Paused = paused schedule.Spec.SkipImmediately = skipImmediately if err := crClient.Update(context.TODO(), schedule); err != nil { return errors.Wrapf(err, "failed to update schedule %s", schedule.Name) } fmt.Printf("Schedule %s %s successfully\n", schedule.Name, msg) } return kubeerrs.NewAggregate(errs) } ================================================ FILE: pkg/cmd/cli/schedule/schedule.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package schedule import ( "github.com/spf13/cobra" "github.com/vmware-tanzu/velero/pkg/client" ) func NewCommand(f client.Factory) *cobra.Command { c := &cobra.Command{ Use: "schedule", Short: "Work with schedules", Long: "Work with schedules", } c.AddCommand( NewCreateCommand(f, "create"), NewGetCommand(f, "get"), NewDescribeCommand(f, "describe"), NewDeleteCommand(f, "delete"), NewPauseCommand(f, "pause"), NewUnpauseCommand(f, "unpause"), ) return c } ================================================ FILE: pkg/cmd/cli/schedule/skip_options.go ================================================ package schedule import ( "github.com/spf13/pflag" "github.com/vmware-tanzu/velero/pkg/cmd/util/flag" ) type SkipOptions struct { SkipImmediately flag.OptionalBool } func NewSkipOptions() *SkipOptions { return &SkipOptions{} } func (o *SkipOptions) BindFlags(flags *pflag.FlagSet) { f := flags.VarPF(&o.SkipImmediately, "skip-immediately", "", "Skip the next scheduled backup immediately") f.NoOptDefVal = "" // default to nil so server options can take precedence } ================================================ FILE: pkg/cmd/cli/schedule/unpause.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package schedule import ( "github.com/spf13/cobra" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" "github.com/vmware-tanzu/velero/pkg/cmd/cli" ) // NewUnpauseCommand creates the command for unpause func NewUnpauseCommand(f client.Factory, use string) *cobra.Command { o := cli.NewSelectOptions("pause", "schedule") pauseOpts := NewPauseOptions() c := &cobra.Command{ Use: use, Short: "Unpause schedules", Example: ` # Unpause a schedule named "schedule-1". velero schedule unpause schedule-1 # Unpause schedules named "schedule-1" and "schedule-2". velero schedule unpause schedule-1 schedule-2 # Unpause all schedules labeled with "foo=bar". velero schedule unpause --selector foo=bar # Unpause all schedules. velero schedule unpause --all`, Run: func(c *cobra.Command, args []string) { cmd.CheckError(o.Complete(args)) cmd.CheckError(o.Validate()) cmd.CheckError(runPause(f, o, false, pauseOpts.SkipOptions.SkipImmediately.Value)) }, } o.BindFlags(c.Flags()) pauseOpts.BindFlags(c.Flags()) return c } ================================================ FILE: pkg/cmd/cli/select_option.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cli import ( "errors" "github.com/spf13/pflag" "golang.org/x/text/cases" "golang.org/x/text/language" "github.com/vmware-tanzu/velero/pkg/cmd/util/flag" ) // SelectOptions defines the options for selecting resources type SelectOptions struct { Names []string All bool Selector flag.LabelSelector CMD string SingularTypeName string } // NewSelectOptions creates a new option for selector func NewSelectOptions(cmd, singularTypeName string) *SelectOptions { return &SelectOptions{ CMD: cmd, SingularTypeName: singularTypeName, } } // Complete fills in the correct values for all the options. func (o *SelectOptions) Complete(args []string) error { o.Names = args return nil } // Validate validates the fields of the SelectOptions struct. func (o *SelectOptions) Validate() error { var ( hasNames = len(o.Names) > 0 hasAll = o.All hasSelector = o.Selector.LabelSelector != nil ) if !xor(hasNames, hasAll, hasSelector) { return errors.New("you must specify exactly one of: specific " + o.SingularTypeName + " name(s), the --all flag, or the --selector flag") } return nil } // BindFlags binds options for this command to flags. func (o *SelectOptions) BindFlags(flags *pflag.FlagSet) { flags.BoolVar(&o.All, "all", o.All, cases.Title(language.Und).String(o.CMD)+" all "+o.SingularTypeName+"s") flags.VarP(&o.Selector, "selector", "l", cases.Title(language.Und).String(o.CMD)+" all "+o.SingularTypeName+"s matching this label selector.") } ================================================ FILE: pkg/cmd/cli/select_option_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cli import ( "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/vmware-tanzu/velero/pkg/cmd/util/flag" ) func TestCompleteOfSelectOption(t *testing.T) { option := &SelectOptions{} args := []string{"arg1", "arg2"} require.NoError(t, option.Complete(args)) assert.Equal(t, args, option.Names) } func TestValidateOfSelectOption(t *testing.T) { option := &SelectOptions{ Names: nil, Selector: flag.LabelSelector{}, All: false, } require.Error(t, option.Validate()) option.All = true assert.NoError(t, option.Validate()) } ================================================ FILE: pkg/cmd/cli/serverstatus/server_status.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package serverstatus import ( "context" "time" "github.com/pkg/errors" "k8s.io/apimachinery/pkg/util/wait" kbclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" veleroclient "github.com/vmware-tanzu/velero/pkg/client" ) type Getter interface { GetServerStatus(kbClient kbclient.Client) (*velerov1api.ServerStatusRequest, error) } type DefaultServerStatusGetter struct { Namespace string Context context.Context } func (g *DefaultServerStatusGetter) GetServerStatus(kbClient kbclient.Client) (*velerov1api.ServerStatusRequest, error) { created := builder.ForServerStatusRequest(g.Namespace, "", "0").ObjectMeta(builder.WithGenerateName("velero-cli-")).Result() if err := veleroclient.CreateRetryGenerateName(kbClient, context.Background(), created); err != nil { return nil, errors.WithStack(err) } ctx, cancel := context.WithCancel(g.Context) defer cancel() key := kbclient.ObjectKey{Name: created.Name, Namespace: g.Namespace} checkFunc := func() { updated := &velerov1api.ServerStatusRequest{} if err := kbClient.Get(ctx, key, updated); err != nil { return } // TODO: once the minimum supported Kubernetes version is v1.9.0, remove the following check. // See http://issue.k8s.io/51046 for details. if updated.Name != created.Name { return } if updated.Status.Phase == velerov1api.ServerStatusRequestPhaseProcessed { created = updated cancel() } } wait.Until(checkFunc, 250*time.Millisecond, ctx.Done()) err := ctx.Err() // context.Canceled error means we have received a processed ServerStatusRequest if err == context.Canceled { err = nil } return created, err } ================================================ FILE: pkg/cmd/cli/snapshotlocation/create.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package snapshotlocation import ( "context" "fmt" "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" "github.com/vmware-tanzu/velero/pkg/cmd/util/flag" "github.com/vmware-tanzu/velero/pkg/cmd/util/output" ) func NewCreateCommand(f client.Factory, use string) *cobra.Command { o := NewCreateOptions() c := &cobra.Command{ Use: use + " NAME", Short: "Create a volume snapshot location", Args: cobra.ExactArgs(1), Run: func(c *cobra.Command, args []string) { cmd.CheckError(o.Complete(args, f)) cmd.CheckError(o.Validate(c, args, f)) cmd.CheckError(o.Run(c, f)) }, } o.BindFlags(c.Flags()) output.BindFlags(c.Flags()) output.ClearOutputFlagDefault(c) return c } type CreateOptions struct { Name string Provider string Config flag.Map Labels flag.Map Credential flag.Map } func NewCreateOptions() *CreateOptions { return &CreateOptions{ Config: flag.NewMap(), Labels: flag.NewMap(), Credential: flag.NewMap(), } } func (o *CreateOptions) BindFlags(flags *pflag.FlagSet) { flags.StringVar(&o.Provider, "provider", o.Provider, "Name of the volume snapshot provider (e.g. aws, azure, gcp).") flags.Var(&o.Config, "config", "Configuration key-value pairs.") flags.Var(&o.Labels, "labels", "Labels to apply to the volume snapshot location.") flags.Var(&o.Credential, "credential", "The credential to be used by this location as a key-value pair, where the key is the Kubernetes Secret name, and the value is the data key name within the Secret. Optional, one value only.") } func (o *CreateOptions) Validate(c *cobra.Command, args []string, f client.Factory) error { if err := output.ValidateFlags(c); err != nil { return err } if o.Provider == "" { return errors.New("--provider is required") } if len(o.Credential.Data()) > 1 { return errors.New("--credential can only contain 1 key/value pair") } return nil } func (o *CreateOptions) Complete(args []string, f client.Factory) error { o.Name = args[0] return nil } func (o *CreateOptions) BuildVolumeSnapshotLocation(namespace string) *api.VolumeSnapshotLocation { volumeSnapshotLocation := &api.VolumeSnapshotLocation{ ObjectMeta: metav1.ObjectMeta{ Namespace: namespace, Name: o.Name, Labels: o.Labels.Data(), }, Spec: api.VolumeSnapshotLocationSpec{ Provider: o.Provider, Config: o.Config.Data(), }, } for secretName, secretKey := range o.Credential.Data() { volumeSnapshotLocation.Spec.Credential = builder.ForSecretKeySelector(secretName, secretKey).Result() break } return volumeSnapshotLocation } func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error { volumeSnapshotLocation := o.BuildVolumeSnapshotLocation(f.Namespace()) if printed, err := output.PrintWithFormat(c, volumeSnapshotLocation); printed || err != nil { return err } client, err := f.KubebuilderClient() if err != nil { return err } if err := client.Create(context.TODO(), volumeSnapshotLocation); err != nil { return errors.WithStack(err) } fmt.Printf("Snapshot volume location %q configured successfully.\n", volumeSnapshotLocation.Name) return nil } ================================================ FILE: pkg/cmd/cli/snapshotlocation/get.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package snapshotlocation import ( "context" "github.com/spf13/cobra" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kbclient "sigs.k8s.io/controller-runtime/pkg/client" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" "github.com/vmware-tanzu/velero/pkg/cmd/util/output" ) func NewGetCommand(f client.Factory, use string) *cobra.Command { var listOptions metav1.ListOptions c := &cobra.Command{ Use: use, Short: "Get snapshot locations", Run: func(c *cobra.Command, args []string) { err := output.ValidateFlags(c) cmd.CheckError(err) client, err := f.KubebuilderClient() cmd.CheckError(err) locations := new(api.VolumeSnapshotLocationList) if len(args) > 0 { for _, name := range args { location := new(api.VolumeSnapshotLocation) err := client.Get(context.TODO(), kbclient.ObjectKey{Namespace: f.Namespace(), Name: name}, location) cmd.CheckError(err) locations.Items = append(locations.Items, *location) } } else { err = client.List(context.TODO(), locations, &kbclient.ListOptions{Namespace: f.Namespace()}) cmd.CheckError(err) } _, err = output.PrintWithFormat(c, locations) cmd.CheckError(err) }, } c.Flags().StringVarP(&listOptions.LabelSelector, "selector", "l", listOptions.LabelSelector, "Only show items matching this label selector") output.BindFlags(c.Flags()) return c } ================================================ FILE: pkg/cmd/cli/snapshotlocation/set.go ================================================ /* Copyright The Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package snapshotlocation import ( "context" "fmt" "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" kbclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" "github.com/vmware-tanzu/velero/pkg/cmd/util/flag" "github.com/vmware-tanzu/velero/pkg/cmd/util/output" ) func NewSetCommand(f client.Factory, use string) *cobra.Command { o := NewSetOptions() c := &cobra.Command{ Use: use + " NAME", Short: "Set specific features for a snapshot location", Args: cobra.ExactArgs(1), Run: func(c *cobra.Command, args []string) { cmd.CheckError(o.Complete(args, f)) cmd.CheckError(o.Validate(c, args, f)) cmd.CheckError(o.Run(c, f)) }, } o.BindFlags(c.Flags()) return c } type SetOptions struct { Name string Credential flag.Map } func NewSetOptions() *SetOptions { return &SetOptions{ Credential: flag.NewMap(), } } func (o *SetOptions) BindFlags(flags *pflag.FlagSet) { flags.Var(&o.Credential, "credential", "Sets the credential to be used by this location as a key-value pair, where the key is the Kubernetes Secret name, and the value is the data key name within the Secret. Optional, one value only.") } func (o *SetOptions) Validate(c *cobra.Command, args []string, f client.Factory) error { if err := output.ValidateFlags(c); err != nil { return err } if len(o.Credential.Data()) > 1 { return errors.New("--credential can only contain 1 key/value pair") } return nil } func (o *SetOptions) Complete(args []string, f client.Factory) error { o.Name = args[0] return nil } func (o *SetOptions) Run(c *cobra.Command, f client.Factory) error { kbClient, err := f.KubebuilderClient() if err != nil { return err } location := &velerov1api.VolumeSnapshotLocation{} err = kbClient.Get(context.Background(), kbclient.ObjectKey{ Namespace: f.Namespace(), Name: o.Name, }, location) if err != nil { return errors.WithStack(err) } for name, key := range o.Credential.Data() { location.Spec.Credential = builder.ForSecretKeySelector(name, key).Result() break } if err := kbClient.Update(context.Background(), location, &kbclient.UpdateOptions{}); err != nil { return errors.WithStack(err) } fmt.Printf("Volume snapshot location %q configured successfully.\n", o.Name) return nil } ================================================ FILE: pkg/cmd/cli/snapshotlocation/snapshot_location.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package snapshotlocation import ( "github.com/spf13/cobra" "github.com/vmware-tanzu/velero/pkg/client" ) func NewCommand(f client.Factory) *cobra.Command { c := &cobra.Command{ Use: "snapshot-location", Short: "Work with snapshot locations", Long: "Work with snapshot locations", } c.AddCommand( NewCreateCommand(f, "create"), NewGetCommand(f, "get"), NewSetCommand(f, "set"), ) return c } ================================================ FILE: pkg/cmd/cli/uninstall/uninstall.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package uninstall import ( "context" "fmt" "reflect" "sync" "time" "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" appsv1api "k8s.io/api/apps/v1" corev1api "k8s.io/api/core/v1" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/labels" kubeerrs "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/wait" kbclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" "github.com/vmware-tanzu/velero/pkg/cmd/util/confirm" "github.com/vmware-tanzu/velero/pkg/controller" "github.com/vmware-tanzu/velero/pkg/install" kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube" ) var gracefulDeletionMaximumDuration = 1 * time.Minute var resToDelete = []kbclient.ObjectList{} // uninstallOptions collects all the options for uninstalling Velero from a Kubernetes cluster. type uninstallOptions struct { wait bool // deprecated force bool } // BindFlags adds command line values to the options struct. func (o *uninstallOptions) BindFlags(flags *pflag.FlagSet) { flags.BoolVar(&o.wait, "wait", o.wait, "Wait for Velero uninstall to be ready. Optional. Deprecated.") flags.BoolVar(&o.force, "force", o.force, "Forces the Velero uninstall. Optional.") } // NewCommand creates a cobra command. func NewCommand(f client.Factory) *cobra.Command { o := &uninstallOptions{} c := &cobra.Command{ Use: "uninstall", Short: "Uninstall Velero", Long: `Uninstall Velero along with the CRDs and clusterrolebinding. The '--namespace' flag can be used to specify the namespace where velero is installed (default: velero). Use '--force' to skip the prompt confirming if you want to uninstall Velero. `, Example: ` # velero uninstall --namespace staging`, Run: func(c *cobra.Command, args []string) { if o.wait { fmt.Println("Warning: the \"--wait\" option is deprecated and will be removed in a future release. The uninstall command always waits for the uninstall to complete.") } // Confirm if not asked to force-skip confirmation if !o.force { fmt.Println("You are about to uninstall Velero.") if !confirm.GetConfirmation() { // Don't do anything unless we get confirmation return } } kbClient, err := f.KubebuilderClient() cmd.CheckError(err) cmd.CheckError(Run(context.Background(), kbClient, f.Namespace())) }, } o.BindFlags(c.Flags()) return c } // Run removes all components that were deployed using the Velero install command func Run(ctx context.Context, kbClient kbclient.Client, namespace string) error { // The CRDs cannot be removed until the namespace is deleted to avoid the problem in issue #3974 so if the namespace deletion fails we error out here if err := deleteNamespace(ctx, kbClient, namespace); err != nil { fmt.Printf("Errors while attempting to uninstall Velero: %q \n", err) return err } var errs []error // ClusterRoleBinding crb := install.ClusterRoleBinding(namespace) key := kbclient.ObjectKey{Name: crb.Name} if err := kbClient.Get(ctx, key, crb); err != nil { if apierrors.IsNotFound(err) { fmt.Printf("Velero ClusterRoleBinding %q does not exist, skipping.\n", crb.Name) } else { errs = append(errs, errors.WithStack(err)) } } else { if err := kbClient.Delete(ctx, crb); err != nil { errs = append(errs, errors.WithStack(err)) } } // CRDs veleroLabelSelector := labels.SelectorFromSet(install.Labels()) opts := []kbclient.DeleteAllOfOption{ kbclient.InNamespace(namespace), kbclient.MatchingLabelsSelector{ Selector: veleroLabelSelector, }, } v1CRDsRemoved := false v1crd := &apiextv1.CustomResourceDefinition{} if err := kbClient.DeleteAllOf(ctx, v1crd, opts...); err != nil { if meta.IsNoMatchError(err) { fmt.Println("V1 Velero CRDs not found, skipping...") } else { errs = append(errs, errors.WithStack(err)) } } else { v1CRDsRemoved = true } // Remove any old Velero v1beta1 CRDs hanging around. v1beta1crd := &apiextv1beta1.CustomResourceDefinition{} if err := kbClient.DeleteAllOf(ctx, v1beta1crd, opts...); err != nil { if meta.IsNoMatchError(err) { if !v1CRDsRemoved { // Only mention this if there were no V1 CRDs removed fmt.Println("V1Beta1 Velero CRDs not found, skipping...") } } else { errs = append(errs, errors.WithStack(err)) } } if kubeerrs.NewAggregate(errs) != nil { fmt.Printf("Errors while attempting to uninstall Velero: %q \n", kubeerrs.NewAggregate(errs)) return kubeerrs.NewAggregate(errs) } fmt.Println("Velero uninstalled ⛵") return nil } func deleteNamespace(ctx context.Context, kbClient kbclient.Client, namespace string) error { // First check if it's already been deleted ns := &corev1api.Namespace{} key := kbclient.ObjectKey{Name: namespace} if err := kbClient.Get(ctx, key, ns); err != nil { if apierrors.IsNotFound(err) { fmt.Printf("Velero namespace %q does not exist, skipping.\n", namespace) return nil } return err } // Deal with resources with attached finalizers to ensure proper handling of those finalizers. if err := deleteResourcesWithFinalizer(ctx, kbClient, namespace); err != nil { return errors.Wrap(err, "Fail to remove finalizer from restores") } if err := kbClient.Delete(ctx, ns); err != nil { if apierrors.IsNotFound(err) { fmt.Printf("Velero namespace %q does not exist, skipping.\n", namespace) return nil } return err } fmt.Println() fmt.Printf("Waiting for velero namespace %q to be deleted\n", namespace) ctx, cancel := context.WithCancel(ctx) defer cancel() var err error checkFunc := func() { if err = kbClient.Get(ctx, key, ns); err != nil { if apierrors.IsNotFound(err) { fmt.Print("\n") err = nil } cancel() return } fmt.Print(".") } // Must wait until the namespace is deleted to avoid the issue https://github.com/vmware-tanzu/velero/issues/3974 wait.Until(checkFunc, 5*time.Millisecond, ctx.Done()) if err != nil { return err } fmt.Printf("Velero namespace %q deleted\n", namespace) return nil } // A few things needed to be noticed here: // 1. When we delete resources with attached finalizers, the corresponding controller will deal with the finalizer then resources can be deleted successfully. // So it is important to delete these resources before deleting the pod that runs that controller. // 2. The controller may encounter errors while handling the finalizer, in such case, the controller will keep trying until it succeeds. // So it is important to set a timeout, once the process exceed the timeout, we will forcedly delete the resources by removing the finalizer from them, // otherwise the deletion process may get stuck indefinitely. // 3. There is only resources finalizer supported as of v1.12. If any new finalizers are added in the future, the corresponding deletion logic can be // incorporated into this function. func deleteResourcesWithFinalizer(ctx context.Context, kbClient kbclient.Client, namespace string) error { fmt.Println("Waiting for resource with attached finalizer to be deleted") return deleteResources(ctx, kbClient, namespace) } func checkResources(ctx context.Context, kbClient kbclient.Client) error { checkCRDs := []string{"restores.velero.io", "datauploads.velero.io", "datadownloads.velero.io"} var err error v1crd := &apiextv1.CustomResourceDefinition{} for _, crd := range checkCRDs { key := kbclient.ObjectKey{Name: crd} if err = kbClient.Get(ctx, key, v1crd); err != nil { if !apierrors.IsNotFound(err) { return errors.Wrapf(err, "Error getting %s crd", crd) } } else { // no error with found CRD that we should delete switch crd { case "restores.velero.io": resToDelete = append(resToDelete, &velerov1api.RestoreList{}) case "datauploads.velero.io": resToDelete = append(resToDelete, &velerov2alpha1api.DataUploadList{}) case "datadownloads.velero.io": resToDelete = append(resToDelete, &velerov2alpha1api.DataDownloadList{}) default: fmt.Printf("Unsupported type %s to be cleared\n", crd) } } } return nil } func deleteResources(ctx context.Context, kbClient kbclient.Client, namespace string) error { // Check if resources crd exists, if it does not exist, return immediately. err := checkResources(ctx, kbClient) if err != nil { return err } // First attempt to gracefully delete all the resources within a specified time frame, If the process exceeds the timeout limit, // it is likely that there may be errors during the finalization of restores. In such cases, we should proceed with forcefully deleting the restores. err = gracefullyDeleteResources(ctx, kbClient, namespace) if err != nil && !wait.Interrupted(err) { return errors.Wrap(err, "Error deleting resources") } if wait.Interrupted(err) { err = forcedlyDeleteResources(ctx, kbClient, namespace) if err != nil { return errors.Wrap(err, "Error deleting resources forcedly") } } return nil } func gracefullyDeleteResources(ctx context.Context, kbClient kbclient.Client, namespace string) error { errorChan := make(chan error) var wg sync.WaitGroup wg.Add(len(resToDelete)) for i := range resToDelete { go func(index int) { defer wg.Done() errorChan <- gracefullyDeleteResource(ctx, kbClient, namespace, resToDelete[index]) }(i) } go func() { wg.Wait() close(errorChan) }() for err := range errorChan { if err != nil { return err } } return waitDeletingResources(ctx, kbClient, namespace) } func gracefullyDeleteResource(ctx context.Context, kbClient kbclient.Client, namespace string, list kbclient.ObjectList) error { if err := kbClient.List(ctx, list, &kbclient.ListOptions{Namespace: namespace}); err != nil { return errors.Wrap(err, "Error getting resources during graceful deletion") } var objectsToDelete []kbclient.Object items := reflect.ValueOf(list).Elem().FieldByName("Items") for i := 0; i < items.Len(); i++ { item := items.Index(i).Addr().Interface() // Type assertion to cast item to the appropriate type switch typedItem := item.(type) { case *velerov1api.Restore: objectsToDelete = append(objectsToDelete, typedItem) case *velerov2alpha1api.DataUpload: objectsToDelete = append(objectsToDelete, typedItem) case *velerov2alpha1api.DataDownload: objectsToDelete = append(objectsToDelete, typedItem) default: return errors.New("Unsupported resource type") } } // Delete collected resources in a batch for _, resource := range objectsToDelete { if err := kbClient.Delete(ctx, resource); err != nil { if apierrors.IsNotFound(err) { continue } return errors.Wrap(err, "Error deleting resources during graceful deletion") } } return nil } func waitDeletingResources(ctx context.Context, kbClient kbclient.Client, namespace string) error { // Wait for the deletion of all the restores within a specified time frame err := wait.PollUntilContextTimeout(ctx, time.Second, gracefulDeletionMaximumDuration, true, func(ctx context.Context) (bool, error) { itemsCount := 0 for i := range resToDelete { if errList := kbClient.List(ctx, resToDelete[i], &kbclient.ListOptions{Namespace: namespace}); errList != nil { return false, errList } itemsCount += reflect.ValueOf(resToDelete[i]).Elem().FieldByName("Items").Len() if itemsCount > 0 { fmt.Print(".") return false, nil } } return true, nil }) return err } func forcedlyDeleteResources(ctx context.Context, kbClient kbclient.Client, namespace string) error { // Delete velero deployment first in case: // 1. finalizers will be added back by resources related controller after they are removed at next step; // 2. new resources attached with finalizer will be created by controller after we remove all the resources' finalizer at next step; deploy := &appsv1api.Deployment{ ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", Name: namespace, }, } err := kbClient.Delete(ctx, deploy) if err != nil && !apierrors.IsNotFound(err) { return errors.Wrap(err, "Error deleting velero deployment during force deletion") } ctxc, cancel := context.WithCancel(ctx) defer cancel() checkFunc := func() { deploy := &appsv1api.Deployment{} key := kbclient.ObjectKey{Namespace: namespace, Name: "velero"} if err = kbClient.Get(ctxc, key, deploy); err != nil { if apierrors.IsNotFound(err) { err = nil } cancel() return } } // Wait until velero deployment are deleted. wait.Until(checkFunc, 100*time.Millisecond, ctxc.Done()) if err != nil { return errors.Wrap(err, "Error deleting velero deployment during force deletion") } return removeResourcesFinalizer(ctx, kbClient, namespace) } func removeResourcesFinalizer(ctx context.Context, kbClient kbclient.Client, namespace string) error { for i := range resToDelete { if err := removeResourceFinalizer(ctx, kbClient, namespace, resToDelete[i]); err != nil { return err } } return nil } func removeResourceFinalizer(ctx context.Context, kbClient kbclient.Client, namespace string, resourceList kbclient.ObjectList) error { listOptions := &kbclient.ListOptions{Namespace: namespace} if err := kbClient.List(ctx, resourceList, listOptions); err != nil { return errors.Wrap(err, fmt.Sprintf("Error getting resources of type %T during force deletion", resourceList)) } items := reflect.ValueOf(resourceList).Elem().FieldByName("Items") var err error for i := 0; i < items.Len(); i++ { item := items.Index(i).Addr().Interface() // Type assertion to cast item to the appropriate type switch typedItem := item.(type) { case *velerov1api.Restore: err = removeFinalizerForObject(typedItem, controller.ExternalResourcesFinalizer, kbClient) case *velerov2alpha1api.DataUpload: err = removeFinalizerForObject(typedItem, controller.DataUploadDownloadFinalizer, kbClient) case *velerov2alpha1api.DataDownload: err = removeFinalizerForObject(typedItem, controller.DataUploadDownloadFinalizer, kbClient) default: err = errors.Errorf("Unsupported resource type %T", typedItem) } if err != nil { return err } } return nil } func removeFinalizerForObject(obj kbclient.Object, finalizer string, kbClient kbclient.Client) error { if controllerutil.ContainsFinalizer(obj, finalizer) { update := obj.DeepCopyObject().(kbclient.Object) original := obj.DeepCopyObject().(kbclient.Object) controllerutil.RemoveFinalizer(update, finalizer) if err := kubeutil.PatchResource(original, update, kbClient); err != nil { return errors.Wrap(err, fmt.Sprintf("Error removing finalizer %q during force deletion", finalizer)) } } return nil } ================================================ FILE: pkg/cmd/cli/version/version.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package version import ( "context" "fmt" "io" "os" "time" "github.com/spf13/cobra" "golang.org/x/mod/semver" kbclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/pkg/buildinfo" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" "github.com/vmware-tanzu/velero/pkg/cmd/cli/serverstatus" ) func NewCommand(f client.Factory) *cobra.Command { var clientOnly bool timeout := 5 * time.Second c := &cobra.Command{ Use: "version", Short: "Print the velero version and associated image", Run: func(c *cobra.Command, args []string) { var kbClient kbclient.Client if !clientOnly { var err error kbClient, err = f.KubebuilderClient() cmd.CheckError(err) } ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() serverStatusGetter := &serverstatus.DefaultServerStatusGetter{ Namespace: f.Namespace(), Context: ctx, } printVersion(os.Stdout, clientOnly, kbClient, serverStatusGetter) }, } c.Flags().DurationVar(&timeout, "timeout", timeout, "Maximum time to wait for server version to be reported. Default is 5 seconds.") c.Flags().BoolVar(&clientOnly, "client-only", clientOnly, "Only get velero client version, not server version") return c } func printVersion(w io.Writer, clientOnly bool, kbClient kbclient.Client, serverStatusGetter serverstatus.Getter) { fmt.Fprintln(w, "Client:") fmt.Fprintf(w, "\tVersion: %s\n", buildinfo.Version) fmt.Fprintf(w, "\tGit commit: %s\n", buildinfo.FormattedGitSHA()) if clientOnly { return } serverStatus, err := serverStatusGetter.GetServerStatus(kbClient) if err != nil { fmt.Fprintf(w, "\n", err) return } fmt.Fprintln(w, "Server:") fmt.Fprintf(w, "\tVersion: %s\n", serverStatus.Status.ServerVersion) serverSemVer := semver.MajorMinor(serverStatus.Status.ServerVersion) cliSemVer := semver.MajorMinor(buildinfo.Version) if serverSemVer != cliSemVer { upgrade := "client" cmp := semver.Compare(cliSemVer, serverSemVer) if cmp == 1 { upgrade = "server" } fmt.Fprintf(w, "# WARNING: the client version does not match the server version. Please update %s\n", upgrade) } } ================================================ FILE: pkg/cmd/cli/version/version_test.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package version import ( "bytes" "fmt" "testing" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" kbclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/buildinfo" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func TestPrintVersion(t *testing.T) { // set up some non-empty buildinfo values, but put them back to their // defaults at the end of the test var ( origVersion = buildinfo.Version origGitSHA = buildinfo.GitSHA origGitTreeState = buildinfo.GitTreeState ) defer func() { buildinfo.Version = origVersion buildinfo.GitSHA = origGitSHA buildinfo.GitTreeState = origGitTreeState }() buildinfo.Version = "v1.0.0" buildinfo.GitSHA = "somegitsha" buildinfo.GitTreeState = "dirty" clientVersion := fmt.Sprintf("Client:\n\tVersion: %s\n\tGit commit: %s\n", buildinfo.Version, buildinfo.FormattedGitSHA()) tests := []struct { name string clientOnly bool serverStatusRequest *velerov1.ServerStatusRequest getterError error want string }{ { name: "client-only", clientOnly: true, want: clientVersion, }, { name: "server status getter error", clientOnly: false, serverStatusRequest: nil, getterError: errors.New("an error"), want: clientVersion + "\n", }, { name: "server status getter returns normally", clientOnly: false, serverStatusRequest: builder.ForServerStatusRequest("velero", "ssr-1", "0").ServerVersion("v1.0.1").Result(), getterError: nil, want: clientVersion + "Server:\n\tVersion: v1.0.1\n", }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { var ( kbClient = velerotest.NewFakeControllerRuntimeClient(t) serverStatusGetter = new(mockServerStatusGetter) buf = new(bytes.Buffer) ) defer serverStatusGetter.AssertExpectations(t) // GetServerStatus should only be called when clientOnly = false if !tc.clientOnly { serverStatusGetter.On("GetServerStatus", kbClient).Return(tc.serverStatusRequest, tc.getterError) } printVersion(buf, tc.clientOnly, kbClient, serverStatusGetter) assert.Equal(t, tc.want, buf.String()) }) } } // mockServerStatusGetter is an autogenerated mock type for the serverStatusGetter type // Code generated by mockery v2.2.1. type mockServerStatusGetter struct { mock.Mock } // GetServerStatus provides a mock function with given fields: mgr func (_m *mockServerStatusGetter) GetServerStatus(kbClient kbclient.Client) (*velerov1.ServerStatusRequest, error) { ret := _m.Called(kbClient) var r0 *velerov1.ServerStatusRequest if rf, ok := ret.Get(0).(func(kbclient.Client) *velerov1.ServerStatusRequest); ok { r0 = rf(kbClient) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(*velerov1.ServerStatusRequest) } } var r1 error if rf, ok := ret.Get(1).(func(kbclient.Client) error); ok { r1 = rf(kbClient) } else { r1 = ret.Error(1) } return r0, r1 } ================================================ FILE: pkg/cmd/const.go ================================================ package cmd var TRUE = "true" ================================================ FILE: pkg/cmd/errors.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cmd import ( "context" "fmt" "os" ) // CheckError prints err to stderr and exits with code 1 if err is not nil. Otherwise, it is a // no-op. func CheckError(err error) { if err != nil { if err != context.Canceled { fmt.Fprintf(os.Stderr, "An error occurred: %v\n", err) } os.Exit(1) } } // Exit prints msg (with optional args), plus a newline, to stderr and exits with code 1. func Exit(msg string, args ...any) { fmt.Fprintf(os.Stderr, msg+"\n", args...) os.Exit(1) } ================================================ FILE: pkg/cmd/server/config/config.go ================================================ package config import ( "fmt" "strings" "time" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/sirupsen/logrus" "github.com/spf13/pflag" "github.com/vmware-tanzu/velero/internal/credentials" "github.com/vmware-tanzu/velero/pkg/cmd/util/flag" "github.com/vmware-tanzu/velero/pkg/constant" podvolumeconfigs "github.com/vmware-tanzu/velero/pkg/podvolume/configs" "github.com/vmware-tanzu/velero/pkg/types" "github.com/vmware-tanzu/velero/pkg/uploader" "github.com/vmware-tanzu/velero/pkg/util/logging" ) const ( // the port where prometheus metrics are exposed defaultMetricsAddress = ":8085" defaultBackupSyncPeriod = time.Minute defaultStoreValidationFrequency = time.Minute defaultPodVolumeOperationTimeout = 240 * time.Minute defaultResourceTerminatingTimeout = 10 * time.Minute // server's client default qps and burst defaultClientQPS float32 = 100.0 defaultClientBurst int = 100 defaultClientPageSize int = 500 defaultProfilerAddress = "localhost:6060" // the default TTL for a backup defaultBackupTTL = 30 * 24 * time.Hour defaultCSISnapshotTimeout = 10 * time.Minute defaultItemOperationTimeout = 4 * time.Hour resourceTimeout = 10 * time.Minute defaultMaxConcurrentK8SConnections = 30 defaultDisableInformerCache = false DefaultItemBlockWorkerCount = 1 DefaultConcurrentBackups = 1 ) var ( // DisableableControllers is a list of controllers that can be disabled DisableableControllers = []string{ constant.ControllerBackupQueue, constant.ControllerBackup, constant.ControllerBackupOperations, constant.ControllerBackupDeletion, constant.ControllerBackupFinalizer, constant.ControllerBackupSync, constant.ControllerDownloadRequest, constant.ControllerGarbageCollection, constant.ControllerBackupRepo, constant.ControllerRestore, constant.ControllerRestoreOperations, constant.ControllerSchedule, constant.ControllerServerStatusRequest, constant.ControllerRestoreFinalizer, } /* High priorities: - Custom Resource Definitions come before Custom Resource so that they can be restored with their corresponding CRD. - Namespaces go second because all namespaced resources depend on them. - Storage Classes are needed to create PVs and PVCs correctly. - VolumeSnapshotClasses are needed to provision volumes using volumesnapshots - VolumeSnapshotContents are needed as they contain the handle to the volume snapshot in the storage provider - VolumeSnapshots are needed to create PVCs using the VolumeSnapshot as their data source. - DataUploads need to restore before PVC for Snapshot DataMover to work, because PVC needs the DataUploadResults to create DataDownloads. - PVs go before PVCs because PVCs depend on them. - PVCs go before pods or controllers so they can be mounted as volumes. - Service accounts go before secrets so service account token secrets can be filled automatically. - Secrets and ConfigMaps go before pods or controllers so they can be mounted as volumes. - Limit ranges go before pods or controllers so pods can use them. - Pods go before controllers so they can be explicitly restored and potentially have pod volume restores run before controllers adopt the pods. - Replica sets go before deployments/other controllers so they can be explicitly restored and be adopted by controllers. - CAPI ClusterClasses go before Clusters. - Endpoints go before Services so no new Endpoints will be created - Services go before Clusters so they can be adopted by AKO-operator and no new Services will be created for the same clusters Low priorities: - Tanzu ClusterBootstraps go last as it can reference any other kind of resources. - ClusterBootstraps go before CAPI Clusters otherwise a new default ClusterBootstrap object is created for the cluster - CAPI Clusters come before ClusterResourceSets because failing to do so means the CAPI controller-manager will panic. Both Clusters and ClusterResourceSets need to come before ClusterResourceSetBinding in order to properly restore workload clusters. See https://github.com/kubernetes-sigs/cluster-api/issues/4105 - apps.kappctrl.k14s.io and packageinstalls.packaging.carvel.dev go after workloads(pod/replicaset/etc.), otherwise the controller may creates new workloads before restoring them */ defaultRestorePriorities = types.Priorities{ HighPriorities: []string{ "customresourcedefinitions", "namespaces", "storageclasses", "volumesnapshotclass.snapshot.storage.k8s.io", "volumesnapshotcontents.snapshot.storage.k8s.io", "volumesnapshots.snapshot.storage.k8s.io", "datauploads.velero.io", "persistentvolumes", "persistentvolumeclaims", "clusterroles", "roles", "serviceaccounts", "clusterrolebindings", "rolebindings", "secrets", "configmaps", "limitranges", "priorityclasses", "pods", // we fully qualify replicasets.apps because prior to Kubernetes 1.16, replicasets also // existed in the extensions API group, but we back up replicasets from "apps" so we want // to ensure that we prioritize restoring from "apps" too, since this is how they're stored // in the backup. "replicasets.apps", "clusterclasses.cluster.x-k8s.io", "endpoints", "services", }, LowPriorities: []string{ "clusterbootstraps.run.tanzu.vmware.com", "clusters.cluster.x-k8s.io", "clusterresourcesets.addons.cluster.x-k8s.io", "apps.kappctrl.k14s.io", "packageinstalls.packaging.carvel.dev", }, } ) type Config struct { PluginDir string MetricsAddress string DefaultBackupLocation string // TODO(2.0) Deprecate defaultBackupLocation BackupSyncPeriod time.Duration PodVolumeOperationTimeout time.Duration ResourceTerminatingTimeout time.Duration DefaultBackupTTL time.Duration DefaultVGSLabelKey string StoreValidationFrequency time.Duration DefaultCSISnapshotTimeout time.Duration DefaultItemOperationTimeout time.Duration ResourceTimeout time.Duration RestoreResourcePriorities types.Priorities DefaultVolumeSnapshotLocations flag.Map RestoreOnly bool DisabledControllers []string ClientQPS float32 ClientBurst int ClientPageSize int ProfilerAddress string LogLevel *logging.LevelFlag LogFormat *logging.FormatFlag RepoMaintenanceFrequency time.Duration GarbageCollectionFrequency time.Duration ItemOperationSyncFrequency time.Duration DefaultVolumesToFsBackup bool UploaderType string MaxConcurrentK8SConnections int DefaultSnapshotMoveData bool DisableInformerCache bool ScheduleSkipImmediately bool CredentialsDirectory string BackupRepoConfig string RepoMaintenanceJobConfig string ItemBlockWorkerCount int ConcurrentBackups int } func GetDefaultConfig() *Config { config := &Config{ PluginDir: "/plugins", MetricsAddress: defaultMetricsAddress, DefaultBackupLocation: "default", DefaultVolumeSnapshotLocations: flag.NewMap().WithKeyValueDelimiter(':'), BackupSyncPeriod: defaultBackupSyncPeriod, DefaultBackupTTL: defaultBackupTTL, DefaultVGSLabelKey: velerov1api.DefaultVGSLabelKey, DefaultCSISnapshotTimeout: defaultCSISnapshotTimeout, DefaultItemOperationTimeout: defaultItemOperationTimeout, ResourceTimeout: resourceTimeout, StoreValidationFrequency: defaultStoreValidationFrequency, PodVolumeOperationTimeout: defaultPodVolumeOperationTimeout, RestoreResourcePriorities: defaultRestorePriorities, ClientQPS: defaultClientQPS, ClientBurst: defaultClientBurst, ClientPageSize: defaultClientPageSize, ProfilerAddress: defaultProfilerAddress, ResourceTerminatingTimeout: defaultResourceTerminatingTimeout, LogLevel: logging.LogLevelFlag(logrus.InfoLevel), LogFormat: logging.NewFormatFlag(), DefaultVolumesToFsBackup: podvolumeconfigs.DefaultVolumesToFsBackup, UploaderType: uploader.KopiaType, MaxConcurrentK8SConnections: defaultMaxConcurrentK8SConnections, DefaultSnapshotMoveData: false, DisableInformerCache: defaultDisableInformerCache, ScheduleSkipImmediately: false, CredentialsDirectory: credentials.DefaultStoreDirectory(), ItemBlockWorkerCount: DefaultItemBlockWorkerCount, ConcurrentBackups: DefaultConcurrentBackups, } return config } func (c *Config) BindFlags(flags *pflag.FlagSet) { flags.Var(c.LogLevel, "log-level", fmt.Sprintf("The level at which to log. Valid values are %s.", strings.Join(c.LogLevel.AllowedValues(), ", "))) flags.Var(c.LogFormat, "log-format", fmt.Sprintf("The format for log output. Valid values are %s.", strings.Join(c.LogFormat.AllowedValues(), ", "))) flags.StringVar(&c.PluginDir, "plugin-dir", c.PluginDir, "Directory containing Velero plugins") flags.StringVar(&c.MetricsAddress, "metrics-address", c.MetricsAddress, "The address to expose prometheus metrics") flags.DurationVar(&c.BackupSyncPeriod, "backup-sync-period", c.BackupSyncPeriod, "How often to ensure all Velero backups in object storage exist as Backup API objects in the cluster. This is the default sync period if none is explicitly specified for a backup storage location.") flags.DurationVar(&c.PodVolumeOperationTimeout, "fs-backup-timeout", c.PodVolumeOperationTimeout, "How long pod volume file system backups/restores should be allowed to run before timing out.") flags.BoolVar(&c.RestoreOnly, "restore-only", c.RestoreOnly, "Run in a mode where only restores are allowed; backups, schedules, and garbage-collection are all disabled. DEPRECATED: this flag will be removed in v2.0. Use read-only backup storage locations instead.") flags.StringSliceVar(&c.DisabledControllers, "disable-controllers", c.DisabledControllers, fmt.Sprintf("List of controllers to disable on startup. Valid values are %s", strings.Join(DisableableControllers, ","))) flags.Var(&c.RestoreResourcePriorities, "restore-resource-priorities", "Desired order of resource restores, the priority list contains two parts which are split by \"-\" element. The resources before \"-\" element are restored first as high priorities, the resources after \"-\" element are restored last as low priorities, and any resource not in the list will be restored alphabetically between the high and low priorities.") flags.StringVar(&c.DefaultBackupLocation, "default-backup-storage-location", c.DefaultBackupLocation, "Name of the default backup storage location. DEPRECATED: this flag will be removed in v2.0. Use \"velero backup-location set --default\" instead.") flags.DurationVar(&c.StoreValidationFrequency, "store-validation-frequency", c.StoreValidationFrequency, "How often to verify if the storage is valid. Optional. Set this to `0s` to disable sync. Default 1 minute.") flags.Float32Var(&c.ClientQPS, "client-qps", c.ClientQPS, "Maximum number of requests per second by the server to the Kubernetes API once the burst limit has been reached.") flags.IntVar(&c.ClientBurst, "client-burst", c.ClientBurst, "Maximum number of requests by the server to the Kubernetes API in a short period of time.") flags.IntVar(&c.ClientPageSize, "client-page-size", c.ClientPageSize, "Page size of requests by the server to the Kubernetes API when listing objects during a backup. Set to 0 to disable paging.") flags.StringVar(&c.ProfilerAddress, "profiler-address", c.ProfilerAddress, "The address to expose the pprof profiler.") flags.DurationVar(&c.ResourceTerminatingTimeout, "terminating-resource-timeout", c.ResourceTerminatingTimeout, "How long to wait on persistent volumes and namespaces to terminate during a restore before timing out.") flags.DurationVar(&c.DefaultBackupTTL, "default-backup-ttl", c.DefaultBackupTTL, "How long to wait by default before backups can be garbage collected.") flags.StringVar(&c.DefaultVGSLabelKey, "volume-group-snapshot-label-key", c.DefaultVGSLabelKey, "Label key for grouping PVCs into VolumeGroupSnapshot. Default value is 'velero.io/volume-group'") flags.DurationVar(&c.RepoMaintenanceFrequency, "default-repo-maintain-frequency", c.RepoMaintenanceFrequency, "How often 'maintain' is run for backup repositories by default.") flags.DurationVar(&c.GarbageCollectionFrequency, "garbage-collection-frequency", c.GarbageCollectionFrequency, "How often garbage collection is run for expired backups.") flags.DurationVar(&c.ItemOperationSyncFrequency, "item-operation-sync-frequency", c.ItemOperationSyncFrequency, "How often to check status on backup/restore operations after backup/restore processing. Default is 10 seconds") flags.BoolVar(&c.DefaultVolumesToFsBackup, "default-volumes-to-fs-backup", c.DefaultVolumesToFsBackup, "Backup all volumes with pod volume file system backup by default.") flags.StringVar(&c.UploaderType, "uploader-type", c.UploaderType, "Type of uploader to handle the transfer of data of pod volumes") flags.DurationVar(&c.DefaultItemOperationTimeout, "default-item-operation-timeout", c.DefaultItemOperationTimeout, "How long to wait on asynchronous BackupItemActions and RestoreItemActions to complete before timing out. Default is 4 hours") flags.DurationVar(&c.ResourceTimeout, "resource-timeout", c.ResourceTimeout, "How long to wait for resource processes which are not covered by other specific timeout parameters. Default is 10 minutes.") flags.IntVar(&c.MaxConcurrentK8SConnections, "max-concurrent-k8s-connections", c.MaxConcurrentK8SConnections, "Max concurrent connections number that Velero can create with kube-apiserver. Default is 30.") flags.BoolVar(&c.DefaultSnapshotMoveData, "default-snapshot-move-data", c.DefaultSnapshotMoveData, "Move data by default for all snapshots supporting data movement.") flags.BoolVar(&c.DisableInformerCache, "disable-informer-cache", c.DisableInformerCache, "Disable informer cache for Get calls on restore. With this enabled, it will speed up restore in cases where there are backup resources which already exist in the cluster, but for very large clusters this will increase velero memory usage. Default is false (don't disable).") flags.BoolVar(&c.ScheduleSkipImmediately, "schedule-skip-immediately", c.ScheduleSkipImmediately, "Skip the first scheduled backup immediately after creating a schedule. Default is false (don't skip).") flags.Var(&c.DefaultVolumeSnapshotLocations, "default-volume-snapshot-locations", "List of unique volume providers and default volume snapshot location (provider1:location-01,provider2:location-02,...)") flags.StringVar( &c.BackupRepoConfig, "backup-repository-configmap", c.BackupRepoConfig, "The name of ConfigMap containing backup repository configurations.", ) flags.StringVar( &c.RepoMaintenanceJobConfig, "repo-maintenance-job-configmap", c.RepoMaintenanceJobConfig, "The name of ConfigMap containing repository maintenance Job configurations.", ) flags.IntVar( &c.ItemBlockWorkerCount, "item-block-worker-count", c.ItemBlockWorkerCount, "Number of worker threads to process ItemBlocks. Default is one. Optional.", ) flags.IntVar( &c.ConcurrentBackups, "concurrent-backups", c.ConcurrentBackups, "Number of backups to process concurrently. Default is one. Optional.", ) } ================================================ FILE: pkg/cmd/server/config/config_test.go ================================================ package config import ( "testing" "github.com/spf13/pflag" "github.com/stretchr/testify/assert" ) func TestGetDefaultConfig(t *testing.T) { config := GetDefaultConfig() assert.Equal(t, 1, config.ItemBlockWorkerCount) } func TestBindFlags(t *testing.T) { config := GetDefaultConfig() config.BindFlags(pflag.CommandLine) assert.Equal(t, 1, config.ItemBlockWorkerCount) } ================================================ FILE: pkg/cmd/server/plugin/plugin.go ================================================ /* Copyright 2017, 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package plugin import ( "github.com/sirupsen/logrus" "github.com/spf13/cobra" apiextensions "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" "github.com/vmware-tanzu/velero/pkg/constant" "github.com/vmware-tanzu/velero/pkg/datamover" dia "github.com/vmware-tanzu/velero/internal/delete/actions/csi" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" bia "github.com/vmware-tanzu/velero/pkg/backup/actions" csibia "github.com/vmware-tanzu/velero/pkg/backup/actions/csi" "github.com/vmware-tanzu/velero/pkg/client" velerodiscovery "github.com/vmware-tanzu/velero/pkg/discovery" "github.com/vmware-tanzu/velero/pkg/features" iba "github.com/vmware-tanzu/velero/pkg/itemblock/actions" veleroplugin "github.com/vmware-tanzu/velero/pkg/plugin/framework" plugincommon "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" ria "github.com/vmware-tanzu/velero/pkg/restore/actions" csiria "github.com/vmware-tanzu/velero/pkg/restore/actions/csi" "github.com/vmware-tanzu/velero/pkg/util/actionhelpers" ) func NewCommand(f client.Factory) *cobra.Command { pluginServer := veleroplugin.NewServer() c := &cobra.Command{ Use: "run-plugins", Hidden: true, Short: "INTERNAL COMMAND ONLY - not intended to be run directly by users", Run: func(c *cobra.Command, args []string) { config := pluginServer.GetConfig() f.SetClientQPS(config.ClientQPS) f.SetClientBurst(config.ClientBurst) pluginServer = pluginServer. RegisterBackupItemAction( "velero.io/pv", newPVBackupItemAction, ). RegisterBackupItemAction( "velero.io/pod", newPodBackupItemAction, ). RegisterBackupItemAction( "velero.io/service-account", newServiceAccountBackupItemAction(f), ). RegisterRestoreItemAction( "velero.io/job", newJobRestoreItemAction, ). RegisterRestoreItemAction( "velero.io/pod", newPodRestoreItemAction, ). RegisterRestoreItemAction( "velero.io/pod-volume-restore", newPodVolumeRestoreItemAction(f), ). RegisterRestoreItemAction( "velero.io/init-restore-hook", newInitRestoreHookPodAction, ). RegisterRestoreItemAction( "velero.io/service", newServiceRestoreItemAction, ). RegisterRestoreItemAction( "velero.io/service-account", newServiceAccountRestoreItemAction, ). RegisterRestoreItemAction( "velero.io/add-pvc-from-pod", newAddPVCFromPodRestoreItemAction, ). RegisterRestoreItemAction( "velero.io/change-storage-class", newChangeStorageClassRestoreItemAction(f), ). RegisterRestoreItemAction( "velero.io/change-image-name", newChangeImageNameRestoreItemAction(f), ). RegisterRestoreItemAction( "velero.io/role-bindings", newRoleBindingItemAction, ). RegisterRestoreItemAction( "velero.io/cluster-role-bindings", newClusterRoleBindingItemAction, ). RegisterRestoreItemAction( "velero.io/crd-preserve-fields", newCRDV1PreserveUnknownFieldsItemAction, ). RegisterRestoreItemAction( "velero.io/pvc", newPVCRestoreItemAction(f), ). RegisterRestoreItemAction( "velero.io/apiservice", newAPIServiceRestoreItemAction, ). RegisterRestoreItemAction( "velero.io/admission-webhook-configuration", newAdmissionWebhookConfigurationAction, ). RegisterRestoreItemAction( "velero.io/secret", newSecretRestoreItemAction(f), ). RegisterRestoreItemAction( "velero.io/dataupload", newDataUploadRetrieveAction(f), ). RegisterDeleteItemAction( "velero.io/dataupload-delete", newDateUploadDeleteItemAction(f), ). RegisterDeleteItemAction( "velero.io/csi-volumesnapshotcontent-delete", newVolumeSnapshotContentDeleteItemAction(f), ). RegisterBackupItemActionV2( "velero.io/csi-pvc-backupper", newPvcBackupItemAction(f), ). RegisterBackupItemActionV2( "velero.io/csi-volumesnapshot-backupper", newVolumeSnapshotBackupItemAction(f), ). RegisterBackupItemActionV2( "velero.io/csi-volumesnapshotcontent-backupper", newVolumeSnapshotContentBackupItemAction, ). RegisterBackupItemActionV2( "velero.io/csi-volumesnapshotclass-backupper", newVolumeSnapshotClassBackupItemAction, ). RegisterRestoreItemActionV2( constant.PluginCSIPVCRestoreRIA, newPvcRestoreItemAction(f), ). RegisterRestoreItemActionV2( constant.PluginCsiVolumeSnapshotRestoreRIA, newVolumeSnapshotRestoreItemAction(f), ). RegisterRestoreItemActionV2( "velero.io/csi-volumesnapshotcontent-restorer", newVolumeSnapshotContentRestoreItemAction(f), ). RegisterRestoreItemActionV2( "velero.io/csi-volumesnapshotclass-restorer", newVolumeSnapshotClassRestoreItemAction, ). RegisterItemBlockAction( "velero.io/pvc", newPVCItemBlockAction(f), ). RegisterItemBlockAction( "velero.io/pod", newPodItemBlockAction, ). RegisterItemBlockAction( "velero.io/service-account", newServiceAccountItemBlockAction(f), ) if !features.IsEnabled(velerov1api.APIGroupVersionsFeatureFlag) { // Do not register crd-remap-version BIA if the API Group feature // flag is enabled, so that the v1 CRD can be backed up. pluginServer = pluginServer.RegisterBackupItemAction( "velero.io/crd-remap-version", newRemapCRDVersionAction(f), ) } pluginServer.Serve() }, FParseErrWhitelist: cobra.FParseErrWhitelist{ // Velero.io word list : ignore UnknownFlags: true, }, } pluginServer.BindFlags(c.Flags()) return c } func newPVBackupItemAction(logger logrus.FieldLogger) (any, error) { return bia.NewPVCAction(logger), nil } func newPodBackupItemAction(logger logrus.FieldLogger) (any, error) { return bia.NewPodAction(logger), nil } func newServiceAccountBackupItemAction(f client.Factory) plugincommon.HandlerInitializer { return func(logger logrus.FieldLogger) (any, error) { // TODO(ncdc): consider a k8s style WantsKubernetesClientSet initialization approach clientset, err := f.KubeClient() if err != nil { return nil, err } discoveryClient, err := f.DiscoveryClient() if err != nil { return nil, err } discoveryHelper, err := velerodiscovery.NewHelper(discoveryClient, logger) if err != nil { return nil, err } action, err := bia.NewServiceAccountAction( logger, actionhelpers.NewClusterRoleBindingListerMap(clientset), discoveryHelper) if err != nil { return nil, err } return action, nil } } func newRemapCRDVersionAction(f client.Factory) plugincommon.HandlerInitializer { return func(logger logrus.FieldLogger) (any, error) { config, err := f.ClientConfig() if err != nil { return nil, err } client, err := apiextensions.NewForConfig(config) if err != nil { return nil, err } discoveryClient, err := f.DiscoveryClient() if err != nil { return nil, err } discoveryHelper, err := velerodiscovery.NewHelper(discoveryClient, logger) if err != nil { return nil, err } return bia.NewRemapCRDVersionAction(logger, client.ApiextensionsV1beta1().CustomResourceDefinitions(), discoveryHelper), nil } } func newJobRestoreItemAction(logger logrus.FieldLogger) (any, error) { return ria.NewJobAction(logger), nil } func newPodRestoreItemAction(logger logrus.FieldLogger) (any, error) { return ria.NewPodAction(logger), nil } func newInitRestoreHookPodAction(logger logrus.FieldLogger) (any, error) { return ria.NewInitRestoreHookPodAction(logger), nil } func newPodVolumeRestoreItemAction(f client.Factory) plugincommon.HandlerInitializer { return func(logger logrus.FieldLogger) (any, error) { client, err := f.KubeClient() if err != nil { return nil, err } crClient, err := f.KubebuilderClient() if err != nil { return nil, err } return ria.NewPodVolumeRestoreAction(logger, client.CoreV1().ConfigMaps(f.Namespace()), crClient, f.Namespace()) } } func newServiceRestoreItemAction(logger logrus.FieldLogger) (any, error) { return ria.NewServiceAction(logger), nil } func newServiceAccountRestoreItemAction(logger logrus.FieldLogger) (any, error) { return ria.NewServiceAccountAction(logger), nil } func newAddPVCFromPodRestoreItemAction(logger logrus.FieldLogger) (any, error) { return ria.NewAddPVCFromPodAction(logger), nil } func newCRDV1PreserveUnknownFieldsItemAction(logger logrus.FieldLogger) (any, error) { return ria.NewCRDV1PreserveUnknownFieldsAction(logger), nil } func newChangeStorageClassRestoreItemAction(f client.Factory) plugincommon.HandlerInitializer { return func(logger logrus.FieldLogger) (any, error) { client, err := f.KubeClient() if err != nil { return nil, err } return ria.NewChangeStorageClassAction( logger, client.CoreV1().ConfigMaps(f.Namespace()), client.StorageV1().StorageClasses(), ), nil } } func newChangeImageNameRestoreItemAction(f client.Factory) plugincommon.HandlerInitializer { return func(logger logrus.FieldLogger) (any, error) { client, err := f.KubeClient() if err != nil { return nil, err } return ria.NewChangeImageNameAction( logger, client.CoreV1().ConfigMaps(f.Namespace()), ), nil } } func newRoleBindingItemAction(logger logrus.FieldLogger) (any, error) { return ria.NewRoleBindingAction(logger), nil } func newClusterRoleBindingItemAction(logger logrus.FieldLogger) (any, error) { return ria.NewClusterRoleBindingAction(logger), nil } func newPVCRestoreItemAction(f client.Factory) plugincommon.HandlerInitializer { return func(logger logrus.FieldLogger) (any, error) { client, err := f.KubeClient() if err != nil { return nil, err } return ria.NewPVCAction( logger, client.CoreV1().ConfigMaps(f.Namespace()), client.CoreV1().Nodes(), ), nil } } func newAPIServiceRestoreItemAction(logger logrus.FieldLogger) (any, error) { return ria.NewAPIServiceAction(logger), nil } func newAdmissionWebhookConfigurationAction(logger logrus.FieldLogger) (any, error) { return ria.NewAdmissionWebhookConfigurationAction(logger), nil } func newSecretRestoreItemAction(f client.Factory) plugincommon.HandlerInitializer { return func(logger logrus.FieldLogger) (any, error) { client, err := f.KubebuilderClient() if err != nil { return nil, err } return ria.NewSecretAction(logger, client), nil } } func newDataUploadRetrieveAction(f client.Factory) plugincommon.HandlerInitializer { return func(logger logrus.FieldLogger) (any, error) { client, err := f.KubebuilderClient() if err != nil { return nil, err } return ria.NewDataUploadRetrieveAction(logger, client), nil } } func newDateUploadDeleteItemAction(f client.Factory) plugincommon.HandlerInitializer { return func(logger logrus.FieldLogger) (any, error) { client, err := f.KubebuilderClient() if err != nil { return nil, err } return datamover.NewDataUploadDeleteAction(logger, client), nil } } // CSI plugin init functions. // BackupItemAction plugins func newPvcBackupItemAction(f client.Factory) plugincommon.HandlerInitializer { return csibia.NewPvcBackupItemAction(f) } func newVolumeSnapshotBackupItemAction(f client.Factory) plugincommon.HandlerInitializer { return csibia.NewVolumeSnapshotBackupItemAction(f) } func newVolumeSnapshotContentBackupItemAction(logger logrus.FieldLogger) (any, error) { return csibia.NewVolumeSnapshotContentBackupItemAction(logger) } func newVolumeSnapshotClassBackupItemAction(logger logrus.FieldLogger) (any, error) { return csibia.NewVolumeSnapshotClassBackupItemAction(logger) } // DeleteItemAction plugins func newVolumeSnapshotContentDeleteItemAction(f client.Factory) plugincommon.HandlerInitializer { return dia.NewVolumeSnapshotContentDeleteItemAction(f) } // RestoreItemAction plugins func newPvcRestoreItemAction(f client.Factory) plugincommon.HandlerInitializer { return csiria.NewPvcRestoreItemAction(f) } func newVolumeSnapshotRestoreItemAction(f client.Factory) plugincommon.HandlerInitializer { return csiria.NewVolumeSnapshotRestoreItemAction(f) } func newVolumeSnapshotContentRestoreItemAction(f client.Factory) plugincommon.HandlerInitializer { return csiria.NewVolumeSnapshotContentRestoreItemAction(f) } func newVolumeSnapshotClassRestoreItemAction(logger logrus.FieldLogger) (any, error) { return csiria.NewVolumeSnapshotClassRestoreItemAction(logger) } // ItemBlockAction plugins func newPVCItemBlockAction(f client.Factory) plugincommon.HandlerInitializer { return iba.NewPVCAction(f) } func newPodItemBlockAction(logger logrus.FieldLogger) (any, error) { return iba.NewPodAction(logger), nil } func newServiceAccountItemBlockAction(f client.Factory) plugincommon.HandlerInitializer { return func(logger logrus.FieldLogger) (any, error) { // TODO(ncdc): consider a k8s style WantsKubernetesClientSet initialization approach clientset, err := f.KubeClient() if err != nil { return nil, err } discoveryClient, err := f.DiscoveryClient() if err != nil { return nil, err } discoveryHelper, err := velerodiscovery.NewHelper(discoveryClient, logger) if err != nil { return nil, err } action, err := iba.NewServiceAccountAction( logger, actionhelpers.NewClusterRoleBindingListerMap(clientset), discoveryHelper) if err != nil { return nil, err } return action, nil } } ================================================ FILE: pkg/cmd/server/server.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package server import ( "context" "fmt" "log" "net/http" "net/http/pprof" "os" "strings" "time" logrusr "github.com/bombsimon/logrusr/v3" volumegroupsnapshotv1beta1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1beta1" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/sirupsen/logrus" "github.com/spf13/cobra" appsv1api "k8s.io/api/apps/v1" batchv1api "k8s.io/api/batch/v1" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" kubeerrs "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/discovery" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/klog/v2" "k8s.io/utils/clock" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" "github.com/vmware-tanzu/velero/internal/credentials" "github.com/vmware-tanzu/velero/internal/hook" "github.com/vmware-tanzu/velero/internal/storage" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" "github.com/vmware-tanzu/velero/pkg/backup" "github.com/vmware-tanzu/velero/pkg/buildinfo" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" "github.com/vmware-tanzu/velero/pkg/cmd/server/config" "github.com/vmware-tanzu/velero/pkg/cmd/util/signals" "github.com/vmware-tanzu/velero/pkg/constant" "github.com/vmware-tanzu/velero/pkg/controller" velerodiscovery "github.com/vmware-tanzu/velero/pkg/discovery" "github.com/vmware-tanzu/velero/pkg/features" "github.com/vmware-tanzu/velero/pkg/itemoperationmap" "github.com/vmware-tanzu/velero/pkg/metrics" "github.com/vmware-tanzu/velero/pkg/nodeagent" "github.com/vmware-tanzu/velero/pkg/persistence" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt/process" "github.com/vmware-tanzu/velero/pkg/podexec" "github.com/vmware-tanzu/velero/pkg/podvolume" "github.com/vmware-tanzu/velero/pkg/repository" repokey "github.com/vmware-tanzu/velero/pkg/repository/keys" repomanager "github.com/vmware-tanzu/velero/pkg/repository/manager" "github.com/vmware-tanzu/velero/pkg/restore" velerotypes "github.com/vmware-tanzu/velero/pkg/types" "github.com/vmware-tanzu/velero/pkg/uploader" "github.com/vmware-tanzu/velero/pkg/util/filesystem" "github.com/vmware-tanzu/velero/pkg/util/kube" "github.com/vmware-tanzu/velero/pkg/util/logging" ) func NewCommand(f client.Factory) *cobra.Command { config := config.GetDefaultConfig() var command = &cobra.Command{ Use: "server", Short: "Run the velero server", Long: "Run the velero server", Hidden: true, Run: func(c *cobra.Command, args []string) { // go-plugin uses log.Println to log when it's waiting for all plugin processes to complete so we need to // set its output to stdout. log.SetOutput(os.Stdout) logLevel := config.LogLevel.Parse() format := config.LogFormat.Parse() // Make sure we log to stdout so cloud log dashboards don't show this as an error. logrus.SetOutput(os.Stdout) // Velero's DefaultLogger logs to stdout, so all is good there. logger := logging.DefaultLogger(logLevel, format) logger.Infof("setting log-level to %s", strings.ToUpper(logLevel.String())) logger.Infof("Starting Velero server %s (%s)", buildinfo.Version, buildinfo.FormattedGitSHA()) if len(features.All()) > 0 { logger.Infof("%d feature flags enabled %s", len(features.All()), features.All()) } else { logger.Info("No feature flags enabled") } f.SetBasename(fmt.Sprintf("%s-%s", c.Parent().Name(), c.Name())) s, err := newServer(f, config, logger) cmd.CheckError(err) cmd.CheckError(s.run()) }, } config.BindFlags(command.Flags()) return command } type server struct { namespace string metricsAddress string kubeClientConfig *rest.Config kubeClient kubernetes.Interface discoveryClient discovery.AggregatedDiscoveryInterface discoveryHelper velerodiscovery.Helper dynamicClient dynamic.Interface // controller-runtime client. the difference from the controller-manager's client // is that the controller-manager's client is limited to list namespaced-scoped // resources in the namespace where Velero is installed, or the cluster-scoped // resources. The crClient doesn't have the limitation. crClient ctrlclient.Client ctx context.Context cancelFunc context.CancelFunc logger logrus.FieldLogger logLevel logrus.Level pluginRegistry process.Registry repoManager repomanager.Manager repoLocker *repository.RepoLocker repoEnsurer *repository.Ensurer metrics *metrics.ServerMetrics config *config.Config mgr manager.Manager credentialFileStore credentials.FileStore credentialSecretStore credentials.SecretStore } func newServer(f client.Factory, config *config.Config, logger *logrus.Logger) (*server, error) { if msg, err := uploader.ValidateUploaderType(config.UploaderType); err != nil { return nil, err } else if msg != "" { logger.Warn(msg) } if config.ClientQPS < 0.0 { return nil, errors.New("client-qps must be positive") } f.SetClientQPS(config.ClientQPS) if config.ClientBurst <= 0 { return nil, errors.New("client-burst must be positive") } f.SetClientBurst(config.ClientBurst) if config.ClientPageSize < 0 { return nil, errors.New("client-page-size must not be negative") } kubeClient, err := f.KubeClient() if err != nil { return nil, err } dynamicClient, err := f.DynamicClient() if err != nil { return nil, err } crClient, err := f.KubebuilderClient() if err != nil { return nil, err } pluginRegistry := process.NewRegistry(config.PluginDir, logger, logger.Level) if err := pluginRegistry.DiscoverPlugins(); err != nil { return nil, err } // cancelFunc is not deferred here because if it was, then ctx would immediately // be canceled once this function exited, making it useless to any informers using later. // That, in turn, causes the velero server to halt when the first informer tries to use it. // Therefore, we must explicitly call it on the error paths in this function. ctx, cancelFunc := context.WithCancel(context.Background()) if len(config.BackupRepoConfig) > 0 { repoConfig := make(map[string]any) if err := kube.VerifyJSONConfigs(ctx, f.Namespace(), crClient, config.BackupRepoConfig, &repoConfig); err != nil { cancelFunc() return nil, err } } if len(config.RepoMaintenanceJobConfig) > 0 { if err := kube.VerifyJSONConfigs(ctx, f.Namespace(), crClient, config.RepoMaintenanceJobConfig, &velerotypes.JobConfigs{}); err != nil { cancelFunc() return nil, err } } clientConfig, err := f.ClientConfig() if err != nil { cancelFunc() return nil, err } scheme := runtime.NewScheme() if err := velerov1api.AddToScheme(scheme); err != nil { cancelFunc() return nil, err } if err := velerov2alpha1api.AddToScheme(scheme); err != nil { cancelFunc() return nil, err } if err := corev1api.AddToScheme(scheme); err != nil { cancelFunc() return nil, err } if err := snapshotv1api.AddToScheme(scheme); err != nil { cancelFunc() return nil, err } if err := volumegroupsnapshotv1beta1.AddToScheme(scheme); err != nil { cancelFunc() return nil, err } if err := batchv1api.AddToScheme(scheme); err != nil { cancelFunc() return nil, err } if err := appsv1api.AddToScheme(scheme); err != nil { cancelFunc() return nil, err } ctrl.SetLogger(logrusr.New(logger)) klog.SetLogger(logrusr.New(logger)) // klog.Logger is used by k8s.io/client-go var mgr manager.Manager retry := 10 for { mgr, err = ctrl.NewManager(clientConfig, ctrl.Options{ Scheme: scheme, Cache: cache.Options{ DefaultNamespaces: map[string]cache.Config{ f.Namespace(): {}, }, }, }) if err == nil { break } retry-- if retry == 0 { break } logger.WithError(err).Warn("Failed to create controller manager, need retry") time.Sleep(time.Second) } if err != nil { cancelFunc() return nil, errors.Wrap(err, "error creating controller manager") } credentialFileStore, err := credentials.NewNamespacedFileStore( mgr.GetClient(), f.Namespace(), config.CredentialsDirectory, filesystem.NewFileSystem(), ) if err != nil { cancelFunc() return nil, err } credentialSecretStore, err := credentials.NewNamespacedSecretStore(mgr.GetClient(), f.Namespace()) if err != nil { cancelFunc() return nil, err } var discoveryClient discovery.AggregatedDiscoveryInterface if discoveryClient, err = f.DiscoveryClient(); err != nil { cancelFunc() return nil, err } s := &server{ namespace: f.Namespace(), metricsAddress: config.MetricsAddress, kubeClientConfig: clientConfig, kubeClient: kubeClient, discoveryClient: discoveryClient, dynamicClient: dynamicClient, crClient: crClient, ctx: ctx, cancelFunc: cancelFunc, logger: logger, logLevel: logger.Level, pluginRegistry: pluginRegistry, config: config, mgr: mgr, credentialFileStore: credentialFileStore, credentialSecretStore: credentialSecretStore, } return s, nil } func (s *server) run() error { signals.CancelOnShutdown(s.cancelFunc, s.logger) if s.config.ProfilerAddress != "" { go s.runProfiler() } // Since s.namespace, which specifies where backups/restores/schedules/etc. should live, // *could* be different from the namespace where the Velero server pod runs, check to make // sure it exists, and fail fast if it doesn't. if err := s.namespaceExists(s.namespace); err != nil { return err } if err := s.initDiscoveryHelper(); err != nil { return err } if err := s.veleroResourcesExist(); err != nil { return err } s.checkNodeAgent() if err := s.initRepoManager(); err != nil { return err } if err := s.setupBeforeControllerRun(); err != nil { return err } if err := s.runControllers(s.config.DefaultVolumeSnapshotLocations.Data()); err != nil { return err } return nil } // setupBeforeControllerRun do any setup that needs to happen before the controllers are started. func (s *server) setupBeforeControllerRun() error { client, err := ctrlclient.New(s.mgr.GetConfig(), ctrlclient.Options{Scheme: s.mgr.GetScheme()}) // the function is called before starting the controller manager, the embedded client isn't ready to use, so create a new one here if err != nil { return errors.WithStack(err) } markInProgressCRsFailed(s.ctx, client, s.namespace, s.logger) if err := setDefaultBackupLocation(s.ctx, client, s.namespace, s.config.DefaultBackupLocation, s.logger); err != nil { return err } return nil } // setDefaultBackupLocation set the BSL that matches the "velero server --default-backup-storage-location" func setDefaultBackupLocation(ctx context.Context, client ctrlclient.Client, namespace, defaultBackupLocation string, logger logrus.FieldLogger) error { if defaultBackupLocation == "" { logger.Debug("No default backup storage location specified. Velero will not automatically select a backup storage location for new backups.") return nil } backupLocation := &velerov1api.BackupStorageLocation{} if err := client.Get(ctx, types.NamespacedName{Namespace: namespace, Name: defaultBackupLocation}, backupLocation); err != nil { if apierrors.IsNotFound(err) { logger.WithField("backupStorageLocation", defaultBackupLocation).WithError(err).Warn("Failed to set default backup storage location at server start") return nil } else { return errors.WithStack(err) } } if !backupLocation.Spec.Default { backupLocation.Spec.Default = true if err := client.Update(ctx, backupLocation); err != nil { return errors.WithStack(err) } logger.WithField("backupStorageLocation", defaultBackupLocation).Info("Set backup storage location as default") } return nil } // namespaceExists returns nil if namespace can be successfully // gotten from the Kubernetes API, or an error otherwise. func (s *server) namespaceExists(namespace string) error { s.logger.WithField("namespace", namespace).Info("Checking existence of namespace.") if _, err := s.kubeClient.CoreV1().Namespaces().Get(s.ctx, namespace, metav1.GetOptions{}); err != nil { return errors.WithStack(err) } s.logger.WithField("namespace", namespace).Info("Namespace exists") return nil } // initDiscoveryHelper instantiates the server's discovery helper and spawns a // goroutine to call Refresh() every 5 minutes. func (s *server) initDiscoveryHelper() error { discoveryHelper, err := velerodiscovery.NewHelper(s.discoveryClient, s.logger) if err != nil { return err } s.discoveryHelper = discoveryHelper go wait.Until( func() { if err := discoveryHelper.Refresh(); err != nil { s.logger.WithError(err).Error("Error refreshing discovery") } }, 5*time.Minute, s.ctx.Done(), ) return nil } // veleroResourcesExist checks for the existence of each Velero CRD via discovery // and returns an error if any of them don't exist. func (s *server) veleroResourcesExist() error { s.logger.Info("Checking existence of Velero custom resource definitions") // add more group versions whenever available gvResources := map[string]sets.Set[string]{ velerov1api.SchemeGroupVersion.String(): velerov1api.CustomResourceKinds(), velerov2alpha1api.SchemeGroupVersion.String(): velerov2alpha1api.CustomResourceKinds(), } for _, lst := range s.discoveryHelper.Resources() { if resources, found := gvResources[lst.GroupVersion]; found { for _, resource := range lst.APIResources { s.logger.WithField("kind", resource.Kind).Info("Found custom resource") resources.Delete(resource.Kind) } } } var errs []error for gv, resources := range gvResources { for kind := range resources { errs = append(errs, errors.Errorf("custom resource %s not found in Velero API group %s", kind, gv)) } } if len(errs) > 0 { errs = append(errs, errors.New("Velero custom resources not found - apply config/crd/v1/bases/*.yaml,config/crd/v2alpha1/bases*.yaml, to update the custom resource definitions")) return kubeerrs.NewAggregate(errs) } s.logger.Info("All Velero custom resource definitions exist") return nil } func (s *server) checkNodeAgent() { // warn if node agent does not exist if kube.WithLinuxNode(s.ctx, s.crClient, s.logger) { if err := nodeagent.IsRunningOnLinux(s.ctx, s.kubeClient, s.namespace); err == nodeagent.ErrDaemonSetNotFound { s.logger.Warn("Velero node agent not found for linux nodes; pod volume backups/restores and data mover backups/restores will not work until it's created") } else if err != nil { s.logger.WithError(errors.WithStack(err)).Warn("Error checking for existence of velero node agent for linux nodes") } } if kube.WithWindowsNode(s.ctx, s.crClient, s.logger) { if err := nodeagent.IsRunningOnWindows(s.ctx, s.kubeClient, s.namespace); err == nodeagent.ErrDaemonSetNotFound { s.logger.Warn("Velero node agent not found for Windows nodes; pod volume backups/restores and data mover backups/restores will not work until it's created") } else if err != nil { s.logger.WithError(errors.WithStack(err)).Warn("Error checking for existence of velero node agent for Windows nodes") } } } func (s *server) initRepoManager() error { // ensure the repo key secret is set up if err := repokey.EnsureCommonRepositoryKey(s.kubeClient.CoreV1(), s.namespace); err != nil { return err } s.repoLocker = repository.NewRepoLocker() s.repoEnsurer = repository.NewEnsurer(s.mgr.GetClient(), s.logger, s.config.ResourceTimeout) s.repoManager = repomanager.NewManager( s.namespace, s.mgr.GetClient(), s.repoLocker, s.credentialFileStore, s.credentialSecretStore, s.logger, ) return nil } func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string) error { s.logger.Info("Starting controllers") go func() { metricsMux := http.NewServeMux() metricsMux.Handle("/metrics", promhttp.Handler()) s.logger.Infof("Starting metric server at address [%s]", s.metricsAddress) server := &http.Server{ Addr: s.metricsAddress, Handler: metricsMux, ReadHeaderTimeout: 3 * time.Second, } if err := server.ListenAndServe(); err != nil { s.logger.Fatalf("Failed to start metric server at [%s]: %v", s.metricsAddress, err) } }() s.metrics = metrics.NewServerMetrics() s.metrics.RegisterAllMetrics() // Initialize manual backup metrics s.metrics.InitSchedule("") newPluginManager := func(logger logrus.FieldLogger) clientmgmt.Manager { return clientmgmt.NewManager(logger, s.logLevel, s.pluginRegistry) } backupStoreGetter := persistence.NewObjectBackupStoreGetterWithSecretStore(s.credentialFileStore, s.credentialSecretStore) backupTracker := controller.NewBackupTracker() // By far, PodVolumeBackup, PodVolumeRestore, BackupStorageLocation controllers // are not included in --disable-controllers list. // This is because of PVB and PVR are used by node agent DaemonSet, // and BSL controller is mandatory for Velero to work. // Note: all runtime type controllers that can be disabled are grouped separately, below: enabledRuntimeControllers := map[string]struct{}{ constant.ControllerBackup: {}, constant.ControllerBackupDeletion: {}, constant.ControllerBackupFinalizer: {}, constant.ControllerBackupOperations: {}, constant.ControllerBackupRepo: {}, constant.ControllerBackupSync: {}, constant.ControllerDownloadRequest: {}, constant.ControllerGarbageCollection: {}, constant.ControllerRestore: {}, constant.ControllerRestoreOperations: {}, constant.ControllerSchedule: {}, constant.ControllerServerStatusRequest: {}, constant.ControllerRestoreFinalizer: {}, constant.ControllerBackupQueue: {}, } if s.config.RestoreOnly { s.logger.Info("Restore only mode - not starting the backup, schedule, delete-backup, or GC controllers") s.config.DisabledControllers = append(s.config.DisabledControllers, constant.ControllerBackup, constant.ControllerBackupDeletion, constant.ControllerBackupFinalizer, constant.ControllerBackupOperations, constant.ControllerGarbageCollection, constant.ControllerSchedule, ) } // Remove disabled controllers so they are not initialized. If a match is not found we want // to halt the system so the user knows this operation was not possible. if err := removeControllers(s.config.DisabledControllers, enabledRuntimeControllers, s.logger); err != nil { log.Fatal(err, "unable to disable a controller") } // Enable BSL controller. No need to check whether it's enabled or not. bslr := controller.NewBackupStorageLocationReconciler( s.ctx, s.mgr.GetClient(), storage.DefaultBackupLocationInfo{ StorageLocation: s.config.DefaultBackupLocation, ServerValidationFrequency: s.config.StoreValidationFrequency, }, newPluginManager, backupStoreGetter, s.metrics, s.logger, ) if err := bslr.SetupWithManager(s.mgr); err != nil { s.logger.Fatal(err, "unable to create controller", "controller", constant.ControllerBackupStorageLocation) } pvbInformer, err := s.mgr.GetCache().GetInformer(s.ctx, &velerov1api.PodVolumeBackup{}) if err != nil { s.logger.Fatal(err, "fail to get controller-runtime informer from manager for PVB") } if _, ok := enabledRuntimeControllers[constant.ControllerBackup]; ok { backupper, err := backup.NewKubernetesBackupper( s.crClient, s.discoveryHelper, client.NewDynamicFactory(s.dynamicClient), podexec.NewPodCommandExecutor(s.kubeClientConfig, s.kubeClient.CoreV1().RESTClient()), podvolume.NewBackupperFactory( s.repoLocker, s.repoEnsurer, s.crClient, pvbInformer, s.logger, ), s.config.PodVolumeOperationTimeout, s.config.DefaultVolumesToFsBackup, s.config.ClientPageSize, s.config.UploaderType, newPluginManager, backupStoreGetter, ) cmd.CheckError(err) if err := controller.NewBackupReconciler( s.ctx, s.discoveryHelper, backupper, s.logger, s.logLevel, newPluginManager, backupTracker, s.mgr.GetClient(), s.config.DefaultBackupLocation, s.config.DefaultVolumesToFsBackup, s.config.DefaultBackupTTL, s.config.DefaultVGSLabelKey, s.config.DefaultCSISnapshotTimeout, s.config.ResourceTimeout, s.config.DefaultItemOperationTimeout, defaultVolumeSnapshotLocations, s.metrics, backupStoreGetter, s.config.LogFormat.Parse(), s.credentialFileStore, s.config.MaxConcurrentK8SConnections, s.config.DefaultSnapshotMoveData, s.config.ItemBlockWorkerCount, s.config.ConcurrentBackups, s.crClient, ).SetupWithManager(s.mgr); err != nil { s.logger.Fatal(err, "unable to create controller", "controller", constant.ControllerBackup) } } if _, ok := enabledRuntimeControllers[constant.ControllerBackupDeletion]; ok { if err := controller.NewBackupDeletionReconciler( s.logger, s.mgr.GetClient(), backupTracker, s.repoManager, s.metrics, s.discoveryHelper, newPluginManager, backupStoreGetter, s.credentialFileStore, s.repoEnsurer, ).SetupWithManager(s.mgr); err != nil { s.logger.Fatal(err, "unable to create controller", "controller", constant.ControllerBackupDeletion) } } backupOpsMap := itemoperationmap.NewBackupItemOperationsMap() if _, ok := enabledRuntimeControllers[constant.ControllerBackupOperations]; ok { r := controller.NewBackupOperationsReconciler( s.logger, s.mgr.GetClient(), s.config.ItemOperationSyncFrequency, newPluginManager, backupStoreGetter, s.metrics, backupOpsMap, ) if err := r.SetupWithManager(s.mgr); err != nil { s.logger.Fatal(err, "unable to create controller", "controller", constant.ControllerBackupOperations) } } if _, ok := enabledRuntimeControllers[constant.ControllerBackupFinalizer]; ok { backupper, err := backup.NewKubernetesBackupper( s.mgr.GetClient(), s.discoveryHelper, client.NewDynamicFactory(s.dynamicClient), podexec.NewPodCommandExecutor(s.kubeClientConfig, s.kubeClient.CoreV1().RESTClient()), podvolume.NewBackupperFactory( s.repoLocker, s.repoEnsurer, s.crClient, pvbInformer, s.logger, ), s.config.PodVolumeOperationTimeout, s.config.DefaultVolumesToFsBackup, s.config.ClientPageSize, s.config.UploaderType, newPluginManager, backupStoreGetter, ) cmd.CheckError(err) r := controller.NewBackupFinalizerReconciler( s.mgr.GetClient(), s.crClient, clock.RealClock{}, backupper, newPluginManager, backupTracker, backupStoreGetter, s.logger, s.metrics, s.config.ResourceTimeout, ) if err := r.SetupWithManager(s.mgr); err != nil { s.logger.Fatal(err, "unable to create controller", "controller", constant.ControllerBackupFinalizer) } } if _, ok := enabledRuntimeControllers[constant.ControllerBackupRepo]; ok { if err := controller.NewBackupRepoReconciler( s.namespace, s.logger, s.mgr.GetClient(), s.repoManager, s.config.RepoMaintenanceFrequency, s.config.BackupRepoConfig, s.config.RepoMaintenanceJobConfig, s.logLevel, s.config.LogFormat, s.metrics, ).SetupWithManager(s.mgr); err != nil { s.logger.Fatal(err, "unable to create controller", "controller", constant.ControllerBackupRepo) } } if _, ok := enabledRuntimeControllers[constant.ControllerBackupSync]; ok { syncPeriod := s.config.BackupSyncPeriod if syncPeriod <= 0 { syncPeriod = time.Minute } backupSyncReconciler := controller.NewBackupSyncReconciler( s.mgr.GetClient(), s.namespace, syncPeriod, newPluginManager, backupStoreGetter, s.logger, ) if err := backupSyncReconciler.SetupWithManager(s.mgr); err != nil { s.logger.Fatal(err, " unable to create controller ", "controller ", constant.ControllerBackupSync) } } restoreOpsMap := itemoperationmap.NewRestoreItemOperationsMap() if _, ok := enabledRuntimeControllers[constant.ControllerRestoreOperations]; ok { r := controller.NewRestoreOperationsReconciler( s.logger, s.namespace, s.mgr.GetClient(), s.config.ItemOperationSyncFrequency, newPluginManager, backupStoreGetter, s.metrics, restoreOpsMap, ) if err := r.SetupWithManager(s.mgr); err != nil { s.logger.Fatal(err, "unable to create controller", "controller", constant.ControllerRestoreOperations) } } if _, ok := enabledRuntimeControllers[constant.ControllerDownloadRequest]; ok { r := controller.NewDownloadRequestReconciler( s.mgr.GetClient(), clock.RealClock{}, newPluginManager, backupStoreGetter, s.logger, backupOpsMap, restoreOpsMap, ) if err := r.SetupWithManager(s.mgr); err != nil { s.logger.Fatal(err, "unable to create controller", "controller", constant.ControllerDownloadRequest) } } if _, ok := enabledRuntimeControllers[constant.ControllerGarbageCollection]; ok { r := controller.NewGCReconciler(s.logger, s.mgr.GetClient(), s.config.GarbageCollectionFrequency) if err := r.SetupWithManager(s.mgr); err != nil { s.logger.Fatal(err, "unable to create controller", "controller", constant.ControllerGarbageCollection) } } pvrInformer, err := s.mgr.GetCache().GetInformer(s.ctx, &velerov1api.PodVolumeRestore{}) if err != nil { s.logger.Fatal(err, "fail to get controller-runtime informer from manager for PVR") } multiHookTracker := hook.NewMultiHookTracker() if _, ok := enabledRuntimeControllers[constant.ControllerRestore]; ok { restorer, err := restore.NewKubernetesRestorer( s.discoveryHelper, client.NewDynamicFactory(s.dynamicClient), s.config.RestoreResourcePriorities, s.kubeClient.CoreV1().Namespaces(), podvolume.NewRestorerFactory( s.repoLocker, s.repoEnsurer, s.kubeClient, s.crClient, pvrInformer, s.logger, ), s.config.PodVolumeOperationTimeout, s.config.ResourceTerminatingTimeout, s.config.ResourceTimeout, s.logger, podexec.NewPodCommandExecutor(s.kubeClientConfig, s.kubeClient.CoreV1().RESTClient()), s.kubeClient.CoreV1().RESTClient(), s.credentialFileStore, s.mgr.GetClient(), multiHookTracker, ) cmd.CheckError(err) r := controller.NewRestoreReconciler( s.ctx, s.namespace, restorer, s.mgr.GetClient(), s.logger, s.logLevel, newPluginManager, backupStoreGetter, s.metrics, s.config.LogFormat.Parse(), s.config.DefaultItemOperationTimeout, s.config.DisableInformerCache, s.crClient, s.config.ResourceTimeout, ) if err = r.SetupWithManager(s.mgr); err != nil { s.logger.Fatal(err, "fail to create controller", "controller", constant.ControllerRestore) } } if _, ok := enabledRuntimeControllers[constant.ControllerSchedule]; ok { if err := controller.NewScheduleReconciler(s.namespace, s.logger, s.mgr.GetClient(), s.metrics, s.config.ScheduleSkipImmediately).SetupWithManager(s.mgr); err != nil { s.logger.Fatal(err, "unable to create controller", "controller", constant.ControllerSchedule) } } if _, ok := enabledRuntimeControllers[constant.ControllerServerStatusRequest]; ok { if err := controller.NewServerStatusRequestReconciler( s.ctx, s.mgr.GetClient(), s.pluginRegistry, clock.RealClock{}, s.logger, ).SetupWithManager(s.mgr); err != nil { s.logger.Fatal(err, "unable to create controller", "controller", constant.ControllerServerStatusRequest) } } if _, ok := enabledRuntimeControllers[constant.ControllerRestoreFinalizer]; ok { if err := controller.NewRestoreFinalizerReconciler( s.logger, s.namespace, s.mgr.GetClient(), newPluginManager, backupStoreGetter, s.metrics, s.crClient, multiHookTracker, s.config.ResourceTimeout, ).SetupWithManager(s.mgr); err != nil { s.logger.Fatal(err, "unable to create controller", "controller", constant.ControllerRestoreFinalizer) } } if _, ok := enabledRuntimeControllers[constant.ControllerBackupQueue]; ok { if err := controller.NewBackupQueueReconciler( s.mgr.GetClient(), s.mgr.GetScheme(), s.logger, s.config.ConcurrentBackups, backupTracker, ).SetupWithManager(s.mgr); err != nil { s.logger.Fatal(err, "unable to create controller", "controller", constant.ControllerBackupQueue) } } s.logger.Info("Server starting...") if err := s.mgr.Start(s.ctx); err != nil { s.logger.Fatal("Problem starting manager", err) } return nil } // removeControllers will remove any controller listed to be disabled from the list // of controllers to be initialized. It will check the runtime controllers. If a match // wasn't found and it returns an error. func removeControllers(disabledControllers []string, enabledRuntimeControllers map[string]struct{}, logger logrus.FieldLogger) error { for _, controllerName := range disabledControllers { if _, ok := enabledRuntimeControllers[controllerName]; ok { logger.Infof("Disabling controller: %s", controllerName) delete(enabledRuntimeControllers, controllerName) } else { msg := fmt.Sprintf("Invalid value for --disable-controllers flag provided: %s. Valid values are: %s", controllerName, strings.Join(config.DisableableControllers, ",")) return errors.New(msg) } } return nil } func (s *server) runProfiler() { mux := http.NewServeMux() mux.HandleFunc("/debug/pprof/", pprof.Index) mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) mux.HandleFunc("/debug/pprof/profile", pprof.Profile) mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) mux.HandleFunc("/debug/pprof/trace", pprof.Trace) server := &http.Server{ Addr: s.config.ProfilerAddress, Handler: mux, ReadHeaderTimeout: 3 * time.Second, } if err := server.ListenAndServe(); err != nil { s.logger.WithError(errors.WithStack(err)).Error("error running profiler http server") } } // if there is a restarting during the reconciling of backups/restores/etc, these CRs may be stuck in progress status // markInProgressCRsFailed tries to mark the in progress CRs as failed when starting the server to avoid the issue func markInProgressCRsFailed(ctx context.Context, client ctrlclient.Client, namespace string, log logrus.FieldLogger) { markInProgressBackupsFailed(ctx, client, namespace, log) markInProgressRestoresFailed(ctx, client, namespace, log) } func markInProgressBackupsFailed(ctx context.Context, client ctrlclient.Client, namespace string, log logrus.FieldLogger) { backups := &velerov1api.BackupList{} if err := client.List(ctx, backups, &ctrlclient.ListOptions{Namespace: namespace}); err != nil { log.WithError(errors.WithStack(err)).Error("failed to list backups") return } for i, backup := range backups.Items { if backup.Status.Phase != velerov1api.BackupPhaseInProgress { log.Debugf("the status of backup %q is %q, skip", backup.GetName(), backup.Status.Phase) continue } updated := backup.DeepCopy() updated.Status.Phase = velerov1api.BackupPhaseFailed updated.Status.FailureReason = fmt.Sprintf("found a backup with status %q during the server starting, mark it as %q", backup.Status.Phase, updated.Status.Phase) updated.Status.CompletionTimestamp = &metav1.Time{Time: time.Now()} if err := client.Patch(ctx, updated, ctrlclient.MergeFrom(&backups.Items[i])); err != nil { log.WithError(errors.WithStack(err)).Errorf("failed to patch backup %q", backup.GetName()) continue } log.WithField("backup", backup.GetName()).Warn(updated.Status.FailureReason) markDataUploadsCancel(ctx, client, backup, log) markPodVolumeBackupsCancel(ctx, client, backup, log) } } func markInProgressRestoresFailed(ctx context.Context, client ctrlclient.Client, namespace string, log logrus.FieldLogger) { restores := &velerov1api.RestoreList{} if err := client.List(ctx, restores, &ctrlclient.ListOptions{Namespace: namespace}); err != nil { log.WithError(errors.WithStack(err)).Error("failed to list restores") return } for i, restore := range restores.Items { if restore.Status.Phase != velerov1api.RestorePhaseInProgress { log.Debugf("the status of restore %q is %q, skip", restore.GetName(), restore.Status.Phase) continue } updated := restore.DeepCopy() updated.Status.Phase = velerov1api.RestorePhaseFailed updated.Status.FailureReason = fmt.Sprintf("found a restore with status %q during the server starting, mark it as %q", restore.Status.Phase, updated.Status.Phase) updated.Status.CompletionTimestamp = &metav1.Time{Time: time.Now()} if err := client.Patch(ctx, updated, ctrlclient.MergeFrom(&restores.Items[i])); err != nil { log.WithError(errors.WithStack(err)).Errorf("failed to patch restore %q", restore.GetName()) continue } log.WithField("restore", restore.GetName()).Warn(updated.Status.FailureReason) markDataDownloadsCancel(ctx, client, restore, log) markPodVolumeRestoresCancel(ctx, client, restore, log) } } func markDataUploadsCancel(ctx context.Context, client ctrlclient.Client, backup velerov1api.Backup, log logrus.FieldLogger) { dataUploads := &velerov2alpha1api.DataUploadList{} if err := client.List(ctx, dataUploads, &ctrlclient.ListOptions{ Namespace: backup.GetNamespace(), LabelSelector: labels.Set(map[string]string{ velerov1api.BackupUIDLabel: string(backup.GetUID()), }).AsSelector(), }); err != nil { log.WithError(errors.WithStack(err)).Error("failed to list dataUploads") return } for i := range dataUploads.Items { du := dataUploads.Items[i] if du.Status.Phase == velerov2alpha1api.DataUploadPhaseAccepted || du.Status.Phase == velerov2alpha1api.DataUploadPhasePrepared || du.Status.Phase == velerov2alpha1api.DataUploadPhaseInProgress || du.Status.Phase == velerov2alpha1api.DataUploadPhaseNew || du.Status.Phase == "" { err := controller.UpdateDataUploadWithRetry(ctx, client, types.NamespacedName{Namespace: du.Namespace, Name: du.Name}, log.WithField("dataupload", du.Name), func(dataUpload *velerov2alpha1api.DataUpload) bool { if dataUpload.Spec.Cancel { return false } dataUpload.Spec.Cancel = true dataUpload.Status.Message = fmt.Sprintf("Dataupload is in status %q during the velero server starting, mark it as cancel", du.Status.Phase) return true }) if err != nil { log.WithError(errors.WithStack(err)).Errorf("failed to mark dataupload %q cancel", du.GetName()) continue } log.WithField("dataupload", du.GetName()).Warn(du.Status.Message) } } } func markDataDownloadsCancel(ctx context.Context, client ctrlclient.Client, restore velerov1api.Restore, log logrus.FieldLogger) { dataDownloads := &velerov2alpha1api.DataDownloadList{} if err := client.List(ctx, dataDownloads, &ctrlclient.ListOptions{ Namespace: restore.GetNamespace(), LabelSelector: labels.Set(map[string]string{ velerov1api.RestoreUIDLabel: string(restore.GetUID()), }).AsSelector(), }); err != nil { log.WithError(errors.WithStack(err)).Error("failed to list dataDownloads") return } for i := range dataDownloads.Items { dd := dataDownloads.Items[i] if dd.Status.Phase == velerov2alpha1api.DataDownloadPhaseAccepted || dd.Status.Phase == velerov2alpha1api.DataDownloadPhasePrepared || dd.Status.Phase == velerov2alpha1api.DataDownloadPhaseInProgress || dd.Status.Phase == velerov2alpha1api.DataDownloadPhaseNew || dd.Status.Phase == "" { err := controller.UpdateDataDownloadWithRetry(ctx, client, types.NamespacedName{Namespace: dd.Namespace, Name: dd.Name}, log.WithField("datadownload", dd.Name), func(dataDownload *velerov2alpha1api.DataDownload) bool { if dataDownload.Spec.Cancel { return false } dataDownload.Spec.Cancel = true dataDownload.Status.Message = fmt.Sprintf("Datadownload is in status %q during the velero server starting, mark it as cancel", dd.Status.Phase) return true }) if err != nil { log.WithError(errors.WithStack(err)).Errorf("failed to mark datadownload %q cancel", dd.GetName()) continue } log.WithField("datadownload", dd.GetName()).Warn(dd.Status.Message) } } } func markPodVolumeBackupsCancel(ctx context.Context, client ctrlclient.Client, backup velerov1api.Backup, log logrus.FieldLogger) { pvbs := &velerov1api.PodVolumeBackupList{} if err := client.List(ctx, pvbs, &ctrlclient.ListOptions{ Namespace: backup.GetNamespace(), LabelSelector: labels.Set(map[string]string{ velerov1api.BackupUIDLabel: string(backup.GetUID()), }).AsSelector(), }); err != nil { log.WithError(errors.WithStack(err)).Error("failed to list PVBs") return } for i := range pvbs.Items { pvb := pvbs.Items[i] if pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseAccepted || pvb.Status.Phase == velerov1api.PodVolumeBackupPhasePrepared || pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseInProgress || pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseNew || pvb.Status.Phase == "" { err := controller.UpdatePVBWithRetry(ctx, client, types.NamespacedName{Namespace: pvb.Namespace, Name: pvb.Name}, log.WithField("PVB", pvb.Name), func(pvb *velerov1api.PodVolumeBackup) bool { if pvb.Spec.Cancel { return false } pvb.Spec.Cancel = true pvb.Status.Message = fmt.Sprintf("PVB is in status %q during the velero server starting, mark it as cancel", pvb.Status.Phase) return true }) if err != nil { log.WithError(errors.WithStack(err)).Errorf("failed to mark PVB %q cancel", pvb.GetName()) continue } log.WithField("PVB is mark for cancel due to server restart", pvb.GetName()).Warn(pvb.Status.Message) } } } func markPodVolumeRestoresCancel(ctx context.Context, client ctrlclient.Client, restore velerov1api.Restore, log logrus.FieldLogger) { pvrs := &velerov1api.PodVolumeRestoreList{} if err := client.List(ctx, pvrs, &ctrlclient.ListOptions{ Namespace: restore.GetNamespace(), LabelSelector: labels.Set(map[string]string{ velerov1api.RestoreUIDLabel: string(restore.GetUID()), }).AsSelector(), }); err != nil { log.WithError(errors.WithStack(err)).Error("failed to list PVRs") return } for i := range pvrs.Items { pvr := pvrs.Items[i] if controller.IsLegacyPVR(&pvr) { log.WithField("PVR", pvr.GetName()).Warn("Found a legacy PVR during velero server restart, cannot stop it") continue } if pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseAccepted || pvr.Status.Phase == velerov1api.PodVolumeRestorePhasePrepared || pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseInProgress || pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseNew || pvr.Status.Phase == "" { err := controller.UpdatePVRWithRetry(ctx, client, types.NamespacedName{Namespace: pvr.Namespace, Name: pvr.Name}, log.WithField("PVR", pvr.Name), func(pvr *velerov1api.PodVolumeRestore) bool { if pvr.Spec.Cancel { return false } pvr.Spec.Cancel = true pvr.Status.Message = fmt.Sprintf("PVR is in status %q during the velero server starting, mark it as cancel", pvr.Status.Phase) return true }) if err != nil { log.WithError(errors.WithStack(err)).Errorf("failed to mark PVR %q cancel", pvr.GetName()) continue } log.WithField("PVR is mark for cancel due to server restart", pvr.GetName()).Warn(pvr.Status.Message) } } } ================================================ FILE: pkg/cmd/server/server_test.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package server import ( "errors" "testing" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" kubefake "k8s.io/client-go/kubernetes/fake" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/client/mocks" "github.com/vmware-tanzu/velero/pkg/cmd/server/config" "github.com/vmware-tanzu/velero/pkg/constant" discovery_mocks "github.com/vmware-tanzu/velero/pkg/discovery/mocks" velerotest "github.com/vmware-tanzu/velero/pkg/test" "github.com/vmware-tanzu/velero/pkg/uploader" ) func TestVeleroResourcesExist(t *testing.T) { var ( fakeDiscoveryHelper = &velerotest.FakeDiscoveryHelper{} server = &server{ logger: velerotest.NewLogger(), discoveryHelper: fakeDiscoveryHelper, } ) // Velero API group doesn't exist in discovery: should error fakeDiscoveryHelper.ResourceList = []*metav1.APIResourceList{ { GroupVersion: "foo/v1", APIResources: []metav1.APIResource{ { Name: "Backups", Kind: "Backup", }, }, }, } require.Error(t, server.veleroResourcesExist()) // Velero v1 API group doesn't contain any custom resources: should error veleroAPIResourceListVelerov1 := &metav1.APIResourceList{ GroupVersion: velerov1api.SchemeGroupVersion.String(), } fakeDiscoveryHelper.ResourceList = append(fakeDiscoveryHelper.ResourceList, veleroAPIResourceListVelerov1) require.Error(t, server.veleroResourcesExist()) // Velero v2alpha1 API group doesn't contain any custom resources: should error veleroAPIResourceListVeleroV2alpha1 := &metav1.APIResourceList{ GroupVersion: velerov2alpha1api.SchemeGroupVersion.String(), } fakeDiscoveryHelper.ResourceList = append(fakeDiscoveryHelper.ResourceList, veleroAPIResourceListVeleroV2alpha1) require.Error(t, server.veleroResourcesExist()) // Velero v1 API group contains all custom resources, but v2alpha1 doesn't contain any custom resources: should error for kind := range velerov1api.CustomResources() { veleroAPIResourceListVelerov1.APIResources = append(veleroAPIResourceListVelerov1.APIResources, metav1.APIResource{ Kind: kind, }) } require.Error(t, server.veleroResourcesExist()) // Velero v1 and v2alpha1 API group contain all custom resources: should not error for kind := range velerov2alpha1api.CustomResources() { veleroAPIResourceListVeleroV2alpha1.APIResources = append(veleroAPIResourceListVeleroV2alpha1.APIResources, metav1.APIResource{ Kind: kind, }) } require.NoError(t, server.veleroResourcesExist()) // Velero API group contains some but not all custom resources: should error veleroAPIResourceListVelerov1.APIResources = veleroAPIResourceListVelerov1.APIResources[:3] assert.Error(t, server.veleroResourcesExist()) } func TestRemoveControllers(t *testing.T) { logger := velerotest.NewLogger() tests := []struct { name string disabledControllers []string errorExpected bool }{ { name: "Remove one disable controller", disabledControllers: []string{ constant.ControllerBackup, }, errorExpected: false, }, { name: "Remove all disable controllers", disabledControllers: []string{ constant.ControllerBackupOperations, constant.ControllerBackup, constant.ControllerBackupDeletion, constant.ControllerBackupSync, constant.ControllerDownloadRequest, constant.ControllerGarbageCollection, constant.ControllerBackupRepo, constant.ControllerRestore, constant.ControllerSchedule, constant.ControllerServerStatusRequest, }, errorExpected: false, }, { name: "Remove with a non-disable controller included", disabledControllers: []string{ constant.ControllerBackup, constant.ControllerBackupStorageLocation, }, errorExpected: true, }, { name: "Remove with a misspelled/non-existing controller name", disabledControllers: []string{ "go", }, errorExpected: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { enabledRuntimeControllers := map[string]struct{}{ constant.ControllerBackupSync: {}, constant.ControllerBackup: {}, constant.ControllerGarbageCollection: {}, constant.ControllerRestore: {}, constant.ControllerServerStatusRequest: {}, constant.ControllerSchedule: {}, constant.ControllerBackupDeletion: {}, constant.ControllerBackupRepo: {}, constant.ControllerDownloadRequest: {}, constant.ControllerBackupOperations: {}, } totalNumOriginalControllers := len(enabledRuntimeControllers) if tt.errorExpected { assert.Error(t, removeControllers(tt.disabledControllers, enabledRuntimeControllers, logger)) } else { require.NoError(t, removeControllers(tt.disabledControllers, enabledRuntimeControllers, logger)) totalNumEnabledControllers := len(enabledRuntimeControllers) assert.Equal(t, totalNumEnabledControllers, totalNumOriginalControllers-len(tt.disabledControllers)) for _, disabled := range tt.disabledControllers { _, ok := enabledRuntimeControllers[disabled] assert.False(t, ok) } } }) } } func TestNewCommand(t *testing.T) { assert.NotNil(t, NewCommand(nil)) } func Test_newServer(t *testing.T) { factory := &mocks.Factory{} logger := logrus.New() // invalid uploader type _, err := newServer(factory, &config.Config{ UploaderType: "invalid", }, logger) require.Error(t, err) // invalid clientQPS _, err = newServer(factory, &config.Config{ UploaderType: uploader.KopiaType, ClientQPS: -1, }, logger) require.Error(t, err) // invalid clientQPS Restic uploader _, err = newServer(factory, &config.Config{ UploaderType: uploader.ResticType, ClientQPS: -1, }, logger) require.Error(t, err) // invalid clientBurst factory.On("SetClientQPS", mock.Anything).Return() _, err = newServer(factory, &config.Config{ UploaderType: uploader.KopiaType, ClientQPS: 1, ClientBurst: -1, }, logger) require.Error(t, err) // invalid clientBclientPageSizeurst factory.On("SetClientQPS", mock.Anything).Return(). On("SetClientBurst", mock.Anything).Return() _, err = newServer(factory, &config.Config{ UploaderType: uploader.KopiaType, ClientQPS: 1, ClientBurst: 1, ClientPageSize: -1, }, logger) require.Error(t, err) // got error when creating client factory.On("SetClientQPS", mock.Anything).Return(). On("SetClientBurst", mock.Anything).Return(). On("KubeClient").Return(nil, nil). On("Client").Return(nil, nil). On("DynamicClient").Return(nil, errors.New("error")) _, err = newServer(factory, &config.Config{ UploaderType: uploader.KopiaType, ClientQPS: 1, ClientBurst: 1, ClientPageSize: 100, }, logger) require.Error(t, err) invalidCM := builder.ForConfigMap("velero", "invalid").Data("invalid", "{\"a\": \"b}").Result() crClient := velerotest.NewFakeControllerRuntimeClient(t, invalidCM) factory.On("KubeClient").Return(crClient, nil). On("Client").Return(nil, nil). On("DynamicClient").Return(nil, errors.New("error")) _, err = newServer(factory, &config.Config{ UploaderType: uploader.KopiaType, BackupRepoConfig: "invalid", }, logger) require.Error(t, err) factory.On("KubeClient").Return(crClient, nil). On("Client").Return(nil, nil). On("DynamicClient").Return(nil, errors.New("error")) _, err = newServer(factory, &config.Config{ UploaderType: uploader.KopiaType, RepoMaintenanceJobConfig: "invalid", }, logger) require.Error(t, err) } func Test_namespaceExists(t *testing.T) { client := kubefake.NewSimpleClientset(&corev1api.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: "velero", }, }) server := &server{ kubeClient: client, logger: logrus.New(), } // namespace doesn't exist require.Error(t, server.namespaceExists("not-exist")) // namespace exists assert.NoError(t, server.namespaceExists("velero")) } func Test_veleroResourcesExist(t *testing.T) { helper := &discovery_mocks.Helper{} server := &server{ discoveryHelper: helper, logger: logrus.New(), } // velero resources don't exist helper.On("Resources").Return(nil) require.Error(t, server.veleroResourcesExist()) // velero resources exist helper.On("Resources").Unset() helper.On("Resources").Return([]*metav1.APIResourceList{ { GroupVersion: velerov1api.SchemeGroupVersion.String(), APIResources: []metav1.APIResource{ {Kind: "Backup"}, {Kind: "Restore"}, {Kind: "Schedule"}, {Kind: "DownloadRequest"}, {Kind: "DeleteBackupRequest"}, {Kind: "PodVolumeBackup"}, {Kind: "PodVolumeRestore"}, {Kind: "BackupRepository"}, {Kind: "BackupStorageLocation"}, {Kind: "VolumeSnapshotLocation"}, {Kind: "ServerStatusRequest"}, }, }, { GroupVersion: velerov2alpha1api.SchemeGroupVersion.String(), APIResources: []metav1.APIResource{ {Kind: "DataUpload"}, {Kind: "DataDownload"}, }, }, }) assert.NoError(t, server.veleroResourcesExist()) } func Test_markInProgressBackupsFailed(t *testing.T) { scheme := runtime.NewScheme() velerov1api.AddToScheme(scheme) c := fake.NewClientBuilder(). WithScheme(scheme). WithLists(&velerov1api.BackupList{ Items: []velerov1api.Backup{ { ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", Name: "backup01", }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseInProgress, }, }, { ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", Name: "backup02", }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseCompleted, }, }, }, }). Build() markInProgressBackupsFailed(t.Context(), c, "velero", logrus.New()) backup01 := &velerov1api.Backup{} require.NoError(t, c.Get(t.Context(), client.ObjectKey{Namespace: "velero", Name: "backup01"}, backup01)) assert.Equal(t, velerov1api.BackupPhaseFailed, backup01.Status.Phase) backup02 := &velerov1api.Backup{} require.NoError(t, c.Get(t.Context(), client.ObjectKey{Namespace: "velero", Name: "backup02"}, backup02)) assert.Equal(t, velerov1api.BackupPhaseCompleted, backup02.Status.Phase) } func Test_markInProgressRestoresFailed(t *testing.T) { scheme := runtime.NewScheme() velerov1api.AddToScheme(scheme) c := fake.NewClientBuilder(). WithScheme(scheme). WithLists(&velerov1api.RestoreList{ Items: []velerov1api.Restore{ { ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", Name: "restore01", }, Status: velerov1api.RestoreStatus{ Phase: velerov1api.RestorePhaseInProgress, }, }, { ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", Name: "restore02", }, Status: velerov1api.RestoreStatus{ Phase: velerov1api.RestorePhaseCompleted, }, }, }, }). Build() markInProgressRestoresFailed(t.Context(), c, "velero", logrus.New()) restore01 := &velerov1api.Restore{} require.NoError(t, c.Get(t.Context(), client.ObjectKey{Namespace: "velero", Name: "restore01"}, restore01)) assert.Equal(t, velerov1api.RestorePhaseFailed, restore01.Status.Phase) restore02 := &velerov1api.Restore{} require.NoError(t, c.Get(t.Context(), client.ObjectKey{Namespace: "velero", Name: "restore02"}, restore02)) assert.Equal(t, velerov1api.RestorePhaseCompleted, restore02.Status.Phase) } func Test_setDefaultBackupLocation(t *testing.T) { scheme := runtime.NewScheme() velerov1api.AddToScheme(scheme) c := fake.NewClientBuilder(). WithScheme(scheme). WithLists(&velerov1api.BackupStorageLocationList{ Items: []velerov1api.BackupStorageLocation{ { ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", Name: "default", }, }, { ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", Name: "non-default", }, }, }, }). Build() setDefaultBackupLocation(t.Context(), c, "velero", "default", logrus.New()) defaultLocation := &velerov1api.BackupStorageLocation{} require.NoError(t, c.Get(t.Context(), client.ObjectKey{Namespace: "velero", Name: "default"}, defaultLocation)) assert.True(t, defaultLocation.Spec.Default) nonDefaultLocation := &velerov1api.BackupStorageLocation{} require.NoError(t, c.Get(t.Context(), client.ObjectKey{Namespace: "velero", Name: "non-default"}, nonDefaultLocation)) assert.False(t, nonDefaultLocation.Spec.Default) // no default location specified c = fake.NewClientBuilder().WithScheme(scheme).Build() err := setDefaultBackupLocation(t.Context(), c, "velero", "", logrus.New()) require.NoError(t, err) // no default location created err = setDefaultBackupLocation(t.Context(), c, "velero", "default", logrus.New()) assert.NoError(t, err) } ================================================ FILE: pkg/cmd/test/const.go ================================================ package test var VeleroNameSpace = "velero-test" var CaptureFlag = "CAPTRUE-OUTPUT" ================================================ FILE: pkg/cmd/util/cacert/bsl_cacert.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cacert import ( "context" "github.com/pkg/errors" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" kbclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) // GetCACertFromBackup fetches the BackupStorageLocation for a backup and returns its cacert func GetCACertFromBackup(ctx context.Context, client kbclient.Client, namespace string, backup *velerov1api.Backup) (string, error) { return GetCACertFromBSL(ctx, client, namespace, backup.Spec.StorageLocation) } // GetCACertFromRestore fetches the BackupStorageLocation for a restore's backup and returns its cacert func GetCACertFromRestore(ctx context.Context, client kbclient.Client, namespace string, restore *velerov1api.Restore) (string, error) { // First get the backup that this restore references backup := &velerov1api.Backup{} key := kbclient.ObjectKey{ Namespace: namespace, Name: restore.Spec.BackupName, } if err := client.Get(ctx, key, backup); err != nil { if apierrors.IsNotFound(err) { // Backup not found is not a fatal error for cacert retrieval return "", nil } return "", errors.Wrapf(err, "error getting backup %s", restore.Spec.BackupName) } return GetCACertFromBackup(ctx, client, namespace, backup) } // GetCACertFromBSL fetches a BackupStorageLocation directly and returns its cacert // Priority order: caCertRef (from Secret) > caCert (inline, deprecated) func GetCACertFromBSL(ctx context.Context, client kbclient.Client, namespace, bslName string) (string, error) { if bslName == "" { return "", nil } bsl := &velerov1api.BackupStorageLocation{} key := kbclient.ObjectKey{ Namespace: namespace, Name: bslName, } if err := client.Get(ctx, key, bsl); err != nil { if apierrors.IsNotFound(err) { // BSL not found is not a fatal error, just means no cacert return "", nil } return "", errors.Wrapf(err, "error getting backup storage location %s", bslName) } if bsl.Spec.ObjectStorage == nil { return "", nil } // Prefer caCertRef over inline caCert if bsl.Spec.ObjectStorage.CACertRef != nil { // Fetch certificate from Secret secret := &corev1api.Secret{} secretKey := types.NamespacedName{ Name: bsl.Spec.ObjectStorage.CACertRef.Name, Namespace: namespace, } if err := client.Get(ctx, secretKey, secret); err != nil { if apierrors.IsNotFound(err) { return "", errors.Errorf("certificate secret %s not found in namespace %s", bsl.Spec.ObjectStorage.CACertRef.Name, namespace) } return "", errors.Wrapf(err, "error getting certificate secret %s", bsl.Spec.ObjectStorage.CACertRef.Name) } keyName := bsl.Spec.ObjectStorage.CACertRef.Key if keyName == "" { return "", errors.New("caCertRef key is empty") } certData, ok := secret.Data[keyName] if !ok { return "", errors.Errorf("key %s not found in secret %s", keyName, bsl.Spec.ObjectStorage.CACertRef.Name) } return string(certData), nil } // Fall back to inline caCert (deprecated) if len(bsl.Spec.ObjectStorage.CACert) > 0 { return string(bsl.Spec.ObjectStorage.CACert), nil } return "", nil } ================================================ FILE: pkg/cmd/util/cacert/bsl_cacert_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cacert import ( "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client/fake" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/util" ) func TestGetCACertFromBackup(t *testing.T) { testCases := []struct { name string backup *velerov1api.Backup bsl *velerov1api.BackupStorageLocation expectedCACert string expectedError bool }{ { name: "backup with BSL containing cacert", backup: builder.ForBackup("test-ns", "test-backup"). StorageLocation("test-bsl"). Result(), bsl: builder.ForBackupStorageLocation("test-ns", "test-bsl"). Provider("aws"). Bucket("test-bucket"). CACert([]byte("test-cacert-content")). Result(), expectedCACert: "test-cacert-content", expectedError: false, }, { name: "backup with BSL without cacert", backup: builder.ForBackup("test-ns", "test-backup"). StorageLocation("test-bsl"). Result(), bsl: builder.ForBackupStorageLocation("test-ns", "test-bsl"). Provider("aws"). Bucket("test-bucket"). Result(), expectedCACert: "", expectedError: false, }, { name: "backup without storage location", backup: builder.ForBackup("test-ns", "test-backup"). Result(), bsl: nil, expectedCACert: "", expectedError: false, }, { name: "BSL not found", backup: builder.ForBackup("test-ns", "test-backup"). StorageLocation("missing-bsl"). Result(), bsl: nil, expectedCACert: "", expectedError: false, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { var objs []runtime.Object objs = append(objs, tc.backup) if tc.bsl != nil { objs = append(objs, tc.bsl) } fakeClient := fake.NewClientBuilder(). WithScheme(util.VeleroScheme). WithRuntimeObjects(objs...). Build() cacert, err := GetCACertFromBackup(t.Context(), fakeClient, "test-ns", tc.backup) if tc.expectedError { assert.Error(t, err) } else { require.NoError(t, err) assert.Equal(t, tc.expectedCACert, cacert) } }) } } func TestGetCACertFromRestore(t *testing.T) { testCases := []struct { name string restore *velerov1api.Restore backup *velerov1api.Backup bsl *velerov1api.BackupStorageLocation expectedCACert string expectedError bool }{ { name: "restore with backup having BSL containing cacert", restore: builder.ForRestore("test-ns", "test-restore"). Backup("test-backup"). Result(), backup: builder.ForBackup("test-ns", "test-backup"). StorageLocation("test-bsl"). Result(), bsl: builder.ForBackupStorageLocation("test-ns", "test-bsl"). Provider("aws"). Bucket("test-bucket"). CACert([]byte("test-cacert-content")). Result(), expectedCACert: "test-cacert-content", expectedError: false, }, { name: "restore with backup not found", restore: builder.ForRestore("test-ns", "test-restore"). Backup("missing-backup"). Result(), backup: nil, bsl: nil, expectedCACert: "", expectedError: false, }, { name: "restore with backup having BSL without cacert", restore: builder.ForRestore("test-ns", "test-restore"). Backup("test-backup"). Result(), backup: builder.ForBackup("test-ns", "test-backup"). StorageLocation("test-bsl"). Result(), bsl: builder.ForBackupStorageLocation("test-ns", "test-bsl"). Provider("aws"). Bucket("test-bucket"). Result(), expectedCACert: "", expectedError: false, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { var objs []runtime.Object objs = append(objs, tc.restore) if tc.backup != nil { objs = append(objs, tc.backup) } if tc.bsl != nil { objs = append(objs, tc.bsl) } fakeClient := fake.NewClientBuilder(). WithScheme(util.VeleroScheme). WithRuntimeObjects(objs...). Build() cacert, err := GetCACertFromRestore(t.Context(), fakeClient, "test-ns", tc.restore) if tc.expectedError { assert.Error(t, err) } else { require.NoError(t, err) assert.Equal(t, tc.expectedCACert, cacert) } }) } } func TestGetCACertFromBSL(t *testing.T) { testCases := []struct { name string bslName string bsl *velerov1api.BackupStorageLocation expectedCACert string expectedError bool }{ { name: "BSL with cacert", bslName: "test-bsl", bsl: builder.ForBackupStorageLocation("test-ns", "test-bsl"). Provider("aws"). Bucket("test-bucket"). CACert([]byte("test-cacert-content")). Result(), expectedCACert: "test-cacert-content", expectedError: false, }, { name: "BSL without cacert", bslName: "test-bsl", bsl: builder.ForBackupStorageLocation("test-ns", "test-bsl"). Provider("aws"). Bucket("test-bucket"). Result(), expectedCACert: "", expectedError: false, }, { name: "empty BSL name", bslName: "", bsl: nil, expectedCACert: "", expectedError: false, }, { name: "BSL not found", bslName: "missing-bsl", bsl: nil, expectedCACert: "", expectedError: false, }, { name: "BSL with invalid CA cert format", bslName: "test-bsl", bsl: builder.ForBackupStorageLocation("test-ns", "test-bsl"). Provider("aws"). Bucket("test-bucket"). CACert([]byte("INVALID CERT DATA WITHOUT PEM HEADERS")). Result(), expectedCACert: "INVALID CERT DATA WITHOUT PEM HEADERS", // We still return it, validation happens during TLS handshake expectedError: false, }, { name: "BSL with malformed PEM certificate", bslName: "test-bsl", bsl: builder.ForBackupStorageLocation("test-ns", "test-bsl"). Provider("aws"). Bucket("test-bucket"). CACert([]byte("-----BEGIN CERTIFICATE-----\nINVALID BASE64 DATA!!!\n-----END CERTIFICATE-----\n")). Result(), expectedCACert: "-----BEGIN CERTIFICATE-----\nINVALID BASE64 DATA!!!\n-----END CERTIFICATE-----\n", expectedError: false, }, { name: "BSL with nil config", bslName: "test-bsl", bsl: &velerov1api.BackupStorageLocation{ ObjectMeta: metav1.ObjectMeta{ Namespace: "test-ns", Name: "test-bsl", }, Spec: velerov1api.BackupStorageLocationSpec{ Provider: "aws", Config: nil, }, }, expectedCACert: "", expectedError: false, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { var objs []runtime.Object if tc.bsl != nil { objs = append(objs, tc.bsl) } fakeClient := fake.NewClientBuilder(). WithScheme(util.VeleroScheme). WithRuntimeObjects(objs...). Build() cacert, err := GetCACertFromBSL(t.Context(), fakeClient, "test-ns", tc.bslName) if tc.expectedError { assert.Error(t, err) } else { require.NoError(t, err) assert.Equal(t, tc.expectedCACert, cacert) } }) } } // TestGetCACertFromBSL_WithCACertRef tests the new caCertRef functionality func TestGetCACertFromBSL_WithCACertRef(t *testing.T) { testCases := []struct { name string bslName string bsl *velerov1api.BackupStorageLocation secret *corev1api.Secret expectedCACert string expectedError bool errorContains string }{ { name: "BSL with caCertRef pointing to valid secret", bslName: "test-bsl", bsl: &velerov1api.BackupStorageLocation{ ObjectMeta: metav1.ObjectMeta{ Namespace: "test-ns", Name: "test-bsl", }, Spec: velerov1api.BackupStorageLocationSpec{ Provider: "aws", StorageType: velerov1api.StorageType{ ObjectStorage: &velerov1api.ObjectStorageLocation{ Bucket: "test-bucket", CACertRef: &corev1api.SecretKeySelector{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "test-secret", }, Key: "ca-bundle.crt", }, }, }, }, }, secret: &corev1api.Secret{ ObjectMeta: metav1.ObjectMeta{ Namespace: "test-ns", Name: "test-secret", }, Data: map[string][]byte{ "ca-bundle.crt": []byte("test-cacert-from-secret"), }, }, expectedCACert: "test-cacert-from-secret", expectedError: false, }, { name: "BSL with both caCertRef and caCert - caCertRef takes precedence", bslName: "test-bsl", bsl: &velerov1api.BackupStorageLocation{ ObjectMeta: metav1.ObjectMeta{ Namespace: "test-ns", Name: "test-bsl", }, Spec: velerov1api.BackupStorageLocationSpec{ Provider: "aws", StorageType: velerov1api.StorageType{ ObjectStorage: &velerov1api.ObjectStorageLocation{ Bucket: "test-bucket", CACert: []byte("inline-cacert-deprecated"), CACertRef: &corev1api.SecretKeySelector{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "test-secret", }, Key: "ca-bundle.crt", }, }, }, }, }, secret: &corev1api.Secret{ ObjectMeta: metav1.ObjectMeta{ Namespace: "test-ns", Name: "test-secret", }, Data: map[string][]byte{ "ca-bundle.crt": []byte("cacert-from-secret-takes-precedence"), }, }, expectedCACert: "cacert-from-secret-takes-precedence", expectedError: false, }, { name: "BSL with caCertRef but secret not found", bslName: "test-bsl", bsl: &velerov1api.BackupStorageLocation{ ObjectMeta: metav1.ObjectMeta{ Namespace: "test-ns", Name: "test-bsl", }, Spec: velerov1api.BackupStorageLocationSpec{ Provider: "aws", StorageType: velerov1api.StorageType{ ObjectStorage: &velerov1api.ObjectStorageLocation{ Bucket: "test-bucket", CACertRef: &corev1api.SecretKeySelector{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "missing-secret", }, Key: "ca-bundle.crt", }, }, }, }, }, secret: nil, expectedCACert: "", expectedError: true, errorContains: "certificate secret missing-secret not found", }, { name: "BSL with caCertRef but key not found in secret", bslName: "test-bsl", bsl: &velerov1api.BackupStorageLocation{ ObjectMeta: metav1.ObjectMeta{ Namespace: "test-ns", Name: "test-bsl", }, Spec: velerov1api.BackupStorageLocationSpec{ Provider: "aws", StorageType: velerov1api.StorageType{ ObjectStorage: &velerov1api.ObjectStorageLocation{ Bucket: "test-bucket", CACertRef: &corev1api.SecretKeySelector{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "test-secret", }, Key: "missing-key", }, }, }, }, }, secret: &corev1api.Secret{ ObjectMeta: metav1.ObjectMeta{ Namespace: "test-ns", Name: "test-secret", }, Data: map[string][]byte{ "ca-bundle.crt": []byte("test-cacert"), }, }, expectedCACert: "", expectedError: true, errorContains: "key missing-key not found in secret test-secret", }, { name: "BSL with caCertRef but empty key", bslName: "test-bsl", bsl: &velerov1api.BackupStorageLocation{ ObjectMeta: metav1.ObjectMeta{ Namespace: "test-ns", Name: "test-bsl", }, Spec: velerov1api.BackupStorageLocationSpec{ Provider: "aws", StorageType: velerov1api.StorageType{ ObjectStorage: &velerov1api.ObjectStorageLocation{ Bucket: "test-bucket", CACertRef: &corev1api.SecretKeySelector{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "test-secret", }, Key: "", }, }, }, }, }, secret: &corev1api.Secret{ ObjectMeta: metav1.ObjectMeta{ Namespace: "test-ns", Name: "test-secret", }, Data: map[string][]byte{ "ca-bundle.crt": []byte("test-cacert"), }, }, expectedCACert: "", expectedError: true, errorContains: "caCertRef key is empty", }, { name: "BSL with caCertRef containing multi-line PEM certificate", bslName: "test-bsl", bsl: &velerov1api.BackupStorageLocation{ ObjectMeta: metav1.ObjectMeta{ Namespace: "test-ns", Name: "test-bsl", }, Spec: velerov1api.BackupStorageLocationSpec{ Provider: "aws", StorageType: velerov1api.StorageType{ ObjectStorage: &velerov1api.ObjectStorageLocation{ Bucket: "test-bucket", CACertRef: &corev1api.SecretKeySelector{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "test-secret", }, Key: "ca.pem", }, }, }, }, }, secret: &corev1api.Secret{ ObjectMeta: metav1.ObjectMeta{ Namespace: "test-ns", Name: "test-secret", }, Data: map[string][]byte{ "ca.pem": []byte("-----BEGIN CERTIFICATE-----\nMIIDETC...\n-----END CERTIFICATE-----\n"), }, }, expectedCACert: "-----BEGIN CERTIFICATE-----\nMIIDETC...\n-----END CERTIFICATE-----\n", expectedError: false, }, { name: "BSL falls back to inline caCert when caCertRef is nil", bslName: "test-bsl", bsl: builder.ForBackupStorageLocation("test-ns", "test-bsl"). Provider("aws"). Bucket("test-bucket"). CACert([]byte("fallback-inline-cacert")). Result(), secret: nil, expectedCACert: "fallback-inline-cacert", expectedError: false, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { var objs []runtime.Object if tc.bsl != nil { objs = append(objs, tc.bsl) } if tc.secret != nil { objs = append(objs, tc.secret) } scheme := runtime.NewScheme() _ = velerov1api.AddToScheme(scheme) _ = corev1api.AddToScheme(scheme) fakeClient := fake.NewClientBuilder(). WithScheme(scheme). WithRuntimeObjects(objs...). Build() cacert, err := GetCACertFromBSL(t.Context(), fakeClient, "test-ns", tc.bslName) if tc.expectedError { require.Error(t, err) if tc.errorContains != "" { assert.Contains(t, err.Error(), tc.errorContains) } } else { require.NoError(t, err) assert.Equal(t, tc.expectedCACert, cacert) } }) } } // TestGetCACertFromBackup_ClientError tests error scenarios where client.Get returns non-NotFound errors func TestGetCACertFromBackup_ClientError(t *testing.T) { testCases := []struct { name string backup *velerov1api.Backup bsl *velerov1api.BackupStorageLocation expectedError string }{ { name: "client error getting BSL", backup: builder.ForBackup("test-ns", "test-backup"). StorageLocation("test-bsl"). Result(), bsl: builder.ForBackupStorageLocation("different-ns", "test-bsl"). // Different namespace to trigger error Provider("aws"). Bucket("test-bucket"). CACert([]byte("test-cacert-content")). Result(), expectedError: "not found", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { var objs []runtime.Object objs = append(objs, tc.backup) if tc.bsl != nil { objs = append(objs, tc.bsl) } fakeClient := fake.NewClientBuilder(). WithScheme(util.VeleroScheme). WithRuntimeObjects(objs...). Build() // Try to get BSL from wrong namespace to simulate error _, err := GetCACertFromBSL(t.Context(), fakeClient, "wrong-ns", tc.backup.Spec.StorageLocation) require.NoError(t, err) // Not found errors are handled gracefully }) } } // TestGetCACertFromRestore_ClientError tests error scenarios for GetCACertFromRestore func TestGetCACertFromRestore_ClientError(t *testing.T) { testCases := []struct { name string restore *velerov1api.Restore backup *velerov1api.Backup expectedError string }{ { name: "backup in different namespace", restore: builder.ForRestore("test-ns", "test-restore"). Backup("test-backup"). Result(), backup: builder.ForBackup("different-ns", "test-backup"). // Different namespace StorageLocation("test-bsl"). Result(), expectedError: "not found", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { var objs []runtime.Object objs = append(objs, tc.restore) if tc.backup != nil { objs = append(objs, tc.backup) } fakeClient := fake.NewClientBuilder(). WithScheme(util.VeleroScheme). WithRuntimeObjects(objs...). Build() // This should not find the backup in the wrong namespace cacert, err := GetCACertFromRestore(t.Context(), fakeClient, "test-ns", tc.restore) require.NoError(t, err) // Not found errors are handled gracefully, returning empty string assert.Empty(t, cacert) }) } } ================================================ FILE: pkg/cmd/util/confirm/confirm.go ================================================ package confirm import ( "bufio" "fmt" "os" "strings" "github.com/spf13/pflag" ) type ConfirmOptions struct { Confirm bool flagDescription string } func NewConfirmOptions() *ConfirmOptions { return &ConfirmOptions{flagDescription: "Confirm action"} } func NewConfirmOptionsWithDescription(desc string) *ConfirmOptions { return &ConfirmOptions{flagDescription: desc} } // Bind confirm flags. func (o *ConfirmOptions) BindFlags(flags *pflag.FlagSet) { flags.BoolVar(&o.Confirm, "confirm", o.Confirm, o.flagDescription) } // GetConfirmation ensures that the user confirms the action before proceeding. func GetConfirmation(prompts ...string) bool { reader := bufio.NewReader(os.Stdin) for { for i := range prompts { fmt.Println(prompts[i]) } fmt.Printf("Are you sure you want to continue (Y/N)? ") confirmation, err := reader.ReadString('\n') if err != nil { fmt.Fprintf(os.Stderr, "error reading user input: %v\n", err) return false } confirmation = strings.TrimSpace(confirmation) if len(confirmation) != 1 { continue } switch strings.ToLower(confirmation) { case "y": return true case "n": return false } } } ================================================ FILE: pkg/cmd/util/downloadrequest/downloadrequest.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package downloadrequest import ( "compress/gzip" "context" "crypto/tls" "crypto/x509" "fmt" "io" "net/http" "net/url" "os" "time" "github.com/google/uuid" "github.com/pkg/errors" kbclient "sigs.k8s.io/controller-runtime/pkg/client" veleroV1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" ) // ErrNotFound is exported for external packages to check for when a file is // not found var ErrNotFound = errors.New("file not found") var ErrDownloadRequestDownloadURLTimeout = errors.New("download request download url timeout, check velero server logs for errors. backup storage location may not be available") func Stream( ctx context.Context, kbClient kbclient.Client, namespace, name string, kind veleroV1api.DownloadTargetKind, w io.Writer, timeout time.Duration, insecureSkipTLSVerify bool, caCertFile string, ) error { return StreamWithBSLCACert(ctx, kbClient, namespace, name, kind, w, timeout, insecureSkipTLSVerify, caCertFile, "") } // StreamWithBSLCACert is like Stream but accepts an additional bslCACert parameter // that contains the cacert from the BackupStorageLocation config func StreamWithBSLCACert( ctx context.Context, kbClient kbclient.Client, namespace, name string, kind veleroV1api.DownloadTargetKind, w io.Writer, timeout time.Duration, insecureSkipTLSVerify bool, caCertFile string, bslCACert string, ) error { ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() downloadURL, err := getDownloadURL(ctx, kbClient, namespace, name, kind) if err != nil { return err } if err := download(ctx, downloadURL, kind, w, insecureSkipTLSVerify, caCertFile, bslCACert); err != nil { return err } return nil } func getDownloadURL( ctx context.Context, kbClient kbclient.Client, namespace, name string, kind veleroV1api.DownloadTargetKind, ) (string, error) { uuid, err := uuid.NewRandom() if err != nil { return "", err } reqName := fmt.Sprintf("%s-%s", name, uuid.String()) created := builder.ForDownloadRequest(namespace, reqName).Target(kind, name).Result() if err := kbClient.Create(ctx, created, &kbclient.CreateOptions{}); err != nil { return "", errors.WithStack(err) } for { select { case <-ctx.Done(): return "", ErrDownloadRequestDownloadURLTimeout case <-time.After(25 * time.Millisecond): updated := &veleroV1api.DownloadRequest{} if err := kbClient.Get(ctx, kbclient.ObjectKey{Name: created.Name, Namespace: namespace}, updated); err != nil { return "", errors.WithStack(err) } if updated.Status.DownloadURL != "" { return updated.Status.DownloadURL, nil } } } } func download( ctx context.Context, downloadURL string, kind veleroV1api.DownloadTargetKind, w io.Writer, insecureSkipTLSVerify bool, caCertFile string, caCertByteString string, ) error { var caPool *x509.CertPool var err error // Initialize caPool once caPool, err = x509.SystemCertPool() if err != nil { caPool = x509.NewCertPool() } // Try to load CA cert from file first if len(caCertFile) > 0 { caCert, err := os.ReadFile(caCertFile) if err != nil { // If caCertFile fails and BSL cert is available, fall back to it if len(caCertByteString) > 0 { fmt.Fprintf(os.Stderr, "Warning: Failed to open CA certificate file %s: %v. Using CA certificate from backup storage location instead.\n", caCertFile, err) caPool.AppendCertsFromPEM([]byte(caCertByteString)) } else { // If no BSL cert available, return the original error return errors.Wrapf(err, "couldn't open cacert") } } else { caPool.AppendCertsFromPEM(caCert) } } else if len(caCertByteString) > 0 { // If no caCertFile specified, use BSL cert if available caPool.AppendCertsFromPEM([]byte(caCertByteString)) } defaultTransport := http.DefaultTransport.(*http.Transport) // same settings as the default transport // aside from TLSClientConfig httpClient := new(http.Client) httpClient.Transport = &http.Transport{ TLSClientConfig: &tls.Config{ InsecureSkipVerify: insecureSkipTLSVerify, //nolint:gosec // This parameter is useful for some scenarios. RootCAs: caPool, }, DialContext: defaultTransport.DialContext, ForceAttemptHTTP2: defaultTransport.ForceAttemptHTTP2, MaxIdleConns: defaultTransport.MaxIdleConns, Proxy: defaultTransport.Proxy, TLSHandshakeTimeout: defaultTransport.TLSHandshakeTimeout, ExpectContinueTimeout: defaultTransport.ExpectContinueTimeout, } httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil) if err != nil { return err } resp, err := httpClient.Do(httpReq) if err != nil { if urlErr, ok := err.(*url.Error); ok { if _, ok := urlErr.Err.(x509.UnknownAuthorityError); ok { return fmt.Errorf("%s\n\nThe --insecure-skip-tls-verify flag can also be used to accept any TLS certificate for the download, but it is susceptible to man-in-the-middle attacks", err.Error()) } } return err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { body, err := io.ReadAll(resp.Body) if err != nil { return errors.Wrapf(err, "request failed: unable to decode response body") } if resp.StatusCode == http.StatusNotFound { return ErrNotFound } return errors.Errorf("request failed: %v", string(body)) } reader := resp.Body if kind != veleroV1api.DownloadTargetKindBackupContents { // need to decompress logs gzipReader, err := gzip.NewReader(resp.Body) if err != nil { return err } defer gzipReader.Close() reader = gzipReader } _, err = io.Copy(w, reader) return err } ================================================ FILE: pkg/cmd/util/downloadrequest/downloadrequest_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package downloadrequest import ( "bytes" "compress/gzip" "crypto/rand" "crypto/rsa" "crypto/tls" "crypto/x509" "crypto/x509/pkix" "encoding/pem" "math/big" "net" "net/http" "net/http/httptest" "os" "strings" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" kbclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/cmd/util/cacert" "github.com/vmware-tanzu/velero/pkg/util" ) // createSelfSignedCertificate creates a self-signed certificate for testing. // This allows us to test the BSL CA certificate functionality by ensuring // that the client properly validates server certificates against the CA cert // provided in the BackupStorageLocation configuration. func createSelfSignedCertificate(t *testing.T) (tls.Certificate, []byte) { t.Helper() // Generate a private key priv, err := rsa.GenerateKey(rand.Reader, 2048) require.NoError(t, err) // Create certificate template for a self-signed certificate template := x509.Certificate{ SerialNumber: big.NewInt(1), Subject: pkix.Name{ Organization: []string{"Test Org"}, Country: []string{"US"}, Province: []string{""}, Locality: []string{"Test City"}, StreetAddress: []string{""}, PostalCode: []string{""}, }, NotBefore: time.Now(), NotAfter: time.Now().Add(365 * 24 * time.Hour), KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, BasicConstraintsValid: true, IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1)}, DNSNames: []string{"localhost"}, } // Create the certificate certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv) require.NoError(t, err) // Encode certificate and key to PEM certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER}) keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)}) // Create tls.Certificate tlsCert, err := tls.X509KeyPair(certPEM, keyPEM) require.NoError(t, err) return tlsCert, certPEM } func TestStream(t *testing.T) { // Create a test server that returns download content testContent := "test backup content" downloadServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) if strings.Contains(r.URL.Path, "log") { // For logs, return gzipped content gzipWriter := gzip.NewWriter(w) gzipWriter.Write([]byte(testContent)) gzipWriter.Close() } else { w.Write([]byte(testContent)) } })) defer downloadServer.Close() testCases := []struct { name string target velerov1api.DownloadTargetKind timeout time.Duration setupClient func(*testing.T, kbclient.WithWatch) expectedError bool expectedErrMessage string validateContent func(*testing.T, *bytes.Buffer) }{ { name: "successful backup log download", target: velerov1api.DownloadTargetKindBackupLog, timeout: 5 * time.Second, setupClient: func(t *testing.T, client kbclient.WithWatch) { t.Helper() // Simulate controller updating the DownloadRequest with URL go func() { time.Sleep(10 * time.Millisecond) list := &velerov1api.DownloadRequestList{} err := client.List(t.Context(), list) assert.NoError(t, err) for _, dr := range list.Items { dr.Status.DownloadURL = downloadServer.URL + "/log" dr.Status.Phase = velerov1api.DownloadRequestPhaseProcessed err := client.Update(t.Context(), &dr) assert.NoError(t, err) } }() }, expectedError: false, validateContent: func(t *testing.T, buf *bytes.Buffer) { t.Helper() // Logs should be decompressed assert.Equal(t, testContent, buf.String()) }, }, { name: "successful backup contents download", target: velerov1api.DownloadTargetKindBackupContents, timeout: 5 * time.Second, setupClient: func(t *testing.T, client kbclient.WithWatch) { t.Helper() // Simulate controller updating the DownloadRequest with URL go func() { time.Sleep(10 * time.Millisecond) list := &velerov1api.DownloadRequestList{} err := client.List(t.Context(), list) assert.NoError(t, err) for _, dr := range list.Items { dr.Status.DownloadURL = downloadServer.URL + "/contents" dr.Status.Phase = velerov1api.DownloadRequestPhaseProcessed err := client.Update(t.Context(), &dr) assert.NoError(t, err) } }() }, expectedError: false, validateContent: func(t *testing.T, buf *bytes.Buffer) { t.Helper() assert.Equal(t, testContent, buf.String()) }, }, { name: "timeout waiting for download URL", target: velerov1api.DownloadTargetKindBackupLog, timeout: 50 * time.Millisecond, setupClient: func(t *testing.T, client kbclient.WithWatch) { t.Helper() }, expectedError: true, expectedErrMessage: "download request download url timeout", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { fakeClient := fake.NewClientBuilder(). WithScheme(util.VeleroScheme). Build() if tc.setupClient != nil { tc.setupClient(t, fakeClient) } var buf bytes.Buffer ctx := t.Context() err := Stream(ctx, fakeClient, "test-ns", "test-backup", tc.target, &buf, tc.timeout, false, "") if tc.expectedError { require.Error(t, err) if tc.expectedErrMessage != "" { assert.Contains(t, err.Error(), tc.expectedErrMessage) } } else { require.NoError(t, err) if tc.validateContent != nil { tc.validateContent(t, &buf) } } }) } } func TestStreamWithBSLCACert(t *testing.T) { // Create a test server that returns download content testContent := "test backup content with BSL CA cert" // Create self-signed certificate for TLS testing tlsCert, serverCACertPEM := createSelfSignedCertificate(t) // Create TLS test server for testing CA certificate functionality tlsServer := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) if strings.Contains(r.URL.Path, "log") { // For logs, return gzipped content gzipWriter := gzip.NewWriter(w) gzipWriter.Write([]byte(testContent)) gzipWriter.Close() } else { w.Write([]byte(testContent)) } })) tlsServer.TLS = &tls.Config{ Certificates: []tls.Certificate{tlsCert}, } tlsServer.StartTLS() defer tlsServer.Close() // Also create a regular HTTP server for non-TLS tests httpServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) if strings.Contains(r.URL.Path, "log") { // For logs, return gzipped content gzipWriter := gzip.NewWriter(w) gzipWriter.Write([]byte(testContent)) gzipWriter.Close() } else { w.Write([]byte(testContent)) } })) defer httpServer.Close() testCases := []struct { name string target velerov1api.DownloadTargetKind bslCACert string timeout time.Duration setupClient func(*testing.T, kbclient.WithWatch) useTLS bool expectedError bool expectedErrMessage string validateContent func(*testing.T, *bytes.Buffer) }{ { name: "successful TLS backup log download with correct BSL CA cert", target: velerov1api.DownloadTargetKindBackupLog, bslCACert: string(serverCACertPEM), timeout: 5 * time.Second, useTLS: true, setupClient: func(t *testing.T, client kbclient.WithWatch) { t.Helper() // Simulate controller updating the DownloadRequest with URL go func() { time.Sleep(10 * time.Millisecond) list := &velerov1api.DownloadRequestList{} err := client.List(t.Context(), list) assert.NoError(t, err) for _, dr := range list.Items { dr.Status.DownloadURL = tlsServer.URL + "/log" dr.Status.Phase = velerov1api.DownloadRequestPhaseProcessed err := client.Update(t.Context(), &dr) assert.NoError(t, err) } }() }, expectedError: false, validateContent: func(t *testing.T, buf *bytes.Buffer) { t.Helper() // Logs should be decompressed assert.Equal(t, testContent, buf.String()) }, }, { name: "successful TLS backup contents download with correct BSL CA cert", target: velerov1api.DownloadTargetKindBackupContents, bslCACert: string(serverCACertPEM), timeout: 5 * time.Second, useTLS: true, setupClient: func(t *testing.T, client kbclient.WithWatch) { t.Helper() // Simulate controller updating the DownloadRequest with URL go func() { time.Sleep(10 * time.Millisecond) list := &velerov1api.DownloadRequestList{} err := client.List(t.Context(), list) assert.NoError(t, err) for _, dr := range list.Items { dr.Status.DownloadURL = tlsServer.URL + "/contents" dr.Status.Phase = velerov1api.DownloadRequestPhaseProcessed err := client.Update(t.Context(), &dr) assert.NoError(t, err) } }() }, expectedError: false, validateContent: func(t *testing.T, buf *bytes.Buffer) { t.Helper() assert.Equal(t, testContent, buf.String()) }, }, { name: "failed TLS download with wrong BSL CA cert", target: velerov1api.DownloadTargetKindBackupContents, bslCACert: "-----BEGIN CERTIFICATE-----\nMIIBkTCB+wIJAKHHIgKwERfFMA0GCSqGSIb3DQEBCwUAMBkxFzAVBgNVBAMTDmRp\nZmZlcmVudC1jZXJ0MB4XDTE5MDQwMTAwMDAwMFoXDTI5MDQwMTAwMDAwMFowGTEX\nMBUGA1UEAxMOZGlmZmVyZW50LWNlcnQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ\nAoGBAOO4V+XrhVEGbTqnO2FM5eVFaM3KMKc3M9/C1aeg3vvY+Th3OhqJBxEYFxXL\nZoSqkwL/E6BjQb0NdSyJY9wdM4Ie3gElcZBKYVpHXYYAVhrepRCRVJEIHdBN8ybr\nFoBBDjd/ID1qy8Gdp3RihPFNvCNx0RWWqPAJtNXWJvCiNRCDAgMBAAEwDQYJKoZI\nhvcNAQELBQADgYEAGEwwGz7HAmH0J3pAJzQKPCb8HJG8hTjD6qkMon3Bp6gZ\n-----END CERTIFICATE-----\n", timeout: 5 * time.Second, useTLS: true, setupClient: func(t *testing.T, client kbclient.WithWatch) { t.Helper() // Simulate controller updating the DownloadRequest with URL go func() { time.Sleep(10 * time.Millisecond) list := &velerov1api.DownloadRequestList{} err := client.List(t.Context(), list) assert.NoError(t, err) for _, dr := range list.Items { dr.Status.DownloadURL = tlsServer.URL + "/contents" dr.Status.Phase = velerov1api.DownloadRequestPhaseProcessed err := client.Update(t.Context(), &dr) assert.NoError(t, err) } }() }, expectedError: true, expectedErrMessage: "x509", }, { name: "successful HTTP download with empty BSL CA cert", target: velerov1api.DownloadTargetKindBackupContents, bslCACert: "", timeout: 5 * time.Second, useTLS: false, setupClient: func(t *testing.T, client kbclient.WithWatch) { t.Helper() // Simulate controller updating the DownloadRequest with URL go func() { time.Sleep(10 * time.Millisecond) list := &velerov1api.DownloadRequestList{} err := client.List(t.Context(), list) assert.NoError(t, err) for _, dr := range list.Items { dr.Status.DownloadURL = httpServer.URL + "/contents" dr.Status.Phase = velerov1api.DownloadRequestPhaseProcessed err := client.Update(t.Context(), &dr) assert.NoError(t, err) } }() }, expectedError: false, validateContent: func(t *testing.T, buf *bytes.Buffer) { t.Helper() assert.Equal(t, testContent, buf.String()) }, }, { name: "timeout waiting for download URL with BSL CA cert", target: velerov1api.DownloadTargetKindBackupLog, bslCACert: "test-ca-cert-content", timeout: 50 * time.Millisecond, setupClient: func(t *testing.T, client kbclient.WithWatch) { t.Helper() }, expectedError: true, expectedErrMessage: "download request download url timeout", }, { name: "failed TLS download with malformed BSL CA cert", target: velerov1api.DownloadTargetKindBackupLog, bslCACert: "-----BEGIN CERTIFICATE-----\nINVALID CERT DATA\n-----END CERTIFICATE-----\n", timeout: 5 * time.Second, useTLS: true, setupClient: func(t *testing.T, client kbclient.WithWatch) { t.Helper() // Simulate controller updating the DownloadRequest with URL go func() { time.Sleep(10 * time.Millisecond) list := &velerov1api.DownloadRequestList{} err := client.List(t.Context(), list) assert.NoError(t, err) for _, dr := range list.Items { dr.Status.DownloadURL = tlsServer.URL + "/log" dr.Status.Phase = velerov1api.DownloadRequestPhaseProcessed err := client.Update(t.Context(), &dr) assert.NoError(t, err) } }() }, expectedError: true, expectedErrMessage: "x509", // Should fail due to malformed cert not being added to pool }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { fakeClient := fake.NewClientBuilder(). WithScheme(util.VeleroScheme). Build() if tc.setupClient != nil { tc.setupClient(t, fakeClient) } var buf bytes.Buffer ctx := t.Context() err := StreamWithBSLCACert(ctx, fakeClient, "test-ns", "test-backup", tc.target, &buf, tc.timeout, false, "", tc.bslCACert) if tc.expectedError { require.Error(t, err) if tc.expectedErrMessage != "" { assert.Contains(t, err.Error(), tc.expectedErrMessage) } } else { require.NoError(t, err) if tc.validateContent != nil { tc.validateContent(t, &buf) } } }) } } func TestDownload(t *testing.T) { testContent := "test content for download" compressedContent := new(bytes.Buffer) gzipWriter := gzip.NewWriter(compressedContent) gzipWriter.Write([]byte(testContent)) gzipWriter.Close() testCases := []struct { name string serverHandler http.HandlerFunc target velerov1api.DownloadTargetKind insecureSkipTLSVerify bool caCertFile string bslCACert string expectedContent string expectedError bool errorType error }{ { name: "successful download with gzip for logs", serverHandler: func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) w.Write(compressedContent.Bytes()) }, target: velerov1api.DownloadTargetKindBackupLog, expectedContent: testContent, expectedError: false, }, { name: "successful download without gzip for backup contents", serverHandler: func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) w.Write([]byte(testContent)) }, target: velerov1api.DownloadTargetKindBackupContents, expectedContent: testContent, expectedError: false, }, { name: "404 not found error", serverHandler: func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNotFound) }, target: velerov1api.DownloadTargetKindBackupLog, expectedError: true, errorType: ErrNotFound, }, { name: "500 internal server error", serverHandler: func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusInternalServerError) w.Write([]byte("internal server error")) }, target: velerov1api.DownloadTargetKindBackupLog, expectedError: true, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { server := httptest.NewServer(tc.serverHandler) defer server.Close() var buf bytes.Buffer err := download( t.Context(), server.URL, tc.target, &buf, tc.insecureSkipTLSVerify, tc.caCertFile, tc.bslCACert, ) if tc.expectedError { require.Error(t, err) if tc.errorType != nil { assert.Equal(t, tc.errorType, err) } } else { require.NoError(t, err) assert.Equal(t, tc.expectedContent, buf.String()) } }) } } // TestStreamWithBSLCACertEndToEnd tests the complete flow from BSL to download with CA cert func TestStreamWithBSLCACertEndToEnd(t *testing.T) { testContent := "end-to-end test content" // Create self-signed certificate for TLS testing tlsCert, serverCACertPEM := createSelfSignedCertificate(t) // Create TLS test server tlsServer := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) if strings.Contains(r.URL.Path, "log") { // For logs, return gzipped content gzipWriter := gzip.NewWriter(w) gzipWriter.Write([]byte(testContent)) gzipWriter.Close() } else { w.Write([]byte(testContent)) } })) tlsServer.TLS = &tls.Config{ Certificates: []tls.Certificate{tlsCert}, } tlsServer.StartTLS() defer tlsServer.Close() // Create BSL with CA cert bsl := builder.ForBackupStorageLocation("test-ns", "test-bsl"). Provider("aws"). Bucket("test-bucket"). CACert(serverCACertPEM). Result() // Create backup that references the BSL backup := builder.ForBackup("test-ns", "test-backup"). StorageLocation("test-bsl"). Result() // Setup fake client with BSL and backup fakeClient := fake.NewClientBuilder(). WithScheme(util.VeleroScheme). WithRuntimeObjects(bsl, backup). Build() // Helper function to simulate controller updating the DownloadRequest simulateControllerUpdate := func() { go func() { time.Sleep(10 * time.Millisecond) list := &velerov1api.DownloadRequestList{} err := fakeClient.List(t.Context(), list) assert.NoError(t, err) for _, dr := range list.Items { dr.Status.DownloadURL = tlsServer.URL + "/log" dr.Status.Phase = velerov1api.DownloadRequestPhaseProcessed err := fakeClient.Update(t.Context(), &dr) assert.NoError(t, err) } }() } // Test the complete flow ctx := t.Context() // First, try to download WITHOUT the CA cert - this should fail simulateControllerUpdate() var bufFail bytes.Buffer err := StreamWithBSLCACert(ctx, fakeClient, "test-ns", "test-backup", velerov1api.DownloadTargetKindBackupLog, &bufFail, 5*time.Second, false, "", "") require.Error(t, err) assert.Contains(t, err.Error(), "x509") // Should fail with certificate validation error // Now get CA cert from BSL through backup cacertStr, err := cacert.GetCACertFromBackup(ctx, fakeClient, "test-ns", backup) require.NoError(t, err) require.Equal(t, string(serverCACertPEM), cacertStr) // Try again with the CA cert - this should succeed simulateControllerUpdate() var bufSuccess bytes.Buffer err = StreamWithBSLCACert(ctx, fakeClient, "test-ns", "test-backup", velerov1api.DownloadTargetKindBackupLog, &bufSuccess, 5*time.Second, false, "", cacertStr) require.NoError(t, err) // Verify content was downloaded and decompressed correctly assert.Equal(t, testContent, bufSuccess.String()) } // TestBackwardCompatibilityWithoutBSLCACert tests that old download requests work without BSL CA cert func TestBackwardCompatibilityWithoutBSLCACert(t *testing.T) { testContent := "backward compatibility test content" // Create HTTP (not HTTPS) server to simulate old behavior where TLS wasn't required httpServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) if strings.Contains(r.URL.Path, "log") { // For logs, return gzipped content gzipWriter := gzip.NewWriter(w) gzipWriter.Write([]byte(testContent)) gzipWriter.Close() } else { w.Write([]byte(testContent)) } })) defer httpServer.Close() // Create BSL without CA cert (simulating old configuration) bsl := builder.ForBackupStorageLocation("test-ns", "test-bsl"). Provider("aws"). Bucket("test-bucket"). // No CACert() call - simulating pre-CA cert support Result() // Create backup that references the BSL backup := builder.ForBackup("test-ns", "test-backup"). StorageLocation("test-bsl"). Result() // Setup fake client with BSL and backup fakeClient := fake.NewClientBuilder(). WithScheme(util.VeleroScheme). WithRuntimeObjects(bsl, backup). Build() // Simulate controller updating the DownloadRequest with HTTP URL (old behavior) go func() { time.Sleep(10 * time.Millisecond) list := &velerov1api.DownloadRequestList{} err := fakeClient.List(t.Context(), list) assert.NoError(t, err) for _, dr := range list.Items { dr.Status.DownloadURL = httpServer.URL + "/log" dr.Status.Phase = velerov1api.DownloadRequestPhaseProcessed err := fakeClient.Update(t.Context(), &dr) assert.NoError(t, err) } }() ctx := t.Context() // Test 1: Stream function (without BSL CA cert parameter) should work var buf1 bytes.Buffer err := Stream(ctx, fakeClient, "test-ns", "test-backup", velerov1api.DownloadTargetKindBackupLog, &buf1, 5*time.Second, false, "") require.NoError(t, err) assert.Equal(t, testContent, buf1.String()) // Test 2: StreamWithBSLCACert with empty BSL CA cert should also work go func() { time.Sleep(10 * time.Millisecond) list := &velerov1api.DownloadRequestList{} err := fakeClient.List(t.Context(), list) assert.NoError(t, err) for _, dr := range list.Items { dr.Status.DownloadURL = httpServer.URL + "/contents" dr.Status.Phase = velerov1api.DownloadRequestPhaseProcessed err := fakeClient.Update(t.Context(), &dr) assert.NoError(t, err) } }() var buf2 bytes.Buffer err = StreamWithBSLCACert(ctx, fakeClient, "test-ns", "test-backup", velerov1api.DownloadTargetKindBackupContents, &buf2, 5*time.Second, false, "", "") require.NoError(t, err) assert.Equal(t, testContent, buf2.String()) // Test 3: Getting CA cert from BSL should return empty string (not error) cacert, err := cacert.GetCACertFromBackup(ctx, fakeClient, "test-ns", backup) require.NoError(t, err) assert.Empty(t, cacert) } // TestMixedEnvironmentHTTPAndHTTPS tests environment with both HTTP and HTTPS endpoints func TestMixedEnvironmentHTTPAndHTTPS(t *testing.T) { testContentHTTP := "http content" testContentHTTPS := "https content" // Create HTTP server httpServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) w.Write([]byte(testContentHTTP)) })) defer httpServer.Close() // Create HTTPS server with self-signed cert tlsCert, serverCACertPEM := createSelfSignedCertificate(t) httpsServer := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) w.Write([]byte(testContentHTTPS)) })) httpsServer.TLS = &tls.Config{ Certificates: []tls.Certificate{tlsCert}, } httpsServer.StartTLS() defer httpsServer.Close() // Create two BSLs - one for HTTP (old) and one for HTTPS (new) bslHTTP := builder.ForBackupStorageLocation("test-ns", "bsl-http"). Provider("aws"). Bucket("http-bucket"). // No CA cert for HTTP Result() bslHTTPS := builder.ForBackupStorageLocation("test-ns", "bsl-https"). Provider("aws"). Bucket("https-bucket"). CACert(serverCACertPEM). Result() // Create backups for each BSL backupHTTP := builder.ForBackup("test-ns", "backup-http"). StorageLocation("bsl-http"). Result() backupHTTPS := builder.ForBackup("test-ns", "backup-https"). StorageLocation("bsl-https"). Result() // Setup fake client fakeClient := fake.NewClientBuilder(). WithScheme(util.VeleroScheme). WithRuntimeObjects(bslHTTP, bslHTTPS, backupHTTP, backupHTTPS). Build() ctx := t.Context() // Test HTTP backup download (backward compatible) go func() { time.Sleep(10 * time.Millisecond) list := &velerov1api.DownloadRequestList{} err := fakeClient.List(t.Context(), list) assert.NoError(t, err) for _, dr := range list.Items { if strings.Contains(dr.Name, "backup-http") { dr.Status.DownloadURL = httpServer.URL dr.Status.Phase = velerov1api.DownloadRequestPhaseProcessed err := fakeClient.Update(t.Context(), &dr) assert.NoError(t, err) } } }() var bufHTTP bytes.Buffer err := Stream(ctx, fakeClient, "test-ns", "backup-http", velerov1api.DownloadTargetKindBackupContents, &bufHTTP, 5*time.Second, false, "") require.NoError(t, err) assert.Equal(t, testContentHTTP, bufHTTP.String()) // Test HTTPS backup download (requires CA cert) go func() { time.Sleep(10 * time.Millisecond) list := &velerov1api.DownloadRequestList{} err := fakeClient.List(t.Context(), list) assert.NoError(t, err) for _, dr := range list.Items { if strings.Contains(dr.Name, "backup-https") { dr.Status.DownloadURL = httpsServer.URL dr.Status.Phase = velerov1api.DownloadRequestPhaseProcessed err := fakeClient.Update(t.Context(), &dr) assert.NoError(t, err) } } }() // First try without CA cert - should fail var bufHTTPSFail bytes.Buffer err = Stream(ctx, fakeClient, "test-ns", "backup-https", velerov1api.DownloadTargetKindBackupContents, &bufHTTPSFail, 5*time.Second, false, "") require.Error(t, err) assert.Contains(t, err.Error(), "x509") // Get CA cert from HTTPS BSL cacertStr, err := cacert.GetCACertFromBackup(ctx, fakeClient, "test-ns", backupHTTPS) require.NoError(t, err) require.Equal(t, string(serverCACertPEM), cacertStr) // Try again with CA cert - should succeed go func() { time.Sleep(10 * time.Millisecond) list := &velerov1api.DownloadRequestList{} err := fakeClient.List(t.Context(), list) assert.NoError(t, err) for _, dr := range list.Items { if strings.Contains(dr.Name, "backup-https") { dr.Status.DownloadURL = httpsServer.URL dr.Status.Phase = velerov1api.DownloadRequestPhaseProcessed err := fakeClient.Update(t.Context(), &dr) assert.NoError(t, err) } } }() var bufHTTPSSuccess bytes.Buffer err = StreamWithBSLCACert(ctx, fakeClient, "test-ns", "backup-https", velerov1api.DownloadTargetKindBackupContents, &bufHTTPSSuccess, 5*time.Second, false, "", cacertStr) require.NoError(t, err) assert.Equal(t, testContentHTTPS, bufHTTPSSuccess.String()) } // TestBSLUpgradeScenario tests the scenario where a BSL is upgraded to include CA cert func TestBSLUpgradeScenario(t *testing.T) { testContent := "bsl upgrade test content" // Create self-signed certificate for TLS testing tlsCert, serverCACertPEM := createSelfSignedCertificate(t) // Create HTTPS server httpsServer := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) if strings.Contains(r.URL.Path, "log") { // For logs, return gzipped content gzipWriter := gzip.NewWriter(w) gzipWriter.Write([]byte(testContent)) gzipWriter.Close() } else { w.Write([]byte(testContent)) } })) httpsServer.TLS = &tls.Config{ Certificates: []tls.Certificate{tlsCert}, } httpsServer.StartTLS() defer httpsServer.Close() // Create BSL initially without CA cert (pre-upgrade state) bsl := builder.ForBackupStorageLocation("test-ns", "test-bsl"). Provider("aws"). Bucket("test-bucket"). // Initially no CA cert Result() // Create an old backup that references this BSL oldBackup := builder.ForBackup("test-ns", "old-backup"). StorageLocation("test-bsl"). Result() // Setup fake client fakeClient := fake.NewClientBuilder(). WithScheme(util.VeleroScheme). WithRuntimeObjects(bsl, oldBackup). Build() ctx := t.Context() // Test 1: Old backup with HTTPS URL should fail without CA cert go func() { time.Sleep(10 * time.Millisecond) list := &velerov1api.DownloadRequestList{} err := fakeClient.List(t.Context(), list) assert.NoError(t, err) for _, dr := range list.Items { dr.Status.DownloadURL = httpsServer.URL + "/log" dr.Status.Phase = velerov1api.DownloadRequestPhaseProcessed err := fakeClient.Update(t.Context(), &dr) assert.NoError(t, err) } }() var bufFail bytes.Buffer err := Stream(ctx, fakeClient, "test-ns", "old-backup", velerov1api.DownloadTargetKindBackupLog, &bufFail, 5*time.Second, false, "") require.Error(t, err) assert.Contains(t, err.Error(), "x509") // Simulate BSL upgrade - get current BSL and update it currentBSL := &velerov1api.BackupStorageLocation{} err = fakeClient.Get(ctx, kbclient.ObjectKey{Namespace: "test-ns", Name: "test-bsl"}, currentBSL) require.NoError(t, err) // Update the BSL to include CA cert // Ensure ObjectStorage is initialized if currentBSL.Spec.ObjectStorage == nil { currentBSL.Spec.ObjectStorage = &velerov1api.ObjectStorageLocation{} } currentBSL.Spec.ObjectStorage.CACert = serverCACertPEM // CA cert added after upgrade err = fakeClient.Update(ctx, currentBSL) require.NoError(t, err) // Test 2: After BSL upgrade, old backup should work with new CA cert cacertStr, err := cacert.GetCACertFromBackup(ctx, fakeClient, "test-ns", oldBackup) require.NoError(t, err) require.Equal(t, string(serverCACertPEM), cacertStr) go func() { time.Sleep(10 * time.Millisecond) list := &velerov1api.DownloadRequestList{} err := fakeClient.List(t.Context(), list) assert.NoError(t, err) for _, dr := range list.Items { dr.Status.DownloadURL = httpsServer.URL + "/log" dr.Status.Phase = velerov1api.DownloadRequestPhaseProcessed err := fakeClient.Update(t.Context(), &dr) assert.NoError(t, err) } }() var bufSuccess bytes.Buffer err = StreamWithBSLCACert(ctx, fakeClient, "test-ns", "old-backup", velerov1api.DownloadTargetKindBackupLog, &bufSuccess, 5*time.Second, false, "", cacertStr) require.NoError(t, err) assert.Equal(t, testContent, bufSuccess.String()) // Test 3: New backup created after upgrade should also work newBackup := builder.ForBackup("test-ns", "new-backup"). StorageLocation("test-bsl"). Result() err = fakeClient.Create(ctx, newBackup) require.NoError(t, err) cacertStr2, err := cacert.GetCACertFromBackup(ctx, fakeClient, "test-ns", newBackup) require.NoError(t, err) require.Equal(t, string(serverCACertPEM), cacertStr2) } // TestConcurrentDownloadsWithBSLCACert tests multiple concurrent downloads with CA cert func TestConcurrentDownloadsWithBSLCACert(t *testing.T) { testContent := "concurrent test content" // Create self-signed certificate for TLS testing tlsCert, serverCACertPEM := createSelfSignedCertificate(t) // Create TLS test server tlsServer := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // Simulate some processing time time.Sleep(10 * time.Millisecond) w.WriteHeader(http.StatusOK) w.Write([]byte(testContent)) })) tlsServer.TLS = &tls.Config{ Certificates: []tls.Certificate{tlsCert}, } tlsServer.StartTLS() defer tlsServer.Close() // Run multiple concurrent downloads numConcurrent := 5 errors := make(chan error, numConcurrent) results := make(chan string, numConcurrent) for i := range numConcurrent { go func(idx int) { var buf bytes.Buffer err := download( t.Context(), tlsServer.URL, velerov1api.DownloadTargetKindBackupContents, &buf, false, "", string(serverCACertPEM), ) if err != nil { errors <- err return } results <- buf.String() }(i) } // Collect results for i := range numConcurrent { select { case err := <-errors: t.Fatalf("Concurrent download %d failed: %v", i, err) case result := <-results: assert.Equal(t, testContent, result) case <-time.After(5 * time.Second): t.Fatal("Timeout waiting for concurrent downloads") } } } func TestDownloadWithBSLCACert(t *testing.T) { testContent := "test content with BSL CA cert" // Create self-signed certificate for TLS testing tlsCert, serverCACertPEM := createSelfSignedCertificate(t) // Create TLS test server with self-signed cert tlsServer := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) w.Write([]byte(testContent)) })) tlsServer.TLS = &tls.Config{ Certificates: []tls.Certificate{tlsCert}, } tlsServer.StartTLS() defer tlsServer.Close() // Create HTTP test server for non-TLS tests httpServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) w.Write([]byte(testContent)) })) defer httpServer.Close() testCases := []struct { name string url string target velerov1api.DownloadTargetKind insecureSkipTLSVerify bool bslCACert string expectedContent string expectedError bool expectedErrContains string }{ { name: "successful TLS download with correct BSL CA cert", url: tlsServer.URL, target: velerov1api.DownloadTargetKindBackupContents, bslCACert: string(serverCACertPEM), expectedContent: testContent, expectedError: false, }, { name: "successful TLS download with insecureSkipTLSVerify", url: tlsServer.URL, target: velerov1api.DownloadTargetKindBackupContents, insecureSkipTLSVerify: true, bslCACert: "", expectedContent: testContent, expectedError: false, }, { name: "failed TLS download without CA cert or insecureSkipTLSVerify", url: tlsServer.URL, target: velerov1api.DownloadTargetKindBackupContents, bslCACert: "", expectedError: true, expectedErrContains: "x509", }, { name: "failed TLS download with wrong BSL CA cert", url: tlsServer.URL, target: velerov1api.DownloadTargetKindBackupContents, bslCACert: "-----BEGIN CERTIFICATE-----\nMIIBkTCB+wIJAKHHIgKwERfFMA0GCSqGSIb3DQEBCwUAMBkxFzAVBgNVBAMTDmRp\nZmZlcmVudC1jZXJ0MB4XDTE5MDQwMTAwMDAwMFoXDTI5MDQwMTAwMDAwMFowGTEX\nMBUGA1UEAxMOZGlmZmVyZW50LWNlcnQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ\nAoGBAOO4V+XrhVEGbTqnO2FM5eVFaM3KMKc3M9/C1aeg3vvY+Th3OhqJBxEYFxXL\nZoSqkwL/E6BjQb0NdSyJY9wdM4Ie3gElcZBKYVpHXYYAVhrepRCRVJEIHdBN8ybr\nFoBBDjd/ID1qy8Gdp3RihPFNvCNx0RWWqPAJtNXWJvCiNRCDAgMBAAEwDQYJKoZI\nhvcNAQELBQADgYEAGEwwGz7HAmH0J3pAJzQKPCb8HJG8hTjD6qkMon3Bp6gZ\n-----END CERTIFICATE-----\n", expectedError: true, expectedErrContains: "x509", }, { name: "successful HTTP download with empty BSL CA cert", url: httpServer.URL, target: velerov1api.DownloadTargetKindBackupContents, bslCACert: "", expectedContent: testContent, expectedError: false, }, { name: "successful TLS download with multiple CA certs in PEM block", url: tlsServer.URL, target: velerov1api.DownloadTargetKindBackupContents, bslCACert: "-----BEGIN CERTIFICATE-----\nMIIBkTCB+wIJAKHHIgKwERfFMA0GCSqGSIb3DQEBCwUAMBkxFzAVBgNVBAMTDmRp\nZmZlcmVudC1jZXJ0MB4XDTE5MDQwMTAwMDAwMFoXDTI5MDQwMTAwMDAwMFowGTEX\nMBUGA1UEAxMOZGlmZmVyZW50LWNlcnQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ\nAoGBAOO4V+XrhVEGbTqnO2FM5eVFaM3KMKc3M9/C1aeg3vvY+Th3OhqJBxEYFxXL\nZoSqkwL/E6BjQb0NdSyJY9wdM4Ie3gElcZBKYVpHXYYAVhrepRCRVJEIHdBN8ybr\nFoBBDjd/ID1qy8Gdp3RihPFNvCNx0RWWqPAJtNXWJvCiNRCDAgMBAAEwDQYJKoZI\nhvcNAQELBQADgYEAGEwwGz7HAmH0J3pAJzQKPCb8HJG8hTjD6qkMon3Bp6gZ\n-----END CERTIFICATE-----\n" + string(serverCACertPEM), // First cert is wrong, but second is correct expectedContent: testContent, expectedError: false, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { var buf bytes.Buffer err := download( t.Context(), tc.url, tc.target, &buf, tc.insecureSkipTLSVerify, "", tc.bslCACert, ) if tc.expectedError { require.Error(t, err) if tc.expectedErrContains != "" { assert.Contains(t, err.Error(), tc.expectedErrContains) } } else { require.NoError(t, err) assert.Equal(t, tc.expectedContent, buf.String()) } }) } } // TestCACertFallback tests the fallback behavior when caCertFile fails func TestCACertFallback(t *testing.T) { testContent := "test content for CA cert fallback" // Create self-signed certificate for TLS testing tlsCert, serverCACertPEM := createSelfSignedCertificate(t) // Create TLS test server tlsServer := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) w.Write([]byte(testContent)) })) tlsServer.TLS = &tls.Config{ Certificates: []tls.Certificate{tlsCert}, } tlsServer.StartTLS() defer tlsServer.Close() // Create a temporary file path that doesn't exist nonExistentFile := "/tmp/non-existent-ca-cert-file.pem" testCases := []struct { name string caCertFile string bslCACert string expectedError bool expectedErrContains string expectedContent string }{ { name: "successful download with BSL cert when caCertFile fails", caCertFile: nonExistentFile, bslCACert: string(serverCACertPEM), expectedError: false, expectedContent: testContent, }, { name: "failed download when both caCertFile and BSL cert are invalid", caCertFile: nonExistentFile, bslCACert: "", expectedError: true, expectedErrContains: "couldn't open cacert", }, { name: "BSL cert used when caCertFile is empty", caCertFile: "", bslCACert: string(serverCACertPEM), expectedError: false, expectedContent: testContent, }, { name: "successful download with valid caCertFile (BSL cert ignored)", caCertFile: "", // Will be set to a valid temp file in test bslCACert: "invalid cert that should be ignored", expectedError: false, expectedContent: testContent, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { caCertFile := tc.caCertFile // For the last test case, create a valid temp file if tc.name == "successful download with valid caCertFile (BSL cert ignored)" { tmpFile, err := os.CreateTemp(t.TempDir(), "test-ca-cert-*.pem") require.NoError(t, err) defer os.Remove(tmpFile.Name()) _, err = tmpFile.Write(serverCACertPEM) require.NoError(t, err) tmpFile.Close() caCertFile = tmpFile.Name() } var buf bytes.Buffer err := download( t.Context(), tlsServer.URL, velerov1api.DownloadTargetKindBackupContents, &buf, false, caCertFile, tc.bslCACert, ) if tc.expectedError { require.Error(t, err) if tc.expectedErrContains != "" { assert.Contains(t, err.Error(), tc.expectedErrContains) } } else { require.NoError(t, err) assert.Equal(t, tc.expectedContent, buf.String()) } }) } } ================================================ FILE: pkg/cmd/util/flag/accessors.go ================================================ /* Copyright 2017, 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package flag import ( "github.com/spf13/cobra" ) // GetOptionalStringFlag returns the value of the specified flag from a // cobra command, or the zero value ("") if the flag was not specified. func GetOptionalStringFlag(cmd *cobra.Command, flagName string) string { s, _ := cmd.Flags().GetString(flagName) return s } // GetOptionalBoolFlag returns the value of the specified flag from a // cobra command, or the zero value (false) if the flag was not specified. func GetOptionalBoolFlag(cmd *cobra.Command, flagName string) bool { b, _ := cmd.Flags().GetBool(flagName) return b } // GetOptionalStringArrayFlag returns the value of the specified flag from a // cobra command, or the zero value if the flag was not specified. func GetOptionalStringArrayFlag(cmd *cobra.Command, flagName string) []string { f := cmd.Flag(flagName) if f == nil { return []string{} } v := f.Value.(*StringArray) return *v } ================================================ FILE: pkg/cmd/util/flag/accessors_test.go ================================================ package flag import ( "testing" "github.com/spf13/cobra" "github.com/stretchr/testify/assert" ) func TestGetOptionalStringFlag(t *testing.T) { flagName := "flag" // not specified cmd := &cobra.Command{} assert.Empty(t, GetOptionalStringFlag(cmd, flagName)) // specified cmd.Flags().String(flagName, "value", "") assert.Equal(t, "value", GetOptionalStringFlag(cmd, flagName)) } func TestGetOptionalBoolFlag(t *testing.T) { flagName := "flag" // not specified cmd := &cobra.Command{} assert.False(t, GetOptionalBoolFlag(cmd, flagName)) // specified cmd.Flags().Bool(flagName, true, "") assert.True(t, GetOptionalBoolFlag(cmd, flagName)) } func TestGetOptionalStringArrayFlag(t *testing.T) { flagName := "flag" // not specified cmd := &cobra.Command{} assert.Equal(t, []string{}, GetOptionalStringArrayFlag(cmd, flagName)) // specified values := NewStringArray("value") cmd.Flags().Var(&values, flagName, "") assert.Equal(t, []string{"value"}, GetOptionalStringArrayFlag(cmd, flagName)) } ================================================ FILE: pkg/cmd/util/flag/array.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package flag import ( "strings" ) // StringArray is a Cobra-compatible named type for defining a // string slice flag. type StringArray []string // NewStringArray returns a StringArray for a provided // slice of values. func NewStringArray(initial ...string) StringArray { return StringArray(initial) } // String returns a comma-separated list of the items // in the string array. func (sa *StringArray) String() string { return strings.Join(*sa, ",") } // Set comma-splits the provided string and assigns // the results to the receiver. It returns an error if // the string is not parseable. func (sa *StringArray) Set(s string) error { *sa = strings.Split(s, ",") return nil } // Type returns a string representation of the // StringArray type. func (sa *StringArray) Type() string { return "stringArray" } ================================================ FILE: pkg/cmd/util/flag/array_test.go ================================================ package flag import ( "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestStringOfStringArray(t *testing.T) { array := NewStringArray("a", "b") assert.Equal(t, "a,b", array.String()) } func TestSetOfStringArray(t *testing.T) { array := NewStringArray() require.NoError(t, array.Set("a,b")) assert.Equal(t, "a,b", array.String()) } func TestTypeOfStringArray(t *testing.T) { array := NewStringArray() assert.Equal(t, "stringArray", array.Type()) } ================================================ FILE: pkg/cmd/util/flag/enum.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package flag import ( "github.com/pkg/errors" ) // Enum is a Cobra-compatible wrapper for defining // a string flag that can be one of a specified set // of values. type Enum struct { allowedValues []string value string } // NewEnum returns a new enum flag with the specified list // of allowed values, and the specified default value if // none is set. func NewEnum(defaultValue string, allowedValues ...string) *Enum { return &Enum{ allowedValues: allowedValues, value: defaultValue, } } // String returns a string representation of the // enum flag. func (e *Enum) String() string { return e.value } // Set assigns the provided string to the enum // receiver. It returns an error if the string // is not an allowed value. func (e *Enum) Set(s string) error { for _, val := range e.allowedValues { if val == s { e.value = s return nil } } return errors.Errorf("invalid value: %q", s) } // Type returns a string representation of the // Enum type. func (e *Enum) Type() string { // we don't want the help text to display anything regarding // the type because the usage text for the flag should capture // the possible options. return "" } // AllowedValues returns a slice of the flag's valid // values. func (e *Enum) AllowedValues() []string { return e.allowedValues } ================================================ FILE: pkg/cmd/util/flag/enum_test.go ================================================ package flag import ( "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestStringOfEnum(t *testing.T) { enum := NewEnum("a", "a", "b", "c") assert.Equal(t, "a", enum.String()) } func TestSetOfEnum(t *testing.T) { enum := NewEnum("a", "a", "b", "c") require.Error(t, enum.Set("d")) require.NoError(t, enum.Set("b")) assert.Equal(t, "b", enum.String()) } func TestTypeOfEnum(t *testing.T) { enum := NewEnum("a", "a", "b", "c") assert.Empty(t, enum.Type()) } func TestAllowedValuesOfEnum(t *testing.T) { enum := NewEnum("a", "a", "b", "c") assert.Equal(t, []string{"a", "b", "c"}, enum.AllowedValues()) } ================================================ FILE: pkg/cmd/util/flag/label_selector_test.go ================================================ package flag import ( "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func TestStringOfLabelSelector(t *testing.T) { ls, err := metav1.ParseToLabelSelector("k1=v1,k2=v2") require.NoError(t, err) selector := &LabelSelector{ LabelSelector: ls, } assert.Equal(t, "k1=v1,k2=v2", selector.String()) } func TestSetOfLabelSelector(t *testing.T) { selector := &LabelSelector{} require.NoError(t, selector.Set("k1=v1,k2=v2")) str := selector.String() assert.True(t, str == "k1=v1,k2=v2" || str == "k2=v2,k2=v2") } func TestTypeOfLabelSelector(t *testing.T) { selector := &LabelSelector{} assert.Equal(t, "labelSelector", selector.Type()) } ================================================ FILE: pkg/cmd/util/flag/labelselector.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package flag import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // LabelSelector is a Cobra-compatible wrapper for defining // a Kubernetes label-selector flag. type LabelSelector struct { LabelSelector *metav1.LabelSelector } // String returns a string representation of the label // selector flag. func (ls *LabelSelector) String() string { return metav1.FormatLabelSelector(ls.LabelSelector) } // Set parses the provided string and assigns the result // to the label-selector receiver. It returns an error if // the string is not parseable. func (ls *LabelSelector) Set(s string) error { parsed, err := metav1.ParseToLabelSelector(s) if err != nil { return err } ls.LabelSelector = parsed return nil } // Type returns a string representation of the // LabelSelector type. func (ls *LabelSelector) Type() string { return "labelSelector" } ================================================ FILE: pkg/cmd/util/flag/map.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package flag import ( "encoding/csv" "fmt" "strings" "github.com/pkg/errors" ) // Map is a Cobra-compatible wrapper for defining a flag containing // map data (i.e. a collection of key-value pairs). type Map struct { data map[string]string entryDelimiter rune keyValueDelimiter rune } // NewMap returns a Map using the default delimiters ('=' between keys and // values, and ',' between map entries, e.g. k1=v1,k2=v2) func NewMap() Map { m := Map{ data: make(map[string]string), } return m.WithEntryDelimiter(',').WithKeyValueDelimiter('=') } // WithEntryDelimiter sets the delimiter to be used between map // entries. // // For example, in "k1=v1&k2=v2", the entry delimiter is '&' func (m Map) WithEntryDelimiter(delimiter rune) Map { m.entryDelimiter = delimiter return m } // WithKeyValueDelimiter sets the delimiter to be used between // keys and values. // // For example, in "k1=v1&k2=v2", the key-value delimiter is '=' func (m Map) WithKeyValueDelimiter(delimiter rune) Map { m.keyValueDelimiter = delimiter return m } // String returns a string representation of the Map flag. func (m *Map) String() string { var a []string for k, v := range m.data { a = append(a, fmt.Sprintf("%s%s%s", k, string(m.keyValueDelimiter), v)) } return strings.Join(a, string(m.entryDelimiter)) } // Set parses the provided string according to the delimiters and // assigns the result to the Map receiver. It returns an error if // the string is not parseable. func (m *Map) Set(s string) error { // use csv.Reader to support parsing input string contains entry delimiters. // e.g. `"k1=a=b,c=d",k2=v2` will be parsed into two parts: `k1=a=b,c=d` and `k2=v2` r := csv.NewReader(strings.NewReader(s)) r.Comma = m.entryDelimiter parts, err := r.Read() if err != nil { return errors.Wrapf(err, "error parsing %q", s) } for _, part := range parts { kvs := strings.SplitN(part, string(m.keyValueDelimiter), 2) if len(kvs) != 2 { return errors.Errorf("error parsing %q", part) } m.data[kvs[0]] = kvs[1] } return nil } // Type returns a string representation of the // Map type. func (m *Map) Type() string { return "mapStringString" } // Data returns the underlying golang map storing // the flag data. func (m *Map) Data() map[string]string { return m.data } ================================================ FILE: pkg/cmd/util/flag/map_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package flag import ( "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestSetOfMap(t *testing.T) { cases := []struct { name string input string error bool expected map[string]string }{ { name: "invalid_input_missing_quote", input: `"k=v`, error: true, }, { name: "invalid_input_contains_no_key_value_delimiter", input: `k`, error: true, }, { name: "valid input", input: `k1=v1,k2=v2`, error: false, expected: map[string]string{ "k1": "v1", "k2": "v2", }, }, { name: "valid input whose value contains entry delimiter", input: `k1=v1,"k2=a=b,c=d"`, error: false, expected: map[string]string{ "k1": "v1", "k2": "a=b,c=d", }, }, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { m := NewMap() err := m.Set(c.input) if c.error { require.Error(t, err) return } assert.Equal(t, c.expected, m.Data()) }) } } func TestStringOfMap(t *testing.T) { m := NewMap() require.NoError(t, m.Set("k1=v1,k2=v2")) str := m.String() assert.True(t, str == "k1=v1,k2=v2" || str == "k2=v2,k1=v1") } func TestTypeOfMap(t *testing.T) { m := NewMap() assert.Equal(t, "mapStringString", m.Type()) } ================================================ FILE: pkg/cmd/util/flag/optional_bool.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package flag import "strconv" type OptionalBool struct { Value *bool } func NewOptionalBool(defaultValue *bool) OptionalBool { return OptionalBool{ Value: defaultValue, } } // String returns a string representation of the // enum flag. func (f *OptionalBool) String() string { switch f.Value { case nil: return "" default: return strconv.FormatBool(*f.Value) } } func (f *OptionalBool) Set(val string) error { if val == "" { f.Value = nil return nil } parsed, err := strconv.ParseBool(val) if err != nil { return err } f.Value = &parsed return nil } func (f *OptionalBool) Type() string { return "optionalBool" } ================================================ FILE: pkg/cmd/util/flag/optional_bool_test.go ================================================ package flag import ( "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestStringOfOptionalBool(t *testing.T) { // nil ob := NewOptionalBool(nil) assert.Equal(t, "", ob.String()) // true b := true ob = NewOptionalBool(&b) assert.Equal(t, "true", ob.String()) // false b = false ob = NewOptionalBool(&b) assert.Equal(t, "false", ob.String()) } func TestSetOfOptionalBool(t *testing.T) { // error ob := NewOptionalBool(nil) require.Error(t, ob.Set("invalid")) // nil ob = NewOptionalBool(nil) require.NoError(t, ob.Set("")) assert.Nil(t, ob.Value) // true ob = NewOptionalBool(nil) require.NoError(t, ob.Set("true")) assert.True(t, *ob.Value) } func TestTypeOfOptionalBool(t *testing.T) { ob := NewOptionalBool(nil) assert.Equal(t, "optionalBool", ob.Type()) } ================================================ FILE: pkg/cmd/util/flag/orlabelselector.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package flag import ( "strings" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // OrLabelSelector is a Cobra-compatible wrapper for defining // a Kubernetes or-label-selector flag. type OrLabelSelector struct { OrLabelSelectors []*metav1.LabelSelector } // String returns a string representation of the or-label // selector flag. func (ls *OrLabelSelector) String() string { orLabels := []string{} for _, v := range ls.OrLabelSelectors { orLabels = append(orLabels, metav1.FormatLabelSelector(v)) } return strings.Join(orLabels, " or ") } // Set parses the provided string and assigns the result // to the or-label-selector receiver. It returns an error if // the string is not parseable. func (ls *OrLabelSelector) Set(s string) error { orItems := strings.Split(s, " or ") ls.OrLabelSelectors = make([]*metav1.LabelSelector, 0) for _, orItem := range orItems { parsed, err := metav1.ParseToLabelSelector(orItem) if err != nil { return err } ls.OrLabelSelectors = append(ls.OrLabelSelectors, parsed) } return nil } // Type returns a string representation of the // OrLabelSelector type. func (ls *OrLabelSelector) Type() string { return "orLabelSelector" } ================================================ FILE: pkg/cmd/util/flag/orlabelselector_test.go ================================================ package flag import ( "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func TestStringOfOrLabelSelector(t *testing.T) { tests := []struct { name string orLabelSelector *OrLabelSelector expectedStr string }{ { name: "or between two labels", orLabelSelector: &OrLabelSelector{ OrLabelSelectors: []*metav1.LabelSelector{ { MatchLabels: map[string]string{"k1": "v1"}, }, { MatchLabels: map[string]string{"k2": "v2"}, }, }, }, expectedStr: "k1=v1 or k2=v2", }, { name: "or between two label groups", orLabelSelector: &OrLabelSelector{ OrLabelSelectors: []*metav1.LabelSelector{ { MatchLabels: map[string]string{"k1": "v1", "k2": "v2"}, }, { MatchLabels: map[string]string{"a1": "b1", "a2": "b2"}, }, }, }, expectedStr: "k1=v1,k2=v2 or a1=b1,a2=b2", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { assert.Equal(t, test.expectedStr, test.orLabelSelector.String()) }) } } func TestSetOfOrLabelSelector(t *testing.T) { tests := []struct { name string inputStr string expectedSelector *OrLabelSelector }{ { name: "or between two labels", inputStr: "k1=v1 or k2=v2", expectedSelector: &OrLabelSelector{ OrLabelSelectors: []*metav1.LabelSelector{ { MatchLabels: map[string]string{"k1": "v1"}, }, { MatchLabels: map[string]string{"k2": "v2"}, }, }, }, }, { name: "or between two label groups", inputStr: "k1=v1,k2=v2 or a1=b1,a2=b2", expectedSelector: &OrLabelSelector{ OrLabelSelectors: []*metav1.LabelSelector{ { MatchLabels: map[string]string{"k1": "v1", "k2": "v2"}, }, { MatchLabels: map[string]string{"a1": "b1", "a2": "b2"}, }, }, }, }, } selector := &OrLabelSelector{} for _, test := range tests { t.Run(test.name, func(t *testing.T) { require.NoError(t, selector.Set(test.inputStr)) assert.Len(t, selector.OrLabelSelectors, len(test.expectedSelector.OrLabelSelectors)) assert.Equal(t, test.expectedSelector.String(), selector.String()) }) } } func TestTypeOfOrLabelSelector(t *testing.T) { selector := &OrLabelSelector{} assert.Equal(t, "orLabelSelector", selector.Type()) } ================================================ FILE: pkg/cmd/util/output/backup_describer.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package output import ( "bytes" "context" "encoding/json" "fmt" "sort" "strconv" "strings" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" "github.com/pkg/errors" "github.com/fatih/color" kbclient "sigs.k8s.io/controller-runtime/pkg/client" veleroapishared "github.com/vmware-tanzu/velero/pkg/apis/velero/shared" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/cmd/util/cacert" "github.com/vmware-tanzu/velero/pkg/cmd/util/downloadrequest" "github.com/vmware-tanzu/velero/pkg/itemoperation" "github.com/vmware-tanzu/velero/internal/volume" "github.com/vmware-tanzu/velero/pkg/util/collections" "github.com/vmware-tanzu/velero/pkg/util/results" ) // DescribeBackup describes a backup in human-readable format. func DescribeBackup( ctx context.Context, kbClient kbclient.Client, backup *velerov1api.Backup, deleteRequests []velerov1api.DeleteBackupRequest, podVolumeBackups []velerov1api.PodVolumeBackup, details bool, insecureSkipTLSVerify bool, caCertFile string, ) string { return Describe(func(d *Describer) { d.DescribeMetadata(backup.ObjectMeta) d.Println() phase := backup.Status.Phase if phase == "" { phase = velerov1api.BackupPhaseNew } phaseString := string(phase) switch phase { case velerov1api.BackupPhaseFailedValidation, velerov1api.BackupPhasePartiallyFailed, velerov1api.BackupPhaseFailed: phaseString = color.RedString(phaseString) case velerov1api.BackupPhaseCompleted: phaseString = color.GreenString(phaseString) case velerov1api.BackupPhaseDeleting: case velerov1api.BackupPhaseWaitingForPluginOperations, velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed: case velerov1api.BackupPhaseFinalizing, velerov1api.BackupPhaseFinalizingPartiallyFailed: case velerov1api.BackupPhaseInProgress: case velerov1api.BackupPhaseNew: case velerov1api.BackupPhaseQueued, velerov1api.BackupPhaseReadyToStart: } logsNote := "" if backup.Status.Phase == velerov1api.BackupPhaseFailed || backup.Status.Phase == velerov1api.BackupPhasePartiallyFailed { logsNote = fmt.Sprintf(" (run `velero backup logs %s` for more information)", backup.Name) } d.Printf("Phase:\t%s%s\n", phaseString, logsNote) if phase == velerov1api.BackupPhaseQueued { d.Printf("Queue position:\t%v\n", backup.Status.QueuePosition) } if backup.Spec.ResourcePolicy != nil { d.Println() DescribeResourcePolicies(d, backup.Spec.ResourcePolicy) } if backup.Spec.UploaderConfig != nil && backup.Spec.UploaderConfig.ParallelFilesUpload > 0 { d.Println() DescribeUploaderConfigForBackup(d, backup.Spec) } status := backup.Status if len(status.ValidationErrors) > 0 { d.Println() d.Printf("Validation errors:") for _, ve := range status.ValidationErrors { d.Printf("\t%s\n", color.RedString(ve)) } } d.Println() DescribeBackupResults(ctx, kbClient, d, backup, insecureSkipTLSVerify, caCertFile) d.Println() DescribeBackupSpec(d, backup.Spec) d.Println() DescribeBackupStatus(ctx, kbClient, d, backup, details, insecureSkipTLSVerify, caCertFile, podVolumeBackups) if len(deleteRequests) > 0 { d.Println() DescribeDeleteBackupRequests(d, deleteRequests) } }) } // DescribeResourcePolicies describes resource policies in human-readable format func DescribeResourcePolicies(d *Describer, resPolicies *corev1api.TypedLocalObjectReference) { d.Printf("Resource policies:\n") d.Printf("\tType:\t%s\n", resPolicies.Kind) d.Printf("\tName:\t%s\n", resPolicies.Name) } // DescribeUploaderConfigForBackup describes uploader config in human-readable format func DescribeUploaderConfigForBackup(d *Describer, spec velerov1api.BackupSpec) { d.Printf("Uploader config:\n") d.Printf("\tParallel files upload:\t%d\n", spec.UploaderConfig.ParallelFilesUpload) } // DescribeBackupSpec describes a backup spec in human-readable format. func DescribeBackupSpec(d *Describer, spec velerov1api.BackupSpec) { // TODO make a helper for this and use it in all the describers. d.Printf("Namespaces:\n") var s string if len(spec.IncludedNamespaces) == 0 { s = "*" } else { s = strings.Join(spec.IncludedNamespaces, ", ") } d.Printf("\tIncluded:\t%s\n", s) if len(spec.ExcludedNamespaces) == 0 { s = emptyDisplay } else { s = strings.Join(spec.ExcludedNamespaces, ", ") } d.Printf("\tExcluded:\t%s\n", s) d.Println() d.Printf("Resources:\n") if collections.UseOldResourceFilters(spec) { if len(spec.IncludedResources) == 0 { s = "*" } else { s = strings.Join(spec.IncludedResources, ", ") } d.Printf("\tIncluded:\t%s\n", s) if len(spec.ExcludedResources) == 0 { s = emptyDisplay } else { s = strings.Join(spec.ExcludedResources, ", ") } d.Printf("\tExcluded:\t%s\n", s) d.Printf("\tCluster-scoped:\t%s\n", BoolPointerString(spec.IncludeClusterResources, "excluded", "included", "auto")) } else { if len(spec.IncludedClusterScopedResources) == 0 { s = emptyDisplay } else { s = strings.Join(spec.IncludedClusterScopedResources, ", ") } d.Printf("\tIncluded cluster-scoped:\t%s\n", s) if len(spec.ExcludedClusterScopedResources) == 0 { s = emptyDisplay } else { s = strings.Join(spec.ExcludedClusterScopedResources, ", ") } d.Printf("\tExcluded cluster-scoped:\t%s\n", s) if len(spec.IncludedNamespaceScopedResources) == 0 { s = "*" } else { s = strings.Join(spec.IncludedNamespaceScopedResources, ", ") } d.Printf("\tIncluded namespace-scoped:\t%s\n", s) if len(spec.ExcludedNamespaceScopedResources) == 0 { s = emptyDisplay } else { s = strings.Join(spec.ExcludedNamespaceScopedResources, ", ") } d.Printf("\tExcluded namespace-scoped:\t%s\n", s) } d.Println() s = emptyDisplay if spec.LabelSelector != nil { s = metav1.FormatLabelSelector(spec.LabelSelector) } d.Printf("Label selector:\t%s\n", s) d.Println() if len(spec.OrLabelSelectors) == 0 { s = emptyDisplay } else { orLabelSelectors := []string{} for _, v := range spec.OrLabelSelectors { orLabelSelectors = append(orLabelSelectors, metav1.FormatLabelSelector(v)) } s = strings.Join(orLabelSelectors, " or ") } d.Printf("Or label selector:\t%s\n", s) d.Println() d.Printf("Storage Location:\t%s\n", spec.StorageLocation) d.Println() d.Printf("Velero-Native Snapshot PVs:\t%s\n", BoolPointerString(spec.SnapshotVolumes, "false", "true", "auto")) if spec.DefaultVolumesToFsBackup != nil { d.Printf("File System Backup (Default):\t%s\n", BoolPointerString(spec.DefaultVolumesToFsBackup, "false", "true", "")) } d.Printf("Snapshot Move Data:\t%s\n", BoolPointerString(spec.SnapshotMoveData, "false", "true", "auto")) if len(spec.DataMover) == 0 { s = defaultDataMover } else { s = spec.DataMover } d.Printf("Data Mover:\t%s\n", s) d.Println() d.Printf("TTL:\t%s\n", spec.TTL.Duration) d.Println() d.Printf("CSISnapshotTimeout:\t%s\n", spec.CSISnapshotTimeout.Duration) d.Printf("ItemOperationTimeout:\t%s\n", spec.ItemOperationTimeout.Duration) d.Println() if len(spec.Hooks.Resources) == 0 { d.Printf("Hooks:\t" + emptyDisplay + "\n") } else { d.Printf("Hooks:\n") d.Printf("\tResources:\n") for _, backupResourceHookSpec := range spec.Hooks.Resources { d.Printf("\t\t%s:\n", backupResourceHookSpec.Name) d.Printf("\t\t\tNamespaces:\n") var s string if len(backupResourceHookSpec.IncludedNamespaces) == 0 { s = "*" } else { s = strings.Join(backupResourceHookSpec.IncludedNamespaces, ", ") } d.Printf("\t\t\t\tIncluded:\t%s\n", s) if len(backupResourceHookSpec.ExcludedNamespaces) == 0 { s = emptyDisplay } else { s = strings.Join(backupResourceHookSpec.ExcludedNamespaces, ", ") } d.Printf("\t\t\t\tExcluded:\t%s\n", s) d.Println() d.Printf("\t\t\tResources:\n") if len(backupResourceHookSpec.IncludedResources) == 0 { s = "*" } else { s = strings.Join(backupResourceHookSpec.IncludedResources, ", ") } d.Printf("\t\t\t\tIncluded:\t%s\n", s) if len(backupResourceHookSpec.ExcludedResources) == 0 { s = emptyDisplay } else { s = strings.Join(backupResourceHookSpec.ExcludedResources, ", ") } d.Printf("\t\t\t\tExcluded:\t%s\n", s) d.Println() s = emptyDisplay if backupResourceHookSpec.LabelSelector != nil { s = metav1.FormatLabelSelector(backupResourceHookSpec.LabelSelector) } d.Printf("\t\t\tLabel selector:\t%s\n", s) for _, hook := range backupResourceHookSpec.PreHooks { if hook.Exec != nil { d.Println() d.Printf("\t\t\tPre Exec Hook:\n") d.Printf("\t\t\t\tContainer:\t%s\n", hook.Exec.Container) d.Printf("\t\t\t\tCommand:\t%s\n", strings.Join(hook.Exec.Command, " ")) d.Printf("\t\t\t\tOn Error:\t%s\n", hook.Exec.OnError) d.Printf("\t\t\t\tTimeout:\t%s\n", hook.Exec.Timeout.Duration) } } for _, hook := range backupResourceHookSpec.PostHooks { if hook.Exec != nil { d.Println() d.Printf("\t\t\tPost Exec Hook:\n") d.Printf("\t\t\t\tContainer:\t%s\n", hook.Exec.Container) d.Printf("\t\t\t\tCommand:\t%s\n", strings.Join(hook.Exec.Command, " ")) d.Printf("\t\t\t\tOn Error:\t%s\n", hook.Exec.OnError) d.Printf("\t\t\t\tTimeout:\t%s\n", hook.Exec.Timeout.Duration) } } } } if spec.OrderedResources != nil { d.Println() d.Printf("OrderedResources:\n") for key, value := range spec.OrderedResources { d.Printf("\t%s: %s\n", key, value) } } } // DescribeBackupStatus describes a backup status in human-readable format. func DescribeBackupStatus(ctx context.Context, kbClient kbclient.Client, d *Describer, backup *velerov1api.Backup, details bool, insecureSkipTLSVerify bool, caCertPath string, podVolumeBackups []velerov1api.PodVolumeBackup) { status := backup.Status // Status.Version has been deprecated, use Status.FormatVersion d.Printf("Backup Format Version:\t%s\n", status.FormatVersion) d.Println() // "" output should only be applicable for backups that failed validation if status.StartTimestamp == nil || status.StartTimestamp.Time.IsZero() { d.Printf("Started:\t%s\n", "") } else { d.Printf("Started:\t%s\n", status.StartTimestamp.Time) } if status.CompletionTimestamp == nil || status.CompletionTimestamp.Time.IsZero() { d.Printf("Completed:\t%s\n", "") } else { d.Printf("Completed:\t%s\n", status.CompletionTimestamp.Time) } d.Println() // Expiration can't be 0, it is always set to a 30-day default. It can be nil // if the controller hasn't processed this Backup yet, in which case this will // just display ``, though this should be temporary. d.Printf("Expiration:\t%s\n", status.Expiration) d.Println() if backup.Status.Progress != nil { if backup.Status.Phase == velerov1api.BackupPhaseInProgress { d.Printf("Estimated total items to be backed up:\t%d\n", backup.Status.Progress.TotalItems) d.Printf("Items backed up so far:\t%d\n", backup.Status.Progress.ItemsBackedUp) } else { d.Printf("Total items to be backed up:\t%d\n", backup.Status.Progress.TotalItems) d.Printf("Items backed up:\t%d\n", backup.Status.Progress.ItemsBackedUp) } d.Println() } describeBackupItemOperations(ctx, kbClient, d, backup, details, insecureSkipTLSVerify, caCertPath) if details { describeBackupResourceList(ctx, kbClient, d, backup, insecureSkipTLSVerify, caCertPath) d.Println() } describeBackupVolumes(ctx, kbClient, d, backup, details, insecureSkipTLSVerify, caCertPath, podVolumeBackups) if status.HookStatus != nil { d.Println() d.Printf("HooksAttempted:\t%d\n", status.HookStatus.HooksAttempted) d.Printf("HooksFailed:\t%d\n", status.HookStatus.HooksFailed) } } func describeBackupItemOperations(ctx context.Context, kbClient kbclient.Client, d *Describer, backup *velerov1api.Backup, details bool, insecureSkipTLSVerify bool, caCertPath string) { status := backup.Status if status.BackupItemOperationsAttempted > 0 { if !details { d.Printf("Backup Item Operations:\t%d of %d completed successfully, %d failed (specify --details for more information)\n", status.BackupItemOperationsCompleted, status.BackupItemOperationsAttempted, status.BackupItemOperationsFailed) return } // Get BSL cacert if available bslCACert, err := cacert.GetCACertFromBackup(ctx, kbClient, backup.Namespace, backup) if err != nil { // Log the error but don't fail - we can still try to download without the BSL cacert d.Printf("WARNING: Error getting cacert from BSL: %v\n", err) bslCACert = "" } buf := new(bytes.Buffer) if err := downloadrequest.StreamWithBSLCACert(ctx, kbClient, backup.Namespace, backup.Name, velerov1api.DownloadTargetKindBackupItemOperations, buf, downloadRequestTimeout, insecureSkipTLSVerify, caCertPath, bslCACert); err != nil { d.Printf("Backup Item Operations:\t\n", err) return } var operations []*itemoperation.BackupOperation if err := json.NewDecoder(buf).Decode(&operations); err != nil { d.Printf("Backup Item Operations:\t\n", err) return } d.Printf("Backup Item Operations:\n") for _, operation := range operations { describeBackupItemOperation(d, operation) } } } func describeBackupResourceList(ctx context.Context, kbClient kbclient.Client, d *Describer, backup *velerov1api.Backup, insecureSkipTLSVerify bool, caCertPath string) { // Get BSL cacert if available bslCACert, err := cacert.GetCACertFromBackup(ctx, kbClient, backup.Namespace, backup) if err != nil { // Log the error but don't fail - we can still try to download without the BSL cacert d.Printf("WARNING: Error getting cacert from BSL: %v\n", err) bslCACert = "" } buf := new(bytes.Buffer) if err := downloadrequest.StreamWithBSLCACert(ctx, kbClient, backup.Namespace, backup.Name, velerov1api.DownloadTargetKindBackupResourceList, buf, downloadRequestTimeout, insecureSkipTLSVerify, caCertPath, bslCACert); err != nil { if err == downloadrequest.ErrNotFound { // the backup resource list could be missing if (other reasons may exist as well): // - the backup was taken prior to v1.1; or // - the backup hasn't completed yet; or // - there was an error uploading the file; or // - the file was manually deleted after upload d.Println("Resource List:\t") } else { d.Printf("Resource List:\t\n", err) } return } var resourceList map[string][]string if err := json.NewDecoder(buf).Decode(&resourceList); err != nil { d.Printf("Resource List:\t\n", err) return } d.Println("Resource List:") // Sort GVKs in output gvks := make([]string, 0, len(resourceList)) for gvk := range resourceList { gvks = append(gvks, gvk) } sort.Strings(gvks) for _, gvk := range gvks { d.Printf("\t%s:\n\t\t- %s\n", gvk, strings.Join(resourceList[gvk], "\n\t\t- ")) } } func describeBackupVolumes( ctx context.Context, kbClient kbclient.Client, d *Describer, backup *velerov1api.Backup, details bool, insecureSkipTLSVerify bool, caCertPath string, podVolumeBackupCRs []velerov1api.PodVolumeBackup, ) { d.Println("Backup Volumes:") // Get BSL cacert if available bslCACert, err := cacert.GetCACertFromBackup(ctx, kbClient, backup.Namespace, backup) if err != nil { // Log the error but don't fail - we can still try to download without the BSL cacert d.Printf("WARNING: Error getting cacert from BSL: %v\n", err) bslCACert = "" } nativeSnapshots := []*volume.BackupVolumeInfo{} csiSnapshots := []*volume.BackupVolumeInfo{} legacyInfoSource := false buf := new(bytes.Buffer) err = downloadrequest.StreamWithBSLCACert(ctx, kbClient, backup.Namespace, backup.Name, velerov1api.DownloadTargetKindBackupVolumeInfos, buf, downloadRequestTimeout, insecureSkipTLSVerify, caCertPath, bslCACert) if err == downloadrequest.ErrNotFound { nativeSnapshots, err = retrieveNativeSnapshotLegacy(ctx, kbClient, backup, insecureSkipTLSVerify, caCertPath, bslCACert) if err != nil { d.Printf("\t\n", err) return } csiSnapshots, err = retrieveCSISnapshotLegacy(ctx, kbClient, backup, insecureSkipTLSVerify, caCertPath, bslCACert) if err != nil { d.Printf("\t\n", err) return } legacyInfoSource = true } else if err != nil { d.Printf("\t\n", err) return } else { var volumeInfos []volume.BackupVolumeInfo if err := json.NewDecoder(buf).Decode(&volumeInfos); err != nil { d.Printf("\t\n", err) return } for i := range volumeInfos { switch volumeInfos[i].BackupMethod { case volume.NativeSnapshot: nativeSnapshots = append(nativeSnapshots, &volumeInfos[i]) case volume.CSISnapshot: csiSnapshots = append(csiSnapshots, &volumeInfos[i]) } } } describeNativeSnapshots(d, details, nativeSnapshots) d.Println() describeCSISnapshots(d, details, csiSnapshots, legacyInfoSource) d.Println() describePodVolumeBackups(d, details, podVolumeBackupCRs) } func retrieveNativeSnapshotLegacy(ctx context.Context, kbClient kbclient.Client, backup *velerov1api.Backup, insecureSkipTLSVerify bool, caCertPath string, bslCACert string) ([]*volume.BackupVolumeInfo, error) { status := backup.Status nativeSnapshots := []*volume.BackupVolumeInfo{} if status.VolumeSnapshotsAttempted == 0 { return nativeSnapshots, nil } buf := new(bytes.Buffer) if err := downloadrequest.StreamWithBSLCACert(ctx, kbClient, backup.Namespace, backup.Name, velerov1api.DownloadTargetKindBackupVolumeSnapshots, buf, downloadRequestTimeout, insecureSkipTLSVerify, caCertPath, bslCACert); err != nil { return nativeSnapshots, errors.Wrapf(err, "error to download native snapshot info") } var snapshots []*volume.Snapshot if err := json.NewDecoder(buf).Decode(&snapshots); err != nil { return nativeSnapshots, errors.Wrapf(err, "error to decode native snapshot info") } for _, snap := range snapshots { volumeInfo := volume.BackupVolumeInfo{ PVName: snap.Spec.PersistentVolumeName, NativeSnapshotInfo: &volume.NativeSnapshotInfo{ SnapshotHandle: snap.Status.ProviderSnapshotID, VolumeType: snap.Spec.VolumeType, VolumeAZ: snap.Spec.VolumeAZ, }, } if snap.Spec.VolumeIOPS != nil { volumeInfo.NativeSnapshotInfo.IOPS = strconv.FormatInt(*snap.Spec.VolumeIOPS, 10) } nativeSnapshots = append(nativeSnapshots, &volumeInfo) } return nativeSnapshots, nil } func retrieveCSISnapshotLegacy(ctx context.Context, kbClient kbclient.Client, backup *velerov1api.Backup, insecureSkipTLSVerify bool, caCertPath string, bslCACert string) ([]*volume.BackupVolumeInfo, error) { status := backup.Status csiSnapshots := []*volume.BackupVolumeInfo{} if status.CSIVolumeSnapshotsAttempted == 0 { return csiSnapshots, nil } vsBuf := new(bytes.Buffer) err := downloadrequest.StreamWithBSLCACert(ctx, kbClient, backup.Namespace, backup.Name, velerov1api.DownloadTargetKindCSIBackupVolumeSnapshots, vsBuf, downloadRequestTimeout, insecureSkipTLSVerify, caCertPath, bslCACert) if err != nil { return csiSnapshots, errors.Wrapf(err, "error to download vs list") } var vsList []snapshotv1api.VolumeSnapshot if err := json.NewDecoder(vsBuf).Decode(&vsList); err != nil { return csiSnapshots, errors.Wrapf(err, "error to decode vs list") } vscBuf := new(bytes.Buffer) err = downloadrequest.StreamWithBSLCACert(ctx, kbClient, backup.Namespace, backup.Name, velerov1api.DownloadTargetKindCSIBackupVolumeSnapshotContents, vscBuf, downloadRequestTimeout, insecureSkipTLSVerify, caCertPath, bslCACert) if err != nil { return csiSnapshots, errors.Wrapf(err, "error to download vsc list") } var vscList []snapshotv1api.VolumeSnapshotContent if err := json.NewDecoder(vscBuf).Decode(&vscList); err != nil { return csiSnapshots, errors.Wrapf(err, "error to decode vsc list") } for _, vsc := range vscList { volInfo := volume.BackupVolumeInfo{ PreserveLocalSnapshot: true, CSISnapshotInfo: &volume.CSISnapshotInfo{ VSCName: vsc.Name, Driver: vsc.Spec.Driver, }, } if vsc.Status != nil && vsc.Status.SnapshotHandle != nil { volInfo.CSISnapshotInfo.SnapshotHandle = *vsc.Status.SnapshotHandle } if vsc.Status != nil && vsc.Status.RestoreSize != nil { volInfo.CSISnapshotInfo.Size = *vsc.Status.RestoreSize } for _, vs := range vsList { if vs.Status.BoundVolumeSnapshotContentName == nil { continue } if vs.Spec.Source.PersistentVolumeClaimName == nil { continue } if *vs.Status.BoundVolumeSnapshotContentName == vsc.Name { volInfo.PVCName = *vs.Spec.Source.PersistentVolumeClaimName volInfo.PVCNamespace = vs.Namespace } } if volInfo.PVCName == "" { volInfo.PVCName = "" } csiSnapshots = append(csiSnapshots, &volInfo) } return csiSnapshots, nil } func describeNativeSnapshots(d *Describer, details bool, infos []*volume.BackupVolumeInfo) { if len(infos) == 0 { d.Printf("\tVelero-Native Snapshots: \n") return } d.Println("\tVelero-Native Snapshots:") for _, info := range infos { describNativeSnapshot(d, details, info) } } func describNativeSnapshot(d *Describer, details bool, info *volume.BackupVolumeInfo) { if details { d.Printf("\t\t%s:\n", info.PVName) d.Printf("\t\t\tSnapshot ID:\t%s\n", info.NativeSnapshotInfo.SnapshotHandle) d.Printf("\t\t\tType:\t%s\n", info.NativeSnapshotInfo.VolumeType) d.Printf("\t\t\tAvailability Zone:\t%s\n", info.NativeSnapshotInfo.VolumeAZ) d.Printf("\t\t\tIOPS:\t%s\n", info.NativeSnapshotInfo.IOPS) d.Printf("\t\t\tResult:\t%s\n", info.Result) } else { d.Printf("\t\t%s: specify --details for more information\n", info.PVName) } } func describeCSISnapshots(d *Describer, details bool, infos []*volume.BackupVolumeInfo, legacyInfoSource bool) { if len(infos) == 0 { if legacyInfoSource { d.Printf("\tCSI Snapshots: \n") } else { d.Printf("\tCSI Snapshots: \n") } return } d.Println("\tCSI Snapshots:") for _, info := range infos { describeCSISnapshot(d, details, info) } } func describeCSISnapshot(d *Describer, details bool, info *volume.BackupVolumeInfo) { d.Printf("\t\t%s:\n", fmt.Sprintf("%s/%s", info.PVCNamespace, info.PVCName)) describeLocalSnapshot(d, details, info) describeDataMovement(d, details, info) } func describeLocalSnapshot(d *Describer, details bool, info *volume.BackupVolumeInfo) { if !info.PreserveLocalSnapshot { return } if details { d.Printf("\t\t\tSnapshot:\n") if !info.SnapshotDataMoved && info.CSISnapshotInfo.OperationID != "" { d.Printf("\t\t\t\tOperation ID: %s\n", info.CSISnapshotInfo.OperationID) } d.Printf("\t\t\t\tSnapshot Content Name: %s\n", info.CSISnapshotInfo.VSCName) d.Printf("\t\t\t\tStorage Snapshot ID: %s\n", info.CSISnapshotInfo.SnapshotHandle) d.Printf("\t\t\t\tSnapshot Size (bytes): %d\n", info.CSISnapshotInfo.Size) d.Printf("\t\t\t\tCSI Driver: %s\n", info.CSISnapshotInfo.Driver) d.Printf("\t\t\t\tResult: %s\n", info.Result) } else { d.Printf("\t\t\tSnapshot: %s\n", "included, specify --details for more information") } } func describeDataMovement(d *Describer, details bool, info *volume.BackupVolumeInfo) { if !info.SnapshotDataMoved { return } if details { d.Printf("\t\t\tData Movement:\n") d.Printf("\t\t\t\tOperation ID: %s\n", info.SnapshotDataMovementInfo.OperationID) dataMover := "velero" if info.SnapshotDataMovementInfo.DataMover != "" { dataMover = info.SnapshotDataMovementInfo.DataMover } d.Printf("\t\t\t\tData Mover: %s\n", dataMover) d.Printf("\t\t\t\tUploader Type: %s\n", info.SnapshotDataMovementInfo.UploaderType) d.Printf("\t\t\t\tMoved data Size (bytes): %d\n", info.SnapshotDataMovementInfo.Size) if info.SnapshotDataMovementInfo.IncrementalSize > 0 { d.Printf("\t\t\t\tIncremental data Size (bytes): %d\n", info.SnapshotDataMovementInfo.IncrementalSize) } d.Printf("\t\t\t\tResult: %s\n", info.Result) } else { d.Printf("\t\t\tData Movement: %s\n", "included, specify --details for more information") } } func describeBackupItemOperation(d *Describer, operation *itemoperation.BackupOperation) { d.Printf("\tOperation for %s %s/%s:\n", operation.Spec.ResourceIdentifier, operation.Spec.ResourceIdentifier.Namespace, operation.Spec.ResourceIdentifier.Name) d.Printf("\t\tBackup Item Action Plugin:\t%s\n", operation.Spec.BackupItemAction) d.Printf("\t\tOperation ID:\t%s\n", operation.Spec.OperationID) if len(operation.Spec.PostOperationItems) > 0 { d.Printf("\t\tItems to Update:\n") } for _, item := range operation.Spec.PostOperationItems { d.Printf("\t\t\t%s %s/%s\n", item, item.Namespace, item.Name) } d.Printf("\t\tPhase:\t%s\n", operation.Status.Phase) if operation.Status.Error != "" { d.Printf("\t\tOperation Error:\t%s\n", operation.Status.Error) } if operation.Status.NTotal > 0 || operation.Status.NCompleted > 0 { d.Printf("\t\tProgress:\t%v of %v complete (%s)\n", operation.Status.NCompleted, operation.Status.NTotal, operation.Status.OperationUnits) } if operation.Status.Description != "" { d.Printf("\t\tProgress description:\t%s\n", operation.Status.Description) } if operation.Status.Created != nil { d.Printf("\t\tCreated:\t%s\n", operation.Status.Created.String()) } if operation.Status.Started != nil { d.Printf("\t\tStarted:\t%s\n", operation.Status.Started.String()) } if operation.Status.Updated != nil { d.Printf("\t\tUpdated:\t%s\n", operation.Status.Updated.String()) } } // DescribeDeleteBackupRequests describes delete backup requests in human-readable format. func DescribeDeleteBackupRequests(d *Describer, requests []velerov1api.DeleteBackupRequest) { d.Printf("Deletion Attempts") if count := failedDeletionCount(requests); count > 0 { d.Printf(" (%d failed)", count) } d.Println(":") started := false for _, req := range requests { if !started { started = true } else { d.Println() } d.Printf("\t%s: %s\n", req.CreationTimestamp.String(), req.Status.Phase) if len(req.Status.Errors) > 0 { d.Printf("\tErrors:\n") for _, err := range req.Status.Errors { d.Printf("\t\t%s\n", err) } } } } func failedDeletionCount(requests []velerov1api.DeleteBackupRequest) int { var count int for _, req := range requests { if req.Status.Phase == velerov1api.DeleteBackupRequestPhaseProcessed && len(req.Status.Errors) > 0 { count++ } } return count } // describePodVolumeBackups describes pod volume backups in human-readable format. func describePodVolumeBackups(d *Describer, details bool, podVolumeBackups []velerov1api.PodVolumeBackup) { // Get the type of pod volume uploader. Since the uploader only comes from a single source, we can // take the uploader type from the first element of the array. var uploaderType string if len(podVolumeBackups) > 0 { uploaderType = podVolumeBackups[0].Spec.UploaderType } else { d.Printf("\tPod Volume Backups: \n") return } if details { d.Printf("\tPod Volume Backups - %s:\n", uploaderType) } else { d.Printf("\tPod Volume Backups - %s (specify --details for more information):\n", uploaderType) } // separate backups by phase (combining and New into a single group) backupsByPhase := groupByPhase(podVolumeBackups) // go through phases in a specific order for _, phase := range []string{ string(velerov1api.PodVolumeBackupPhaseCompleted), string(velerov1api.PodVolumeBackupPhaseFailed), string(velerov1api.PodVolumeBackupPhaseCanceled), "In Progress", string(velerov1api.PodVolumeBackupPhaseCanceling), string(velerov1api.PodVolumeBackupPhasePrepared), string(velerov1api.PodVolumeBackupPhaseAccepted), string(velerov1api.PodVolumeBackupPhaseNew), } { if len(backupsByPhase[phase]) == 0 { continue } // if we're not printing details, just report the phase and count if !details { d.Printf("\t\t%s:\t%d\n", phase, len(backupsByPhase[phase])) continue } // group the backups in the current phase by pod (i.e. "ns/name") backupsByPod := new(volumesByPod) for _, backup := range backupsByPhase[phase] { backupsByPod.Add(backup.Spec.Pod.Namespace, backup.Spec.Pod.Name, backup.Spec.Volume, phase, backup.Status.Progress, backup.Status.IncrementalBytes) } d.Printf("\t\t%s:\n", phase) for _, backupGroup := range backupsByPod.Sorted() { sort.Strings(backupGroup.volumes) // print volumes backed up for this pod d.Printf("\t\t\t%s: %s\n", backupGroup.label, strings.Join(backupGroup.volumes, ", ")) } } } func groupByPhase(backups []velerov1api.PodVolumeBackup) map[string][]velerov1api.PodVolumeBackup { backupsByPhase := make(map[string][]velerov1api.PodVolumeBackup) phaseToGroup := map[velerov1api.PodVolumeBackupPhase]string{ velerov1api.PodVolumeBackupPhaseCompleted: string(velerov1api.PodVolumeBackupPhaseCompleted), velerov1api.PodVolumeBackupPhaseFailed: string(velerov1api.PodVolumeBackupPhaseFailed), velerov1api.PodVolumeBackupPhaseCanceled: string(velerov1api.PodVolumeBackupPhaseCanceled), velerov1api.PodVolumeBackupPhaseInProgress: "In Progress", velerov1api.PodVolumeBackupPhaseCanceling: string(velerov1api.PodVolumeBackupPhaseCanceling), velerov1api.PodVolumeBackupPhasePrepared: string(velerov1api.PodVolumeBackupPhasePrepared), velerov1api.PodVolumeBackupPhaseAccepted: string(velerov1api.PodVolumeBackupPhaseAccepted), velerov1api.PodVolumeBackupPhaseNew: string(velerov1api.PodVolumeBackupPhaseNew), "": string(velerov1api.PodVolumeBackupPhaseNew), } for _, backup := range backups { group := phaseToGroup[backup.Status.Phase] backupsByPhase[group] = append(backupsByPhase[group], backup) } return backupsByPhase } type podVolumeGroup struct { label string volumes []string } // volumesByPod stores podVolumeGroups, where the grouping // label is "namespace/name". type volumesByPod struct { volumesByPodMap map[string]*podVolumeGroup volumesByPodSlice []*podVolumeGroup } // Add adds a pod volume with the specified pod namespace, name // and volume to the appropriate group. // Used for both backup and restore func (v *volumesByPod) Add(namespace, name, volume, phase string, progress veleroapishared.DataMoveOperationProgress, incrementalBytes int64) { if v.volumesByPodMap == nil { v.volumesByPodMap = make(map[string]*podVolumeGroup) } key := fmt.Sprintf("%s/%s", namespace, name) // append backup progress percentage if backup is in progress if phase == "In Progress" && progress.TotalBytes != 0 { volume = fmt.Sprintf("%s (%.2f%%)", volume, float64(progress.BytesDone)/float64(progress.TotalBytes)*100) } else if phase == string(velerov1api.PodVolumeBackupPhaseCompleted) && incrementalBytes > 0 { volume = fmt.Sprintf("%s (size: %v, incremental size: %v)", volume, progress.TotalBytes, incrementalBytes) } else if (phase == string(velerov1api.PodVolumeBackupPhaseCompleted) || phase == string(velerov1api.PodVolumeRestorePhaseCompleted)) && progress.TotalBytes > 0 { volume = fmt.Sprintf("%s (size: %v)", volume, progress.TotalBytes) } if group, ok := v.volumesByPodMap[key]; !ok { group := &podVolumeGroup{ label: key, volumes: []string{volume}, } v.volumesByPodMap[key] = group v.volumesByPodSlice = append(v.volumesByPodSlice, group) } else { group.volumes = append(group.volumes, volume) } } // Sorted returns a slice of all pod volume groups, ordered by // label. func (v *volumesByPod) Sorted() []*podVolumeGroup { sort.Slice(v.volumesByPodSlice, func(i, j int) bool { return v.volumesByPodSlice[i].label <= v.volumesByPodSlice[j].label }) return v.volumesByPodSlice } // DescribeBackupResults describes errors and warnings in human-readable format. func DescribeBackupResults(ctx context.Context, kbClient kbclient.Client, d *Describer, backup *velerov1api.Backup, insecureSkipTLSVerify bool, caCertPath string) { if backup.Status.Warnings == 0 && backup.Status.Errors == 0 { return } // Get BSL cacert if available bslCACert, err := cacert.GetCACertFromBackup(ctx, kbClient, backup.Namespace, backup) if err != nil { // Log the error but don't fail - we can still try to download without the BSL cacert d.Printf("WARNING: Error getting cacert from BSL: %v\n", err) bslCACert = "" } var buf bytes.Buffer var resultMap map[string]results.Result // If err 'ErrNotFound' occurs, it means the backup bundle in the bucket has already been there before the backup-result file is introduced. // We only display the count of errors and warnings in this case. err = downloadrequest.StreamWithBSLCACert(ctx, kbClient, backup.Namespace, backup.Name, velerov1api.DownloadTargetKindBackupResults, &buf, downloadRequestTimeout, insecureSkipTLSVerify, caCertPath, bslCACert) if err == downloadrequest.ErrNotFound { d.Printf("Errors:\t%d\n", backup.Status.Errors) d.Printf("Warnings:\t%d\n", backup.Status.Warnings) return } else if err != nil { d.Printf("Warnings:\t\n\nErrors:\t\n", err, err) return } if err := json.NewDecoder(&buf).Decode(&resultMap); err != nil { d.Printf("Warnings:\t\n\nErrors:\t\n", err, err) return } if backup.Status.Warnings > 0 { d.Println() describeResult(d, "Warnings", resultMap["warnings"]) } if backup.Status.Errors > 0 { d.Println() describeResult(d, "Errors", resultMap["errors"]) } } ================================================ FILE: pkg/cmd/util/output/backup_describer_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package output import ( "bytes" "testing" "text/tabwriter" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" "github.com/vmware-tanzu/velero/internal/volume" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/itemoperation" ) func TestDescribeUploaderConfig(t *testing.T) { input := builder.ForBackup("test-ns", "test-backup-1").ParallelFilesUpload(10).Result().Spec d := &Describer{ Prefix: "", out: &tabwriter.Writer{}, buf: &bytes.Buffer{}, } d.out.Init(d.buf, 0, 8, 2, ' ', 0) DescribeUploaderConfigForBackup(d, input) d.out.Flush() expect := `Uploader config: Parallel files upload: 10 ` assert.Equal(t, expect, d.buf.String()) } func TestDescribeResourcePolicies(t *testing.T) { input := &corev1api.TypedLocalObjectReference{ Kind: "configmap", Name: "test-resource-policy", } d := &Describer{ Prefix: "", out: &tabwriter.Writer{}, buf: &bytes.Buffer{}, } d.out.Init(d.buf, 0, 8, 2, ' ', 0) DescribeResourcePolicies(d, input) d.out.Flush() expect := `Resource policies: Type: configmap Name: test-resource-policy ` assert.Equal(t, expect, d.buf.String()) } func TestDescribeBackupSpec(t *testing.T) { input1 := builder.ForBackup("test-ns", "test-backup-1"). IncludedNamespaces("inc-ns-1", "inc-ns-2"). ExcludedNamespaces("exc-ns-1", "exc-ns-2"). IncludedResources("inc-res-1", "inc-res-2"). ExcludedResources("exc-res-1", "exc-res-2"). StorageLocation("backup-location"). TTL(72 * time.Hour). CSISnapshotTimeout(10 * time.Minute). DataMover("mover"). Hooks(velerov1api.BackupHooks{ Resources: []velerov1api.BackupResourceHookSpec{ { Name: "hook-1", PreHooks: []velerov1api.BackupResourceHook{ { Exec: &velerov1api.ExecHook{ Container: "hook-container-1", Command: []string{"pre"}, OnError: velerov1api.HookErrorModeContinue, }, }, }, PostHooks: []velerov1api.BackupResourceHook{ { Exec: &velerov1api.ExecHook{ Container: "hook-container-1", Command: []string{"post"}, OnError: velerov1api.HookErrorModeContinue, }, }, }, IncludedNamespaces: []string{"hook-inc-ns-1", "hook-inc-ns-2"}, ExcludedNamespaces: []string{"hook-exc-ns-1", "hook-exc-ns-2"}, IncludedResources: []string{"hook-inc-res-1", "hook-inc-res-2"}, ExcludedResources: []string{"hook-exc-res-1", "hook-exc-res-2"}, }, }, }).Result().Spec expect1 := `Namespaces: Included: inc-ns-1, inc-ns-2 Excluded: exc-ns-1, exc-ns-2 Resources: Included: inc-res-1, inc-res-2 Excluded: exc-res-1, exc-res-2 Cluster-scoped: auto Label selector: Or label selector: Storage Location: backup-location Velero-Native Snapshot PVs: auto Snapshot Move Data: auto Data Mover: mover TTL: 72h0m0s CSISnapshotTimeout: 10m0s ItemOperationTimeout: 0s Hooks: Resources: hook-1: Namespaces: Included: hook-inc-ns-1, hook-inc-ns-2 Excluded: hook-exc-ns-1, hook-exc-ns-2 Resources: Included: hook-inc-res-1, hook-inc-res-2 Excluded: hook-exc-res-1, hook-exc-res-2 Label selector: Pre Exec Hook: Container: hook-container-1 Command: pre On Error: Continue Timeout: 0s Post Exec Hook: Container: hook-container-1 Command: post On Error: Continue Timeout: 0s ` input2 := builder.ForBackup("test-ns", "test-backup-2"). IncludedNamespaces("inc-ns-1", "inc-ns-2"). ExcludedNamespaces("exc-ns-1", "exc-ns-2"). IncludedClusterScopedResources("inc-cluster-res-1", "inc-cluster-res-2"). IncludedNamespaceScopedResources("inc-ns-res-1", "inc-ns-res-2"). ExcludedClusterScopedResources("exc-cluster-res-1", "exc-cluster-res-2"). ExcludedNamespaceScopedResources("exc-ns-res-1", "exc-ns-res-2"). StorageLocation("backup-location"). TTL(72 * time.Hour). CSISnapshotTimeout(10 * time.Minute). DataMover("mover"). Result().Spec expect2 := `Namespaces: Included: inc-ns-1, inc-ns-2 Excluded: exc-ns-1, exc-ns-2 Resources: Included cluster-scoped: inc-cluster-res-1, inc-cluster-res-2 Excluded cluster-scoped: exc-cluster-res-1, exc-cluster-res-2 Included namespace-scoped: inc-ns-res-1, inc-ns-res-2 Excluded namespace-scoped: exc-ns-res-1, exc-ns-res-2 Label selector: Or label selector: Storage Location: backup-location Velero-Native Snapshot PVs: auto Snapshot Move Data: auto Data Mover: mover TTL: 72h0m0s CSISnapshotTimeout: 10m0s ItemOperationTimeout: 0s Hooks: ` input3 := builder.ForBackup("test-ns", "test-backup-3"). StorageLocation("backup-location"). OrderedResources(map[string]string{ "kind1": "rs1-1, rs1-2", }).Hooks(velerov1api.BackupHooks{ Resources: []velerov1api.BackupResourceHookSpec{ { Name: "hook-1", PreHooks: []velerov1api.BackupResourceHook{ { Exec: &velerov1api.ExecHook{ Container: "hook-container-1", Command: []string{"pre"}, OnError: velerov1api.HookErrorModeContinue, }, }, }, PostHooks: []velerov1api.BackupResourceHook{ { Exec: &velerov1api.ExecHook{ Container: "hook-container-1", Command: []string{"post"}, OnError: velerov1api.HookErrorModeContinue, }, }, }, }, }, }).Result().Spec expect3 := `Namespaces: Included: * Excluded: Resources: Included cluster-scoped: Excluded cluster-scoped: Included namespace-scoped: * Excluded namespace-scoped: Label selector: Or label selector: Storage Location: backup-location Velero-Native Snapshot PVs: auto Snapshot Move Data: auto Data Mover: velero TTL: 0s CSISnapshotTimeout: 0s ItemOperationTimeout: 0s Hooks: Resources: hook-1: Namespaces: Included: * Excluded: Resources: Included: * Excluded: Label selector: Pre Exec Hook: Container: hook-container-1 Command: pre On Error: Continue Timeout: 0s Post Exec Hook: Container: hook-container-1 Command: post On Error: Continue Timeout: 0s OrderedResources: kind1: rs1-1, rs1-2 ` input4 := builder.ForBackup("test-ns", "test-backup-4"). DefaultVolumesToFsBackup(true). StorageLocation("backup-location"). Result().Spec expect4 := `Namespaces: Included: * Excluded: Resources: Included cluster-scoped: Excluded cluster-scoped: Included namespace-scoped: * Excluded namespace-scoped: Label selector: Or label selector: Storage Location: backup-location Velero-Native Snapshot PVs: auto File System Backup (Default): true Snapshot Move Data: auto Data Mover: velero TTL: 0s CSISnapshotTimeout: 0s ItemOperationTimeout: 0s Hooks: ` input5 := builder.ForBackup("test-ns", "test-backup-5"). DefaultVolumesToFsBackup(false). StorageLocation("backup-location"). Result().Spec expect5 := `Namespaces: Included: * Excluded: Resources: Included cluster-scoped: Excluded cluster-scoped: Included namespace-scoped: * Excluded namespace-scoped: Label selector: Or label selector: Storage Location: backup-location Velero-Native Snapshot PVs: auto File System Backup (Default): false Snapshot Move Data: auto Data Mover: velero TTL: 0s CSISnapshotTimeout: 0s ItemOperationTimeout: 0s Hooks: ` testcases := []struct { name string input velerov1api.BackupSpec expect string }{ { name: "old resource filter with hooks", input: input1, expect: expect1, }, { name: "new resource filter", input: input2, expect: expect2, }, { name: "old resource filter with hooks and ordered resources", input: input3, expect: expect3, }, { name: "DefaultVolumesToFsBackup is true", input: input4, expect: expect4, }, { name: "DefaultVolumesToFsBackup is false", input: input5, expect: expect5, }, } for _, tc := range testcases { t.Run(tc.name, func(tt *testing.T) { d := &Describer{ Prefix: "", out: &tabwriter.Writer{}, buf: &bytes.Buffer{}, } d.out.Init(d.buf, 0, 8, 2, ' ', 0) DescribeBackupSpec(d, tc.input) d.out.Flush() assert.Equal(tt, tc.expect, d.buf.String()) }) } } func TestDescribeNativeSnapshots(t *testing.T) { testcases := []struct { name string volumeInfo []*volume.BackupVolumeInfo inputDetails bool expect string }{ { name: "no details", volumeInfo: []*volume.BackupVolumeInfo{ { BackupMethod: volume.NativeSnapshot, PVName: "pv-1", NativeSnapshotInfo: &volume.NativeSnapshotInfo{ SnapshotHandle: "snapshot-1", VolumeType: "ebs", VolumeAZ: "us-east-2", IOPS: "1000 mbps", }, }, }, expect: ` Velero-Native Snapshots: pv-1: specify --details for more information `, }, { name: "details", volumeInfo: []*volume.BackupVolumeInfo{ { BackupMethod: volume.NativeSnapshot, PVName: "pv-1", Result: volume.VolumeResultSucceeded, NativeSnapshotInfo: &volume.NativeSnapshotInfo{ SnapshotHandle: "snapshot-1", VolumeType: "ebs", VolumeAZ: "us-east-2", IOPS: "1000 mbps", }, }, }, inputDetails: true, expect: ` Velero-Native Snapshots: pv-1: Snapshot ID: snapshot-1 Type: ebs Availability Zone: us-east-2 IOPS: 1000 mbps Result: succeeded `, }, } for _, tc := range testcases { t.Run(tc.name, func(tt *testing.T) { d := &Describer{ Prefix: "", out: &tabwriter.Writer{}, buf: &bytes.Buffer{}, } d.out.Init(d.buf, 0, 8, 2, ' ', 0) describeNativeSnapshots(d, tc.inputDetails, tc.volumeInfo) d.out.Flush() assert.Equal(t, tc.expect, d.buf.String()) }) } } func TestCSISnapshots(t *testing.T) { testcases := []struct { name string volumeInfo []*volume.BackupVolumeInfo inputDetails bool expect string legacyInfoSource bool }{ { name: "empty info, not legacy", volumeInfo: []*volume.BackupVolumeInfo{}, expect: ` CSI Snapshots: `, }, { name: "empty info, legacy", volumeInfo: []*volume.BackupVolumeInfo{}, legacyInfoSource: true, expect: ` CSI Snapshots: `, }, { name: "no details, local snapshot", volumeInfo: []*volume.BackupVolumeInfo{ { BackupMethod: volume.CSISnapshot, PVCNamespace: "pvc-ns-1", PVCName: "pvc-1", PreserveLocalSnapshot: true, CSISnapshotInfo: &volume.CSISnapshotInfo{ SnapshotHandle: "snapshot-1", Size: 1024, Driver: "fake-driver", VSCName: "vsc-1", OperationID: "fake-operation-1", }, }, }, expect: ` CSI Snapshots: pvc-ns-1/pvc-1: Snapshot: included, specify --details for more information `, }, { name: "details, local snapshot", volumeInfo: []*volume.BackupVolumeInfo{ { BackupMethod: volume.CSISnapshot, PVCNamespace: "pvc-ns-2", PVCName: "pvc-2", PreserveLocalSnapshot: true, Result: volume.VolumeResultSucceeded, CSISnapshotInfo: &volume.CSISnapshotInfo{ SnapshotHandle: "snapshot-2", Size: 1024, Driver: "fake-driver", VSCName: "vsc-2", OperationID: "fake-operation-2", }, }, }, inputDetails: true, expect: ` CSI Snapshots: pvc-ns-2/pvc-2: Snapshot: Operation ID: fake-operation-2 Snapshot Content Name: vsc-2 Storage Snapshot ID: snapshot-2 Snapshot Size (bytes): 1024 CSI Driver: fake-driver Result: succeeded `, }, { name: "no details, data movement", volumeInfo: []*volume.BackupVolumeInfo{ { BackupMethod: volume.CSISnapshot, PVCNamespace: "pvc-ns-3", PVCName: "pvc-3", SnapshotDataMoved: true, SnapshotDataMovementInfo: &volume.SnapshotDataMovementInfo{ DataMover: "velero", UploaderType: "fake-uploader", SnapshotHandle: "fake-repo-id-3", OperationID: "fake-operation-3", }, }, }, expect: ` CSI Snapshots: pvc-ns-3/pvc-3: Data Movement: included, specify --details for more information `, }, { name: "details, data movement", volumeInfo: []*volume.BackupVolumeInfo{ { BackupMethod: volume.CSISnapshot, PVCNamespace: "pvc-ns-4", PVCName: "pvc-4", SnapshotDataMoved: true, Result: volume.VolumeResultSucceeded, SnapshotDataMovementInfo: &volume.SnapshotDataMovementInfo{ DataMover: "velero", UploaderType: "fake-uploader", SnapshotHandle: "fake-repo-id-4", OperationID: "fake-operation-4", }, }, }, inputDetails: true, expect: ` CSI Snapshots: pvc-ns-4/pvc-4: Data Movement: Operation ID: fake-operation-4 Data Mover: velero Uploader Type: fake-uploader Moved data Size (bytes): 0 Result: succeeded `, }, { name: "details, data movement, data mover is empty", volumeInfo: []*volume.BackupVolumeInfo{ { BackupMethod: volume.CSISnapshot, PVCNamespace: "pvc-ns-5", PVCName: "pvc-5", Result: volume.VolumeResultFailed, SnapshotDataMoved: true, SnapshotDataMovementInfo: &volume.SnapshotDataMovementInfo{ UploaderType: "fake-uploader", SnapshotHandle: "fake-repo-id-5", OperationID: "fake-operation-5", Size: 100, IncrementalSize: 50, Phase: velerov2alpha1.DataUploadPhaseFailed, }, }, }, inputDetails: true, expect: ` CSI Snapshots: pvc-ns-5/pvc-5: Data Movement: Operation ID: fake-operation-5 Data Mover: velero Uploader Type: fake-uploader Moved data Size (bytes): 100 Incremental data Size (bytes): 50 Result: failed `, }, } for _, tc := range testcases { t.Run(tc.name, func(tt *testing.T) { d := &Describer{ Prefix: "", out: &tabwriter.Writer{}, buf: &bytes.Buffer{}, } d.out.Init(d.buf, 0, 8, 2, ' ', 0) describeCSISnapshots(d, tc.inputDetails, tc.volumeInfo, tc.legacyInfoSource) d.out.Flush() assert.Equal(t, tc.expect, d.buf.String()) }) } } func TestDescribePodVolumeBackups(t *testing.T) { pvb1 := builder.ForPodVolumeBackup("test-ns", "test-pvb1"). UploaderType("kopia"). Phase(velerov1api.PodVolumeBackupPhaseCompleted). BackupStorageLocation("bsl-1"). Volume("vol-1"). PodName("pod-1"). PodNamespace("pod-ns-1"). SnapshotID("snap-1").Result() pvb2 := builder.ForPodVolumeBackup("test-ns1", "test-pvb2"). UploaderType("kopia"). Phase(velerov1api.PodVolumeBackupPhaseCompleted). BackupStorageLocation("bsl-1"). Volume("vol-2"). PodName("pod-2"). PodNamespace("pod-ns-1"). SnapshotID("snap-2").Result() pvb3 := builder.ForPodVolumeBackup("test-ns1", "test-pvb3"). UploaderType("kopia"). Phase(velerov1api.PodVolumeBackupPhaseFailed). BackupStorageLocation("bsl-1"). Volume("vol-3"). PodName("pod-3"). PodNamespace("pod-ns-1"). SnapshotID("snap-3").Result() pvb4 := builder.ForPodVolumeBackup("test-ns1", "test-pvb4"). UploaderType("kopia"). Phase(velerov1api.PodVolumeBackupPhaseCanceled). BackupStorageLocation("bsl-1"). Volume("vol-4"). PodName("pod-4"). PodNamespace("pod-ns-1"). SnapshotID("snap-4").Result() pvb5 := builder.ForPodVolumeBackup("test-ns1", "test-pvb5"). UploaderType("kopia"). Phase(velerov1api.PodVolumeBackupPhaseInProgress). BackupStorageLocation("bsl-1"). Volume("vol-5"). PodName("pod-5"). PodNamespace("pod-ns-1"). SnapshotID("snap-5").Result() pvb6 := builder.ForPodVolumeBackup("test-ns1", "test-pvb6"). UploaderType("kopia"). Phase(velerov1api.PodVolumeBackupPhaseCanceling). BackupStorageLocation("bsl-1"). Volume("vol-6"). PodName("pod-6"). PodNamespace("pod-ns-1"). SnapshotID("snap-6").Result() pvb7 := builder.ForPodVolumeBackup("test-ns1", "test-pvb7"). UploaderType("kopia"). Phase(velerov1api.PodVolumeBackupPhasePrepared). BackupStorageLocation("bsl-1"). Volume("vol-7"). PodName("pod-7"). PodNamespace("pod-ns-1"). SnapshotID("snap-7").Result() pvb8 := builder.ForPodVolumeBackup("test-ns1", "test-pvb6"). UploaderType("kopia"). Phase(velerov1api.PodVolumeBackupPhaseAccepted). BackupStorageLocation("bsl-1"). Volume("vol-8"). PodName("pod-8"). PodNamespace("pod-ns-1"). SnapshotID("snap-8").Result() testcases := []struct { name string inputPVBList []velerov1api.PodVolumeBackup inputDetails bool expect string }{ { name: "empty list", inputPVBList: []velerov1api.PodVolumeBackup{}, inputDetails: true, expect: ` Pod Volume Backups: `, }, { name: "2 completed pvbs no details", inputPVBList: []velerov1api.PodVolumeBackup{*pvb1, *pvb2}, inputDetails: false, expect: ` Pod Volume Backups - kopia (specify --details for more information): Completed: 2 `, }, { name: "2 completed pvbs with details", inputPVBList: []velerov1api.PodVolumeBackup{*pvb1, *pvb2}, inputDetails: true, expect: ` Pod Volume Backups - kopia: Completed: pod-ns-1/pod-1: vol-1 pod-ns-1/pod-2: vol-2 `, }, { name: "all phases with details", inputPVBList: []velerov1api.PodVolumeBackup{*pvb1, *pvb2, *pvb3, *pvb4, *pvb5, *pvb6, *pvb7, *pvb8}, inputDetails: true, expect: ` Pod Volume Backups - kopia: Completed: pod-ns-1/pod-1: vol-1 pod-ns-1/pod-2: vol-2 Failed: pod-ns-1/pod-3: vol-3 Canceled: pod-ns-1/pod-4: vol-4 In Progress: pod-ns-1/pod-5: vol-5 Canceling: pod-ns-1/pod-6: vol-6 Prepared: pod-ns-1/pod-7: vol-7 Accepted: pod-ns-1/pod-8: vol-8 `, }, } for _, tc := range testcases { t.Run(tc.name, func(tt *testing.T) { d := &Describer{ Prefix: "", out: &tabwriter.Writer{}, buf: &bytes.Buffer{}, } d.out.Init(d.buf, 0, 8, 2, ' ', 0) describePodVolumeBackups(d, tc.inputDetails, tc.inputPVBList) d.out.Flush() assert.Equal(tt, tc.expect, d.buf.String()) }) } } func TestDescribeDeleteBackupRequests(t *testing.T) { t1, err1 := time.Parse("2006-Jan-02", "2023-Jun-26") require.NoError(t, err1) dbr1 := builder.ForDeleteBackupRequest("velero", "dbr1"). ObjectMeta(builder.WithCreationTimestamp(t1)). BackupName("bak-1"). Phase(velerov1api.DeleteBackupRequestPhaseProcessed). Errors("some error").Result() t2, err2 := time.Parse("2006-Jan-02", "2023-Jun-25") require.NoError(t, err2) dbr2 := builder.ForDeleteBackupRequest("velero", "dbr2"). ObjectMeta(builder.WithCreationTimestamp(t2)). BackupName("bak-2"). Phase(velerov1api.DeleteBackupRequestPhaseInProgress).Result() testcases := []struct { name string input []velerov1api.DeleteBackupRequest expect string }{ { name: "empty list", input: []velerov1api.DeleteBackupRequest{}, expect: `Deletion Attempts: `, }, { name: "list with one failed and one in-progress request", input: []velerov1api.DeleteBackupRequest{*dbr1, *dbr2}, expect: `Deletion Attempts (1 failed): 2023-06-26 00:00:00 +0000 UTC: Processed Errors: some error 2023-06-25 00:00:00 +0000 UTC: InProgress `, }, } for _, tc := range testcases { t.Run(tc.name, func(tt *testing.T) { d := &Describer{ Prefix: "", out: &tabwriter.Writer{}, buf: &bytes.Buffer{}, } d.out.Init(d.buf, 0, 8, 2, ' ', 0) DescribeDeleteBackupRequests(d, tc.input) d.out.Flush() assert.Equal(tt, tc.expect, d.buf.String()) }) } } func TestDescribeBackupItemOperation(t *testing.T) { t1, err1 := time.Parse("2006-Jan-02", "2023-Jun-26") require.NoError(t, err1) t2, err2 := time.Parse("2006-Jan-02", "2023-Jun-25") require.NoError(t, err2) t3, err3 := time.Parse("2006-Jan-02", "2023-Jun-24") require.NoError(t, err3) input := builder.ForBackupOperation(). BackupName("backup-1"). OperationID("op-1"). BackupItemAction("action-1"). ResourceIdentifier("group", "rs-type", "ns", "rs-name"). Status(*builder.ForOperationStatus(). Phase(itemoperation.OperationPhaseFailed). Error("operation error"). Progress(50, 100, "bytes"). Description("operation description"). Created(t3). Started(t2). Updated(t1). Result()).Result() expected := ` Operation for rs-type.group ns/rs-name: Backup Item Action Plugin: action-1 Operation ID: op-1 Phase: Failed Operation Error: operation error Progress: 50 of 100 complete (bytes) Progress description: operation description Created: 2023-06-24 00:00:00 +0000 UTC Started: 2023-06-25 00:00:00 +0000 UTC Updated: 2023-06-26 00:00:00 +0000 UTC ` d := &Describer{ Prefix: "", out: &tabwriter.Writer{}, buf: &bytes.Buffer{}, } d.out.Init(d.buf, 0, 8, 2, ' ', 0) describeBackupItemOperation(d, input) d.out.Flush() assert.Equal(t, expected, d.buf.String()) } ================================================ FILE: pkg/cmd/util/output/backup_printer.go ================================================ /* Copyright 2017, 2019, 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package output import ( "fmt" "regexp" "sort" "strconv" "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/duration" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) var ( backupColumns = []metav1.TableColumnDefinition{ // name needs Type and Format defined for the decorator to identify it: // https://github.com/kubernetes/kubernetes/blob/v1.15.3/pkg/printers/tableprinter.go#L204 {Name: "Name", Type: "string", Format: "name"}, {Name: "Status"}, {Name: "Errors"}, {Name: "Warnings"}, {Name: "Created"}, {Name: "Expires"}, {Name: "Storage Location"}, {Name: "Queue Position"}, {Name: "Selector"}, } ) func printBackupList(list *velerov1api.BackupList) []metav1.TableRow { sortBackupsByPrefixAndTimestamp(list) rows := make([]metav1.TableRow, 0, len(list.Items)) for i := range list.Items { rows = append(rows, printBackup(&list.Items[i])...) } return rows } // sort by default alphabetically, but if backups stem from a common schedule // (detected by the presence of a 14-digit timestamp suffix), then within that // group, sort by newest to oldest (i.e. prefix ASC, suffix DESC) var timestampSuffix = regexp.MustCompile("-[0-9]{14}$") func sortBackupsByPrefixAndTimestamp(list *velerov1api.BackupList) { sort.Slice(list.Items, func(i, j int) bool { iSuffixIndex := timestampSuffix.FindStringIndex(list.Items[i].Name) jSuffixIndex := timestampSuffix.FindStringIndex(list.Items[j].Name) // one/both don't have a timestamp suffix, so sort alphabetically if iSuffixIndex == nil || jSuffixIndex == nil { return list.Items[i].Name < list.Items[j].Name } // different prefixes, so sort alphabetically if list.Items[i].Name[0:iSuffixIndex[0]] != list.Items[j].Name[0:jSuffixIndex[0]] { return list.Items[i].Name < list.Items[j].Name } // same prefixes, so sort based on suffix (desc) return list.Items[i].Name[iSuffixIndex[0]:] >= list.Items[j].Name[jSuffixIndex[0]:] }) } func printBackup(backup *velerov1api.Backup) []metav1.TableRow { row := metav1.TableRow{ Object: runtime.RawExtension{Object: backup}, } var expiration time.Time if backup.Status.Expiration != nil { expiration = backup.Status.Expiration.Time } if expiration.IsZero() && backup.Spec.TTL.Duration > 0 { expiration = backup.CreationTimestamp.Add(backup.Spec.TTL.Duration) } status := string(backup.Status.Phase) if status == "" { status = string(velerov1api.BackupPhaseNew) } if backup.DeletionTimestamp != nil && !backup.DeletionTimestamp.Time.IsZero() { status = "Deleting" } row.Cells = append(row.Cells, backup.Name, status, backup.Status.Errors, backup.Status.Warnings, backup.Status.StartTimestamp, humanReadableTimeFromNow(expiration), backup.Spec.StorageLocation, queuePosition(backup.Status.QueuePosition), metav1.FormatLabelSelector(backup.Spec.LabelSelector), ) return []metav1.TableRow{row} } func humanReadableTimeFromNow(when time.Time) string { if when.IsZero() { return "n/a" } now := time.Now() switch { case when == now || when.After(now): return duration.ShortHumanDuration(when.Sub(now)) default: return fmt.Sprintf("%s ago", duration.ShortHumanDuration(now.Sub(when))) } } func queuePosition(pos int) string { if pos == 0 { return "" } else { return strconv.Itoa(pos) } } ================================================ FILE: pkg/cmd/util/output/backup_printer_test.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package output import ( "testing" "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) func TestSortBackups(t *testing.T) { tests := []struct { name string backupList *v1.BackupList expected []v1.Backup }{ { name: "non-timestamped backups", backupList: &v1.BackupList{Items: []v1.Backup{ {ObjectMeta: metav1.ObjectMeta{Name: "a"}}, {ObjectMeta: metav1.ObjectMeta{Name: "c"}}, {ObjectMeta: metav1.ObjectMeta{Name: "b"}}, }}, expected: []v1.Backup{ {ObjectMeta: metav1.ObjectMeta{Name: "a"}}, {ObjectMeta: metav1.ObjectMeta{Name: "b"}}, {ObjectMeta: metav1.ObjectMeta{Name: "c"}}, }, }, { name: "timestamped backups", backupList: &v1.BackupList{Items: []v1.Backup{ {ObjectMeta: metav1.ObjectMeta{Name: "schedule-20170102030405"}}, {ObjectMeta: metav1.ObjectMeta{Name: "schedule-20170102030406"}}, {ObjectMeta: metav1.ObjectMeta{Name: "schedule-20170102030407"}}, }}, expected: []v1.Backup{ {ObjectMeta: metav1.ObjectMeta{Name: "schedule-20170102030407"}}, {ObjectMeta: metav1.ObjectMeta{Name: "schedule-20170102030406"}}, {ObjectMeta: metav1.ObjectMeta{Name: "schedule-20170102030405"}}, }, }, { name: "non-timestamped and timestamped backups", backupList: &v1.BackupList{Items: []v1.Backup{ {ObjectMeta: metav1.ObjectMeta{Name: "schedule-20170102030405"}}, {ObjectMeta: metav1.ObjectMeta{Name: "schedule-20170102030406"}}, {ObjectMeta: metav1.ObjectMeta{Name: "a"}}, {ObjectMeta: metav1.ObjectMeta{Name: "schedule-20170102030407"}}, }}, expected: []v1.Backup{ {ObjectMeta: metav1.ObjectMeta{Name: "a"}}, {ObjectMeta: metav1.ObjectMeta{Name: "schedule-20170102030407"}}, {ObjectMeta: metav1.ObjectMeta{Name: "schedule-20170102030406"}}, {ObjectMeta: metav1.ObjectMeta{Name: "schedule-20170102030405"}}, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { sortBackupsByPrefixAndTimestamp(test.backupList) if assert.Len(t, test.backupList.Items, len(test.expected)) { for i := range test.expected { assert.Equal(t, test.expected[i].Name, test.backupList.Items[i].Name) } } }) } } ================================================ FILE: pkg/cmd/util/output/backup_repo_printer.go ================================================ /* Copyright 2018, 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package output import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) var ( backupRepoColumns = []metav1.TableColumnDefinition{ // name needs Type and Format defined for the decorator to identify it: // https://github.com/kubernetes/kubernetes/blob/v1.15.3/pkg/printers/tableprinter.go#L204 {Name: "Name", Type: "string", Format: "name"}, {Name: "Status"}, {Name: "Last Maintenance"}, } ) func printBackupRepoList(list *v1.BackupRepositoryList) []metav1.TableRow { rows := make([]metav1.TableRow, 0, len(list.Items)) for i := range list.Items { rows = append(rows, printBackupRepo(&list.Items[i])...) } return rows } func printBackupRepo(repo *v1.BackupRepository) []metav1.TableRow { row := metav1.TableRow{ Object: runtime.RawExtension{Object: repo}, } status := repo.Status.Phase if status == "" { status = v1.BackupRepositoryPhaseNew } var lastMaintenance string if repo.Status.LastMaintenanceTime == nil || repo.Status.LastMaintenanceTime.IsZero() { lastMaintenance = "" } else { lastMaintenance = repo.Status.LastMaintenanceTime.String() } row.Cells = append(row.Cells, repo.Name, status, lastMaintenance, ) return []metav1.TableRow{row} } ================================================ FILE: pkg/cmd/util/output/backup_storage_location_printer.go ================================================ /* Copyright 2018, 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package output import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/cmd" ) var ( backupStorageLocationColumns = []metav1.TableColumnDefinition{ // name needs Type and Format defined for the decorator to identify it: // https://github.com/kubernetes/kubernetes/blob/v1.15.3/pkg/printers/tableprinter.go#L204 {Name: "Name", Type: "string", Format: "name"}, {Name: "Provider"}, {Name: "Bucket/Prefix"}, {Name: "Phase"}, {Name: "Last Validated"}, {Name: "Access Mode"}, {Name: "Default"}, } ) func printBackupStorageLocationList(list *velerov1api.BackupStorageLocationList) []metav1.TableRow { rows := make([]metav1.TableRow, 0, len(list.Items)) for i := range list.Items { rows = append(rows, printBackupStorageLocation(&list.Items[i])...) } return rows } func printBackupStorageLocation(location *velerov1api.BackupStorageLocation) []metav1.TableRow { row := metav1.TableRow{ Object: runtime.RawExtension{Object: location}, } isDefault := "" if location.Spec.Default { isDefault = cmd.TRUE } bucketAndPrefix := location.Spec.ObjectStorage.Bucket if location.Spec.ObjectStorage.Prefix != "" { bucketAndPrefix += "/" + location.Spec.ObjectStorage.Prefix } accessMode := location.Spec.AccessMode if accessMode == "" { accessMode = velerov1api.BackupStorageLocationAccessModeReadWrite } status := location.Status.Phase if status == "" { status = "Unknown" } lastValidated := location.Status.LastValidationTime LastValidatedStr := "Unknown" if lastValidated != nil { LastValidatedStr = lastValidated.String() } row.Cells = append(row.Cells, location.Name, location.Spec.Provider, bucketAndPrefix, status, LastValidatedStr, accessMode, isDefault, ) return []metav1.TableRow{row} } ================================================ FILE: pkg/cmd/util/output/backup_structured_describer.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package output import ( "bytes" "context" "encoding/json" "fmt" "strings" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kbclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/internal/volume" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/cmd/util/cacert" "github.com/vmware-tanzu/velero/pkg/cmd/util/downloadrequest" "github.com/vmware-tanzu/velero/pkg/util/results" ) // DescribeBackupInSF describes a backup in structured format. func DescribeBackupInSF( ctx context.Context, kbClient kbclient.Client, backup *velerov1api.Backup, deleteRequests []velerov1api.DeleteBackupRequest, podVolumeBackups []velerov1api.PodVolumeBackup, details bool, insecureSkipTLSVerify bool, caCertFile string, outputFormat string, ) string { return DescribeInSF(func(d *StructuredDescriber) { d.DescribeMetadata(backup.ObjectMeta) d.Describe("phase", backup.Status.Phase) if backup.Spec.ResourcePolicy != nil { DescribeResourcePoliciesInSF(d, backup.Spec.ResourcePolicy) } status := backup.Status if len(status.ValidationErrors) > 0 { d.Describe("validationErrors", status.ValidationErrors) } DescribeBackupResultsInSF(ctx, kbClient, d, backup, insecureSkipTLSVerify, caCertFile) DescribeBackupSpecInSF(d, backup.Spec) DescribeBackupStatusInSF(ctx, kbClient, d, backup, details, insecureSkipTLSVerify, caCertFile, podVolumeBackups) if len(deleteRequests) > 0 { DescribeDeleteBackupRequestsInSF(d, deleteRequests) } }, outputFormat) } // DescribeBackupSpecInSF describes a backup spec in structured format. func DescribeBackupSpecInSF(d *StructuredDescriber, spec velerov1api.BackupSpec) { backupSpecInfo := make(map[string]any) var s string // describe namespaces namespaceInfo := make(map[string]any) if len(spec.IncludedNamespaces) == 0 { s = "*" } else { s = strings.Join(spec.IncludedNamespaces, ", ") } namespaceInfo["included"] = s if len(spec.ExcludedNamespaces) == 0 { s = emptyDisplay } else { s = strings.Join(spec.ExcludedNamespaces, ", ") } namespaceInfo["excluded"] = s backupSpecInfo["namespaces"] = namespaceInfo // describe resources resourcesInfo := make(map[string]string) if len(spec.IncludedResources) == 0 { s = "*" } else { s = strings.Join(spec.IncludedResources, ", ") } resourcesInfo["included"] = s if len(spec.ExcludedResources) == 0 { s = emptyDisplay } else { s = strings.Join(spec.ExcludedResources, ", ") } resourcesInfo["excluded"] = s resourcesInfo["clusterScoped"] = BoolPointerString(spec.IncludeClusterResources, "excluded", "included", "auto") backupSpecInfo["resources"] = resourcesInfo // describe label selector s = emptyDisplay if spec.LabelSelector != nil { s = metav1.FormatLabelSelector(spec.LabelSelector) } backupSpecInfo["labelSelector"] = s // describe storage location backupSpecInfo["storageLocation"] = spec.StorageLocation // describe snapshot volumes backupSpecInfo["veleroNativeSnapshotPVs"] = BoolPointerString(spec.SnapshotVolumes, "false", "true", "auto") // describe snapshot move data backupSpecInfo["veleroSnapshotMoveData"] = BoolPointerString(spec.SnapshotMoveData, "false", "true", "auto") // describe data mover if len(spec.DataMover) == 0 { s = emptyDisplay } else { s = spec.DataMover } backupSpecInfo["dataMover"] = s // describe TTL backupSpecInfo["TTL"] = spec.TTL.Duration.String() // describe CSI snapshot timeout backupSpecInfo["CSISnapshotTimeout"] = spec.CSISnapshotTimeout.Duration.String() // describe hooks hooksInfo := make(map[string]any) hooksResources := make(map[string]any) for _, backupResourceHookSpec := range spec.Hooks.Resources { ResourceDetails := make(map[string]any) var s string namespaceInfo := make(map[string]string) if len(backupResourceHookSpec.IncludedNamespaces) == 0 { s = "*" } else { s = strings.Join(backupResourceHookSpec.IncludedNamespaces, ", ") } namespaceInfo["included"] = s if len(backupResourceHookSpec.ExcludedNamespaces) == 0 { s = emptyDisplay } else { s = strings.Join(backupResourceHookSpec.ExcludedNamespaces, ", ") } namespaceInfo["excluded"] = s ResourceDetails["namespaces"] = namespaceInfo resourcesInfo := make(map[string]string) if len(backupResourceHookSpec.IncludedResources) == 0 { s = "*" } else { s = strings.Join(backupResourceHookSpec.IncludedResources, ", ") } resourcesInfo["included"] = s if len(backupResourceHookSpec.ExcludedResources) == 0 { s = emptyDisplay } else { s = strings.Join(backupResourceHookSpec.ExcludedResources, ", ") } resourcesInfo["excluded"] = s ResourceDetails["resources"] = resourcesInfo s = emptyDisplay if backupResourceHookSpec.LabelSelector != nil { s = metav1.FormatLabelSelector(backupResourceHookSpec.LabelSelector) } ResourceDetails["labelSelector"] = s preHooks := make([]map[string]any, 0) for _, hook := range backupResourceHookSpec.PreHooks { if hook.Exec != nil { preExecHook := make(map[string]any) preExecHook["container"] = hook.Exec.Container preExecHook["command"] = strings.Join(hook.Exec.Command, " ") preExecHook["onError:"] = hook.Exec.OnError preExecHook["timeout"] = hook.Exec.Timeout.Duration.String() preHooks = append(preHooks, preExecHook) } } ResourceDetails["preExecHook"] = preHooks postHooks := make([]map[string]any, 0) for _, hook := range backupResourceHookSpec.PostHooks { if hook.Exec != nil { postExecHook := make(map[string]any) postExecHook["container"] = hook.Exec.Container postExecHook["command"] = strings.Join(hook.Exec.Command, " ") postExecHook["onError:"] = hook.Exec.OnError postExecHook["timeout"] = hook.Exec.Timeout.Duration.String() postHooks = append(postHooks, postExecHook) } } ResourceDetails["postExecHook"] = postHooks hooksResources[backupResourceHookSpec.Name] = ResourceDetails } if len(spec.Hooks.Resources) > 0 { hooksInfo["resources"] = hooksResources backupSpecInfo["hooks"] = hooksInfo } // desrcibe ordered resources if spec.OrderedResources != nil { backupSpecInfo["orderedResources"] = spec.OrderedResources } d.Describe("spec", backupSpecInfo) } // DescribeBackupStatusInSF describes a backup status in structured format. func DescribeBackupStatusInSF(ctx context.Context, kbClient kbclient.Client, d *StructuredDescriber, backup *velerov1api.Backup, details bool, insecureSkipTLSVerify bool, caCertPath string, podVolumeBackups []velerov1api.PodVolumeBackup) { status := backup.Status backupStatusInfo := make(map[string]any) // Status.Version has been deprecated, use Status.FormatVersion backupStatusInfo["backupFormatVersion"] = status.FormatVersion // "" output should only be applicable for backups that failed validation if status.StartTimestamp == nil || status.StartTimestamp.Time.IsZero() { backupStatusInfo["started"] = "" } else { backupStatusInfo["started"] = status.StartTimestamp.Time.String() } if status.CompletionTimestamp == nil || status.CompletionTimestamp.Time.IsZero() { backupStatusInfo["completed"] = "" } else { backupStatusInfo["completed"] = status.CompletionTimestamp.Time.String() } // Expiration can't be 0, it is always set to a 30-day default. It can be nil // if the controller hasn't processed this Backup yet, in which case this will // just display ``, though this should be temporary. backupStatusInfo["expiration"] = status.Expiration.String() defer d.Describe("status", backupStatusInfo) if backup.Status.Progress != nil { if backup.Status.Phase == velerov1api.BackupPhaseInProgress { backupStatusInfo["estimatedTotalItemsToBeBackedUp"] = backup.Status.Progress.TotalItems backupStatusInfo["itemsBackedUpSoFar"] = backup.Status.Progress.ItemsBackedUp } else { backupStatusInfo["totalItemsToBeBackedUp"] = backup.Status.Progress.TotalItems backupStatusInfo["itemsBackedUp"] = backup.Status.Progress.ItemsBackedUp } } if details { describeBackupResourceListInSF(ctx, kbClient, backupStatusInfo, backup, insecureSkipTLSVerify, caCertPath) } describeBackupVolumesInSF(ctx, kbClient, backup, details, insecureSkipTLSVerify, caCertPath, podVolumeBackups, backupStatusInfo) if status.HookStatus != nil { backupStatusInfo["hooksAttempted"] = status.HookStatus.HooksAttempted backupStatusInfo["hooksFailed"] = status.HookStatus.HooksFailed } } func describeBackupResourceListInSF(ctx context.Context, kbClient kbclient.Client, backupStatusInfo map[string]any, backup *velerov1api.Backup, insecureSkipTLSVerify bool, caCertPath string) { // Get BSL cacert if available bslCACert, err := cacert.GetCACertFromBackup(ctx, kbClient, backup.Namespace, backup) if err != nil { // Log the error but don't fail - we can still try to download without the BSL cacert backupStatusInfo["warningGettingBSLCACert"] = fmt.Sprintf("Warning: Error getting cacert from BSL: %v", err) bslCACert = "" } // In consideration of decoding structured output conveniently, the two separate fields were created here(in func describeBackupResourceList, there is only one field describing either error message or resource list) // the field of 'errorGettingResourceList' gives specific error message when it fails to get resources list // the field of 'resourceList' lists the rearranged resources buf := new(bytes.Buffer) if err := downloadrequest.StreamWithBSLCACert(ctx, kbClient, backup.Namespace, backup.Name, velerov1api.DownloadTargetKindBackupResourceList, buf, downloadRequestTimeout, insecureSkipTLSVerify, caCertPath, bslCACert); err != nil { if err == downloadrequest.ErrNotFound { // the backup resource list could be missing if (other reasons may exist as well): // - the backup was taken prior to v1.1; or // - the backup hasn't completed yet; or // - there was an error uploading the file; or // - the file was manually deleted after upload backupStatusInfo["errorGettingResourceList"] = "" } else { backupStatusInfo["errorGettingResourceList"] = fmt.Sprintf("", err) } return } var resourceList map[string][]string if err := json.NewDecoder(buf).Decode(&resourceList); err != nil { backupStatusInfo["errorGettingResourceList"] = fmt.Sprintf("\n", err) return } backupStatusInfo["resourceList"] = resourceList } func describeBackupVolumesInSF(ctx context.Context, kbClient kbclient.Client, backup *velerov1api.Backup, details bool, insecureSkipTLSVerify bool, caCertPath string, podVolumeBackupCRs []velerov1api.PodVolumeBackup, backupStatusInfo map[string]any) { backupVolumes := make(map[string]any) // Get BSL cacert if available bslCACert, err := cacert.GetCACertFromBackup(ctx, kbClient, backup.Namespace, backup) if err != nil { // Log the error but don't fail - we can still try to download without the BSL cacert backupVolumes["warningGettingBSLCACert"] = fmt.Sprintf("Warning: Error getting cacert from BSL: %v", err) bslCACert = "" } nativeSnapshots := []*volume.BackupVolumeInfo{} csiSnapshots := []*volume.BackupVolumeInfo{} legacyInfoSource := false buf := new(bytes.Buffer) err = downloadrequest.StreamWithBSLCACert(ctx, kbClient, backup.Namespace, backup.Name, velerov1api.DownloadTargetKindBackupVolumeInfos, buf, downloadRequestTimeout, insecureSkipTLSVerify, caCertPath, bslCACert) if err == downloadrequest.ErrNotFound { nativeSnapshots, err = retrieveNativeSnapshotLegacy(ctx, kbClient, backup, insecureSkipTLSVerify, caCertPath, bslCACert) if err != nil { backupVolumes["errorConcludeNativeSnapshot"] = fmt.Sprintf("error concluding native snapshot info: %v", err) return } csiSnapshots, err = retrieveCSISnapshotLegacy(ctx, kbClient, backup, insecureSkipTLSVerify, caCertPath, bslCACert) if err != nil { backupVolumes["errorConcludeCSISnapshot"] = fmt.Sprintf("error concluding CSI snapshot info: %v", err) return } legacyInfoSource = true } else if err != nil { backupVolumes["errorGetBackupVolumeInfo"] = fmt.Sprintf("error getting backup volume info: %v", err) return } else { var volumeInfos []volume.BackupVolumeInfo if err := json.NewDecoder(buf).Decode(&volumeInfos); err != nil { backupVolumes["errorReadBackupVolumeInfo"] = fmt.Sprintf("error reading backup volume info: %v", err) return } for i := range volumeInfos { switch volumeInfos[i].BackupMethod { case volume.NativeSnapshot: nativeSnapshots = append(nativeSnapshots, &volumeInfos[i]) case volume.CSISnapshot: csiSnapshots = append(csiSnapshots, &volumeInfos[i]) } } } describeNativeSnapshotsInSF(details, nativeSnapshots, backupVolumes) describeCSISnapshotsInSF(details, csiSnapshots, backupVolumes, legacyInfoSource) describePodVolumeBackupsInSF(podVolumeBackupCRs, details, backupVolumes) backupStatusInfo["backupVolumes"] = backupVolumes } func describeNativeSnapshotsInSF(details bool, infos []*volume.BackupVolumeInfo, backupVolumes map[string]any) { if len(infos) == 0 { backupVolumes["nativeSnapshots"] = "" return } snapshotDetails := make(map[string]any) for _, info := range infos { describNativeSnapshotInSF(details, info, snapshotDetails) } backupVolumes["nativeSnapshots"] = snapshotDetails } func describNativeSnapshotInSF(details bool, info *volume.BackupVolumeInfo, snapshotDetails map[string]any) { if details { snapshotInfo := make(map[string]string) snapshotInfo["snapshotID"] = info.NativeSnapshotInfo.SnapshotHandle snapshotInfo["type"] = info.NativeSnapshotInfo.VolumeType snapshotInfo["availabilityZone"] = info.NativeSnapshotInfo.VolumeAZ snapshotInfo["IOPS"] = info.NativeSnapshotInfo.IOPS snapshotInfo["result"] = string(info.Result) snapshotDetails[info.PVName] = snapshotInfo } else { snapshotDetails[info.PVName] = "specify --details for more information" } } func describeCSISnapshotsInSF(details bool, infos []*volume.BackupVolumeInfo, backupVolumes map[string]any, legacyInfoSource bool) { if len(infos) == 0 { if legacyInfoSource { backupVolumes["csiSnapshots"] = "" } else { backupVolumes["csiSnapshots"] = "" } return } snapshotDetails := make(map[string]any) for _, info := range infos { describeCSISnapshotInSF(details, info, snapshotDetails) } backupVolumes["csiSnapshots"] = snapshotDetails } func describeCSISnapshotInSF(details bool, info *volume.BackupVolumeInfo, snapshotDetails map[string]any) { snapshotDetail := make(map[string]any) describeLocalSnapshotInSF(details, info, snapshotDetail) describeDataMovementInSF(details, info, snapshotDetail) snapshotDetails[fmt.Sprintf("%s/%s", info.PVCNamespace, info.PVCName)] = snapshotDetail } // describeLocalSnapshotInSF describes CSI volume snapshot contents in structured format. func describeLocalSnapshotInSF(details bool, info *volume.BackupVolumeInfo, snapshotDetail map[string]any) { if !info.PreserveLocalSnapshot { return } if details { localSnapshot := make(map[string]any) if !info.SnapshotDataMoved { localSnapshot["operationID"] = info.CSISnapshotInfo.OperationID } localSnapshot["snapshotContentName"] = info.CSISnapshotInfo.VSCName localSnapshot["storageSnapshotID"] = info.CSISnapshotInfo.SnapshotHandle localSnapshot["snapshotSize(bytes)"] = info.CSISnapshotInfo.Size localSnapshot["csiDriver"] = info.CSISnapshotInfo.Driver localSnapshot["result"] = string(info.Result) snapshotDetail["snapshot"] = localSnapshot } else { snapshotDetail["snapshot"] = "included, specify --details for more information" } } func describeDataMovementInSF(details bool, info *volume.BackupVolumeInfo, snapshotDetail map[string]any) { if !info.SnapshotDataMoved { return } if details { dataMovement := make(map[string]any) dataMovement["operationID"] = info.SnapshotDataMovementInfo.OperationID dataMover := "velero" if info.SnapshotDataMovementInfo.DataMover != "" { dataMover = info.SnapshotDataMovementInfo.DataMover } dataMovement["dataMover"] = dataMover dataMovement["uploaderType"] = info.SnapshotDataMovementInfo.UploaderType dataMovement["result"] = string(info.Result) if info.SnapshotDataMovementInfo.Size > 0 || info.SnapshotDataMovementInfo.IncrementalSize > 0 { dataMovement["size"] = info.SnapshotDataMovementInfo.Size dataMovement["incrementalSize"] = info.SnapshotDataMovementInfo.IncrementalSize } snapshotDetail["dataMovement"] = dataMovement } else { snapshotDetail["dataMovement"] = "included, specify --details for more information" } } // DescribeDeleteBackupRequestsInSF describes delete backup requests in structured format. func DescribeDeleteBackupRequestsInSF(d *StructuredDescriber, requests []velerov1api.DeleteBackupRequest) { deletionAttempts := make(map[string]any) if count := failedDeletionCount(requests); count > 0 { deletionAttempts["failed"] = count } deletionRequests := make([]map[string]any, 0) for _, req := range requests { deletionReq := make(map[string]any) deletionReq["creationTimestamp"] = req.CreationTimestamp.String() deletionReq["phase"] = req.Status.Phase if len(req.Status.Errors) > 0 { deletionReq["errors"] = req.Status.Errors } deletionRequests = append(deletionRequests, deletionReq) } deletionAttempts["deleteBackupRequests"] = deletionRequests d.Describe("deletionAttempts", deletionAttempts) } // describePodVolumeBackupsInSF describes pod volume backups in structured format. func describePodVolumeBackupsInSF(backups []velerov1api.PodVolumeBackup, details bool, backupVolumes map[string]any) { podVolumeBackupsInfo := make(map[string]any) // Get the type of pod volume uploader. Since the uploader only comes from a single source, we can // take the uploader type from the first element of the array. var uploaderType string if len(backups) > 0 { uploaderType = backups[0].Spec.UploaderType } else { backupVolumes["podVolumeBackups"] = "" return } // type display the type of pod volume backups podVolumeBackupsInfo["uploderType"] = uploaderType podVolumeBackupsDetails := make(map[string]any) // separate backups by phase (combining and New into a single group) backupsByPhase := groupByPhase(backups) // go through phases in a specific order for _, phase := range []string{ string(velerov1api.PodVolumeBackupPhaseCompleted), string(velerov1api.PodVolumeBackupPhaseFailed), string(velerov1api.PodVolumeBackupPhaseCanceled), "In Progress", string(velerov1api.PodVolumeBackupPhaseCanceling), string(velerov1api.PodVolumeBackupPhasePrepared), string(velerov1api.PodVolumeBackupPhaseAccepted), string(velerov1api.PodVolumeBackupPhaseNew), } { if len(backupsByPhase[phase]) == 0 { continue } // if we're not printing details, just report the phase and count if !details { podVolumeBackupsDetails[phase] = len(backupsByPhase[phase]) continue } // group the backups in the current phase by pod (i.e. "ns/name") backupsByPod := new(volumesByPod) for _, backup := range backupsByPhase[phase] { backupsByPod.Add(backup.Spec.Pod.Namespace, backup.Spec.Pod.Name, backup.Spec.Volume, phase, backup.Status.Progress, backup.Status.IncrementalBytes) } backupsByPods := make([]map[string]string, 0) for _, backupGroup := range backupsByPod.volumesByPodSlice { // print volumes backed up for this pod backupsByPods = append(backupsByPods, map[string]string{backupGroup.label: strings.Join(backupGroup.volumes, ", ")}) } podVolumeBackupsDetails[phase] = backupsByPods } // Pod Volume Backups Details display the detailed pod volume backups info podVolumeBackupsInfo["podVolumeBackupsDetails"] = podVolumeBackupsDetails backupVolumes["podVolumeBackups"] = podVolumeBackupsInfo } // DescribeBackupResultsInSF describes errors and warnings in structured format. func DescribeBackupResultsInSF(ctx context.Context, kbClient kbclient.Client, d *StructuredDescriber, backup *velerov1api.Backup, insecureSkipTLSVerify bool, caCertPath string) { if backup.Status.Warnings == 0 && backup.Status.Errors == 0 { return } // Get BSL cacert if available bslCACert, err := cacert.GetCACertFromBackup(ctx, kbClient, backup.Namespace, backup) if err != nil { // Log the error but don't fail - we can still try to download without the BSL cacert warnings := make(map[string]any) warnings["warningGettingBSLCACert"] = fmt.Sprintf("Warning: Error getting cacert from BSL: %v", err) d.Describe("warningsGettingBSLCACert", warnings) bslCACert = "" } var buf bytes.Buffer var resultMap map[string]results.Result errors, warnings := make(map[string]any), make(map[string]any) defer func() { d.Describe("errors", errors) d.Describe("warnings", warnings) }() // If 'ErrNotFound' occurs, it means the backup bundle in the bucket has already been there before the backup-result file is introduced. // We only display the count of errors and warnings in this case. err = downloadrequest.StreamWithBSLCACert(ctx, kbClient, backup.Namespace, backup.Name, velerov1api.DownloadTargetKindBackupResults, &buf, downloadRequestTimeout, insecureSkipTLSVerify, caCertPath, bslCACert) if err == downloadrequest.ErrNotFound { errors["count"] = backup.Status.Errors warnings["count"] = backup.Status.Warnings return } else if err != nil { errors["errorGettingErrors"] = fmt.Errorf("", err) warnings["errorGettingWarnings"] = fmt.Errorf("", err) return } if err := json.NewDecoder(&buf).Decode(&resultMap); err != nil { errors["errorGettingErrors"] = fmt.Errorf("", err) warnings["errorGettingWarnings"] = fmt.Errorf("", err) return } if backup.Status.Warnings > 0 { describeResultInSF(warnings, resultMap["warnings"]) } if backup.Status.Errors > 0 { describeResultInSF(errors, resultMap["errors"]) } } // DescribeResourcePoliciesInSF describes resource policies in structured format. func DescribeResourcePoliciesInSF(d *StructuredDescriber, resPolicies *corev1api.TypedLocalObjectReference) { policiesInfo := make(map[string]any) policiesInfo["type"] = resPolicies.Kind policiesInfo["name"] = resPolicies.Name d.Describe("resourcePolicies", policiesInfo) } func describeResultInSF(m map[string]any, result results.Result) { m["velero"], m["cluster"], m["namespace"] = []string{}, []string{}, []string{} if len(result.Velero) > 0 { m["velero"] = result.Velero } if len(result.Cluster) > 0 { m["cluster"] = result.Cluster } if len(result.Namespaces) > 0 { m["namespace"] = result.Namespaces } } ================================================ FILE: pkg/cmd/util/output/backup_structured_describer_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package output import ( "reflect" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" "github.com/vmware-tanzu/velero/internal/volume" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/util/results" ) func TestDescribeBackupInSF(t *testing.T) { sd := &StructuredDescriber{ output: make(map[string]any), format: "", } backupBuilder1 := builder.ForBackup("test-ns", "test-backup") backupBuilder1.IncludedNamespaces("inc-ns-1", "inc-ns-2"). ExcludedNamespaces("exc-ns-1", "exc-ns-2"). IncludedResources("inc-res-1", "inc-res-2"). ExcludedResources("exc-res-1", "exc-res-2"). StorageLocation("backup-location"). TTL(72 * time.Hour). CSISnapshotTimeout(10 * time.Minute). DataMover("mover"). Hooks(velerov1api.BackupHooks{ Resources: []velerov1api.BackupResourceHookSpec{ { Name: "hook-1", PreHooks: []velerov1api.BackupResourceHook{ { Exec: &velerov1api.ExecHook{ Container: "hook-container-1", Command: []string{"pre"}, OnError: velerov1api.HookErrorModeContinue, }, }, }, PostHooks: []velerov1api.BackupResourceHook{ { Exec: &velerov1api.ExecHook{ Container: "hook-container-1", Command: []string{"post"}, OnError: velerov1api.HookErrorModeContinue, }, }, }, IncludedNamespaces: []string{"hook-inc-ns-1", "hook-inc-ns-2"}, ExcludedNamespaces: []string{"hook-exc-ns-1", "hook-exc-ns-2"}, IncludedResources: []string{"hook-inc-res-1", "hook-inc-res-2"}, ExcludedResources: []string{"hook-exc-res-1", "hook-exc-res-2"}, }, }, }) expect1 := map[string]any{ "spec": map[string]any{ "namespaces": map[string]any{ "included": "inc-ns-1, inc-ns-2", "excluded": "exc-ns-1, exc-ns-2", }, "resources": map[string]string{ "included": "inc-res-1, inc-res-2", "excluded": "exc-res-1, exc-res-2", "clusterScoped": "auto", }, "dataMover": "mover", "labelSelector": emptyDisplay, "storageLocation": "backup-location", "veleroNativeSnapshotPVs": "auto", "TTL": "72h0m0s", "CSISnapshotTimeout": "10m0s", "veleroSnapshotMoveData": "auto", "hooks": map[string]any{ "resources": map[string]any{ "hook-1": map[string]any{ "labelSelector": emptyDisplay, "namespaces": map[string]string{ "included": "hook-inc-ns-1, hook-inc-ns-2", "excluded": "hook-exc-ns-1, hook-exc-ns-2", }, "preExecHook": []map[string]any{ { "container": "hook-container-1", "command": "pre", "onError:": velerov1api.HookErrorModeContinue, "timeout": "0s", }, }, "postExecHook": []map[string]any{ { "container": "hook-container-1", "command": "post", "onError:": velerov1api.HookErrorModeContinue, "timeout": "0s", }, }, "resources": map[string]string{ "included": "hook-inc-res-1, hook-inc-res-2", "excluded": "hook-exc-res-1, hook-exc-res-2", }, }, }, }, }, } DescribeBackupSpecInSF(sd, backupBuilder1.Result().Spec) assert.True(t, reflect.DeepEqual(sd.output, expect1)) backupBuilder2 := builder.ForBackup("test-ns-2", "test-backup-2"). StorageLocation("backup-location"). OrderedResources(map[string]string{ "kind1": "rs1-1, rs1-2", "kind2": "rs2-1, rs2-2", }).Hooks(velerov1api.BackupHooks{ Resources: []velerov1api.BackupResourceHookSpec{ { Name: "hook-1", PreHooks: []velerov1api.BackupResourceHook{ { Exec: &velerov1api.ExecHook{ Container: "hook-container-1", Command: []string{"pre"}, OnError: velerov1api.HookErrorModeContinue, }, }, }, PostHooks: []velerov1api.BackupResourceHook{ { Exec: &velerov1api.ExecHook{ Container: "hook-container-1", Command: []string{"post"}, OnError: velerov1api.HookErrorModeContinue, }, }, }, }, }, }) expect2 := map[string]any{ "spec": map[string]any{ "namespaces": map[string]any{ "included": "*", "excluded": emptyDisplay, }, "resources": map[string]string{ "included": "*", "excluded": emptyDisplay, "clusterScoped": "auto", }, "dataMover": emptyDisplay, "labelSelector": emptyDisplay, "storageLocation": "backup-location", "veleroNativeSnapshotPVs": "auto", "TTL": "0s", "CSISnapshotTimeout": "0s", "veleroSnapshotMoveData": "auto", "hooks": map[string]any{ "resources": map[string]any{ "hook-1": map[string]any{ "labelSelector": emptyDisplay, "namespaces": map[string]string{ "included": "*", "excluded": emptyDisplay, }, "preExecHook": []map[string]any{ { "container": "hook-container-1", "command": "pre", "onError:": velerov1api.HookErrorModeContinue, "timeout": "0s", }, }, "postExecHook": []map[string]any{ { "container": "hook-container-1", "command": "post", "onError:": velerov1api.HookErrorModeContinue, "timeout": "0s", }, }, "resources": map[string]string{ "included": "*", "excluded": emptyDisplay, }, }, }, }, "orderedResources": map[string]string{ "kind1": "rs1-1, rs1-2", "kind2": "rs2-1, rs2-2", }, }, } DescribeBackupSpecInSF(sd, backupBuilder2.Result().Spec) assert.True(t, reflect.DeepEqual(sd.output, expect2)) } func TestDescribePodVolumeBackupsInSF(t *testing.T) { pvbBuilder1 := builder.ForPodVolumeBackup("test-ns1", "test-pvb1") pvb1 := pvbBuilder1.BackupStorageLocation("backup-location"). UploaderType("kopia"). Phase(velerov1api.PodVolumeBackupPhaseCompleted). BackupStorageLocation("bsl-1"). Volume("vol-1"). PodName("pod-1"). PodNamespace("pod-ns-1"). SnapshotID("snap-1").Result() pvbBuilder2 := builder.ForPodVolumeBackup("test-ns1", "test-pvb2") pvb2 := pvbBuilder2.BackupStorageLocation("backup-location"). UploaderType("kopia"). Phase(velerov1api.PodVolumeBackupPhaseCompleted). BackupStorageLocation("bsl-1"). Volume("vol-2"). PodName("pod-2"). PodNamespace("pod-ns-1"). SnapshotID("snap-2").Result() pvb3 := builder.ForPodVolumeBackup("test-ns1", "test-pvb3"). UploaderType("kopia"). Phase(velerov1api.PodVolumeBackupPhaseFailed). BackupStorageLocation("bsl-1"). Volume("vol-3"). PodName("pod-3"). PodNamespace("pod-ns-1"). SnapshotID("snap-3").Result() pvb4 := builder.ForPodVolumeBackup("test-ns1", "test-pvb4"). UploaderType("kopia"). Phase(velerov1api.PodVolumeBackupPhaseCanceled). BackupStorageLocation("bsl-1"). Volume("vol-4"). PodName("pod-4"). PodNamespace("pod-ns-1"). SnapshotID("snap-4").Result() pvb5 := builder.ForPodVolumeBackup("test-ns1", "test-pvb5"). UploaderType("kopia"). Phase(velerov1api.PodVolumeBackupPhaseInProgress). BackupStorageLocation("bsl-1"). Volume("vol-5"). PodName("pod-5"). PodNamespace("pod-ns-1"). SnapshotID("snap-5").Result() pvb6 := builder.ForPodVolumeBackup("test-ns1", "test-pvb6"). UploaderType("kopia"). Phase(velerov1api.PodVolumeBackupPhaseCanceling). BackupStorageLocation("bsl-1"). Volume("vol-6"). PodName("pod-6"). PodNamespace("pod-ns-1"). SnapshotID("snap-6").Result() pvb7 := builder.ForPodVolumeBackup("test-ns1", "test-pvb7"). UploaderType("kopia"). Phase(velerov1api.PodVolumeBackupPhasePrepared). BackupStorageLocation("bsl-1"). Volume("vol-7"). PodName("pod-7"). PodNamespace("pod-ns-1"). SnapshotID("snap-7").Result() pvb8 := builder.ForPodVolumeBackup("test-ns1", "test-pvb6"). UploaderType("kopia"). Phase(velerov1api.PodVolumeBackupPhaseAccepted). BackupStorageLocation("bsl-1"). Volume("vol-8"). PodName("pod-8"). PodNamespace("pod-ns-1"). SnapshotID("snap-8").Result() testcases := []struct { name string inputPVBList []velerov1api.PodVolumeBackup inputDetails bool expect map[string]any }{ { name: "empty list", inputPVBList: []velerov1api.PodVolumeBackup{}, inputDetails: false, expect: map[string]any{"podVolumeBackups": ""}, }, { name: "2 completed pvbs", inputPVBList: []velerov1api.PodVolumeBackup{*pvb1, *pvb2}, inputDetails: true, expect: map[string]any{ "podVolumeBackups": map[string]any{ "podVolumeBackupsDetails": map[string]any{ "Completed": []map[string]string{ {"pod-ns-1/pod-1": "vol-1"}, {"pod-ns-1/pod-2": "vol-2"}, }, }, "uploderType": "kopia", }, }, }, { name: "all phases", inputPVBList: []velerov1api.PodVolumeBackup{*pvb1, *pvb2, *pvb3, *pvb4, *pvb5, *pvb6, *pvb7, *pvb8}, inputDetails: true, expect: map[string]any{ "podVolumeBackups": map[string]any{ "podVolumeBackupsDetails": map[string]any{ "Completed": []map[string]string{ {"pod-ns-1/pod-1": "vol-1"}, {"pod-ns-1/pod-2": "vol-2"}, }, "Failed": []map[string]string{ {"pod-ns-1/pod-3": "vol-3"}, }, "Canceled": []map[string]string{ {"pod-ns-1/pod-4": "vol-4"}, }, "In Progress": []map[string]string{ {"pod-ns-1/pod-5": "vol-5"}, }, "Canceling": []map[string]string{ {"pod-ns-1/pod-6": "vol-6"}, }, "Prepared": []map[string]string{ {"pod-ns-1/pod-7": "vol-7"}, }, "Accepted": []map[string]string{ {"pod-ns-1/pod-8": "vol-8"}, }, }, "uploderType": "kopia", }, }, }, } for _, tc := range testcases { t.Run(tc.name, func(tt *testing.T) { output := make(map[string]any) describePodVolumeBackupsInSF(tc.inputPVBList, tc.inputDetails, output) assert.True(tt, reflect.DeepEqual(output, tc.expect)) }) } } func TestDescribeNativeSnapshotsInSF(t *testing.T) { testcases := []struct { name string volumeInfo []*volume.BackupVolumeInfo inputDetails bool expect map[string]any }{ { name: "no details", volumeInfo: []*volume.BackupVolumeInfo{ { BackupMethod: volume.NativeSnapshot, PVName: "pv-1", NativeSnapshotInfo: &volume.NativeSnapshotInfo{ SnapshotHandle: "snapshot-1", VolumeType: "ebs", VolumeAZ: "us-east-2", IOPS: "1000 mbps", }, }, }, expect: map[string]any{ "nativeSnapshots": map[string]any{ "pv-1": "specify --details for more information", }, }, }, { name: "details", volumeInfo: []*volume.BackupVolumeInfo{ { BackupMethod: volume.NativeSnapshot, PVName: "pv-1", Result: volume.VolumeResultSucceeded, NativeSnapshotInfo: &volume.NativeSnapshotInfo{ SnapshotHandle: "snapshot-1", VolumeType: "ebs", VolumeAZ: "us-east-2", IOPS: "1000 mbps", }, }, }, inputDetails: true, expect: map[string]any{ "nativeSnapshots": map[string]any{ "pv-1": map[string]string{ "snapshotID": "snapshot-1", "type": "ebs", "availabilityZone": "us-east-2", "IOPS": "1000 mbps", "result": "succeeded", }, }, }, }, } for _, tc := range testcases { t.Run(tc.name, func(tt *testing.T) { output := make(map[string]any) describeNativeSnapshotsInSF(tc.inputDetails, tc.volumeInfo, output) assert.True(tt, reflect.DeepEqual(output, tc.expect)) }) } } func TestDescribeCSISnapshotsInSF(t *testing.T) { testcases := []struct { name string volumeInfo []*volume.BackupVolumeInfo inputDetails bool expect map[string]any legacyInfoSource bool }{ { name: "empty info, not legacy", volumeInfo: []*volume.BackupVolumeInfo{}, expect: map[string]any{ "csiSnapshots": "", }, }, { name: "empty info, legacy", volumeInfo: []*volume.BackupVolumeInfo{}, legacyInfoSource: true, expect: map[string]any{ "csiSnapshots": "", }, }, { name: "no details, local snapshot", volumeInfo: []*volume.BackupVolumeInfo{ { BackupMethod: volume.CSISnapshot, PVCNamespace: "pvc-ns-1", PVCName: "pvc-1", PreserveLocalSnapshot: true, CSISnapshotInfo: &volume.CSISnapshotInfo{ SnapshotHandle: "snapshot-1", Size: 1024, Driver: "fake-driver", VSCName: "vsc-1", OperationID: "fake-operation-1", }, }, }, expect: map[string]any{ "csiSnapshots": map[string]any{ "pvc-ns-1/pvc-1": map[string]any{ "snapshot": "included, specify --details for more information", }, }, }, }, { name: "details, local snapshot", volumeInfo: []*volume.BackupVolumeInfo{ { BackupMethod: volume.CSISnapshot, PVCNamespace: "pvc-ns-2", PVCName: "pvc-2", PreserveLocalSnapshot: true, Result: volume.VolumeResultSucceeded, CSISnapshotInfo: &volume.CSISnapshotInfo{ SnapshotHandle: "snapshot-2", Size: 1024, Driver: "fake-driver", VSCName: "vsc-2", OperationID: "fake-operation-2", }, }, }, inputDetails: true, expect: map[string]any{ "csiSnapshots": map[string]any{ "pvc-ns-2/pvc-2": map[string]any{ "snapshot": map[string]any{ "operationID": "fake-operation-2", "snapshotContentName": "vsc-2", "storageSnapshotID": "snapshot-2", "snapshotSize(bytes)": int64(1024), "csiDriver": "fake-driver", "result": "succeeded", }, }, }, }, }, { name: "no details, data movement", volumeInfo: []*volume.BackupVolumeInfo{ { BackupMethod: volume.CSISnapshot, PVCNamespace: "pvc-ns-3", PVCName: "pvc-3", SnapshotDataMoved: true, SnapshotDataMovementInfo: &volume.SnapshotDataMovementInfo{ DataMover: "velero", UploaderType: "fake-uploader", SnapshotHandle: "fake-repo-id-3", OperationID: "fake-operation-3", }, }, }, expect: map[string]any{ "csiSnapshots": map[string]any{ "pvc-ns-3/pvc-3": map[string]any{ "dataMovement": "included, specify --details for more information", }, }, }, }, { name: "details, data movement", volumeInfo: []*volume.BackupVolumeInfo{ { BackupMethod: volume.CSISnapshot, PVCNamespace: "pvc-ns-4", PVCName: "pvc-4", SnapshotDataMoved: true, Result: volume.VolumeResultSucceeded, SnapshotDataMovementInfo: &volume.SnapshotDataMovementInfo{ DataMover: "velero", UploaderType: "fake-uploader", SnapshotHandle: "fake-repo-id-4", OperationID: "fake-operation-4", }, }, }, inputDetails: true, expect: map[string]any{ "csiSnapshots": map[string]any{ "pvc-ns-4/pvc-4": map[string]any{ "dataMovement": map[string]any{ "operationID": "fake-operation-4", "dataMover": "velero", "uploaderType": "fake-uploader", "result": "succeeded", }, }, }, }, }, { name: "details, data movement, data mover is empty", volumeInfo: []*volume.BackupVolumeInfo{ { BackupMethod: volume.CSISnapshot, PVCNamespace: "pvc-ns-4", Result: volume.VolumeResultFailed, PVCName: "pvc-4", SnapshotDataMoved: true, SnapshotDataMovementInfo: &volume.SnapshotDataMovementInfo{ UploaderType: "fake-uploader", SnapshotHandle: "fake-repo-id-4", OperationID: "fake-operation-4", }, }, }, inputDetails: true, expect: map[string]any{ "csiSnapshots": map[string]any{ "pvc-ns-4/pvc-4": map[string]any{ "dataMovement": map[string]any{ "operationID": "fake-operation-4", "dataMover": "velero", "uploaderType": "fake-uploader", "result": "failed", }, }, }, }, }, } for _, tc := range testcases { t.Run(tc.name, func(tt *testing.T) { output := make(map[string]any) describeCSISnapshotsInSF(tc.inputDetails, tc.volumeInfo, output, tc.legacyInfoSource) assert.True(tt, reflect.DeepEqual(output, tc.expect)) }) } } func TestDescribeResourcePoliciesInSF(t *testing.T) { input := &corev1api.TypedLocalObjectReference{ Kind: "configmap", Name: "resource-policy-1", } expect := map[string]any{ "resourcePolicies": map[string]any{ "type": "configmap", "name": "resource-policy-1", }, } sd := &StructuredDescriber{ output: make(map[string]any), format: "", } DescribeResourcePoliciesInSF(sd, input) assert.True(t, reflect.DeepEqual(sd.output, expect)) } func TestDescribeBackupResultInSF(t *testing.T) { input := results.Result{ Velero: []string{"msg-1", "msg-2"}, Cluster: []string{"cluster-1", "cluster-2"}, Namespaces: map[string][]string{ "ns-1": {"ns-1-msg-1", "ns-1-msg-2"}, }, } got := map[string]any{} expect := map[string]any{ "velero": []string{"msg-1", "msg-2"}, "cluster": []string{"cluster-1", "cluster-2"}, "namespace": map[string][]string{ "ns-1": {"ns-1-msg-1", "ns-1-msg-2"}, }, } describeResultInSF(got, input) assert.True(t, reflect.DeepEqual(got, expect)) } func TestDescribeDeleteBackupRequestsInSF(t *testing.T) { t1, err1 := time.Parse("2006-Jan-02", "2023-Jun-26") require.NoError(t, err1) dbr1 := builder.ForDeleteBackupRequest("velero", "dbr1"). ObjectMeta(builder.WithCreationTimestamp(t1)). BackupName("bak-1"). Phase(velerov1api.DeleteBackupRequestPhaseProcessed). Errors("some error").Result() t2, err2 := time.Parse("2006-Jan-02", "2023-Jun-25") require.NoError(t, err2) dbr2 := builder.ForDeleteBackupRequest("velero", "dbr2"). ObjectMeta(builder.WithCreationTimestamp(t2)). BackupName("bak-2"). Phase(velerov1api.DeleteBackupRequestPhaseInProgress).Result() testcases := []struct { name string input []velerov1api.DeleteBackupRequest expect map[string]any }{ { name: "empty list", input: []velerov1api.DeleteBackupRequest{}, expect: map[string]any{ "deletionAttempts": map[string]any{ "deleteBackupRequests": []map[string]any{}, }, }, }, { name: "list with one failed and one in-progress request", input: []velerov1api.DeleteBackupRequest{*dbr1, *dbr2}, expect: map[string]any{ "deletionAttempts": map[string]any{ "failed": int(1), "deleteBackupRequests": []map[string]any{ { "creationTimestamp": t1.String(), "phase": velerov1api.DeleteBackupRequestPhaseProcessed, "errors": []string{ "some error", }, }, { "creationTimestamp": t2.String(), "phase": velerov1api.DeleteBackupRequestPhaseInProgress, }, }, }, }, }, } for _, tc := range testcases { t.Run(tc.name, func(tt *testing.T) { sd := &StructuredDescriber{ output: make(map[string]any), format: "", } DescribeDeleteBackupRequestsInSF(sd, tc.input) assert.True(tt, reflect.DeepEqual(sd.output, tc.expect)) }) } } ================================================ FILE: pkg/cmd/util/output/describe.go ================================================ /* Copyright 2021 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package output import ( "bytes" "encoding/json" "fmt" "sort" "strings" "text/tabwriter" "github.com/fatih/color" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) type Describer struct { Prefix string out *tabwriter.Writer buf *bytes.Buffer } func Describe(fn func(d *Describer)) string { d := Describer{ out: new(tabwriter.Writer), buf: new(bytes.Buffer), } d.out.Init(d.buf, 0, 8, 2, ' ', 0) fn(&d) d.out.Flush() return d.buf.String() } func (d *Describer) Printf(msg string, args ...any) { fmt.Fprint(d.out, d.Prefix) fmt.Fprintf(d.out, msg, args...) } func (d *Describer) Println(args ...any) { fmt.Fprint(d.out, d.Prefix) fmt.Fprintln(d.out, args...) } // DescribeMetadata describes standard object metadata in a consistent manner. func (d *Describer) DescribeMetadata(metadata metav1.ObjectMeta) { d.Printf("Name:\t%s\n", color.New(color.Bold).SprintFunc()(metadata.Name)) d.Printf("Namespace:\t%s\n", metadata.Namespace) d.DescribeMap("Labels", metadata.Labels) d.DescribeMap("Annotations", metadata.Annotations) } // DescribeMap describes a map of key-value pairs using name as the heading. func (d *Describer) DescribeMap(name string, m map[string]string) { d.Printf("%s:\t", name) first := true prefix := "" if len(m) > 0 { keys := make([]string, 0, len(m)) for key := range m { keys = append(keys, key) } sort.Strings(keys) for _, key := range keys { d.Printf("%s%s=%s\n", prefix, key, m[key]) if first { first = false prefix = "\t" } } } else { d.Printf("\n") } } // DescribeSlice describes a slice of strings using name as the heading. The output is prefixed by // "preindent" number of tabs. func (d *Describer) DescribeSlice(preindent int, name string, s []string) { pretab := strings.Repeat("\t", preindent) d.Printf("%s%s:\t", pretab, name) first := true prefix := "" if len(s) > 0 { for _, x := range s { d.Printf("%s%s\n", prefix, x) if first { first = false prefix = pretab + "\t" } } } else { d.Printf("%s\n", pretab) } } // BoolPointerString returns the appropriate string based on the bool pointer's value. func BoolPointerString(b *bool, falseString, trueString, nilString string) string { if b == nil { return nilString } if *b { return trueString } return falseString } type StructuredDescriber struct { output map[string]any format string } // NewStructuredDescriber creates a StructuredDescriber. func NewStructuredDescriber(format string) *StructuredDescriber { return &StructuredDescriber{ output: make(map[string]any), format: format, } } // DescribeInSF returns the structured output based on the func // that applies StructuredDescriber to collect outputs. // This function takes arg 'format' for future format extension. func DescribeInSF(fn func(d *StructuredDescriber), format string) string { d := NewStructuredDescriber(format) fn(d) return d.JSONEncode() } // Describe adds all types of argument to d.output. func (d *StructuredDescriber) Describe(name string, arg any) { d.output[name] = arg } // DescribeMetadata describes standard object metadata. func (d *StructuredDescriber) DescribeMetadata(metadata metav1.ObjectMeta) { metadataInfo := make(map[string]any) metadataInfo["name"] = metadata.Name metadataInfo["namespace"] = metadata.Namespace metadataInfo["labels"] = metadata.Labels metadataInfo["annotations"] = metadata.Annotations d.Describe("metadata", metadataInfo) } // JSONEncode encodes d.output to json func (d *StructuredDescriber) JSONEncode() string { byteBuffer := &bytes.Buffer{} encoder := json.NewEncoder(byteBuffer) encoder.SetEscapeHTML(false) encoder.SetIndent("", " ") err := encoder.Encode(d.output) if err != nil { fmt.Printf("fail to encode %s", err.Error()) return "" } return byteBuffer.String() } ================================================ FILE: pkg/cmd/util/output/describe_test.go ================================================ package output import ( "bytes" "fmt" "reflect" "testing" "text/tabwriter" "github.com/fatih/color" "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func TestBoolPointerString(t *testing.T) { trueStr := "true" falseStr := "FALSE" nilStr := "nul" truee := true falsee := false testcases := []struct { name string input *bool expect string }{ { name: "nil", input: nil, expect: nilStr, }, { name: "true", input: &truee, expect: trueStr, }, { name: "false", input: &falsee, expect: falseStr, }, } for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { got := BoolPointerString(tc.input, falseStr, trueStr, nilStr) assert.Equal(t, tc.expect, got) }) } } func TestDescriber_DescribeMetadata(t *testing.T) { input := metav1.ObjectMeta{ Name: "test", Namespace: "test-ns", Labels: map[string]string{ "test-key2": "v2", "test-key0": "v0", }, Annotations: nil, } expect := new(bytes.Buffer) d := &Describer{ Prefix: "pref-", out: &tabwriter.Writer{}, buf: &bytes.Buffer{}, } d.out.Init(d.buf, 0, 8, 0, ' ', 0) fmt.Fprintf(expect, "pref-Name: %s\n", color.New(color.Bold).SprintFunc()("test")) fmt.Fprintf(expect, "pref-Namespace: %s\n", "test-ns") fmt.Fprintf(expect, "pref-Labels: %s\n", "pref-test-key0=v0") fmt.Fprintf(expect, "pref- test-key2=v2\n") fmt.Fprintf(expect, "pref-Annotations:%s\n", "pref-") d.DescribeMetadata(input) d.out.Flush() assert.Equal(t, expect.String(), d.buf.String()) } func TestDescriber_DescribeSlice(t *testing.T) { input := []string{"a", "b", "c"} expect := new(bytes.Buffer) d := &Describer{ Prefix: "pref-", out: &tabwriter.Writer{}, buf: &bytes.Buffer{}, } d.out.Init(d.buf, 0, 8, 0, ' ', 0) fmt.Fprintf(expect, "pref-test:pref-a\n") fmt.Fprintf(expect, "pref- b\n") fmt.Fprintf(expect, "pref- c\n") d.DescribeSlice(4, "test", input) d.out.Flush() assert.Equal(t, expect.String(), d.buf.String()) var input2 []string expect2 := new(bytes.Buffer) d2 := &Describer{ Prefix: "pref-", out: &tabwriter.Writer{}, buf: &bytes.Buffer{}, } d2.out.Init(d2.buf, 0, 4, 0, ' ', 0) fmt.Fprintf(expect2, "pref-test:pref-\n") d2.DescribeSlice(4, "test", input2) d2.out.Flush() assert.Equal(t, expect2.String(), d2.buf.String()) } func TestStructuredDescriber_JSONEncode(t *testing.T) { testcases := []struct { name string inputMap map[string]any expect string }{ { name: "invalid json", inputMap: map[string]any{}, expect: "{}\n", }, { name: "valid json", inputMap: map[string]any{"k1": "v1"}, expect: `{ "k1": "v1" } `, }, } for _, tc := range testcases { t.Run(tc.name, func(tt *testing.T) { d := &StructuredDescriber{ output: tc.inputMap, } got := d.JSONEncode() assert.Equal(tt, tc.expect, got) }) } } func TestStructuredDescriber_DescribeMetadata(t *testing.T) { d := NewStructuredDescriber("") input := metav1.ObjectMeta{ Name: "test", Namespace: "test-ns", Labels: map[string]string{ "label-1": "v1", "label-2": "v2", }, Annotations: map[string]string{ "annotation-1": "v1", "annotation-2": "v2", }, } expect := map[string]any{ "metadata": map[string]any{ "name": "test", "namespace": "test-ns", "labels": map[string]string{ "label-1": "v1", "label-2": "v2", }, "annotations": map[string]string{ "annotation-1": "v1", "annotation-2": "v2", }, }, } d.DescribeMetadata(input) assert.True(t, reflect.DeepEqual(expect, d.output)) } ================================================ FILE: pkg/cmd/util/output/output.go ================================================ /* Copyright 2017, 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package output import ( "fmt" "os" "time" "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/cli-runtime/pkg/printers" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/cmd/util/flag" "github.com/vmware-tanzu/velero/pkg/util/encode" ) const ( downloadRequestTimeout = 30 * time.Second emptyDisplay = "" defaultDataMover = "velero" ) // BindFlags defines a set of output-specific flags within the provided // FlagSet. func BindFlags(flags *pflag.FlagSet) { flags.StringP("output", "o", "table", "Output display format. For create commands, display the object but do not send it to the server. Valid formats are 'table', 'json', and 'yaml'. 'table' is not valid for the install command.") labelColumns := flag.NewStringArray() flags.VarP(&labelColumns, "label-columns", "L", "Accepts a comma separated list of labels that are going to be presented as columns. Names are case-sensitive. You can also use multiple flag options like -L label1 -L label2...") flags.Bool("show-labels", false, "Show labels in the last column") } // BindFlagsSimple defines the output format flag only. func BindFlagsSimple(flags *pflag.FlagSet) { flags.StringP("output", "o", "table", "Output display format. For create commands, display the object but do not send it to the server. Valid formats are 'table', 'json', and 'yaml'. 'table' is not valid for the install command.") } // ClearOutputFlagDefault sets the current and default value // of the "output" flag to the empty string. func ClearOutputFlagDefault(cmd *cobra.Command) { f := cmd.Flag("output") if f == nil { return } f.DefValue = "" if err := f.Value.Set(""); err != nil { fmt.Printf("error clear the default value of output flag: %s\n", err.Error()) } } // GetOutputFlagValue returns the value of the "output" flag // in the provided command, or the zero value if not present. func GetOutputFlagValue(cmd *cobra.Command) string { return flag.GetOptionalStringFlag(cmd, "output") } // GetLabelColumnsValues returns the value of the "label-columns" flag // in the provided command, or the zero value if not present. func GetLabelColumnsValues(cmd *cobra.Command) []string { return flag.GetOptionalStringArrayFlag(cmd, "label-columns") } // GetShowLabelsValue returns the value of the "show-labels" flag // in the provided command, or the zero value if not present. func GetShowLabelsValue(cmd *cobra.Command) bool { return flag.GetOptionalBoolFlag(cmd, "show-labels") } // ValidateFlags returns an error if any of the output-related flags // were specified with invalid values, or nil otherwise. func ValidateFlags(cmd *cobra.Command) error { if err := validateOutputFlag(cmd); err != nil { return err } return nil } func validateOutputFlag(cmd *cobra.Command) error { output := GetOutputFlagValue(cmd) switch output { case "", "json", "yaml": case "table": if cmd.Name() == "install" { return errors.New("'table' format is not supported with 'install' command") } default: return errors.Errorf("invalid output format %q - valid values are 'table', 'json', and 'yaml'", output) } return nil } // PrintWithFormat prints the provided object in the format specified by // the command's flags. func PrintWithFormat(c *cobra.Command, obj runtime.Object) (bool, error) { format := GetOutputFlagValue(c) if format == "" { return false, nil } switch format { case "table": return printTable(c, obj) case "json", "yaml": return printEncoded(obj, format) } return false, errors.Errorf("unsupported output format %q; valid values are 'table', 'json', and 'yaml'", format) } func printEncoded(obj runtime.Object, format string) (bool, error) { // assume we're printing obj toPrint := obj if meta.IsListType(obj) { list, _ := meta.ExtractList(obj) if len(list) == 1 { // if obj was a list and there was only 1 item, just print that 1 instead of a list toPrint = list[0] } } encoded, err := encode.Encode(toPrint, format) if err != nil { return false, err } fmt.Println(string(encoded)) return true, nil } func printTable(cmd *cobra.Command, obj runtime.Object) (bool, error) { // 1. generate table var table *metav1.Table switch objType := obj.(type) { case *velerov1api.Backup: table = &metav1.Table{ ColumnDefinitions: backupColumns, Rows: printBackup(objType), } case *velerov1api.BackupList: table = &metav1.Table{ ColumnDefinitions: backupColumns, Rows: printBackupList(objType), } case *velerov1api.Restore: table = &metav1.Table{ ColumnDefinitions: restoreColumns, Rows: printRestore(objType), } case *velerov1api.RestoreList: table = &metav1.Table{ ColumnDefinitions: restoreColumns, Rows: printRestoreList(objType), } case *velerov1api.Schedule: table = &metav1.Table{ ColumnDefinitions: scheduleColumns, Rows: printSchedule(objType), } case *velerov1api.ScheduleList: table = &metav1.Table{ ColumnDefinitions: scheduleColumns, Rows: printScheduleList(objType), } case *velerov1api.BackupRepository: table = &metav1.Table{ ColumnDefinitions: backupRepoColumns, Rows: printBackupRepo(objType), } case *velerov1api.BackupRepositoryList: table = &metav1.Table{ ColumnDefinitions: backupRepoColumns, Rows: printBackupRepoList(objType), } case *velerov1api.BackupStorageLocation: table = &metav1.Table{ ColumnDefinitions: backupStorageLocationColumns, Rows: printBackupStorageLocation(objType), } case *velerov1api.BackupStorageLocationList: table = &metav1.Table{ ColumnDefinitions: backupStorageLocationColumns, Rows: printBackupStorageLocationList(objType), } case *velerov1api.VolumeSnapshotLocation: table = &metav1.Table{ ColumnDefinitions: volumeSnapshotLocationColumns, Rows: printVolumeSnapshotLocation(objType), } case *velerov1api.VolumeSnapshotLocationList: table = &metav1.Table{ ColumnDefinitions: volumeSnapshotLocationColumns, Rows: printVolumeSnapshotLocationList(objType), } case *velerov1api.ServerStatusRequest: table = &metav1.Table{ ColumnDefinitions: pluginColumns, Rows: printPluginList(objType), } default: return false, errors.Errorf("type %T is not supported", obj) } // 2. print table tablePrinter, err := NewPrinter(cmd) if err != nil { return false, err } err = tablePrinter.PrintObj(table, os.Stdout) if err != nil { return false, err } return true, nil } // NewPrinter returns a printer for doing human-readable table printing of // Velero objects. func NewPrinter(cmd *cobra.Command) (printers.ResourcePrinter, error) { options := printers.PrintOptions{ ShowLabels: GetShowLabelsValue(cmd), ColumnLabels: GetLabelColumnsValues(cmd), } printer := printers.NewTablePrinter(options) return printer, nil } ================================================ FILE: pkg/cmd/util/output/output_test.go ================================================ package output import ( "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/runtime" velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "testing" ) func TestBindFlags(t *testing.T) { cmd := &cobra.Command{} BindFlags(cmd.Flags()) assert.NotNil(t, cmd.Flags().Lookup("output")) assert.NotNil(t, cmd.Flags().Lookup("label-columns")) assert.NotNil(t, cmd.Flags().Lookup("show-labels")) assert.Nil(t, cmd.Flags().Lookup("not-exist")) } func TestBindFlagsSimple(t *testing.T) { cmd := &cobra.Command{} BindFlagsSimple(cmd.Flags()) assert.NotNil(t, cmd.Flags().Lookup("output")) assert.Nil(t, cmd.Flags().Lookup("label-columns")) assert.Nil(t, cmd.Flags().Lookup("show-labels")) } func TestClearOutputFlagDefault(t *testing.T) { cmd := &cobra.Command{} ClearOutputFlagDefault(cmd) assert.Nil(t, cmd.Flags().Lookup("output")) BindFlags(cmd.Flags()) cmd.Flags().Set("output", "json") ClearOutputFlagDefault(cmd) assert.Empty(t, cmd.Flags().Lookup("output").Value.String()) } func cmdWithFormat(use string, format string) *cobra.Command { cmd := &cobra.Command{ Use: use, } BindFlags(cmd.Flags()) cmd.Flags().Set("output", format) return cmd } func TestValidateFlags(t *testing.T) { testcases := []struct { name string input *cobra.Command hasErr bool }{ { name: "unknown format", input: cmdWithFormat("whatever", "unknown"), hasErr: true, }, { name: "json format", input: cmdWithFormat("whatever", "json"), hasErr: false, }, { name: "yaml format", input: cmdWithFormat("whatever", "yaml"), hasErr: false, }, { name: "empty format", input: cmdWithFormat("whatever", ""), hasErr: false, }, { name: "install with table format", input: cmdWithFormat("install", "table"), hasErr: true, }, { name: "other with table format", input: cmdWithFormat("other", "table"), hasErr: false, }, } for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { err := ValidateFlags(tc.input) if tc.hasErr { assert.Error(t, err) } else { assert.NoError(t, err) } }) } } func TestPrintWithFormat(t *testing.T) { testcases := []struct { name string input struct { cmd *cobra.Command obj runtime.Object } hasErr bool printed bool }{ { name: "empty format", input: struct { cmd *cobra.Command obj runtime.Object }{ cmd: cmdWithFormat("describe", ""), }, hasErr: false, printed: false, }, { name: "json format backup", input: struct { cmd *cobra.Command obj runtime.Object }{ cmd: cmdWithFormat("describe", "json"), obj: &velerov1.Backup{}, }, hasErr: false, printed: true, }, { name: "table format backup", input: struct { cmd *cobra.Command obj runtime.Object }{ cmd: cmdWithFormat("describe", "table"), obj: &velerov1.Backup{}, }, hasErr: false, printed: true, }, { name: "json format backup list", input: struct { cmd *cobra.Command obj runtime.Object }{ cmd: cmdWithFormat("describe", "json"), obj: &velerov1.BackupList{ Items: []velerov1.Backup{ {}, }, }, }, hasErr: false, printed: true, }, { name: "table format backup list", input: struct { cmd *cobra.Command obj runtime.Object }{ cmd: cmdWithFormat("describe", "table"), obj: &velerov1.BackupList{ Items: []velerov1.Backup{ {}, }, }, }, hasErr: false, printed: true, }, { name: "table format backup", input: struct { cmd *cobra.Command obj runtime.Object }{ cmd: cmdWithFormat("describe", "table"), obj: &velerov1.Backup{}, }, hasErr: false, printed: true, }, { name: "table format restore list", input: struct { cmd *cobra.Command obj runtime.Object }{ cmd: cmdWithFormat("describe", "table"), obj: &velerov1.RestoreList{ Items: []velerov1.Restore{ {}, }, }, }, hasErr: false, printed: true, }, { name: "table format restore", input: struct { cmd *cobra.Command obj runtime.Object }{ cmd: cmdWithFormat("describe", "table"), obj: &velerov1.Restore{}, }, hasErr: false, printed: true, }, { name: "table format schedule list", input: struct { cmd *cobra.Command obj runtime.Object }{ cmd: cmdWithFormat("describe", "table"), obj: &velerov1.ScheduleList{ Items: []velerov1.Schedule{ {}, }, }, }, hasErr: false, printed: true, }, { name: "table format schedule", input: struct { cmd *cobra.Command obj runtime.Object }{ cmd: cmdWithFormat("describe", "table"), obj: &velerov1.Schedule{}, }, hasErr: false, printed: true, }, { name: "table format backup repository list", input: struct { cmd *cobra.Command obj runtime.Object }{ cmd: cmdWithFormat("describe", "table"), obj: &velerov1.BackupRepositoryList{ Items: []velerov1.BackupRepository{ {}, }, }, }, hasErr: false, printed: true, }, { name: "table format backup repository", input: struct { cmd *cobra.Command obj runtime.Object }{ cmd: cmdWithFormat("describe", "table"), obj: &velerov1.BackupRepository{}, }, hasErr: false, printed: true, }, { name: "table format backup location list", input: struct { cmd *cobra.Command obj runtime.Object }{ cmd: cmdWithFormat("describe", "table"), obj: &velerov1.BackupStorageLocationList{ Items: []velerov1.BackupStorageLocation{ { Spec: velerov1.BackupStorageLocationSpec{ Provider: "aws", StorageType: velerov1.StorageType{ ObjectStorage: &velerov1.ObjectStorageLocation{ Bucket: "bucket", }, }, }, }, }, }, }, hasErr: false, printed: true, }, { name: "table format backup location", input: struct { cmd *cobra.Command obj runtime.Object }{ cmd: cmdWithFormat("describe", "table"), obj: &velerov1.BackupStorageLocation{ Spec: velerov1.BackupStorageLocationSpec{ Provider: "aws", StorageType: velerov1.StorageType{ ObjectStorage: &velerov1.ObjectStorageLocation{ Bucket: "bucket", }, }, }, }, }, hasErr: false, printed: true, }, { name: "table format volume snapshot location list", input: struct { cmd *cobra.Command obj runtime.Object }{ cmd: cmdWithFormat("describe", "table"), obj: &velerov1.VolumeSnapshotLocationList{ Items: []velerov1.VolumeSnapshotLocation{ {}, }, }, }, hasErr: false, printed: true, }, { name: "table format volume snapshot location", input: struct { cmd *cobra.Command obj runtime.Object }{ cmd: cmdWithFormat("describe", "table"), obj: &velerov1.VolumeSnapshotLocation{}, }, hasErr: false, printed: true, }, { name: "table format volume snapshot location", input: struct { cmd *cobra.Command obj runtime.Object }{ cmd: cmdWithFormat("describe", "table"), obj: &velerov1.VolumeSnapshotLocation{}, }, hasErr: false, printed: true, }, { name: "table format plugin list via server status", input: struct { cmd *cobra.Command obj runtime.Object }{ cmd: cmdWithFormat("describe", "table"), obj: &velerov1.ServerStatusRequest{}, }, hasErr: false, printed: true, }, } for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { p, err := PrintWithFormat(tc.input.cmd, tc.input.obj) if tc.hasErr { require.Error(t, err) } else { require.NoError(t, err) } assert.Equal(t, tc.printed, p) }) } } ================================================ FILE: pkg/cmd/util/output/plugin_printer.go ================================================ /* Copyright 2019, 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package output import ( "sort" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) var ( pluginColumns = []metav1.TableColumnDefinition{ // name needs Type and Format defined for the decorator to identify it: // https://github.com/kubernetes/kubernetes/blob/v1.15.3/pkg/printers/tableprinter.go#L204 {Name: "Name", Type: "string", Format: "name"}, {Name: "Kind"}, } ) func printPluginList(list *velerov1api.ServerStatusRequest) []metav1.TableRow { plugins := list.Status.Plugins sortByKindAndName(plugins) rows := make([]metav1.TableRow, 0, len(plugins)) for _, plugin := range plugins { rows = append(rows, printPlugin(plugin)...) } return rows } func sortByKindAndName(plugins []velerov1api.PluginInfo) { sort.Slice(plugins, func(i, j int) bool { if plugins[i].Kind != plugins[j].Kind { return plugins[i].Kind < plugins[j].Kind } return plugins[i].Name < plugins[j].Name }) } func printPlugin(plugin velerov1api.PluginInfo) []metav1.TableRow { row := metav1.TableRow{} row.Cells = append(row.Cells, plugin.Name, plugin.Kind) return []metav1.TableRow{row} } ================================================ FILE: pkg/cmd/util/output/restore_describer.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package output import ( "bytes" "context" "encoding/json" "errors" "fmt" "sort" "strings" "github.com/vmware-tanzu/velero/internal/volume" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kbclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/fatih/color" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/cmd/util/cacert" "github.com/vmware-tanzu/velero/pkg/cmd/util/downloadrequest" "github.com/vmware-tanzu/velero/pkg/itemoperation" "github.com/vmware-tanzu/velero/pkg/util/boolptr" "github.com/vmware-tanzu/velero/pkg/util/results" ) func DescribeRestore( ctx context.Context, kbClient kbclient.Client, restore *velerov1api.Restore, podVolumeRestores []velerov1api.PodVolumeRestore, details bool, insecureSkipTLSVerify bool, caCertFile string, ) string { return Describe(func(d *Describer) { d.DescribeMetadata(restore.ObjectMeta) d.Println() phase := restore.Status.Phase if phase == "" { phase = velerov1api.RestorePhaseNew } phaseString := string(phase) // Append "Deleting" to phaseString if deletionTimestamp is marked. if !restore.DeletionTimestamp.IsZero() { phaseString += " (Deleting)" } switch phase { case velerov1api.RestorePhaseCompleted: phaseString = color.GreenString(phaseString) case velerov1api.RestorePhaseFailedValidation, velerov1api.RestorePhasePartiallyFailed, velerov1api.RestorePhaseFailed: phaseString = color.RedString(phaseString) } resultsNote := "" if phase == velerov1api.RestorePhaseFailed || phase == velerov1api.RestorePhasePartiallyFailed { resultsNote = fmt.Sprintf(" (run 'velero restore logs %s' for more information)", restore.Name) } d.Printf("Phase:\t%s%s\n", phaseString, resultsNote) if restore.Status.Progress != nil { if restore.Status.Phase == velerov1api.RestorePhaseInProgress { d.Printf("Estimated total items to be restored:\t%d\n", restore.Status.Progress.TotalItems) d.Printf("Items restored so far:\t%d\n", restore.Status.Progress.ItemsRestored) } else { d.Printf("Total items to be restored:\t%d\n", restore.Status.Progress.TotalItems) d.Printf("Items restored:\t%d\n", restore.Status.Progress.ItemsRestored) } } d.Println() // "" output should only be applicable for restore that failed validation if restore.Status.StartTimestamp == nil || restore.Status.StartTimestamp.IsZero() { d.Printf("Started:\t%s\n", "") } else { d.Printf("Started:\t%s\n", restore.Status.StartTimestamp) } if restore.Status.CompletionTimestamp == nil || restore.Status.CompletionTimestamp.IsZero() { d.Printf("Completed:\t%s\n", "") } else { d.Printf("Completed:\t%s\n", restore.Status.CompletionTimestamp) } if len(restore.Status.ValidationErrors) > 0 { d.Println() d.Printf("Validation errors:") for _, ve := range restore.Status.ValidationErrors { d.Printf("\t%s\n", color.RedString(ve)) } } describeRestoreResults(ctx, kbClient, d, restore, insecureSkipTLSVerify, caCertFile) d.Println() d.Printf("Backup:\t%s\n", restore.Spec.BackupName) d.Println() d.Printf("Namespaces:\n") var s string if len(restore.Spec.IncludedNamespaces) == 0 { s = "all namespaces found in the backup" } else if len(restore.Spec.IncludedNamespaces) == 1 && restore.Spec.IncludedNamespaces[0] == "*" { s = "all namespaces found in the backup" } else { s = strings.Join(restore.Spec.IncludedNamespaces, ", ") } d.Printf("\tIncluded:\t%s\n", s) if len(restore.Spec.ExcludedNamespaces) == 0 { s = emptyDisplay } else { s = strings.Join(restore.Spec.ExcludedNamespaces, ", ") } d.Printf("\tExcluded:\t%s\n", s) d.Println() d.Printf("Resources:\n") if len(restore.Spec.IncludedResources) == 0 { s = "*" } else { s = strings.Join(restore.Spec.IncludedResources, ", ") } d.Printf("\tIncluded:\t%s\n", s) if len(restore.Spec.ExcludedResources) == 0 { s = emptyDisplay } else { s = strings.Join(restore.Spec.ExcludedResources, ", ") } d.Printf("\tExcluded:\t%s\n", s) d.Printf("\tCluster-scoped:\t%s\n", BoolPointerString(restore.Spec.IncludeClusterResources, "excluded", "included", "auto")) d.Println() d.DescribeMap("Namespace mappings", restore.Spec.NamespaceMapping) d.Println() s = emptyDisplay if restore.Spec.LabelSelector != nil { s = metav1.FormatLabelSelector(restore.Spec.LabelSelector) } d.Printf("Label selector:\t%s\n", s) d.Println() if len(restore.Spec.OrLabelSelectors) == 0 { s = emptyDisplay } else { orLabelSelectors := []string{} for _, v := range restore.Spec.OrLabelSelectors { orLabelSelectors = append(orLabelSelectors, metav1.FormatLabelSelector(v)) } s = strings.Join(orLabelSelectors, " or ") } d.Printf("Or label selector:\t%s\n", s) d.Println() d.Printf("Restore PVs:\t%s\n", BoolPointerString(restore.Spec.RestorePVs, "false", "true", "auto")) if len(podVolumeRestores) > 0 { d.Println() describePodVolumeRestores(d, podVolumeRestores, details) } // Get BSL cacert if available bslCACert, err := cacert.GetCACertFromRestore(ctx, kbClient, restore.Namespace, restore) if err != nil { // Log the error but don't fail - we can still try to download without the BSL cacert d.Printf("WARNING: Error getting cacert from BSL: %v\n", err) bslCACert = "" } buf := new(bytes.Buffer) if err := downloadrequest.StreamWithBSLCACert(ctx, kbClient, restore.Namespace, restore.Name, velerov1api.DownloadTargetKindRestoreVolumeInfo, buf, downloadRequestTimeout, insecureSkipTLSVerify, caCertFile, bslCACert); err == nil { var restoreVolInfo []volume.RestoreVolumeInfo if err := json.NewDecoder(buf).Decode(&restoreVolInfo); err != nil { d.Printf("\t\n", err) } else { describeCSISnapshotsRestores(d, restoreVolInfo, details) } } else if err != nil && !errors.Is(err, downloadrequest.ErrNotFound) { // For the restores by older versions of velero, it will see NotFound Error when downloading the volume info. // In that case, no errors will be printed. d.Printf("\t\n", err) } d.Println() s = emptyDisplay if restore.Spec.ExistingResourcePolicy != "" { s = string(restore.Spec.ExistingResourcePolicy) } d.Printf("Existing Resource Policy: \t%s\n", s) d.Printf("ItemOperationTimeout:\t%s\n", restore.Spec.ItemOperationTimeout.Duration) d.Println() d.Printf("Preserve Service NodePorts:\t%s\n", BoolPointerString(restore.Spec.PreserveNodePorts, "false", "true", "auto")) if restore.Spec.ResourceModifier != nil { d.Println() DescribeResourceModifier(d, restore.Spec.ResourceModifier) } describeUploaderConfigForRestore(d, restore.Spec) d.Println() describeRestoreItemOperations(ctx, kbClient, d, restore, details, insecureSkipTLSVerify, caCertFile) if restore.Status.HookStatus != nil { d.Println() d.Printf("HooksAttempted: \t%d\n", restore.Status.HookStatus.HooksAttempted) d.Printf("HooksFailed: \t%d\n", restore.Status.HookStatus.HooksFailed) } if details { d.Println() describeRestoreResourceList(ctx, kbClient, d, restore, insecureSkipTLSVerify, caCertFile) } }) } // describeUploaderConfigForRestore describes uploader config in human-readable format func describeUploaderConfigForRestore(d *Describer, spec velerov1api.RestoreSpec) { if spec.UploaderConfig != nil { d.Println() d.Printf("Uploader config:\n") if boolptr.IsSetToTrue(spec.UploaderConfig.WriteSparseFiles) { d.Printf("\tWrite Sparse Files:\t%v\n", boolptr.IsSetToTrue(spec.UploaderConfig.WriteSparseFiles)) } if spec.UploaderConfig.ParallelFilesDownload > 0 { d.Printf("\tParallel Restore:\t%d\n", spec.UploaderConfig.ParallelFilesDownload) } } } func describeRestoreItemOperations(ctx context.Context, kbClient kbclient.Client, d *Describer, restore *velerov1api.Restore, details bool, insecureSkipTLSVerify bool, caCertPath string) { status := restore.Status if status.RestoreItemOperationsAttempted > 0 { if !details { d.Printf("Restore Item Operations:\t%d of %d completed successfully, %d failed (specify --details for more information)\n", status.RestoreItemOperationsCompleted, status.RestoreItemOperationsAttempted, status.RestoreItemOperationsFailed) return } // Get BSL cacert if available bslCACert, err := cacert.GetCACertFromRestore(ctx, kbClient, restore.Namespace, restore) if err != nil { // Log the error but don't fail - we can still try to download without the BSL cacert d.Printf("WARNING: Error getting cacert from BSL: %v\n", err) bslCACert = "" } buf := new(bytes.Buffer) if err := downloadrequest.StreamWithBSLCACert(ctx, kbClient, restore.Namespace, restore.Name, velerov1api.DownloadTargetKindRestoreItemOperations, buf, downloadRequestTimeout, insecureSkipTLSVerify, caCertPath, bslCACert); err != nil { d.Printf("Restore Item Operations:\t\n", err) return } var operations []*itemoperation.RestoreOperation if err := json.NewDecoder(buf).Decode(&operations); err != nil { d.Printf("Restore Item Operations:\t\n", err) return } d.Printf("Restore Item Operations:\n") for _, operation := range operations { describeRestoreItemOperation(d, operation) } } } func describeRestoreResults(ctx context.Context, kbClient kbclient.Client, d *Describer, restore *velerov1api.Restore, insecureSkipTLSVerify bool, caCertPath string) { if restore.Status.Warnings == 0 && restore.Status.Errors == 0 { return } // Get BSL cacert if available bslCACert, err := cacert.GetCACertFromRestore(ctx, kbClient, restore.Namespace, restore) if err != nil { // Log the error but don't fail - we can still try to download without the BSL cacert d.Printf("WARNING: Error getting cacert from BSL: %v\n", err) bslCACert = "" } var buf bytes.Buffer var resultMap map[string]results.Result if err := downloadrequest.StreamWithBSLCACert(ctx, kbClient, restore.Namespace, restore.Name, velerov1api.DownloadTargetKindRestoreResults, &buf, downloadRequestTimeout, insecureSkipTLSVerify, caCertPath, bslCACert); err != nil { d.Printf("Warnings:\t\n\nErrors:\t\n", err, err) return } if err := json.NewDecoder(&buf).Decode(&resultMap); err != nil { d.Printf("Warnings:\t\n\nErrors:\t\n", err, err) return } if restore.Status.Warnings > 0 { d.Println() describeResult(d, "Warnings", resultMap["warnings"]) } if restore.Status.Errors > 0 { d.Println() describeResult(d, "Errors", resultMap["errors"]) } } func describeResult(d *Describer, name string, result results.Result) { d.Printf("%s:\n", name) d.DescribeSlice(1, "Velero", result.Velero) d.DescribeSlice(1, "Cluster", result.Cluster) if len(result.Namespaces) == 0 { d.Printf("\tNamespaces: \n") } else { d.Printf("\tNamespaces:\n") for ns, warnings := range result.Namespaces { d.DescribeSlice(2, ns, warnings) } } } func describeRestoreItemOperation(d *Describer, operation *itemoperation.RestoreOperation) { d.Printf("\tOperation for %s %s/%s:\n", operation.Spec.ResourceIdentifier, operation.Spec.ResourceIdentifier.Namespace, operation.Spec.ResourceIdentifier.Name) d.Printf("\t\tRestore Item Action Plugin:\t%s\n", operation.Spec.RestoreItemAction) d.Printf("\t\tOperation ID:\t%s\n", operation.Spec.OperationID) d.Printf("\t\tPhase:\t%s\n", operation.Status.Phase) if operation.Status.Error != "" { d.Printf("\t\tOperation Error:\t%s\n", operation.Status.Error) } if operation.Status.NTotal > 0 || operation.Status.NCompleted > 0 { d.Printf("\t\tProgress:\t%v of %v complete (%s)\n", operation.Status.NCompleted, operation.Status.NTotal, operation.Status.OperationUnits) } if operation.Status.Description != "" { d.Printf("\t\tProgress description:\t%s\n", operation.Status.Description) } if operation.Status.Created != nil { d.Printf("\t\tCreated:\t%s\n", operation.Status.Created.String()) } if operation.Status.Started != nil { d.Printf("\t\tStarted:\t%s\n", operation.Status.Started.String()) } if operation.Status.Updated != nil { d.Printf("\t\tUpdated:\t%s\n", operation.Status.Updated.String()) } } // describePodVolumeRestores describes pod volume restores in human-readable format. func describePodVolumeRestores(d *Describer, restores []velerov1api.PodVolumeRestore, details bool) { // Get the type of pod volume uploader. Since the uploader only comes from a single source, we can // take the uploader type from the first element of the array. var uploaderType string if len(restores) > 0 { uploaderType = restores[0].Spec.UploaderType } else { return } if details { d.Printf("%s Restores:\n", uploaderType) } else { d.Printf("%s Restores (specify --details for more information):\n", uploaderType) } // separate restores by phase (combining and New into a single group) restoresByPhase := groupRestoresByPhase(restores) // go through phases in a specific order for _, phase := range []string{ string(velerov1api.PodVolumeRestorePhaseCompleted), string(velerov1api.PodVolumeRestorePhaseCanceled), string(velerov1api.PodVolumeRestorePhaseFailed), "In Progress", string(velerov1api.PodVolumeRestorePhasePrepared), string(velerov1api.PodVolumeRestorePhaseAccepted), string(velerov1api.PodVolumeRestorePhaseNew), } { if len(restoresByPhase[phase]) == 0 { continue } // if we're not printing details, just report the phase and count if !details { d.Printf("\t%s:\t%d\n", phase, len(restoresByPhase[phase])) continue } // group the restores in the current phase by pod (i.e. "ns/name") restoresByPod := new(volumesByPod) for _, restore := range restoresByPhase[phase] { restoresByPod.Add(restore.Spec.Pod.Namespace, restore.Spec.Pod.Name, restore.Spec.Volume, phase, restore.Status.Progress, 0) } d.Printf("\t%s:\n", phase) for _, restoreGroup := range restoresByPod.Sorted() { sort.Strings(restoreGroup.volumes) // print volumes restored up for this pod d.Printf("\t\t%s: %s\n", restoreGroup.label, strings.Join(restoreGroup.volumes, ", ")) } } } // describeCSISnapshotsRestores describes PVC restored via CSISnapshots, incl. data-movement, in human-readable format. func describeCSISnapshotsRestores(d *Describer, restoreVolInfo []volume.RestoreVolumeInfo, details bool) { d.Println() var nonDMInfoList, dmInfoList []volume.RestoreVolumeInfo for _, info := range restoreVolInfo { if info.RestoreMethod != volume.CSISnapshot { continue } if info.SnapshotDataMoved { dmInfoList = append(dmInfoList, info) } else { nonDMInfoList = append(nonDMInfoList, info) } } if len(nonDMInfoList) == 0 && len(dmInfoList) == 0 { d.Printf("CSI Snapshot Restores: \n") return } d.Printf("CSI Snapshot Restores:\n") for _, info := range nonDMInfoList { // All CSI snapshots are restored via PVC d.Printf("\t%s/%s:\n", info.PVCNamespace, info.PVCName) if details { d.Printf("\t\tSnapshot:\n") d.Printf("\t\t\tSnapshot Content Name: %s\n", info.CSISnapshotInfo.VSCName) d.Printf("\t\t\tStorage Snapshot ID: %s\n", info.CSISnapshotInfo.SnapshotHandle) d.Printf("\t\t\tCSI Driver: %s\n", info.CSISnapshotInfo.Driver) } else { d.Printf("\t\tSnapshot: specify --details for more information\n") } } for _, info := range dmInfoList { d.Printf("\t%s/%s:\n", info.PVCNamespace, info.PVCName) if details { d.Printf("\t\tData Movement:\n") d.Printf("\t\t\tOperation ID: %s\n", info.SnapshotDataMovementInfo.OperationID) d.Printf("\t\t\tData Mover: %s\n", info.SnapshotDataMovementInfo.DataMover) d.Printf("\t\t\tUploader Type: %s\n", info.SnapshotDataMovementInfo.UploaderType) } else { d.Printf("\t\tData Movement: specify --details for more information\n") } } } func groupRestoresByPhase(restores []velerov1api.PodVolumeRestore) map[string][]velerov1api.PodVolumeRestore { restoresByPhase := make(map[string][]velerov1api.PodVolumeRestore) phaseToGroup := map[velerov1api.PodVolumeRestorePhase]string{ velerov1api.PodVolumeRestorePhaseCompleted: string(velerov1api.PodVolumeRestorePhaseCompleted), velerov1api.PodVolumeRestorePhaseCanceled: string(velerov1api.PodVolumeRestorePhaseCanceled), velerov1api.PodVolumeRestorePhaseFailed: string(velerov1api.PodVolumeRestorePhaseFailed), velerov1api.PodVolumeRestorePhaseInProgress: "In Progress", velerov1api.PodVolumeRestorePhasePrepared: string(velerov1api.PodVolumeRestorePhasePrepared), velerov1api.PodVolumeRestorePhaseAccepted: string(velerov1api.PodVolumeRestorePhaseAccepted), velerov1api.PodVolumeRestorePhaseNew: string(velerov1api.PodVolumeRestorePhaseNew), "": string(velerov1api.PodVolumeRestorePhaseNew), } for _, restore := range restores { group := phaseToGroup[restore.Status.Phase] restoresByPhase[group] = append(restoresByPhase[group], restore) } return restoresByPhase } func describeRestoreResourceList(ctx context.Context, kbClient kbclient.Client, d *Describer, restore *velerov1api.Restore, insecureSkipTLSVerify bool, caCertPath string) { // Get BSL cacert if available bslCACert, err := cacert.GetCACertFromRestore(ctx, kbClient, restore.Namespace, restore) if err != nil { // Log the error but don't fail - we can still try to download without the BSL cacert d.Printf("WARNING: Error getting cacert from BSL: %v\n", err) bslCACert = "" } buf := new(bytes.Buffer) if err := downloadrequest.StreamWithBSLCACert(ctx, kbClient, restore.Namespace, restore.Name, velerov1api.DownloadTargetKindRestoreResourceList, buf, downloadRequestTimeout, insecureSkipTLSVerify, caCertPath, bslCACert); err != nil { if err == downloadrequest.ErrNotFound { d.Println("Resource List:\t") } else { d.Printf("Resource List:\t\n", err) } return } var resourceList map[string][]string if err := json.NewDecoder(buf).Decode(&resourceList); err != nil { d.Printf("Resource List:\t\n", err) return } d.Println("Resource List:") // Sort GVKs in output gvks := make([]string, 0, len(resourceList)) for gvk := range resourceList { gvks = append(gvks, gvk) } sort.Strings(gvks) for _, gvk := range gvks { d.Printf("\t%s:\n\t\t- %s\n", gvk, strings.Join(resourceList[gvk], "\n\t\t- ")) } } // DescribeResourceModifier describes resource policies in human-readable format func DescribeResourceModifier(d *Describer, resModifier *corev1api.TypedLocalObjectReference) { d.Printf("Resource modifier:\n") d.Printf("\tType:\t%s\n", resModifier.Kind) d.Printf("\tName:\t%s\n", resModifier.Name) } ================================================ FILE: pkg/cmd/util/output/restore_describer_test.go ================================================ package output import ( "bytes" "fmt" "testing" "text/tabwriter" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" "github.com/vmware-tanzu/velero/internal/volume" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/itemoperation" "github.com/vmware-tanzu/velero/pkg/util/boolptr" "github.com/vmware-tanzu/velero/pkg/util/results" ) func TestDescribeResult(t *testing.T) { testcases := []struct { name string inputName string inputResult results.Result expect string }{ { name: "result without ns warns", inputName: "restore-1", inputResult: results.Result{ Velero: []string{"velero-msg-1", "velero-msg-2"}, Cluster: []string{"cluster-msg-1", "cluster-msg-2"}, Namespaces: map[string][]string{}, }, expect: `restore-1: Velero: velero-msg-1 velero-msg-2 Cluster: cluster-msg-1 cluster-msg-2 Namespaces: `, }, { name: "result with ns warns", inputName: "restore-2", inputResult: results.Result{ Velero: []string{"velero-msg-1", "velero-msg-2"}, Cluster: []string{"cluster-msg-1", "cluster-msg-2"}, Namespaces: map[string][]string{ "ns-1": {"ns-1-warn-1", "ns-1-warn-2"}, }, }, expect: `restore-2: Velero: velero-msg-1 velero-msg-2 Cluster: cluster-msg-1 cluster-msg-2 Namespaces: ns-1: ns-1-warn-1 ns-1-warn-2 `, }, } for _, tc := range testcases { t.Run(tc.name, func(tt *testing.T) { d := &Describer{ Prefix: "", out: &tabwriter.Writer{}, buf: &bytes.Buffer{}, } d.out.Init(d.buf, 0, 8, 2, ' ', 0) describeResult(d, tc.inputName, tc.inputResult) d.out.Flush() assert.Equal(tt, tc.expect, d.buf.String()) }) } } func TestDescribeRestoreItemOperation(t *testing.T) { t1, err1 := time.Parse("2006-Jan-02", "2023-Jun-26") require.NoError(t, err1) t2, err2 := time.Parse("2006-Jan-02", "2023-Jun-25") require.NoError(t, err2) t3, err3 := time.Parse("2006-Jan-02", "2023-Jun-24") require.NoError(t, err3) input := builder.ForRestoreOperation(). RestoreName("restore-1"). OperationID("op-1"). RestoreItemAction("action-1"). ResourceIdentifier("group", "rs-type", "ns", "rs-name"). Status(*builder.ForOperationStatus(). Phase(itemoperation.OperationPhaseFailed). Error("operation error"). Progress(50, 100, "bytes"). Description("operation description"). Created(t3). Started(t2). Updated(t1). Result()).Result() expected := ` Operation for rs-type.group ns/rs-name: Restore Item Action Plugin: action-1 Operation ID: op-1 Phase: Failed Operation Error: operation error Progress: 50 of 100 complete (bytes) Progress description: operation description Created: 2023-06-24 00:00:00 +0000 UTC Started: 2023-06-25 00:00:00 +0000 UTC Updated: 2023-06-26 00:00:00 +0000 UTC ` d := &Describer{ Prefix: "", out: &tabwriter.Writer{}, buf: &bytes.Buffer{}, } d.out.Init(d.buf, 0, 8, 2, ' ', 0) describeRestoreItemOperation(d, input) d.out.Flush() assert.Equal(t, expected, d.buf.String()) } func TestDescribePodVolumeRestores(t *testing.T) { pvr1 := builder.ForPodVolumeRestore("velero", "pvr-1"). UploaderType("kopia"). Phase(velerov1api.PodVolumeRestorePhaseCompleted). BackupStorageLocation("bsl-1"). Volume("vol-1"). PodName("pod-1"). PodNamespace("pod-ns-1"). SnapshotID("snap-1").Result() pvr2 := builder.ForPodVolumeRestore("velero", "pvr-2"). UploaderType("kopia"). Phase(velerov1api.PodVolumeRestorePhaseCompleted). BackupStorageLocation("bsl-1"). Volume("vol-2"). PodName("pod-2"). PodNamespace("pod-ns-1"). SnapshotID("snap-2").Result() testcases := []struct { name string inputPVRList []velerov1api.PodVolumeRestore inputDetails bool expect string }{ { name: "empty list", inputPVRList: []velerov1api.PodVolumeRestore{}, inputDetails: true, expect: ``, }, { name: "2 completed pvrs no details", inputPVRList: []velerov1api.PodVolumeRestore{*pvr1, *pvr2}, inputDetails: false, expect: `kopia Restores (specify --details for more information): Completed: 2 `, }, { name: "2 completed pvrs with details", inputPVRList: []velerov1api.PodVolumeRestore{*pvr1, *pvr2}, inputDetails: true, expect: `kopia Restores: Completed: pod-ns-1/pod-1: vol-1 pod-ns-1/pod-2: vol-2 `, }, } for _, tc := range testcases { t.Run(tc.name, func(tt *testing.T) { d := &Describer{ Prefix: "", out: &tabwriter.Writer{}, buf: &bytes.Buffer{}, } d.out.Init(d.buf, 0, 8, 2, ' ', 0) describePodVolumeRestores(d, tc.inputPVRList, tc.inputDetails) d.out.Flush() assert.Equal(tt, tc.expect, d.buf.String()) }) } } func TestDescribeUploaderConfigForRestore(t *testing.T) { cases := []struct { name string spec velerov1api.RestoreSpec expected string }{ { name: "UploaderConfigNil", spec: velerov1api.RestoreSpec{}, // Create a RestoreSpec with nil UploaderConfig expected: "", }, { name: "test", spec: velerov1api.RestoreSpec{ UploaderConfig: &velerov1api.UploaderConfigForRestore{ WriteSparseFiles: boolptr.True(), ParallelFilesDownload: 4, }, }, expected: "\nUploader config:\n Write Sparse Files: true\n Parallel Restore: 4\n", }, { name: "WriteSparseFiles test", spec: velerov1api.RestoreSpec{ UploaderConfig: &velerov1api.UploaderConfigForRestore{ WriteSparseFiles: boolptr.True(), }, }, expected: "\nUploader config:\n Write Sparse Files: true\n", }, { name: "ParallelFilesDownload test", spec: velerov1api.RestoreSpec{ UploaderConfig: &velerov1api.UploaderConfigForRestore{ ParallelFilesDownload: 4, }, }, expected: "\nUploader config:\n Parallel Restore: 4\n", }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { d := &Describer{ Prefix: "", out: &tabwriter.Writer{}, buf: &bytes.Buffer{}, } d.out.Init(d.buf, 0, 8, 2, ' ', 0) describeUploaderConfigForRestore(d, tc.spec) d.out.Flush() assert.Equal(t, tc.expected, d.buf.String(), "Output should match expected") }) } } func TestDescribeCSISnapshotsRestore(t *testing.T) { cases := []struct { name string inputVolInfoList []volume.RestoreVolumeInfo inputDetail bool expect string }{ { name: "empty list", inputVolInfoList: []volume.RestoreVolumeInfo{}, inputDetail: true, expect: ` CSI Snapshot Restores: `, }, { name: "list with non CSI snapshot", inputVolInfoList: []volume.RestoreVolumeInfo{ { PVCName: "pvc-2", PVCNamespace: "ns-2", PVName: "pv-2", RestoreMethod: volume.NativeSnapshot, NativeSnapshotInfo: &volume.NativeSnapshotInfo{ SnapshotHandle: "snap-1", }, }, }, inputDetail: true, expect: ` CSI Snapshot Restores: `, }, { name: "CSI restore without data movement, detailed", inputVolInfoList: []volume.RestoreVolumeInfo{ { PVCName: "pvc-1", PVCNamespace: "ns-1", PVName: "pv-1", RestoreMethod: volume.CSISnapshot, CSISnapshotInfo: &volume.CSISnapshotInfo{ SnapshotHandle: "snapshot-handle-1", Size: 1234, Driver: "csi.test.driver", VSCName: "content-1", }, }, }, inputDetail: true, expect: ` CSI Snapshot Restores: ns-1/pvc-1: Snapshot: Snapshot Content Name: content-1 Storage Snapshot ID: snapshot-handle-1 CSI Driver: csi.test.driver `, }, { name: "CSI restore with data movement, detailed", inputVolInfoList: []volume.RestoreVolumeInfo{ { PVCName: "pvc-3", PVCNamespace: "ns-3", PVName: "pv-3", RestoreMethod: volume.CSISnapshot, SnapshotDataMoved: true, SnapshotDataMovementInfo: &volume.SnapshotDataMovementInfo{ OperationID: "op-3", DataMover: "velero", UploaderType: "kopia", Size: 1234, }, }, }, inputDetail: true, expect: ` CSI Snapshot Restores: ns-3/pvc-3: Data Movement: Operation ID: op-3 Data Mover: velero Uploader Type: kopia `, }, { name: "vol info with different entries, without details", inputVolInfoList: []volume.RestoreVolumeInfo{ { PVCName: "pvc-3", PVCNamespace: "ns-3", PVName: "pv-3", RestoreMethod: volume.CSISnapshot, SnapshotDataMoved: true, SnapshotDataMovementInfo: &volume.SnapshotDataMovementInfo{ OperationID: "op-3", DataMover: "velero", UploaderType: "kopia", Size: 1234, }, }, { PVCName: "pvc-2", PVCNamespace: "ns-2", PVName: "pv-2", RestoreMethod: volume.NativeSnapshot, NativeSnapshotInfo: &volume.NativeSnapshotInfo{ SnapshotHandle: "snap-1", }, }, { PVCName: "pvc-1", PVCNamespace: "ns-1", PVName: "pv-1", RestoreMethod: volume.CSISnapshot, CSISnapshotInfo: &volume.CSISnapshotInfo{ SnapshotHandle: "snapshot-handle-1", Size: 1234, Driver: "csi.test.driver", VSCName: "content-1", }, }, }, inputDetail: false, expect: ` CSI Snapshot Restores: ns-1/pvc-1: Snapshot: specify --details for more information ns-3/pvc-3: Data Movement: specify --details for more information `, }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { d := &Describer{ Prefix: "", out: &tabwriter.Writer{}, buf: &bytes.Buffer{}, } d.out.Init(d.buf, 0, 8, 2, ' ', 0) describeCSISnapshotsRestores(d, tc.inputVolInfoList, tc.inputDetail) d.out.Flush() assert.Equal(t, tc.expect, d.buf.String()) }) } } func TestDescribeResourceModifier(t *testing.T) { d := &Describer{ Prefix: "", out: &tabwriter.Writer{}, buf: &bytes.Buffer{}, } d.out.Init(d.buf, 0, 8, 2, ' ', 0) DescribeResourceModifier(d, &corev1api.TypedLocalObjectReference{ APIGroup: &corev1api.SchemeGroupVersion.Group, Kind: "ConfigMap", Name: "resourceModifier", }) d.out.Flush() expectOutput := `Resource modifier: Type: ConfigMap Name: resourceModifier ` fmt.Println(d.buf.String()) require.Equal(t, expectOutput, d.buf.String()) } ================================================ FILE: pkg/cmd/util/output/restore_printer.go ================================================ /* Copyright 2017, 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package output import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) var ( restoreColumns = []metav1.TableColumnDefinition{ // name needs Type and Format defined for the decorator to identify it: // https://github.com/kubernetes/kubernetes/blob/v1.15.3/pkg/printers/tableprinter.go#L204 {Name: "Name", Type: "string", Format: "name"}, {Name: "Backup"}, {Name: "Status"}, {Name: "Started"}, {Name: "Completed"}, {Name: "Errors"}, {Name: "Warnings"}, {Name: "Created"}, {Name: "Selector"}, } ) func printRestoreList(list *v1.RestoreList) []metav1.TableRow { rows := make([]metav1.TableRow, 0, len(list.Items)) for i := range list.Items { rows = append(rows, printRestore(&list.Items[i])...) } return rows } func printRestore(restore *v1.Restore) []metav1.TableRow { row := metav1.TableRow{ Object: runtime.RawExtension{Object: restore}, } status := restore.Status.Phase if status == "" { status = v1.RestorePhaseNew } row.Cells = append(row.Cells, restore.Name, restore.Spec.BackupName, status, restore.Status.StartTimestamp, restore.Status.CompletionTimestamp, restore.Status.Errors, restore.Status.Warnings, restore.CreationTimestamp.Time, metav1.FormatLabelSelector(restore.Spec.LabelSelector), ) return []metav1.TableRow{row} } ================================================ FILE: pkg/cmd/util/output/schedule_describe_test.go ================================================ package output import ( "testing" "github.com/stretchr/testify/assert" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" ) func TestDescribeSchedule(t *testing.T) { input1 := builder.ForSchedule("velero", "schedule-1"). Phase(velerov1api.SchedulePhaseFailedValidation). ValidationError("validation failed").Result() expect1 := `Name: schedule-1 Namespace: velero Labels: Annotations: Phase: FailedValidation Validation errors: validation failed Paused: false Schedule: Backup Template: Namespaces: Included: * Excluded: Resources: Included cluster-scoped: Excluded cluster-scoped: Included namespace-scoped: * Excluded namespace-scoped: Label selector: Or label selector: Storage Location: Velero-Native Snapshot PVs: auto Snapshot Move Data: auto Data Mover: velero TTL: 0s CSISnapshotTimeout: 0s ItemOperationTimeout: 0s Hooks: Last Backup: ` input2 := builder.ForSchedule("velero", "schedule-2"). Phase(velerov1api.SchedulePhaseEnabled). CronSchedule("0 0 * * *"). Template(builder.ForBackup("velero", "backup-1").ParallelFilesUpload(10).Result().Spec). LastBackupTime("2023-06-25 15:04:05").Result() expect2 := `Name: schedule-2 Namespace: velero Labels: Annotations: Phase: Enabled Uploader config: Parallel files upload: 10 Paused: false Schedule: 0 0 * * * Backup Template: Namespaces: Included: * Excluded: Resources: Included cluster-scoped: Excluded cluster-scoped: Included namespace-scoped: * Excluded namespace-scoped: Label selector: Or label selector: Storage Location: Velero-Native Snapshot PVs: auto Snapshot Move Data: auto Data Mover: velero TTL: 0s CSISnapshotTimeout: 0s ItemOperationTimeout: 0s Hooks: Last Backup: 2023-06-25 15:04:05 +0000 UTC ` input3 := builder.ForSchedule("velero", "schedule-3"). Phase(velerov1api.SchedulePhaseEnabled). CronSchedule("0 0 * * *"). Template(builder.ForBackup("velero", "backup-1").DefaultVolumesToFsBackup(true).Result().Spec). LastBackupTime("2023-06-25 15:04:05").Result() expect3 := `Name: schedule-3 Namespace: velero Labels: Annotations: Phase: Enabled Paused: false Schedule: 0 0 * * * Backup Template: Namespaces: Included: * Excluded: Resources: Included cluster-scoped: Excluded cluster-scoped: Included namespace-scoped: * Excluded namespace-scoped: Label selector: Or label selector: Storage Location: Velero-Native Snapshot PVs: auto File System Backup (Default): true Snapshot Move Data: auto Data Mover: velero TTL: 0s CSISnapshotTimeout: 0s ItemOperationTimeout: 0s Hooks: Last Backup: 2023-06-25 15:04:05 +0000 UTC ` input4 := builder.ForSchedule("velero", "schedule-4"). Phase(velerov1api.SchedulePhaseEnabled). CronSchedule("0 0 * * *"). Template(builder.ForBackup("velero", "backup-1").DefaultVolumesToFsBackup(false).Result().Spec). LastBackupTime("2023-06-25 15:04:05").Result() expect4 := `Name: schedule-4 Namespace: velero Labels: Annotations: Phase: Enabled Paused: false Schedule: 0 0 * * * Backup Template: Namespaces: Included: * Excluded: Resources: Included cluster-scoped: Excluded cluster-scoped: Included namespace-scoped: * Excluded namespace-scoped: Label selector: Or label selector: Storage Location: Velero-Native Snapshot PVs: auto File System Backup (Default): false Snapshot Move Data: auto Data Mover: velero TTL: 0s CSISnapshotTimeout: 0s ItemOperationTimeout: 0s Hooks: Last Backup: 2023-06-25 15:04:05 +0000 UTC ` testcases := []struct { name string input *velerov1api.Schedule expect string }{ { name: "schedule failed in validation", input: input1, expect: expect1, }, { name: "schedule enabled", input: input2, expect: expect2, }, { name: "schedule with DefaultVolumesToFsBackup is true", input: input3, expect: expect3, }, { name: "schedule with DefaultVolumesToFsBackup is false", input: input4, expect: expect4, }, } for _, tc := range testcases { t.Run(tc.name, func(tt *testing.T) { assert.Equal(tt, tc.expect, DescribeSchedule(tc.input)) }) } } ================================================ FILE: pkg/cmd/util/output/schedule_describer.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package output import ( "fmt" "github.com/fatih/color" v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) func DescribeSchedule(schedule *v1.Schedule) string { return Describe(func(d *Describer) { d.DescribeMetadata(schedule.ObjectMeta) d.Println() phase := schedule.Status.Phase if phase == "" { phase = v1.SchedulePhaseNew } phaseString := string(phase) switch phase { case v1.SchedulePhaseEnabled: phaseString = color.GreenString(phaseString) case v1.SchedulePhaseFailedValidation: phaseString = color.RedString(phaseString) } d.Printf("Phase:\t%s\n", phaseString) if schedule.Spec.Template.ResourcePolicy != nil { d.Println() DescribeResourcePolicies(d, schedule.Spec.Template.ResourcePolicy) } if schedule.Spec.Template.UploaderConfig != nil && schedule.Spec.Template.UploaderConfig.ParallelFilesUpload > 0 { d.Println() DescribeUploaderConfigForBackup(d, schedule.Spec.Template) } status := schedule.Status if len(status.ValidationErrors) > 0 { d.Println() d.Printf("Validation errors:") for _, ve := range status.ValidationErrors { d.Printf("\t%s\n", color.RedString(ve)) } } d.Println() d.Printf("Paused:\t%t\n", schedule.Spec.Paused) d.Println() DescribeScheduleSpec(d, schedule.Spec) d.Println() DescribeScheduleStatus(d, schedule.Status) }) } func DescribeScheduleSpec(d *Describer, spec v1.ScheduleSpec) { d.Printf("Schedule:\t%s\n", spec.Schedule) d.Println() d.Println("Backup Template:") d.Prefix = "\t" DescribeBackupSpec(d, spec.Template) d.Prefix = "" } func DescribeScheduleStatus(d *Describer, status v1.ScheduleStatus) { lastBackup := "" if status.LastBackup != nil && !status.LastBackup.Time.IsZero() { lastBackup = fmt.Sprintf("%v", status.LastBackup.Time) } d.Printf("Last Backup:\t%s\n", lastBackup) } ================================================ FILE: pkg/cmd/util/output/schedule_printer.go ================================================ /* Copyright 2017, 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package output import ( "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) var ( scheduleColumns = []metav1.TableColumnDefinition{ // name needs Type and Format defined for the decorator to identify it: // https://github.com/kubernetes/kubernetes/blob/v1.15.3/pkg/printers/tableprinter.go#L204 {Name: "Name", Type: "string", Format: "name"}, {Name: "Status"}, {Name: "Created"}, {Name: "Schedule"}, {Name: "Backup TTL"}, {Name: "Last Backup"}, {Name: "Selector"}, {Name: "Paused"}, } ) func printScheduleList(list *v1.ScheduleList) []metav1.TableRow { rows := make([]metav1.TableRow, 0, len(list.Items)) for i := range list.Items { rows = append(rows, printSchedule(&list.Items[i])...) } return rows } func printSchedule(schedule *v1.Schedule) []metav1.TableRow { row := metav1.TableRow{ Object: runtime.RawExtension{Object: schedule}, } status := schedule.Status.Phase if status == "" { status = v1.SchedulePhaseNew } var lastBackupTime time.Time if schedule.Status.LastBackup != nil { lastBackupTime = schedule.Status.LastBackup.Time } row.Cells = append(row.Cells, schedule.Name, status, schedule.CreationTimestamp.Time, schedule.Spec.Schedule, schedule.Spec.Template.TTL.Duration, humanReadableTimeFromNow(lastBackupTime), metav1.FormatLabelSelector(schedule.Spec.Template.LabelSelector), schedule.Spec.Paused, ) return []metav1.TableRow{row} } ================================================ FILE: pkg/cmd/util/output/volume_snapshot_location_printer.go ================================================ /* Copyright 2018, 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package output import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) var ( volumeSnapshotLocationColumns = []metav1.TableColumnDefinition{ // name needs Type and Format defined for the decorator to identify it: // https://github.com/kubernetes/kubernetes/blob/v1.15.3/pkg/printers/tableprinter.go#L204 {Name: "Name", Type: "string", Format: "name"}, {Name: "Provider"}, } ) func printVolumeSnapshotLocationList(list *v1.VolumeSnapshotLocationList) []metav1.TableRow { rows := make([]metav1.TableRow, 0, len(list.Items)) for i := range list.Items { rows = append(rows, printVolumeSnapshotLocation(&list.Items[i])...) } return rows } func printVolumeSnapshotLocation(location *v1.VolumeSnapshotLocation) []metav1.TableRow { row := metav1.TableRow{ Object: runtime.RawExtension{Object: location}, } row.Cells = append(row.Cells, location.Name, location.Spec.Provider, ) return []metav1.TableRow{row} } ================================================ FILE: pkg/cmd/util/signals/signals.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package signals import ( "context" "os" "os/signal" "syscall" "github.com/sirupsen/logrus" ) // CancelOnShutdown starts a goroutine that will call cancelFunc when // either SIGINT or SIGTERM is received func CancelOnShutdown(cancelFunc context.CancelFunc, logger logrus.FieldLogger) { sigs := make(chan os.Signal, 1) signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) go func() { sig := <-sigs logger.Infof("Received signal %s, shutting down", sig) cancelFunc() }() } ================================================ FILE: pkg/cmd/velero/velero.go ================================================ /* Copyright 2021 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package velero import ( "flag" "fmt" "os" "github.com/fatih/color" "github.com/spf13/cobra" "k8s.io/klog/v2" "github.com/vmware-tanzu/velero/pkg/cmd/cli/debug" "github.com/vmware-tanzu/velero/pkg/cmd/cli/podvolume" "github.com/vmware-tanzu/velero/pkg/cmd/cli/repomantenance" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd/cli/backup" "github.com/vmware-tanzu/velero/pkg/cmd/cli/backuplocation" "github.com/vmware-tanzu/velero/pkg/cmd/cli/bug" cliclient "github.com/vmware-tanzu/velero/pkg/cmd/cli/client" "github.com/vmware-tanzu/velero/pkg/cmd/cli/completion" "github.com/vmware-tanzu/velero/pkg/cmd/cli/create" "github.com/vmware-tanzu/velero/pkg/cmd/cli/delete" "github.com/vmware-tanzu/velero/pkg/cmd/cli/describe" "github.com/vmware-tanzu/velero/pkg/cmd/cli/get" "github.com/vmware-tanzu/velero/pkg/cmd/cli/install" "github.com/vmware-tanzu/velero/pkg/cmd/cli/plugin" "github.com/vmware-tanzu/velero/pkg/cmd/cli/repo" "github.com/vmware-tanzu/velero/pkg/cmd/cli/restore" "github.com/vmware-tanzu/velero/pkg/cmd/cli/schedule" "github.com/vmware-tanzu/velero/pkg/cmd/cli/snapshotlocation" "github.com/vmware-tanzu/velero/pkg/cmd/cli/uninstall" "github.com/vmware-tanzu/velero/pkg/cmd/cli/version" "github.com/vmware-tanzu/velero/pkg/cmd/server" runplugin "github.com/vmware-tanzu/velero/pkg/cmd/server/plugin" veleroflag "github.com/vmware-tanzu/velero/pkg/cmd/util/flag" "github.com/vmware-tanzu/velero/pkg/features" "github.com/vmware-tanzu/velero/pkg/cmd/cli/datamover" "github.com/vmware-tanzu/velero/pkg/cmd/cli/nodeagent" ) func NewCommand(name string) *cobra.Command { // Load the config here so that we can extract features from it. config, err := client.LoadConfig() if err != nil { fmt.Fprintf(os.Stderr, "WARNING: Error reading config file: %v\n", err) } // Declare cmdFeatures and cmdColorzied here so we can access them in the PreRun hooks // without doing a chain of calls into the command's FlagSet var cmdFeatures veleroflag.StringArray var cmdColorzied veleroflag.OptionalBool c := &cobra.Command{ Use: name, Short: "Back up and restore Kubernetes cluster resources.", Long: `Velero is a tool for managing disaster recovery, specifically for Kubernetes cluster resources. It provides a simple, configurable, and operationally robust way to back up your application state and associated data. If you're familiar with kubectl, Velero supports a similar model, allowing you to execute commands such as 'velero get backup' and 'velero create schedule'. The same operations can also be performed as 'velero backup get' and 'velero schedule create'.`, // PersistentPreRun will run before all subcommands EXCEPT in the following conditions: // - a subcommand defines its own PersistentPreRun function // - the command is run without arguments or with --help and only prints the usage info PersistentPreRun: func(cmd *cobra.Command, args []string) { features.Enable(config.Features()...) features.Enable(cmdFeatures...) switch { case cmdColorzied.Value != nil: color.NoColor = !*cmdColorzied.Value default: color.NoColor = !config.Colorized() } }, } f := client.NewFactory(name, config) f.BindFlags(c.PersistentFlags()) // Bind features directly to the root command so it's available to all callers. c.PersistentFlags().Var(&cmdFeatures, "features", "Comma-separated list of features to enable for this Velero process. Combines with values from $HOME/.config/velero/config.json if present") // Color will be enabled or disabled for all subcommands c.PersistentFlags().Var(&cmdColorzied, "colorized", "Show colored output in TTY. Overrides 'colorized' value from $HOME/.config/velero/config.json if present. Enabled by default") c.AddCommand( backup.NewCommand(f), schedule.NewCommand(f), restore.NewCommand(f), server.NewCommand(f), nodeagent.NewCommand(f), version.NewCommand(f), get.NewCommand(f), install.NewCommand(f), uninstall.NewCommand(f), describe.NewCommand(f), create.NewCommand(f), runplugin.NewCommand(f), plugin.NewCommand(f), delete.NewCommand(f), cliclient.NewCommand(), completion.NewCommand(), repo.NewCommand(f), bug.NewCommand(), backuplocation.NewCommand(f), snapshotlocation.NewCommand(f), debug.NewCommand(f), repomantenance.NewCommand(f), datamover.NewCommand(f), podvolume.NewCommand(f), ) // init and add the klog flags klog.InitFlags(flag.CommandLine) c.PersistentFlags().AddGoFlagSet(flag.CommandLine) return c } ================================================ FILE: pkg/constant/constant.go ================================================ package constant const ( ControllerBackupQueue = "backup-queue" ControllerBackup = "backup" ControllerBackupOperations = "backup-operations" ControllerBackupDeletion = "backup-deletion" ControllerBackupFinalizer = "backup-finalizer" ControllerBackupRepo = "backup-repo" ControllerBackupStorageLocation = "backup-storage-location" ControllerBackupSync = "backup-sync" ControllerDataDownload = "data-download" ControllerDataUpload = "data-upload" ControllerDownloadRequest = "download-request" ControllerGarbageCollection = "gc" ControllerPodVolumeBackup = "pod-volume-backup" ControllerPodVolumeRestore = "pod-volume-restore" ControllerRestore = "restore" ControllerRestoreOperations = "restore-operations" ControllerSchedule = "schedule" ControllerServerStatusRequest = "server-status-request" ControllerRestoreFinalizer = "restore-finalizer" PluginCSIPVCRestoreRIA = "velero.io/csi-pvc-restorer" PluginCsiVolumeSnapshotRestoreRIA = "velero.io/csi-volumesnapshot-restorer" DefaultEphemeralStorageRequest = "0" DefaultEphemeralStorageLimit = "0" ) ================================================ FILE: pkg/controller/backup_controller.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package controller import ( "bytes" "context" "fmt" "os" "slices" "time" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" kerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/utils/clock" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" kbclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/predicate" "github.com/vmware-tanzu/velero/internal/credentials" "github.com/vmware-tanzu/velero/internal/resourcepolicies" "github.com/vmware-tanzu/velero/internal/storage" "github.com/vmware-tanzu/velero/internal/volume" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" pkgbackup "github.com/vmware-tanzu/velero/pkg/backup" "github.com/vmware-tanzu/velero/pkg/constant" "github.com/vmware-tanzu/velero/pkg/discovery" "github.com/vmware-tanzu/velero/pkg/features" "github.com/vmware-tanzu/velero/pkg/label" "github.com/vmware-tanzu/velero/pkg/metrics" "github.com/vmware-tanzu/velero/pkg/persistence" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" "github.com/vmware-tanzu/velero/pkg/plugin/framework" "github.com/vmware-tanzu/velero/pkg/util/boolptr" "github.com/vmware-tanzu/velero/pkg/util/collections" "github.com/vmware-tanzu/velero/pkg/util/encode" kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube" "github.com/vmware-tanzu/velero/pkg/util/logging" "github.com/vmware-tanzu/velero/pkg/util/results" veleroutil "github.com/vmware-tanzu/velero/pkg/util/velero" ) const ( backupResyncPeriod = time.Minute ) var autoExcludeNamespaceScopedResources = []string{ // CSI VolumeSnapshot and VolumeSnapshotContent are intermediate resources. // Velero only handle the VS and VSC created during backup, // not during resource collecting. "volumesnapshots.snapshot.storage.k8s.io", } var autoExcludeClusterScopedResources = []string{ // CSI VolumeSnapshot and VolumeSnapshotContent are intermediate resources. // Velero only handle the VS and VSC created during backup, // not during resource collecting. "volumesnapshotcontents.snapshot.storage.k8s.io", } type backupReconciler struct { ctx context.Context logger logrus.FieldLogger discoveryHelper discovery.Helper backupper pkgbackup.Backupper kbClient kbclient.Client clock clock.WithTickerAndDelayedExecution backupLogLevel logrus.Level newPluginManager func(logrus.FieldLogger) clientmgmt.Manager backupTracker BackupTracker defaultBackupLocation string defaultVolumesToFsBackup bool defaultBackupTTL time.Duration defaultVGSLabelKey string defaultCSISnapshotTimeout time.Duration resourceTimeout time.Duration defaultItemOperationTimeout time.Duration defaultSnapshotLocations map[string]string metrics *metrics.ServerMetrics backupStoreGetter persistence.ObjectBackupStoreGetter formatFlag logging.Format credentialFileStore credentials.FileStore maxConcurrentK8SConnections int defaultSnapshotMoveData bool globalCRClient kbclient.Client itemBlockWorkerCount int concurrentBackups int } func NewBackupReconciler( ctx context.Context, discoveryHelper discovery.Helper, backupper pkgbackup.Backupper, logger logrus.FieldLogger, backupLogLevel logrus.Level, newPluginManager func(logrus.FieldLogger) clientmgmt.Manager, backupTracker BackupTracker, kbClient kbclient.Client, defaultBackupLocation string, defaultVolumesToFsBackup bool, defaultBackupTTL time.Duration, defaultVGSLabelKey string, defaultCSISnapshotTimeout time.Duration, resourceTimeout time.Duration, defaultItemOperationTimeout time.Duration, defaultSnapshotLocations map[string]string, metrics *metrics.ServerMetrics, backupStoreGetter persistence.ObjectBackupStoreGetter, formatFlag logging.Format, credentialStore credentials.FileStore, maxConcurrentK8SConnections int, defaultSnapshotMoveData bool, itemBlockWorkerCount int, concurrentBackups int, globalCRClient kbclient.Client, ) *backupReconciler { b := &backupReconciler{ ctx: ctx, discoveryHelper: discoveryHelper, backupper: backupper, clock: &clock.RealClock{}, logger: logger, backupLogLevel: backupLogLevel, newPluginManager: newPluginManager, backupTracker: backupTracker, kbClient: kbClient, defaultBackupLocation: defaultBackupLocation, defaultVolumesToFsBackup: defaultVolumesToFsBackup, defaultBackupTTL: defaultBackupTTL, defaultVGSLabelKey: defaultVGSLabelKey, defaultCSISnapshotTimeout: defaultCSISnapshotTimeout, resourceTimeout: resourceTimeout, defaultItemOperationTimeout: defaultItemOperationTimeout, defaultSnapshotLocations: defaultSnapshotLocations, metrics: metrics, backupStoreGetter: backupStoreGetter, formatFlag: formatFlag, credentialFileStore: credentialStore, maxConcurrentK8SConnections: maxConcurrentK8SConnections, defaultSnapshotMoveData: defaultSnapshotMoveData, itemBlockWorkerCount: itemBlockWorkerCount, concurrentBackups: max(concurrentBackups, 1), globalCRClient: globalCRClient, } b.updateTotalBackupMetric() return b } func (b *backupReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&velerov1api.Backup{}, builder.WithPredicates(predicate.Funcs{ UpdateFunc: func(ue event.UpdateEvent) bool { backup := ue.ObjectNew.(*velerov1api.Backup) return backup.Status.Phase == velerov1api.BackupPhaseReadyToStart }, CreateFunc: func(ce event.CreateEvent) bool { return false }, DeleteFunc: func(de event.DeleteEvent) bool { return false }, GenericFunc: func(ge event.GenericEvent) bool { return false }, })). WithOptions(controller.Options{ MaxConcurrentReconciles: b.concurrentBackups, }). Named(constant.ControllerBackup). Complete(b) } func (b *backupReconciler) updateTotalBackupMetric() { go func() { // Wait for 5 seconds to let controller-runtime to setup k8s clients. time.Sleep(5 * time.Second) wait.Until( func() { // recompute backup_total metric backups := &velerov1api.BackupList{} err := b.kbClient.List(context.Background(), backups, &kbclient.ListOptions{LabelSelector: labels.Everything()}) if err != nil { b.logger.Error(err, "Error computing backup_total metric") } else { b.metrics.SetBackupTotal(int64(len(backups.Items))) } // recompute backup_last_successful_timestamp metric for each // schedule (including the empty schedule, i.e. ad-hoc backups) for schedule, timestamp := range getLastSuccessBySchedule(backups.Items) { b.metrics.SetBackupLastSuccessfulTimestamp(schedule, timestamp) } }, backupResyncPeriod, b.ctx.Done(), ) }() } // getLastSuccessBySchedule finds the most recent completed backup for each schedule // and returns a map of schedule name -> completion time of the most recent completed // backup. This map includes an entry for ad-hoc/non-scheduled backups, where the key // is the empty string. func getLastSuccessBySchedule(backups []velerov1api.Backup) map[string]time.Time { lastSuccessBySchedule := map[string]time.Time{} for _, backup := range backups { if backup.Status.Phase != velerov1api.BackupPhaseCompleted { continue } if backup.Status.CompletionTimestamp == nil { continue } schedule := backup.Labels[velerov1api.ScheduleNameLabel] timestamp := backup.Status.CompletionTimestamp.Time if timestamp.After(lastSuccessBySchedule[schedule]) { lastSuccessBySchedule[schedule] = timestamp } } return lastSuccessBySchedule } func (b *backupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := b.logger.WithFields(logrus.Fields{ "controller": constant.ControllerBackup, "backuprequest": req.String(), }) log.Debug("Getting backup") original := &velerov1api.Backup{} err := b.kbClient.Get(ctx, req.NamespacedName, original) if err != nil { if apierrors.IsNotFound(err) { log.Debug("backup not found") return ctrl.Result{}, nil } log.WithError(err).Error("error getting backup") return ctrl.Result{}, err } // Double-check we have the correct phase. In the unlikely event that multiple controller // instances are running, it's possible for controller A to succeed in changing the phase to // InProgress, while controller B's attempt to patch the phase fails. When controller B // reprocesses the same backup, it will either show up as New (informer hasn't seen the update // yet) or as InProgress. In the former case, the patch attempt will fail again, until the // informer sees the update. In the latter case, after the informer has seen the update to // InProgress, we still need this check so we can return nil to indicate we've finished processing // this key (even though it was a no-op). switch original.Status.Phase { case velerov1api.BackupPhaseReadyToStart: // only process ReadytToStart backups default: b.logger.WithFields(logrus.Fields{ "backup": kubeutil.NamespaceAndName(original), "phase": original.Status.Phase, }).Debug("Backup is not handled") return ctrl.Result{}, nil } log.Debug("Preparing backup request") request := b.prepareBackupRequest(ctx, original, log) // delete worker pool after reconcile defer request.StopWorkerPool() if len(request.Status.ValidationErrors) > 0 { request.Status.Phase = velerov1api.BackupPhaseFailedValidation } else { request.Status.Phase = velerov1api.BackupPhaseInProgress request.Status.StartTimestamp = &metav1.Time{Time: b.clock.Now()} } // update status to // BackupPhaseFailedValidation // BackupPhaseInProgress // if patch fail, backup can reconcile again as phase would still be "" or New if err := kubeutil.PatchResource(original, request.Backup, b.kbClient); err != nil { return ctrl.Result{}, errors.Wrapf(err, "error updating Backup status to %s", request.Status.Phase) } backupScheduleName := request.GetLabels()[velerov1api.ScheduleNameLabel] b.backupTracker.Add(request.Namespace, request.Name) defer func() { switch request.Status.Phase { case velerov1api.BackupPhaseCompleted, velerov1api.BackupPhasePartiallyFailed, velerov1api.BackupPhaseFailed, velerov1api.BackupPhaseFailedValidation: b.backupTracker.Delete(request.Namespace, request.Name) case velerov1api.BackupPhaseWaitingForPluginOperations, velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed, velerov1api.BackupPhaseFinalizing, velerov1api.BackupPhaseFinalizingPartiallyFailed: b.backupTracker.AddPostProcessing(request.Namespace, request.Name) } }() if request.Status.Phase == velerov1api.BackupPhaseFailedValidation { log.Debug("failed to validate backup status") b.metrics.RegisterBackupValidationFailure(backupScheduleName) b.metrics.RegisterBackupLastStatus(backupScheduleName, metrics.BackupLastStatusFailure) return ctrl.Result{}, nil } // store ref to just-updated item for creating patch original = request.Backup.DeepCopy() log.Debug("Running backup") b.metrics.RegisterBackupAttempt(backupScheduleName) // execution & upload of backup if err := b.runBackup(request); err != nil { // even though runBackup sets the backup's phase prior // to uploading artifacts to object storage, we have to // check for an error again here and update the phase if // one is found, because there could've been an error // while uploading artifacts to object storage, which would // result in the backup being Failed. log.WithError(err).Error("backup failed") request.Status.Phase = velerov1api.BackupPhaseFailed request.Status.FailureReason = err.Error() } switch request.Status.Phase { case velerov1api.BackupPhaseCompleted: b.metrics.RegisterBackupSuccess(backupScheduleName) b.metrics.RegisterBackupLastStatus(backupScheduleName, metrics.BackupLastStatusSucc) case velerov1api.BackupPhasePartiallyFailed: b.metrics.RegisterBackupPartialFailure(backupScheduleName) b.metrics.RegisterBackupLastStatus(backupScheduleName, metrics.BackupLastStatusFailure) case velerov1api.BackupPhaseFailed: b.metrics.RegisterBackupFailed(backupScheduleName) b.metrics.RegisterBackupLastStatus(backupScheduleName, metrics.BackupLastStatusFailure) case velerov1api.BackupPhaseFailedValidation: b.metrics.RegisterBackupValidationFailure(backupScheduleName) b.metrics.RegisterBackupLastStatus(backupScheduleName, metrics.BackupLastStatusFailure) } log.Info("Updating backup's status") // Phases were updated in runBackup() // This patch with retry update Phase from InProgress to // BackupPhaseWaitingForPluginOperations -> backup_operations_controller.go will now reconcile // BackupPhaseWaitingForPluginOperationsPartiallyFailed -> backup_operations_controller.go will now reconcile // BackupPhaseFinalizing -> backup_finalizer_controller.go will now reconcile // BackupPhaseFinalizingPartiallyFailed -> backup_finalizer_controller.go will now reconcile // BackupPhaseFailed if err := kubeutil.PatchResourceWithRetriesOnErrors(b.resourceTimeout, original, request.Backup, b.kbClient); err != nil { log.WithError(err).Errorf("error updating backup's status from %v to %v", original.Status.Phase, request.Backup.Status.Phase) } return ctrl.Result{}, nil } func (b *backupReconciler) prepareBackupRequest(ctx context.Context, backup *velerov1api.Backup, logger logrus.FieldLogger) *pkgbackup.Request { request := &pkgbackup.Request{ Backup: backup.DeepCopy(), // don't modify items in the cache SkippedPVTracker: pkgbackup.NewSkipPVTracker(), BackedUpItems: pkgbackup.NewBackedUpItemsMap(), WorkerPool: pkgbackup.StartItemBlockWorkerPool(ctx, b.itemBlockWorkerCount, logger), } request.VolumesInformation.Init() // set backup major version - deprecated, use Status.FormatVersion request.Status.Version = pkgbackup.BackupVersion // set backup major, minor, and patch version request.Status.FormatVersion = pkgbackup.BackupFormatVersion if request.Spec.TTL.Duration == 0 { // set default backup TTL request.Spec.TTL.Duration = b.defaultBackupTTL } if len(request.Spec.VolumeGroupSnapshotLabelKey) == 0 { request.Spec.VolumeGroupSnapshotLabelKey = b.defaultVGSLabelKey } if request.Spec.CSISnapshotTimeout.Duration == 0 { // set default CSI VolumeSnapshot timeout request.Spec.CSISnapshotTimeout.Duration = b.defaultCSISnapshotTimeout } if request.Spec.ItemOperationTimeout.Duration == 0 { // set default item operation timeout request.Spec.ItemOperationTimeout.Duration = b.defaultItemOperationTimeout } // calculate expiration request.Status.Expiration = &metav1.Time{Time: b.clock.Now().Add(request.Spec.TTL.Duration)} // TODO: After we drop the support for backup v1 CR. Remove this code block after DefaultVolumesToRestic is removed from CRD // For now, for CRs created by old versions, we need to respect the DefaultVolumesToRestic value if it is set true if boolptr.IsSetToTrue(request.Spec.DefaultVolumesToRestic) { logger.Warn("DefaultVolumesToRestic field will be deprecated, use DefaultVolumesToFsBackup instead. Automatically remap it to DefaultVolumesToFsBackup") request.Spec.DefaultVolumesToFsBackup = request.Spec.DefaultVolumesToRestic } if request.Spec.DefaultVolumesToFsBackup == nil { request.Spec.DefaultVolumesToFsBackup = &b.defaultVolumesToFsBackup } if request.Spec.SnapshotMoveData == nil { request.Spec.SnapshotMoveData = &b.defaultSnapshotMoveData } // find which storage location to use var serverSpecified bool if request.Spec.StorageLocation == "" { // when the user doesn't specify a location, use the server default unless there is an existing BSL marked as default // TODO(2.0) b.defaultBackupLocation will be deprecated request.Spec.StorageLocation = b.defaultBackupLocation locationList, err := storage.ListBackupStorageLocations(context.Background(), b.kbClient, request.Namespace) if err == nil { for _, location := range locationList.Items { if location.Spec.Default { request.Spec.StorageLocation = location.Name break } } } serverSpecified = true } // get the storage location, and store the BackupStorageLocation API obj on the request storageLocation := &velerov1api.BackupStorageLocation{} if err := b.kbClient.Get(context.Background(), kbclient.ObjectKey{ Namespace: request.Namespace, Name: request.Spec.StorageLocation, }, storageLocation); err != nil { if apierrors.IsNotFound(err) { if serverSpecified { // TODO(2.0) remove this. For now, without mentioning "server default" it could be confusing trying to grasp where the default came from. request.Status.ValidationErrors = append(request.Status.ValidationErrors, fmt.Sprintf("an existing backup storage location was not specified at backup creation time and the server default %s does not exist. Please address this issue (see `velero backup-location -h` for options) and create a new backup. Error: %v", request.Spec.StorageLocation, err)) } else { request.Status.ValidationErrors = append(request.Status.ValidationErrors, fmt.Sprintf("an existing backup storage location was not specified at backup creation time and the default %s was not found. Please address this issue (see `velero backup-location -h` for options) and create a new backup. Error: %v", request.Spec.StorageLocation, err)) } } else { request.Status.ValidationErrors = append(request.Status.ValidationErrors, fmt.Sprintf("error getting backup storage location: %v", err)) } } else { request.StorageLocation = storageLocation if request.StorageLocation.Spec.AccessMode == velerov1api.BackupStorageLocationAccessModeReadOnly { request.Status.ValidationErrors = append(request.Status.ValidationErrors, fmt.Sprintf("backup can't be created because backup storage location %s is currently in read-only mode", request.StorageLocation.Name)) } if !veleroutil.BSLIsAvailable(*request.StorageLocation) { request.Status.ValidationErrors = append( request.Status.ValidationErrors, fmt.Sprintf("backup can't be created because BackupStorageLocation %s is in Unavailable status.", request.StorageLocation.Name), ) } } // add the storage location as a label for easy filtering later. if request.Labels == nil { request.Labels = make(map[string]string) } request.Labels[velerov1api.StorageLocationLabel] = label.GetValidName(request.Spec.StorageLocation) // validate and get the backup's VolumeSnapshotLocations, and store the // VolumeSnapshotLocation API objs on the request if locs, errs := b.validateAndGetSnapshotLocations(request.Backup); len(errs) > 0 { request.Status.ValidationErrors = append(request.Status.ValidationErrors, errs...) } else { request.Spec.VolumeSnapshotLocations = nil for _, loc := range locs { request.Spec.VolumeSnapshotLocations = append(request.Spec.VolumeSnapshotLocations, loc.Name) request.SnapshotLocations = append(request.SnapshotLocations, loc) } } // Getting all information of cluster version - useful for future skip-level migration if request.Annotations == nil { request.Annotations = make(map[string]string) } request.Annotations[velerov1api.SourceClusterK8sGitVersionAnnotation] = b.discoveryHelper.ServerVersion().String() request.Annotations[velerov1api.SourceClusterK8sMajorVersionAnnotation] = b.discoveryHelper.ServerVersion().Major request.Annotations[velerov1api.SourceClusterK8sMinorVersionAnnotation] = b.discoveryHelper.ServerVersion().Minor request.Annotations[velerov1api.ResourceTimeoutAnnotation] = b.resourceTimeout.String() // Add namespaces with label velero.io/exclude-from-backup=true into request.Spec.ExcludedNamespaces // Essentially, adding the label velero.io/exclude-from-backup=true to a namespace would be equivalent to setting spec.ExcludedNamespaces namespaces := corev1api.NamespaceList{} if err := b.kbClient.List(context.Background(), &namespaces, kbclient.MatchingLabels{velerov1api.ExcludeFromBackupLabel: "true"}); err == nil { for _, ns := range namespaces.Items { request.Spec.ExcludedNamespaces = append(request.Spec.ExcludedNamespaces, ns.Name) } } else { request.Status.ValidationErrors = append(request.Status.ValidationErrors, fmt.Sprintf("error getting namespace list: %v", err)) } // validate whether Included/Excluded resources and IncludedClusterResource are mixed with // Included/Excluded cluster-scoped/namespace-scoped resources. if oldAndNewFilterParametersUsedTogether(request.Spec) { validatedError := fmt.Sprintf("include-resources, exclude-resources and include-cluster-resources are old filter parameters.\n" + "include-cluster-scoped-resources, exclude-cluster-scoped-resources, include-namespace-scoped-resources and exclude-namespace-scoped-resources are new filter parameters.\n" + "They cannot be used together") request.Status.ValidationErrors = append(request.Status.ValidationErrors, validatedError) } if collections.UseOldResourceFilters(request.Spec) { // validate the included/excluded resources ieErr := collections.ValidateIncludesExcludes(request.Spec.IncludedResources, request.Spec.ExcludedResources) if len(ieErr) > 0 { for _, err := range ieErr { request.Status.ValidationErrors = append(request.Status.ValidationErrors, fmt.Sprintf("Invalid included/excluded resource lists: %v", err)) } } else { request.Spec.IncludedResources, request.Spec.ExcludedResources = modifyResourceIncludeExclude( request.Spec.IncludedResources, request.Spec.ExcludedResources, append(autoExcludeNamespaceScopedResources, autoExcludeClusterScopedResources...), ) } } else { // validate the cluster-scoped included/excluded resources clusterErr := collections.ValidateScopedIncludesExcludes(request.Spec.IncludedClusterScopedResources, request.Spec.ExcludedClusterScopedResources) if len(clusterErr) > 0 { for _, err := range clusterErr { request.Status.ValidationErrors = append(request.Status.ValidationErrors, fmt.Sprintf("Invalid cluster-scoped included/excluded resource lists: %s", err)) } } else { request.Spec.IncludedClusterScopedResources, request.Spec.ExcludedClusterScopedResources = modifyResourceIncludeExclude( request.Spec.IncludedClusterScopedResources, request.Spec.ExcludedClusterScopedResources, autoExcludeClusterScopedResources, ) } // validate the namespace-scoped included/excluded resources namespaceErr := collections.ValidateScopedIncludesExcludes(request.Spec.IncludedNamespaceScopedResources, request.Spec.ExcludedNamespaceScopedResources) if len(namespaceErr) > 0 { for _, err := range namespaceErr { request.Status.ValidationErrors = append(request.Status.ValidationErrors, fmt.Sprintf("Invalid namespace-scoped included/excluded resource lists: %s", err)) } } else { request.Spec.IncludedNamespaceScopedResources, request.Spec.ExcludedNamespaceScopedResources = modifyResourceIncludeExclude( request.Spec.IncludedNamespaceScopedResources, request.Spec.ExcludedNamespaceScopedResources, autoExcludeNamespaceScopedResources, ) } } // validate the included/excluded namespaces for _, err := range collections.ValidateNamespaceIncludesExcludes(request.Spec.IncludedNamespaces, request.Spec.ExcludedNamespaces) { request.Status.ValidationErrors = append(request.Status.ValidationErrors, fmt.Sprintf("Invalid included/excluded namespace lists: %v", err)) } // validate that only one exists orLabelSelector or just labelSelector (singular) if request.Spec.OrLabelSelectors != nil && request.Spec.LabelSelector != nil { request.Status.ValidationErrors = append(request.Status.ValidationErrors, "encountered labelSelector as well as orLabelSelectors in backup spec, only one can be specified") } resourcePolicies, err := resourcepolicies.GetResourcePoliciesFromBackup(*request.Backup, b.kbClient, logger) if err != nil { request.Status.ValidationErrors = append(request.Status.ValidationErrors, err.Error()) } if resourcePolicies != nil && resourcePolicies.GetIncludeExcludePolicy() != nil && collections.UseOldResourceFilters(request.Spec) { request.Status.ValidationErrors = append(request.Status.ValidationErrors, "include-resources, exclude-resources and include-cluster-resources are old filter parameters.\n"+ "They cannot be used with include-exclude policies.") } request.ResPolicies = resourcePolicies return request } // validateAndGetSnapshotLocations gets a collection of VolumeSnapshotLocation objects that // this backup will use (returned as a map of provider name -> VSL), and ensures: // - each location name in .spec.volumeSnapshotLocations exists as a location // - exactly 1 location per provider // - a given provider's default location name is added to .spec.volumeSnapshotLocations if one // is not explicitly specified for the provider (if there's only one location for the provider, // it will automatically be used) // // if backup has snapshotVolume disabled then it returns empty VSL func (b *backupReconciler) validateAndGetSnapshotLocations(backup *velerov1api.Backup) (map[string]*velerov1api.VolumeSnapshotLocation, []string) { errors := []string{} providerLocations := make(map[string]*velerov1api.VolumeSnapshotLocation) for _, locationName := range backup.Spec.VolumeSnapshotLocations { // validate each locationName exists as a VolumeSnapshotLocation location := &velerov1api.VolumeSnapshotLocation{} if err := b.kbClient.Get(context.Background(), kbclient.ObjectKey{Namespace: backup.Namespace, Name: locationName}, location); err != nil { if apierrors.IsNotFound(err) { errors = append(errors, fmt.Sprintf("a VolumeSnapshotLocation CRD for the location %s with the name specified in the backup spec needs to be created before this snapshot can be executed. Error: %v", locationName, err)) } else { errors = append(errors, fmt.Sprintf("error getting volume snapshot location named %s: %v", locationName, err)) } continue } // ensure we end up with exactly 1 location *per provider* if providerLocation, ok := providerLocations[location.Spec.Provider]; ok { // if > 1 location name per provider as in ["aws-us-east-1" | "aws-us-west-1"] (same provider, multiple names) if providerLocation.Name != locationName { errors = append(errors, fmt.Sprintf("more than one VolumeSnapshotLocation name specified for provider %s: %s; unexpected name was %s", location.Spec.Provider, locationName, providerLocation.Name)) continue } } else { // keep track of all valid existing locations, per provider providerLocations[location.Spec.Provider] = location } } if len(errors) > 0 { return nil, errors } volumeSnapshotLocations := &velerov1api.VolumeSnapshotLocationList{} err := b.kbClient.List(context.Background(), volumeSnapshotLocations, &kbclient.ListOptions{Namespace: backup.Namespace, LabelSelector: labels.Everything()}) if err != nil { errors = append(errors, fmt.Sprintf("error listing volume snapshot locations: %v", err)) return nil, errors } // build a map of provider->list of all locations for the provider allProviderLocations := make(map[string][]*velerov1api.VolumeSnapshotLocation) for i := range volumeSnapshotLocations.Items { loc := volumeSnapshotLocations.Items[i] allProviderLocations[loc.Spec.Provider] = append(allProviderLocations[loc.Spec.Provider], &loc) } // go through each provider and make sure we have/can get a VSL // for it for provider, locations := range allProviderLocations { if _, ok := providerLocations[provider]; ok { // backup's spec had a location named for this provider continue } if len(locations) > 1 { // more than one possible location for the provider: check // the defaults defaultLocation := b.defaultSnapshotLocations[provider] if defaultLocation == "" { errors = append(errors, fmt.Sprintf("provider %s has more than one possible volume snapshot location, and none were specified explicitly or as a default", provider)) continue } location := &velerov1api.VolumeSnapshotLocation{} if err := b.kbClient.Get(context.Background(), kbclient.ObjectKey{Namespace: backup.Namespace, Name: defaultLocation}, location); err != nil { errors = append(errors, fmt.Sprintf("error getting volume snapshot location named %s: %v", defaultLocation, err)) continue } providerLocations[provider] = location continue } // exactly one location for the provider: use it providerLocations[provider] = locations[0] } if len(errors) > 0 { return nil, errors } // add credential to config for each location for _, location := range providerLocations { err = volume.UpdateVolumeSnapshotLocationWithCredentialConfig(location, b.credentialFileStore) if err != nil { errors = append(errors, fmt.Sprintf("error adding credentials to volume snapshot location named %s: %v", location.Name, err)) continue } } if len(errors) > 0 { return nil, errors } return providerLocations, nil } // runBackup runs and uploads a validated backup. Any error returned from this function // causes the backup to be Failed; if no error is returned, the backup's status's Errors // field is checked to see if the backup was a partial failure. func (b *backupReconciler) runBackup(backup *pkgbackup.Request) error { b.logger.WithField(constant.ControllerBackup, kubeutil.NamespaceAndName(backup)).Info("Setting up backup log") // Log the backup to both a backup log file and to stdout. This will help see what happened if the upload of the // backup log failed for whatever reason. logCounter := logging.NewLogHook() backupLog, err := logging.NewTempFileLogger(b.backupLogLevel, b.formatFlag, logCounter, logrus.Fields{constant.ControllerBackup: kubeutil.NamespaceAndName(backup)}) if err != nil { return errors.Wrap(err, "error creating dual mode logger for backup") } defer backupLog.Dispose(b.logger.WithField(constant.ControllerBackup, kubeutil.NamespaceAndName(backup))) backupLog.Info("Setting up backup temp file") backupFile, err := os.CreateTemp("", "") if err != nil { return errors.Wrap(err, "error creating temp file for backup") } defer closeAndRemoveFile(backupFile, backupLog) backupLog.Info("Setting up plugin manager") pluginManager := b.newPluginManager(backupLog) defer pluginManager.CleanupClients() backupLog.Info("Getting backup item actions") actions, err := pluginManager.GetBackupItemActionsV2() if err != nil { return err } backupLog.Info("Getting ItemBlock actions") ibActions, err := pluginManager.GetItemBlockActions() if err != nil { return err } backupLog.Info("Setting up backup store to check for backup existence") backupStore, err := b.backupStoreGetter.Get(backup.StorageLocation, pluginManager, backupLog) if err != nil { return err } exists, err := backupStore.BackupExists(backup.StorageLocation.Spec.StorageType.ObjectStorage.Bucket, backup.Name) if exists || err != nil { backup.Status.Phase = velerov1api.BackupPhaseFailed backup.Status.CompletionTimestamp = &metav1.Time{Time: b.clock.Now()} if err != nil { return errors.Wrapf(err, "error checking if backup already exists in object storage") } return errors.Errorf("backup already exists in object storage") } backupItemActionsResolver := framework.NewBackupItemActionResolverV2(actions) itemBlockActionResolver := framework.NewItemBlockActionResolver(ibActions) var fatalErrs []error if err := b.backupper.BackupWithResolvers(backupLog, backup, backupFile, backupItemActionsResolver, itemBlockActionResolver, pluginManager); err != nil { fatalErrs = append(fatalErrs, err) } // native snapshots phase will either be failed or completed right away // https://github.com/vmware-tanzu/velero/blob/de3ea52f0cc478e99efa7b9524c7f353514261a4/pkg/backup/item_backupper.go#L632-L639 backup.Status.VolumeSnapshotsAttempted = len(backup.VolumeSnapshots.Get()) for _, snap := range backup.VolumeSnapshots.Get() { if snap.Status.Phase == volume.SnapshotPhaseCompleted { backup.Status.VolumeSnapshotsCompleted++ } } volumeSnapshots, volumeSnapshotContents, volumeSnapshotClasses := pkgbackup.GetBackupCSIResources(b.kbClient, b.globalCRClient, backup.Backup, backupLog) // Update CSIVolumeSnapshotsAttempted backup.Status.CSIVolumeSnapshotsAttempted = len(volumeSnapshots) // Iterate over backup item operations and update progress. // Any errors on operations at this point should be added to backup errors. // If any operations are still not complete, then back will not be set to // Completed yet. inProgressOperations, _, opsCompleted, opsFailed, errs := getBackupItemOperationProgress(backup.Backup, pluginManager, *backup.GetItemOperationsList()) if len(errs) > 0 { for _, err := range errs { backupLog.Error(err) } } backup.Status.BackupItemOperationsAttempted = len(*backup.GetItemOperationsList()) backup.Status.BackupItemOperationsCompleted = opsCompleted backup.Status.BackupItemOperationsFailed = opsFailed backup.Status.Warnings = logCounter.GetCount(logrus.WarnLevel) backup.Status.Errors = logCounter.GetCount(logrus.ErrorLevel) backupWarnings := logCounter.GetEntries(logrus.WarnLevel) backupErrors := logCounter.GetEntries(logrus.ErrorLevel) results := map[string]results.Result{ "warnings": backupWarnings, "errors": backupErrors, } backupLog.DoneForPersist(b.logger.WithField(constant.ControllerBackup, kubeutil.NamespaceAndName(backup))) // Assign finalize phase as close to end as possible so that any errors // logged to backupLog are captured. This is done before uploading the // artifacts to object storage so that the JSON representation of the // backup in object storage has the terminal phase set. switch { case len(fatalErrs) > 0: backup.Status.Phase = velerov1api.BackupPhaseFailed case logCounter.GetCount(logrus.ErrorLevel) > 0: if inProgressOperations { backup.Status.Phase = velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed } else { backup.Status.Phase = velerov1api.BackupPhaseFinalizingPartiallyFailed } default: if inProgressOperations { backup.Status.Phase = velerov1api.BackupPhaseWaitingForPluginOperations } else { backup.Status.Phase = velerov1api.BackupPhaseFinalizing } } // Mark completion timestamp before serializing and uploading. // Otherwise, the JSON file in object storage has a CompletionTimestamp of 'null'. if backup.Status.Phase == velerov1api.BackupPhaseFailed || backup.Status.Phase == velerov1api.BackupPhasePartiallyFailed || backup.Status.Phase == velerov1api.BackupPhaseCompleted { backup.Status.CompletionTimestamp = &metav1.Time{Time: b.clock.Now()} } recordBackupMetrics(backupLog, backup.Backup, backupFile, b.metrics, false) // re-instantiate the backup store because credentials could have changed since the original // instantiation, if this was a long-running backup backupLog.Info("Setting up backup store to persist the backup") backupStore, err = b.backupStoreGetter.Get(backup.StorageLocation, pluginManager, backupLog) if err != nil { return err } if logFile, err := backupLog.GetPersistFile(); err != nil { fatalErrs = append(fatalErrs, errors.Wrap(err, "error getting backup log file")) } else { if errs := persistBackup(backup, backupFile, logFile, backupStore, volumeSnapshots, volumeSnapshotContents, volumeSnapshotClasses, results, b.globalCRClient, backupLog); len(errs) > 0 { fatalErrs = append(fatalErrs, errs...) } } b.logger.WithField(constant.ControllerBackup, kubeutil.NamespaceAndName(backup)).Infof("Initial backup processing complete, moving to %s", backup.Status.Phase) // if we return a non-nil error, the calling function will update // the backup's phase to Failed. return kerrors.NewAggregate(fatalErrs) } func recordBackupMetrics(log logrus.FieldLogger, backup *velerov1api.Backup, backupFile *os.File, serverMetrics *metrics.ServerMetrics, finalize bool) { backupScheduleName := backup.GetLabels()[velerov1api.ScheduleNameLabel] if backupFile != nil { var backupSizeBytes int64 if backupFileStat, err := backupFile.Stat(); err != nil { log.WithError(errors.WithStack(err)).Error("Error getting backup file info") } else { backupSizeBytes = backupFileStat.Size() } serverMetrics.SetBackupTarballSizeBytesGauge(backupScheduleName, backupSizeBytes) } if backup.Status.CompletionTimestamp != nil { backupDuration := backup.Status.CompletionTimestamp.Time.Sub(backup.Status.StartTimestamp.Time) backupDurationSeconds := float64(backupDuration / time.Second) serverMetrics.RegisterBackupDuration(backupScheduleName, backupDurationSeconds) } if !finalize { serverMetrics.RegisterVolumeSnapshotAttempts(backupScheduleName, backup.Status.VolumeSnapshotsAttempted) serverMetrics.RegisterVolumeSnapshotSuccesses(backupScheduleName, backup.Status.VolumeSnapshotsCompleted) serverMetrics.RegisterVolumeSnapshotFailures(backupScheduleName, backup.Status.VolumeSnapshotsAttempted-backup.Status.VolumeSnapshotsCompleted) if features.IsEnabled(velerov1api.CSIFeatureFlag) { serverMetrics.RegisterCSISnapshotAttempts(backupScheduleName, backup.Name, backup.Status.CSIVolumeSnapshotsAttempted) } if backup.Status.Progress != nil { serverMetrics.RegisterBackupItemsTotalGauge(backupScheduleName, backup.Status.Progress.TotalItems) } serverMetrics.RegisterBackupItemsErrorsGauge(backupScheduleName, backup.Status.Errors) if backup.Status.Warnings > 0 { serverMetrics.RegisterBackupWarning(backupScheduleName) } } else if features.IsEnabled(velerov1api.CSIFeatureFlag) { serverMetrics.RegisterCSISnapshotSuccesses(backupScheduleName, backup.Name, backup.Status.CSIVolumeSnapshotsCompleted) serverMetrics.RegisterCSISnapshotFailures(backupScheduleName, backup.Name, backup.Status.CSIVolumeSnapshotsAttempted-backup.Status.CSIVolumeSnapshotsCompleted) } } func persistBackup(backup *pkgbackup.Request, backupContents, backupLog *os.File, backupStore persistence.BackupStore, csiVolumeSnapshots []snapshotv1api.VolumeSnapshot, csiVolumeSnapshotContents []snapshotv1api.VolumeSnapshotContent, csiVolumeSnapshotClasses []snapshotv1api.VolumeSnapshotClass, results map[string]results.Result, crClient kbclient.Client, logger logrus.FieldLogger, ) []error { persistErrs := []error{} backupJSON := new(bytes.Buffer) if err := encode.To(backup.Backup, "json", backupJSON); err != nil { persistErrs = append(persistErrs, errors.Wrap(err, "error encoding backup")) } // Velero-native volume snapshots (as opposed to CSI ones) nativeVolumeSnapshots, errs := encode.ToJSONGzip(backup.VolumeSnapshots.Get(), "native volumesnapshots list") if errs != nil { persistErrs = append(persistErrs, errs...) } var backupItemOperations *bytes.Buffer backupItemOperations, errs = encode.ToJSONGzip(backup.GetItemOperationsList(), "backup item operations list") if errs != nil { persistErrs = append(persistErrs, errs...) } podVolumeBackups, errs := encode.ToJSONGzip(backup.PodVolumeBackups, "pod volume backups list") if errs != nil { persistErrs = append(persistErrs, errs...) } csiSnapshotClassesJSON, errs := encode.ToJSONGzip(csiVolumeSnapshotClasses, "csi volume snapshot classes list") if errs != nil { persistErrs = append(persistErrs, errs...) } backupResourceList, errs := encode.ToJSONGzip(backup.BackupResourceList(), "backup resources list") if errs != nil { persistErrs = append(persistErrs, errs...) } backupResult, errs := encode.ToJSONGzip(results, "backup results") if errs != nil { persistErrs = append(persistErrs, errs...) } backup.FillVolumesInformation() volumeInfoJSON, errs := encode.ToJSONGzip(backup.VolumesInformation.Result( csiVolumeSnapshots, csiVolumeSnapshotContents, csiVolumeSnapshotClasses, crClient, logger, ), "backup volumes information") if errs != nil { persistErrs = append(persistErrs, errs...) } if len(persistErrs) > 0 { // Don't upload the JSON files or backup tarball if encoding to json fails. backupJSON = nil backupContents = nil nativeVolumeSnapshots = nil backupItemOperations = nil backupResourceList = nil csiSnapshotClassesJSON = nil backupResult = nil volumeInfoJSON = nil } backupInfo := persistence.BackupInfo{ Name: backup.Name, Metadata: backupJSON, Contents: backupContents, Log: backupLog, BackupResults: backupResult, PodVolumeBackups: podVolumeBackups, VolumeSnapshots: nativeVolumeSnapshots, BackupItemOperations: backupItemOperations, BackupResourceList: backupResourceList, CSIVolumeSnapshotClasses: csiSnapshotClassesJSON, BackupVolumeInfo: volumeInfoJSON, } if err := backupStore.PutBackup(backupInfo); err != nil { persistErrs = append(persistErrs, err) } return persistErrs } func closeAndRemoveFile(file *os.File, log logrus.FieldLogger) { if file == nil { log.Debug("Skipping removal of file due to nil file pointer") return } if err := file.Close(); err != nil { log.WithError(err).WithField("file", file.Name()).Error("error closing file") } if err := os.Remove(file.Name()); err != nil { log.WithError(err).WithField("file", file.Name()).Error("error removing file") } } func oldAndNewFilterParametersUsedTogether(backupSpec velerov1api.BackupSpec) bool { haveOldResourceFilterParameters := len(backupSpec.IncludedResources) > 0 || (len(backupSpec.ExcludedResources) > 0) || (backupSpec.IncludeClusterResources != nil) haveNewResourceFilterParameters := len(backupSpec.IncludedClusterScopedResources) > 0 || (len(backupSpec.ExcludedClusterScopedResources) > 0) || (len(backupSpec.IncludedNamespaceScopedResources) > 0) || (len(backupSpec.ExcludedNamespaceScopedResources) > 0) return haveOldResourceFilterParameters && haveNewResourceFilterParameters } func modifyResourceIncludeExclude(include, exclude, addedExclude []string) (modifiedInclude, modifiedExclude []string) { modifiedInclude = include modifiedExclude = exclude excludeStrSet := sets.NewString(exclude...) for _, ex := range addedExclude { if !excludeStrSet.Has(ex) { modifiedExclude = append(modifiedExclude, ex) } } for _, exElem := range modifiedExclude { for inIndex, inElem := range modifiedInclude { if inElem == exElem { modifiedInclude = slices.Delete(modifiedInclude, inIndex, inIndex+1) } } } return modifiedInclude, modifiedExclude } ================================================ FILE: pkg/controller/backup_controller_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package controller import ( "bytes" "fmt" "io" "reflect" "sort" "strings" "testing" "time" "github.com/google/go-cmp/cmp" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/version" "k8s.io/utils/clock" testclocks "k8s.io/utils/clock/testing" ctrl "sigs.k8s.io/controller-runtime" kbclient "sigs.k8s.io/controller-runtime/pkg/client" fakeClient "sigs.k8s.io/controller-runtime/pkg/client/fake" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" pkgbackup "github.com/vmware-tanzu/velero/pkg/backup" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/discovery" "github.com/vmware-tanzu/velero/pkg/features" "github.com/vmware-tanzu/velero/pkg/itemoperation" "github.com/vmware-tanzu/velero/pkg/metrics" "github.com/vmware-tanzu/velero/pkg/persistence" persistencemocks "github.com/vmware-tanzu/velero/pkg/persistence/mocks" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" "github.com/vmware-tanzu/velero/pkg/plugin/framework" pluginmocks "github.com/vmware-tanzu/velero/pkg/plugin/mocks" biav2 "github.com/vmware-tanzu/velero/pkg/plugin/velero/backupitemaction/v2" ibav1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/itemblockaction/v1" velerotest "github.com/vmware-tanzu/velero/pkg/test" "github.com/vmware-tanzu/velero/pkg/util/boolptr" kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube" "github.com/vmware-tanzu/velero/pkg/util/logging" ) type fakeBackupper struct { mock.Mock } func (b *fakeBackupper) Backup(logger logrus.FieldLogger, backup *pkgbackup.Request, backupFile io.Writer, actions []biav2.BackupItemAction, itemBlockActions []ibav1.ItemBlockAction, volumeSnapshotterGetter pkgbackup.VolumeSnapshotterGetter) error { args := b.Called(logger, backup, backupFile, actions, itemBlockActions, volumeSnapshotterGetter) return args.Error(0) } func (b *fakeBackupper) BackupWithResolvers(logger logrus.FieldLogger, backup *pkgbackup.Request, backupFile io.Writer, backupItemActionResolver framework.BackupItemActionResolverV2, itemBlockActionResolver framework.ItemBlockActionResolver, volumeSnapshotterGetter pkgbackup.VolumeSnapshotterGetter, ) error { args := b.Called(logger, backup, backupFile, backupItemActionResolver, volumeSnapshotterGetter) return args.Error(0) } func (b *fakeBackupper) FinalizeBackup( logger logrus.FieldLogger, backup *pkgbackup.Request, inBackupFile io.Reader, outBackupFile io.Writer, backupItemActionResolver framework.BackupItemActionResolverV2, asyncBIAOperations []*itemoperation.BackupOperation, backupStore persistence.BackupStore, ) error { args := b.Called(logger, backup, inBackupFile, outBackupFile, backupItemActionResolver, asyncBIAOperations) return args.Error(0) } func defaultBackup() *builder.BackupBuilder { return builder.ForBackup(velerov1api.DefaultNamespace, "backup-1").Phase(velerov1api.BackupPhaseReadyToStart) } func namedBackup(name string) *builder.BackupBuilder { return builder.ForBackup(velerov1api.DefaultNamespace, name).Phase(velerov1api.BackupPhaseReadyToStart) } func TestProcessBackupNonProcessedItems(t *testing.T) { tests := []struct { name string key string backup *velerov1api.Backup }{ { name: "New backup is not processed", key: "velero/backup-1", backup: defaultBackup().Phase(velerov1api.BackupPhaseNew).Result(), }, { name: "Queued backup is not processed", key: "velero/backup-1", backup: defaultBackup().Phase(velerov1api.BackupPhaseQueued).Result(), }, { name: "FailedValidation backup is not processed", key: "velero/backup-1", backup: defaultBackup().Phase(velerov1api.BackupPhaseFailedValidation).Result(), }, { name: "InProgress backup is not processed", key: "velero/backup-1", backup: defaultBackup().Phase(velerov1api.BackupPhaseInProgress).Result(), }, { name: "Completed backup is not processed", key: "velero/backup-1", backup: defaultBackup().Phase(velerov1api.BackupPhaseCompleted).Result(), }, { name: "Failed backup is not processed", key: "velero/backup-1", backup: defaultBackup().Phase(velerov1api.BackupPhaseFailed).Result(), }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { formatFlag := logging.FormatText logger := logging.DefaultLogger(logrus.DebugLevel, formatFlag) c := &backupReconciler{ kbClient: velerotest.NewFakeControllerRuntimeClient(t), formatFlag: formatFlag, logger: logger, } if test.backup != nil { require.NoError(t, c.kbClient.Create(t.Context(), test.backup)) } actualResult, err := c.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Namespace: test.backup.Namespace, Name: test.backup.Name}}) assert.Equal(t, ctrl.Result{}, actualResult) assert.NoError(t, err) // Any backup that would actually proceed to validation will cause a segfault because this // test hasn't set up the necessary controller dependencies for validation/etc. So the lack // of segfaults during test execution here imply that backups are not being processed, which // is what we expect. }) } } func TestProcessBackupValidationFailures(t *testing.T) { defaultBackupLocation := builder.ForBackupStorageLocation("velero", "loc-1").Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result() tests := []struct { name string backup *velerov1api.Backup backupLocation *velerov1api.BackupStorageLocation expectedErrs []string }{ { name: "invalid included/excluded resources fails validation", backup: defaultBackup().IncludedResources("foo").ExcludedResources("foo").Result(), backupLocation: defaultBackupLocation, expectedErrs: []string{"Invalid included/excluded resource lists: excludes list cannot contain an item in the includes list: foo"}, }, { name: "invalid included/excluded namespaces fails validation", backup: defaultBackup().IncludedNamespaces("foo").ExcludedNamespaces("foo").Result(), backupLocation: defaultBackupLocation, expectedErrs: []string{"Invalid included/excluded namespace lists: excludes list cannot contain an item in the includes list: foo"}, }, { name: "non-existent backup location fails validation", backup: defaultBackup().StorageLocation("nonexistent").Result(), expectedErrs: []string{"an existing backup storage location was not specified at backup creation time and the default nonexistent was not found. Please address this issue (see `velero backup-location -h` for options) and create a new backup. Error: backupstoragelocations.velero.io \"nonexistent\" not found"}, }, { name: "backup for read-only backup location fails validation", backup: defaultBackup().StorageLocation("read-only").Result(), backupLocation: builder.ForBackupStorageLocation("velero", "read-only").AccessMode(velerov1api.BackupStorageLocationAccessModeReadOnly).Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result(), expectedErrs: []string{"backup can't be created because backup storage location read-only is currently in read-only mode"}, }, { name: "labelSelector as well as orLabelSelectors both are specified in backup request fails validation", backup: defaultBackup().LabelSelector(&metav1.LabelSelector{MatchLabels: map[string]string{"a": "b"}}).OrLabelSelector([]*metav1.LabelSelector{ {MatchLabels: map[string]string{"a1": "b1"}}, {MatchLabels: map[string]string{"a2": "b2"}}, {MatchLabels: map[string]string{"a3": "b3"}}, {MatchLabels: map[string]string{"a4": "b4"}}, }).Result(), backupLocation: defaultBackupLocation, expectedErrs: []string{"encountered labelSelector as well as orLabelSelectors in backup spec, only one can be specified"}, }, { name: "use old filter parameters and new filter parameters together", backup: defaultBackup().IncludeClusterResources(true).IncludedNamespaceScopedResources("Deployment").IncludedNamespaces("default").Result(), backupLocation: defaultBackupLocation, expectedErrs: []string{"include-resources, exclude-resources and include-cluster-resources are old filter parameters.\ninclude-cluster-scoped-resources, exclude-cluster-scoped-resources, include-namespace-scoped-resources and exclude-namespace-scoped-resources are new filter parameters.\nThey cannot be used together"}, }, { name: "BSL in unavailable state", backup: defaultBackup().StorageLocation("unavailable").Result(), backupLocation: builder.ForBackupStorageLocation("velero", "unavailable").Phase(velerov1api.BackupStorageLocationPhaseUnavailable).Result(), expectedErrs: []string{"backup can't be created because BackupStorageLocation unavailable is in Unavailable status."}, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { formatFlag := logging.FormatText logger := logging.DefaultLogger(logrus.DebugLevel, formatFlag) apiServer := velerotest.NewAPIServer(t) discoveryHelper, err := discovery.NewHelper(apiServer.DiscoveryClient, logger) require.NoError(t, err) var fakeClient kbclient.Client if test.backupLocation != nil { fakeClient = velerotest.NewFakeControllerRuntimeClient(t, test.backupLocation) } else { fakeClient = velerotest.NewFakeControllerRuntimeClient(t) } c := &backupReconciler{ logger: logger, discoveryHelper: discoveryHelper, kbClient: fakeClient, defaultBackupLocation: defaultBackupLocation.Name, clock: &clock.RealClock{}, formatFlag: formatFlag, metrics: metrics.NewServerMetrics(), backupTracker: NewBackupTracker(), } require.NotNil(t, test.backup) require.NoError(t, c.kbClient.Create(t.Context(), test.backup)) actualResult, err := c.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Namespace: test.backup.Namespace, Name: test.backup.Name}}) assert.Equal(t, ctrl.Result{}, actualResult) require.NoError(t, err) res := &velerov1api.Backup{} err = c.kbClient.Get(t.Context(), kbclient.ObjectKey{Namespace: test.backup.Namespace, Name: test.backup.Name}, res) require.NoError(t, err) assert.Equal(t, velerov1api.BackupPhaseFailedValidation, res.Status.Phase) assert.Equal(t, test.expectedErrs, res.Status.ValidationErrors) // Any backup that would actually proceed to processing will cause a segfault because this // test hasn't set up the necessary controller dependencies for running backups. So the lack // of segfaults during test execution here imply that backups are not being processed, which // is what we expect. }) } } func TestBackupLocationLabel(t *testing.T) { tests := []struct { name string backup *velerov1api.Backup backupLocation *velerov1api.BackupStorageLocation expectedBackupLocation string }{ { name: "valid backup location name should be used as a label", backup: defaultBackup().Result(), backupLocation: builder.ForBackupStorageLocation("velero", "loc-1").Result(), expectedBackupLocation: "loc-1", }, { name: "invalid storage location name should be handled while creating label", backup: defaultBackup().Result(), backupLocation: builder.ForBackupStorageLocation("velero", "defaultdefaultdefaultdefaultdefaultdefaultdefaultdefaultdefaultdefault").Result(), expectedBackupLocation: "defaultdefaultdefaultdefaultdefaultdefaultdefaultdefaultd58343f", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { formatFlag := logging.FormatText var ( logger = logging.DefaultLogger(logrus.DebugLevel, formatFlag) fakeClient = velerotest.NewFakeControllerRuntimeClient(t) ) apiServer := velerotest.NewAPIServer(t) discoveryHelper, err := discovery.NewHelper(apiServer.DiscoveryClient, logger) require.NoError(t, err) c := &backupReconciler{ discoveryHelper: discoveryHelper, kbClient: fakeClient, defaultBackupLocation: test.backupLocation.Name, clock: &clock.RealClock{}, formatFlag: formatFlag, } res := c.prepareBackupRequest(ctx, test.backup, logger) defer res.WorkerPool.Stop() assert.NotNil(t, res) assert.Equal(t, test.expectedBackupLocation, res.Labels[velerov1api.StorageLocationLabel]) }) } } func Test_prepareBackupRequest_BackupStorageLocation(t *testing.T) { var ( defaultBackupTTL = metav1.Duration{Duration: 24 * 30 * time.Hour} defaultBackupLocation = "default-location" ) now, err := time.Parse(time.RFC1123Z, time.RFC1123Z) require.NoError(t, err) tests := []struct { name string backup *velerov1api.Backup backupLocationNameInBackup string backupLocationInAPIServer *velerov1api.BackupStorageLocation defaultBackupLocationInAPIServer *velerov1api.BackupStorageLocation expectedBackupLocation string expectedSuccess bool expectedValidationError string }{ { name: "BackupLocation is specified in backup CR'spec and it can be found in ApiServer", backup: defaultBackup().Result(), backupLocationNameInBackup: "test-backup-location", backupLocationInAPIServer: builder.ForBackupStorageLocation("velero", "test-backup-location").Result(), defaultBackupLocationInAPIServer: builder.ForBackupStorageLocation("velero", "default-location").Result(), expectedBackupLocation: "test-backup-location", expectedSuccess: true, }, { name: "BackupLocation is specified in backup CR'spec and it can't be found in ApiServer", backup: defaultBackup().Result(), backupLocationNameInBackup: "test-backup-location", backupLocationInAPIServer: nil, defaultBackupLocationInAPIServer: nil, expectedSuccess: false, expectedValidationError: "an existing backup storage location was not specified at backup creation time and the default test-backup-location was not found. Please address this issue (see `velero backup-location -h` for options) and create a new backup. Error: backupstoragelocations.velero.io \"test-backup-location\" not found", }, { name: "Using default BackupLocation and it can be found in ApiServer", backup: defaultBackup().Result(), backupLocationNameInBackup: "", backupLocationInAPIServer: builder.ForBackupStorageLocation("velero", "test-backup-location").Result(), defaultBackupLocationInAPIServer: builder.ForBackupStorageLocation("velero", "default-location").Result(), expectedBackupLocation: defaultBackupLocation, expectedSuccess: true, }, { name: "Using default BackupLocation and it can't be found in ApiServer", backup: defaultBackup().Result(), backupLocationNameInBackup: "", backupLocationInAPIServer: nil, defaultBackupLocationInAPIServer: nil, expectedSuccess: false, expectedValidationError: fmt.Sprintf("an existing backup storage location was not specified at backup creation time and the server default %s does not exist. Please address this issue (see `velero backup-location -h` for options) and create a new backup. Error: backupstoragelocations.velero.io \"%s\" not found", defaultBackupLocation, defaultBackupLocation), }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { // Arrange var ( formatFlag = logging.FormatText logger = logging.DefaultLogger(logrus.DebugLevel, formatFlag) apiServer = velerotest.NewAPIServer(t) ) // objects that should init with client objects := make([]runtime.Object, 0) if test.backupLocationInAPIServer != nil { objects = append(objects, test.backupLocationInAPIServer) } if test.defaultBackupLocationInAPIServer != nil { objects = append(objects, test.defaultBackupLocationInAPIServer) } fakeClient := velerotest.NewFakeControllerRuntimeClient(t, objects...) discoveryHelper, err := discovery.NewHelper(apiServer.DiscoveryClient, logger) require.NoError(t, err) c := &backupReconciler{ discoveryHelper: discoveryHelper, defaultBackupLocation: defaultBackupLocation, kbClient: fakeClient, defaultBackupTTL: defaultBackupTTL.Duration, clock: testclocks.NewFakeClock(now), formatFlag: formatFlag, } test.backup.Spec.StorageLocation = test.backupLocationNameInBackup // Run res := c.prepareBackupRequest(ctx, test.backup, logger) defer res.WorkerPool.Stop() // Assert if test.expectedSuccess { assert.Equal(t, test.expectedBackupLocation, res.Spec.StorageLocation) assert.NotNil(t, res) } else { // in every test case, we only trigger one error at once if len(res.Status.ValidationErrors) > 1 { assert.Fail(t, "multi error found in request") } assert.Equal(t, test.expectedValidationError, res.Status.ValidationErrors[0]) } }) } } func TestDefaultBackupTTL(t *testing.T) { defaultBackupTTL := metav1.Duration{Duration: 24 * 30 * time.Hour} now, err := time.Parse(time.RFC1123Z, time.RFC1123Z) require.NoError(t, err) now = now.Local() tests := []struct { name string backup *velerov1api.Backup backupLocation *velerov1api.BackupStorageLocation expectedTTL metav1.Duration expectedExpiration metav1.Time }{ { name: "backup with no TTL specified", backup: defaultBackup().Result(), expectedTTL: defaultBackupTTL, expectedExpiration: metav1.NewTime(now.Add(defaultBackupTTL.Duration)), }, { name: "backup with TTL specified", backup: defaultBackup().TTL(time.Hour).Result(), expectedTTL: metav1.Duration{Duration: 1 * time.Hour}, expectedExpiration: metav1.NewTime(now.Add(1 * time.Hour)), }, } for _, test := range tests { formatFlag := logging.FormatText var ( fakeClient kbclient.Client logger = logging.DefaultLogger(logrus.DebugLevel, formatFlag) ) t.Run(test.name, func(t *testing.T) { apiServer := velerotest.NewAPIServer(t) discoveryHelper, err := discovery.NewHelper(apiServer.DiscoveryClient, logger) require.NoError(t, err) // add the test's backup storage location if it's different than the default if test.backupLocation != nil { fakeClient = velerotest.NewFakeControllerRuntimeClient(t, test.backupLocation) } else { fakeClient = velerotest.NewFakeControllerRuntimeClient(t) } c := &backupReconciler{ logger: logger, discoveryHelper: discoveryHelper, kbClient: fakeClient, defaultBackupTTL: defaultBackupTTL.Duration, clock: testclocks.NewFakeClock(now), formatFlag: formatFlag, } res := c.prepareBackupRequest(ctx, test.backup, logger) defer res.WorkerPool.Stop() assert.NotNil(t, res) assert.Equal(t, test.expectedTTL, res.Spec.TTL) assert.Equal(t, test.expectedExpiration, *res.Status.Expiration) }) } } func TestPrepareBackupRequest_SetsVGSLabelKey(t *testing.T) { now, err := time.Parse(time.RFC1123Z, time.RFC1123Z) require.NoError(t, err) now = now.Local() tests := []struct { name string backup *velerov1api.Backup serverFlagKey string expectedLabelKey string }{ { name: "backup with spec label key set", backup: defaultBackup(). VolumeGroupSnapshotLabelKey("spec-key"). Result(), serverFlagKey: "server-key", expectedLabelKey: "spec-key", }, { name: "backup with no spec key, uses server flag", backup: namedBackup("backup-2").Result(), serverFlagKey: "server-key", expectedLabelKey: "server-key", }, { name: "backup with no spec or server flag, uses default", backup: namedBackup("backup-3").Result(), serverFlagKey: velerov1api.DefaultVGSLabelKey, expectedLabelKey: velerov1api.DefaultVGSLabelKey, }, } for _, test := range tests { formatFlag := logging.FormatText logger := logging.DefaultLogger(logrus.DebugLevel, formatFlag) t.Run(test.name, func(t *testing.T) { fakeClient := velerotest.NewFakeControllerRuntimeClient(t, test.backup) apiServer := velerotest.NewAPIServer(t) discoveryHelper, err := discovery.NewHelper(apiServer.DiscoveryClient, logger) require.NoError(t, err) c := &backupReconciler{ logger: logger, kbClient: fakeClient, defaultVGSLabelKey: test.serverFlagKey, discoveryHelper: discoveryHelper, clock: testclocks.NewFakeClock(now), } res := c.prepareBackupRequest(ctx, test.backup, logger) defer res.WorkerPool.Stop() assert.NotNil(t, res) assert.Equal(t, test.expectedLabelKey, res.Spec.VolumeGroupSnapshotLabelKey) }) } } func TestDefaultVolumesToResticDeprecation(t *testing.T) { tests := []struct { name string backup *velerov1api.Backup globalVal bool expectGlobal bool expectRemap bool expectVal bool }{ { name: "DefaultVolumesToRestic is not set, DefaultVolumesToFsBackup is not set", backup: defaultBackup().Result(), globalVal: true, expectGlobal: true, expectVal: true, }, { name: "DefaultVolumesToRestic is not set, DefaultVolumesToFsBackup is set to false", backup: defaultBackup().DefaultVolumesToFsBackup(false).Result(), globalVal: true, expectVal: false, }, { name: "DefaultVolumesToRestic is not set, DefaultVolumesToFsBackup is set to true", backup: defaultBackup().DefaultVolumesToFsBackup(true).Result(), globalVal: false, expectVal: true, }, { name: "DefaultVolumesToRestic is set to false, DefaultVolumesToFsBackup is not set", backup: defaultBackup().DefaultVolumesToRestic(false).Result(), globalVal: false, expectGlobal: true, expectVal: false, }, { name: "DefaultVolumesToRestic is set to false, DefaultVolumesToFsBackup is set to true", backup: defaultBackup().DefaultVolumesToRestic(false).DefaultVolumesToFsBackup(true).Result(), globalVal: false, expectVal: true, }, { name: "DefaultVolumesToRestic is set to false, DefaultVolumesToFsBackup is set to false", backup: defaultBackup().DefaultVolumesToRestic(false).DefaultVolumesToFsBackup(false).Result(), globalVal: true, expectVal: false, }, { name: "DefaultVolumesToRestic is set to true, DefaultVolumesToFsBackup is not set", backup: defaultBackup().DefaultVolumesToRestic(true).Result(), globalVal: false, expectRemap: true, expectVal: true, }, { name: "DefaultVolumesToRestic is set to true, DefaultVolumesToFsBackup is set to false", backup: defaultBackup().DefaultVolumesToRestic(true).DefaultVolumesToFsBackup(false).Result(), globalVal: false, expectRemap: true, expectVal: true, }, { name: "DefaultVolumesToRestic is set to true, DefaultVolumesToFsBackup is set to true", backup: defaultBackup().DefaultVolumesToRestic(true).DefaultVolumesToFsBackup(true).Result(), globalVal: false, expectRemap: true, expectVal: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { formatFlag := logging.FormatText var ( logger = logging.DefaultLogger(logrus.DebugLevel, formatFlag) fakeClient = velerotest.NewFakeControllerRuntimeClient(t) ) apiServer := velerotest.NewAPIServer(t) discoveryHelper, err := discovery.NewHelper(apiServer.DiscoveryClient, logger) require.NoError(t, err) c := &backupReconciler{ logger: logger, discoveryHelper: discoveryHelper, kbClient: fakeClient, clock: &clock.RealClock{}, formatFlag: formatFlag, defaultVolumesToFsBackup: test.globalVal, } res := c.prepareBackupRequest(ctx, test.backup, logger) defer res.WorkerPool.Stop() assert.NotNil(t, res) assert.NotNil(t, res.Spec.DefaultVolumesToFsBackup) if test.expectRemap { assert.Equal(t, res.Spec.DefaultVolumesToRestic, res.Spec.DefaultVolumesToFsBackup) } else if test.expectGlobal { assert.NotSame(t, res.Spec.DefaultVolumesToRestic, res.Spec.DefaultVolumesToFsBackup) assert.Equal(t, &c.defaultVolumesToFsBackup, res.Spec.DefaultVolumesToFsBackup) } else { assert.NotSame(t, res.Spec.DefaultVolumesToRestic, res.Spec.DefaultVolumesToFsBackup) assert.NotEqual(t, &c.defaultVolumesToFsBackup, res.Spec.DefaultVolumesToFsBackup) } assert.Equal(t, test.expectVal, *res.Spec.DefaultVolumesToFsBackup) }) } } func TestProcessBackupCompletions(t *testing.T) { defaultBackupLocation := builder.ForBackupStorageLocation("velero", "loc-1").Default(true).Bucket("store-1").Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result() now, err := time.Parse(time.RFC1123Z, time.RFC1123Z) require.NoError(t, err) now = now.Local() timestamp := metav1.NewTime(now) tests := []struct { name string backup *velerov1api.Backup backupLocation *velerov1api.BackupStorageLocation defaultVolumesToFsBackup bool defaultSnapshotMoveData bool enableCSI bool expectedResult *velerov1api.Backup backupExists bool existenceCheckError error volumeSnapshot *snapshotv1api.VolumeSnapshot }{ // Finalizing { name: "backup with no backup location gets the default", backup: defaultBackup().Result(), backupLocation: defaultBackupLocation, defaultVolumesToFsBackup: true, expectedResult: &velerov1api.Backup{ TypeMeta: metav1.TypeMeta{ Kind: "Backup", APIVersion: "velero.io/v1", }, ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "backup-1", Annotations: map[string]string{ "velero.io/source-cluster-k8s-major-version": "1", "velero.io/source-cluster-k8s-minor-version": "16", "velero.io/source-cluster-k8s-gitversion": "v1.16.4", "velero.io/resource-timeout": "0s", }, Labels: map[string]string{ "velero.io/storage-location": "loc-1", }, }, Spec: velerov1api.BackupSpec{ StorageLocation: defaultBackupLocation.Name, DefaultVolumesToFsBackup: boolptr.True(), SnapshotMoveData: boolptr.False(), ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, Version: 1, FormatVersion: "1.1.0", StartTimestamp: ×tamp, Expiration: ×tamp, }, }, }, { name: "backup with a specific backup location keeps it", backup: defaultBackup().StorageLocation("alt-loc").Result(), backupLocation: builder.ForBackupStorageLocation("velero", "alt-loc").Bucket("store-1").Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result(), defaultVolumesToFsBackup: false, expectedResult: &velerov1api.Backup{ TypeMeta: metav1.TypeMeta{ Kind: "Backup", APIVersion: "velero.io/v1", }, ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "backup-1", Annotations: map[string]string{ "velero.io/source-cluster-k8s-major-version": "1", "velero.io/source-cluster-k8s-minor-version": "16", "velero.io/source-cluster-k8s-gitversion": "v1.16.4", "velero.io/resource-timeout": "0s", }, Labels: map[string]string{ "velero.io/storage-location": "alt-loc", }, }, Spec: velerov1api.BackupSpec{ StorageLocation: "alt-loc", DefaultVolumesToFsBackup: boolptr.False(), SnapshotMoveData: boolptr.False(), ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, Version: 1, FormatVersion: "1.1.0", StartTimestamp: ×tamp, Expiration: ×tamp, }, }, }, { name: "backup for a location with ReadWrite access mode gets processed", backup: defaultBackup().StorageLocation("read-write").Result(), backupLocation: builder.ForBackupStorageLocation("velero", "read-write"). Bucket("store-1"). AccessMode(velerov1api.BackupStorageLocationAccessModeReadWrite). Phase(velerov1api.BackupStorageLocationPhaseAvailable). Result(), defaultVolumesToFsBackup: true, expectedResult: &velerov1api.Backup{ TypeMeta: metav1.TypeMeta{ Kind: "Backup", APIVersion: "velero.io/v1", }, ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "backup-1", Annotations: map[string]string{ "velero.io/source-cluster-k8s-major-version": "1", "velero.io/source-cluster-k8s-minor-version": "16", "velero.io/source-cluster-k8s-gitversion": "v1.16.4", "velero.io/resource-timeout": "0s", }, Labels: map[string]string{ "velero.io/storage-location": "read-write", }, }, Spec: velerov1api.BackupSpec{ StorageLocation: "read-write", DefaultVolumesToFsBackup: boolptr.True(), SnapshotMoveData: boolptr.False(), ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, Version: 1, FormatVersion: "1.1.0", StartTimestamp: ×tamp, Expiration: ×tamp, }, }, }, { name: "backup with a TTL has expiration set", backup: defaultBackup().TTL(10 * time.Minute).Result(), backupLocation: defaultBackupLocation, defaultVolumesToFsBackup: false, expectedResult: &velerov1api.Backup{ TypeMeta: metav1.TypeMeta{ Kind: "Backup", APIVersion: "velero.io/v1", }, ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "backup-1", Annotations: map[string]string{ "velero.io/source-cluster-k8s-major-version": "1", "velero.io/source-cluster-k8s-minor-version": "16", "velero.io/source-cluster-k8s-gitversion": "v1.16.4", "velero.io/resource-timeout": "0s", }, Labels: map[string]string{ "velero.io/storage-location": "loc-1", }, }, Spec: velerov1api.BackupSpec{ TTL: metav1.Duration{Duration: 10 * time.Minute}, StorageLocation: defaultBackupLocation.Name, DefaultVolumesToFsBackup: boolptr.False(), SnapshotMoveData: boolptr.False(), ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, Version: 1, FormatVersion: "1.1.0", Expiration: &metav1.Time{Time: now.Add(10 * time.Minute)}, StartTimestamp: ×tamp, }, }, }, { name: "backup without an existing backup will succeed", backupExists: false, backup: defaultBackup().Result(), backupLocation: defaultBackupLocation, defaultVolumesToFsBackup: true, expectedResult: &velerov1api.Backup{ TypeMeta: metav1.TypeMeta{ Kind: "Backup", APIVersion: "velero.io/v1", }, ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "backup-1", Annotations: map[string]string{ "velero.io/source-cluster-k8s-major-version": "1", "velero.io/source-cluster-k8s-minor-version": "16", "velero.io/source-cluster-k8s-gitversion": "v1.16.4", "velero.io/resource-timeout": "0s", }, Labels: map[string]string{ "velero.io/storage-location": "loc-1", }, }, Spec: velerov1api.BackupSpec{ StorageLocation: defaultBackupLocation.Name, DefaultVolumesToFsBackup: boolptr.True(), SnapshotMoveData: boolptr.False(), ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, Version: 1, FormatVersion: "1.1.0", StartTimestamp: ×tamp, Expiration: ×tamp, }, }, }, { name: "backup specifying a false value for 'DefaultVolumesToFsBackup' keeps it", backupExists: false, backup: defaultBackup().DefaultVolumesToFsBackup(false).Result(), backupLocation: defaultBackupLocation, // value set in the controller is different from that specified in the backup defaultVolumesToFsBackup: true, expectedResult: &velerov1api.Backup{ TypeMeta: metav1.TypeMeta{ Kind: "Backup", APIVersion: "velero.io/v1", }, ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "backup-1", Annotations: map[string]string{ "velero.io/source-cluster-k8s-major-version": "1", "velero.io/source-cluster-k8s-minor-version": "16", "velero.io/source-cluster-k8s-gitversion": "v1.16.4", "velero.io/resource-timeout": "0s", }, Labels: map[string]string{ "velero.io/storage-location": "loc-1", }, }, Spec: velerov1api.BackupSpec{ StorageLocation: defaultBackupLocation.Name, DefaultVolumesToFsBackup: boolptr.False(), SnapshotMoveData: boolptr.False(), ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, Version: 1, FormatVersion: "1.1.0", StartTimestamp: ×tamp, Expiration: ×tamp, }, }, }, { name: "backup specifying a true value for 'DefaultVolumesToFsBackup' keeps it", backupExists: false, backup: defaultBackup().DefaultVolumesToFsBackup(true).Result(), backupLocation: defaultBackupLocation, // value set in the controller is different from that specified in the backup defaultVolumesToFsBackup: false, expectedResult: &velerov1api.Backup{ TypeMeta: metav1.TypeMeta{ Kind: "Backup", APIVersion: "velero.io/v1", }, ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "backup-1", Annotations: map[string]string{ "velero.io/source-cluster-k8s-major-version": "1", "velero.io/source-cluster-k8s-minor-version": "16", "velero.io/source-cluster-k8s-gitversion": "v1.16.4", "velero.io/resource-timeout": "0s", }, Labels: map[string]string{ "velero.io/storage-location": "loc-1", }, }, Spec: velerov1api.BackupSpec{ StorageLocation: defaultBackupLocation.Name, DefaultVolumesToFsBackup: boolptr.True(), SnapshotMoveData: boolptr.False(), ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, Version: 1, FormatVersion: "1.1.0", StartTimestamp: ×tamp, Expiration: ×tamp, }, }, }, { name: "backup specifying no value for 'DefaultVolumesToFsBackup' gets the default true value", backupExists: false, backup: defaultBackup().Result(), backupLocation: defaultBackupLocation, // value set in the controller is different from that specified in the backup defaultVolumesToFsBackup: true, expectedResult: &velerov1api.Backup{ TypeMeta: metav1.TypeMeta{ Kind: "Backup", APIVersion: "velero.io/v1", }, ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "backup-1", Annotations: map[string]string{ "velero.io/source-cluster-k8s-major-version": "1", "velero.io/source-cluster-k8s-minor-version": "16", "velero.io/source-cluster-k8s-gitversion": "v1.16.4", "velero.io/resource-timeout": "0s", }, Labels: map[string]string{ "velero.io/storage-location": "loc-1", }, }, Spec: velerov1api.BackupSpec{ StorageLocation: defaultBackupLocation.Name, DefaultVolumesToFsBackup: boolptr.True(), SnapshotMoveData: boolptr.False(), ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, Version: 1, FormatVersion: "1.1.0", StartTimestamp: ×tamp, Expiration: ×tamp, }, }, }, { name: "backup specifying no value for 'DefaultVolumesToFsBackup' gets the default false value", backupExists: false, backup: defaultBackup().Result(), backupLocation: defaultBackupLocation, // value set in the controller is different from that specified in the backup defaultVolumesToFsBackup: false, expectedResult: &velerov1api.Backup{ TypeMeta: metav1.TypeMeta{ Kind: "Backup", APIVersion: "velero.io/v1", }, ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "backup-1", Annotations: map[string]string{ "velero.io/source-cluster-k8s-major-version": "1", "velero.io/source-cluster-k8s-minor-version": "16", "velero.io/source-cluster-k8s-gitversion": "v1.16.4", "velero.io/resource-timeout": "0s", }, Labels: map[string]string{ "velero.io/storage-location": "loc-1", }, }, Spec: velerov1api.BackupSpec{ StorageLocation: defaultBackupLocation.Name, DefaultVolumesToFsBackup: boolptr.False(), SnapshotMoveData: boolptr.False(), ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, Version: 1, FormatVersion: "1.1.0", StartTimestamp: ×tamp, Expiration: ×tamp, }, }, }, // Failed { name: "backup with existing backup will fail", backupExists: true, backup: defaultBackup().Result(), backupLocation: defaultBackupLocation, defaultVolumesToFsBackup: true, expectedResult: &velerov1api.Backup{ TypeMeta: metav1.TypeMeta{ Kind: "Backup", APIVersion: "velero.io/v1", }, ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "backup-1", Annotations: map[string]string{ "velero.io/source-cluster-k8s-major-version": "1", "velero.io/source-cluster-k8s-minor-version": "16", "velero.io/source-cluster-k8s-gitversion": "v1.16.4", "velero.io/resource-timeout": "0s", }, Labels: map[string]string{ "velero.io/storage-location": "loc-1", }, }, Spec: velerov1api.BackupSpec{ StorageLocation: defaultBackupLocation.Name, DefaultVolumesToFsBackup: boolptr.True(), SnapshotMoveData: boolptr.False(), ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFailed, FailureReason: "backup already exists in object storage", Version: 1, FormatVersion: "1.1.0", StartTimestamp: ×tamp, CompletionTimestamp: ×tamp, Expiration: ×tamp, }, }, }, { name: "error when checking if backup exists will cause backup to fail", backup: defaultBackup().Result(), existenceCheckError: errors.New("Backup already exists in object storage"), backupLocation: defaultBackupLocation, defaultVolumesToFsBackup: true, expectedResult: &velerov1api.Backup{ TypeMeta: metav1.TypeMeta{ Kind: "Backup", APIVersion: "velero.io/v1", }, ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "backup-1", Annotations: map[string]string{ "velero.io/source-cluster-k8s-major-version": "1", "velero.io/source-cluster-k8s-minor-version": "16", "velero.io/source-cluster-k8s-gitversion": "v1.16.4", "velero.io/resource-timeout": "0s", }, Labels: map[string]string{ "velero.io/storage-location": "loc-1", }, }, Spec: velerov1api.BackupSpec{ StorageLocation: defaultBackupLocation.Name, DefaultVolumesToFsBackup: boolptr.True(), SnapshotMoveData: boolptr.False(), ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFailed, FailureReason: "error checking if backup already exists in object storage: Backup already exists in object storage", Version: 1, FormatVersion: "1.1.0", StartTimestamp: ×tamp, CompletionTimestamp: ×tamp, Expiration: ×tamp, }, }, }, { name: "backup with snapshot data movement when CSI feature is enabled", backup: defaultBackup().SnapshotMoveData(true).Result(), backupLocation: defaultBackupLocation, defaultVolumesToFsBackup: false, enableCSI: true, expectedResult: &velerov1api.Backup{ TypeMeta: metav1.TypeMeta{ Kind: "Backup", APIVersion: "velero.io/v1", }, ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "backup-1", Annotations: map[string]string{ "velero.io/source-cluster-k8s-major-version": "1", "velero.io/source-cluster-k8s-minor-version": "16", "velero.io/source-cluster-k8s-gitversion": "v1.16.4", "velero.io/resource-timeout": "0s", }, Labels: map[string]string{ "velero.io/storage-location": "loc-1", }, }, Spec: velerov1api.BackupSpec{ StorageLocation: defaultBackupLocation.Name, DefaultVolumesToFsBackup: boolptr.False(), SnapshotMoveData: boolptr.True(), ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, Version: 1, FormatVersion: "1.1.0", StartTimestamp: ×tamp, Expiration: ×tamp, CSIVolumeSnapshotsAttempted: 0, CSIVolumeSnapshotsCompleted: 0, }, }, volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").VolumeSnapshotClass("testClass").Status().BoundVolumeSnapshotContentName("testVSC").RestoreSize("10G").SourcePVC("testPVC").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(), }, { name: "backup with snapshot data movement set to false when CSI feature is enabled", backup: defaultBackup().SnapshotMoveData(false).Result(), backupLocation: defaultBackupLocation, defaultVolumesToFsBackup: false, enableCSI: true, expectedResult: &velerov1api.Backup{ TypeMeta: metav1.TypeMeta{ Kind: "Backup", APIVersion: "velero.io/v1", }, ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "backup-1", Annotations: map[string]string{ "velero.io/source-cluster-k8s-major-version": "1", "velero.io/source-cluster-k8s-minor-version": "16", "velero.io/source-cluster-k8s-gitversion": "v1.16.4", "velero.io/resource-timeout": "0s", }, Labels: map[string]string{ "velero.io/storage-location": "loc-1", }, }, Spec: velerov1api.BackupSpec{ StorageLocation: defaultBackupLocation.Name, DefaultVolumesToFsBackup: boolptr.False(), SnapshotMoveData: boolptr.False(), ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, Version: 1, FormatVersion: "1.1.0", StartTimestamp: ×tamp, Expiration: ×tamp, CSIVolumeSnapshotsAttempted: 1, CSIVolumeSnapshotsCompleted: 0, }, }, volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").VolumeSnapshotClass("testClass").Status().BoundVolumeSnapshotContentName("testVSC").RestoreSize("10G").SourcePVC("testPVC").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(), }, { name: "backup with snapshot data movement not set when CSI feature is enabled", backup: defaultBackup().Result(), backupLocation: defaultBackupLocation, defaultVolumesToFsBackup: false, enableCSI: true, expectedResult: &velerov1api.Backup{ TypeMeta: metav1.TypeMeta{ Kind: "Backup", APIVersion: "velero.io/v1", }, ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "backup-1", Annotations: map[string]string{ "velero.io/source-cluster-k8s-major-version": "1", "velero.io/source-cluster-k8s-minor-version": "16", "velero.io/source-cluster-k8s-gitversion": "v1.16.4", "velero.io/resource-timeout": "0s", }, Labels: map[string]string{ "velero.io/storage-location": "loc-1", }, }, Spec: velerov1api.BackupSpec{ StorageLocation: defaultBackupLocation.Name, DefaultVolumesToFsBackup: boolptr.False(), SnapshotMoveData: boolptr.False(), ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, Version: 1, FormatVersion: "1.1.0", StartTimestamp: ×tamp, Expiration: ×tamp, CSIVolumeSnapshotsAttempted: 1, CSIVolumeSnapshotsCompleted: 0, }, }, volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").VolumeSnapshotClass("testClass").Status().BoundVolumeSnapshotContentName("testVSC").RestoreSize("10G").SourcePVC("testPVC").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(), }, { name: "backup with snapshot data movement set to true and defaultSnapshotMoveData set to false", backup: defaultBackup().SnapshotMoveData(true).Result(), backupLocation: defaultBackupLocation, defaultVolumesToFsBackup: false, defaultSnapshotMoveData: false, expectedResult: &velerov1api.Backup{ TypeMeta: metav1.TypeMeta{ Kind: "Backup", APIVersion: "velero.io/v1", }, ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "backup-1", Annotations: map[string]string{ "velero.io/source-cluster-k8s-major-version": "1", "velero.io/source-cluster-k8s-minor-version": "16", "velero.io/source-cluster-k8s-gitversion": "v1.16.4", "velero.io/resource-timeout": "0s", }, Labels: map[string]string{ "velero.io/storage-location": "loc-1", }, }, Spec: velerov1api.BackupSpec{ StorageLocation: defaultBackupLocation.Name, DefaultVolumesToFsBackup: boolptr.False(), SnapshotMoveData: boolptr.True(), ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, Version: 1, FormatVersion: "1.1.0", StartTimestamp: ×tamp, Expiration: ×tamp, CSIVolumeSnapshotsAttempted: 0, CSIVolumeSnapshotsCompleted: 0, }, }, volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").VolumeSnapshotClass("testClass").Status().BoundVolumeSnapshotContentName("testVSC").RestoreSize("10G").SourcePVC("testPVC").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(), }, { name: "backup with snapshot data movement set to false and defaultSnapshotMoveData set to true", backup: defaultBackup().SnapshotMoveData(false).Result(), backupLocation: defaultBackupLocation, defaultVolumesToFsBackup: false, defaultSnapshotMoveData: true, enableCSI: true, expectedResult: &velerov1api.Backup{ TypeMeta: metav1.TypeMeta{ Kind: "Backup", APIVersion: "velero.io/v1", }, ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "backup-1", Annotations: map[string]string{ "velero.io/source-cluster-k8s-major-version": "1", "velero.io/source-cluster-k8s-minor-version": "16", "velero.io/source-cluster-k8s-gitversion": "v1.16.4", "velero.io/resource-timeout": "0s", }, Labels: map[string]string{ "velero.io/storage-location": "loc-1", }, }, Spec: velerov1api.BackupSpec{ StorageLocation: defaultBackupLocation.Name, DefaultVolumesToFsBackup: boolptr.False(), SnapshotMoveData: boolptr.False(), ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, Version: 1, FormatVersion: "1.1.0", StartTimestamp: ×tamp, Expiration: ×tamp, CSIVolumeSnapshotsAttempted: 1, CSIVolumeSnapshotsCompleted: 0, }, }, volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").VolumeSnapshotClass("testClass").Status().BoundVolumeSnapshotContentName("testVSC").RestoreSize("10G").SourcePVC("testPVC").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(), }, { name: "backup with snapshot data movement not set and defaultSnapshotMoveData set to true", backup: defaultBackup().Result(), backupLocation: defaultBackupLocation, defaultVolumesToFsBackup: false, defaultSnapshotMoveData: true, expectedResult: &velerov1api.Backup{ TypeMeta: metav1.TypeMeta{ Kind: "Backup", APIVersion: "velero.io/v1", }, ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "backup-1", Annotations: map[string]string{ "velero.io/source-cluster-k8s-major-version": "1", "velero.io/source-cluster-k8s-minor-version": "16", "velero.io/source-cluster-k8s-gitversion": "v1.16.4", "velero.io/resource-timeout": "0s", }, Labels: map[string]string{ "velero.io/storage-location": "loc-1", }, }, Spec: velerov1api.BackupSpec{ StorageLocation: defaultBackupLocation.Name, DefaultVolumesToFsBackup: boolptr.False(), SnapshotMoveData: boolptr.True(), ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, Version: 1, FormatVersion: "1.1.0", StartTimestamp: ×tamp, Expiration: ×tamp, CSIVolumeSnapshotsAttempted: 0, CSIVolumeSnapshotsCompleted: 0, }, }, volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").VolumeSnapshotClass("testClass").Status().BoundVolumeSnapshotContentName("testVSC").RestoreSize("10G").SourcePVC("testPVC").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(), }, { name: "backup with namespace-scoped and cluster-scoped resource filters", backup: defaultBackup(). ExcludedClusterScopedResources("clusterroles"). IncludedClusterScopedResources("storageclasses"). ExcludedNamespaceScopedResources("secrets"). IncludedNamespaceScopedResources("pods").Result(), backupLocation: defaultBackupLocation, defaultVolumesToFsBackup: false, defaultSnapshotMoveData: true, expectedResult: &velerov1api.Backup{ TypeMeta: metav1.TypeMeta{ Kind: "Backup", APIVersion: "velero.io/v1", }, ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "backup-1", Annotations: map[string]string{ "velero.io/source-cluster-k8s-major-version": "1", "velero.io/source-cluster-k8s-minor-version": "16", "velero.io/source-cluster-k8s-gitversion": "v1.16.4", "velero.io/resource-timeout": "0s", }, Labels: map[string]string{ "velero.io/storage-location": "loc-1", }, }, Spec: velerov1api.BackupSpec{ StorageLocation: defaultBackupLocation.Name, DefaultVolumesToFsBackup: boolptr.False(), SnapshotMoveData: boolptr.True(), IncludedClusterScopedResources: []string{"storageclasses"}, ExcludedClusterScopedResources: append([]string{"clusterroles"}, autoExcludeClusterScopedResources...), IncludedNamespaceScopedResources: []string{"pods"}, ExcludedNamespaceScopedResources: append([]string{"secrets"}, autoExcludeNamespaceScopedResources...), }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, Version: 1, FormatVersion: "1.1.0", StartTimestamp: ×tamp, Expiration: ×tamp, CSIVolumeSnapshotsAttempted: 0, CSIVolumeSnapshotsCompleted: 0, }, }, volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").VolumeSnapshotClass("testClass").Status().BoundVolumeSnapshotContentName("testVSC").RestoreSize("10G").SourcePVC("testPVC").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(), }, { name: "backup's include filter overlap with default exclude resources", backup: defaultBackup(). ExcludedClusterScopedResources("clusterroles"). IncludedClusterScopedResources("storageclasses", "volumesnapshotcontents.snapshot.storage.k8s.io"). ExcludedNamespaceScopedResources("secrets"). IncludedNamespaceScopedResources("pods", "volumesnapshots.snapshot.storage.k8s.io").Result(), backupLocation: defaultBackupLocation, defaultVolumesToFsBackup: false, defaultSnapshotMoveData: true, expectedResult: &velerov1api.Backup{ TypeMeta: metav1.TypeMeta{ Kind: "Backup", APIVersion: "velero.io/v1", }, ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "backup-1", Annotations: map[string]string{ "velero.io/source-cluster-k8s-major-version": "1", "velero.io/source-cluster-k8s-minor-version": "16", "velero.io/source-cluster-k8s-gitversion": "v1.16.4", "velero.io/resource-timeout": "0s", }, Labels: map[string]string{ "velero.io/storage-location": "loc-1", }, }, Spec: velerov1api.BackupSpec{ StorageLocation: defaultBackupLocation.Name, DefaultVolumesToFsBackup: boolptr.False(), SnapshotMoveData: boolptr.True(), IncludedClusterScopedResources: []string{"storageclasses"}, ExcludedClusterScopedResources: append([]string{"clusterroles"}, autoExcludeClusterScopedResources...), IncludedNamespaceScopedResources: []string{"pods"}, ExcludedNamespaceScopedResources: append([]string{"secrets"}, autoExcludeNamespaceScopedResources...), }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, Version: 1, FormatVersion: "1.1.0", StartTimestamp: ×tamp, Expiration: ×tamp, CSIVolumeSnapshotsAttempted: 0, CSIVolumeSnapshotsCompleted: 0, }, }, volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").VolumeSnapshotClass("testClass").Status().BoundVolumeSnapshotContentName("testVSC").RestoreSize("10G").SourcePVC("testPVC").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(), }, } snapshotHandle := "testSnapshotID" for _, test := range tests { t.Run(test.name, func(t *testing.T) { formatFlag := logging.FormatText var ( logger = logging.DefaultLogger(logrus.DebugLevel, formatFlag) pluginManager = new(pluginmocks.Manager) backupStore = new(persistencemocks.BackupStore) backupper = new(fakeBackupper) fakeGlobalClient = velerotest.NewFakeControllerRuntimeClient(t) ) var fakeClient kbclient.Client // add the test's backup storage location if it's different than the default if test.backupLocation != nil && test.backupLocation != defaultBackupLocation { fakeClient = velerotest.NewFakeControllerRuntimeClient(t, test.backupLocation, builder.ForVolumeSnapshotClass("testClass").Driver("testDriver").Result(), builder.ForVolumeSnapshotContent("testVSC").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).VolumeSnapshotClassName("testClass").Status(&snapshotv1api.VolumeSnapshotContentStatus{ SnapshotHandle: &snapshotHandle, }).Result(), ) } else { fakeClient = velerotest.NewFakeControllerRuntimeClient(t, builder.ForVolumeSnapshotClass("testClass").Driver("testDriver").Result(), builder.ForVolumeSnapshotContent("testVSC").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).VolumeSnapshotClassName("testClass").Status(&snapshotv1api.VolumeSnapshotContentStatus{ SnapshotHandle: &snapshotHandle, }).Result(), ) } if test.volumeSnapshot != nil { require.NoError(t, fakeGlobalClient.Create(t.Context(), test.volumeSnapshot)) } apiServer := velerotest.NewAPIServer(t) apiServer.DiscoveryClient.FakedServerVersion = &version.Info{ Major: "1", Minor: "16", GitVersion: "v1.16.4", GitCommit: "FakeTest", GitTreeState: "", BuildDate: "", GoVersion: "", Compiler: "", Platform: "", } discoveryHelper, err := discovery.NewHelper(apiServer.DiscoveryClient, logger) require.NoError(t, err) c := &backupReconciler{ logger: logger, discoveryHelper: discoveryHelper, kbClient: fakeClient, defaultBackupLocation: defaultBackupLocation.Name, defaultVolumesToFsBackup: test.defaultVolumesToFsBackup, defaultSnapshotMoveData: test.defaultSnapshotMoveData, backupTracker: NewBackupTracker(), metrics: metrics.NewServerMetrics(), clock: testclocks.NewFakeClock(now), newPluginManager: func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager }, backupStoreGetter: NewFakeSingleObjectBackupStoreGetter(backupStore), backupper: backupper, formatFlag: formatFlag, globalCRClient: fakeGlobalClient, } pluginManager.On("GetBackupItemActionsV2").Return(nil, nil) pluginManager.On("GetItemBlockActions").Return(nil, nil) pluginManager.On("CleanupClients").Return(nil) backupper.On("Backup", mock.Anything, mock.Anything, mock.Anything, []biav2.BackupItemAction(nil), pluginManager).Return(nil) backupper.On("BackupWithResolvers", mock.Anything, mock.Anything, mock.Anything, framework.BackupItemActionResolverV2{}, pluginManager).Return(nil) backupStore.On("BackupExists", test.backupLocation.Spec.StorageType.ObjectStorage.Bucket, test.backup.Name).Return(test.backupExists, test.existenceCheckError) // Ensure we have a CompletionTimestamp when uploading and that the backup name matches the backup in the object store. // Failures will display the bytes in buf. hasNameAndCompletionTimestampIfCompleted := func(info persistence.BackupInfo) bool { buf := new(bytes.Buffer) buf.ReadFrom(info.Metadata) return info.Name == test.backup.Name && (!(strings.Contains(buf.String(), `"phase": "Completed"`) || strings.Contains(buf.String(), `"phase": "Failed"`) || strings.Contains(buf.String(), `"phase": "PartiallyFailed"`)) || strings.Contains(buf.String(), `"completionTimestamp": "2006-01-02T22:04:05Z"`)) } backupStore.On("PutBackup", mock.MatchedBy(hasNameAndCompletionTimestampIfCompleted)).Return(nil) // add the test's backup to the informer/lister store require.NotNil(t, test.backup) require.NoError(t, c.kbClient.Create(t.Context(), test.backup)) // add the default backup storage location to the clientset and the informer/lister store require.NoError(t, fakeClient.Create(t.Context(), defaultBackupLocation)) // Enable CSI feature flag for SnapshotDataMovement test. if test.enableCSI { features.Enable(velerov1api.CSIFeatureFlag) } actualResult, err := c.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Namespace: test.backup.Namespace, Name: test.backup.Name}}) assert.Equal(t, ctrl.Result{}, actualResult) require.NoError(t, err) // Disable CSI feature to not impact other test cases. if test.enableCSI { features.Disable(velerov1api.CSIFeatureFlag) } res := &velerov1api.Backup{} err = c.kbClient.Get(t.Context(), kbclient.ObjectKey{Namespace: test.backup.Namespace, Name: test.backup.Name}, res) require.NoError(t, err) res.ResourceVersion = "" assert.Equal(t, test.expectedResult, res) // reset defaultBackupLocation resourceVersion defaultBackupLocation.ObjectMeta.ResourceVersion = "" }) } } func TestValidateAndGetSnapshotLocations(t *testing.T) { defaultBSL := builder.ForBackupStorageLocation(velerov1api.DefaultNamespace, "bsl").Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result() tests := []struct { name string backup *velerov1api.Backup locations []*velerov1api.VolumeSnapshotLocation defaultLocations map[string]string bsl velerov1api.BackupStorageLocation expectedVolumeSnapshotLocationNames []string // adding these in the expected order will allow to test with better msgs in case of a test failure expectedErrors string expectedSuccess bool }{ { name: "location name does not correspond to any existing location", backup: defaultBackup().Phase(velerov1api.BackupPhaseNew).VolumeSnapshotLocations("random-name").Result(), locations: []*velerov1api.VolumeSnapshotLocation{ builder.ForVolumeSnapshotLocation(velerov1api.DefaultNamespace, "aws-us-east-1").Provider("aws").Result(), builder.ForVolumeSnapshotLocation(velerov1api.DefaultNamespace, "aws-us-west-1").Provider("aws").Result(), builder.ForVolumeSnapshotLocation(velerov1api.DefaultNamespace, "some-name").Provider("fake-provider").Result(), }, expectedErrors: "a VolumeSnapshotLocation CRD for the location random-name with the name specified in the backup spec needs to be created before this snapshot can be executed. Error: volumesnapshotlocations.velero.io \"random-name\" not found", expectedSuccess: false, bsl: *defaultBSL, }, { name: "duplicate locationName per provider: should filter out dups", backup: defaultBackup().Phase(velerov1api.BackupPhaseNew).VolumeSnapshotLocations("aws-us-west-1", "aws-us-west-1").Result(), locations: []*velerov1api.VolumeSnapshotLocation{ builder.ForVolumeSnapshotLocation(velerov1api.DefaultNamespace, "aws-us-east-1").Provider("aws").Result(), builder.ForVolumeSnapshotLocation(velerov1api.DefaultNamespace, "aws-us-west-1").Provider("aws").Result(), }, expectedVolumeSnapshotLocationNames: []string{"aws-us-west-1"}, expectedSuccess: true, bsl: *defaultBSL, }, { name: "multiple non-dupe location names per provider should error", backup: defaultBackup().Phase(velerov1api.BackupPhaseNew).VolumeSnapshotLocations("aws-us-east-1", "aws-us-west-1").Result(), locations: []*velerov1api.VolumeSnapshotLocation{ builder.ForVolumeSnapshotLocation(velerov1api.DefaultNamespace, "aws-us-east-1").Provider("aws").Result(), builder.ForVolumeSnapshotLocation(velerov1api.DefaultNamespace, "aws-us-west-1").Provider("aws").Result(), builder.ForVolumeSnapshotLocation(velerov1api.DefaultNamespace, "some-name").Provider("fake-provider").Result(), }, expectedErrors: "more than one VolumeSnapshotLocation name specified for provider aws: aws-us-west-1; unexpected name was aws-us-east-1", expectedSuccess: false, bsl: *defaultBSL, }, { name: "no location name for the provider exists, only one VSL for the provider: use it", backup: defaultBackup().Phase(velerov1api.BackupPhaseNew).Result(), locations: []*velerov1api.VolumeSnapshotLocation{ builder.ForVolumeSnapshotLocation(velerov1api.DefaultNamespace, "aws-us-east-1").Provider("aws").Result(), }, expectedVolumeSnapshotLocationNames: []string{"aws-us-east-1"}, expectedSuccess: true, bsl: *defaultBSL, }, { name: "no location name for the provider exists, no default, more than one VSL for the provider: error", backup: defaultBackup().Phase(velerov1api.BackupPhaseNew).Result(), locations: []*velerov1api.VolumeSnapshotLocation{ builder.ForVolumeSnapshotLocation(velerov1api.DefaultNamespace, "aws-us-east-1").Provider("aws").Result(), builder.ForVolumeSnapshotLocation(velerov1api.DefaultNamespace, "aws-us-west-1").Provider("aws").Result(), }, expectedErrors: "provider aws has more than one possible volume snapshot location, and none were specified explicitly or as a default", bsl: *defaultBSL, }, { name: "no location name for the provider exists, more than one VSL for the provider: the provider's default should be added", backup: defaultBackup().Phase(velerov1api.BackupPhaseNew).Result(), defaultLocations: map[string]string{"aws": "aws-us-east-1"}, locations: []*velerov1api.VolumeSnapshotLocation{ builder.ForVolumeSnapshotLocation(velerov1api.DefaultNamespace, "aws-us-east-1").Provider("aws").Result(), builder.ForVolumeSnapshotLocation(velerov1api.DefaultNamespace, "aws-us-west-1").Provider("aws").Result(), }, expectedVolumeSnapshotLocationNames: []string{"aws-us-east-1"}, expectedSuccess: true, bsl: *defaultBSL, }, { name: "no existing location name and no default location name given", backup: defaultBackup().Phase(velerov1api.BackupPhaseNew).Result(), expectedSuccess: true, bsl: *defaultBSL, }, { name: "multiple location names for a provider, default location name for another provider", backup: defaultBackup().Phase(velerov1api.BackupPhaseNew).VolumeSnapshotLocations("aws-us-west-1", "aws-us-west-1").Result(), defaultLocations: map[string]string{"fake-provider": "some-name"}, locations: []*velerov1api.VolumeSnapshotLocation{ builder.ForVolumeSnapshotLocation(velerov1api.DefaultNamespace, "aws-us-west-1").Provider("aws").Result(), builder.ForVolumeSnapshotLocation(velerov1api.DefaultNamespace, "some-name").Provider("fake-provider").Result(), }, expectedVolumeSnapshotLocationNames: []string{"aws-us-west-1", "some-name"}, expectedSuccess: true, bsl: *defaultBSL, }, { name: "location name does not correspond to any existing location and snapshotvolume disabled; should return error", backup: defaultBackup().Phase(velerov1api.BackupPhaseNew).VolumeSnapshotLocations("random-name").SnapshotVolumes(false).Result(), locations: []*velerov1api.VolumeSnapshotLocation{ builder.ForVolumeSnapshotLocation(velerov1api.DefaultNamespace, "aws-us-east-1").Provider("aws").Result(), builder.ForVolumeSnapshotLocation(velerov1api.DefaultNamespace, "aws-us-west-1").Provider("aws").Result(), builder.ForVolumeSnapshotLocation(velerov1api.DefaultNamespace, "some-name").Provider("fake-provider").Result(), }, expectedVolumeSnapshotLocationNames: nil, expectedErrors: "a VolumeSnapshotLocation CRD for the location random-name with the name specified in the backup spec needs to be created before this snapshot can be executed. Error: volumesnapshotlocations.velero.io \"random-name\" not found", expectedSuccess: false, bsl: *defaultBSL, }, { name: "duplicate locationName per provider and snapshotvolume disabled; should return only one BSL", backup: defaultBackup().Phase(velerov1api.BackupPhaseNew).VolumeSnapshotLocations("aws-us-west-1", "aws-us-west-1").SnapshotVolumes(false).Result(), locations: []*velerov1api.VolumeSnapshotLocation{ builder.ForVolumeSnapshotLocation(velerov1api.DefaultNamespace, "aws-us-east-1").Provider("aws").Result(), builder.ForVolumeSnapshotLocation(velerov1api.DefaultNamespace, "aws-us-west-1").Provider("aws").Result(), }, expectedVolumeSnapshotLocationNames: []string{"aws-us-west-1"}, expectedSuccess: true, bsl: *defaultBSL, }, { name: "no location name for the provider exists, only one VSL created and snapshotvolume disabled; should return the VSL", backup: defaultBackup().Phase(velerov1api.BackupPhaseNew).SnapshotVolumes(false).Result(), locations: []*velerov1api.VolumeSnapshotLocation{ builder.ForVolumeSnapshotLocation(velerov1api.DefaultNamespace, "aws-us-east-1").Provider("aws").Result(), }, expectedVolumeSnapshotLocationNames: []string{"aws-us-east-1"}, expectedSuccess: true, bsl: *defaultBSL, }, { name: "multiple location names for a provider, no default location and backup has no location defined, but snapshotvolume disabled, should return error", backup: defaultBackup().Phase(velerov1api.BackupPhaseNew).SnapshotVolumes(false).Result(), locations: []*velerov1api.VolumeSnapshotLocation{ builder.ForVolumeSnapshotLocation(velerov1api.DefaultNamespace, "aws-us-west-1").Provider("aws").Result(), builder.ForVolumeSnapshotLocation(velerov1api.DefaultNamespace, "aws-us-east-1").Provider("aws").Result(), }, expectedVolumeSnapshotLocationNames: nil, expectedErrors: "provider aws has more than one possible volume snapshot location, and none were specified explicitly or as a default", bsl: *defaultBSL, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { formatFlag := logging.FormatText logger := logging.DefaultLogger(logrus.DebugLevel, formatFlag) c := &backupReconciler{ logger: logger, defaultSnapshotLocations: test.defaultLocations, kbClient: velerotest.NewFakeControllerRuntimeClient(t), } // set up a Backup object to represent what we expect to be passed to backupper.Backup() backup := test.backup.DeepCopy() backup.Spec.VolumeSnapshotLocations = test.backup.Spec.VolumeSnapshotLocations for _, location := range test.locations { require.NoError(t, c.kbClient.Create(t.Context(), location)) } providerLocations, errs := c.validateAndGetSnapshotLocations(backup) if test.expectedSuccess { for _, err := range errs { require.NoError(t, errors.New(err), "validateAndGetSnapshotLocations unexpected error: %v", err) } var locations []string for _, loc := range providerLocations { locations = append(locations, loc.Name) } sort.Strings(test.expectedVolumeSnapshotLocationNames) sort.Strings(locations) require.Equal(t, test.expectedVolumeSnapshotLocationNames, locations) } else { require.NotEmpty(t, errs, "validateAndGetSnapshotLocations expected error") require.Contains(t, errs, test.expectedErrors) } }) } } // Test_getLastSuccessBySchedule verifies that the getLastSuccessBySchedule helper function correctly returns // the completion timestamp of the most recent completed backup for each schedule, including an entry for ad-hoc // or non-scheduled backups. func Test_getLastSuccessBySchedule(t *testing.T) { buildBackup := func(phase velerov1api.BackupPhase, completion time.Time, schedule string) velerov1api.Backup { b := builder.ForBackup("", ""). ObjectMeta(builder.WithLabels(velerov1api.ScheduleNameLabel, schedule)). Phase(phase) if !completion.IsZero() { b.CompletionTimestamp(completion) } return *b.Result() } // create a static "base time" that can be used to easily construct completion timestamps // by using the .Add(...) method. baseTime, err := time.Parse(time.RFC1123, time.RFC1123) require.NoError(t, err) tests := []struct { name string backups []velerov1api.Backup want map[string]time.Time }{ { name: "when backups is nil, an empty map is returned", backups: nil, want: map[string]time.Time{}, }, { name: "when backups is empty, an empty map is returned", backups: []velerov1api.Backup{}, want: map[string]time.Time{}, }, { name: "when multiple completed backups for a schedule exist, the latest one is returned", backups: []velerov1api.Backup{ buildBackup(velerov1api.BackupPhaseCompleted, baseTime, "schedule-1"), buildBackup(velerov1api.BackupPhaseCompleted, baseTime.Add(time.Second), "schedule-1"), buildBackup(velerov1api.BackupPhaseCompleted, baseTime.Add(-time.Second), "schedule-1"), }, want: map[string]time.Time{ "schedule-1": baseTime.Add(time.Second), }, }, { name: "when the most recent backup for a schedule is Failed, the timestamp of the most recent Completed one is returned", backups: []velerov1api.Backup{ buildBackup(velerov1api.BackupPhaseCompleted, baseTime, "schedule-1"), buildBackup(velerov1api.BackupPhaseFailed, baseTime.Add(time.Second), "schedule-1"), buildBackup(velerov1api.BackupPhaseCompleted, baseTime.Add(-time.Second), "schedule-1"), }, want: map[string]time.Time{ "schedule-1": baseTime, }, }, { name: "when there are no Completed backups for a schedule, it's not returned", backups: []velerov1api.Backup{ buildBackup(velerov1api.BackupPhaseInProgress, baseTime, "schedule-1"), buildBackup(velerov1api.BackupPhaseFailed, baseTime.Add(time.Second), "schedule-1"), buildBackup(velerov1api.BackupPhasePartiallyFailed, baseTime.Add(-time.Second), "schedule-1"), }, want: map[string]time.Time{}, }, { name: "when backups exist without a schedule, the most recent Completed one is returned", backups: []velerov1api.Backup{ buildBackup(velerov1api.BackupPhaseCompleted, baseTime, ""), buildBackup(velerov1api.BackupPhaseFailed, baseTime.Add(time.Second), ""), buildBackup(velerov1api.BackupPhaseCompleted, baseTime.Add(-time.Second), ""), }, want: map[string]time.Time{ "": baseTime, }, }, { name: "when backups exist for multiple schedules, the most recent Completed timestamp for each schedule is returned", backups: []velerov1api.Backup{ // ad-hoc backups (no schedule) buildBackup(velerov1api.BackupPhaseCompleted, baseTime.Add(30*time.Minute), ""), buildBackup(velerov1api.BackupPhaseFailed, baseTime.Add(time.Hour), ""), buildBackup(velerov1api.BackupPhaseCompleted, baseTime.Add(-time.Second), ""), // schedule-1 buildBackup(velerov1api.BackupPhaseCompleted, baseTime, "schedule-1"), buildBackup(velerov1api.BackupPhaseFailed, baseTime.Add(time.Second), "schedule-1"), buildBackup(velerov1api.BackupPhaseCompleted, baseTime.Add(-time.Second), "schedule-1"), // schedule-2 buildBackup(velerov1api.BackupPhaseCompleted, baseTime.Add(24*time.Hour), "schedule-2"), buildBackup(velerov1api.BackupPhaseCompleted, baseTime.Add(48*time.Hour), "schedule-2"), buildBackup(velerov1api.BackupPhaseCompleted, baseTime.Add(72*time.Hour), "schedule-2"), // schedule-3 buildBackup(velerov1api.BackupPhaseNew, baseTime, "schedule-3"), buildBackup(velerov1api.BackupPhaseInProgress, baseTime.Add(time.Minute), "schedule-3"), buildBackup(velerov1api.BackupPhasePartiallyFailed, baseTime.Add(2*time.Minute), "schedule-3"), }, want: map[string]time.Time{ "": baseTime.Add(30 * time.Minute), "schedule-1": baseTime, "schedule-2": baseTime.Add(72 * time.Hour), }, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { assert.Equal(t, tc.want, getLastSuccessBySchedule(tc.backups)) }) } } // Unit tests to make sure that the backup's status is updated correctly during reconcile. // To clear up confusion whether status can be updated with Patch alone without status writer and not kbClient.Status().Patch() func TestPatchResourceWorksWithStatus(t *testing.T) { type args struct { original *velerov1api.Backup updated *velerov1api.Backup } tests := []struct { name string args args wantErr bool }{ { name: "patch backup status", args: args{ original: defaultBackup().SnapshotMoveData(false).Result(), updated: defaultBackup().SnapshotMoveData(false).WithStatus(velerov1api.BackupStatus{ CSIVolumeSnapshotsCompleted: 1, }).Result(), }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { scheme := runtime.NewScheme() error := velerov1api.AddToScheme(scheme) if error != nil { t.Errorf("PatchResource() error = %v", error) } fakeClient := fakeClient.NewClientBuilder().WithScheme(scheme).WithObjects(tt.args.original).Build() fromCluster := &velerov1api.Backup{ ObjectMeta: metav1.ObjectMeta{ Name: tt.args.original.Name, Namespace: tt.args.original.Namespace, }, } // check original exists if err := fakeClient.Get(t.Context(), kbclient.ObjectKeyFromObject(tt.args.updated), fromCluster); err != nil { t.Errorf("PatchResource() error = %v", err) } // ignore resourceVersion tt.args.updated.ResourceVersion = fromCluster.ResourceVersion tt.args.original.ResourceVersion = fromCluster.ResourceVersion if err := kubeutil.PatchResource(tt.args.original, tt.args.updated, fakeClient); (err != nil) != tt.wantErr { t.Errorf("PatchResource() error = %v, wantErr %v", err, tt.wantErr) } // check updated exists if err := fakeClient.Get(t.Context(), kbclient.ObjectKeyFromObject(tt.args.updated), fromCluster); err != nil { t.Errorf("PatchResource() error = %v", err) } // check fromCluster is equal to updated if !reflect.DeepEqual(fromCluster, tt.args.updated) { t.Error(cmp.Diff(fromCluster, tt.args.updated)) } }) } } ================================================ FILE: pkg/controller/backup_deletion_controller.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package controller import ( "context" "encoding/json" "fmt" "time" jsonpatch "github.com/evanphx/json-patch/v5" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" kubeerrs "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/utils/clock" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/internal/credentials" "github.com/vmware-tanzu/velero/internal/delete" "github.com/vmware-tanzu/velero/internal/volume" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" "github.com/vmware-tanzu/velero/pkg/constant" "github.com/vmware-tanzu/velero/pkg/discovery" "github.com/vmware-tanzu/velero/pkg/label" "github.com/vmware-tanzu/velero/pkg/metrics" "github.com/vmware-tanzu/velero/pkg/persistence" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" vsv1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/volumesnapshotter/v1" "github.com/vmware-tanzu/velero/pkg/podvolume" "github.com/vmware-tanzu/velero/pkg/repository" repomanager "github.com/vmware-tanzu/velero/pkg/repository/manager" repotypes "github.com/vmware-tanzu/velero/pkg/repository/types" "github.com/vmware-tanzu/velero/pkg/util/boolptr" "github.com/vmware-tanzu/velero/pkg/util/csi" "github.com/vmware-tanzu/velero/pkg/util/filesystem" "github.com/vmware-tanzu/velero/pkg/util/kube" veleroutil "github.com/vmware-tanzu/velero/pkg/util/velero" ) const ( deleteBackupRequestMaxAge = 24 * time.Hour ) type backupDeletionReconciler struct { client.Client logger logrus.FieldLogger backupTracker BackupTracker repoMgr repomanager.Manager metrics *metrics.ServerMetrics clock clock.Clock discoveryHelper discovery.Helper newPluginManager func(logrus.FieldLogger) clientmgmt.Manager backupStoreGetter persistence.ObjectBackupStoreGetter credentialStore credentials.FileStore repoEnsurer *repository.Ensurer } // NewBackupDeletionReconciler creates a new backup deletion reconciler. func NewBackupDeletionReconciler( logger logrus.FieldLogger, client client.Client, backupTracker BackupTracker, repoMgr repomanager.Manager, metrics *metrics.ServerMetrics, helper discovery.Helper, newPluginManager func(logrus.FieldLogger) clientmgmt.Manager, backupStoreGetter persistence.ObjectBackupStoreGetter, credentialStore credentials.FileStore, repoEnsurer *repository.Ensurer, ) *backupDeletionReconciler { return &backupDeletionReconciler{ Client: client, logger: logger, backupTracker: backupTracker, repoMgr: repoMgr, metrics: metrics, clock: clock.RealClock{}, discoveryHelper: helper, newPluginManager: newPluginManager, backupStoreGetter: backupStoreGetter, credentialStore: credentialStore, repoEnsurer: repoEnsurer, } } func (r *backupDeletionReconciler) SetupWithManager(mgr ctrl.Manager) error { // Make sure the expired requests can be deleted eventually s := kube.NewPeriodicalEnqueueSource(r.logger.WithField("controller", constant.ControllerBackupDeletion), mgr.GetClient(), &velerov1api.DeleteBackupRequestList{}, time.Hour, kube.PeriodicalEnqueueSourceOption{}) return ctrl.NewControllerManagedBy(mgr). For(&velerov1api.DeleteBackupRequest{}). WatchesRawSource(s). Complete(r) } // +kubebuilder:rbac:groups=velero.io,resources=deletebackuprequests,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=velero.io,resources=deletebackuprequests/status,verbs=get;update;patch // +kubebuilder:rbac:groups=velero.io,resources=backups,verbs=delete func (r *backupDeletionReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := r.logger.WithFields(logrus.Fields{ "controller": constant.ControllerBackupDeletion, "deletebackuprequest": req.String(), }) log.Debug("Getting deletebackuprequest") dbr := &velerov1api.DeleteBackupRequest{} if err := r.Get(ctx, req.NamespacedName, dbr); err != nil { if apierrors.IsNotFound(err) { log.Debug("Unable to find the deletebackuprequest") return ctrl.Result{}, nil } log.WithError(err).Error("Error getting deletebackuprequest") return ctrl.Result{}, err } // Since we use the reconciler along with the PeriodicalEnqueueSource, there may be reconciliation triggered by // stale requests. if dbr.Status.Phase == velerov1api.DeleteBackupRequestPhaseProcessed || dbr.Status.Phase == velerov1api.DeleteBackupRequestPhaseInProgress { age := r.clock.Now().Sub(dbr.CreationTimestamp.Time) if age >= deleteBackupRequestMaxAge { // delete the expired request log.Debugf("The request is expired, status: %s, deleting it.", dbr.Status.Phase) if err := r.Delete(ctx, dbr); err != nil { log.WithError(err).Error("Error deleting DeleteBackupRequest") } } else { log.Infof("The request has status '%s', skip.", dbr.Status.Phase) } return ctrl.Result{}, nil } // Make sure we have the backup name if dbr.Spec.BackupName == "" { err := r.patchDeleteBackupRequestWithError(ctx, dbr, errors.New("spec.backupName is required")) return ctrl.Result{}, err } log = log.WithField("backup", dbr.Spec.BackupName) // Remove any existing deletion requests for this backup so we only have // one at a time if errs := r.deleteExistingDeletionRequests(ctx, dbr, log); errs != nil { return ctrl.Result{}, kubeerrs.NewAggregate(errs) } // Don't allow deleting an in-progress backup if r.backupTracker.Contains(dbr.Namespace, dbr.Spec.BackupName) { err := r.patchDeleteBackupRequestWithError(ctx, dbr, errors.New("backup is still in progress")) return ctrl.Result{}, err } // Get the backup we're trying to delete backup := &velerov1api.Backup{} if err := r.Get(ctx, types.NamespacedName{ Namespace: dbr.Namespace, Name: dbr.Spec.BackupName, }, backup); apierrors.IsNotFound(err) { // Couldn't find backup - update status to Processed and record the not-found error err = r.patchDeleteBackupRequestWithError(ctx, dbr, errors.New("backup not found")) return ctrl.Result{}, err } else if err != nil { return ctrl.Result{}, errors.Wrap(err, "error getting backup") } // Don't allow deleting backups in read-only storage locations location := &velerov1api.BackupStorageLocation{} if err := r.Get(context.Background(), client.ObjectKey{ Namespace: backup.Namespace, Name: backup.Spec.StorageLocation, }, location); err != nil { if apierrors.IsNotFound(err) { err := r.patchDeleteBackupRequestWithError(ctx, dbr, fmt.Errorf("backup storage location %s not found", backup.Spec.StorageLocation)) return ctrl.Result{}, err } return ctrl.Result{}, errors.Wrap(err, "error getting backup storage location") } if location.Spec.AccessMode == velerov1api.BackupStorageLocationAccessModeReadOnly { err := r.patchDeleteBackupRequestWithError(ctx, dbr, fmt.Errorf("cannot delete backup because backup storage location %s is currently in read-only mode", location.Name)) return ctrl.Result{}, err } if !veleroutil.BSLIsAvailable(*location) { err := r.patchDeleteBackupRequestWithError(ctx, dbr, fmt.Errorf("cannot delete backup because backup storage location %s is currently in Unavailable state", location.Name)) return ctrl.Result{}, err } // if the request object has no labels defined, initialize an empty map since // we will be updating labels if dbr.Labels == nil { dbr.Labels = map[string]string{} } // Update status to InProgress and set backup-name and backup-uid label if needed dbr, err := r.patchDeleteBackupRequest(ctx, dbr, func(r *velerov1api.DeleteBackupRequest) { r.Status.Phase = velerov1api.DeleteBackupRequestPhaseInProgress if r.Labels[velerov1api.BackupNameLabel] == "" { r.Labels[velerov1api.BackupNameLabel] = label.GetValidName(dbr.Spec.BackupName) } if r.Labels[velerov1api.BackupUIDLabel] == "" { r.Labels[velerov1api.BackupUIDLabel] = string(backup.UID) } }) if err != nil { return ctrl.Result{}, err } // Set backup status to Deleting backup, err = r.patchBackup(ctx, backup, func(b *velerov1api.Backup) { b.Status.Phase = velerov1api.BackupPhaseDeleting }) if err != nil { log.WithError(err).Error("Error setting backup phase to deleting") err2 := r.patchDeleteBackupRequestWithError(ctx, dbr, errors.Wrap(err, "error setting backup phase to deleting")) return ctrl.Result{}, err2 } backupScheduleName := backup.GetLabels()[velerov1api.ScheduleNameLabel] r.metrics.RegisterBackupDeletionAttempt(backupScheduleName) pluginManager := r.newPluginManager(log) defer pluginManager.CleanupClients() backupStore, err := r.backupStoreGetter.Get(location, pluginManager, log) if err != nil { log.WithError(err).Error("Error getting the backup store") err2 := r.patchDeleteBackupRequestWithError(ctx, dbr, errors.Wrap(err, "error getting the backup store")) return ctrl.Result{}, err2 } actions, err := pluginManager.GetDeleteItemActions() log.Debugf("%d actions before invoking actions", len(actions)) if err != nil { log.WithError(err).Error("Error getting delete item actions") err2 := r.patchDeleteBackupRequestWithError(ctx, dbr, errors.New("error getting delete item actions")) return ctrl.Result{}, err2 } // don't defer CleanupClients here, since it was already called above. var errs []string if len(actions) > 0 { // Download the tarball backupFile, err := downloadToTempFile(backup.Name, backupStore, log) if err != nil { log.WithError(err).Errorf("Unable to download tarball for backup %s, skipping associated DeleteItemAction plugins", backup.Name) log.Info("Cleaning up CSI volumesnapshots") r.deleteCSIVolumeSnapshotsIfAny(ctx, backup, log) } else { defer closeAndRemoveFile(backupFile, r.logger) deleteCtx := &delete.Context{ Backup: backup, BackupReader: backupFile, Actions: actions, Log: r.logger, DiscoveryHelper: r.discoveryHelper, Filesystem: filesystem.NewFileSystem(), } // Optimization: wrap in a gofunc? Would be useful for large backups with lots of objects. // but what do we do with the error returned? We can't just swallow it as that may lead to dangling resources. err = delete.InvokeDeleteActions(deleteCtx) if err != nil { log.WithError(err).Error("Error invoking delete item actions") err2 := r.patchDeleteBackupRequestWithError(ctx, dbr, errors.New("error invoking delete item actions")) return ctrl.Result{}, err2 } } } if backupStore != nil { log.Info("Removing PV snapshots") if snapshots, err := backupStore.GetBackupVolumeSnapshots(backup.Name); err != nil { errs = append(errs, errors.Wrap(err, "error getting backup's volume snapshots").Error()) } else { volumeSnapshotters := make(map[string]vsv1.VolumeSnapshotter) for _, snapshot := range snapshots { log.WithField("providerSnapshotID", snapshot.Status.ProviderSnapshotID).Info("Removing snapshot associated with backup") volumeSnapshotter, ok := volumeSnapshotters[snapshot.Spec.Location] if !ok { if volumeSnapshotter, err = r.volumeSnapshottersForVSL(ctx, backup.Namespace, snapshot.Spec.Location, pluginManager); err != nil { errs = append(errs, err.Error()) continue } volumeSnapshotters[snapshot.Spec.Location] = volumeSnapshotter } if err := volumeSnapshotter.DeleteSnapshot(snapshot.Status.ProviderSnapshotID); err != nil { errs = append(errs, errors.Wrapf(err, "error deleting snapshot %s", snapshot.Status.ProviderSnapshotID).Error()) } } } } log.Info("Removing pod volume snapshots") if deleteErrs := r.deletePodVolumeSnapshots(ctx, backup); len(deleteErrs) > 0 { for _, err := range deleteErrs { errs = append(errs, err.Error()) } } if boolptr.IsSetToTrue(backup.Spec.SnapshotMoveData) { log.Info("Removing snapshot data by data mover") if deleteErrs := r.deleteMovedSnapshots(ctx, backup); len(deleteErrs) > 0 { for _, err := range deleteErrs { errs = append(errs, err.Error()) } } duList := &velerov2alpha1.DataUploadList{} log.Info("Removing local datauploads") if err := r.Client.List(ctx, duList, &client.ListOptions{ Namespace: backup.Namespace, LabelSelector: labels.SelectorFromSet(map[string]string{ velerov1api.BackupNameLabel: label.GetValidName(backup.Name), }), }); err != nil { log.WithError(err).Error("Error listing datauploads") errs = append(errs, err.Error()) } else { for i := range duList.Items { du := duList.Items[i] if err := r.Delete(ctx, &du); err != nil { errs = append(errs, err.Error()) } } } } if backupStore != nil { log.Info("Removing backup from backup storage") if err := backupStore.DeleteBackup(backup.Name); err != nil { errs = append(errs, err.Error()) } } log.Info("Removing restores") restoreList := &velerov1api.RestoreList{} selector := labels.Everything() if err := r.List(ctx, restoreList, &client.ListOptions{ Namespace: backup.Namespace, LabelSelector: selector, }); err != nil { log.WithError(errors.WithStack(err)).Error("Error listing restore API objects") } else { // Restore files in object storage will be handled by restore finalizer, so we simply need to initiate a delete request on restores here. for i, restore := range restoreList.Items { if restore.Spec.BackupName != backup.Name { continue } restoreLog := log.WithField("restore", kube.NamespaceAndName(&restoreList.Items[i])) restoreLog.Info("Deleting restore referencing backup") if err := r.Delete(ctx, &restoreList.Items[i]); err != nil { errs = append(errs, errors.Wrapf(err, "error deleting restore %s", kube.NamespaceAndName(&restoreList.Items[i])).Error()) } } // Wait for the deletion of restores within certain amount of time. // Notice that there could be potential errors during the finalization process, which may result in the failure to delete the restore. // Therefore, it is advisable to set a timeout period for waiting. err := wait.PollUntilContextTimeout(ctx, time.Second, time.Minute, true, func(ctx context.Context) (bool, error) { restoreList := &velerov1api.RestoreList{} if err := r.List(ctx, restoreList, &client.ListOptions{Namespace: backup.Namespace, LabelSelector: selector}); err != nil { return false, err } cnt := 0 for _, restore := range restoreList.Items { if restore.Spec.BackupName != backup.Name { continue } cnt++ } if cnt > 0 { return false, nil } else { return true, nil } }) if err != nil { log.WithError(err).Error("Error polling for deletion of restores") errs = append(errs, errors.Wrapf(err, "error deleting restore %s", err).Error()) } } if len(errs) == 0 { // Only try to delete the backup object from kube if everything preceding went smoothly if err := r.Delete(ctx, backup); err != nil { errs = append(errs, errors.Wrapf(err, "error deleting backup %s", kube.NamespaceAndName(backup)).Error()) } } if len(errs) == 0 { r.metrics.RegisterBackupDeletionSuccess(backupScheduleName) } else { r.metrics.RegisterBackupDeletionFailed(backupScheduleName) } // Update status to processed and record errors if _, err := r.patchDeleteBackupRequest(ctx, dbr, func(r *velerov1api.DeleteBackupRequest) { r.Status.Phase = velerov1api.DeleteBackupRequestPhaseProcessed r.Status.Errors = errs }); err != nil { return ctrl.Result{}, err } // Everything deleted correctly, so we can delete all DeleteBackupRequests for this backup if len(errs) == 0 { labelSelector, err := labels.Parse(fmt.Sprintf("%s=%s,%s=%s", velerov1api.BackupNameLabel, label.GetValidName(backup.Name), velerov1api.BackupUIDLabel, backup.UID)) if err != nil { // Should not be here r.logger.WithError(err).WithField("backup", kube.NamespaceAndName(backup)).Error("error creating label selector for the backup for deleting DeleteBackupRequests") return ctrl.Result{}, nil } alldbr := &velerov1api.DeleteBackupRequest{} err = r.DeleteAllOf(ctx, alldbr, client.MatchingLabelsSelector{ Selector: labelSelector, }, client.InNamespace(dbr.Namespace)) if err != nil { // If this errors, all we can do is log it. r.logger.WithError(err).WithField("backup", kube.NamespaceAndName(backup)).Error("error deleting all associated DeleteBackupRequests after successfully deleting the backup") } } log.Infof("Reconciliation done") return ctrl.Result{}, nil } func (r *backupDeletionReconciler) volumeSnapshottersForVSL( ctx context.Context, namespace, vslName string, pluginManager clientmgmt.Manager, ) (vsv1.VolumeSnapshotter, error) { vsl := &velerov1api.VolumeSnapshotLocation{} if err := r.Client.Get(ctx, types.NamespacedName{ Namespace: namespace, Name: vslName, }, vsl); err != nil { return nil, errors.Wrapf(err, "error getting volume snapshot location %s", vslName) } // add credential to config err := volume.UpdateVolumeSnapshotLocationWithCredentialConfig(vsl, r.credentialStore) if err != nil { return nil, errors.WithStack(err) } volumeSnapshotter, err := pluginManager.GetVolumeSnapshotter(vsl.Spec.Provider) if err != nil { return nil, errors.Wrapf(err, "error getting volume snapshotter for provider %s", vsl.Spec.Provider) } if err = volumeSnapshotter.Init(vsl.Spec.Config); err != nil { return nil, errors.Wrapf(err, "error initializing volume snapshotter for volume snapshot location %s", vslName) } return volumeSnapshotter, nil } func (r *backupDeletionReconciler) deleteExistingDeletionRequests(ctx context.Context, req *velerov1api.DeleteBackupRequest, log logrus.FieldLogger) []error { log.Info("Removing existing deletion requests for backup") dbrList := &velerov1api.DeleteBackupRequestList{} selector := label.NewSelectorForBackup(req.Spec.BackupName) if err := r.List(ctx, dbrList, &client.ListOptions{ Namespace: req.Namespace, LabelSelector: selector, }); err != nil { return []error{errors.Wrap(err, "error listing existing DeleteBackupRequests for backup")} } var errs []error for i, dbr := range dbrList.Items { if dbr.Name == req.Name { continue } if err := r.Delete(ctx, &dbrList.Items[i]); err != nil { errs = append(errs, errors.WithStack(err)) } else { log.Infof("deletion request '%s' removed.", dbr.Name) } } return errs } // deleteCSIVolumeSnapshotsIfAny clean up the CSI snapshots created by the backup, this should be called when the backup is failed // when it's running, e.g. due to velero pod restart, and the backup.tar is failed to be downloaded from storage. func (r *backupDeletionReconciler) deleteCSIVolumeSnapshotsIfAny(ctx context.Context, backup *velerov1api.Backup, log logrus.FieldLogger) { vsList := snapshotv1api.VolumeSnapshotList{} if err := r.Client.List(ctx, &vsList, &client.ListOptions{ LabelSelector: labels.SelectorFromSet(map[string]string{ velerov1api.BackupNameLabel: label.GetValidName(backup.Name), }), }); err != nil { log.WithError(err).Warnf("Could not list volume snapshots, abort") return } for _, item := range vsList.Items { vs := item csi.CleanupVolumeSnapshot(&vs, r.Client, log) } } func (r *backupDeletionReconciler) deletePodVolumeSnapshots(ctx context.Context, backup *velerov1api.Backup) []error { if r.repoMgr == nil { return nil } directSnapshots, err := getSnapshotsInBackup(ctx, backup, r.Client) if err != nil { return []error{err} } return batchDeleteSnapshots(ctx, r.repoEnsurer, r.repoMgr, directSnapshots, backup, r.logger) } var batchDeleteSnapshotFunc = batchDeleteSnapshots func (r *backupDeletionReconciler) deleteMovedSnapshots(ctx context.Context, backup *velerov1api.Backup) []error { if r.repoMgr == nil { return nil } list := &corev1api.ConfigMapList{} if err := r.Client.List(ctx, list, &client.ListOptions{ Namespace: backup.Namespace, LabelSelector: labels.SelectorFromSet( map[string]string{ velerov1api.BackupNameLabel: label.GetValidName(backup.Name), velerov1api.DataUploadSnapshotInfoLabel: "true", }), }); err != nil { return []error{errors.Wrapf(err, "failed to retrieve config for snapshot info")} } var errs []error directSnapshots := map[string][]repotypes.SnapshotIdentifier{} for i := range list.Items { cm := list.Items[i] if len(cm.Data) == 0 { errs = append(errs, errors.New("no snapshot info in config")) continue } b, err := json.Marshal(cm.Data) if err != nil { errs = append(errs, errors.Wrapf(err, "fail to marshal the snapshot info into JSON")) continue } snapshot := repotypes.SnapshotIdentifier{} if err := json.Unmarshal(b, &snapshot); err != nil { errs = append(errs, errors.Wrapf(err, "failed to unmarshal snapshot info")) continue } if snapshot.SnapshotID == "" || snapshot.VolumeNamespace == "" || snapshot.RepositoryType == "" { errs = append(errs, errors.Errorf("invalid snapshot, ID %s, namespace %s, repository %s", snapshot.SnapshotID, snapshot.VolumeNamespace, snapshot.RepositoryType)) continue } if directSnapshots[snapshot.VolumeNamespace] == nil { directSnapshots[snapshot.VolumeNamespace] = []repotypes.SnapshotIdentifier{} } directSnapshots[snapshot.VolumeNamespace] = append(directSnapshots[snapshot.VolumeNamespace], snapshot) r.logger.Infof("Deleting snapshot %s, namespace: %s, repo type: %s", snapshot.SnapshotID, snapshot.VolumeNamespace, snapshot.RepositoryType) } for i := range list.Items { cm := list.Items[i] if err := r.Client.Delete(ctx, &cm); err != nil { r.logger.Warnf("Failed to delete snapshot info configmap %s/%s: %v", cm.Namespace, cm.Name, err) } } if len(directSnapshots) > 0 { deleteErrs := batchDeleteSnapshotFunc(ctx, r.repoEnsurer, r.repoMgr, directSnapshots, backup, r.logger) errs = append(errs, deleteErrs...) } return errs } func (r *backupDeletionReconciler) patchDeleteBackupRequest(ctx context.Context, req *velerov1api.DeleteBackupRequest, mutate func(*velerov1api.DeleteBackupRequest)) (*velerov1api.DeleteBackupRequest, error) { original := req.DeepCopy() mutate(req) if err := r.Patch(ctx, req, client.MergeFrom(original)); err != nil { return nil, errors.Wrap(err, "error patching the deletebackuprquest") } return req, nil } func (r *backupDeletionReconciler) patchDeleteBackupRequestWithError(ctx context.Context, req *velerov1api.DeleteBackupRequest, err error) error { _, err = r.patchDeleteBackupRequest(ctx, req, func(r *velerov1api.DeleteBackupRequest) { r.Status.Phase = velerov1api.DeleteBackupRequestPhaseProcessed r.Status.Errors = []string{err.Error()} }) return err } func (r *backupDeletionReconciler) patchBackup(ctx context.Context, backup *velerov1api.Backup, mutate func(*velerov1api.Backup)) (*velerov1api.Backup, error) { //TODO: The patchHelper can't be used here because the `backup/xxx/status` does not exist, until the backup resource is refactored // Record original json oldData, err := json.Marshal(backup) if err != nil { return nil, errors.Wrap(err, "error marshaling original Backup") } newBackup := backup.DeepCopy() mutate(newBackup) newData, err := json.Marshal(newBackup) if err != nil { return nil, errors.Wrap(err, "error marshaling updated Backup") } patchBytes, err := jsonpatch.CreateMergePatch(oldData, newData) if err != nil { return nil, errors.Wrap(err, "error creating json merge patch for Backup") } if err := r.Client.Patch(ctx, backup, client.RawPatch(types.MergePatchType, patchBytes)); err != nil { return nil, errors.Wrap(err, "error patching Backup") } return backup, nil } // getSnapshotsInBackup returns a list of all pod volume snapshot ids associated with // a given Velero backup. func getSnapshotsInBackup(ctx context.Context, backup *velerov1api.Backup, kbClient client.Client) (map[string][]repotypes.SnapshotIdentifier, error) { podVolumeBackups := &velerov1api.PodVolumeBackupList{} options := &client.ListOptions{ LabelSelector: labels.Set(map[string]string{ velerov1api.BackupNameLabel: label.GetValidName(backup.Name), }).AsSelector(), } err := kbClient.List(ctx, podVolumeBackups, options) if err != nil { return nil, errors.WithStack(err) } return podvolume.GetSnapshotIdentifier(podVolumeBackups), nil } func batchDeleteSnapshots(ctx context.Context, repoEnsurer *repository.Ensurer, repoMgr repomanager.Manager, directSnapshots map[string][]repotypes.SnapshotIdentifier, backup *velerov1api.Backup, logger logrus.FieldLogger) []error { var errs []error for volumeNamespace, snapshots := range directSnapshots { batchForget := []string{} for _, snapshot := range snapshots { batchForget = append(batchForget, snapshot.SnapshotID) } // For volumes in one backup, the BSL and repositoryType should always be the same repoType := snapshots[0].RepositoryType repo, err := repoEnsurer.EnsureRepo(ctx, backup.Namespace, volumeNamespace, backup.Spec.StorageLocation, repoType) if err != nil { errs = append(errs, errors.Wrapf(err, "error to ensure repo %s-%s-%s, skip deleting PVB snapshots %v", backup.Spec.StorageLocation, volumeNamespace, repoType, batchForget)) continue } if forgetErrs := repoMgr.BatchForget(ctx, repo, batchForget); len(forgetErrs) > 0 { errs = append(errs, forgetErrs...) continue } logger.Infof("Batch deleted snapshots %v", batchForget) } return errs } ================================================ FILE: pkg/controller/backup_deletion_controller_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package controller import ( "bytes" "encoding/json" "errors" "fmt" "io" "reflect" "time" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" "context" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "strings" "testing" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/internal/volume" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" pkgbackup "github.com/vmware-tanzu/velero/pkg/backup" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/metrics" persistencemocks "github.com/vmware-tanzu/velero/pkg/persistence/mocks" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" pluginmocks "github.com/vmware-tanzu/velero/pkg/plugin/mocks" "github.com/vmware-tanzu/velero/pkg/plugin/velero" "github.com/vmware-tanzu/velero/pkg/plugin/velero/mocks" "github.com/vmware-tanzu/velero/pkg/repository" repomanager "github.com/vmware-tanzu/velero/pkg/repository/manager" repomocks "github.com/vmware-tanzu/velero/pkg/repository/mocks" repotypes "github.com/vmware-tanzu/velero/pkg/repository/types" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) type backupDeletionControllerTestData struct { fakeClient client.Client volumeSnapshotter *velerotest.FakeVolumeSnapshotter backupStore *persistencemocks.BackupStore controller *backupDeletionReconciler req ctrl.Request } func defaultTestDbr() *velerov1api.DeleteBackupRequest { req := pkgbackup.NewDeleteBackupRequest("foo", "uid") req.Namespace = velerov1api.DefaultNamespace req.Name = "foo-abcde" return req } func setupBackupDeletionControllerTest(t *testing.T, req *velerov1api.DeleteBackupRequest, objects ...runtime.Object) *backupDeletionControllerTestData { t.Helper() var ( fakeClient = velerotest.NewFakeControllerRuntimeClient(t, append(objects, req)...) volumeSnapshotter = &velerotest.FakeVolumeSnapshotter{SnapshotsTaken: sets.NewString()} pluginManager = &pluginmocks.Manager{} backupStore = &persistencemocks.BackupStore{} ) data := &backupDeletionControllerTestData{ fakeClient: fakeClient, volumeSnapshotter: volumeSnapshotter, backupStore: backupStore, controller: NewBackupDeletionReconciler( velerotest.NewLogger(), fakeClient, NewBackupTracker(), nil, // repository manager metrics.NewServerMetrics(), nil, // discovery helper func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager }, NewFakeSingleObjectBackupStoreGetter(backupStore), velerotest.NewFakeCredentialsFileStore("", nil), nil, ), req: ctrl.Request{NamespacedName: types.NamespacedName{Namespace: req.Namespace, Name: req.Name}}, } pluginManager.On("CleanupClients").Return(nil) return data } func TestBackupDeletionControllerReconcile(t *testing.T) { t.Run("failed to get backup store", func(t *testing.T) { backup := builder.ForBackup(velerov1api.DefaultNamespace, "foo").StorageLocation("default").Result() location := &velerov1api.BackupStorageLocation{ ObjectMeta: metav1.ObjectMeta{ Namespace: backup.Namespace, Name: backup.Spec.StorageLocation, }, Spec: velerov1api.BackupStorageLocationSpec{ Provider: "objStoreProvider", StorageType: velerov1api.StorageType{ ObjectStorage: &velerov1api.ObjectStorageLocation{ Bucket: "bucket", }, }, }, Status: velerov1api.BackupStorageLocationStatus{ Phase: velerov1api.BackupStorageLocationPhaseAvailable, }, } dbr := defaultTestDbr() td := setupBackupDeletionControllerTest(t, dbr, location, backup) td.controller.backupStoreGetter = &fakeErrorBackupStoreGetter{} _, err := td.controller.Reconcile(ctx, td.req) require.NoError(t, err) res := &velerov1api.DeleteBackupRequest{} td.fakeClient.Get(ctx, td.req.NamespacedName, res) assert.Equal(t, "Processed", string(res.Status.Phase)) assert.Len(t, res.Status.Errors, 1) assert.True(t, strings.HasPrefix(res.Status.Errors[0], "error getting the backup store")) }) t.Run("missing spec.backupName", func(t *testing.T) { dbr := defaultTestDbr() dbr.Spec.BackupName = "" td := setupBackupDeletionControllerTest(t, dbr) _, err := td.controller.Reconcile(ctx, td.req) require.NoError(t, err) res := &velerov1api.DeleteBackupRequest{} err = td.fakeClient.Get(ctx, td.req.NamespacedName, res) require.NoError(t, err) assert.Equal(t, "Processed", string(res.Status.Phase)) assert.Len(t, res.Status.Errors, 1) assert.Equal(t, "spec.backupName is required", res.Status.Errors[0]) }) t.Run("existing deletion requests for the backup are deleted", func(t *testing.T) { input := defaultTestDbr() td := setupBackupDeletionControllerTest(t, input) // add the backup to the tracker so the execution of reconcile doesn't progress // past checking for an in-progress backup. this makes validation easier. td.controller.backupTracker.Add(td.req.Namespace, input.Spec.BackupName) existing := &velerov1api.DeleteBackupRequest{ ObjectMeta: metav1.ObjectMeta{ Namespace: td.req.Namespace, Name: "bar", Labels: map[string]string{ velerov1api.BackupNameLabel: input.Spec.BackupName, }, }, Spec: velerov1api.DeleteBackupRequestSpec{ BackupName: input.Spec.BackupName, }, } err := td.fakeClient.Create(t.Context(), existing) require.NoError(t, err) existing2 := &velerov1api.DeleteBackupRequest{ ObjectMeta: metav1.ObjectMeta{ Namespace: td.req.Namespace, Name: "bar-2", Labels: map[string]string{ velerov1api.BackupNameLabel: "some-other-backup", }, }, Spec: velerov1api.DeleteBackupRequestSpec{ BackupName: "some-other-backup", }, } err = td.fakeClient.Create(t.Context(), existing2) require.NoError(t, err) _, err = td.controller.Reconcile(t.Context(), td.req) require.NoError(t, err) // verify "existing" is deleted err = td.fakeClient.Get(t.Context(), types.NamespacedName{ Namespace: existing.Namespace, Name: existing.Name, }, &velerov1api.DeleteBackupRequest{}) assert.True(t, apierrors.IsNotFound(err), "Expected not found error, but actual value of error: %v", err) // verify "existing2" remains assert.NoError(t, td.fakeClient.Get(t.Context(), types.NamespacedName{ Namespace: existing2.Namespace, Name: existing2.Name, }, &velerov1api.DeleteBackupRequest{})) }) t.Run("deleting an in progress backup isn't allowed", func(t *testing.T) { dbr := defaultTestDbr() td := setupBackupDeletionControllerTest(t, dbr) td.controller.backupTracker.Add(td.req.Namespace, dbr.Spec.BackupName) _, err := td.controller.Reconcile(t.Context(), td.req) require.NoError(t, err) res := &velerov1api.DeleteBackupRequest{} err = td.fakeClient.Get(ctx, td.req.NamespacedName, res) require.NoError(t, err) assert.Equal(t, "Processed", string(res.Status.Phase)) assert.Len(t, res.Status.Errors, 1) assert.Equal(t, "backup is still in progress", res.Status.Errors[0]) }) t.Run("unable to find backup", func(t *testing.T) { td := setupBackupDeletionControllerTest(t, defaultTestDbr()) _, err := td.controller.Reconcile(t.Context(), td.req) require.NoError(t, err) res := &velerov1api.DeleteBackupRequest{} err = td.fakeClient.Get(ctx, td.req.NamespacedName, res) require.NoError(t, err) assert.Equal(t, "Processed", string(res.Status.Phase)) assert.Len(t, res.Status.Errors, 1) assert.Equal(t, "backup not found", res.Status.Errors[0]) }) t.Run("unable to find backup storage location", func(t *testing.T) { backup := builder.ForBackup(velerov1api.DefaultNamespace, "foo").StorageLocation("default").Result() td := setupBackupDeletionControllerTest(t, defaultTestDbr(), backup) _, err := td.controller.Reconcile(t.Context(), td.req) require.NoError(t, err) res := &velerov1api.DeleteBackupRequest{} err = td.fakeClient.Get(ctx, td.req.NamespacedName, res) require.NoError(t, err) assert.Equal(t, "Processed", string(res.Status.Phase)) assert.Len(t, res.Status.Errors, 1) assert.Equal(t, "backup storage location default not found", res.Status.Errors[0]) }) t.Run("backup storage location is in read-only mode", func(t *testing.T) { backup := builder.ForBackup(velerov1api.DefaultNamespace, "foo").StorageLocation("default").Result() location := builder.ForBackupStorageLocation("velero", "default").Phase(velerov1api.BackupStorageLocationPhaseAvailable).AccessMode(velerov1api.BackupStorageLocationAccessModeReadOnly).Result() td := setupBackupDeletionControllerTest(t, defaultTestDbr(), location, backup) _, err := td.controller.Reconcile(t.Context(), td.req) require.NoError(t, err) res := &velerov1api.DeleteBackupRequest{} err = td.fakeClient.Get(ctx, td.req.NamespacedName, res) require.NoError(t, err) assert.Equal(t, "Processed", string(res.Status.Phase)) assert.Len(t, res.Status.Errors, 1) assert.Equal(t, "cannot delete backup because backup storage location default is currently in read-only mode", res.Status.Errors[0]) }) t.Run("backup storage location is in unavailable state", func(t *testing.T) { backup := builder.ForBackup(velerov1api.DefaultNamespace, "foo").StorageLocation("default").Result() location := builder.ForBackupStorageLocation("velero", "default").Phase(velerov1api.BackupStorageLocationPhaseUnavailable).Result() td := setupBackupDeletionControllerTest(t, defaultTestDbr(), location, backup) _, err := td.controller.Reconcile(t.Context(), td.req) require.NoError(t, err) res := &velerov1api.DeleteBackupRequest{} err = td.fakeClient.Get(ctx, td.req.NamespacedName, res) require.NoError(t, err) assert.Equal(t, "Processed", string(res.Status.Phase)) assert.Len(t, res.Status.Errors, 1) assert.Equal(t, "cannot delete backup because backup storage location default is currently in Unavailable state", res.Status.Errors[0]) }) t.Run("full delete, no errors", func(t *testing.T) { input := defaultTestDbr() // Clear out resource labels to make sure the controller adds them and does not // panic when encountering a nil Labels map // (https://github.com/vmware-tanzu/velero/issues/1546) input.Labels = nil backup := builder.ForBackup(velerov1api.DefaultNamespace, "foo").Result() backup.UID = "uid" backup.Spec.StorageLocation = "primary" restore1 := builder.ForRestore(velerov1api.DefaultNamespace, "restore-1").Phase(velerov1api.RestorePhaseCompleted).Backup("foo").Result() restore2 := builder.ForRestore(velerov1api.DefaultNamespace, "restore-2").Phase(velerov1api.RestorePhaseCompleted).Backup("foo").Result() restore3 := builder.ForRestore(velerov1api.DefaultNamespace, "restore-3").Phase(velerov1api.RestorePhaseCompleted).Backup("some-other-backup").Result() location := &velerov1api.BackupStorageLocation{ ObjectMeta: metav1.ObjectMeta{ Namespace: backup.Namespace, Name: backup.Spec.StorageLocation, }, Spec: velerov1api.BackupStorageLocationSpec{ Provider: "objStoreProvider", StorageType: velerov1api.StorageType{ ObjectStorage: &velerov1api.ObjectStorageLocation{ Bucket: "bucket", }, }, }, Status: velerov1api.BackupStorageLocationStatus{ Phase: velerov1api.BackupStorageLocationPhaseAvailable, }, } snapshotLocation := &velerov1api.VolumeSnapshotLocation{ ObjectMeta: metav1.ObjectMeta{ Namespace: backup.Namespace, Name: "vsl-1", }, Spec: velerov1api.VolumeSnapshotLocationSpec{ Provider: "provider-1", }, } td := setupBackupDeletionControllerTest(t, input, backup, restore1, restore2, restore3, location, snapshotLocation) td.volumeSnapshotter.SnapshotsTaken.Insert("snap-1") snapshots := []*volume.Snapshot{ { Spec: volume.SnapshotSpec{ Location: "vsl-1", }, Status: volume.SnapshotStatus{ ProviderSnapshotID: "snap-1", }, }, } pluginManager := &pluginmocks.Manager{} pluginManager.On("GetVolumeSnapshotter", "provider-1").Return(td.volumeSnapshotter, nil) pluginManager.On("GetDeleteItemActions").Return(nil, nil) pluginManager.On("CleanupClients") td.controller.newPluginManager = func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager } td.backupStore.On("GetBackupVolumeSnapshots", input.Spec.BackupName).Return(snapshots, nil) td.backupStore.On("GetBackupContents", input.Spec.BackupName).Return(io.NopCloser(bytes.NewReader([]byte("hello world"))), nil) td.backupStore.On("DeleteBackup", input.Spec.BackupName).Return(nil) _, err := td.controller.Reconcile(t.Context(), td.req) require.NoError(t, err) // the dbr should be deleted res := &velerov1api.DeleteBackupRequest{} err = td.fakeClient.Get(ctx, td.req.NamespacedName, res) assert.True(t, apierrors.IsNotFound(err), "Expected not found error, but actual value of error: %v", err) if err == nil { t.Logf("status of the dbr: %s, errors in dbr: %v", res.Status.Phase, res.Status.Errors) } // backup CR, restore CR restore-1 and restore-2 should be deleted err = td.fakeClient.Get(t.Context(), types.NamespacedName{ Namespace: velerov1api.DefaultNamespace, Name: backup.Name, }, &velerov1api.Backup{}) assert.True(t, apierrors.IsNotFound(err), "Expected not found error, but actual value of error: %v", err) err = td.fakeClient.Get(t.Context(), types.NamespacedName{ Namespace: velerov1api.DefaultNamespace, Name: "restore-1", }, &velerov1api.Restore{}) assert.True(t, apierrors.IsNotFound(err), "Expected not found error, but actual value of error: %v", err) err = td.fakeClient.Get(t.Context(), types.NamespacedName{ Namespace: velerov1api.DefaultNamespace, Name: "restore-2", }, &velerov1api.Restore{}) assert.True(t, apierrors.IsNotFound(err), "Expected not found error, but actual value of error: %v", err) // restore-3 should remain err = td.fakeClient.Get(t.Context(), types.NamespacedName{ Namespace: velerov1api.DefaultNamespace, Name: "restore-3", }, &velerov1api.Restore{}) require.NoError(t, err) td.backupStore.AssertCalled(t, "DeleteBackup", input.Spec.BackupName) // Make sure snapshot was deleted assert.Equal(t, 0, td.volumeSnapshotter.SnapshotsTaken.Len()) }) t.Run("full delete, no errors, with backup name greater than 63 chars", func(t *testing.T) { backup := defaultBackup(). ObjectMeta( builder.WithName("the-really-long-backup-name-that-is-much-more-than-63-characters"), ). Result() backup.UID = "uid" backup.Spec.StorageLocation = "primary" restore1 := builder.ForRestore(backup.Namespace, "restore-1"). Phase(velerov1api.RestorePhaseCompleted). Backup(backup.Name). Result() restore2 := builder.ForRestore(backup.Namespace, "restore-2"). Phase(velerov1api.RestorePhaseCompleted). Backup(backup.Name). Result() restore3 := builder.ForRestore(backup.Namespace, "restore-3"). Phase(velerov1api.RestorePhaseCompleted). Backup("some-other-backup"). Result() dbr := pkgbackup.NewDeleteBackupRequest(backup.Name, "uid") dbr.Namespace = velerov1api.DefaultNamespace dbr.Name = "foo-abcde" // Clear out resource labels to make sure the controller adds them dbr.Labels = nil location := &velerov1api.BackupStorageLocation{ ObjectMeta: metav1.ObjectMeta{ Namespace: backup.Namespace, Name: backup.Spec.StorageLocation, }, Spec: velerov1api.BackupStorageLocationSpec{ Provider: "objStoreProvider", StorageType: velerov1api.StorageType{ ObjectStorage: &velerov1api.ObjectStorageLocation{ Bucket: "bucket", }, }, }, Status: velerov1api.BackupStorageLocationStatus{ Phase: velerov1api.BackupStorageLocationPhaseAvailable, }, } snapshotLocation := &velerov1api.VolumeSnapshotLocation{ ObjectMeta: metav1.ObjectMeta{ Namespace: backup.Namespace, Name: "vsl-1", }, Spec: velerov1api.VolumeSnapshotLocationSpec{ Provider: "provider-1", }, } td := setupBackupDeletionControllerTest(t, dbr, backup, restore1, restore2, restore3, location, snapshotLocation) snapshots := []*volume.Snapshot{ { Spec: volume.SnapshotSpec{ Location: "vsl-1", }, Status: volume.SnapshotStatus{ ProviderSnapshotID: "snap-1", }, }, } pluginManager := &pluginmocks.Manager{} pluginManager.On("GetVolumeSnapshotter", "provider-1").Return(td.volumeSnapshotter, nil) pluginManager.On("GetDeleteItemActions").Return(nil, nil) pluginManager.On("CleanupClients") td.controller.newPluginManager = func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager } td.backupStore.On("GetBackupVolumeSnapshots", dbr.Spec.BackupName).Return(snapshots, nil) td.backupStore.On("GetBackupContents", dbr.Spec.BackupName).Return(io.NopCloser(bytes.NewReader([]byte("hello world"))), nil) td.backupStore.On("DeleteBackup", dbr.Spec.BackupName).Return(nil) td.backupStore.On("DeleteRestore", "restore-1").Return(nil) td.backupStore.On("DeleteRestore", "restore-2").Return(nil) td.volumeSnapshotter.SnapshotsTaken.Insert("snap-1") _, err := td.controller.Reconcile(t.Context(), td.req) require.NoError(t, err) // the dbr should be deleted res := &velerov1api.DeleteBackupRequest{} err = td.fakeClient.Get(ctx, td.req.NamespacedName, res) assert.True(t, apierrors.IsNotFound(err), "Expected not found error, but actual value of error: %v", err) if err == nil { t.Logf("status of the dbr: %s, errors in dbr: %v", res.Status.Phase, res.Status.Errors) } // backup CR, restore CR restore-1 and restore-2 should be deleted err = td.fakeClient.Get(t.Context(), types.NamespacedName{ Namespace: velerov1api.DefaultNamespace, Name: backup.Name, }, &velerov1api.Backup{}) assert.True(t, apierrors.IsNotFound(err), "Expected not found error, but actual value of error: %v", err) err = td.fakeClient.Get(t.Context(), types.NamespacedName{ Namespace: velerov1api.DefaultNamespace, Name: "restore-1", }, &velerov1api.Restore{}) assert.True(t, apierrors.IsNotFound(err), "Expected not found error, but actual value of error: %v", err) err = td.fakeClient.Get(t.Context(), types.NamespacedName{ Namespace: velerov1api.DefaultNamespace, Name: "restore-2", }, &velerov1api.Restore{}) assert.True(t, apierrors.IsNotFound(err), "Expected not found error, but actual value of error: %v", err) // restore-3 should remain err = td.fakeClient.Get(t.Context(), types.NamespacedName{ Namespace: velerov1api.DefaultNamespace, Name: "restore-3", }, &velerov1api.Restore{}) require.NoError(t, err) // Make sure snapshot was deleted assert.Equal(t, 0, td.volumeSnapshotter.SnapshotsTaken.Len()) }) t.Run("backup is not downloaded when there are no DeleteItemAction plugins", func(t *testing.T) { backup := builder.ForBackup(velerov1api.DefaultNamespace, "foo").Result() backup.UID = "uid" backup.Spec.StorageLocation = "primary" input := defaultTestDbr() // Clear out resource labels to make sure the controller adds them and does not // panic when encountering a nil Labels map // (https://github.com/vmware-tanzu/velero/issues/1546) input.Labels = nil location := &velerov1api.BackupStorageLocation{ ObjectMeta: metav1.ObjectMeta{ Namespace: backup.Namespace, Name: backup.Spec.StorageLocation, }, Spec: velerov1api.BackupStorageLocationSpec{ Provider: "objStoreProvider", StorageType: velerov1api.StorageType{ ObjectStorage: &velerov1api.ObjectStorageLocation{ Bucket: "bucket", }, }, }, Status: velerov1api.BackupStorageLocationStatus{ Phase: velerov1api.BackupStorageLocationPhaseAvailable, }, } snapshotLocation := &velerov1api.VolumeSnapshotLocation{ ObjectMeta: metav1.ObjectMeta{ Namespace: backup.Namespace, Name: "vsl-1", }, Spec: velerov1api.VolumeSnapshotLocationSpec{ Provider: "provider-1", }, } td := setupBackupDeletionControllerTest(t, defaultTestDbr(), backup, location, snapshotLocation) td.volumeSnapshotter.SnapshotsTaken.Insert("snap-1") snapshots := []*volume.Snapshot{ { Spec: volume.SnapshotSpec{ Location: "vsl-1", }, Status: volume.SnapshotStatus{ ProviderSnapshotID: "snap-1", }, }, } pluginManager := &pluginmocks.Manager{} pluginManager.On("GetVolumeSnapshotter", "provider-1").Return(td.volumeSnapshotter, nil) pluginManager.On("GetDeleteItemActions").Return([]velero.DeleteItemAction{}, nil) pluginManager.On("CleanupClients") td.controller.newPluginManager = func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager } td.backupStore.On("GetBackupVolumeSnapshots", input.Spec.BackupName).Return(snapshots, nil) td.backupStore.On("DeleteBackup", input.Spec.BackupName).Return(nil) _, err := td.controller.Reconcile(t.Context(), td.req) require.NoError(t, err) td.backupStore.AssertNotCalled(t, "GetBackupContents", mock.Anything) td.backupStore.AssertCalled(t, "DeleteBackup", input.Spec.BackupName) // the dbr should be deleted res := &velerov1api.DeleteBackupRequest{} err = td.fakeClient.Get(ctx, td.req.NamespacedName, res) assert.True(t, apierrors.IsNotFound(err), "Expected not found error, but actual value of error: %v", err) if err == nil { t.Logf("status of the dbr: %s, errors in dbr: %v", res.Status.Phase, res.Status.Errors) } // backup CR should be deleted err = td.fakeClient.Get(t.Context(), types.NamespacedName{ Namespace: velerov1api.DefaultNamespace, Name: backup.Name, }, &velerov1api.Backup{}) assert.True(t, apierrors.IsNotFound(err), "Expected not found error, but actual value of error: %v", err) // Make sure snapshot was deleted assert.Equal(t, 0, td.volumeSnapshotter.SnapshotsTaken.Len()) }) t.Run("backup is still deleted if downloading tarball fails for DeleteItemAction plugins", func(t *testing.T) { backup := builder.ForBackup(velerov1api.DefaultNamespace, "foo").Result() backup.UID = "uid" backup.Spec.StorageLocation = "primary" input := defaultTestDbr() // Clear out resource labels to make sure the controller adds them and does not // panic when encountering a nil Labels map // (https://github.com/vmware-tanzu/velero/issues/1546) input.Labels = nil location := &velerov1api.BackupStorageLocation{ ObjectMeta: metav1.ObjectMeta{ Namespace: backup.Namespace, Name: backup.Spec.StorageLocation, }, Spec: velerov1api.BackupStorageLocationSpec{ Provider: "objStoreProvider", StorageType: velerov1api.StorageType{ ObjectStorage: &velerov1api.ObjectStorageLocation{ Bucket: "bucket", }, }, }, Status: velerov1api.BackupStorageLocationStatus{ Phase: velerov1api.BackupStorageLocationPhaseAvailable, }, } snapshotLocation := &velerov1api.VolumeSnapshotLocation{ ObjectMeta: metav1.ObjectMeta{ Namespace: backup.Namespace, Name: "vsl-1", }, Spec: velerov1api.VolumeSnapshotLocationSpec{ Provider: "provider-1", }, } csiSnapshot := builder.ForVolumeSnapshot("user-ns", "vs-1").ObjectMeta( builder.WithLabelsMap(map[string]string{ "velero.io/backup-name": "foo", })).SourcePVC("some-pvc").Result() td := setupBackupDeletionControllerTest(t, defaultTestDbr(), backup, location, snapshotLocation, csiSnapshot) td.volumeSnapshotter.SnapshotsTaken.Insert("snap-1") snapshots := []*volume.Snapshot{ { Spec: volume.SnapshotSpec{ Location: "vsl-1", }, Status: volume.SnapshotStatus{ ProviderSnapshotID: "snap-1", }, }, } pluginManager := &pluginmocks.Manager{} pluginManager.On("GetVolumeSnapshotter", "provider-1").Return(td.volumeSnapshotter, nil) pluginManager.On("GetDeleteItemActions").Return([]velero.DeleteItemAction{new(mocks.DeleteItemAction)}, nil) pluginManager.On("CleanupClients") td.controller.newPluginManager = func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager } td.backupStore.On("GetBackupVolumeSnapshots", input.Spec.BackupName).Return(snapshots, nil) td.backupStore.On("GetBackupContents", input.Spec.BackupName).Return(nil, fmt.Errorf("error downloading tarball")) td.backupStore.On("DeleteBackup", input.Spec.BackupName).Return(nil) _, err := td.controller.Reconcile(t.Context(), td.req) require.NoError(t, err) td.backupStore.AssertCalled(t, "GetBackupContents", input.Spec.BackupName) td.backupStore.AssertCalled(t, "DeleteBackup", input.Spec.BackupName) // the dbr should be deleted res := &velerov1api.DeleteBackupRequest{} err = td.fakeClient.Get(ctx, td.req.NamespacedName, res) assert.True(t, apierrors.IsNotFound(err), "Expected not found error, but actual value of error: %v", err) if err == nil { t.Logf("status of the dbr: %s, errors in dbr: %v", res.Status.Phase, res.Status.Errors) } // backup CR should be deleted err = td.fakeClient.Get(t.Context(), types.NamespacedName{ Namespace: velerov1api.DefaultNamespace, Name: backup.Name, }, &velerov1api.Backup{}) assert.True(t, apierrors.IsNotFound(err), "Expected not found error, but actual value of error: %v", err) // leaked CSI snapshot should be deleted err = td.fakeClient.Get(t.Context(), types.NamespacedName{ Namespace: "user-ns", Name: "vs-1", }, &snapshotv1api.VolumeSnapshot{}) assert.True(t, apierrors.IsNotFound(err), "Expected not found error for the leaked CSI snapshot, but actual value of error: %v", err) // Make sure snapshot was deleted assert.Equal(t, 0, td.volumeSnapshotter.SnapshotsTaken.Len()) }) t.Run("Expired request will be deleted if the status is processed", func(t *testing.T) { expired := time.Date(2018, 4, 3, 12, 0, 0, 0, time.UTC) input := defaultTestDbr() input.CreationTimestamp = metav1.Time{ Time: expired, } input.Status.Phase = velerov1api.DeleteBackupRequestPhaseProcessed td := setupBackupDeletionControllerTest(t, input) td.backupStore.On("DeleteBackup", mock.Anything).Return(nil) _, err := td.controller.Reconcile(t.Context(), td.req) require.NoError(t, err) res := &velerov1api.DeleteBackupRequest{} err = td.fakeClient.Get(ctx, td.req.NamespacedName, res) assert.True(t, apierrors.IsNotFound(err), "Expected not found error, but actual value of error: %v", err) td.backupStore.AssertNotCalled(t, "DeleteBackup", mock.Anything) }) t.Run("Expired request will not be deleted if the status is not processed", func(t *testing.T) { expired := time.Date(2018, 4, 3, 12, 0, 0, 0, time.UTC) input := defaultTestDbr() input.CreationTimestamp = metav1.Time{ Time: expired, } input.Status.Phase = velerov1api.DeleteBackupRequestPhaseNew td := setupBackupDeletionControllerTest(t, input) td.backupStore.On("DeleteBackup", mock.Anything).Return(nil) _, err := td.controller.Reconcile(t.Context(), td.req) require.NoError(t, err) res := &velerov1api.DeleteBackupRequest{} err = td.fakeClient.Get(ctx, td.req.NamespacedName, res) require.NoError(t, err) assert.Equal(t, "Processed", string(res.Status.Phase)) assert.Len(t, res.Status.Errors, 1) assert.Equal(t, "backup not found", res.Status.Errors[0]) }) } func TestGetSnapshotsInBackup(t *testing.T) { tests := []struct { name string podVolumeBackups []velerov1api.PodVolumeBackup expected map[string][]repotypes.SnapshotIdentifier longBackupNameEnabled bool }{ { name: "no pod volume backups", podVolumeBackups: nil, expected: map[string][]repotypes.SnapshotIdentifier{}, }, { name: "no pod volume backups with matching label", podVolumeBackups: []velerov1api.PodVolumeBackup{ { ObjectMeta: metav1.ObjectMeta{Name: "foo", Labels: map[string]string{velerov1api.BackupNameLabel: "non-matching-backup-1"}}, Spec: velerov1api.PodVolumeBackupSpec{ Pod: corev1api.ObjectReference{Name: "pod-1", Namespace: "ns-1"}, }, Status: velerov1api.PodVolumeBackupStatus{SnapshotID: "snap-1"}, }, { ObjectMeta: metav1.ObjectMeta{Name: "bar", Labels: map[string]string{velerov1api.BackupNameLabel: "non-matching-backup-2"}}, Spec: velerov1api.PodVolumeBackupSpec{ Pod: corev1api.ObjectReference{Name: "pod-2", Namespace: "ns-2"}, }, Status: velerov1api.PodVolumeBackupStatus{SnapshotID: "snap-2"}, }, }, expected: map[string][]repotypes.SnapshotIdentifier{}, }, { name: "some pod volume backups with matching label", podVolumeBackups: []velerov1api.PodVolumeBackup{ { ObjectMeta: metav1.ObjectMeta{Name: "foo", Labels: map[string]string{velerov1api.BackupNameLabel: "non-matching-backup-1"}}, Spec: velerov1api.PodVolumeBackupSpec{ Pod: corev1api.ObjectReference{Name: "pod-1", Namespace: "ns-1"}, }, Status: velerov1api.PodVolumeBackupStatus{SnapshotID: "snap-1"}, }, { ObjectMeta: metav1.ObjectMeta{Name: "bar", Labels: map[string]string{velerov1api.BackupNameLabel: "non-matching-backup-2"}}, Spec: velerov1api.PodVolumeBackupSpec{ Pod: corev1api.ObjectReference{Name: "pod-2", Namespace: "ns-2"}, }, Status: velerov1api.PodVolumeBackupStatus{SnapshotID: "snap-2"}, }, { ObjectMeta: metav1.ObjectMeta{Name: "completed-pvb", Labels: map[string]string{velerov1api.BackupNameLabel: "backup-1"}}, Spec: velerov1api.PodVolumeBackupSpec{ Pod: corev1api.ObjectReference{Name: "pod-1", Namespace: "ns-1"}, }, Status: velerov1api.PodVolumeBackupStatus{SnapshotID: "snap-3"}, }, { ObjectMeta: metav1.ObjectMeta{Name: "completed-pvb-2", Labels: map[string]string{velerov1api.BackupNameLabel: "backup-1"}}, Spec: velerov1api.PodVolumeBackupSpec{ Pod: corev1api.ObjectReference{Name: "pod-1", Namespace: "ns-1"}, }, Status: velerov1api.PodVolumeBackupStatus{SnapshotID: "snap-4"}, }, { ObjectMeta: metav1.ObjectMeta{Name: "incomplete-or-failed-pvb", Labels: map[string]string{velerov1api.BackupNameLabel: "backup-1"}}, Spec: velerov1api.PodVolumeBackupSpec{ Pod: corev1api.ObjectReference{Name: "pod-1", Namespace: "ns-2"}, }, Status: velerov1api.PodVolumeBackupStatus{SnapshotID: ""}, }, }, expected: map[string][]repotypes.SnapshotIdentifier{ "ns-1": { { VolumeNamespace: "ns-1", SnapshotID: "snap-3", RepositoryType: "restic", }, { VolumeNamespace: "ns-1", SnapshotID: "snap-4", RepositoryType: "restic", }, }, }, }, { name: "some pod volume backups with matching label and backup name greater than 63 chars", longBackupNameEnabled: true, podVolumeBackups: []velerov1api.PodVolumeBackup{ { ObjectMeta: metav1.ObjectMeta{Name: "foo", Labels: map[string]string{velerov1api.BackupNameLabel: "non-matching-backup-1"}}, Spec: velerov1api.PodVolumeBackupSpec{ Pod: corev1api.ObjectReference{Name: "pod-1", Namespace: "ns-1"}, }, Status: velerov1api.PodVolumeBackupStatus{SnapshotID: "snap-1"}, }, { ObjectMeta: metav1.ObjectMeta{Name: "bar", Labels: map[string]string{velerov1api.BackupNameLabel: "non-matching-backup-2"}}, Spec: velerov1api.PodVolumeBackupSpec{ Pod: corev1api.ObjectReference{Name: "pod-2", Namespace: "ns-2"}, }, Status: velerov1api.PodVolumeBackupStatus{SnapshotID: "snap-2"}, }, { ObjectMeta: metav1.ObjectMeta{Name: "completed-pvb", Labels: map[string]string{velerov1api.BackupNameLabel: "the-really-long-backup-name-that-is-much-more-than-63-cha6ca4bc"}}, Spec: velerov1api.PodVolumeBackupSpec{ Pod: corev1api.ObjectReference{Name: "pod-1", Namespace: "ns-1"}, }, Status: velerov1api.PodVolumeBackupStatus{SnapshotID: "snap-3"}, }, { ObjectMeta: metav1.ObjectMeta{Name: "completed-pvb-2", Labels: map[string]string{velerov1api.BackupNameLabel: "backup-1"}}, Spec: velerov1api.PodVolumeBackupSpec{ Pod: corev1api.ObjectReference{Name: "pod-1", Namespace: "ns-1"}, }, Status: velerov1api.PodVolumeBackupStatus{SnapshotID: "snap-4"}, }, { ObjectMeta: metav1.ObjectMeta{Name: "incomplete-or-failed-pvb", Labels: map[string]string{velerov1api.BackupNameLabel: "backup-1"}}, Spec: velerov1api.PodVolumeBackupSpec{ Pod: corev1api.ObjectReference{Name: "pod-1", Namespace: "ns-2"}, }, Status: velerov1api.PodVolumeBackupStatus{SnapshotID: ""}, }, }, expected: map[string][]repotypes.SnapshotIdentifier{ "ns-1": { { VolumeNamespace: "ns-1", SnapshotID: "snap-3", RepositoryType: "restic", }, }, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { var ( clientBuilder = velerotest.NewFakeControllerRuntimeClientBuilder(t) veleroBackup = &velerov1api.Backup{} ) veleroBackup.Name = "backup-1" if test.longBackupNameEnabled { veleroBackup.Name = "the-really-long-backup-name-that-is-much-more-than-63-characters" } clientBuilder.WithLists(&velerov1api.PodVolumeBackupList{ Items: test.podVolumeBackups, }) res, err := getSnapshotsInBackup(t.Context(), veleroBackup, clientBuilder.Build()) require.NoError(t, err) assert.True(t, reflect.DeepEqual(res, test.expected)) }) } } func batchDeleteSucceed(ctx context.Context, repoEnsurer *repository.Ensurer, repoMgr repomanager.Manager, directSnapshots map[string][]repotypes.SnapshotIdentifier, backup *velerov1api.Backup, logger logrus.FieldLogger) []error { return nil } func batchDeleteFail(ctx context.Context, repoEnsurer *repository.Ensurer, repoMgr repomanager.Manager, directSnapshots map[string][]repotypes.SnapshotIdentifier, backup *velerov1api.Backup, logger logrus.FieldLogger) []error { return []error{ errors.New("fake-delete-1"), errors.New("fake-delete-2"), } } func generateSnapshotData(snapshot *repotypes.SnapshotIdentifier) (map[string]string, error) { if snapshot == nil { return nil, nil } b, err := json.Marshal(snapshot) if err != nil { return nil, err } data := make(map[string]string) if err := json.Unmarshal(b, &data); err != nil { return nil, err } return data, nil } func TestDeleteMovedSnapshots(t *testing.T) { tests := []struct { name string repoMgr repomanager.Manager batchDeleteSucceed bool backupName string snapshots []*repotypes.SnapshotIdentifier expected []string }{ { name: "repoMgr is nil", }, { name: "no cm", repoMgr: repomocks.NewManager(t), }, { name: "bad cm info", repoMgr: repomocks.NewManager(t), backupName: "backup-01", snapshots: []*repotypes.SnapshotIdentifier{nil}, expected: []string{"no snapshot info in config"}, }, { name: "invalid snapshots", repoMgr: repomocks.NewManager(t), backupName: "backup-01", snapshots: []*repotypes.SnapshotIdentifier{ { RepositoryType: "repo-1", VolumeNamespace: "ns-1", }, { SnapshotID: "snapshot-1", VolumeNamespace: "ns-1", }, { SnapshotID: "snapshot-1", RepositoryType: "repo-1", }, }, batchDeleteSucceed: true, expected: []string{ "invalid snapshot, ID , namespace ns-1, repository repo-1", "invalid snapshot, ID snapshot-1, namespace ns-1, repository ", "invalid snapshot, ID snapshot-1, namespace , repository repo-1", }, }, { name: "batch delete succeed", repoMgr: repomocks.NewManager(t), backupName: "backup-01", snapshots: []*repotypes.SnapshotIdentifier{ { SnapshotID: "snapshot-1", RepositoryType: "repo-1", VolumeNamespace: "ns-1", }, }, batchDeleteSucceed: true, expected: []string{}, }, { name: "batch delete fail", repoMgr: repomocks.NewManager(t), backupName: "backup-01", snapshots: []*repotypes.SnapshotIdentifier{ { RepositoryType: "repo-1", VolumeNamespace: "ns-1", }, { SnapshotID: "snapshot-1", RepositoryType: "repo-1", VolumeNamespace: "ns-1", }, }, expected: []string{"invalid snapshot, ID , namespace ns-1, repository repo-1", "fake-delete-1", "fake-delete-2"}, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { objs := []runtime.Object{} for i, snapshot := range test.snapshots { snapshotData, err := generateSnapshotData(snapshot) require.NoError(t, err) cm := corev1api.ConfigMap{ TypeMeta: metav1.TypeMeta{ APIVersion: corev1api.SchemeGroupVersion.String(), Kind: "ConfigMap", }, ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", Name: fmt.Sprintf("du-info-%d", i), Labels: map[string]string{ velerov1api.BackupNameLabel: test.backupName, velerov1api.DataUploadSnapshotInfoLabel: "true", }, }, Data: snapshotData, } objs = append(objs, &cm) } veleroBackup := &velerov1api.Backup{} controller := NewBackupDeletionReconciler( velerotest.NewLogger(), velerotest.NewFakeControllerRuntimeClient(t, objs...), NewBackupTracker(), test.repoMgr, metrics.NewServerMetrics(), nil, // discovery helper func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager }, NewFakeSingleObjectBackupStoreGetter(backupStore), velerotest.NewFakeCredentialsFileStore("", nil), nil, ) veleroBackup.Name = test.backupName if test.batchDeleteSucceed { batchDeleteSnapshotFunc = batchDeleteSucceed } else { batchDeleteSnapshotFunc = batchDeleteFail } errs := controller.deleteMovedSnapshots(t.Context(), veleroBackup) if test.expected == nil { assert.Nil(t, errs) } else { assert.Len(t, errs, len(test.expected)) for i := range test.expected { assert.EqualError(t, errs[i], test.expected[i]) } } }) } } ================================================ FILE: pkg/controller/backup_finalizer_controller.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package controller import ( "bytes" "context" "os" "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clocks "k8s.io/utils/clock" ctrl "sigs.k8s.io/controller-runtime" kbclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" pkgbackup "github.com/vmware-tanzu/velero/pkg/backup" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/constant" "github.com/vmware-tanzu/velero/pkg/itemoperation" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/metrics" "github.com/vmware-tanzu/velero/pkg/persistence" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" "github.com/vmware-tanzu/velero/pkg/plugin/framework" "github.com/vmware-tanzu/velero/pkg/util/encode" ) // backupFinalizerReconciler reconciles a Backup object type backupFinalizerReconciler struct { client kbclient.Client globalCRClient kbclient.Client clock clocks.WithTickerAndDelayedExecution backupper pkgbackup.Backupper newPluginManager func(logrus.FieldLogger) clientmgmt.Manager backupTracker BackupTracker metrics *metrics.ServerMetrics backupStoreGetter persistence.ObjectBackupStoreGetter log logrus.FieldLogger resourceTimeout time.Duration } // NewBackupFinalizerReconciler initializes and returns backupFinalizerReconciler struct. func NewBackupFinalizerReconciler( client kbclient.Client, globalCRClient kbclient.Client, clock clocks.WithTickerAndDelayedExecution, backupper pkgbackup.Backupper, newPluginManager func(logrus.FieldLogger) clientmgmt.Manager, backupTracker BackupTracker, backupStoreGetter persistence.ObjectBackupStoreGetter, log logrus.FieldLogger, metrics *metrics.ServerMetrics, resourceTimeout time.Duration, ) *backupFinalizerReconciler { return &backupFinalizerReconciler{ client: client, globalCRClient: globalCRClient, clock: clock, backupper: backupper, newPluginManager: newPluginManager, backupTracker: backupTracker, backupStoreGetter: backupStoreGetter, log: log, metrics: metrics, } } // +kubebuilder:rbac:groups=velero.io,resources=backups,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=velero.io,resources=backups/status,verbs=get;update;patch func (r *backupFinalizerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := r.log.WithFields(logrus.Fields{ "controller": "backup-finalizer", "backup": req.NamespacedName, }) // Fetch the Backup instance. log.Debug("Getting Backup") backup := &velerov1api.Backup{} if err := r.client.Get(ctx, req.NamespacedName, backup); err != nil { if apierrors.IsNotFound(err) { log.Debug("Unable to find Backup") return ctrl.Result{}, nil } log.WithError(err).Error("Error getting Backup") return ctrl.Result{}, errors.WithStack(err) } switch backup.Status.Phase { case velerov1api.BackupPhaseFinalizing, velerov1api.BackupPhaseFinalizingPartiallyFailed: // only process backups finalizing after plugin operations are complete default: log.Debug("Backup is not awaiting finalizing, skipping") return ctrl.Result{}, nil } original := backup.DeepCopy() defer func() { switch backup.Status.Phase { case velerov1api.BackupPhaseCompleted, velerov1api.BackupPhasePartiallyFailed, velerov1api.BackupPhaseFailed, velerov1api.BackupPhaseFailedValidation: r.backupTracker.Delete(backup.Namespace, backup.Name) } // Always attempt to Patch the backup object and status after each reconciliation. // // if this patch fails, there may not be another opportunity to update the backup object without external update event. // so we retry // This retries updating Finalzing/FinalizingPartiallyFailed to Completed/PartiallyFailed if err := client.RetryOnErrorMaxBackOff(r.resourceTimeout, func() error { return r.client.Patch(ctx, backup, kbclient.MergeFrom(original)) }); err != nil { log.WithError(err).Error("Error updating backup") return } }() location := &velerov1api.BackupStorageLocation{} if err := r.client.Get(ctx, kbclient.ObjectKey{ Namespace: backup.Namespace, Name: backup.Spec.StorageLocation, }, location); err != nil { return ctrl.Result{}, errors.WithStack(err) } pluginManager := r.newPluginManager(log) defer pluginManager.CleanupClients() backupStore, err := r.backupStoreGetter.Get(location, pluginManager, log) if err != nil { log.WithError(err).Error("Error getting a backup store") return ctrl.Result{}, errors.WithStack(err) } // Download item operations list and backup contents operations, err := backupStore.GetBackupItemOperations(backup.Name) if err != nil { log.WithError(err).Error("Error getting backup item operations") return ctrl.Result{}, errors.WithStack(err) } backupRequest := &pkgbackup.Request{ Backup: backup, StorageLocation: location, SkippedPVTracker: pkgbackup.NewSkipPVTracker(), BackedUpItems: pkgbackup.NewBackedUpItemsMap(), } var outBackupFile *os.File if len(operations) > 0 { log.Info("Setting up finalized backup temp file") inBackupFile, err := downloadToTempFile(backup.Name, backupStore, log) if err != nil { return ctrl.Result{}, errors.Wrap(err, "error downloading backup") } defer closeAndRemoveFile(inBackupFile, log) outBackupFile, err = os.CreateTemp("", "") if err != nil { log.WithError(err).Error("error creating temp file for backup") return ctrl.Result{}, errors.WithStack(err) } defer closeAndRemoveFile(outBackupFile, log) log.Info("Getting backup item actions") actions, err := pluginManager.GetBackupItemActionsV2() if err != nil { log.WithError(err).Error("error getting Backup Item Actions") return ctrl.Result{}, errors.WithStack(err) } backupItemActionsResolver := framework.NewBackupItemActionResolverV2(actions) // Call itemBackupper.BackupItem for the list of items updated by async operations err = r.backupper.FinalizeBackup( log, backupRequest, inBackupFile, outBackupFile, backupItemActionsResolver, operations, backupStore, ) if err != nil { log.WithError(err).Error("error finalizing Backup") return ctrl.Result{}, errors.WithStack(err) } } backupScheduleName := backupRequest.GetLabels()[velerov1api.ScheduleNameLabel] switch backup.Status.Phase { case velerov1api.BackupPhaseFinalizing: backup.Status.Phase = velerov1api.BackupPhaseCompleted r.metrics.RegisterBackupSuccess(backupScheduleName) r.metrics.RegisterBackupLastStatus(backupScheduleName, metrics.BackupLastStatusSucc) case velerov1api.BackupPhaseFinalizingPartiallyFailed: backup.Status.Phase = velerov1api.BackupPhasePartiallyFailed r.metrics.RegisterBackupPartialFailure(backupScheduleName) r.metrics.RegisterBackupLastStatus(backupScheduleName, metrics.BackupLastStatusFailure) } backup.Status.CompletionTimestamp = &metav1.Time{Time: r.clock.Now()} backup.Status.CSIVolumeSnapshotsCompleted = updateCSIVolumeSnapshotsCompleted(operations) recordBackupMetrics(log, backup, outBackupFile, r.metrics, true) // update backup metadata in object store backupJSON := new(bytes.Buffer) if err := encode.To(backup, "json", backupJSON); err != nil { return ctrl.Result{}, errors.Wrap(err, "error encoding backup json") } err = backupStore.PutBackupMetadata(backup.Name, backupJSON) if err != nil { return ctrl.Result{}, errors.Wrap(err, "error uploading backup json") } if len(operations) > 0 { err = backupStore.PutBackupContents(backup.Name, outBackupFile) if err != nil { return ctrl.Result{}, errors.Wrap(err, "error uploading backup final contents") } } return ctrl.Result{}, nil } func (r *backupFinalizerReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&velerov1api.Backup{}). Named(constant.ControllerBackupFinalizer). Complete(r) } // updateCSIVolumeSnapshotsCompleted calculate the completed VS number according to // the backup's async operation list. func updateCSIVolumeSnapshotsCompleted( operations []*itemoperation.BackupOperation) int { completedNum := 0 for index := range operations { if operations[index].Spec.ResourceIdentifier.String() == kuberesource.VolumeSnapshots.String() && operations[index].Status.Phase == itemoperation.OperationPhaseCompleted { completedNum++ } } return completedNum } ================================================ FILE: pkg/controller/backup_finalizer_controller_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package controller import ( "bytes" "io" "testing" "time" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" testclocks "k8s.io/utils/clock/testing" ctrl "sigs.k8s.io/controller-runtime" kbclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/features" "github.com/vmware-tanzu/velero/pkg/itemoperation" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/metrics" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" "github.com/vmware-tanzu/velero/pkg/plugin/framework" "github.com/vmware-tanzu/velero/pkg/plugin/velero" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func mockBackupFinalizerReconciler(fakeClient kbclient.Client, fakeGlobalClient kbclient.Client, fakeClock *testclocks.FakeClock) (*backupFinalizerReconciler, *fakeBackupper) { backupper := new(fakeBackupper) return NewBackupFinalizerReconciler( fakeClient, fakeGlobalClient, fakeClock, backupper, func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager }, NewBackupTracker(), NewFakeSingleObjectBackupStoreGetter(backupStore), logrus.StandardLogger(), metrics.NewServerMetrics(), 10*time.Minute, ), backupper } func TestBackupFinalizerReconcile(t *testing.T) { fakeClock := testclocks.NewFakeClock(time.Now()) metav1Now := metav1.NewTime(fakeClock.Now()) defaultBackupLocation := builder.ForBackupStorageLocation(velerov1api.DefaultNamespace, "default").Result() tests := []struct { name string backup *velerov1api.Backup backupOperations []*itemoperation.BackupOperation backupLocation *velerov1api.BackupStorageLocation enableCSI bool expectError bool expectPhase velerov1api.BackupPhase expectedCompletedVS int }{ { name: "Finalizing backup is completed", backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-1"). StorageLocation("default"). ObjectMeta(builder.WithUID("foo")). StartTimestamp(fakeClock.Now()). Phase(velerov1api.BackupPhaseFinalizing).Result(), backupLocation: defaultBackupLocation, expectPhase: velerov1api.BackupPhaseCompleted, backupOperations: []*itemoperation.BackupOperation{ { Spec: itemoperation.BackupOperationSpec{ BackupName: "backup-1", BackupUID: "foo", BackupItemAction: "foo", ResourceIdentifier: velero.ResourceIdentifier{ GroupResource: kuberesource.Pods, Namespace: "ns-1", Name: "pod-1", }, PostOperationItems: []velero.ResourceIdentifier{ { GroupResource: kuberesource.Secrets, Namespace: "ns-1", Name: "secret-1", }, }, OperationID: "operation-1", }, Status: itemoperation.OperationStatus{ Phase: itemoperation.OperationPhaseCompleted, Created: &metav1Now, }, }, }, }, { name: "FinalizingPartiallyFailed backup is partially failed", backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-2"). StorageLocation("default"). ObjectMeta(builder.WithUID("foo")). StartTimestamp(fakeClock.Now()). Phase(velerov1api.BackupPhaseFinalizingPartiallyFailed).Result(), backupLocation: defaultBackupLocation, expectPhase: velerov1api.BackupPhasePartiallyFailed, backupOperations: []*itemoperation.BackupOperation{ { Spec: itemoperation.BackupOperationSpec{ BackupName: "backup-2", BackupUID: "foo", BackupItemAction: "foo", ResourceIdentifier: velero.ResourceIdentifier{ GroupResource: kuberesource.Pods, Namespace: "ns-2", Name: "pod-2", }, PostOperationItems: []velero.ResourceIdentifier{ { GroupResource: kuberesource.Secrets, Namespace: "ns-2", Name: "secret-2", }, }, OperationID: "operation-2", }, Status: itemoperation.OperationStatus{ Phase: itemoperation.OperationPhaseCompleted, Created: &metav1Now, }, }, }, }, { name: "Test calculate backup.Status.BackupItemOperationsCompleted", backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-3"). StorageLocation("default"). ObjectMeta(builder.WithUID("foo")). StartTimestamp(fakeClock.Now()). WithStatus(velerov1api.BackupStatus{ StartTimestamp: &metav1Now, CompletionTimestamp: &metav1Now, CSIVolumeSnapshotsAttempted: 1, Phase: velerov1api.BackupPhaseFinalizing, }). Result(), backupLocation: defaultBackupLocation, enableCSI: true, expectPhase: velerov1api.BackupPhaseCompleted, expectedCompletedVS: 1, backupOperations: []*itemoperation.BackupOperation{ { Spec: itemoperation.BackupOperationSpec{ BackupName: "backup-3", BackupUID: "foo", BackupItemAction: "foo", ResourceIdentifier: velero.ResourceIdentifier{ GroupResource: kuberesource.VolumeSnapshots, Namespace: "ns-1", Name: "vs-1", }, PostOperationItems: []velero.ResourceIdentifier{ { GroupResource: kuberesource.Secrets, Namespace: "ns-1", Name: "secret-1", }, }, OperationID: "operation-3", }, Status: itemoperation.OperationStatus{ Phase: itemoperation.OperationPhaseCompleted, Created: &metav1Now, }, }, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { if test.backup == nil { return } initObjs := []runtime.Object{} initObjs = append(initObjs, test.backup) if test.backupLocation != nil { initObjs = append(initObjs, test.backupLocation) } if test.enableCSI { features.Enable(velerov1api.CSIFeatureFlag) defer features.Enable() } fakeClient := velerotest.NewFakeControllerRuntimeClient(t, initObjs...) fakeGlobalClient := velerotest.NewFakeControllerRuntimeClient(t, initObjs...) reconciler, backupper := mockBackupFinalizerReconciler(fakeClient, fakeGlobalClient, fakeClock) pluginManager.On("CleanupClients").Return(nil) backupStore.On("GetBackupItemOperations", test.backup.Name).Return(test.backupOperations, nil) backupStore.On("GetBackupContents", mock.Anything).Return(io.NopCloser(bytes.NewReader([]byte("hello world"))), nil) backupStore.On("PutBackupContents", mock.Anything, mock.Anything).Return(nil) backupStore.On("PutBackupMetadata", mock.Anything, mock.Anything).Return(nil) backupStore.On("GetBackupVolumeInfos", mock.Anything).Return(nil, nil) backupStore.On("PutBackupVolumeInfos", mock.Anything, mock.Anything).Return(nil) pluginManager.On("GetBackupItemActionsV2").Return(nil, nil) backupper.On("FinalizeBackup", mock.Anything, mock.Anything, mock.Anything, mock.Anything, framework.BackupItemActionResolverV2{}, mock.Anything, mock.Anything).Return(nil) _, err := reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: test.backup.Namespace, Name: test.backup.Name}}) gotErr := err != nil assert.Equal(t, test.expectError, gotErr) backupAfter := velerov1api.Backup{} err = fakeClient.Get(t.Context(), types.NamespacedName{ Namespace: test.backup.Namespace, Name: test.backup.Name, }, &backupAfter) require.NoError(t, err) assert.Equal(t, test.expectPhase, backupAfter.Status.Phase) assert.Equal(t, test.expectedCompletedVS, backupAfter.Status.CSIVolumeSnapshotsCompleted) }) } } ================================================ FILE: pkg/controller/backup_operations_controller.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package controller import ( "bytes" "context" "fmt" "time" v2 "github.com/vmware-tanzu/velero/pkg/plugin/velero/backupitemaction/v2" "github.com/pkg/errors" "github.com/sirupsen/logrus" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clocks "k8s.io/utils/clock" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/predicate" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/constant" "github.com/vmware-tanzu/velero/pkg/itemoperation" "github.com/vmware-tanzu/velero/pkg/itemoperationmap" "github.com/vmware-tanzu/velero/pkg/metrics" "github.com/vmware-tanzu/velero/pkg/persistence" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" "github.com/vmware-tanzu/velero/pkg/util/encode" "github.com/vmware-tanzu/velero/pkg/util/kube" ) const ( defaultBackupOperationsFrequency = 10 * time.Second ) type backupOperationsReconciler struct { client.Client logger logrus.FieldLogger clock clocks.WithTickerAndDelayedExecution frequency time.Duration itemOperationsMap *itemoperationmap.BackupItemOperationsMap newPluginManager func(logger logrus.FieldLogger) clientmgmt.Manager backupStoreGetter persistence.ObjectBackupStoreGetter metrics *metrics.ServerMetrics } func NewBackupOperationsReconciler( logger logrus.FieldLogger, client client.Client, frequency time.Duration, newPluginManager func(logrus.FieldLogger) clientmgmt.Manager, backupStoreGetter persistence.ObjectBackupStoreGetter, metrics *metrics.ServerMetrics, itemOperationsMap *itemoperationmap.BackupItemOperationsMap, ) *backupOperationsReconciler { abor := &backupOperationsReconciler{ Client: client, logger: logger, clock: clocks.RealClock{}, frequency: frequency, itemOperationsMap: itemOperationsMap, newPluginManager: newPluginManager, backupStoreGetter: backupStoreGetter, metrics: metrics, } if abor.frequency <= 0 { abor.frequency = defaultBackupOperationsFrequency } return abor } func (c *backupOperationsReconciler) SetupWithManager(mgr ctrl.Manager) error { gp := kube.NewGenericEventPredicate(func(object client.Object) bool { backup := object.(*velerov1api.Backup) return (backup.Status.Phase == velerov1api.BackupPhaseWaitingForPluginOperations || backup.Status.Phase == velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed) }) s := kube.NewPeriodicalEnqueueSource(c.logger.WithField("controller", constant.ControllerBackupOperations), mgr.GetClient(), &velerov1api.BackupList{}, c.frequency, kube.PeriodicalEnqueueSourceOption{ Predicates: []predicate.Predicate{gp}, }) return ctrl.NewControllerManagedBy(mgr). For(&velerov1api.Backup{}, builder.WithPredicates(kube.FalsePredicate{})). WatchesRawSource(s). Named(constant.ControllerBackupOperations). Complete(c) } // +kubebuilder:rbac:groups=velero.io,resources=backups,verbs=get;list;watch;update // +kubebuilder:rbac:groups=velero.io,resources=backups/status,verbs=get // +kubebuilder:rbac:groups=velero.io,resources=backupstoragelocations,verbs=get func (c *backupOperationsReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := c.logger.WithField("backup operations for backup", req.String()) log.Debug("backupOperationsReconciler getting backup") original := &velerov1api.Backup{} if err := c.Get(ctx, req.NamespacedName, original); err != nil { if apierrors.IsNotFound(err) { log.WithError(err).Error("backup not found") return ctrl.Result{}, nil } return ctrl.Result{}, errors.Wrapf(err, "error getting backup %s", req.String()) } backup := original.DeepCopy() log.Debugf("backup: %s", backup.Name) log = c.logger.WithFields( logrus.Fields{ "backup": req.String(), }, ) switch backup.Status.Phase { case velerov1api.BackupPhaseWaitingForPluginOperations, velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed: // only process backups waiting for plugin operations to complete default: log.Debug("Backup has no ongoing plugin operations, skipping") return ctrl.Result{}, nil } loc := &velerov1api.BackupStorageLocation{} if err := c.Get(ctx, client.ObjectKey{ Namespace: req.Namespace, Name: backup.Spec.StorageLocation, }, loc); err != nil { if apierrors.IsNotFound(err) { log.Warnf("Cannot check progress on Backup operations because backup storage location %s does not exist; marking backup PartiallyFailed", backup.Spec.StorageLocation) backup.Status.Phase = velerov1api.BackupPhasePartiallyFailed } else { log.Warnf("Cannot check progress on Backup operations because backup storage location %s could not be retrieved: %s; marking backup PartiallyFailed", backup.Spec.StorageLocation, err.Error()) backup.Status.Phase = velerov1api.BackupPhasePartiallyFailed } err2 := c.updateBackupAndOperationsJSON(ctx, original, backup, nil, &itemoperationmap.OperationsForBackup{ErrsSinceUpdate: []string{err.Error()}}, false, false) if err2 != nil { log.WithError(err2).Error("error updating Backup") } return ctrl.Result{}, errors.Wrap(err, "error getting backup storage location") } if loc.Spec.AccessMode == velerov1api.BackupStorageLocationAccessModeReadOnly { log.Infof("Cannot check progress on Backup operations because backup storage location %s is currently in read-only mode; marking backup PartiallyFailed", loc.Name) backup.Status.Phase = velerov1api.BackupPhasePartiallyFailed err := c.updateBackupAndOperationsJSON(ctx, original, backup, nil, &itemoperationmap.OperationsForBackup{ErrsSinceUpdate: []string{"BSL is read-only"}}, false, false) if err != nil { log.WithError(err).Error("error updating Backup") } return ctrl.Result{}, nil } pluginManager := c.newPluginManager(c.logger) defer pluginManager.CleanupClients() backupStore, err := c.backupStoreGetter.Get(loc, pluginManager, c.logger) if err != nil { return ctrl.Result{}, errors.Wrap(err, "error getting backup store") } operations, err := c.itemOperationsMap.GetOperationsForBackup(backupStore, backup.Name) if err != nil { err2 := c.updateBackupAndOperationsJSON(ctx, original, backup, backupStore, &itemoperationmap.OperationsForBackup{ErrsSinceUpdate: []string{err.Error()}}, false, false) if err2 != nil { return ctrl.Result{}, errors.Wrap(err2, "error updating Backup") } return ctrl.Result{}, errors.Wrap(err, "error getting backup operations") } stillInProgress, changes, opsCompleted, opsFailed, errs := getBackupItemOperationProgress(backup, pluginManager, operations.Operations) // if len(errs)>0, need to update backup errors and error log operations.ErrsSinceUpdate = append(operations.ErrsSinceUpdate, errs...) backup.Status.Errors += len(operations.ErrsSinceUpdate) completionChanges := false if backup.Status.BackupItemOperationsCompleted != opsCompleted || backup.Status.BackupItemOperationsFailed != opsFailed { completionChanges = true backup.Status.BackupItemOperationsCompleted = opsCompleted backup.Status.BackupItemOperationsFailed = opsFailed } if changes { operations.ChangesSinceUpdate = true } if len(operations.ErrsSinceUpdate) > 0 { backup.Status.Phase = velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed } // if stillInProgress is false, backup moves to finalize phase and needs update // if operations.ErrsSinceUpdate is not empty, then backup phase needs to change to // BackupPhaseWaitingForPluginOperationsPartiallyFailed and needs update // If the only changes are incremental progress, then no write is necessary, progress can remain in memory if !stillInProgress { if backup.Status.Phase == velerov1api.BackupPhaseWaitingForPluginOperations { log.Infof("Marking backup %s Finalizing", backup.Name) backup.Status.Phase = velerov1api.BackupPhaseFinalizing } else { log.Infof("Marking backup %s FinalizingPartiallyFailed", backup.Name) backup.Status.Phase = velerov1api.BackupPhaseFinalizingPartiallyFailed } } err = c.updateBackupAndOperationsJSON(ctx, original, backup, backupStore, operations, changes, completionChanges) if err != nil { return ctrl.Result{}, errors.Wrap(err, "error updating Backup") } return ctrl.Result{}, nil } func (c *backupOperationsReconciler) updateBackupAndOperationsJSON( ctx context.Context, original, backup *velerov1api.Backup, backupStore persistence.BackupStore, operations *itemoperationmap.OperationsForBackup, changes bool, completionChanges bool) error { backupScheduleName := backup.GetLabels()[velerov1api.ScheduleNameLabel] if len(operations.ErrsSinceUpdate) > 0 { c.metrics.RegisterBackupItemsErrorsGauge(backupScheduleName, backup.Status.Errors) // FIXME: download/upload results once https://github.com/vmware-tanzu/velero/pull/5576 is merged } removeIfComplete := true defer func() { // remove local operations list if complete if removeIfComplete && (backup.Status.Phase == velerov1api.BackupPhaseCompleted || backup.Status.Phase == velerov1api.BackupPhasePartiallyFailed || backup.Status.Phase == velerov1api.BackupPhaseFinalizing || backup.Status.Phase == velerov1api.BackupPhaseFinalizingPartiallyFailed) { c.itemOperationsMap.DeleteOperationsForBackup(backup.Name) } else if changes { c.itemOperationsMap.PutOperationsForBackup(operations, backup.Name) } }() // update backup and upload progress if errs or complete if len(operations.ErrsSinceUpdate) > 0 || backup.Status.Phase == velerov1api.BackupPhaseCompleted || backup.Status.Phase == velerov1api.BackupPhasePartiallyFailed || backup.Status.Phase == velerov1api.BackupPhaseFinalizing || backup.Status.Phase == velerov1api.BackupPhaseFinalizingPartiallyFailed { // update file store if backupStore != nil { backupJSON := new(bytes.Buffer) if err := encode.To(backup, "json", backupJSON); err != nil { removeIfComplete = false return errors.Wrap(err, "error encoding backup json") } err := backupStore.PutBackupMetadata(backup.Name, backupJSON) if err != nil { removeIfComplete = false return errors.Wrap(err, "error uploading backup json") } if err := c.itemOperationsMap.UploadProgressAndPutOperationsForBackup(backupStore, operations, backup.Name); err != nil { removeIfComplete = false return err } } // update backup err := c.Client.Patch(ctx, backup, client.MergeFrom(original)) if err != nil { removeIfComplete = false return errors.Wrapf(err, "error updating Backup %s", backup.Name) } } else if completionChanges { // If backup is still incomplete and no new errors are found but there are some new operations // completed, patch backup to reflect new completion numbers, but don't upload detailed json file err := c.Client.Patch(ctx, backup, client.MergeFrom(original)) if err != nil { return errors.Wrapf(err, "error updating Backup %s", backup.Name) } } return nil } // check progress of backupItemOperations // return: inProgressOperations, changes, completedCount, failedCount, errs func getBackupItemOperationProgress( backup *velerov1api.Backup, pluginManager clientmgmt.Manager, operationsList []*itemoperation.BackupOperation) (bool, bool, int, int, []string) { inProgressOperations := false changes := false var errs []string var completedCount, failedCount int for _, operation := range operationsList { if operation.Status.Phase == itemoperation.OperationPhaseNew || operation.Status.Phase == itemoperation.OperationPhaseInProgress { bia, err := pluginManager.GetBackupItemActionV2(operation.Spec.BackupItemAction) if err != nil { operation.Status.Phase = itemoperation.OperationPhaseFailed operation.Status.Error = err.Error() errs = append(errs, wrapErrMsg(err.Error(), bia)) changes = true failedCount++ continue } operationProgress, err := bia.Progress(operation.Spec.OperationID, backup) if err != nil { operation.Status.Phase = itemoperation.OperationPhaseFailed operation.Status.Error = err.Error() errs = append(errs, wrapErrMsg(err.Error(), bia)) changes = true failedCount++ continue } if operation.Status.NCompleted != operationProgress.NCompleted { operation.Status.NCompleted = operationProgress.NCompleted changes = true } if operation.Status.NTotal != operationProgress.NTotal { operation.Status.NTotal = operationProgress.NTotal changes = true } if operation.Status.OperationUnits != operationProgress.OperationUnits { operation.Status.OperationUnits = operationProgress.OperationUnits changes = true } if operation.Status.Description != operationProgress.Description { operation.Status.Description = operationProgress.Description changes = true } started := metav1.NewTime(operationProgress.Started) if operation.Status.Started == nil && !operationProgress.Started.IsZero() || operation.Status.Started != nil && *(operation.Status.Started) != started { operation.Status.Started = &started changes = true } updated := metav1.NewTime(operationProgress.Updated) if operation.Status.Updated == nil && !operationProgress.Updated.IsZero() || operation.Status.Updated != nil && *(operation.Status.Updated) != updated { operation.Status.Updated = &updated changes = true } if operationProgress.Completed { if operationProgress.Err != "" { operation.Status.Phase = itemoperation.OperationPhaseFailed operation.Status.Error = operationProgress.Err errs = append(errs, wrapErrMsg(operationProgress.Err, bia)) changes = true failedCount++ continue } operation.Status.Phase = itemoperation.OperationPhaseCompleted changes = true completedCount++ continue } // cancel operation if past timeout period if operation.Status.Created.Time.Add(backup.Spec.ItemOperationTimeout.Duration).Before(time.Now()) { _ = bia.Cancel(operation.Spec.OperationID, backup) operation.Status.Phase = itemoperation.OperationPhaseFailed operation.Status.Error = "Asynchronous action timed out" errs = append(errs, wrapErrMsg(operation.Status.Error, bia)) changes = true failedCount++ continue } if operation.Status.Phase == itemoperation.OperationPhaseNew && operation.Status.Started != nil { operation.Status.Phase = itemoperation.OperationPhaseInProgress changes = true } // if we reach this point, the operation is still running inProgressOperations = true } else if operation.Status.Phase == itemoperation.OperationPhaseCompleted { completedCount++ } else if operation.Status.Phase == itemoperation.OperationPhaseFailed { failedCount++ } } return inProgressOperations, changes, completedCount, failedCount, errs } // wrap the error message to include the BIA name func wrapErrMsg(errMsg string, bia v2.BackupItemAction) string { plugin := "unknown" if bia != nil { plugin = bia.Name() } if len(errMsg) > 0 { errMsg += ", " } return fmt.Sprintf("%splugin: %s", errMsg, plugin) } ================================================ FILE: pkg/controller/backup_operations_controller_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package controller import ( "testing" "time" v2 "github.com/vmware-tanzu/velero/pkg/plugin/velero/backupitemaction/v2" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" testclocks "k8s.io/utils/clock/testing" ctrl "sigs.k8s.io/controller-runtime" kbclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/itemoperation" "github.com/vmware-tanzu/velero/pkg/itemoperationmap" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/metrics" persistencemocks "github.com/vmware-tanzu/velero/pkg/persistence/mocks" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" pluginmocks "github.com/vmware-tanzu/velero/pkg/plugin/mocks" "github.com/vmware-tanzu/velero/pkg/plugin/velero" biav2mocks "github.com/vmware-tanzu/velero/pkg/plugin/velero/mocks/backupitemaction/v2" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) var ( pluginManager = &pluginmocks.Manager{} backupStore = &persistencemocks.BackupStore{} bia = &biav2mocks.BackupItemAction{} ) func mockBackupOperationsReconciler(fakeClient kbclient.Client, fakeClock *testclocks.FakeClock, freq time.Duration) *backupOperationsReconciler { abor := NewBackupOperationsReconciler( logrus.StandardLogger(), fakeClient, freq, func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager }, NewFakeSingleObjectBackupStoreGetter(backupStore), metrics.NewServerMetrics(), itemoperationmap.NewBackupItemOperationsMap(), ) abor.clock = fakeClock return abor } func TestBackupOperationsReconcile(t *testing.T) { fakeClock := testclocks.NewFakeClock(time.Now()) metav1Now := metav1.NewTime(fakeClock.Now()) defaultBackupLocation := builder.ForBackupStorageLocation(velerov1api.DefaultNamespace, "default").Result() tests := []struct { name string backup *velerov1api.Backup backupOperations []*itemoperation.BackupOperation backupLocation *velerov1api.BackupStorageLocation operationComplete bool operationErr string expectError bool expectPhase velerov1api.BackupPhase }{ { name: "WaitingForPluginOperations backup with completed operations is Finalizing", backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-11"). StorageLocation("default"). ItemOperationTimeout(60 * time.Minute). ObjectMeta(builder.WithUID("foo-11")). Phase(velerov1api.BackupPhaseWaitingForPluginOperations).Result(), backupLocation: defaultBackupLocation, operationComplete: true, expectPhase: velerov1api.BackupPhaseFinalizing, backupOperations: []*itemoperation.BackupOperation{ { Spec: itemoperation.BackupOperationSpec{ BackupName: "backup-11", BackupUID: "foo-11", BackupItemAction: "foo-11", ResourceIdentifier: velero.ResourceIdentifier{ GroupResource: kuberesource.Pods, Namespace: "ns-1", Name: "pod-1", }, OperationID: "operation-11", }, Status: itemoperation.OperationStatus{ Phase: itemoperation.OperationPhaseNew, Created: &metav1Now, }, }, }, }, { name: "WaitingForPluginOperations backup with incomplete operations is still incomplete", backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-12"). StorageLocation("default"). ItemOperationTimeout(60 * time.Minute). ObjectMeta(builder.WithUID("foo-12")). Phase(velerov1api.BackupPhaseWaitingForPluginOperations).Result(), backupLocation: defaultBackupLocation, operationComplete: false, expectPhase: velerov1api.BackupPhaseWaitingForPluginOperations, backupOperations: []*itemoperation.BackupOperation{ { Spec: itemoperation.BackupOperationSpec{ BackupName: "backup-12", BackupUID: "foo-12", BackupItemAction: "foo-12", ResourceIdentifier: velero.ResourceIdentifier{ GroupResource: kuberesource.Pods, Namespace: "ns-1", Name: "pod-1", }, OperationID: "operation-12", }, Status: itemoperation.OperationStatus{ Phase: itemoperation.OperationPhaseNew, Created: &metav1Now, }, }, }, }, { name: "WaitingForPluginOperations backup with completed failed operations is FinalizingPartiallyFailed", backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-13"). StorageLocation("default"). ItemOperationTimeout(60 * time.Minute). ObjectMeta(builder.WithUID("foo-13")). Phase(velerov1api.BackupPhaseWaitingForPluginOperations).Result(), backupLocation: defaultBackupLocation, operationComplete: true, operationErr: "failed", expectPhase: velerov1api.BackupPhaseFinalizingPartiallyFailed, backupOperations: []*itemoperation.BackupOperation{ { Spec: itemoperation.BackupOperationSpec{ BackupName: "backup-13", BackupUID: "foo-13", BackupItemAction: "foo-13", ResourceIdentifier: velero.ResourceIdentifier{ GroupResource: kuberesource.Pods, Namespace: "ns-1", Name: "pod-1", }, OperationID: "operation-13", }, Status: itemoperation.OperationStatus{ Phase: itemoperation.OperationPhaseNew, Created: &metav1Now, }, }, }, }, { name: "WaitingForPluginOperationsPartiallyFailed backup with completed operations is FinalizingPartiallyFailed", backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-14"). StorageLocation("default"). ItemOperationTimeout(60 * time.Minute). ObjectMeta(builder.WithUID("foo-14")). Phase(velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed).Result(), backupLocation: defaultBackupLocation, operationComplete: true, expectPhase: velerov1api.BackupPhaseFinalizingPartiallyFailed, backupOperations: []*itemoperation.BackupOperation{ { Spec: itemoperation.BackupOperationSpec{ BackupName: "backup-14", BackupUID: "foo-14", BackupItemAction: "foo-14", ResourceIdentifier: velero.ResourceIdentifier{ GroupResource: kuberesource.Pods, Namespace: "ns-1", Name: "pod-1", }, OperationID: "operation-14", }, Status: itemoperation.OperationStatus{ Phase: itemoperation.OperationPhaseNew, Created: &metav1Now, }, }, }, }, { name: "WaitingForPluginOperationsPartiallyFailed backup with incomplete operations is still incomplete", backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-15"). StorageLocation("default"). ItemOperationTimeout(60 * time.Minute). ObjectMeta(builder.WithUID("foo-15")). Phase(velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed).Result(), backupLocation: defaultBackupLocation, operationComplete: false, expectPhase: velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed, backupOperations: []*itemoperation.BackupOperation{ { Spec: itemoperation.BackupOperationSpec{ BackupName: "backup-15", BackupUID: "foo-15", BackupItemAction: "foo-15", ResourceIdentifier: velero.ResourceIdentifier{ GroupResource: kuberesource.Pods, Namespace: "ns-1", Name: "pod-1", }, OperationID: "operation-15", }, Status: itemoperation.OperationStatus{ Phase: itemoperation.OperationPhaseNew, Created: &metav1Now, }, }, }, }, { name: "WaitingForPluginOperationsPartiallyFailed backup with completed failed operations is FinalizingPartiallyFailed", backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-16"). StorageLocation("default"). ItemOperationTimeout(60 * time.Minute). ObjectMeta(builder.WithUID("foo-16")). Phase(velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed).Result(), backupLocation: defaultBackupLocation, operationComplete: true, operationErr: "failed", expectPhase: velerov1api.BackupPhaseFinalizingPartiallyFailed, backupOperations: []*itemoperation.BackupOperation{ { Spec: itemoperation.BackupOperationSpec{ BackupName: "backup-16", BackupUID: "foo-16", BackupItemAction: "foo-16", ResourceIdentifier: velero.ResourceIdentifier{ GroupResource: kuberesource.Pods, Namespace: "ns-1", Name: "pod-1", }, OperationID: "operation-16", }, Status: itemoperation.OperationStatus{ Phase: itemoperation.OperationPhaseNew, Created: &metav1Now, }, }, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { if test.backup == nil { return } initObjs := []runtime.Object{} initObjs = append(initObjs, test.backup) if test.backupLocation != nil { initObjs = append(initObjs, test.backupLocation) } fakeClient := velerotest.NewFakeControllerRuntimeClient(t, initObjs...) reconciler := mockBackupOperationsReconciler(fakeClient, fakeClock, defaultBackupOperationsFrequency) pluginManager.On("CleanupClients").Return(nil) backupStore.On("GetBackupItemOperations", test.backup.Name).Return(test.backupOperations, nil) backupStore.On("PutBackupItemOperations", mock.Anything, mock.Anything).Return(nil) backupStore.On("PutBackupMetadata", mock.Anything, mock.Anything).Return(nil) for _, operation := range test.backupOperations { bia.On("Name").Return("test") bia.On("Progress", operation.Spec.OperationID, mock.Anything). Return(velero.OperationProgress{ Completed: test.operationComplete, Err: test.operationErr, }, nil) pluginManager.On("GetBackupItemActionV2", operation.Spec.BackupItemAction).Return(bia, nil) } _, err := reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: test.backup.Namespace, Name: test.backup.Name}}) gotErr := err != nil assert.Equal(t, test.expectError, gotErr) backupAfter := velerov1api.Backup{} err = fakeClient.Get(t.Context(), types.NamespacedName{ Namespace: test.backup.Namespace, Name: test.backup.Name, }, &backupAfter) require.NoError(t, err) assert.Equal(t, test.expectPhase, backupAfter.Status.Phase) }) } } func TestWrapErrMsg(t *testing.T) { bia2 := &biav2mocks.BackupItemAction{} bia2.On("Name").Return("test-bia") cases := []struct { name string inputErr string plugin v2.BackupItemAction expect string }{ { name: "empty error message", inputErr: "", plugin: bia2, expect: "plugin: test-bia", }, { name: "nil bia", inputErr: "some error happened", plugin: nil, expect: "some error happened, plugin: unknown", }, { name: "regular error and bia", inputErr: "some error happened", plugin: bia2, expect: "some error happened, plugin: test-bia", }, } for _, test := range cases { t.Run(test.name, func(t *testing.T) { got := wrapErrMsg(test.inputErr, test.plugin) assert.Equal(t, test.expect, got) }) } } ================================================ FILE: pkg/controller/backup_queue_controller.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package controller import ( "context" "slices" "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/constant" "github.com/vmware-tanzu/velero/pkg/util/collections" "github.com/vmware-tanzu/velero/pkg/util/kube" ) // backupQueueReconciler reconciles a Backup object type backupQueueReconciler struct { client.Client Scheme *runtime.Scheme logger logrus.FieldLogger concurrentBackups int backupTracker BackupTracker frequency time.Duration } const ( defaultQueuedBackupRecheckFrequency = time.Minute ) // NewBackupQueueReconciler returns a new backupQueueReconciler func NewBackupQueueReconciler( client client.Client, scheme *runtime.Scheme, logger logrus.FieldLogger, concurrentBackups int, backupTracker BackupTracker, ) *backupQueueReconciler { return &backupQueueReconciler{ Client: client, Scheme: scheme, logger: logger, concurrentBackups: max(concurrentBackups, 1), backupTracker: backupTracker, frequency: defaultQueuedBackupRecheckFrequency, } } func queuePositionOrderFunc(objList client.ObjectList) client.ObjectList { backupList := objList.(*velerov1api.BackupList) slices.SortFunc(backupList.Items, func(backup1, backup2 velerov1api.Backup) int { if backup1.Status.QueuePosition < backup2.Status.QueuePosition { return -1 } else if backup1.Status.QueuePosition == backup2.Status.QueuePosition { return 0 } else { return 1 } }) return backupList } // SetupWithManager adds the reconciler to the manager func (r *backupQueueReconciler) SetupWithManager(mgr ctrl.Manager) error { // For periodic requeue, only consider Queued backups, order by QueuePosition gp := kube.NewGenericEventPredicate(func(object client.Object) bool { backup := object.(*velerov1api.Backup) return backup.Status.Phase == velerov1api.BackupPhaseQueued }) s := kube.NewPeriodicalEnqueueSource(r.logger.WithField("controller", constant.ControllerBackupQueue), mgr.GetClient(), &velerov1api.BackupList{}, r.frequency, kube.PeriodicalEnqueueSourceOption{ Predicates: []predicate.Predicate{gp}, OrderFunc: queuePositionOrderFunc, }) return ctrl.NewControllerManagedBy(mgr). For(&velerov1api.Backup{}, builder.WithPredicates(predicate.Funcs{ UpdateFunc: func(ue event.UpdateEvent) bool { backup := ue.ObjectNew.(*velerov1api.Backup) return backup.Status.Phase == "" || backup.Status.Phase == velerov1api.BackupPhaseNew }, CreateFunc: func(ce event.CreateEvent) bool { backup := ce.Object.(*velerov1api.Backup) return backup.Status.Phase == "" || backup.Status.Phase == velerov1api.BackupPhaseNew }, DeleteFunc: func(de event.DeleteEvent) bool { return false }, GenericFunc: func(ge event.GenericEvent) bool { return false }, })). Watches( &velerov1api.Backup{}, handler.EnqueueRequestsFromMapFunc(r.findQueuedBackupsToRequeue), builder.WithPredicates(predicate.Funcs{ UpdateFunc: func(ue event.UpdateEvent) bool { oldBackup := ue.ObjectOld.(*velerov1api.Backup) newBackup := ue.ObjectNew.(*velerov1api.Backup) return oldBackup.Status.Phase == velerov1api.BackupPhaseInProgress && newBackup.Status.Phase != velerov1api.BackupPhaseInProgress || oldBackup.Status.Phase != velerov1api.BackupPhaseQueued && newBackup.Status.Phase == velerov1api.BackupPhaseQueued && r.backupTracker.RunningCount() < r.concurrentBackups }, CreateFunc: func(event.CreateEvent) bool { return false }, DeleteFunc: func(de event.DeleteEvent) bool { return false }, GenericFunc: func(ge event.GenericEvent) bool { return false }, })). WatchesRawSource(s). Named(constant.ControllerBackupQueue). Complete(r) } func (r *backupQueueReconciler) detectNamespaceConflict(ctx context.Context, backup *velerov1api.Backup, earlierBackups []velerov1api.Backup) (bool, string, []string, error) { nsList := &corev1api.NamespaceList{} if err := r.Client.List(ctx, nsList); err != nil { return false, "", nil, err } var clusterNamespaces []string for _, ns := range nsList.Items { clusterNamespaces = append(clusterNamespaces, ns.Name) } foundConflict, conflictBackup := detectNSConflictsInternal(backup, earlierBackups, clusterNamespaces) return foundConflict, conflictBackup, clusterNamespaces, nil } func detectNSConflictsInternal(backup *velerov1api.Backup, earlierBackups []velerov1api.Backup, clusterNamespaces []string) (bool, string) { backupNamespaces := sets.NewString(namespacesForBackup(backup, clusterNamespaces)...) for _, earlierBackup := range earlierBackups { // This will never be true for the primary backup, but for the secondary // runnability check for queued backups ahead of the current backup, we // only care about backups ahead of it. // Backup isn't earlier than this one, skip if earlierBackup.Status.Phase == velerov1api.BackupPhaseQueued && earlierBackup.Status.QueuePosition >= backup.Status.QueuePosition { continue } if backupNamespaces.HasAny(namespacesForBackup(&earlierBackup, clusterNamespaces)...) { return true, earlierBackup.Name } } return false, "" } // Returns true if there are backups ahead of the current backup that are runnable // This could happen if velero just reconciled the one earlier in the queue and rejected it // due to too many running backups, but a backup completed in between that reconcile and this one // so exit, as the recent completion has triggered another reconcile of all queued backups func (r *backupQueueReconciler) checkForEarlierRunnableBackups(backup *velerov1api.Backup, earlierBackups []velerov1api.Backup, clusterNamespaces []string) (bool, string) { for _, earlierBackup := range earlierBackups { // if this backup is queued and ahead of current backup, check for conflicts if earlierBackup.Status.Phase != velerov1api.BackupPhaseQueued || earlierBackup.Status.QueuePosition >= backup.Status.QueuePosition { continue } conflict, _ := detectNSConflictsInternal(&earlierBackup, earlierBackups, clusterNamespaces) // !conflict means we've found an earlier backup that is currently runnable // so current reconcile should exit to run this one if !conflict { return true, earlierBackup.Name } } return false, "" } func namespacesForBackup(backup *velerov1api.Backup, clusterNamespaces []string) []string { // Ignore error here. If a backup has invalid namespace wildcards, the backup controller // will validate and fail it. Consider the ns list empty for conflict detection purposes. nsList, err := collections.NewNamespaceIncludesExcludes().Includes(backup.Spec.IncludedNamespaces...).Excludes(backup.Spec.ExcludedNamespaces...).ActiveNamespaces(clusterNamespaces).ResolveNamespaceList() if err != nil { return []string{} } return nsList } func (r *backupQueueReconciler) getMaxQueuePosition(lister *queuedBackupsLister) int { queuedBackups := lister.orderedQueued() maxPos := 0 if len(queuedBackups) > 0 { maxPos = queuedBackups[len(queuedBackups)-1].Status.QueuePosition } return maxPos } func (r *backupQueueReconciler) findQueuedBackupsToRequeue(ctx context.Context, obj client.Object) []reconcile.Request { backup := obj.(*velerov1api.Backup) requests := []reconcile.Request{} allBackups := &velerov1api.BackupList{} if err := r.Client.List(ctx, allBackups, &client.ListOptions{Namespace: backup.Namespace}); err != nil { r.logger.WithError(err).Error("error listing backups") return requests } backups := r.newQueuedBackupsLister(allBackups).orderedQueued() for _, item := range backups { requests = append(requests, reconcile.Request{ NamespacedName: types.NamespacedName{ Namespace: item.GetNamespace(), Name: item.GetName(), }, }) } return requests } // Reconcile reconciles a Backup object func (r *backupQueueReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := r.logger.WithField("backup", req.NamespacedName.String()) log.Debug("Getting backup") backup := &velerov1api.Backup{} if err := r.Get(ctx, req.NamespacedName, backup); err != nil { log.WithError(err).Error("unable to get backup") return ctrl.Result{}, client.IgnoreNotFound(err) } switch backup.Status.Phase { case "", velerov1api.BackupPhaseNew: // queue new backup allBackups := &velerov1api.BackupList{} if err := r.Client.List(ctx, allBackups, &client.ListOptions{Namespace: backup.Namespace}); err != nil { r.logger.WithError(err).Error("error listing backups") return ctrl.Result{}, nil } lister := r.newQueuedBackupsLister(allBackups) maxQueuePosition := r.getMaxQueuePosition(lister) original := backup.DeepCopy() backup.Status.Phase = velerov1api.BackupPhaseQueued backup.Status.QueuePosition = maxQueuePosition + 1 log.Infof("Queueing backup %v, queue position %v", backup.Name, backup.Status.QueuePosition) if err := kube.PatchResource(original, backup, r.Client); err != nil { return ctrl.Result{}, errors.Wrapf(err, "error updating Backup status to %s", backup.Status.Phase) } case velerov1api.BackupPhaseQueued: // handle queued backup // Find backups ahead of this one (InProgress, ReadyToStart, or Queued with higher position) allBackups := &velerov1api.BackupList{} if err := r.Client.List(ctx, allBackups, &client.ListOptions{Namespace: backup.Namespace}); err != nil { r.logger.WithError(err).Error("error listing backups") return ctrl.Result{}, nil } lister := r.newQueuedBackupsLister(allBackups) if r.backupTracker.RunningCount() >= r.concurrentBackups { log.Debugf("%v concurrent backups are already running, leaving %v queued", r.concurrentBackups, backup.Name) return ctrl.Result{}, nil } earlierBackups := lister.earlierThan(backup.Status.QueuePosition) foundConflict, conflictBackup, clusterNamespaces, err := r.detectNamespaceConflict(ctx, backup, earlierBackups) if err != nil { log.WithError(err).Error("error listing namespaces") return ctrl.Result{}, nil } if foundConflict { log.Infof("Backup %v has namespace conflict with %v, leaving queued", backup.Name, conflictBackup) return ctrl.Result{}, nil } foundEarlierRunnable, earlierRunnable := r.checkForEarlierRunnableBackups(backup, earlierBackups, clusterNamespaces) if foundEarlierRunnable { log.Infof("Earlier queued backup %v is runnable, leaving %v queued", earlierRunnable, backup.Name) return ctrl.Result{}, nil } log.Infof("Dequeueing backup %v, moving to ReadyToStart", backup.Name) original := backup.DeepCopy() backup.Status.Phase = velerov1api.BackupPhaseReadyToStart backup.Status.QueuePosition = 0 if err := kube.PatchResource(original, backup, r.Client); err != nil { return ctrl.Result{}, errors.Wrapf(err, "error updating Backup status to %s", backup.Status.Phase) } r.backupTracker.AddReadyToStart(backup.Namespace, backup.Name) log.Debug("Updating queuePosition for remaining queued backups") queuedBackups := lister.orderedQueued() newQueuePos := 1 for _, queuedBackup := range queuedBackups { if queuedBackup.Name != backup.Name { original := queuedBackup.DeepCopy() queuedBackup.Status.QueuePosition = newQueuePos if err := kube.PatchResource(original, &queuedBackup, r.Client); err != nil { log.WithError(errors.Wrapf(err, "error updating Backup %s queuePosition to %v", queuedBackup.Name, newQueuePos)) return ctrl.Result{}, nil } newQueuePos++ } } return ctrl.Result{}, nil default: log.Debug("Backup is not New or Queued, skipping") return ctrl.Result{}, nil } return ctrl.Result{}, nil } // queuedBackupsLister manages a list of all backups Queued, ReadyToStart, or InProgress // with methods to return specific subsets as needed type queuedBackupsLister struct { backups *velerov1api.BackupList } func (r *backupQueueReconciler) newQueuedBackupsLister(backupList *velerov1api.BackupList) *queuedBackupsLister { backups := []velerov1api.Backup{} for _, backup := range backupList.Items { if backup.Status.Phase == velerov1api.BackupPhaseQueued || backup.Status.Phase == velerov1api.BackupPhaseInProgress || backup.Status.Phase == velerov1api.BackupPhaseReadyToStart { backups = append(backups, backup) } } backupList.Items = backups return &queuedBackupsLister{backupList} } func (l *queuedBackupsLister) earlierThan(queuePos int) []velerov1api.Backup { backups := []velerov1api.Backup{} for _, backup := range l.backups.Items { // InProgress and ReadyToStart backups have QueuePosition==0 if backup.Status.QueuePosition < queuePos { backups = append(backups, backup) } } return backups } func (l *queuedBackupsLister) orderedQueued() []velerov1api.Backup { var returnList []velerov1api.Backup orderedBackupList := queuePositionOrderFunc(l.backups).(*velerov1api.BackupList) for _, item := range orderedBackupList.Items { if item.Status.Phase == velerov1api.BackupPhaseQueued { returnList = append(returnList, item) } } return returnList } ================================================ FILE: pkg/controller/backup_queue_controller_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package controller import ( "testing" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" //metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" //"sigs.k8s.io/controller-runtime/pkg/client/fake" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func TestBackupQueueReconciler(t *testing.T) { scheme := runtime.NewScheme() velerov1api.AddToScheme(scheme) tests := []struct { name string priorBackups []*velerov1api.Backup namespaces []string backup *velerov1api.Backup concurrentBackups int expectError bool expectPhase velerov1api.BackupPhase expectQueuePosition int }{ { name: "New Backup gets queued", backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Result(), expectPhase: velerov1api.BackupPhaseQueued, expectQueuePosition: 1, }, { name: "InProgress Backup is ignored", backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).Result(), expectPhase: velerov1api.BackupPhaseInProgress, }, { name: "Second New Backup gets queued with queuePosition 2", priorBackups: []*velerov1api.Backup{ builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).Result(), }, backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-12").Result(), expectPhase: velerov1api.BackupPhaseQueued, expectQueuePosition: 2, }, { name: "Queued Backup moves to ReadyToStart if no others are running", backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseQueued).Result(), expectPhase: velerov1api.BackupPhaseReadyToStart, }, { name: "Queued Backup remains queued if no spaces available", priorBackups: []*velerov1api.Backup{ builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).Result(), builder.ForBackup(velerov1api.DefaultNamespace, "backup-12").Phase(velerov1api.BackupPhaseInProgress).Result(), }, concurrentBackups: 2, backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-20").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).Result(), expectPhase: velerov1api.BackupPhaseQueued, expectQueuePosition: 1, }, { name: "Queued Backup remains queued if no spaces available including ReadyToStart", priorBackups: []*velerov1api.Backup{ builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).Result(), builder.ForBackup(velerov1api.DefaultNamespace, "backup-12").Phase(velerov1api.BackupPhaseReadyToStart).Result(), }, concurrentBackups: 2, backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-20").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).Result(), expectPhase: velerov1api.BackupPhaseQueued, expectQueuePosition: 1, }, { name: "Queued Backup remains queued if earlier runnable backup is also queued", priorBackups: []*velerov1api.Backup{ builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).Result(), builder.ForBackup(velerov1api.DefaultNamespace, "backup-12").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).Result(), }, concurrentBackups: 3, backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-20").Phase(velerov1api.BackupPhaseQueued).QueuePosition(2).Result(), expectPhase: velerov1api.BackupPhaseQueued, expectQueuePosition: 2, }, { name: "Queued Backup remains queued if in conflict with running backup", priorBackups: []*velerov1api.Backup{ builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).IncludedNamespaces("foo").Result(), }, namespaces: []string{"foo"}, concurrentBackups: 3, backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-20").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).IncludedNamespaces("foo").Result(), expectPhase: velerov1api.BackupPhaseQueued, expectQueuePosition: 1, }, { name: "Queued Backup remains queued if in conflict with ReadyToStart backup", priorBackups: []*velerov1api.Backup{ builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseReadyToStart).IncludedNamespaces("foo").Result(), }, namespaces: []string{"foo"}, concurrentBackups: 3, backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-20").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).IncludedNamespaces("foo").Result(), expectPhase: velerov1api.BackupPhaseQueued, expectQueuePosition: 1, }, { name: "Queued Backup remains queued if in conflict with earlier queued backup", priorBackups: []*velerov1api.Backup{ builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).IncludedNamespaces("foo").Result(), }, namespaces: []string{"foo", "bar"}, concurrentBackups: 3, backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-20").Phase(velerov1api.BackupPhaseQueued).QueuePosition(2).IncludedNamespaces("foo", "bar").Result(), expectPhase: velerov1api.BackupPhaseQueued, expectQueuePosition: 2, }, { name: "Queued Backup remains queued if earlier non-ns-conflict backup exists", priorBackups: []*velerov1api.Backup{ builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).IncludedNamespaces("bar").Result(), builder.ForBackup(velerov1api.DefaultNamespace, "backup-12").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).IncludedNamespaces("foo").Result(), }, namespaces: []string{"foo", "bar", "baz"}, concurrentBackups: 3, backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-20").Phase(velerov1api.BackupPhaseQueued).QueuePosition(2).IncludedNamespaces("baz").Result(), expectPhase: velerov1api.BackupPhaseQueued, expectQueuePosition: 2, }, { name: "Running all-namespace backup conflicts with queued one-namespace backup ", priorBackups: []*velerov1api.Backup{ builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).IncludedNamespaces("*").Result(), }, namespaces: []string{"foo", "bar"}, concurrentBackups: 3, backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-20").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).IncludedNamespaces("foo").Result(), expectPhase: velerov1api.BackupPhaseQueued, expectQueuePosition: 1, }, { name: "Running one-namespace backup conflicts with queued all-namespace backup ", priorBackups: []*velerov1api.Backup{ builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).IncludedNamespaces("bar").Result(), }, namespaces: []string{"foo", "bar"}, concurrentBackups: 3, backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-20").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).IncludedNamespaces("*").Result(), expectPhase: velerov1api.BackupPhaseQueued, expectQueuePosition: 1, }, { name: "Queued Backup moves to ReadyToStart if running count < concurrentBackups", priorBackups: []*velerov1api.Backup{ builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).Result(), builder.ForBackup(velerov1api.DefaultNamespace, "backup-12").Phase(velerov1api.BackupPhaseInProgress).Result(), }, concurrentBackups: 3, backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-20").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).Result(), expectPhase: velerov1api.BackupPhaseReadyToStart, }, { name: "Queued Backup moves to ReadyToStart if running count < concurrentBackups and no ns conflict found", priorBackups: []*velerov1api.Backup{ builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseReadyToStart).IncludedNamespaces("foo").Result(), }, namespaces: []string{"foo", "bar"}, concurrentBackups: 3, backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-20").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).IncludedNamespaces("bar").Result(), expectPhase: velerov1api.BackupPhaseReadyToStart, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { if test.backup == nil { return } backupTracker := NewBackupTracker() initObjs := []runtime.Object{} for _, priorBackup := range test.priorBackups { initObjs = append(initObjs, priorBackup) if priorBackup.Status.Phase == velerov1api.BackupPhaseReadyToStart { backupTracker.AddReadyToStart(priorBackup.Namespace, priorBackup.Name) } else if priorBackup.Status.Phase == velerov1api.BackupPhaseInProgress { backupTracker.Add(priorBackup.Namespace, priorBackup.Name) } } for _, ns := range test.namespaces { initObjs = append(initObjs, builder.ForNamespace(ns).Result()) } initObjs = append(initObjs, test.backup) fakeClient := velerotest.NewFakeControllerRuntimeClient(t, initObjs...) logger := logrus.New() log := logger.WithField("controller", "backup-queue-test") r := NewBackupQueueReconciler(fakeClient, scheme, log, test.concurrentBackups, backupTracker) req := ctrl.Request{NamespacedName: types.NamespacedName{Namespace: test.backup.Namespace, Name: test.backup.Name}} res, err := r.Reconcile(t.Context(), req) gotErr := err != nil require.NoError(t, err) assert.Equal(t, ctrl.Result{}, res) assert.Equal(t, test.expectError, gotErr) backupAfter := velerov1api.Backup{} err = fakeClient.Get(t.Context(), types.NamespacedName{ Namespace: test.backup.Namespace, Name: test.backup.Name, }, &backupAfter) require.NoError(t, err) assert.Equal(t, test.expectPhase, backupAfter.Status.Phase) assert.Equal(t, test.expectQueuePosition, backupAfter.Status.QueuePosition) }) } } ================================================ FILE: pkg/controller/backup_repository_controller.go ================================================ /* Copyright 2018, 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package controller import ( "bytes" "context" "encoding/json" "fmt" "reflect" "slices" "time" "github.com/petar/GoLLRB/llrb" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" clocks "k8s.io/utils/clock" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/constant" "github.com/vmware-tanzu/velero/pkg/label" "github.com/vmware-tanzu/velero/pkg/metrics" repoconfig "github.com/vmware-tanzu/velero/pkg/repository/config" "github.com/vmware-tanzu/velero/pkg/repository/maintenance" repomanager "github.com/vmware-tanzu/velero/pkg/repository/manager" "github.com/vmware-tanzu/velero/pkg/util/kube" "github.com/vmware-tanzu/velero/pkg/util/logging" ) const ( repoSyncPeriod = 5 * time.Minute defaultMaintainFrequency = 7 * 24 * time.Hour defaultMaintenanceStatusQueueLength = 3 ) type BackupRepoReconciler struct { client.Client namespace string logger logrus.FieldLogger clock clocks.WithTickerAndDelayedExecution maintenanceFrequency time.Duration backupRepoConfig string repositoryManager repomanager.Manager repoMaintenanceConfig string logLevel logrus.Level logFormat *logging.FormatFlag metrics *metrics.ServerMetrics } func NewBackupRepoReconciler( namespace string, logger logrus.FieldLogger, client client.Client, repositoryManager repomanager.Manager, maintenanceFrequency time.Duration, backupRepoConfig string, repoMaintenanceConfig string, logLevel logrus.Level, logFormat *logging.FormatFlag, metrics *metrics.ServerMetrics, ) *BackupRepoReconciler { c := &BackupRepoReconciler{ client, namespace, logger, clocks.RealClock{}, maintenanceFrequency, backupRepoConfig, repositoryManager, repoMaintenanceConfig, logLevel, logFormat, metrics, } return c } func (r *BackupRepoReconciler) SetupWithManager(mgr ctrl.Manager) error { s := kube.NewPeriodicalEnqueueSource( r.logger.WithField("controller", constant.ControllerBackupRepo), mgr.GetClient(), &velerov1api.BackupRepositoryList{}, repoSyncPeriod, kube.PeriodicalEnqueueSourceOption{}, ) return ctrl.NewControllerManagedBy(mgr). For(&velerov1api.BackupRepository{}, builder.WithPredicates(kube.SpecChangePredicate{})). WatchesRawSource(s). Watches( // mark BackupRepository as invalid when BSL is created, updated or deleted. // BSL may be recreated after deleting, so also include the create event &velerov1api.BackupStorageLocation{}, kube.EnqueueRequestsFromMapUpdateFunc(r.invalidateBackupReposForBSL), builder.WithPredicates( // Combine three predicates together to guarantee // only BSL's Delete Event and Update Event can enqueue. // We don't care about BSL's Generic Event and Create Event, // because BSL's periodical enqueue triggers Generic Event, // and the BackupRepository controller restart will triggers BSL create event. kube.NewUpdateEventPredicate( r.needInvalidBackupRepo, ), kube.NewGenericEventPredicate( func(client.Object) bool { return false }, ), kube.NewCreateEventPredicate( func(client.Object) bool { return false }, ), ), ). Complete(r) } func (r *BackupRepoReconciler) invalidateBackupReposForBSL(ctx context.Context, bslObj client.Object) []reconcile.Request { bsl := bslObj.(*velerov1api.BackupStorageLocation) list := &velerov1api.BackupRepositoryList{} options := &client.ListOptions{ LabelSelector: labels.Set(map[string]string{ velerov1api.StorageLocationLabel: label.GetValidName(bsl.Name), }).AsSelector(), } if err := r.List(context.TODO(), list, options); err != nil { r.logger.WithField("BSL", bsl.Name).WithError(err).Error("unable to list BackupRepositories") return []reconcile.Request{} } requests := []reconcile.Request{} for i := range list.Items { r.logger.WithField("BSL", bsl.Name).Infof("Invalidating Backup Repository %s", list.Items[i].Name) if err := r.patchBackupRepository(context.Background(), &list.Items[i], repoNotReady("re-establish on BSL change, create or delete")); err != nil { r.logger.WithField("BSL", bsl.Name).WithError(err).Errorf("fail to patch BackupRepository %s", list.Items[i].Name) continue } requests = append(requests, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: list.Items[i].Namespace, Name: list.Items[i].Name}}) } return requests } // needInvalidBackupRepo returns true if the BSL's storage type, bucket, prefix, CACert, or config has changed func (r *BackupRepoReconciler) needInvalidBackupRepo(oldObj client.Object, newObj client.Object) bool { oldBSL := oldObj.(*velerov1api.BackupStorageLocation) newBSL := newObj.(*velerov1api.BackupStorageLocation) oldStorage := oldBSL.Spec.StorageType.ObjectStorage newStorage := newBSL.Spec.StorageType.ObjectStorage oldConfig := oldBSL.Spec.Config newConfig := newBSL.Spec.Config if oldStorage == nil { oldStorage = &velerov1api.ObjectStorageLocation{} } if newStorage == nil { newStorage = &velerov1api.ObjectStorageLocation{} } logger := r.logger.WithField("BSL", newBSL.Name) if oldStorage.Bucket != newStorage.Bucket { logger.WithFields(logrus.Fields{ "old bucket": oldStorage.Bucket, "new bucket": newStorage.Bucket, }).Info("BSL's bucket has changed, invalid backup repositories") return true } if oldStorage.Prefix != newStorage.Prefix { logger.WithFields(logrus.Fields{ "old prefix": oldStorage.Prefix, "new prefix": newStorage.Prefix, }).Info("BSL's prefix has changed, invalid backup repositories") return true } // Check if either CACert or CACertRef has changed if !bytes.Equal(oldStorage.CACert, newStorage.CACert) { logger.Info("BSL's CACert has changed, invalid backup repositories") return true } // Check if CACertRef has changed if (oldStorage.CACertRef == nil && newStorage.CACertRef != nil) || (oldStorage.CACertRef != nil && newStorage.CACertRef == nil) || (oldStorage.CACertRef != nil && newStorage.CACertRef != nil && (oldStorage.CACertRef.Name != newStorage.CACertRef.Name || oldStorage.CACertRef.Key != newStorage.CACertRef.Key)) { logger.Info("BSL's CACertRef has changed, invalid backup repositories") return true } if !reflect.DeepEqual(oldConfig, newConfig) { logger.Info("BSL's storage config has changed, invalid backup repositories") return true } return false } func (r *BackupRepoReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := r.logger.WithField("backupRepo", req.String()) backupRepo := &velerov1api.BackupRepository{} if err := r.Get(ctx, req.NamespacedName, backupRepo); err != nil { if apierrors.IsNotFound(err) { log.Warnf("backup repository %s in namespace %s is not found", req.Name, req.Namespace) return ctrl.Result{}, nil } log.WithError(err).Error("error getting backup repository") return ctrl.Result{}, err } bsl, bslErr := r.getBSL(ctx, backupRepo) if bslErr != nil { log.WithError(bslErr).Error("Fail to get BSL for BackupRepository. Skip reconciling.") return ctrl.Result{}, nil } if backupRepo.Status.Phase == "" || backupRepo.Status.Phase == velerov1api.BackupRepositoryPhaseNew { if err := r.initializeRepo(ctx, backupRepo, bsl, log); err != nil { log.WithError(err).Error("error initialize repository") return ctrl.Result{}, errors.WithStack(err) } return ctrl.Result{}, nil } // If the repository is ready or not-ready, check it for stale locks, but if // this fails for any reason, it's non-critical so we still continue on to the // rest of the "process" logic. log.Debug("Checking repository for stale locks") if err := r.repositoryManager.UnlockRepo(backupRepo); err != nil { log.WithError(err).Error("Error checking repository for stale locks") } switch backupRepo.Status.Phase { case velerov1api.BackupRepositoryPhaseNotReady: ready, err := r.checkNotReadyRepo(ctx, backupRepo, bsl, log) if err != nil { return ctrl.Result{}, err } else if !ready { return ctrl.Result{}, nil } fallthrough case velerov1api.BackupRepositoryPhaseReady: if bsl.Spec.AccessMode == velerov1api.BackupStorageLocationAccessModeReadOnly { log.Debugf("Skip running maintenance for BackupRepository, because its BSL is in the ReadOnly mode.") return ctrl.Result{}, nil } if err := r.recallMaintenance(ctx, backupRepo, log); err != nil { return ctrl.Result{}, errors.Wrap(err, "error handling incomplete repo maintenance jobs") } if err := r.runMaintenanceIfDue(ctx, backupRepo, log); err != nil { return ctrl.Result{}, errors.Wrap(err, "error check and run repo maintenance jobs") } // Get the configured number of maintenance jobs to keep from ConfigMap keepJobs, err := maintenance.GetKeepLatestMaintenanceJobs(ctx, r.Client, log, r.namespace, r.repoMaintenanceConfig, backupRepo) if err != nil { log.WithError(err).Warn("Failed to get keepLatestMaintenanceJobs from ConfigMap, using CLI parameter value") } if err := maintenance.DeleteOldJobs(r.Client, *backupRepo, keepJobs, log); err != nil { log.WithError(err).Warn("Failed to delete old maintenance jobs") } } return ctrl.Result{}, nil } func (r *BackupRepoReconciler) getBSL(ctx context.Context, req *velerov1api.BackupRepository) (*velerov1api.BackupStorageLocation, error) { loc := new(velerov1api.BackupStorageLocation) if err := r.Get(ctx, client.ObjectKey{ Namespace: req.Namespace, Name: req.Spec.BackupStorageLocation, }, loc); err != nil { return nil, err } return loc, nil } func (r *BackupRepoReconciler) getIdentifierByBSL(bsl *velerov1api.BackupStorageLocation, req *velerov1api.BackupRepository) (string, error) { repoIdentifier, err := repoconfig.GetRepoIdentifier(bsl, req.Spec.VolumeNamespace) if err != nil { return "", errors.Wrapf(err, "error to get identifier for repo %s", req.Name) } return repoIdentifier, nil } func (r *BackupRepoReconciler) initializeRepo(ctx context.Context, req *velerov1api.BackupRepository, bsl *velerov1api.BackupStorageLocation, log logrus.FieldLogger) error { log.WithField("repoConfig", r.backupRepoConfig).Info("Initializing backup repository") var repoIdentifier string // Only get restic identifier for restic repositories if req.Spec.RepositoryType == "" || req.Spec.RepositoryType == velerov1api.BackupRepositoryTypeRestic { var err error repoIdentifier, err = r.getIdentifierByBSL(bsl, req) if err != nil { return r.patchBackupRepository(ctx, req, func(rr *velerov1api.BackupRepository) { rr.Status.Message = err.Error() rr.Status.Phase = velerov1api.BackupRepositoryPhaseNotReady if rr.Spec.MaintenanceFrequency.Duration <= 0 { rr.Spec.MaintenanceFrequency = metav1.Duration{Duration: r.getRepositoryMaintenanceFrequency(req)} } }) } } config, err := getBackupRepositoryConfig(ctx, r, r.backupRepoConfig, r.namespace, req.Name, req.Spec.RepositoryType, log) if err != nil { log.WithError(err).Warn("Failed to get repo config, repo config is ignored") } else if config != nil { log.Infof("Init repo with config %v", config) } // defaulting - if the patch fails, return an error so the item is returned to the queue if err := r.patchBackupRepository(ctx, req, func(rr *velerov1api.BackupRepository) { // Only set ResticIdentifier for restic repositories if rr.Spec.RepositoryType == "" || rr.Spec.RepositoryType == velerov1api.BackupRepositoryTypeRestic { rr.Spec.ResticIdentifier = repoIdentifier } if rr.Spec.MaintenanceFrequency.Duration <= 0 { rr.Spec.MaintenanceFrequency = metav1.Duration{Duration: r.getRepositoryMaintenanceFrequency(req)} } rr.Spec.RepositoryConfig = config }); err != nil { return err } if err := ensureRepo(req, r.repositoryManager); err != nil { return r.patchBackupRepository(ctx, req, repoNotReady(err.Error())) } return r.patchBackupRepository(ctx, req, func(rr *velerov1api.BackupRepository) { rr.Status.Phase = velerov1api.BackupRepositoryPhaseReady rr.Status.LastMaintenanceTime = &metav1.Time{Time: time.Now()} }) } func (r *BackupRepoReconciler) getRepositoryMaintenanceFrequency(req *velerov1api.BackupRepository) time.Duration { if r.maintenanceFrequency > 0 { r.logger.WithField("frequency", r.maintenanceFrequency).Info("Set user defined maintenance frequency") return r.maintenanceFrequency } frequency, err := r.repositoryManager.DefaultMaintenanceFrequency(req) if err != nil || frequency <= 0 { r.logger.WithError(err).WithField("returned frequency", frequency).Warn("Failed to get maitanance frequency, use the default one") frequency = defaultMaintainFrequency } else { r.logger.WithField("frequency", frequency).Info("Set maintenance according to repository suggestion") } return frequency } // ensureRepo calls repo manager's PrepareRepo to ensure the repo is ready for use. // An error is returned if the repository can't be connected to or initialized. func ensureRepo(repo *velerov1api.BackupRepository, repoManager repomanager.Manager) error { return repoManager.PrepareRepo(repo) } func (r *BackupRepoReconciler) recallMaintenance(ctx context.Context, req *velerov1api.BackupRepository, log logrus.FieldLogger) error { history, err := maintenance.WaitAllJobsComplete(ctx, r.Client, req, defaultMaintenanceStatusQueueLength, log) if err != nil { return errors.Wrapf(err, "error waiting incomplete repo maintenance job for repo %s", req.Name) } consolidated := consolidateHistory(history, req.Status.RecentMaintenance) if consolidated == nil { return nil } lastMaintenanceTime := getLastMaintenanceTimeFromHistory(consolidated) log.Warn("Updating backup repository because of unrecorded histories") return r.patchBackupRepository(ctx, req, func(rr *velerov1api.BackupRepository) { if lastMaintenanceTime != nil && (rr.Status.LastMaintenanceTime == nil || lastMaintenanceTime.After(rr.Status.LastMaintenanceTime.Time)) { if rr.Status.LastMaintenanceTime != nil { log.Warnf("Updating backup repository last maintenance time (%v) from history (%v)", rr.Status.LastMaintenanceTime.Time, lastMaintenanceTime.Time) } else { log.Warnf("Setting backup repository last maintenance time from history (%v)", lastMaintenanceTime.Time) } rr.Status.LastMaintenanceTime = lastMaintenanceTime } rr.Status.RecentMaintenance = consolidated }) } type maintenanceStatusWrapper struct { status *velerov1api.BackupRepositoryMaintenanceStatus } func (w maintenanceStatusWrapper) Less(other llrb.Item) bool { return w.status.StartTimestamp.Before(other.(maintenanceStatusWrapper).status.StartTimestamp) } func consolidateHistory(coming, cur []velerov1api.BackupRepositoryMaintenanceStatus) []velerov1api.BackupRepositoryMaintenanceStatus { if len(coming) == 0 { return nil } if slices.EqualFunc(cur, coming, func(a, b velerov1api.BackupRepositoryMaintenanceStatus) bool { return a.StartTimestamp.Equal(b.StartTimestamp) }) { return nil } consolidator := llrb.New() for i := range cur { consolidator.ReplaceOrInsert(maintenanceStatusWrapper{&cur[i]}) } for i := range coming { consolidator.ReplaceOrInsert(maintenanceStatusWrapper{&coming[i]}) } truncated := []velerov1api.BackupRepositoryMaintenanceStatus{} for consolidator.Len() > 0 { if len(truncated) == defaultMaintenanceStatusQueueLength { break } item := consolidator.DeleteMax() truncated = append(truncated, *item.(maintenanceStatusWrapper).status) } slices.Reverse(truncated) if slices.EqualFunc(cur, truncated, func(a, b velerov1api.BackupRepositoryMaintenanceStatus) bool { return a.StartTimestamp.Equal(b.StartTimestamp) }) { return nil } return truncated } func getLastMaintenanceTimeFromHistory(history []velerov1api.BackupRepositoryMaintenanceStatus) *metav1.Time { time := history[0].CompleteTimestamp for i := range history { if history[i].CompleteTimestamp == nil { continue } if time == nil || time.Before(history[i].CompleteTimestamp) { time = history[i].CompleteTimestamp } } return time } var funcStartMaintenanceJob = maintenance.StartNewJob var funcWaitMaintenanceJobComplete = maintenance.WaitJobComplete func (r *BackupRepoReconciler) runMaintenanceIfDue(ctx context.Context, req *velerov1api.BackupRepository, log logrus.FieldLogger) error { startTime := r.clock.Now() if !dueForMaintenance(req, startTime) { log.Debug("not due for maintenance") return nil } log.Info("Running maintenance on backup repository") job, err := funcStartMaintenanceJob(r.Client, ctx, req, r.repoMaintenanceConfig, r.logLevel, r.logFormat, log) if err != nil { log.WithError(err).Warn("Starting repo maintenance failed") // Record failure metric when job fails to start if r.metrics != nil { r.metrics.RegisterRepoMaintenanceFailure(req.Name) } return r.patchBackupRepository(ctx, req, func(rr *velerov1api.BackupRepository) { updateRepoMaintenanceHistory(rr, velerov1api.BackupRepositoryMaintenanceFailed, &metav1.Time{Time: startTime}, nil, fmt.Sprintf("Failed to start maintenance job, err: %v", err)) }) } // when WaitMaintenanceJobComplete fails, the maintenance result will be left aside temporarily // If the maintenenance still completes later, recallMaintenance recalls the left once and update LastMaintenanceTime and history status, err := funcWaitMaintenanceJobComplete(r.Client, ctx, job, r.namespace, log) if err != nil { return errors.Wrapf(err, "error waiting repo maintenance completion status") } if status.Result == velerov1api.BackupRepositoryMaintenanceFailed { log.WithError(err).Warn("Pruning repository failed") // Record failure metric if r.metrics != nil { r.metrics.RegisterRepoMaintenanceFailure(req.Name) if status.StartTimestamp != nil && status.CompleteTimestamp != nil { duration := status.CompleteTimestamp.Sub(status.StartTimestamp.Time).Seconds() r.metrics.ObserveRepoMaintenanceDuration(req.Name, duration) } } return r.patchBackupRepository(ctx, req, func(rr *velerov1api.BackupRepository) { updateRepoMaintenanceHistory(rr, velerov1api.BackupRepositoryMaintenanceFailed, status.StartTimestamp, status.CompleteTimestamp, status.Message) }) } // Record success metric if r.metrics != nil { r.metrics.RegisterRepoMaintenanceSuccess(req.Name) if status.StartTimestamp != nil && status.CompleteTimestamp != nil { duration := status.CompleteTimestamp.Sub(status.StartTimestamp.Time).Seconds() r.metrics.ObserveRepoMaintenanceDuration(req.Name, duration) } } return r.patchBackupRepository(ctx, req, func(rr *velerov1api.BackupRepository) { rr.Status.LastMaintenanceTime = &metav1.Time{Time: status.CompleteTimestamp.Time} updateRepoMaintenanceHistory(rr, velerov1api.BackupRepositoryMaintenanceSucceeded, status.StartTimestamp, status.CompleteTimestamp, status.Message) }) } func updateRepoMaintenanceHistory(repo *velerov1api.BackupRepository, result velerov1api.BackupRepositoryMaintenanceResult, startTime, completionTime *metav1.Time, message string) { latest := velerov1api.BackupRepositoryMaintenanceStatus{ Result: result, StartTimestamp: startTime, CompleteTimestamp: completionTime, Message: message, } startingPos := 0 if len(repo.Status.RecentMaintenance) >= defaultMaintenanceStatusQueueLength { startingPos = len(repo.Status.RecentMaintenance) - defaultMaintenanceStatusQueueLength + 1 } repo.Status.RecentMaintenance = append(repo.Status.RecentMaintenance[startingPos:], latest) } func dueForMaintenance(req *velerov1api.BackupRepository, now time.Time) bool { return req.Status.LastMaintenanceTime == nil || req.Status.LastMaintenanceTime.Add(req.Spec.MaintenanceFrequency.Duration).Before(now) } func (r *BackupRepoReconciler) checkNotReadyRepo(ctx context.Context, req *velerov1api.BackupRepository, bsl *velerov1api.BackupStorageLocation, log logrus.FieldLogger) (bool, error) { log.Info("Checking backup repository for readiness") // Only check and update restic identifier for restic repositories if req.Spec.RepositoryType == "" || req.Spec.RepositoryType == velerov1api.BackupRepositoryTypeRestic { repoIdentifier, err := r.getIdentifierByBSL(bsl, req) if err != nil { return false, r.patchBackupRepository(ctx, req, repoNotReady(err.Error())) } if repoIdentifier != req.Spec.ResticIdentifier { if err := r.patchBackupRepository(ctx, req, func(rr *velerov1api.BackupRepository) { rr.Spec.ResticIdentifier = repoIdentifier }); err != nil { return false, err } } } // we need to ensure it (first check, if check fails, attempt to init) // because we don't know if it's been successfully initialized yet. if err := ensureRepo(req, r.repositoryManager); err != nil { return false, r.patchBackupRepository(ctx, req, repoNotReady(err.Error())) } err := r.patchBackupRepository(ctx, req, repoReady()) if err != nil { return false, err } return true, nil } func repoNotReady(msg string) func(*velerov1api.BackupRepository) { return func(r *velerov1api.BackupRepository) { r.Status.Phase = velerov1api.BackupRepositoryPhaseNotReady r.Status.Message = msg } } func repoReady() func(*velerov1api.BackupRepository) { return func(r *velerov1api.BackupRepository) { r.Status.Phase = velerov1api.BackupRepositoryPhaseReady r.Status.Message = "" } } // patchBackupRepository mutates req with the provided mutate function, and patches it // through the Kube API. After executing this function, req will be updated with both // the mutation and the results of the Patch() API call. func (r *BackupRepoReconciler) patchBackupRepository(ctx context.Context, req *velerov1api.BackupRepository, mutate func(*velerov1api.BackupRepository)) error { original := req.DeepCopy() mutate(req) if err := r.Patch(ctx, req, client.MergeFrom(original)); err != nil { return errors.Wrap(err, "error patching BackupRepository") } return nil } func getBackupRepositoryConfig(ctx context.Context, ctrlClient client.Client, configName, namespace, repoName, repoType string, log logrus.FieldLogger) (map[string]string, error) { if configName == "" { return nil, nil } loc := &corev1api.ConfigMap{} if err := ctrlClient.Get(ctx, client.ObjectKey{ Namespace: namespace, Name: configName, }, loc); err != nil { return nil, errors.Wrapf(err, "error getting configMap %s", configName) } jsonData, found := loc.Data[repoType] if !found { log.Infof("No data for repo type %s in config map %s", repoType, configName) return nil, nil } var unmarshalled map[string]any if err := json.Unmarshal([]byte(jsonData), &unmarshalled); err != nil { return nil, errors.Wrapf(err, "error unmarshalling config data from %s for repo %s, repo type %s", configName, repoName, repoType) } result := map[string]string{} for k, v := range unmarshalled { result[k] = fmt.Sprintf("%v", v) } return result, nil } ================================================ FILE: pkg/controller/backup_repository_controller_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package controller import ( "context" "errors" "testing" "time" "github.com/prometheus/client_golang/prometheus" dto "github.com/prometheus/client_model/go" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/clock" ctrl "sigs.k8s.io/controller-runtime" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/metrics" "github.com/vmware-tanzu/velero/pkg/repository" "github.com/vmware-tanzu/velero/pkg/repository/maintenance" repomaintenance "github.com/vmware-tanzu/velero/pkg/repository/maintenance" repomanager "github.com/vmware-tanzu/velero/pkg/repository/manager" repomokes "github.com/vmware-tanzu/velero/pkg/repository/mocks" repotypes "github.com/vmware-tanzu/velero/pkg/repository/types" velerotest "github.com/vmware-tanzu/velero/pkg/test" "github.com/vmware-tanzu/velero/pkg/util/logging" "sigs.k8s.io/controller-runtime/pkg/client" clientFake "sigs.k8s.io/controller-runtime/pkg/client/fake" batchv1api "k8s.io/api/batch/v1" ) const testMaintenanceFrequency = 10 * time.Minute func mockBackupRepoReconciler(t *testing.T, mockOn string, arg any, ret ...any) *BackupRepoReconciler { t.Helper() mgr := &repomokes.Manager{} if mockOn != "" { mgr.On(mockOn, arg).Return(ret...) } return NewBackupRepoReconciler( velerov1api.DefaultNamespace, velerotest.NewLogger(), velerotest.NewFakeControllerRuntimeClient(t), mgr, testMaintenanceFrequency, "fake-repo-config", "", logrus.InfoLevel, nil, nil, ) } func mockBackupRepositoryCR() *velerov1api.BackupRepository { return &velerov1api.BackupRepository{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "repo", }, Spec: velerov1api.BackupRepositorySpec{ MaintenanceFrequency: metav1.Duration{Duration: testMaintenanceFrequency}, }, } } func TestPatchBackupRepository(t *testing.T) { rr := mockBackupRepositoryCR() reconciler := mockBackupRepoReconciler(t, "", nil, nil) err := reconciler.Client.Create(t.Context(), rr) require.NoError(t, err) err = reconciler.patchBackupRepository(t.Context(), rr, repoReady()) require.NoError(t, err) assert.Equal(t, velerov1api.BackupRepositoryPhaseReady, rr.Status.Phase) err = reconciler.patchBackupRepository(t.Context(), rr, repoNotReady("not ready")) require.NoError(t, err) assert.NotEqual(t, velerov1api.BackupRepositoryPhaseReady, rr.Status.Phase) } func TestCheckNotReadyRepo(t *testing.T) { // Test for restic repository t.Run("restic repository", func(t *testing.T) { rr := mockBackupRepositoryCR() rr.Spec.BackupStorageLocation = "default" rr.Spec.ResticIdentifier = "fake-identifier" rr.Spec.VolumeNamespace = "volume-ns-1" rr.Spec.RepositoryType = velerov1api.BackupRepositoryTypeRestic reconciler := mockBackupRepoReconciler(t, "PrepareRepo", rr, nil) err := reconciler.Client.Create(t.Context(), rr) require.NoError(t, err) location := velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Config: map[string]string{"resticRepoPrefix": "s3:test.amazonaws.com/bucket/restic"}, }, ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: rr.Spec.BackupStorageLocation, }, } _, err = reconciler.checkNotReadyRepo(t.Context(), rr, &location, reconciler.logger) require.NoError(t, err) assert.Equal(t, velerov1api.BackupRepositoryPhaseReady, rr.Status.Phase) assert.Equal(t, "s3:test.amazonaws.com/bucket/restic/volume-ns-1", rr.Spec.ResticIdentifier) }) // Test for kopia repository t.Run("kopia repository", func(t *testing.T) { rr := mockBackupRepositoryCR() rr.Spec.BackupStorageLocation = "default" rr.Spec.VolumeNamespace = "volume-ns-1" rr.Spec.RepositoryType = velerov1api.BackupRepositoryTypeKopia reconciler := mockBackupRepoReconciler(t, "PrepareRepo", rr, nil) err := reconciler.Client.Create(t.Context(), rr) require.NoError(t, err) location := velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Config: map[string]string{"resticRepoPrefix": "s3:test.amazonaws.com/bucket/restic"}, }, ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: rr.Spec.BackupStorageLocation, }, } _, err = reconciler.checkNotReadyRepo(t.Context(), rr, &location, reconciler.logger) require.NoError(t, err) assert.Equal(t, velerov1api.BackupRepositoryPhaseReady, rr.Status.Phase) // ResticIdentifier should remain empty for kopia assert.Empty(t, rr.Spec.ResticIdentifier) }) // Test for empty repository type (defaults to restic) t.Run("empty repository type", func(t *testing.T) { rr := mockBackupRepositoryCR() rr.Spec.BackupStorageLocation = "default" rr.Spec.ResticIdentifier = "fake-identifier" rr.Spec.VolumeNamespace = "volume-ns-1" // Deliberately leave RepositoryType empty reconciler := mockBackupRepoReconciler(t, "PrepareRepo", rr, nil) err := reconciler.Client.Create(t.Context(), rr) require.NoError(t, err) location := velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Config: map[string]string{"resticRepoPrefix": "s3:test.amazonaws.com/bucket/restic"}, }, ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: rr.Spec.BackupStorageLocation, }, } _, err = reconciler.checkNotReadyRepo(t.Context(), rr, &location, reconciler.logger) require.NoError(t, err) assert.Equal(t, velerov1api.BackupRepositoryPhaseReady, rr.Status.Phase) assert.Equal(t, "s3:test.amazonaws.com/bucket/restic/volume-ns-1", rr.Spec.ResticIdentifier) }) } func startMaintenanceJobFail(client.Client, context.Context, *velerov1api.BackupRepository, string, logrus.Level, *logging.FormatFlag, logrus.FieldLogger) (string, error) { return "", errors.New("fake-start-error") } func startMaintenanceJobSucceed(client.Client, context.Context, *velerov1api.BackupRepository, string, logrus.Level, *logging.FormatFlag, logrus.FieldLogger) (string, error) { return "fake-job-name", nil } func waitMaintenanceJobCompleteFail(client.Client, context.Context, string, string, logrus.FieldLogger) (velerov1api.BackupRepositoryMaintenanceStatus, error) { return velerov1api.BackupRepositoryMaintenanceStatus{}, errors.New("fake-wait-error") } func waitMaintenanceJobCompleteFunc(now time.Time, result velerov1api.BackupRepositoryMaintenanceResult, message string) func(client.Client, context.Context, string, string, logrus.FieldLogger) (velerov1api.BackupRepositoryMaintenanceStatus, error) { completionTimeStamp := &metav1.Time{Time: now.Add(time.Hour)} if result == velerov1api.BackupRepositoryMaintenanceFailed { completionTimeStamp = nil } return func(client.Client, context.Context, string, string, logrus.FieldLogger) (velerov1api.BackupRepositoryMaintenanceStatus, error) { return velerov1api.BackupRepositoryMaintenanceStatus{ StartTimestamp: &metav1.Time{Time: now}, CompleteTimestamp: completionTimeStamp, Result: result, Message: message, }, nil } } type fakeClock struct { now time.Time } func (f *fakeClock) After(time.Duration) <-chan time.Time { return nil } func (f *fakeClock) NewTicker(time.Duration) clock.Ticker { return nil } func (f *fakeClock) NewTimer(time.Duration) clock.Timer { return nil } func (f *fakeClock) Now() time.Time { return f.now } func (f *fakeClock) Since(time.Time) time.Duration { return 0 } func (f *fakeClock) Sleep(time.Duration) {} func (f *fakeClock) Tick(time.Duration) <-chan time.Time { return nil } func (f *fakeClock) AfterFunc(time.Duration, func()) clock.Timer { return nil } func TestRunMaintenanceIfDue(t *testing.T) { now := time.Now().Round(time.Second) tests := []struct { name string repo *velerov1api.BackupRepository startJobFunc func(client.Client, context.Context, *velerov1api.BackupRepository, string, logrus.Level, *logging.FormatFlag, logrus.FieldLogger) (string, error) waitJobFunc func(client.Client, context.Context, string, string, logrus.FieldLogger) (velerov1api.BackupRepositoryMaintenanceStatus, error) expectedMaintenanceTime time.Time expectedHistory []velerov1api.BackupRepositoryMaintenanceStatus expectedErr string }{ { name: "not due", repo: &velerov1api.BackupRepository{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "repo", }, Spec: velerov1api.BackupRepositorySpec{ MaintenanceFrequency: metav1.Duration{Duration: time.Hour}, }, Status: velerov1api.BackupRepositoryStatus{ LastMaintenanceTime: &metav1.Time{Time: now}, RecentMaintenance: []velerov1api.BackupRepositoryMaintenanceStatus{ { StartTimestamp: &metav1.Time{Time: now.Add(-time.Hour)}, CompleteTimestamp: &metav1.Time{Time: now}, Result: velerov1api.BackupRepositoryMaintenanceSucceeded, }, }, }, }, expectedMaintenanceTime: now, expectedHistory: []velerov1api.BackupRepositoryMaintenanceStatus{ { StartTimestamp: &metav1.Time{Time: now.Add(-time.Hour)}, CompleteTimestamp: &metav1.Time{Time: now}, Result: velerov1api.BackupRepositoryMaintenanceSucceeded, }, }, }, { name: "start failed", repo: &velerov1api.BackupRepository{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "repo", }, Spec: velerov1api.BackupRepositorySpec{ MaintenanceFrequency: metav1.Duration{Duration: time.Hour}, }, Status: velerov1api.BackupRepositoryStatus{ LastMaintenanceTime: &metav1.Time{Time: now.Add(-time.Hour - time.Minute)}, RecentMaintenance: []velerov1api.BackupRepositoryMaintenanceStatus{ { StartTimestamp: &metav1.Time{Time: now.Add(-time.Hour * 2)}, CompleteTimestamp: &metav1.Time{Time: now.Add(-time.Hour)}, Result: velerov1api.BackupRepositoryMaintenanceSucceeded, }, }, }, }, startJobFunc: startMaintenanceJobFail, expectedMaintenanceTime: now.Add(-time.Hour - time.Minute), expectedHistory: []velerov1api.BackupRepositoryMaintenanceStatus{ { StartTimestamp: &metav1.Time{Time: now.Add(-time.Hour * 2)}, CompleteTimestamp: &metav1.Time{Time: now.Add(-time.Hour)}, Result: velerov1api.BackupRepositoryMaintenanceSucceeded, }, { StartTimestamp: &metav1.Time{Time: now}, Result: velerov1api.BackupRepositoryMaintenanceFailed, Message: "Failed to start maintenance job, err: fake-start-error", }, }, }, { name: "wait failed", repo: &velerov1api.BackupRepository{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "repo", }, Spec: velerov1api.BackupRepositorySpec{ MaintenanceFrequency: metav1.Duration{Duration: time.Hour}, }, Status: velerov1api.BackupRepositoryStatus{ LastMaintenanceTime: &metav1.Time{Time: now.Add(-time.Hour - time.Minute)}, RecentMaintenance: []velerov1api.BackupRepositoryMaintenanceStatus{ { StartTimestamp: &metav1.Time{Time: now.Add(-time.Hour * 2)}, CompleteTimestamp: &metav1.Time{Time: now.Add(-time.Hour)}, Result: velerov1api.BackupRepositoryMaintenanceSucceeded, }, }, }, }, startJobFunc: startMaintenanceJobSucceed, waitJobFunc: waitMaintenanceJobCompleteFail, expectedErr: "error waiting repo maintenance completion status: fake-wait-error", expectedMaintenanceTime: now.Add(-time.Hour - time.Minute), expectedHistory: []velerov1api.BackupRepositoryMaintenanceStatus{ { StartTimestamp: &metav1.Time{Time: now.Add(-time.Hour * 2)}, CompleteTimestamp: &metav1.Time{Time: now.Add(-time.Hour)}, Result: velerov1api.BackupRepositoryMaintenanceSucceeded, }, }, }, { name: "maintenance failed", repo: &velerov1api.BackupRepository{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "repo", }, Spec: velerov1api.BackupRepositorySpec{ MaintenanceFrequency: metav1.Duration{Duration: time.Hour}, }, Status: velerov1api.BackupRepositoryStatus{ LastMaintenanceTime: &metav1.Time{Time: now.Add(-time.Hour - time.Minute)}, RecentMaintenance: []velerov1api.BackupRepositoryMaintenanceStatus{ { StartTimestamp: &metav1.Time{Time: now.Add(-time.Hour * 2)}, CompleteTimestamp: &metav1.Time{Time: now.Add(-time.Hour)}, Result: velerov1api.BackupRepositoryMaintenanceSucceeded, }, }, }, }, startJobFunc: startMaintenanceJobSucceed, waitJobFunc: waitMaintenanceJobCompleteFunc(now, velerov1api.BackupRepositoryMaintenanceFailed, "fake-maintenance-message"), expectedMaintenanceTime: now.Add(-time.Hour - time.Minute), expectedHistory: []velerov1api.BackupRepositoryMaintenanceStatus{ { StartTimestamp: &metav1.Time{Time: now.Add(-time.Hour * 2)}, CompleteTimestamp: &metav1.Time{Time: now.Add(-time.Hour)}, Result: velerov1api.BackupRepositoryMaintenanceSucceeded, }, { StartTimestamp: &metav1.Time{Time: now}, Result: velerov1api.BackupRepositoryMaintenanceFailed, Message: "fake-maintenance-message", }, }, }, { name: "maintenance succeeded", repo: &velerov1api.BackupRepository{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "repo", }, Spec: velerov1api.BackupRepositorySpec{ MaintenanceFrequency: metav1.Duration{Duration: time.Hour}, }, Status: velerov1api.BackupRepositoryStatus{ LastMaintenanceTime: &metav1.Time{Time: now.Add(-time.Hour - time.Minute)}, RecentMaintenance: []velerov1api.BackupRepositoryMaintenanceStatus{ { StartTimestamp: &metav1.Time{Time: now.Add(-time.Hour * 2)}, CompleteTimestamp: &metav1.Time{Time: now.Add(-time.Hour)}, Result: velerov1api.BackupRepositoryMaintenanceSucceeded, }, }, }, }, startJobFunc: startMaintenanceJobSucceed, waitJobFunc: waitMaintenanceJobCompleteFunc(now, velerov1api.BackupRepositoryMaintenanceSucceeded, ""), expectedMaintenanceTime: now.Add(time.Hour), expectedHistory: []velerov1api.BackupRepositoryMaintenanceStatus{ { StartTimestamp: &metav1.Time{Time: now.Add(-time.Hour * 2)}, CompleteTimestamp: &metav1.Time{Time: now.Add(-time.Hour)}, Result: velerov1api.BackupRepositoryMaintenanceSucceeded, }, { StartTimestamp: &metav1.Time{Time: now}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour)}, Result: velerov1api.BackupRepositoryMaintenanceSucceeded, }, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { reconciler := mockBackupRepoReconciler(t, "", test.repo, nil) reconciler.clock = &fakeClock{now} err := reconciler.Client.Create(t.Context(), test.repo) require.NoError(t, err) funcStartMaintenanceJob = test.startJobFunc funcWaitMaintenanceJobComplete = test.waitJobFunc err = reconciler.runMaintenanceIfDue(t.Context(), test.repo, velerotest.NewLogger()) if test.expectedErr == "" { require.NoError(t, err) } assert.Equal(t, test.expectedMaintenanceTime, test.repo.Status.LastMaintenanceTime.Time) assert.Len(t, test.repo.Status.RecentMaintenance, len(test.expectedHistory)) for i := 0; i < len(test.expectedHistory); i++ { assert.Equal(t, test.expectedHistory[i].StartTimestamp.Time, test.repo.Status.RecentMaintenance[i].StartTimestamp.Time) if test.expectedHistory[i].CompleteTimestamp == nil { assert.Nil(t, test.repo.Status.RecentMaintenance[i].CompleteTimestamp) } else { assert.Equal(t, test.expectedHistory[i].CompleteTimestamp.Time, test.repo.Status.RecentMaintenance[i].CompleteTimestamp.Time) } assert.Equal(t, test.expectedHistory[i].Result, test.repo.Status.RecentMaintenance[i].Result) assert.Equal(t, test.expectedHistory[i].Message, test.repo.Status.RecentMaintenance[i].Message) } }) } } func TestInitializeRepo(t *testing.T) { rr := mockBackupRepositoryCR() rr.Spec.BackupStorageLocation = "default" reconciler := mockBackupRepoReconciler(t, "PrepareRepo", rr, nil) err := reconciler.Client.Create(t.Context(), rr) require.NoError(t, err) location := velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Config: map[string]string{"resticRepoPrefix": "s3:test.amazonaws.com/bucket/restic"}, }, ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: rr.Spec.BackupStorageLocation, }, } err = reconciler.initializeRepo(t.Context(), rr, &location, reconciler.logger) require.NoError(t, err) assert.Equal(t, velerov1api.BackupRepositoryPhaseReady, rr.Status.Phase) } func TestBackupRepoReconcile(t *testing.T) { tests := []struct { name string repo *velerov1api.BackupRepository expectNil bool }{ { name: "test on api server not found", repo: &velerov1api.BackupRepository{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "unknown", }, Spec: velerov1api.BackupRepositorySpec{ MaintenanceFrequency: metav1.Duration{Duration: testMaintenanceFrequency}, }, }, expectNil: true, }, { name: "test on initialize repo", repo: &velerov1api.BackupRepository{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "repo", }, Spec: velerov1api.BackupRepositorySpec{ MaintenanceFrequency: metav1.Duration{Duration: testMaintenanceFrequency}, }, }, expectNil: true, }, { name: "test on repo with new phase", repo: &velerov1api.BackupRepository{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "repo", }, Spec: velerov1api.BackupRepositorySpec{ MaintenanceFrequency: metav1.Duration{Duration: testMaintenanceFrequency}, }, Status: velerov1api.BackupRepositoryStatus{ Phase: velerov1api.BackupRepositoryPhaseNew, }, }, expectNil: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { reconciler := mockBackupRepoReconciler(t, "", test.repo, nil) err := reconciler.Client.Create(t.Context(), test.repo) require.NoError(t, err) _, err = reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: test.repo.Namespace, Name: "repo"}}) if test.expectNil { assert.NoError(t, err) } else { assert.Error(t, err) } }) } } func TestGetRepositoryMaintenanceFrequency(t *testing.T) { tests := []struct { name string mgr repotypes.SnapshotIdentifier repo *velerov1api.BackupRepository freqReturn time.Duration freqError error userDefinedFreq time.Duration expectFreq time.Duration }{ { name: "user defined valid", userDefinedFreq: time.Hour, expectFreq: time.Hour, }, { name: "repo return valid", freqReturn: time.Hour * 2, expectFreq: time.Hour * 2, }, { name: "fall to default", userDefinedFreq: -1, freqError: errors.New("fake-error"), expectFreq: defaultMaintainFrequency, }, { name: "fall to default, no freq error", freqReturn: -1, expectFreq: defaultMaintainFrequency, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { mgr := repomokes.Manager{} mgr.On("DefaultMaintenanceFrequency", mock.Anything).Return(test.freqReturn, test.freqError) reconciler := NewBackupRepoReconciler( velerov1api.DefaultNamespace, velerotest.NewLogger(), velerotest.NewFakeControllerRuntimeClient(t), &mgr, test.userDefinedFreq, "", "", logrus.InfoLevel, nil, nil, ) freq := reconciler.getRepositoryMaintenanceFrequency(test.repo) assert.Equal(t, test.expectFreq, freq) }) } } func TestNeedInvalidBackupRepo(t *testing.T) { tests := []struct { name string oldBSL *velerov1api.BackupStorageLocation newBSL *velerov1api.BackupStorageLocation expect bool }{ { name: "no change", oldBSL: &velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Provider: "old-provider", }, }, newBSL: &velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Provider: "new-provider", }, }, expect: false, }, { name: "other part change", oldBSL: &velerov1api.BackupStorageLocation{}, newBSL: &velerov1api.BackupStorageLocation{}, expect: false, }, { name: "bucket change", oldBSL: &velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ StorageType: velerov1api.StorageType{ ObjectStorage: &velerov1api.ObjectStorageLocation{ Bucket: "old-bucket", }, }, }, }, newBSL: &velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ StorageType: velerov1api.StorageType{ ObjectStorage: &velerov1api.ObjectStorageLocation{ Bucket: "new-bucket", }, }, }, }, expect: true, }, { name: "prefix change", oldBSL: &velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ StorageType: velerov1api.StorageType{ ObjectStorage: &velerov1api.ObjectStorageLocation{ Prefix: "old-prefix", }, }, }, }, newBSL: &velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ StorageType: velerov1api.StorageType{ ObjectStorage: &velerov1api.ObjectStorageLocation{ Prefix: "new-prefix", }, }, }, }, expect: true, }, { name: "CACert change", oldBSL: &velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ StorageType: velerov1api.StorageType{ ObjectStorage: &velerov1api.ObjectStorageLocation{ CACert: []byte{0x11, 0x12, 0x13}, }, }, }, }, newBSL: &velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ StorageType: velerov1api.StorageType{ ObjectStorage: &velerov1api.ObjectStorageLocation{ CACert: []byte{0x21, 0x22, 0x23}, }, }, }, }, expect: true, }, { name: "config change", oldBSL: &velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Config: map[string]string{ "key1": "value1", }, }, }, newBSL: &velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Config: map[string]string{ "key2": "value2", }, }, }, expect: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { reconciler := NewBackupRepoReconciler( velerov1api.DefaultNamespace, velerotest.NewLogger(), velerotest.NewFakeControllerRuntimeClient(t), nil, time.Duration(0), "", "", logrus.InfoLevel, nil, nil, ) need := reconciler.needInvalidBackupRepo(test.oldBSL, test.newBSL) assert.Equal(t, test.expect, need) }) } } func TestGetBackupRepositoryConfig(t *testing.T) { configWithNoData := &corev1api.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "config-1", Namespace: velerov1api.DefaultNamespace, }, } configWithWrongData := &corev1api.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "config-1", Namespace: velerov1api.DefaultNamespace, }, Data: map[string]string{ "fake-repo-type": "", }, } configWithData := &corev1api.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "config-1", Namespace: velerov1api.DefaultNamespace, }, Data: map[string]string{ "fake-repo-type": "{\"cacheLimitMB\": 1000, \"enableCompression\": true, \"fullMaintenanceInterval\": \"fastGC\"}", "fake-repo-type-1": "{\"cacheLimitMB\": 1, \"enableCompression\": false}", }, } tests := []struct { name string congiName string repoName string repoType string kubeClientObj []runtime.Object expectedErr string expectedResult map[string]string }{ { name: "empty configName", }, { name: "get error", congiName: "config-1", expectedErr: "error getting configMap config-1: configmaps \"config-1\" not found", }, { name: "no config for repo", congiName: "config-1", repoName: "fake-repo", repoType: "fake-repo-type", kubeClientObj: []runtime.Object{ configWithNoData, }, }, { name: "unmarshall error", congiName: "config-1", repoName: "fake-repo", repoType: "fake-repo-type", kubeClientObj: []runtime.Object{ configWithWrongData, }, expectedErr: "error unmarshalling config data from config-1 for repo fake-repo, repo type fake-repo-type: unexpected end of JSON input", }, { name: "succeed", congiName: "config-1", repoName: "fake-repo", repoType: "fake-repo-type", kubeClientObj: []runtime.Object{ configWithData, }, expectedResult: map[string]string{ "cacheLimitMB": "1000", "enableCompression": "true", "fullMaintenanceInterval": "fastGC", }, }, } scheme := runtime.NewScheme() corev1api.AddToScheme(scheme) for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeClientBuilder := clientFake.NewClientBuilder() fakeClientBuilder = fakeClientBuilder.WithScheme(scheme) fakeClient := fakeClientBuilder.WithRuntimeObjects(test.kubeClientObj...).Build() result, err := getBackupRepositoryConfig(t.Context(), fakeClient, test.congiName, velerov1api.DefaultNamespace, test.repoName, test.repoType, velerotest.NewLogger()) if test.expectedErr != "" { assert.EqualError(t, err, test.expectedErr) } else { require.NoError(t, err) assert.Equal(t, test.expectedResult, result) } }) } } func TestUpdateRepoMaintenanceHistory(t *testing.T) { standardTime := time.Now() backupRepoWithoutHistory := &velerov1api.BackupRepository{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "repo", }, } backupRepoWithHistory := &velerov1api.BackupRepository{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "repo", }, Status: velerov1api.BackupRepositoryStatus{ RecentMaintenance: []velerov1api.BackupRepositoryMaintenanceStatus{ { StartTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 24)}, CompleteTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 23)}, Message: "fake-history-message-1", }, }, }, } backupRepoWithFullHistory := &velerov1api.BackupRepository{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "repo", }, Status: velerov1api.BackupRepositoryStatus{ RecentMaintenance: []velerov1api.BackupRepositoryMaintenanceStatus{ { StartTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 24)}, CompleteTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 23)}, Message: "fake-history-message-2", }, { StartTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 22)}, CompleteTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 21)}, Message: "fake-history-message-3", }, { StartTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 20)}, CompleteTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 19)}, Message: "fake-history-message-4", }, }, }, } backupRepoWithOverFullHistory := &velerov1api.BackupRepository{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "repo", }, Status: velerov1api.BackupRepositoryStatus{ RecentMaintenance: []velerov1api.BackupRepositoryMaintenanceStatus{ { StartTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 24)}, CompleteTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 23)}, Message: "fake-history-message-5", }, { StartTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 22)}, CompleteTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 21)}, Message: "fake-history-message-6", }, { StartTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 20)}, CompleteTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 19)}, Message: "fake-history-message-7", }, { StartTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 18)}, CompleteTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 17)}, Message: "fake-history-message-8", }, }, }, } tests := []struct { name string backupRepo *velerov1api.BackupRepository result velerov1api.BackupRepositoryMaintenanceResult expectedHistory []velerov1api.BackupRepositoryMaintenanceStatus }{ { name: "empty history", backupRepo: backupRepoWithoutHistory, result: velerov1api.BackupRepositoryMaintenanceSucceeded, expectedHistory: []velerov1api.BackupRepositoryMaintenanceStatus{ { StartTimestamp: &metav1.Time{Time: standardTime}, CompleteTimestamp: &metav1.Time{Time: standardTime.Add(time.Hour)}, Message: "fake-message-0", }, }, }, { name: "less than history queue length", backupRepo: backupRepoWithHistory, result: velerov1api.BackupRepositoryMaintenanceSucceeded, expectedHistory: []velerov1api.BackupRepositoryMaintenanceStatus{ { StartTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 24)}, CompleteTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 23)}, Message: "fake-history-message-1", }, { StartTimestamp: &metav1.Time{Time: standardTime}, CompleteTimestamp: &metav1.Time{Time: standardTime.Add(time.Hour)}, Message: "fake-message-0", }, }, }, { name: "full history", backupRepo: backupRepoWithFullHistory, result: velerov1api.BackupRepositoryMaintenanceSucceeded, expectedHistory: []velerov1api.BackupRepositoryMaintenanceStatus{ { StartTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 22)}, CompleteTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 21)}, Message: "fake-history-message-3", }, { StartTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 20)}, CompleteTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 19)}, Message: "fake-history-message-4", }, { StartTimestamp: &metav1.Time{Time: standardTime}, CompleteTimestamp: &metav1.Time{Time: standardTime.Add(time.Hour)}, Message: "fake-message-0", }, }, }, { name: "over full history", backupRepo: backupRepoWithOverFullHistory, result: velerov1api.BackupRepositoryMaintenanceSucceeded, expectedHistory: []velerov1api.BackupRepositoryMaintenanceStatus{ { StartTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 20)}, CompleteTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 19)}, Message: "fake-history-message-7", }, { StartTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 18)}, CompleteTimestamp: &metav1.Time{Time: standardTime.Add(-time.Hour * 17)}, Message: "fake-history-message-8", }, { StartTimestamp: &metav1.Time{Time: standardTime}, CompleteTimestamp: &metav1.Time{Time: standardTime.Add(time.Hour)}, Message: "fake-message-0", }, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { updateRepoMaintenanceHistory(test.backupRepo, test.result, &metav1.Time{Time: standardTime}, &metav1.Time{Time: standardTime.Add(time.Hour)}, "fake-message-0") for at := range test.backupRepo.Status.RecentMaintenance { assert.Equal(t, test.expectedHistory[at].StartTimestamp.Time, test.backupRepo.Status.RecentMaintenance[at].StartTimestamp.Time) assert.Equal(t, test.expectedHistory[at].CompleteTimestamp.Time, test.backupRepo.Status.RecentMaintenance[at].CompleteTimestamp.Time) assert.Equal(t, test.expectedHistory[at].Message, test.backupRepo.Status.RecentMaintenance[at].Message) } }) } } func TestRecallMaintenance(t *testing.T) { now := time.Now().Round(time.Second) schemeFail := runtime.NewScheme() velerov1api.AddToScheme(schemeFail) scheme := runtime.NewScheme() batchv1api.AddToScheme(scheme) corev1api.AddToScheme(scheme) velerov1api.AddToScheme(scheme) jobSucceeded := &batchv1api.Job{ ObjectMeta: metav1.ObjectMeta{ Name: "job1", Namespace: velerov1api.DefaultNamespace, Labels: map[string]string{maintenance.RepositoryNameLabel: "repo"}, CreationTimestamp: metav1.Time{Time: now.Add(time.Hour)}, }, Status: batchv1api.JobStatus{ StartTime: &metav1.Time{Time: now.Add(time.Hour)}, CompletionTime: &metav1.Time{Time: now.Add(time.Hour * 2)}, Succeeded: 1, }, } jobPodSucceeded := builder.ForPod(velerov1api.DefaultNamespace, "job1").Labels(map[string]string{"job-name": "job1"}).ContainerStatuses(&corev1api.ContainerStatus{ State: corev1api.ContainerState{ Terminated: &corev1api.ContainerStateTerminated{}, }, }).Result() tests := []struct { name string kubeClientObj []runtime.Object runtimeScheme *runtime.Scheme repoLastMatainTime metav1.Time expectNewHistory []velerov1api.BackupRepositoryMaintenanceStatus expectTimeUpdate *metav1.Time expectedErr string }{ { name: "wait completion error", runtimeScheme: schemeFail, expectedErr: "error waiting incomplete repo maintenance job for repo repo: error listing maintenance job for repo repo: no kind is registered for the type v1.JobList in scheme", }, { name: "no consolidate result", runtimeScheme: scheme, }, { name: "no update last time", runtimeScheme: scheme, kubeClientObj: []runtime.Object{ jobSucceeded, jobPodSucceeded, }, repoLastMatainTime: metav1.Time{Time: now.Add(time.Hour * 5)}, expectNewHistory: []velerov1api.BackupRepositoryMaintenanceStatus{ { StartTimestamp: &metav1.Time{Time: now.Add(time.Hour)}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour * 2)}, Result: velerov1api.BackupRepositoryMaintenanceSucceeded, }, }, }, { name: "update last time", runtimeScheme: scheme, kubeClientObj: []runtime.Object{ jobSucceeded, jobPodSucceeded, }, expectNewHistory: []velerov1api.BackupRepositoryMaintenanceStatus{ { StartTimestamp: &metav1.Time{Time: now.Add(time.Hour)}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour * 2)}, Result: velerov1api.BackupRepositoryMaintenanceSucceeded, }, }, expectTimeUpdate: &metav1.Time{Time: now.Add(time.Hour * 2)}, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { r := mockBackupRepoReconciler(t, "", nil, nil) backupRepo := mockBackupRepositoryCR() backupRepo.Status.LastMaintenanceTime = &test.repoLastMatainTime test.kubeClientObj = append(test.kubeClientObj, backupRepo) fakeClientBuilder := clientFake.NewClientBuilder() fakeClientBuilder = fakeClientBuilder.WithScheme(test.runtimeScheme) fakeClient := fakeClientBuilder.WithRuntimeObjects(test.kubeClientObj...).Build() r.Client = fakeClient lastTm := backupRepo.Status.LastMaintenanceTime err := r.recallMaintenance(t.Context(), backupRepo, velerotest.NewLogger()) if test.expectedErr != "" { assert.ErrorContains(t, err, test.expectedErr) } else { assert.NoError(t, err) if test.expectNewHistory == nil { assert.Nil(t, backupRepo.Status.RecentMaintenance) } else { assert.Len(t, backupRepo.Status.RecentMaintenance, len(test.expectNewHistory)) for i := 0; i < len(test.expectNewHistory); i++ { assert.Equal(t, test.expectNewHistory[i].StartTimestamp.Time, backupRepo.Status.RecentMaintenance[i].StartTimestamp.Time) assert.Equal(t, test.expectNewHistory[i].CompleteTimestamp.Time, backupRepo.Status.RecentMaintenance[i].CompleteTimestamp.Time) assert.Equal(t, test.expectNewHistory[i].Result, backupRepo.Status.RecentMaintenance[i].Result) assert.Equal(t, test.expectNewHistory[i].Message, backupRepo.Status.RecentMaintenance[i].Message) } } if test.expectTimeUpdate != nil { assert.Equal(t, test.expectTimeUpdate.Time, backupRepo.Status.LastMaintenanceTime.Time) } else { assert.Equal(t, lastTm, backupRepo.Status.LastMaintenanceTime) } } }) } } func TestConsolidateHistory(t *testing.T) { now := time.Now() tests := []struct { name string cur []velerov1api.BackupRepositoryMaintenanceStatus coming []velerov1api.BackupRepositoryMaintenanceStatus expected []velerov1api.BackupRepositoryMaintenanceStatus }{ { name: "zero coming", cur: []velerov1api.BackupRepositoryMaintenanceStatus{ { StartTimestamp: &metav1.Time{Time: now}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour)}, Result: velerov1api.BackupRepositoryMaintenanceSucceeded, Message: "fake-maintenance-message", }, }, expected: nil, }, { name: "identical coming", cur: []velerov1api.BackupRepositoryMaintenanceStatus{ { StartTimestamp: &metav1.Time{Time: now}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour)}, Result: velerov1api.BackupRepositoryMaintenanceSucceeded, Message: "fake-maintenance-message", }, }, coming: []velerov1api.BackupRepositoryMaintenanceStatus{ { StartTimestamp: &metav1.Time{Time: now}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour)}, }, }, expected: nil, }, { name: "less than limit", cur: []velerov1api.BackupRepositoryMaintenanceStatus{ { StartTimestamp: &metav1.Time{Time: now}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour)}, Result: velerov1api.BackupRepositoryMaintenanceSucceeded, Message: "fake-maintenance-message", }, { StartTimestamp: &metav1.Time{Time: now.Add(time.Hour)}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour * 2)}, Result: velerov1api.BackupRepositoryMaintenanceSucceeded, Message: "fake-maintenance-message-2", }, }, coming: []velerov1api.BackupRepositoryMaintenanceStatus{ { StartTimestamp: &metav1.Time{Time: now.Add(time.Hour * 2)}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour * 3)}, Result: velerov1api.BackupRepositoryMaintenanceSucceeded, Message: "fake-maintenance-message-3", }, }, expected: []velerov1api.BackupRepositoryMaintenanceStatus{ { StartTimestamp: &metav1.Time{Time: now}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour)}, Result: velerov1api.BackupRepositoryMaintenanceSucceeded, Message: "fake-maintenance-message", }, { StartTimestamp: &metav1.Time{Time: now.Add(time.Hour)}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour * 2)}, Result: velerov1api.BackupRepositoryMaintenanceSucceeded, Message: "fake-maintenance-message-2", }, { StartTimestamp: &metav1.Time{Time: now.Add(time.Hour * 2)}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour * 3)}, Result: velerov1api.BackupRepositoryMaintenanceSucceeded, Message: "fake-maintenance-message-3", }, }, }, { name: "more than limit", cur: []velerov1api.BackupRepositoryMaintenanceStatus{ { StartTimestamp: &metav1.Time{Time: now}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour)}, Result: velerov1api.BackupRepositoryMaintenanceSucceeded, Message: "fake-maintenance-message", }, { StartTimestamp: &metav1.Time{Time: now.Add(time.Hour)}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour * 2)}, Result: velerov1api.BackupRepositoryMaintenanceSucceeded, Message: "fake-maintenance-message-2", }, }, coming: []velerov1api.BackupRepositoryMaintenanceStatus{ { StartTimestamp: &metav1.Time{Time: now.Add(time.Hour * 2)}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour * 3)}, Result: velerov1api.BackupRepositoryMaintenanceSucceeded, Message: "fake-maintenance-message-3", }, { StartTimestamp: &metav1.Time{Time: now.Add(time.Hour * 3)}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour * 4)}, Result: velerov1api.BackupRepositoryMaintenanceSucceeded, Message: "fake-maintenance-message-4", }, }, expected: []velerov1api.BackupRepositoryMaintenanceStatus{ { StartTimestamp: &metav1.Time{Time: now.Add(time.Hour)}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour * 2)}, Result: velerov1api.BackupRepositoryMaintenanceSucceeded, Message: "fake-maintenance-message-2", }, { StartTimestamp: &metav1.Time{Time: now.Add(time.Hour * 2)}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour * 3)}, Result: velerov1api.BackupRepositoryMaintenanceSucceeded, Message: "fake-maintenance-message-3", }, { StartTimestamp: &metav1.Time{Time: now.Add(time.Hour * 3)}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour * 4)}, Result: velerov1api.BackupRepositoryMaintenanceSucceeded, Message: "fake-maintenance-message-4", }, }, }, { name: "more than limit 2", cur: []velerov1api.BackupRepositoryMaintenanceStatus{ { StartTimestamp: &metav1.Time{Time: now}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour)}, Result: velerov1api.BackupRepositoryMaintenanceSucceeded, Message: "fake-maintenance-message", }, { StartTimestamp: &metav1.Time{Time: now.Add(time.Hour)}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour * 2)}, Result: velerov1api.BackupRepositoryMaintenanceSucceeded, Message: "fake-maintenance-message-2", }, { StartTimestamp: &metav1.Time{Time: now.Add(time.Hour * 2)}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour * 3)}, Result: velerov1api.BackupRepositoryMaintenanceSucceeded, Message: "fake-maintenance-message-3", }, }, coming: []velerov1api.BackupRepositoryMaintenanceStatus{ { StartTimestamp: &metav1.Time{Time: now.Add(time.Hour)}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour * 2)}, Result: velerov1api.BackupRepositoryMaintenanceSucceeded, Message: "fake-maintenance-message-2", }, { StartTimestamp: &metav1.Time{Time: now.Add(time.Hour * 2)}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour * 3)}, Result: velerov1api.BackupRepositoryMaintenanceSucceeded, Message: "fake-maintenance-message-3", }, { StartTimestamp: &metav1.Time{Time: now.Add(time.Hour * 3)}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour * 4)}, Result: velerov1api.BackupRepositoryMaintenanceSucceeded, Message: "fake-maintenance-message-4", }, }, expected: []velerov1api.BackupRepositoryMaintenanceStatus{ { StartTimestamp: &metav1.Time{Time: now.Add(time.Hour)}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour * 2)}, Result: velerov1api.BackupRepositoryMaintenanceSucceeded, Message: "fake-maintenance-message-2", }, { StartTimestamp: &metav1.Time{Time: now.Add(time.Hour * 2)}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour * 3)}, Result: velerov1api.BackupRepositoryMaintenanceSucceeded, Message: "fake-maintenance-message-3", }, { StartTimestamp: &metav1.Time{Time: now.Add(time.Hour * 3)}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour * 4)}, Result: velerov1api.BackupRepositoryMaintenanceSucceeded, Message: "fake-maintenance-message-4", }, }, }, { name: "coming is not used", cur: []velerov1api.BackupRepositoryMaintenanceStatus{ { StartTimestamp: &metav1.Time{Time: now.Add(time.Hour)}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour * 2)}, Result: velerov1api.BackupRepositoryMaintenanceSucceeded, Message: "fake-maintenance-message", }, { StartTimestamp: &metav1.Time{Time: now.Add(time.Hour * 2)}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour * 3)}, Result: velerov1api.BackupRepositoryMaintenanceSucceeded, Message: "fake-maintenance-message-3", }, { StartTimestamp: &metav1.Time{Time: now.Add(time.Hour * 3)}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour * 4)}, Result: velerov1api.BackupRepositoryMaintenanceSucceeded, Message: "fake-maintenance-message-4", }, }, coming: []velerov1api.BackupRepositoryMaintenanceStatus{ { StartTimestamp: &metav1.Time{Time: now}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour)}, }, }, expected: nil, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { consolidated := consolidateHistory(test.coming, test.cur) if test.expected == nil { assert.Nil(t, consolidated) } else { assert.Len(t, consolidated, len(test.expected)) for i := 0; i < len(test.expected); i++ { assert.Equal(t, *test.expected[i].StartTimestamp, *consolidated[i].StartTimestamp) assert.Equal(t, *test.expected[i].CompleteTimestamp, *consolidated[i].CompleteTimestamp) assert.Equal(t, test.expected[i].Result, consolidated[i].Result) assert.Equal(t, test.expected[i].Message, consolidated[i].Message) } assert.Nil(t, consolidateHistory(test.coming, consolidated)) } }) } } func TestGetLastMaintenanceTimeFromHistory(t *testing.T) { now := time.Now() tests := []struct { name string history []velerov1api.BackupRepositoryMaintenanceStatus expected time.Time }{ { name: "first one is nil", history: []velerov1api.BackupRepositoryMaintenanceStatus{ { StartTimestamp: &metav1.Time{Time: now}, Result: velerov1api.BackupRepositoryMaintenanceFailed, Message: "fake-maintenance-message", }, { StartTimestamp: &metav1.Time{Time: now.Add(time.Hour)}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour * 2)}, Result: velerov1api.BackupRepositoryMaintenanceSucceeded, Message: "fake-maintenance-message-2", }, { StartTimestamp: &metav1.Time{Time: now.Add(time.Hour * 2)}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour * 3)}, Result: velerov1api.BackupRepositoryMaintenanceSucceeded, Message: "fake-maintenance-message-3", }, }, expected: now.Add(time.Hour * 3), }, { name: "another one is nil", history: []velerov1api.BackupRepositoryMaintenanceStatus{ { StartTimestamp: &metav1.Time{Time: now}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour)}, Result: velerov1api.BackupRepositoryMaintenanceSucceeded, Message: "fake-maintenance-message", }, { StartTimestamp: &metav1.Time{Time: now.Add(time.Hour)}, Result: velerov1api.BackupRepositoryMaintenanceFailed, Message: "fake-maintenance-message-2", }, { StartTimestamp: &metav1.Time{Time: now.Add(time.Hour * 2)}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour * 3)}, Result: velerov1api.BackupRepositoryMaintenanceSucceeded, Message: "fake-maintenance-message-3", }, }, expected: now.Add(time.Hour * 3), }, { name: "disordered", history: []velerov1api.BackupRepositoryMaintenanceStatus{ { StartTimestamp: &metav1.Time{Time: now.Add(time.Hour * 2)}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour * 3)}, Result: velerov1api.BackupRepositoryMaintenanceSucceeded, Message: "fake-maintenance-message-3", }, { StartTimestamp: &metav1.Time{Time: now}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour)}, Result: velerov1api.BackupRepositoryMaintenanceSucceeded, Message: "fake-maintenance-message", }, { StartTimestamp: &metav1.Time{Time: now.Add(time.Hour)}, Result: velerov1api.BackupRepositoryMaintenanceFailed, Message: "fake-maintenance-message-2", }, }, expected: now.Add(time.Hour * 3), }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { time := getLastMaintenanceTimeFromHistory(test.history) assert.Equal(t, test.expected, time.Time) }) } } func TestDeleteOldMaintenanceJobWithConfigMap(t *testing.T) { tests := []struct { name string repo *velerov1api.BackupRepository expectedKeptJobs int maintenanceJobs []batchv1api.Job bsl *velerov1api.BackupStorageLocation repoMaintenanceJob *corev1api.ConfigMap }{ { name: "test with global config", repo: &velerov1api.BackupRepository{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "repo", }, Spec: velerov1api.BackupRepositorySpec{ MaintenanceFrequency: metav1.Duration{Duration: testMaintenanceFrequency}, BackupStorageLocation: "default", VolumeNamespace: "test-ns", RepositoryType: "restic", }, Status: velerov1api.BackupRepositoryStatus{ Phase: velerov1api.BackupRepositoryPhaseReady, }, }, expectedKeptJobs: 5, maintenanceJobs: []batchv1api.Job{ *builder.ForJob("velero", "job-01").ObjectMeta(builder.WithLabels(repomaintenance.RepositoryNameLabel, "repo")).Succeeded(1).Result(), *builder.ForJob("velero", "job-02").ObjectMeta(builder.WithLabels(repomaintenance.RepositoryNameLabel, "repo")).Succeeded(1).Result(), *builder.ForJob("velero", "job-03").ObjectMeta(builder.WithLabels(repomaintenance.RepositoryNameLabel, "repo")).Succeeded(1).Result(), *builder.ForJob("velero", "job-04").ObjectMeta(builder.WithLabels(repomaintenance.RepositoryNameLabel, "repo")).Succeeded(1).Result(), *builder.ForJob("velero", "job-05").ObjectMeta(builder.WithLabels(repomaintenance.RepositoryNameLabel, "repo")).Succeeded(1).Result(), *builder.ForJob("velero", "job-06").ObjectMeta(builder.WithLabels(repomaintenance.RepositoryNameLabel, "repo")).Succeeded(1).Result(), }, bsl: builder.ForBackupStorageLocation("velero", "default").Result(), repoMaintenanceJob: &corev1api.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "repo-maintenance-job-config", }, Data: map[string]string{ "global": `{"keepLatestMaintenanceJobs": 5}`, }, }, }, { name: "test with specific repo config overriding global", repo: &velerov1api.BackupRepository{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "repo", }, Spec: velerov1api.BackupRepositorySpec{ MaintenanceFrequency: metav1.Duration{Duration: testMaintenanceFrequency}, BackupStorageLocation: "default", VolumeNamespace: "test-ns", RepositoryType: "restic", }, Status: velerov1api.BackupRepositoryStatus{ Phase: velerov1api.BackupRepositoryPhaseReady, }, }, expectedKeptJobs: 2, maintenanceJobs: []batchv1api.Job{ *builder.ForJob("velero", "job-01").ObjectMeta(builder.WithLabels(repomaintenance.RepositoryNameLabel, "repo")).Succeeded(1).Result(), *builder.ForJob("velero", "job-02").ObjectMeta(builder.WithLabels(repomaintenance.RepositoryNameLabel, "repo")).Succeeded(1).Result(), *builder.ForJob("velero", "job-03").ObjectMeta(builder.WithLabels(repomaintenance.RepositoryNameLabel, "repo")).Succeeded(1).Result(), }, bsl: builder.ForBackupStorageLocation("velero", "default").Result(), repoMaintenanceJob: &corev1api.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "repo-maintenance-job-config", }, Data: map[string]string{ "global": `{"keepLatestMaintenanceJobs": 5}`, "test-ns-default-restic": `{"keepLatestMaintenanceJobs": 2}`, }, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { objects := []runtime.Object{test.repo, test.bsl} if test.repoMaintenanceJob != nil { objects = append(objects, test.repoMaintenanceJob) } crClient := velerotest.NewFakeControllerRuntimeClient(t, objects...) for _, job := range test.maintenanceJobs { require.NoError(t, crClient.Create(t.Context(), &job)) } repoLocker := repository.NewRepoLocker() mgr := repomanager.NewManager("", crClient, repoLocker, nil, nil, nil) repoMaintenanceConfigName := "" if test.repoMaintenanceJob != nil { repoMaintenanceConfigName = test.repoMaintenanceJob.Name } reconciler := NewBackupRepoReconciler( velerov1api.DefaultNamespace, velerotest.NewLogger(), crClient, mgr, time.Duration(0), "", repoMaintenanceConfigName, logrus.InfoLevel, nil, nil, ) _, err := reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: test.repo.Namespace, Name: "repo"}}) require.NoError(t, err) jobList := new(batchv1api.JobList) require.NoError(t, reconciler.Client.List(t.Context(), jobList, &client.ListOptions{Namespace: "velero"})) assert.Len(t, jobList.Items, test.expectedKeptJobs, "Expected %d jobs to be kept, but got %d", test.expectedKeptJobs, len(jobList.Items)) }) } } func TestInitializeRepoWithRepositoryTypes(t *testing.T) { scheme := runtime.NewScheme() corev1api.AddToScheme(scheme) velerov1api.AddToScheme(scheme) // Test for restic repository t.Run("restic repository", func(t *testing.T) { rr := mockBackupRepositoryCR() rr.Spec.BackupStorageLocation = "default" rr.Spec.VolumeNamespace = "volume-ns-1" rr.Spec.RepositoryType = velerov1api.BackupRepositoryTypeRestic location := &velerov1api.BackupStorageLocation{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "default", }, Spec: velerov1api.BackupStorageLocationSpec{ Provider: "aws", StorageType: velerov1api.StorageType{ ObjectStorage: &velerov1api.ObjectStorageLocation{ Bucket: "test-bucket", Prefix: "test-prefix", }, }, Config: map[string]string{ "region": "us-east-1", }, }, } fakeClient := clientFake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(rr, location).Build() mgr := &repomokes.Manager{} mgr.On("PrepareRepo", rr).Return(nil) reconciler := NewBackupRepoReconciler( velerov1api.DefaultNamespace, velerotest.NewLogger(), fakeClient, mgr, testMaintenanceFrequency, "", "", logrus.InfoLevel, nil, nil, ) err := reconciler.initializeRepo(t.Context(), rr, location, reconciler.logger) require.NoError(t, err) // Verify ResticIdentifier is set for restic assert.NotEmpty(t, rr.Spec.ResticIdentifier) assert.Contains(t, rr.Spec.ResticIdentifier, "volume-ns-1") assert.Equal(t, velerov1api.BackupRepositoryPhaseReady, rr.Status.Phase) }) // Test for kopia repository t.Run("kopia repository", func(t *testing.T) { rr := mockBackupRepositoryCR() rr.Spec.BackupStorageLocation = "default" rr.Spec.VolumeNamespace = "volume-ns-1" rr.Spec.RepositoryType = velerov1api.BackupRepositoryTypeKopia location := &velerov1api.BackupStorageLocation{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "default", }, Spec: velerov1api.BackupStorageLocationSpec{ Provider: "aws", StorageType: velerov1api.StorageType{ ObjectStorage: &velerov1api.ObjectStorageLocation{ Bucket: "test-bucket", Prefix: "test-prefix", }, }, Config: map[string]string{ "region": "us-east-1", }, }, } fakeClient := clientFake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(rr, location).Build() mgr := &repomokes.Manager{} mgr.On("PrepareRepo", rr).Return(nil) reconciler := NewBackupRepoReconciler( velerov1api.DefaultNamespace, velerotest.NewLogger(), fakeClient, mgr, testMaintenanceFrequency, "", "", logrus.InfoLevel, nil, nil, ) err := reconciler.initializeRepo(t.Context(), rr, location, reconciler.logger) require.NoError(t, err) // Verify ResticIdentifier is NOT set for kopia assert.Empty(t, rr.Spec.ResticIdentifier) assert.Equal(t, velerov1api.BackupRepositoryPhaseReady, rr.Status.Phase) }) // Test for empty repository type (defaults to restic) t.Run("empty repository type", func(t *testing.T) { rr := mockBackupRepositoryCR() rr.Spec.BackupStorageLocation = "default" rr.Spec.VolumeNamespace = "volume-ns-1" // Leave RepositoryType empty location := &velerov1api.BackupStorageLocation{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "default", }, Spec: velerov1api.BackupStorageLocationSpec{ Provider: "aws", StorageType: velerov1api.StorageType{ ObjectStorage: &velerov1api.ObjectStorageLocation{ Bucket: "test-bucket", Prefix: "test-prefix", }, }, Config: map[string]string{ "region": "us-east-1", }, }, } fakeClient := clientFake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(rr, location).Build() mgr := &repomokes.Manager{} mgr.On("PrepareRepo", rr).Return(nil) reconciler := NewBackupRepoReconciler( velerov1api.DefaultNamespace, velerotest.NewLogger(), fakeClient, mgr, testMaintenanceFrequency, "", "", logrus.InfoLevel, nil, nil, ) err := reconciler.initializeRepo(t.Context(), rr, location, reconciler.logger) require.NoError(t, err) // Verify ResticIdentifier is set when type is empty (defaults to restic) assert.NotEmpty(t, rr.Spec.ResticIdentifier) assert.Contains(t, rr.Spec.ResticIdentifier, "volume-ns-1") assert.Equal(t, velerov1api.BackupRepositoryPhaseReady, rr.Status.Phase) }) } func TestRepoMaintenanceMetricsRecording(t *testing.T) { now := time.Now().Round(time.Second) tests := []struct { name string repo *velerov1api.BackupRepository startJobFunc func(client.Client, context.Context, *velerov1api.BackupRepository, string, logrus.Level, *logging.FormatFlag, logrus.FieldLogger) (string, error) waitJobFunc func(client.Client, context.Context, string, string, logrus.FieldLogger) (velerov1api.BackupRepositoryMaintenanceStatus, error) expectSuccess bool expectFailure bool expectDuration bool }{ { name: "metrics recorded on successful maintenance", repo: &velerov1api.BackupRepository{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "test-repo-success", }, Spec: velerov1api.BackupRepositorySpec{ MaintenanceFrequency: metav1.Duration{Duration: time.Hour}, }, Status: velerov1api.BackupRepositoryStatus{ LastMaintenanceTime: &metav1.Time{Time: now.Add(-2 * time.Hour)}, }, }, startJobFunc: startMaintenanceJobSucceed, waitJobFunc: waitMaintenanceJobCompleteFunc(now, velerov1api.BackupRepositoryMaintenanceSucceeded, ""), expectSuccess: true, expectFailure: false, expectDuration: true, }, { name: "metrics recorded on failed maintenance", repo: &velerov1api.BackupRepository{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "test-repo-failure", }, Spec: velerov1api.BackupRepositorySpec{ MaintenanceFrequency: metav1.Duration{Duration: time.Hour}, }, Status: velerov1api.BackupRepositoryStatus{ LastMaintenanceTime: &metav1.Time{Time: now.Add(-2 * time.Hour)}, }, }, startJobFunc: startMaintenanceJobSucceed, waitJobFunc: func(client.Client, context.Context, string, string, logrus.FieldLogger) (velerov1api.BackupRepositoryMaintenanceStatus, error) { return velerov1api.BackupRepositoryMaintenanceStatus{ StartTimestamp: &metav1.Time{Time: now}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Minute)}, // Job ran for 1 minute then failed Result: velerov1api.BackupRepositoryMaintenanceFailed, Message: "test error", }, nil }, expectSuccess: false, expectFailure: true, expectDuration: true, }, { name: "metrics recorded on job start failure", repo: &velerov1api.BackupRepository{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "test-repo-start-fail", }, Spec: velerov1api.BackupRepositorySpec{ MaintenanceFrequency: metav1.Duration{Duration: time.Hour}, }, Status: velerov1api.BackupRepositoryStatus{ LastMaintenanceTime: &metav1.Time{Time: now.Add(-2 * time.Hour)}, }, }, startJobFunc: startMaintenanceJobFail, expectSuccess: false, expectFailure: true, expectDuration: false, // No duration when job fails to start }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { // Create metrics instance m := metrics.NewServerMetrics() // Create reconciler with metrics reconciler := mockBackupRepoReconciler(t, "", test.repo, nil) reconciler.metrics = m reconciler.clock = &fakeClock{now} err := reconciler.Client.Create(t.Context(), test.repo) require.NoError(t, err) // Set up job functions funcStartMaintenanceJob = test.startJobFunc funcWaitMaintenanceJobComplete = test.waitJobFunc // Run maintenance _ = reconciler.runMaintenanceIfDue(t.Context(), test.repo, velerotest.NewLogger()) // Verify metrics were recorded successCount := getMaintenanceMetricValue(t, m, "repo_maintenance_success_total", test.repo.Name) failureCount := getMaintenanceMetricValue(t, m, "repo_maintenance_failure_total", test.repo.Name) durationCount := getMaintenanceDurationCount(t, m, test.repo.Name) if test.expectSuccess { assert.Equal(t, float64(1), successCount, "Success metric should be recorded") } else { assert.Equal(t, float64(0), successCount, "Success metric should not be recorded") } if test.expectFailure { assert.Equal(t, float64(1), failureCount, "Failure metric should be recorded") } else { assert.Equal(t, float64(0), failureCount, "Failure metric should not be recorded") } if test.expectDuration { assert.Equal(t, uint64(1), durationCount, "Duration metric should be recorded") } else { assert.Equal(t, uint64(0), durationCount, "Duration metric should not be recorded") } }) } } // Helper to get maintenance metric value from ServerMetrics func getMaintenanceMetricValue(t *testing.T, m *metrics.ServerMetrics, metricName, repoName string) float64 { t.Helper() metricMap := m.Metrics() collector, ok := metricMap[metricName] if !ok { return 0 } ch := make(chan prometheus.Metric, 1) collector.Collect(ch) close(ch) for metric := range ch { dto := &dto.Metric{} err := metric.Write(dto) require.NoError(t, err) for _, label := range dto.Label { if *label.Name == "repository_name" && *label.Value == repoName { if dto.Counter != nil { return *dto.Counter.Value } } } } return 0 } // Helper to get maintenance duration histogram count func getMaintenanceDurationCount(t *testing.T, m *metrics.ServerMetrics, repoName string) uint64 { t.Helper() metricMap := m.Metrics() collector, ok := metricMap["repo_maintenance_duration_seconds"] if !ok { return 0 } ch := make(chan prometheus.Metric, 1) collector.Collect(ch) close(ch) for metric := range ch { dto := &dto.Metric{} err := metric.Write(dto) require.NoError(t, err) for _, label := range dto.Label { if *label.Name == "repository_name" && *label.Value == repoName { if dto.Histogram != nil { return *dto.Histogram.SampleCount } } } } return 0 } ================================================ FILE: pkg/controller/backup_storage_location_controller.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package controller import ( "context" "regexp" "strings" "time" "github.com/vmware-tanzu/velero/pkg/metrics" "github.com/pkg/errors" "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/predicate" "github.com/vmware-tanzu/velero/internal/storage" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/constant" "github.com/vmware-tanzu/velero/pkg/persistence" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" "github.com/vmware-tanzu/velero/pkg/util/kube" ) const ( // keep the enqueue period a smaller value to make sure the BSL can be validated as expected. // The BSL validation frequency is 1 minute by default, if we set the enqueue period as 1 minute, // this will cause the actual validation interval for each BSL to be 2 minutes bslValidationEnqueuePeriod = 10 * time.Second ) // sanitizeStorageError cleans up verbose HTTP responses from cloud provider errors, // particularly Azure which includes full HTTP response details and XML in error messages. // It extracts the error code and message while removing HTTP headers and response bodies. // It also scrubs sensitive information like SAS tokens from URLs. func sanitizeStorageError(err error) string { if err == nil { return "" } errMsg := err.Error() // Scrub sensitive information from URLs (SAS tokens, credentials, etc.) // Azure SAS token parameters: sig, se, st, sp, spr, sv, sr, sip, srt, ss // These appear as query parameters in URLs like: ?sig=value&se=value sasParamsRegex := regexp.MustCompile(`([?&])(sig|se|st|sp|spr|sv|sr|sip|srt|ss)=([^&\s<>\n]+)`) errMsg = sasParamsRegex.ReplaceAllString(errMsg, `${1}${2}=***REDACTED***`) // Check if this looks like an Azure HTTP response error // Azure errors contain patterns like "RESPONSE 404:" and "ERROR CODE:" if !strings.Contains(errMsg, "RESPONSE") || !strings.Contains(errMsg, "ERROR CODE:") { // Not an Azure-style error, return as-is return errMsg } // Extract the error code (e.g., "ContainerNotFound", "BlobNotFound") errorCodeRegex := regexp.MustCompile(`ERROR CODE:\s*(\w+)`) errorCodeMatch := errorCodeRegex.FindStringSubmatch(errMsg) var errorCode string if len(errorCodeMatch) > 1 { errorCode = errorCodeMatch[1] } // Extract the error message from the XML or plain text // Look for message between tags or after "RESPONSE XXX:" var errorMessage string // Try to extract from XML first messageRegex := regexp.MustCompile(`(.*?)`) messageMatch := messageRegex.FindStringSubmatch(errMsg) if len(messageMatch) > 1 { errorMessage = messageMatch[1] // Remove RequestId and Time from the message if idx := strings.Index(errorMessage, "\nRequestId:"); idx != -1 { errorMessage = errorMessage[:idx] } } else { // Try to extract from plain text response (e.g., "RESPONSE 404: 404 The specified container does not exist.") responseRegex := regexp.MustCompile(`RESPONSE\s+\d+:\s+\d+\s+([^\n]+)`) responseMatch := responseRegex.FindStringSubmatch(errMsg) if len(responseMatch) > 1 { errorMessage = strings.TrimSpace(responseMatch[1]) } } // Build a clean error message var cleanMsg string if errorCode != "" && errorMessage != "" { cleanMsg = errorCode + ": " + errorMessage } else if errorCode != "" { cleanMsg = errorCode } else if errorMessage != "" { cleanMsg = errorMessage } else { // Fallback: try to extract the desc part from gRPC error descRegex := regexp.MustCompile(`desc\s*=\s*(.+)`) descMatch := descRegex.FindStringSubmatch(errMsg) if len(descMatch) > 1 { // Take everything up to the first newline or "RESPONSE" marker desc := descMatch[1] if idx := strings.Index(desc, "\n"); idx != -1 { desc = desc[:idx] } if idx := strings.Index(desc, "RESPONSE"); idx != -1 { desc = strings.TrimSpace(desc[:idx]) } cleanMsg = desc } else { // Last resort: return first line if idx := strings.Index(errMsg, "\n"); idx != -1 { cleanMsg = errMsg[:idx] } else { cleanMsg = errMsg } } } // Preserve the prefix part of the error (e.g., "rpc error: code = Unknown desc = ") // but replace the verbose description with our clean message if strings.Contains(errMsg, "desc = ") { parts := strings.SplitN(errMsg, "desc = ", 2) if len(parts) == 2 { return parts[0] + "desc = " + cleanMsg } } return cleanMsg } // BackupStorageLocationReconciler reconciles a BackupStorageLocation object type backupStorageLocationReconciler struct { ctx context.Context client client.Client defaultBackupLocationInfo storage.DefaultBackupLocationInfo // use variables to refer to these functions so they can be // replaced with fakes for testing. newPluginManager func(logrus.FieldLogger) clientmgmt.Manager backupStoreGetter persistence.ObjectBackupStoreGetter metrics *metrics.ServerMetrics log logrus.FieldLogger } // NewBackupStorageLocationReconciler initialize and return a backupStorageLocationReconciler struct func NewBackupStorageLocationReconciler( ctx context.Context, client client.Client, defaultBackupLocationInfo storage.DefaultBackupLocationInfo, newPluginManager func(logrus.FieldLogger) clientmgmt.Manager, backupStoreGetter persistence.ObjectBackupStoreGetter, metrics *metrics.ServerMetrics, log logrus.FieldLogger) *backupStorageLocationReconciler { return &backupStorageLocationReconciler{ ctx: ctx, client: client, defaultBackupLocationInfo: defaultBackupLocationInfo, newPluginManager: newPluginManager, backupStoreGetter: backupStoreGetter, metrics: metrics, log: log, } } // +kubebuilder:rbac:groups=velero.io,resources=backupstoragelocations,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=velero.io,resources=backupstoragelocations/status,verbs=get;update;patch func (r *backupStorageLocationReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { var unavailableErrors []string var location velerov1api.BackupStorageLocation log := r.log.WithField("controller", constant.ControllerBackupStorageLocation).WithField(constant.ControllerBackupStorageLocation, req.NamespacedName.String()) log.Debug("Validating availability of BackupStorageLocation") locationList, err := storage.ListBackupStorageLocations(r.ctx, r.client, req.Namespace) if err != nil { log.WithError(err).Error("No BackupStorageLocations found, at least one is required") return ctrl.Result{}, nil } pluginManager := r.newPluginManager(log) defer pluginManager.CleanupClients() // find the BSL that matches the request for _, bsl := range locationList.Items { if bsl.Name == req.Name && bsl.Namespace == req.Namespace { location = bsl } } if location.Name == "" || location.Namespace == "" { log.WithError(err).Error("BackupStorageLocation is not found") return ctrl.Result{}, nil } // decide the default BSL defaultFound, err := r.ensureSingleDefaultBSL(locationList) if err != nil { log.WithError(err).Error("failed to ensure single default bsl") return ctrl.Result{}, nil } func() { var err error original := location.DeepCopy() defer func() { location.Status.LastValidationTime = &metav1.Time{Time: time.Now().UTC()} if err != nil { log.Info("BackupStorageLocation is invalid, marking as unavailable") err = errors.Wrapf(err, "BackupStorageLocation %q is unavailable", location.Name) unavailableErrors = append(unavailableErrors, sanitizeStorageError(err)) location.Status.Phase = velerov1api.BackupStorageLocationPhaseUnavailable location.Status.Message = sanitizeStorageError(err) } else { log.Info("BackupStorageLocations is valid, marking as available") location.Status.Phase = velerov1api.BackupStorageLocationPhaseAvailable location.Status.Message = "" } if err := r.client.Patch(r.ctx, &location, client.MergeFrom(original)); err != nil { log.WithError(err).Error("Error updating BackupStorageLocation phase") } }() // Validate the BackupStorageLocation spec if err = location.Validate(); err != nil { log.WithError(err).Error("BackupStorageLocation spec is invalid") return } backupStore, err := r.backupStoreGetter.Get(&location, pluginManager, log) if err != nil { log.WithError(err).Error("Error getting a backup store") return } log.Info("Validating BackupStorageLocation") err = backupStore.IsValid() if err != nil { log.WithError(err).Error("fail to validate backup store") return } }() r.logReconciledPhase(defaultFound, locationList, unavailableErrors) return ctrl.Result{}, nil } func (r *backupStorageLocationReconciler) logReconciledPhase(defaultFound bool, locationList velerov1api.BackupStorageLocationList, errs []string) { var availableBSLs []*velerov1api.BackupStorageLocation var unAvailableBSLs []*velerov1api.BackupStorageLocation var unknownBSLs []*velerov1api.BackupStorageLocation log := r.log.WithField("controller", constant.ControllerBackupStorageLocation) for i, location := range locationList.Items { phase := location.Status.Phase switch phase { case velerov1api.BackupStorageLocationPhaseAvailable: availableBSLs = append(availableBSLs, &locationList.Items[i]) r.metrics.RegisterBackupLocationAvailable(locationList.Items[i].Name) case velerov1api.BackupStorageLocationPhaseUnavailable: unAvailableBSLs = append(unAvailableBSLs, &locationList.Items[i]) r.metrics.RegisterBackupLocationUnavailable(locationList.Items[i].Name) default: unknownBSLs = append(unknownBSLs, &locationList.Items[i]) } } numAvailable := len(availableBSLs) numUnavailable := len(unAvailableBSLs) numUnknown := len(unknownBSLs) if numUnavailable+numUnknown == len(locationList.Items) { // no available BSL if len(errs) > 0 { log.Errorf("Current BackupStorageLocations available/unavailable/unknown: %v/%v/%v, %s)", numAvailable, numUnavailable, numUnknown, strings.Join(errs, "; ")) } else { log.Errorf("Current BackupStorageLocations available/unavailable/unknown: %v/%v/%v)", numAvailable, numUnavailable, numUnknown) } } else if numUnavailable > 0 { // some but not all BSL unavailable log.Warnf("Unavailable BackupStorageLocations detected: available/unavailable/unknown: %v/%v/%v, %s)", numAvailable, numUnavailable, numUnknown, strings.Join(errs, "; ")) } if !defaultFound { log.Warn("There is no existing BackupStorageLocation set as default. Please see `velero backup-location -h` for options.") } } func (r *backupStorageLocationReconciler) SetupWithManager(mgr ctrl.Manager) error { gp := kube.NewGenericEventPredicate(func(object client.Object) bool { location := object.(*velerov1api.BackupStorageLocation) return storage.IsReadyToValidate(location.Spec.ValidationFrequency, location.Status.LastValidationTime, r.defaultBackupLocationInfo.ServerValidationFrequency, r.log.WithField("controller", constant.ControllerBackupStorageLocation)) }) g := kube.NewPeriodicalEnqueueSource( r.log.WithField("controller", constant.ControllerBackupStorageLocation), mgr.GetClient(), &velerov1api.BackupStorageLocationList{}, bslValidationEnqueuePeriod, kube.PeriodicalEnqueueSourceOption{ Predicates: []predicate.Predicate{gp}, }, ) return ctrl.NewControllerManagedBy(mgr). // As the "status.LastValidationTime" field is always updated, this triggers new reconciling process, skip the update event that include no spec change to avoid the reconcile loop For(&velerov1api.BackupStorageLocation{}, builder.WithPredicates(kube.SpecChangePredicate{})). WatchesRawSource(g). Named(constant.ControllerBackupStorageLocation). Complete(r) } // ensureSingleDefaultBSL ensures that there is only one default BSL in the namespace. // the default BSL priority is as follows: // 1. follow the user's setting (the most recent validation BSL is the default BSL) // 2. follow the server's setting ("velero server --default-backup-storage-location") func (r *backupStorageLocationReconciler) ensureSingleDefaultBSL(locationList velerov1api.BackupStorageLocationList) (bool, error) { // get all default BSLs var defaultBSLs []*velerov1api.BackupStorageLocation var defaultFound bool for i, location := range locationList.Items { if location.Spec.Default { defaultBSLs = append(defaultBSLs, &locationList.Items[i]) } } if len(defaultBSLs) > 1 { // more than 1 default BSL // find the most recent updated default BSL var mostRecentCreatedBSL *velerov1api.BackupStorageLocation defaultFound = true for _, bsl := range defaultBSLs { if mostRecentCreatedBSL == nil { mostRecentCreatedBSL = bsl continue } // For lack of a better way to compare timestamps, we use the CreationTimestamp // it cloud not really find the most recent updated BSL, but it is good enough for now bslTimestamp := bsl.CreationTimestamp mostRecentTimestamp := mostRecentCreatedBSL.CreationTimestamp if mostRecentTimestamp.Before(&bslTimestamp) { mostRecentCreatedBSL = bsl } } // unset all other default BSLs for _, bsl := range defaultBSLs { if bsl.Name != mostRecentCreatedBSL.Name { bsl.Spec.Default = false if err := r.client.Update(r.ctx, bsl); err != nil { return defaultFound, errors.Wrapf(err, "failed to unset default backup storage location %q", bsl.Name) } r.log.Debugf("update default backup storage location %q to false", bsl.Name) } } } else if len(defaultBSLs) == 0 { // no default BSL defaultFound = false } else { // only 1 default BSL defaultFound = true } return defaultFound, nil } ================================================ FILE: pkg/controller/backup_storage_location_controller_test.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package controller import ( "testing" "time" "github.com/vmware-tanzu/velero/pkg/metrics" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" "github.com/vmware-tanzu/velero/internal/storage" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" persistencemocks "github.com/vmware-tanzu/velero/pkg/persistence/mocks" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" pluginmocks "github.com/vmware-tanzu/velero/pkg/plugin/mocks" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) var _ = Describe("Backup Storage Location Reconciler", func() { It("Should successfully patch a backup storage location object status phase according to whether its storage is valid or not", func() { tests := []struct { backupLocation *velerov1api.BackupStorageLocation isValidError error expectedIsDefault bool expectedPhase velerov1api.BackupStorageLocationPhase }{ { backupLocation: builder.ForBackupStorageLocation("ns-1", "location-1").ValidationFrequency(1 * time.Second).Default(true).Result(), isValidError: nil, expectedIsDefault: true, expectedPhase: velerov1api.BackupStorageLocationPhaseAvailable, }, { backupLocation: builder.ForBackupStorageLocation("ns-1", "location-2").ValidationFrequency(1 * time.Second).Result(), isValidError: errors.New("an error"), expectedIsDefault: false, expectedPhase: velerov1api.BackupStorageLocationPhaseUnavailable, }, } // Setup var ( pluginManager = &pluginmocks.Manager{} backupStores = make(map[string]*persistencemocks.BackupStore) ) pluginManager.On("CleanupClients").Return(nil) locations := new(velerov1api.BackupStorageLocationList) for i, test := range tests { location := test.backupLocation locations.Items = append(locations.Items, *location) backupStores[location.Name] = &persistencemocks.BackupStore{} backupStore := backupStores[location.Name] backupStore.On("IsValid").Return(tests[i].isValidError) } // Setup reconciler Expect(velerov1api.AddToScheme(scheme.Scheme)).To(Succeed()) r := backupStorageLocationReconciler{ ctx: ctx, client: fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(locations).Build(), defaultBackupLocationInfo: storage.DefaultBackupLocationInfo{ StorageLocation: "location-1", ServerValidationFrequency: 0, }, newPluginManager: func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager }, backupStoreGetter: NewFakeObjectBackupStoreGetter(backupStores), metrics: metrics.NewServerMetrics(), log: velerotest.NewLogger(), } // Assertions for i, location := range locations.Items { actualResult, err := r.Reconcile(ctx, ctrl.Request{ NamespacedName: types.NamespacedName{Namespace: location.Namespace, Name: location.Name}, }) Expect(actualResult).To(BeEquivalentTo(ctrl.Result{})) Expect(err).ToNot(HaveOccurred()) key := client.ObjectKey{Name: location.Name, Namespace: location.Namespace} instance := &velerov1api.BackupStorageLocation{} err = r.client.Get(ctx, key, instance) Expect(err).ToNot(HaveOccurred()) Expect(instance.Spec.Default).To(BeIdenticalTo(tests[i].expectedIsDefault)) Expect(instance.Status.Phase).To(BeIdenticalTo(tests[i].expectedPhase)) } }) It("Should successfully patch a backup storage location object spec default if the BSL is the default one", func() { tests := []struct { backupLocation *velerov1api.BackupStorageLocation isValidError error expectedIsDefault bool }{ { backupLocation: builder.ForBackupStorageLocation("ns-1", "location-1").ValidationFrequency(1 * time.Second).Default(false).Result(), isValidError: nil, expectedIsDefault: false, }, { backupLocation: builder.ForBackupStorageLocation("ns-1", "location-2").ValidationFrequency(1 * time.Second).Default(true).Result(), isValidError: nil, expectedIsDefault: true, }, } // Setup var ( pluginManager = &pluginmocks.Manager{} backupStores = make(map[string]*persistencemocks.BackupStore) ) pluginManager.On("CleanupClients").Return(nil) locations := new(velerov1api.BackupStorageLocationList) for i, test := range tests { location := test.backupLocation locations.Items = append(locations.Items, *location) backupStores[location.Name] = &persistencemocks.BackupStore{} backupStore := backupStores[location.Name] backupStore.On("IsValid").Return(tests[i].isValidError) } // Setup reconciler Expect(velerov1api.AddToScheme(scheme.Scheme)).To(Succeed()) r := backupStorageLocationReconciler{ ctx: ctx, client: fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(locations).Build(), defaultBackupLocationInfo: storage.DefaultBackupLocationInfo{ StorageLocation: "default", ServerValidationFrequency: 0, }, newPluginManager: func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager }, backupStoreGetter: NewFakeObjectBackupStoreGetter(backupStores), metrics: metrics.NewServerMetrics(), log: velerotest.NewLogger(), } // Assertions for i, location := range locations.Items { actualResult, err := r.Reconcile(ctx, ctrl.Request{ NamespacedName: types.NamespacedName{Namespace: location.Namespace, Name: location.Name}, }) Expect(actualResult).To(BeEquivalentTo(ctrl.Result{})) Expect(err).ToNot(HaveOccurred()) key := client.ObjectKey{Name: location.Name, Namespace: location.Namespace} instance := &velerov1api.BackupStorageLocation{} err = r.client.Get(ctx, key, instance) Expect(err).ToNot(HaveOccurred()) Expect(instance.Spec.Default).To(BeIdenticalTo(tests[i].expectedIsDefault)) } }) }) func TestEnsureSingleDefaultBSL(t *testing.T) { tests := []struct { name string locations velerov1api.BackupStorageLocationList defaultBackupInfo storage.DefaultBackupLocationInfo expectedDefaultSet bool expectedError error }{ { name: "MultipleDefaults", locations: func() velerov1api.BackupStorageLocationList { var locations velerov1api.BackupStorageLocationList locations.Items = append(locations.Items, *builder.ForBackupStorageLocation(velerov1api.DefaultNamespace, "location-1").LastValidationTime(time.Now()).Default(true).Result()) locations.Items = append(locations.Items, *builder.ForBackupStorageLocation(velerov1api.DefaultNamespace, "location-2").LastValidationTime(time.Now().Add(-1 * time.Hour)).Default(true).Result()) return locations }(), expectedDefaultSet: true, expectedError: nil, }, { name: "NoDefault with exist default bsl in defaultBackupInfo", locations: func() velerov1api.BackupStorageLocationList { var locations velerov1api.BackupStorageLocationList locations.Items = append(locations.Items, *builder.ForBackupStorageLocation(velerov1api.DefaultNamespace, "location-1").Default(false).Result()) locations.Items = append(locations.Items, *builder.ForBackupStorageLocation(velerov1api.DefaultNamespace, "location-2").Default(false).Result()) return locations }(), defaultBackupInfo: storage.DefaultBackupLocationInfo{ StorageLocation: "location-2", }, expectedDefaultSet: false, expectedError: nil, }, { name: "NoDefault with non-exist default bsl in defaultBackupInfo", locations: func() velerov1api.BackupStorageLocationList { var locations velerov1api.BackupStorageLocationList locations.Items = append(locations.Items, *builder.ForBackupStorageLocation(velerov1api.DefaultNamespace, "location-1").Default(false).Result()) locations.Items = append(locations.Items, *builder.ForBackupStorageLocation(velerov1api.DefaultNamespace, "location-2").Default(false).Result()) return locations }(), defaultBackupInfo: storage.DefaultBackupLocationInfo{ StorageLocation: "location-3", }, expectedDefaultSet: false, expectedError: nil, }, { name: "SingleDefault", locations: func() velerov1api.BackupStorageLocationList { var locations velerov1api.BackupStorageLocationList locations.Items = append(locations.Items, *builder.ForBackupStorageLocation(velerov1api.DefaultNamespace, "location-1").Default(true).Result()) locations.Items = append(locations.Items, *builder.ForBackupStorageLocation(velerov1api.DefaultNamespace, "location-2").Default(false).Result()) return locations }(), expectedDefaultSet: true, expectedError: nil, }, } for _, test := range tests { // Setup reconciler require.NoError(t, velerov1api.AddToScheme(scheme.Scheme)) t.Run(test.name, func(t *testing.T) { r := &backupStorageLocationReconciler{ ctx: t.Context(), client: fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(&test.locations).Build(), defaultBackupLocationInfo: test.defaultBackupInfo, metrics: metrics.NewServerMetrics(), log: velerotest.NewLogger(), } defaultFound, err := r.ensureSingleDefaultBSL(test.locations) assert.Equal(t, test.expectedDefaultSet, defaultFound) assert.Equal(t, test.expectedError, err) }) } } func TestBSLReconcile(t *testing.T) { tests := []struct { name string locationList velerov1api.BackupStorageLocationList defaultFound bool expectedError error }{ { name: "NoBSL", locationList: velerov1api.BackupStorageLocationList{}, defaultFound: false, expectedError: nil, }, { name: "BSLNotFound", locationList: func() velerov1api.BackupStorageLocationList { var locations velerov1api.BackupStorageLocationList locations.Items = append(locations.Items, *builder.ForBackupStorageLocation(velerov1api.DefaultNamespace, "location-2").Result()) return locations }(), defaultFound: false, expectedError: nil, }, } pluginManager := &pluginmocks.Manager{} pluginManager.On("CleanupClients").Return(nil) for _, test := range tests { // Setup reconciler require.NoError(t, velerov1api.AddToScheme(scheme.Scheme)) t.Run(test.name, func(t *testing.T) { r := &backupStorageLocationReconciler{ ctx: t.Context(), client: fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(&test.locationList).Build(), newPluginManager: func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager }, metrics: metrics.NewServerMetrics(), log: velerotest.NewLogger(), } result, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: velerov1api.DefaultNamespace, Name: "location-1"}}) assert.Equal(t, test.expectedError, err) assert.Equal(t, ctrl.Result{}, result) }) } } func TestSanitizeStorageError(t *testing.T) { tests := []struct { name string input error expected string }{ { name: "Nil error", input: nil, expected: "", }, { name: "Simple error without Azure formatting", input: errors.New("simple error message"), expected: "simple error message", }, { name: "AWS style error", input: errors.New("NoSuchBucket: The specified bucket does not exist"), expected: "NoSuchBucket: The specified bucket does not exist", }, { name: "Azure container not found error with full HTTP response", input: errors.New(`rpc error: code = Unknown desc = GET https://oadp100711zl59k.blob.core.windows.net/oadp100711zl59k1 -------------------------------------------------------------------------------- RESPONSE 404: 404 The specified container does not exist. ERROR CODE: ContainerNotFound -------------------------------------------------------------------------------- ContainerNotFoundThe specified container does not exist. RequestId:63cf34d8-801e-0078-09b4-2e4682000000 Time:2024-11-04T12:23:04.5623627Z -------------------------------------------------------------------------------- `), expected: "rpc error: code = Unknown desc = ContainerNotFound: The specified container does not exist.", }, { name: "Azure blob not found error", input: errors.New(`rpc error: code = Unknown desc = GET https://storage.blob.core.windows.net/container/blob -------------------------------------------------------------------------------- RESPONSE 404: 404 The specified blob does not exist. ERROR CODE: BlobNotFound -------------------------------------------------------------------------------- BlobNotFoundThe specified blob does not exist. RequestId:12345678-1234-1234-1234-123456789012 Time:2024-11-04T12:23:04.5623627Z -------------------------------------------------------------------------------- `), expected: "rpc error: code = Unknown desc = BlobNotFound: The specified blob does not exist.", }, { name: "Azure error with plain text response (no XML)", input: errors.New(`rpc error: code = Unknown desc = GET https://storage.blob.core.windows.net/container -------------------------------------------------------------------------------- RESPONSE 404: 404 The specified container does not exist. ERROR CODE: ContainerNotFound -------------------------------------------------------------------------------- `), expected: "rpc error: code = Unknown desc = ContainerNotFound: The specified container does not exist.", }, { name: "Azure error without XML message but with error code", input: errors.New(`rpc error: code = Unknown desc = operation failed RESPONSE 403: 403 Forbidden ERROR CODE: AuthorizationFailure -------------------------------------------------------------------------------- `), expected: "rpc error: code = Unknown desc = AuthorizationFailure: Forbidden", }, { name: "Error with Azure SAS token in URL", input: errors.New(`rpc error: code = Unknown desc = GET https://storage.blob.core.windows.net/backup?sv=2020-08-04&sig=abc123secrettoken&se=2024-12-31T23:59:59Z&sp=rwdl -------------------------------------------------------------------------------- RESPONSE 404: 404 The specified container does not exist. ERROR CODE: ContainerNotFound -------------------------------------------------------------------------------- `), expected: "rpc error: code = Unknown desc = ContainerNotFound: The specified container does not exist.", }, { name: "Error with multiple SAS parameters", input: errors.New(`GET https://mystorageaccount.blob.core.windows.net/container?sv=2020-08-04&ss=b&srt=sco&sp=rwdlac&se=2024-12-31&st=2024-01-01&sip=168.1.5.60&spr=https&sig=SIGNATURE_HASH`), expected: "GET https://mystorageaccount.blob.core.windows.net/container?sv=***REDACTED***&ss=***REDACTED***&srt=***REDACTED***&sp=***REDACTED***&se=***REDACTED***&st=***REDACTED***&sip=***REDACTED***&spr=***REDACTED***&sig=***REDACTED***", }, { name: "Simple URL without SAS tokens unchanged", input: errors.New("GET https://storage.blob.core.windows.net/container/blob"), expected: "GET https://storage.blob.core.windows.net/container/blob", }, { name: "Azure error with SAS token in full HTTP response", input: errors.New(`rpc error: code = Unknown desc = GET https://oadp100711zl59k.blob.core.windows.net/backup?sig=secretsignature123&se=2024-12-31 -------------------------------------------------------------------------------- RESPONSE 404: 404 The specified container does not exist. ERROR CODE: ContainerNotFound -------------------------------------------------------------------------------- ContainerNotFoundThe specified container does not exist. RequestId:63cf34d8-801e-0078-09b4-2e4682000000 Time:2024-11-04T12:23:04.5623627Z -------------------------------------------------------------------------------- `), expected: "rpc error: code = Unknown desc = ContainerNotFound: The specified container does not exist.", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { actual := sanitizeStorageError(test.input) assert.Equal(t, test.expected, actual) }) } } ================================================ FILE: pkg/controller/backup_sync_controller.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package controller import ( "context" "fmt" "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/predicate" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/constant" "github.com/vmware-tanzu/velero/pkg/label" "github.com/vmware-tanzu/velero/pkg/persistence" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" "github.com/vmware-tanzu/velero/pkg/util/kube" veleroutil "github.com/vmware-tanzu/velero/pkg/util/velero" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" ) const ( backupSyncReconcilePeriod = time.Minute ) type backupSyncReconciler struct { client client.Client namespace string defaultBackupSyncPeriod time.Duration newPluginManager func(logrus.FieldLogger) clientmgmt.Manager backupStoreGetter persistence.ObjectBackupStoreGetter logger logrus.FieldLogger } // NewBackupSyncReconciler is used to generate BackupSync reconciler structure. func NewBackupSyncReconciler( client client.Client, namespace string, defaultBackupSyncPeriod time.Duration, newPluginManager func(logrus.FieldLogger) clientmgmt.Manager, backupStoreGetter persistence.ObjectBackupStoreGetter, logger logrus.FieldLogger) *backupSyncReconciler { return &backupSyncReconciler{ client: client, namespace: namespace, defaultBackupSyncPeriod: defaultBackupSyncPeriod, newPluginManager: newPluginManager, backupStoreGetter: backupStoreGetter, logger: logger, } } // Reconcile syncs between the backups in cluster and backups metadata in object store. func (b *backupSyncReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := b.logger.WithField("controller", constant.ControllerBackupSync) log = log.WithField("backupLocation", req.String()) log.Debug("Begin to sync between backups' metadata in BSL object storage and cluster's existing backups.") location := &velerov1api.BackupStorageLocation{} err := b.client.Get(ctx, req.NamespacedName, location) if err != nil { if apierrors.IsNotFound(err) { log.Debug("BackupStorageLocation is not found") return ctrl.Result{}, nil } return ctrl.Result{}, errors.Wrapf(err, "error getting BackupStorageLocation %s", req.String()) } if !veleroutil.BSLIsAvailable(*location) { log.Errorf("BackupStorageLocation is in unavailable state, skip syncing backup from it.") return ctrl.Result{}, nil } pluginManager := b.newPluginManager(log) defer pluginManager.CleanupClients() log.Debug("Checking backup location for backups to sync into cluster") backupStore, err := b.backupStoreGetter.Get(location, pluginManager, log) if err != nil { log.WithError(err).Error("Error getting backup store for this location") return ctrl.Result{}, nil } // get a list of all the backups that are stored in the backup storage location res, err := backupStore.ListBackups() if err != nil { log.WithError(err).Error("Error listing backups in backup store") return ctrl.Result{}, nil } backupStoreBackups := sets.New[string](res...) log.WithField("backupCount", len(backupStoreBackups)).Debug("Got backups from backup store") // get a list of all the backups that exist as custom resources in the cluster var clusterBackupList velerov1api.BackupList listOption := client.ListOptions{ LabelSelector: labels.Everything(), Namespace: b.namespace, } err = b.client.List(ctx, &clusterBackupList, &listOption) if err != nil { log.WithError(errors.WithStack(err)).Error("Error getting backups from cluster, proceeding with sync into cluster") } else { log.WithField("backupCount", len(clusterBackupList.Items)).Debug("Got backups from cluster") } // get a list of backups that *are* in the backup storage location and *aren't* in the cluster clusterBackupsSet := sets.New[string]() for _, b := range clusterBackupList.Items { clusterBackupsSet.Insert(b.Name) } backupsToSync := backupStoreBackups.Difference(clusterBackupsSet) if count := backupsToSync.Len(); count > 0 { log.Infof("Found %v backups in the backup location that do not exist in the cluster and need to be synced", count) } else { log.Debug("No backups found in the backup location that need to be synced into the cluster") } // sync each backup for backupName := range backupsToSync { log = log.WithField("backup", backupName) log.Info("Attempting to sync backup into cluster") exist, err := backupStore.BackupExists(location.Spec.ObjectStorage.Bucket, backupName) if err != nil { log.WithError(errors.WithStack(err)).Error("Error checking backup exist from backup store") continue } if !exist { log.Debugf("backup %s doesn't exist in backup store, skip", backupName) continue } backup, err := backupStore.GetBackupMetadata(backupName) if err != nil { log.WithError(errors.WithStack(err)).Error("Error getting backup metadata from backup store") continue } if backup.Status.Phase == velerov1api.BackupPhaseWaitingForPluginOperations || backup.Status.Phase == velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed || backup.Status.Phase == velerov1api.BackupPhaseFinalizing || backup.Status.Phase == velerov1api.BackupPhaseFinalizingPartiallyFailed { if backup.Status.Expiration == nil || backup.Status.Expiration.After(time.Now()) { log.Debugf("Skipping non-expired incomplete backup %v", backup.Name) continue } log.Debugf("%v Backup is past expiration, syncing for garbage collection", backup.Status.Phase) backup.Status.Phase = velerov1api.BackupPhasePartiallyFailed } backup.Namespace = b.namespace backup.ResourceVersion = "" // update the StorageLocation field and label since the name of the location // may be different in this cluster than in the cluster that created the // backup. backup.Spec.StorageLocation = location.Name if backup.Labels == nil { backup.Labels = make(map[string]string) } backup.Labels[velerov1api.StorageLocationLabel] = label.GetValidName(backup.Spec.StorageLocation) //check for the ownership references. If they do not exist, remove them. backup.ObjectMeta.OwnerReferences = b.filterBackupOwnerReferences(ctx, backup, log) // attempt to create backup custom resource via API err = b.client.Create(ctx, backup, &client.CreateOptions{}) switch { case err != nil && apierrors.IsAlreadyExists(err): log.Debug("Backup already exists in cluster") continue case err != nil && !apierrors.IsAlreadyExists(err): log.WithError(errors.WithStack(err)).Error("Error syncing backup into cluster") continue default: log.Info("Successfully synced backup into cluster") } // process the pod volume backups from object store, if any podVolumeBackups, err := backupStore.GetPodVolumeBackups(backupName) if err != nil { log.WithError(errors.WithStack(err)).Error("Error getting pod volume backups for this backup from backup store") continue } for _, podVolumeBackup := range podVolumeBackups { log := log.WithField("podVolumeBackup", podVolumeBackup.Name) log.Debug("Checking this pod volume backup to see if it needs to be synced into the cluster") for i, ownerRef := range podVolumeBackup.OwnerReferences { if ownerRef.APIVersion == velerov1api.SchemeGroupVersion.String() && ownerRef.Kind == "Backup" && ownerRef.Name == backup.Name { log.WithField("uid", backup.UID).Debugf("Updating pod volume backup's owner reference UID") podVolumeBackup.OwnerReferences[i].UID = backup.UID } } if _, ok := podVolumeBackup.Labels[velerov1api.BackupUIDLabel]; ok { podVolumeBackup.Labels[velerov1api.BackupUIDLabel] = string(backup.UID) } podVolumeBackup.Namespace = backup.Namespace podVolumeBackup.ResourceVersion = "" podVolumeBackup.Spec.BackupStorageLocation = location.Name err = b.client.Create(ctx, podVolumeBackup, &client.CreateOptions{}) switch { case err != nil && apierrors.IsAlreadyExists(err): log.Debug("Pod volume backup already exists in cluster") continue case err != nil && !apierrors.IsAlreadyExists(err): log.WithError(errors.WithStack(err)).Error("Error syncing pod volume backup into cluster") continue default: log.Debug("Synced pod volume backup into cluster") } } } b.deleteOrphanedBackups(ctx, location.Name, backupStoreBackups, log) // update the location's last-synced time field statusPatch := client.MergeFrom(location.DeepCopy()) location.Status.LastSyncedTime = &metav1.Time{Time: time.Now().UTC()} if err := b.client.Patch(ctx, location, statusPatch); err != nil { log.WithError(errors.WithStack(err)).Error("Error patching backup location's last-synced time") return ctrl.Result{}, nil } return ctrl.Result{}, nil } func (b *backupSyncReconciler) filterBackupOwnerReferences(ctx context.Context, backup *velerov1api.Backup, log logrus.FieldLogger) []metav1.OwnerReference { listedReferences := backup.ObjectMeta.OwnerReferences foundReferences := make([]metav1.OwnerReference, 0) for _, v := range listedReferences { switch v.Kind { case "Schedule": schedule := new(velerov1api.Schedule) err := b.client.Get(ctx, types.NamespacedName{ Name: v.Name, Namespace: backup.Namespace, }, schedule) switch { case err != nil && apierrors.IsNotFound(err): log.Warnf("Removing missing schedule ownership reference %s/%s from backup", backup.Namespace, v.Name) continue case schedule.UID != v.UID: log.Warnf("Removing schedule ownership reference with mismatched UIDs. Expected %s, got %s", v.UID, schedule.UID) continue case err != nil && !apierrors.IsNotFound(err): log.WithError(errors.WithStack(err)).Error("Error finding schedule ownership reference, keeping schedule on backup") } default: log.Warnf("Unable to check ownership reference for unknown kind, %s", v.Kind) } foundReferences = append(foundReferences, v) } return foundReferences } // SetupWithManager is used to setup controller and its watching sources. func (b *backupSyncReconciler) SetupWithManager(mgr ctrl.Manager) error { gp := kube.NewGenericEventPredicate(func(object client.Object) bool { location := object.(*velerov1api.BackupStorageLocation) return b.locationFilterFunc(location) }) backupSyncSource := kube.NewPeriodicalEnqueueSource( b.logger.WithField("controller", constant.ControllerBackupSync), mgr.GetClient(), &velerov1api.BackupStorageLocationList{}, backupSyncReconcilePeriod, kube.PeriodicalEnqueueSourceOption{ OrderFunc: backupSyncSourceOrderFunc, Predicates: []predicate.Predicate{gp}, }, ) return ctrl.NewControllerManagedBy(mgr). // Filter all BSL events, because this controller is supposed to run periodically, not by event. For(&velerov1api.BackupStorageLocation{}, builder.WithPredicates(kube.FalsePredicate{})). WatchesRawSource(backupSyncSource). Named(constant.ControllerBackupSync). Complete(b) } // deleteOrphanedBackups deletes backup objects (CRDs) from Kubernetes that have the specified location // and a phase of Completed, but no corresponding backup in object storage. func (b *backupSyncReconciler) deleteOrphanedBackups(ctx context.Context, locationName string, backupStoreBackups sets.Set[string], log logrus.FieldLogger) { var backupList velerov1api.BackupList listOption := client.ListOptions{ LabelSelector: labels.Set(map[string]string{ velerov1api.StorageLocationLabel: label.GetValidName(locationName), }).AsSelector(), } err := b.client.List(ctx, &backupList, &listOption) if err != nil { log.WithError(errors.WithStack(err)).Error("Error listing backups from cluster") return } if len(backupList.Items) == 0 { return } for i, backup := range backupList.Items { log = log.WithField("backup", backup.Name) if !(backup.Status.Phase == velerov1api.BackupPhaseCompleted || backup.Status.Phase == velerov1api.BackupPhasePartiallyFailed) || backupStoreBackups.Has(backup.Name) { continue } if err := b.client.Delete(ctx, &backupList.Items[i], &client.DeleteOptions{}); err != nil { log.WithError(errors.WithStack(err)).Error("Error deleting orphaned backup from cluster") } } } // backupSyncSourceOrderFunc returns a new slice with the default backup location first (if it exists), // followed by the rest of the locations in no particular order. func backupSyncSourceOrderFunc(objList client.ObjectList) client.ObjectList { inputBSLList := objList.(*velerov1api.BackupStorageLocationList) resultBSLList := &velerov1api.BackupStorageLocationList{} bslArray := make([]runtime.Object, 0) if len(inputBSLList.Items) <= 0 { return objList } for i := range inputBSLList.Items { location := inputBSLList.Items[i] // sync the default backup storage location first, if it exists if location.Spec.Default { // put the default location first bslArray = append(bslArray, &inputBSLList.Items[i]) // append everything before the default for _, bsl := range inputBSLList.Items[:i] { cpBsl := bsl bslArray = append(bslArray, &cpBsl) } // append everything after the default for _, bsl := range inputBSLList.Items[i+1:] { cpBsl := bsl bslArray = append(bslArray, &cpBsl) } if err := meta.SetList(resultBSLList, bslArray); err != nil { fmt.Printf("fail to sort BSL list: %s", err.Error()) return &velerov1api.BackupStorageLocationList{} } return resultBSLList } } // No default BSL found. Return the input. return objList } func (b *backupSyncReconciler) locationFilterFunc(location *velerov1api.BackupStorageLocation) bool { syncPeriod := b.defaultBackupSyncPeriod if location.Spec.BackupSyncPeriod != nil { syncPeriod = location.Spec.BackupSyncPeriod.Duration if syncPeriod == 0 { b.logger.Debug("Backup sync period for this location is set to 0, skipping sync") return false } if syncPeriod < 0 { b.logger.Debug("Backup sync period must be non-negative") syncPeriod = b.defaultBackupSyncPeriod } } lastSync := location.Status.LastSyncedTime if lastSync != nil { b.logger.Debug("Checking if backups need to be synced at this time for this location") nextSync := lastSync.Add(syncPeriod) if time.Now().UTC().Before(nextSync) { return false } } return true } ================================================ FILE: pkg/controller/backup_sync_controller_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package controller import ( "context" "fmt" "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/sirupsen/logrus" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation" testclocks "k8s.io/utils/clock/testing" ctrl "sigs.k8s.io/controller-runtime" ctrlClient "sigs.k8s.io/controller-runtime/pkg/client" ctrlfake "sigs.k8s.io/controller-runtime/pkg/client/fake" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/label" persistencemocks "github.com/vmware-tanzu/velero/pkg/persistence/mocks" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" pluginmocks "github.com/vmware-tanzu/velero/pkg/plugin/mocks" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func defaultLocation(namespace string) *velerov1api.BackupStorageLocation { return &velerov1api.BackupStorageLocation{ ObjectMeta: metav1.ObjectMeta{ Namespace: namespace, Name: "location-1", }, Spec: velerov1api.BackupStorageLocationSpec{ Provider: "objStoreProvider", StorageType: velerov1api.StorageType{ ObjectStorage: &velerov1api.ObjectStorageLocation{ Bucket: "bucket-1", }, }, Default: true, }, Status: velerov1api.BackupStorageLocationStatus{ Phase: velerov1api.BackupStorageLocationPhaseAvailable, }, } } func defaultLocationsList(namespace string) []*velerov1api.BackupStorageLocation { return []*velerov1api.BackupStorageLocation{ { ObjectMeta: metav1.ObjectMeta{ Namespace: namespace, Name: "location-0", }, Spec: velerov1api.BackupStorageLocationSpec{ Provider: "objStoreProvider", StorageType: velerov1api.StorageType{ ObjectStorage: &velerov1api.ObjectStorageLocation{ Bucket: "bucket-1", }, }, }, }, { ObjectMeta: metav1.ObjectMeta{ Namespace: namespace, Name: "location-1", }, Spec: velerov1api.BackupStorageLocationSpec{ Provider: "objStoreProvider", StorageType: velerov1api.StorageType{ ObjectStorage: &velerov1api.ObjectStorageLocation{ Bucket: "bucket-1", }, }, Default: true, }, }, { ObjectMeta: metav1.ObjectMeta{ Namespace: namespace, Name: "location-2", }, Spec: velerov1api.BackupStorageLocationSpec{ Provider: "objStoreProvider", StorageType: velerov1api.StorageType{ ObjectStorage: &velerov1api.ObjectStorageLocation{ Bucket: "bucket-1", }, }, }, }, { ObjectMeta: metav1.ObjectMeta{ Namespace: namespace, Name: "location-3", }, Spec: velerov1api.BackupStorageLocationSpec{ Provider: "objStoreProvider", StorageType: velerov1api.StorageType{ ObjectStorage: &velerov1api.ObjectStorageLocation{ Bucket: "bucket-1", }, }, }, }, } } func defaultLocationWithLongerLocationName(namespace string) *velerov1api.BackupStorageLocation { return &velerov1api.BackupStorageLocation{ ObjectMeta: metav1.ObjectMeta{ Namespace: namespace, Name: "the-really-long-location-name-that-is-much-more-than-63-characters-1", }, Spec: velerov1api.BackupStorageLocationSpec{ Provider: "objStoreProvider", StorageType: velerov1api.StorageType{ ObjectStorage: &velerov1api.ObjectStorageLocation{ Bucket: "bucket-1", }, }, }, Status: velerov1api.BackupStorageLocationStatus{ Phase: velerov1api.BackupStorageLocationPhaseAvailable, }, } } func numBackups(c ctrlClient.WithWatch) (int, error) { var existingK8SBackups velerov1api.BackupList err := c.List(context.TODO(), &existingK8SBackups, &ctrlClient.ListOptions{}) if err != nil { return 0, err } return len(existingK8SBackups.Items), nil } var _ = Describe("Backup Sync Reconciler", func() { It("Test Backup Sync Reconciler basic function", func() { fakeClock := testclocks.NewFakeClock(time.Now()) type cloudBackupData struct { backup *velerov1api.Backup podVolumeBackups []*velerov1api.PodVolumeBackup backupShouldSkipSync bool // backups waiting for plugin operations should not sync } tests := []struct { name string namespace string location *velerov1api.BackupStorageLocation cloudBackups []*cloudBackupData existingBackups []*velerov1api.Backup existingPodVolumeBackups []*velerov1api.PodVolumeBackup longLocationNameEnabled bool }{ { name: "no cloud backups", namespace: "ns-1", location: defaultLocation("ns-1"), }, { name: "unavailable BSL", namespace: "ns-1", location: builder.ForBackupStorageLocation("ns-1", "default").Phase(velerov1api.BackupStorageLocationPhaseUnavailable).Result(), cloudBackups: []*cloudBackupData{ { backup: builder.ForBackup("ns-1", "backup-1").Result(), backupShouldSkipSync: true, }, { backup: builder.ForBackup("ns-1", "backup-2").Result(), backupShouldSkipSync: true, }, }, }, { name: "normal case", namespace: "ns-1", location: defaultLocation("ns-1"), cloudBackups: []*cloudBackupData{ { backup: builder.ForBackup("ns-1", "backup-1").Result(), }, { backup: builder.ForBackup("ns-1", "backup-2").Result(), }, }, }, { name: "backups waiting for plugin operations aren't synced", namespace: "ns-1", location: defaultLocation("ns-1"), cloudBackups: []*cloudBackupData{ { backup: builder.ForBackup("ns-1", "backup-1"). Phase(velerov1api.BackupPhaseWaitingForPluginOperations).Result(), backupShouldSkipSync: true, }, { backup: builder.ForBackup("ns-1", "backup-2"). Phase(velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed).Result(), backupShouldSkipSync: true, }, { backup: builder.ForBackup("ns-1", "backup-3"). Phase(velerov1api.BackupPhaseWaitingForPluginOperations).Result(), podVolumeBackups: []*velerov1api.PodVolumeBackup{ builder.ForPodVolumeBackup("ns-1", "pvb-1").Result(), }, backupShouldSkipSync: true, }, { backup: builder.ForBackup("ns-1", "backup-4"). Phase(velerov1api.BackupPhaseFinalizing).Result(), backupShouldSkipSync: true, }, { backup: builder.ForBackup("ns-1", "backup-5"). Phase(velerov1api.BackupPhaseFinalizingPartiallyFailed).Result(), backupShouldSkipSync: true, }, { backup: builder.ForBackup("ns-1", "backup-6"). Phase(velerov1api.BackupPhaseFinalizing).Result(), podVolumeBackups: []*velerov1api.PodVolumeBackup{ builder.ForPodVolumeBackup("ns-1", "pvb-2").Result(), }, backupShouldSkipSync: true, }, }, }, { name: "expired backups waiting for plugin operations are synced", namespace: "ns-1", location: defaultLocation("ns-1"), cloudBackups: []*cloudBackupData{ { backup: builder.ForBackup("ns-1", "backup-1"). Phase(velerov1api.BackupPhaseWaitingForPluginOperations). Expiration(fakeClock.Now().Add(-time.Hour)).Result(), backupShouldSkipSync: true, }, { backup: builder.ForBackup("ns-1", "backup-2"). Phase(velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed). Expiration(fakeClock.Now().Add(-time.Hour)).Result(), backupShouldSkipSync: true, }, { backup: builder.ForBackup("ns-1", "backup-3"). Phase(velerov1api.BackupPhaseWaitingForPluginOperations). Expiration(fakeClock.Now().Add(-time.Hour)).Result(), podVolumeBackups: []*velerov1api.PodVolumeBackup{ builder.ForPodVolumeBackup("ns-1", "pvb-1").Result(), }, backupShouldSkipSync: true, }, { backup: builder.ForBackup("ns-1", "backup-4"). Phase(velerov1api.BackupPhaseFinalizing). Expiration(fakeClock.Now().Add(-time.Hour)).Result(), backupShouldSkipSync: true, }, { backup: builder.ForBackup("ns-1", "backup-5"). Phase(velerov1api.BackupPhaseFinalizingPartiallyFailed). Expiration(fakeClock.Now().Add(-time.Hour)).Result(), backupShouldSkipSync: true, }, { backup: builder.ForBackup("ns-1", "backup-6"). Phase(velerov1api.BackupPhaseFinalizing). Expiration(fakeClock.Now().Add(-time.Hour)).Result(), podVolumeBackups: []*velerov1api.PodVolumeBackup{ builder.ForPodVolumeBackup("ns-1", "pvb-2").Result(), }, backupShouldSkipSync: true, }, }, }, { name: "all synced backups get created in Velero server's namespace", namespace: "velero", location: defaultLocation("velero"), cloudBackups: []*cloudBackupData{ { backup: builder.ForBackup("ns-1", "backup-1").Result(), }, { backup: builder.ForBackup("ns-1", "backup-2").Result(), }, }, }, { name: "new backups get synced when some cloud backups already exist in the cluster", namespace: "ns-1", location: defaultLocation("ns-1"), cloudBackups: []*cloudBackupData{ { backup: builder.ForBackup("ns-1", "backup-1").Result(), }, { backup: builder.ForBackup("ns-1", "backup-2").Result(), }, }, existingBackups: []*velerov1api.Backup{ // add a label to each existing backup so we can differentiate it from the cloud // backup during verification builder.ForBackup("ns-1", "backup-1").StorageLocation("location-1").ObjectMeta(builder.WithLabels("i-exist", "true")).Result(), builder.ForBackup("ns-1", "backup-3").StorageLocation("location-2").ObjectMeta(builder.WithLabels("i-exist", "true")).Result(), }, }, { name: "existing backups without a StorageLocation get it filled in", namespace: "ns-1", location: defaultLocation("ns-1"), cloudBackups: []*cloudBackupData{ { backup: builder.ForBackup("ns-1", "backup-1").Result(), }, }, existingBackups: []*velerov1api.Backup{ // add a label to each existing backup so we can differentiate it from the cloud // backup during verification builder.ForBackup("ns-1", "backup-1").ObjectMeta(builder.WithLabels("i-exist", "true")).StorageLocation("location-1").Result(), }, }, { name: "backup storage location names and labels get updated", namespace: "ns-1", location: defaultLocation("ns-1"), cloudBackups: []*cloudBackupData{ { backup: builder.ForBackup("ns-1", "backup-1").StorageLocation("foo").ObjectMeta(builder.WithLabels(velerov1api.StorageLocationLabel, "foo")).Result(), }, { backup: builder.ForBackup("ns-1", "backup-2").Result(), }, }, }, { name: "backup storage location names and labels get updated with location name greater than 63 chars", namespace: "ns-1", location: defaultLocationWithLongerLocationName("ns-1"), longLocationNameEnabled: true, cloudBackups: []*cloudBackupData{ { backup: builder.ForBackup("ns-1", "backup-1").StorageLocation("foo").ObjectMeta(builder.WithLabels(velerov1api.StorageLocationLabel, "foo")).Result(), }, { backup: builder.ForBackup("ns-1", "backup-2").Result(), }, }, }, { name: "all synced backups and pod volume backups get created in Velero server's namespace", namespace: "ns-1", location: defaultLocation("ns-1"), cloudBackups: []*cloudBackupData{ { backup: builder.ForBackup("ns-1", "backup-1").Result(), podVolumeBackups: []*velerov1api.PodVolumeBackup{ builder.ForPodVolumeBackup("ns-1", "pvb-1").Result(), }, }, { backup: builder.ForBackup("ns-1", "backup-2").Result(), podVolumeBackups: []*velerov1api.PodVolumeBackup{ builder.ForPodVolumeBackup("ns-1", "pvb-2").Result(), }, }, }, }, { name: "new pod volume backups get synched when some pod volume backups already exist in the cluster", namespace: "ns-1", location: defaultLocation("ns-1"), cloudBackups: []*cloudBackupData{ { backup: builder.ForBackup("ns-1", "backup-1").Result(), podVolumeBackups: []*velerov1api.PodVolumeBackup{ builder.ForPodVolumeBackup("ns-1", "pvb-1").Result(), }, }, { backup: builder.ForBackup("ns-1", "backup-2").Result(), podVolumeBackups: []*velerov1api.PodVolumeBackup{ builder.ForPodVolumeBackup("ns-1", "pvb-3").Result(), }, }, }, existingPodVolumeBackups: []*velerov1api.PodVolumeBackup{ builder.ForPodVolumeBackup("ns-1", "pvb-1").Result(), builder.ForPodVolumeBackup("ns-1", "pvb-2").Result(), }, }, } for _, test := range tests { var ( client = ctrlfake.NewClientBuilder().Build() pluginManager = &pluginmocks.Manager{} backupStores = make(map[string]*persistencemocks.BackupStore) ) pluginManager.On("CleanupClients").Return(nil) r := backupSyncReconciler{ client: client, namespace: test.namespace, defaultBackupSyncPeriod: time.Second * 10, newPluginManager: func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager }, backupStoreGetter: NewFakeObjectBackupStoreGetter(backupStores), logger: velerotest.NewLogger(), } if test.location != nil { Expect(r.client.Create(ctx, test.location)).ShouldNot(HaveOccurred()) backupStores[test.location.Name] = &persistencemocks.BackupStore{} backupStore, ok := backupStores[test.location.Name] Expect(ok).To(BeTrue(), "no mock backup store for location %s", test.location.Name) var backupNames []string for _, backup := range test.cloudBackups { backupNames = append(backupNames, backup.backup.Name) backupStore.On("GetBackupMetadata", backup.backup.Name).Return(backup.backup, nil) backupStore.On("GetPodVolumeBackups", backup.backup.Name).Return(backup.podVolumeBackups, nil) backupStore.On("BackupExists", "bucket-1", backup.backup.Name).Return(true, nil) } backupStore.On("ListBackups").Return(backupNames, nil) } for _, existingBackup := range test.existingBackups { err := client.Create(context.TODO(), existingBackup, &ctrlClient.CreateOptions{}) Expect(err).ShouldNot(HaveOccurred()) } for _, existingPodVolumeBackup := range test.existingPodVolumeBackups { err := client.Create(context.TODO(), existingPodVolumeBackup, &ctrlClient.CreateOptions{}) Expect(err).ShouldNot(HaveOccurred()) } actualResult, err := r.Reconcile(ctx, ctrl.Request{ NamespacedName: types.NamespacedName{Namespace: test.location.Namespace, Name: test.location.Name}, }) Expect(actualResult).To(BeEquivalentTo(ctrl.Result{})) Expect(err).ToNot(HaveOccurred()) // process the cloud backups for _, cloudBackupData := range test.cloudBackups { obj := &velerov1api.Backup{} err := client.Get( context.TODO(), types.NamespacedName{ Namespace: cloudBackupData.backup.Namespace, Name: cloudBackupData.backup.Name}, obj) if cloudBackupData.backupShouldSkipSync && (cloudBackupData.backup.Status.Expiration == nil || cloudBackupData.backup.Status.Expiration.After(fakeClock.Now())) { Expect(apierrors.IsNotFound(err)).To(BeTrue()) } else { Expect(err).ToNot(HaveOccurred()) // did this cloud backup already exist in the cluster? var existing *velerov1api.Backup for _, obj := range test.existingBackups { if obj.Name == cloudBackupData.backup.Name { existing = obj break } } if existing != nil { // if this cloud backup already exists in the cluster, make sure that what we get from the // client is the existing backup, not the cloud one. // verify that the in-cluster backup has its storage location populated, if it's not already. expected := existing.DeepCopy() expected.Spec.StorageLocation = test.location.Name Expect(expected).To(BeEquivalentTo(obj)) } else { // verify that the storage location field and label are set properly Expect(test.location.Name).To(BeEquivalentTo(obj.Spec.StorageLocation)) locationName := test.location.Name if test.longLocationNameEnabled { locationName = label.GetValidName(locationName) } Expect(locationName).To(BeEquivalentTo(obj.Labels[velerov1api.StorageLocationLabel])) Expect(len(obj.Labels[velerov1api.StorageLocationLabel])).To(BeNumerically("<=", validation.DNS1035LabelMaxLength)) } } // process the cloud pod volume backups for this backup, if any for _, podVolumeBackup := range cloudBackupData.podVolumeBackups { objPodVolumeBackup := &velerov1api.PodVolumeBackup{} err := client.Get( context.TODO(), types.NamespacedName{ Namespace: podVolumeBackup.Namespace, Name: podVolumeBackup.Name, }, objPodVolumeBackup) if cloudBackupData.backupShouldSkipSync && (cloudBackupData.backup.Status.Expiration == nil || cloudBackupData.backup.Status.Expiration.After(fakeClock.Now())) { Expect(apierrors.IsNotFound(err)).To(BeTrue()) } else { Expect(err).ShouldNot(HaveOccurred()) // did this cloud pod volume backup already exist in the cluster? var existingPodVolumeBackup *velerov1api.PodVolumeBackup for _, objPodVolumeBackup := range test.existingPodVolumeBackups { if objPodVolumeBackup.Name == podVolumeBackup.Name { existingPodVolumeBackup = objPodVolumeBackup break } } if existingPodVolumeBackup != nil { // if this cloud pod volume backup already exists in the cluster, make sure that what we get from the // client is the existing backup, not the cloud one. expected := existingPodVolumeBackup.DeepCopy() Expect(expected).To(BeEquivalentTo(objPodVolumeBackup)) } } } } } }) It("Test deleting orphaned backups.", func() { longLabelName := "the-really-long-location-name-that-is-much-more-than-63-characters" baseBuilder := func(name string) *builder.BackupBuilder { return builder.ForBackup("ns-1", name).ObjectMeta(builder.WithLabels(velerov1api.StorageLocationLabel, "default")) } tests := []struct { name string cloudBackups sets.Set[string] k8sBackups []*velerov1api.Backup namespace string expectedDeletes sets.Set[string] useLongBSLName bool }{ { name: "no overlapping backups", namespace: "ns-1", cloudBackups: sets.New[string]("backup-1", "backup-2", "backup-3"), k8sBackups: []*velerov1api.Backup{ baseBuilder("backupA").Phase(velerov1api.BackupPhaseCompleted).Result(), baseBuilder("backupB").Phase(velerov1api.BackupPhaseCompleted).Result(), baseBuilder("backupC").Phase(velerov1api.BackupPhasePartiallyFailed).Result(), }, expectedDeletes: sets.New[string]("backupA", "backupB", "backupC"), }, { name: "some overlapping backups", namespace: "ns-1", cloudBackups: sets.New[string]("backup-1", "backup-2", "backup-3"), k8sBackups: []*velerov1api.Backup{ baseBuilder("backup-1").Phase(velerov1api.BackupPhaseCompleted).Result(), baseBuilder("backup-2").Phase(velerov1api.BackupPhaseCompleted).Result(), baseBuilder("backup-B").Phase(velerov1api.BackupPhaseCompleted).Result(), baseBuilder("backup-C").Phase(velerov1api.BackupPhasePartiallyFailed).Result(), }, expectedDeletes: sets.New[string]("backup-B", "backup-C"), }, { name: "all overlapping backups", namespace: "ns-1", cloudBackups: sets.New[string]("backup-1", "backup-2", "backup-3"), k8sBackups: []*velerov1api.Backup{ baseBuilder("backup-1").Phase(velerov1api.BackupPhaseCompleted).Result(), baseBuilder("backup-2").Phase(velerov1api.BackupPhaseCompleted).Result(), baseBuilder("backup-3").Phase(velerov1api.BackupPhasePartiallyFailed).Result(), }, expectedDeletes: sets.New[string](), }, { name: "no overlapping backups but including backups that are not complete", namespace: "ns-1", cloudBackups: sets.New[string]("backup-1", "backup-2", "backup-3"), k8sBackups: []*velerov1api.Backup{ baseBuilder("backupA").Phase(velerov1api.BackupPhaseCompleted).Result(), baseBuilder("backupB").Phase(velerov1api.BackupPhasePartiallyFailed).Result(), baseBuilder("Deleting").Phase(velerov1api.BackupPhaseDeleting).Result(), baseBuilder("Failed").Phase(velerov1api.BackupPhaseFailed).Result(), baseBuilder("FailedValidation").Phase(velerov1api.BackupPhaseFailedValidation).Result(), baseBuilder("InProgress").Phase(velerov1api.BackupPhaseInProgress).Result(), baseBuilder("New").Phase(velerov1api.BackupPhaseNew).Result(), }, expectedDeletes: sets.New[string]("backupA", "backupB"), }, { name: "all overlapping backups and all backups that are not complete", namespace: "ns-1", cloudBackups: sets.New[string]("backup-1", "backup-2", "backup-3"), k8sBackups: []*velerov1api.Backup{ baseBuilder("backup-1").Phase(velerov1api.BackupPhaseFailed).Result(), baseBuilder("backup-2").Phase(velerov1api.BackupPhaseFailedValidation).Result(), baseBuilder("backup-3").Phase(velerov1api.BackupPhaseInProgress).Result(), }, expectedDeletes: sets.New[string](), }, { name: "no completed backups in other locations are deleted", namespace: "ns-1", cloudBackups: sets.New[string]("backup-1", "backup-2", "backup-3"), k8sBackups: []*velerov1api.Backup{ baseBuilder("backup-1").Phase(velerov1api.BackupPhaseCompleted).Result(), baseBuilder("backup-2").Phase(velerov1api.BackupPhaseCompleted).Result(), baseBuilder("backup-C").Phase(velerov1api.BackupPhaseCompleted).Result(), baseBuilder("backup-D").Phase(velerov1api.BackupPhasePartiallyFailed).Result(), baseBuilder("backup-4").ObjectMeta(builder.WithLabels(velerov1api.StorageLocationLabel, "alternate")).Phase(velerov1api.BackupPhaseCompleted).Result(), baseBuilder("backup-5").ObjectMeta(builder.WithLabels(velerov1api.StorageLocationLabel, "alternate")).Phase(velerov1api.BackupPhaseCompleted).Result(), baseBuilder("backup-6").ObjectMeta(builder.WithLabels(velerov1api.StorageLocationLabel, "alternate")).Phase(velerov1api.BackupPhasePartiallyFailed).Result(), }, expectedDeletes: sets.New[string]("backup-C", "backup-D"), }, { name: "some overlapping backups", namespace: "ns-1", cloudBackups: sets.New[string]("backup-1", "backup-2", "backup-3"), k8sBackups: []*velerov1api.Backup{ builder.ForBackup("ns-1", "backup-1"). ObjectMeta( builder.WithLabels(velerov1api.StorageLocationLabel, "the-really-long-location-name-that-is-much-more-than-63-c69e779"), ). Phase(velerov1api.BackupPhaseCompleted). Result(), builder.ForBackup("ns-1", "backup-2"). ObjectMeta( builder.WithLabels(velerov1api.StorageLocationLabel, "the-really-long-location-name-that-is-much-more-than-63-c69e779"), ). Phase(velerov1api.BackupPhaseCompleted). Result(), builder.ForBackup("ns-1", "backup-C"). ObjectMeta( builder.WithLabels(velerov1api.StorageLocationLabel, "the-really-long-location-name-that-is-much-more-than-63-c69e779"), ). Phase(velerov1api.BackupPhaseCompleted). Result(), builder.ForBackup("ns-1", "backup-D"). ObjectMeta( builder.WithLabels(velerov1api.StorageLocationLabel, "the-really-long-location-name-that-is-much-more-than-63-c69e779"), ). Phase(velerov1api.BackupPhasePartiallyFailed). Result(), }, expectedDeletes: sets.New[string]("backup-C", "backup-D"), useLongBSLName: true, }, } for _, test := range tests { var ( client = ctrlfake.NewClientBuilder().Build() pluginManager = &pluginmocks.Manager{} backupStores = make(map[string]*persistencemocks.BackupStore) ) r := backupSyncReconciler{ client: client, namespace: test.namespace, defaultBackupSyncPeriod: time.Second * 10, newPluginManager: func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager }, backupStoreGetter: NewFakeObjectBackupStoreGetter(backupStores), logger: velerotest.NewLogger(), } for _, backup := range test.k8sBackups { // add test backup to client err := client.Create(context.TODO(), backup, &ctrlClient.CreateOptions{}) Expect(err).ShouldNot(HaveOccurred()) } bslName := "default" if test.useLongBSLName { bslName = longLabelName } r.deleteOrphanedBackups(ctx, bslName, test.cloudBackups, velerotest.NewLogger()) numBackups, err := numBackups(client) Expect(err).ShouldNot(HaveOccurred()) fmt.Println("") expected := len(test.k8sBackups) - len(test.expectedDeletes) Expect(expected).To(BeEquivalentTo(numBackups)) } }) It("Test moving default BSL at the head of BSL array.", func() { locationList := &velerov1api.BackupStorageLocationList{} objArray := make([]runtime.Object, 0) // Generate BSL array. locations := defaultLocationsList("velero") for _, bsl := range locations { objArray = append(objArray, bsl) } meta.SetList(locationList, objArray) testObjList := backupSyncSourceOrderFunc(locationList) testObjArray, err := meta.ExtractList(testObjList) Expect(err).ShouldNot(HaveOccurred()) expectLocation := testObjArray[0].(*velerov1api.BackupStorageLocation) Expect(expectLocation.Spec.Default).To(BeEquivalentTo(true)) // If BSL list without default BSL is passed in, the output should be same with input. locationList.Items = testObjList.(*velerov1api.BackupStorageLocationList).Items[1:] testObjList = backupSyncSourceOrderFunc(locationList) Expect(testObjList).To(BeEquivalentTo(locationList)) }) When("testing validateOwnerReferences", func() { testCases := []struct { name string backup *velerov1api.Backup toCreate []ctrlClient.Object expectedReferences []metav1.OwnerReference }{ { name: "handles empty owner references", backup: &velerov1api.Backup{ ObjectMeta: metav1.ObjectMeta{ OwnerReferences: []metav1.OwnerReference{}, }, }, expectedReferences: []metav1.OwnerReference{}, }, { name: "handles missing schedule", backup: &velerov1api.Backup{ ObjectMeta: metav1.ObjectMeta{ OwnerReferences: []metav1.OwnerReference{ { Kind: "Schedule", Name: "some name", }, }, }, }, expectedReferences: []metav1.OwnerReference{}, }, { name: "handles existing reference", backup: &velerov1api.Backup{ ObjectMeta: metav1.ObjectMeta{ OwnerReferences: []metav1.OwnerReference{ { Kind: "Schedule", Name: "existing-schedule", }, }, Namespace: "test-namespace", }, }, toCreate: []ctrlClient.Object{ &velerov1api.Schedule{ ObjectMeta: metav1.ObjectMeta{ Name: "existing-schedule", Namespace: "test-namespace", }, }, }, expectedReferences: []metav1.OwnerReference{ { Kind: "Schedule", Name: "existing-schedule", }, }, }, { name: "handles existing mismatched UID", backup: &velerov1api.Backup{ ObjectMeta: metav1.ObjectMeta{ OwnerReferences: []metav1.OwnerReference{ { Kind: "Schedule", Name: "existing-schedule", UID: "backup-UID", }, }, Namespace: "test-namespace", }, }, toCreate: []ctrlClient.Object{ &velerov1api.Schedule{ ObjectMeta: metav1.ObjectMeta{ Name: "existing-schedule", Namespace: "test-namespace", UID: "schedule-UID", }, }, }, expectedReferences: []metav1.OwnerReference{}, }, { name: "handles multiple references", backup: &velerov1api.Backup{ ObjectMeta: metav1.ObjectMeta{ OwnerReferences: []metav1.OwnerReference{ { Kind: "Schedule", Name: "existing-schedule", UID: "1", }, { Kind: "Schedule", Name: "missing-schedule", UID: "2", }, { Kind: "Schedule", Name: "mismatched-uid-schedule", UID: "3", }, { Kind: "Schedule", Name: "another-existing-schedule", UID: "4", }, }, Namespace: "test-namespace", }, }, toCreate: []ctrlClient.Object{ &velerov1api.Schedule{ ObjectMeta: metav1.ObjectMeta{ Name: "existing-schedule", Namespace: "test-namespace", UID: "1", }, }, &velerov1api.Schedule{ ObjectMeta: metav1.ObjectMeta{ Name: "mismatched-uid-schedule", Namespace: "test-namespace", UID: "not-3", }, }, &velerov1api.Schedule{ ObjectMeta: metav1.ObjectMeta{ Name: "another-existing-schedule", Namespace: "test-namespace", UID: "4", }, }, }, expectedReferences: []metav1.OwnerReference{ { Kind: "Schedule", Name: "existing-schedule", UID: "1", }, { Kind: "Schedule", Name: "another-existing-schedule", UID: "4", }, }, }, } for _, test := range testCases { It(test.name, func() { logger := velerotest.NewLogger() b := backupSyncReconciler{ client: ctrlfake.NewClientBuilder().Build(), } //create all required schedules as needed. for _, creatable := range test.toCreate { err := b.client.Create(context.Background(), creatable) Expect(err).ShouldNot(HaveOccurred()) } references := b.filterBackupOwnerReferences(context.Background(), test.backup, logger) Expect(references).To(BeEquivalentTo(test.expectedReferences)) }) } }) }) ================================================ FILE: pkg/controller/backup_tracker.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package controller import ( "fmt" "sync" "k8s.io/apimachinery/pkg/util/sets" ) // BackupTracker keeps track of in-progress backups. type BackupTracker interface { // Add informs the tracker that a backup is ReadyToStart. AddReadyToStart(ns, name string) // Add informs the tracker that a backup is in progress. Add(ns, name string) // Add informs the tracker that a backup has moved beyond InProgress AddPostProcessing(ns, name string) // Delete informs the tracker that a backup has reached a terminal state. Delete(ns, name string) // Contains returns true if backup is InProgress or post-InProgress Contains(ns, name string) bool // RunningCount returns the number of backups which are ReadyToStart or InProgress RunningCount() int } type backupTracker struct { lock sync.RWMutex readyToStartBackups sets.Set[string] inProgressBackups sets.Set[string] postProgressBackups sets.Set[string] } // NewBackupTracker returns a new BackupTracker. func NewBackupTracker() BackupTracker { return &backupTracker{ readyToStartBackups: sets.New[string](), inProgressBackups: sets.New[string](), postProgressBackups: sets.New[string](), } } func (bt *backupTracker) AddReadyToStart(ns, name string) { bt.lock.Lock() defer bt.lock.Unlock() bt.readyToStartBackups.Insert(backupTrackerKey(ns, name)) } func (bt *backupTracker) Add(ns, name string) { bt.lock.Lock() defer bt.lock.Unlock() key := backupTrackerKey(ns, name) bt.readyToStartBackups.Delete(key) bt.inProgressBackups.Insert(key) } func (bt *backupTracker) AddPostProcessing(ns, name string) { bt.lock.Lock() defer bt.lock.Unlock() key := backupTrackerKey(ns, name) bt.readyToStartBackups.Delete(key) bt.inProgressBackups.Delete(key) bt.postProgressBackups.Insert(key) } func (bt *backupTracker) Delete(ns, name string) { bt.lock.Lock() defer bt.lock.Unlock() key := backupTrackerKey(ns, name) bt.readyToStartBackups.Delete(key) bt.inProgressBackups.Delete(key) bt.postProgressBackups.Delete(key) } // Contains returns true if backup is InProgress or post-InProgress // ignores ReadyToStart, since this is used to determine whether // a backup is in progress and thus not able to be deleted now. func (bt *backupTracker) Contains(ns, name string) bool { bt.lock.RLock() defer bt.lock.RUnlock() key := backupTrackerKey(ns, name) return bt.inProgressBackups.Has(key) || bt.postProgressBackups.Has(key) } // RunningCount returns the number of backups which are ReadyToStart or InProgress // used by queue controller to determine whether a new backup can be started. func (bt *backupTracker) RunningCount() int { bt.lock.RLock() defer bt.lock.RUnlock() return bt.inProgressBackups.Len() + bt.readyToStartBackups.Len() } func backupTrackerKey(ns, name string) string { return fmt.Sprintf("%s/%s", ns, name) } ================================================ FILE: pkg/controller/backup_tracker_test.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package controller import ( "testing" "github.com/stretchr/testify/assert" ) func TestBackupTracker(t *testing.T) { bt := NewBackupTracker() assert.False(t, bt.Contains("ns", "name")) bt.Add("ns", "name") assert.True(t, bt.Contains("ns", "name")) bt.Add("ns2", "name2") assert.True(t, bt.Contains("ns", "name")) assert.True(t, bt.Contains("ns2", "name2")) bt.Delete("ns", "name") assert.False(t, bt.Contains("ns", "name")) assert.True(t, bt.Contains("ns2", "name2")) bt.Delete("ns2", "name2") assert.False(t, bt.Contains("ns2", "name2")) } ================================================ FILE: pkg/controller/data_download_controller.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package controller import ( "context" "fmt" "strings" "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" "k8s.io/utils/clock" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" "github.com/vmware-tanzu/velero/pkg/apis/velero/shared" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" "github.com/vmware-tanzu/velero/pkg/constant" datamover "github.com/vmware-tanzu/velero/pkg/datamover" "github.com/vmware-tanzu/velero/pkg/datapath" "github.com/vmware-tanzu/velero/pkg/exposer" "github.com/vmware-tanzu/velero/pkg/metrics" "github.com/vmware-tanzu/velero/pkg/nodeagent" repository "github.com/vmware-tanzu/velero/pkg/repository/manager" velerotypes "github.com/vmware-tanzu/velero/pkg/types" "github.com/vmware-tanzu/velero/pkg/uploader" "github.com/vmware-tanzu/velero/pkg/util" "github.com/vmware-tanzu/velero/pkg/util/kube" ) // DataDownloadReconciler reconciles a DataDownload object type DataDownloadReconciler struct { client client.Client kubeClient kubernetes.Interface mgr manager.Manager logger logrus.FieldLogger Clock clock.WithTickerAndDelayedExecution restoreExposer exposer.GenericRestoreExposer nodeName string dataPathMgr *datapath.Manager vgdpCounter *exposer.VgdpCounter loadAffinity []*kube.LoadAffinity restorePVCConfig velerotypes.RestorePVC backupRepoConfigs map[string]string cacheVolumeConfigs *velerotypes.CachePVC podResources corev1api.ResourceRequirements preparingTimeout time.Duration metrics *metrics.ServerMetrics cancelledDataDownload map[string]time.Time dataMovePriorityClass string repoConfigMgr repository.ConfigManager podLabels map[string]string podAnnotations map[string]string } func NewDataDownloadReconciler( client client.Client, mgr manager.Manager, kubeClient kubernetes.Interface, dataPathMgr *datapath.Manager, counter *exposer.VgdpCounter, loadAffinity []*kube.LoadAffinity, restorePVCConfig velerotypes.RestorePVC, backupRepoConfigs map[string]string, cacheVolumeConfigs *velerotypes.CachePVC, podResources corev1api.ResourceRequirements, nodeName string, preparingTimeout time.Duration, logger logrus.FieldLogger, metrics *metrics.ServerMetrics, dataMovePriorityClass string, repoConfigMgr repository.ConfigManager, podLabels map[string]string, podAnnotations map[string]string, ) *DataDownloadReconciler { return &DataDownloadReconciler{ client: client, kubeClient: kubeClient, mgr: mgr, logger: logger.WithField("controller", "DataDownload"), Clock: &clock.RealClock{}, nodeName: nodeName, restoreExposer: exposer.NewGenericRestoreExposer(kubeClient, logger), restorePVCConfig: restorePVCConfig, backupRepoConfigs: backupRepoConfigs, cacheVolumeConfigs: cacheVolumeConfigs, dataPathMgr: dataPathMgr, vgdpCounter: counter, loadAffinity: loadAffinity, podResources: podResources, preparingTimeout: preparingTimeout, metrics: metrics, cancelledDataDownload: make(map[string]time.Time), dataMovePriorityClass: dataMovePriorityClass, repoConfigMgr: repoConfigMgr, podLabels: podLabels, podAnnotations: podAnnotations, } } // +kubebuilder:rbac:groups=velero.io,resources=datadownloads,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=velero.io,resources=datadownloads/status,verbs=get;update;patch // +kubebuilder:rbac:groups="",resources=pods,verbs=get // +kubebuilder:rbac:groups="",resources=persistentvolumes,verbs=get // +kubebuilder:rbac:groups="",resources=persistentvolumerclaims,verbs=get func (r *DataDownloadReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := r.logger.WithFields(logrus.Fields{ "controller": "datadownload", "datadownload": req.NamespacedName, }) log.Infof("Reconcile %s", req.Name) dd := &velerov2alpha1api.DataDownload{} if err := r.client.Get(ctx, types.NamespacedName{Namespace: req.Namespace, Name: req.Name}, dd); err != nil { if apierrors.IsNotFound(err) { log.Warn("DataDownload not found, skip") return ctrl.Result{}, nil } log.WithError(err).Error("Unable to get the DataDownload") return ctrl.Result{}, err } if !datamover.IsBuiltInUploader(dd.Spec.DataMover) { log.WithField("data mover", dd.Spec.DataMover).Info("it is not one built-in data mover which is not supported by Velero") return ctrl.Result{}, nil } // Logic for clear resources when datadownload been deleted if !isDataDownloadInFinalState(dd) { if !controllerutil.ContainsFinalizer(dd, DataUploadDownloadFinalizer) { if err := UpdateDataDownloadWithRetry(ctx, r.client, req.NamespacedName, log, func(dd *velerov2alpha1api.DataDownload) bool { if controllerutil.ContainsFinalizer(dd, DataUploadDownloadFinalizer) { return false } controllerutil.AddFinalizer(dd, DataUploadDownloadFinalizer) return true }); err != nil { log.WithError(err).Errorf("failed to add finalizer for dd %s/%s", dd.Namespace, dd.Name) return ctrl.Result{}, err } return ctrl.Result{}, nil } if !dd.DeletionTimestamp.IsZero() { if !dd.Spec.Cancel { // when delete cr we need to clear up internal resources created by Velero, here we use the cancel mechanism // to help clear up resources instead of clear them directly in case of some conflict with Expose action log.Warnf("Cancel dd under phase %s because it is being deleted", dd.Status.Phase) if err := UpdateDataDownloadWithRetry(ctx, r.client, req.NamespacedName, log, func(dataDownload *velerov2alpha1api.DataDownload) bool { if dataDownload.Spec.Cancel { return false } dataDownload.Spec.Cancel = true dataDownload.Status.Message = "Cancel datadownload because it is being deleted" return true }); err != nil { log.WithError(err).Errorf("failed to set cancel flag for dd %s/%s", dd.Namespace, dd.Name) return ctrl.Result{}, err } return ctrl.Result{}, nil } } } else { delete(r.cancelledDataDownload, dd.Name) // put the finalizer remove action here for all cr will goes to the final status, we could check finalizer and do remove action in final status // instead of intermediate state. // remove finalizer no matter whether the cr is being deleted or not for it is no longer needed when internal resources are all cleaned up // also in final status cr won't block the direct delete of the velero namespace if controllerutil.ContainsFinalizer(dd, DataUploadDownloadFinalizer) { if err := UpdateDataDownloadWithRetry(ctx, r.client, req.NamespacedName, log, func(dd *velerov2alpha1api.DataDownload) bool { if !controllerutil.ContainsFinalizer(dd, DataUploadDownloadFinalizer) { return false } controllerutil.RemoveFinalizer(dd, DataUploadDownloadFinalizer) return true }); err != nil { log.WithError(err).Error("error to remove finalizer") return ctrl.Result{}, err } return ctrl.Result{}, nil } } if dd.Spec.Cancel { if spotted, found := r.cancelledDataDownload[dd.Name]; !found { r.cancelledDataDownload[dd.Name] = r.Clock.Now() } else { delay := cancelDelayOthers if dd.Status.Phase == velerov2alpha1api.DataDownloadPhaseInProgress { delay = cancelDelayInProgress } if time.Since(spotted) > delay { log.Infof("Data download %s is canceled in Phase %s but not handled in rasonable time", dd.GetName(), dd.Status.Phase) if r.tryCancelDataDownload(ctx, dd, "") { delete(r.cancelledDataDownload, dd.Name) } return ctrl.Result{}, nil } } } if dd.Status.Phase == "" || dd.Status.Phase == velerov2alpha1api.DataDownloadPhaseNew { if dd.Spec.Cancel { log.Debugf("Data download is canceled in Phase %s", dd.Status.Phase) r.tryCancelDataDownload(ctx, dd, "") return ctrl.Result{}, nil } if r.vgdpCounter != nil && r.vgdpCounter.IsConstrained(ctx, r.logger) { log.Debug("Data path initiation is constrained, requeue later") return ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, nil } if _, err := r.getTargetPVC(ctx, dd); err != nil { log.WithField("error", err).Debugf("Cannot find target PVC for DataDownload yet. Retry later.") return ctrl.Result{Requeue: true}, nil } log.Info("Data download starting") accepted, err := r.acceptDataDownload(ctx, dd) if err != nil { return ctrl.Result{}, errors.Wrapf(err, "error accepting the data download %s", dd.Name) } if !accepted { log.Debug("Data download is not accepted") return ctrl.Result{}, nil } log.Info("Data download is accepted") exposeParam, err := r.setupExposeParam(dd) if err != nil { return r.errorOut(ctx, dd, err, "failed to set exposer parameters", log) } // Expose() will trigger to create one pod whose volume is restored by a given volume snapshot, // but the pod maybe is not in the same node of the current controller, so we need to return it here. // And then only the controller who is in the same node could do the rest work. err = r.restoreExposer.Expose(ctx, getDataDownloadOwnerObject(dd), exposeParam) if err != nil { return r.errorOut(ctx, dd, err, "error to expose snapshot", log) } log.Info("Restore is exposed") return ctrl.Result{}, nil } else if dd.Status.Phase == velerov2alpha1api.DataDownloadPhaseAccepted { if peekErr := r.restoreExposer.PeekExposed(ctx, getDataDownloadOwnerObject(dd)); peekErr != nil { log.Errorf("Cancel dd %s/%s because of expose error %s", dd.Namespace, dd.Name, peekErr) diags := strings.Split(r.restoreExposer.DiagnoseExpose(ctx, getDataDownloadOwnerObject(dd)), "\n") for _, diag := range diags { log.Warnf("[Diagnose DD expose]%s", diag) } r.tryCancelDataDownload(ctx, dd, fmt.Sprintf("found a datadownload %s/%s with expose error: %s. mark it as cancel", dd.Namespace, dd.Name, peekErr)) } else if dd.Status.AcceptedTimestamp != nil { if time.Since(dd.Status.AcceptedTimestamp.Time) >= r.preparingTimeout { r.onPrepareTimeout(ctx, dd) } } return ctrl.Result{}, nil } else if dd.Status.Phase == velerov2alpha1api.DataDownloadPhasePrepared { log.Infof("Data download is prepared and should be processed by %s (%s)", dd.Status.Node, r.nodeName) if dd.Status.Node != r.nodeName { return ctrl.Result{}, nil } if dd.Spec.Cancel { log.Debugf("Data download is been canceled %s in Phase %s", dd.GetName(), dd.Status.Phase) r.OnDataDownloadCancelled(ctx, dd.GetNamespace(), dd.GetName()) return ctrl.Result{}, nil } asyncBR := r.dataPathMgr.GetAsyncBR(dd.Name) if asyncBR != nil { log.Info("Cancellable data path is already started") return ctrl.Result{}, nil } result, err := r.restoreExposer.GetExposed(ctx, getDataDownloadOwnerObject(dd), r.client, r.nodeName, dd.Spec.OperationTimeout.Duration) if err != nil { return r.errorOut(ctx, dd, err, "restore exposer is not ready", log) } else if result == nil { return r.errorOut(ctx, dd, errors.New("no expose result is available for the current node"), "exposed snapshot is not ready", log) } log.Info("Restore PVC is ready and creating data path routine") // Need to first create file system BR and get data path instance then update data download status callbacks := datapath.Callbacks{ OnCompleted: r.OnDataDownloadCompleted, OnFailed: r.OnDataDownloadFailed, OnCancelled: r.OnDataDownloadCancelled, OnProgress: r.OnDataDownloadProgress, } asyncBR, err = r.dataPathMgr.CreateMicroServiceBRWatcher(ctx, r.client, r.kubeClient, r.mgr, datapath.TaskTypeRestore, dd.Name, dd.Namespace, result.ByPod.HostingPod.Name, result.ByPod.HostingContainer, dd.Name, callbacks, false, log) if err != nil { if err == datapath.ConcurrentLimitExceed { log.Debug("Data path instance is concurrent limited requeue later") return ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, nil } else { return r.errorOut(ctx, dd, err, "error to create data path", log) } } if err := r.initCancelableDataPath(ctx, asyncBR, result, log); err != nil { log.WithError(err).Errorf("Failed to init cancelable data path for %s", dd.Name) r.closeDataPath(ctx, dd.Name) return r.errorOut(ctx, dd, err, "error initializing data path", log) } // Update status to InProgress terminated := false if err := UpdateDataDownloadWithRetry(ctx, r.client, types.NamespacedName{Namespace: dd.Namespace, Name: dd.Name}, log, func(dd *velerov2alpha1api.DataDownload) bool { if isDataDownloadInFinalState(dd) { terminated = true return false } dd.Status.Phase = velerov2alpha1api.DataDownloadPhaseInProgress dd.Status.StartTimestamp = &metav1.Time{Time: r.Clock.Now()} delete(dd.Labels, exposer.ExposeOnGoingLabel) return true }); err != nil { log.WithError(err).Warnf("Failed to update datadownload %s to InProgress, will data path close and retry", dd.Name) r.closeDataPath(ctx, dd.Name) return ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, nil } if terminated { log.Warnf("datadownload %s is terminated during transition from prepared", dd.Name) r.closeDataPath(ctx, dd.Name) return ctrl.Result{}, nil } log.Info("Data download is marked as in progress") if err := r.startCancelableDataPath(asyncBR, dd, result, log); err != nil { log.WithError(err).Errorf("Failed to start cancelable data path for %s", dd.Name) r.closeDataPath(ctx, dd.Name) return r.errorOut(ctx, dd, err, "error starting data path", log) } return ctrl.Result{}, nil } else if dd.Status.Phase == velerov2alpha1api.DataDownloadPhaseInProgress { if dd.Spec.Cancel { if dd.Status.Node != r.nodeName { return ctrl.Result{}, nil } log.Info("In progress data download is being canceled") asyncBR := r.dataPathMgr.GetAsyncBR(dd.Name) if asyncBR == nil { r.OnDataDownloadCancelled(ctx, dd.GetNamespace(), dd.GetName()) return ctrl.Result{}, nil } // Update status to Canceling. if err := UpdateDataDownloadWithRetry(ctx, r.client, types.NamespacedName{Namespace: dd.Namespace, Name: dd.Name}, log, func(dd *velerov2alpha1api.DataDownload) bool { if isDataDownloadInFinalState(dd) { log.Warnf("datadownload %s is terminated, abort setting it to canceling", dd.Name) return false } dd.Status.Phase = velerov2alpha1api.DataDownloadPhaseCanceling return true }); err != nil { log.WithError(err).Error("error updating data download into canceling status") return ctrl.Result{}, err } asyncBR.Cancel() return ctrl.Result{}, nil } return ctrl.Result{}, nil } return ctrl.Result{}, nil } func (r *DataDownloadReconciler) initCancelableDataPath(ctx context.Context, asyncBR datapath.AsyncBR, res *exposer.ExposeResult, log logrus.FieldLogger) error { log.Info("Init cancelable dataDownload") if err := asyncBR.Init(ctx, nil); err != nil { return errors.Wrap(err, "error initializing asyncBR") } log.Infof("async restore init for pod %s, volume %s", res.ByPod.HostingPod.Name, res.ByPod.VolumeName) return nil } func (r *DataDownloadReconciler) startCancelableDataPath(asyncBR datapath.AsyncBR, dd *velerov2alpha1api.DataDownload, res *exposer.ExposeResult, log logrus.FieldLogger) error { log.Info("Start cancelable dataDownload") if err := asyncBR.StartRestore(dd.Spec.SnapshotID, datapath.AccessPoint{ ByPath: res.ByPod.VolumeName, }, dd.Spec.DataMoverConfig); err != nil { return errors.Wrapf(err, "error starting async restore for pod %s, volume %s", res.ByPod.HostingPod.Name, res.ByPod.VolumeName) } log.Infof("Async restore started for pod %s, volume %s", res.ByPod.HostingPod.Name, res.ByPod.VolumeName) return nil } func (r *DataDownloadReconciler) OnDataDownloadCompleted(ctx context.Context, namespace string, ddName string, result datapath.Result) { defer r.dataPathMgr.RemoveAsyncBR(ddName) log := r.logger.WithField("datadownload", ddName) log.Info("Async fs restore data path completed") var dd velerov2alpha1api.DataDownload if err := r.client.Get(ctx, types.NamespacedName{Name: ddName, Namespace: namespace}, &dd); err != nil { log.WithError(err).Warn("Failed to get datadownload on completion") return } objRef := getDataDownloadOwnerObject(&dd) err := r.restoreExposer.RebindVolume(ctx, objRef, dd.Spec.TargetVolume.PVC, dd.Spec.TargetVolume.Namespace, dd.Spec.OperationTimeout.Duration) if err != nil { log.WithError(err).Error("Failed to rebind PV to target PVC on completion") return } log.Info("Cleaning up exposed environment") r.restoreExposer.CleanUp(ctx, objRef) if err := UpdateDataDownloadWithRetry(ctx, r.client, types.NamespacedName{Namespace: dd.Namespace, Name: dd.Name}, log, func(dd *velerov2alpha1api.DataDownload) bool { if isDataDownloadInFinalState(dd) { return false } dd.Status.Phase = velerov2alpha1api.DataDownloadPhaseCompleted dd.Status.CompletionTimestamp = &metav1.Time{Time: r.Clock.Now()} delete(dd.Labels, exposer.ExposeOnGoingLabel) return true }); err != nil { log.WithError(err).Error("error updating data download status") } else { log.Infof("Data download is marked as %s", dd.Status.Phase) r.metrics.RegisterDataDownloadSuccess(r.nodeName) } } func (r *DataDownloadReconciler) OnDataDownloadFailed(ctx context.Context, namespace string, ddName string, err error) { defer r.dataPathMgr.RemoveAsyncBR(ddName) log := r.logger.WithField("datadownload", ddName) log.WithError(err).Error("Async fs restore data path failed") var dd velerov2alpha1api.DataDownload if getErr := r.client.Get(ctx, types.NamespacedName{Name: ddName, Namespace: namespace}, &dd); getErr != nil { log.WithError(getErr).Warn("Failed to get data download on failure") } else { _, _ = r.errorOut(ctx, &dd, err, "data path restore failed", log) } } func (r *DataDownloadReconciler) OnDataDownloadCancelled(ctx context.Context, namespace string, ddName string) { defer r.dataPathMgr.RemoveAsyncBR(ddName) log := r.logger.WithField("datadownload", ddName) log.Warn("Async fs backup data path canceled") var dd velerov2alpha1api.DataDownload if getErr := r.client.Get(ctx, types.NamespacedName{Name: ddName, Namespace: namespace}, &dd); getErr != nil { log.WithError(getErr).Warn("Failed to get datadownload on cancel") return } // cleans up any objects generated during the snapshot expose r.restoreExposer.CleanUp(ctx, getDataDownloadOwnerObject(&dd)) if err := UpdateDataDownloadWithRetry(ctx, r.client, types.NamespacedName{Namespace: dd.Namespace, Name: dd.Name}, log, func(dd *velerov2alpha1api.DataDownload) bool { if isDataDownloadInFinalState(dd) { return false } dd.Status.Phase = velerov2alpha1api.DataDownloadPhaseCanceled if dd.Status.StartTimestamp.IsZero() { dd.Status.StartTimestamp = &metav1.Time{Time: r.Clock.Now()} } dd.Status.CompletionTimestamp = &metav1.Time{Time: r.Clock.Now()} delete(dd.Labels, exposer.ExposeOnGoingLabel) return true }); err != nil { log.WithError(err).Error("error updating data download status") } else { r.metrics.RegisterDataDownloadCancel(r.nodeName) delete(r.cancelledDataDownload, dd.Name) } } func (r *DataDownloadReconciler) tryCancelDataDownload(ctx context.Context, dd *velerov2alpha1api.DataDownload, message string) bool { log := r.logger.WithField("datadownload", dd.Name) succeeded, err := funcExclusiveUpdateDataDownload(ctx, r.client, dd, func(dataDownload *velerov2alpha1api.DataDownload) { dataDownload.Status.Phase = velerov2alpha1api.DataDownloadPhaseCanceled if dataDownload.Status.StartTimestamp.IsZero() { dataDownload.Status.StartTimestamp = &metav1.Time{Time: r.Clock.Now()} } dataDownload.Status.CompletionTimestamp = &metav1.Time{Time: r.Clock.Now()} if message != "" { dataDownload.Status.Message = message } delete(dataDownload.Labels, exposer.ExposeOnGoingLabel) }) if err != nil { log.WithError(err).Error("error updating datadownload status") return false } else if !succeeded { log.Warn("conflict in updating datadownload status and will try it again later") return false } // success update r.metrics.RegisterDataDownloadCancel(r.nodeName) r.restoreExposer.CleanUp(ctx, getDataDownloadOwnerObject(dd)) log.Warn("data download is canceled") return true } func (r *DataDownloadReconciler) OnDataDownloadProgress(ctx context.Context, namespace string, ddName string, progress *uploader.Progress) { log := r.logger.WithField("datadownload", ddName) if err := UpdateDataDownloadWithRetry(ctx, r.client, types.NamespacedName{Namespace: namespace, Name: ddName}, log, func(dd *velerov2alpha1api.DataDownload) bool { dd.Status.Progress = shared.DataMoveOperationProgress{TotalBytes: progress.TotalBytes, BytesDone: progress.BytesDone} return true }); err != nil { log.WithError(err).Error("Failed to update progress") } } // SetupWithManager registers the DataDownload controller. // The fresh new DataDownload CR first created will trigger to create one pod (long time, maybe failure or unknown status) by one of the datadownload controllers // then the request will get out of the Reconcile queue immediately by not blocking others' CR handling, in order to finish the rest data download process we need to // re-enqueue the previous related request once the related pod is in running status to keep going on the rest logic. and below logic will avoid handling the unwanted // pod status and also avoid block others CR handling func (r *DataDownloadReconciler) SetupWithManager(mgr ctrl.Manager) error { gp := kube.NewGenericEventPredicate(func(object client.Object) bool { dd := object.(*velerov2alpha1api.DataDownload) if dd.Status.Phase == velerov2alpha1api.DataDownloadPhaseAccepted { return true } if dd.Spec.Cancel && !isDataDownloadInFinalState(dd) { return true } if isDataDownloadInFinalState(dd) && !dd.DeletionTimestamp.IsZero() { return true } return false }) s := kube.NewPeriodicalEnqueueSource(r.logger.WithField("controller", constant.ControllerDataDownload), r.client, &velerov2alpha1api.DataDownloadList{}, preparingMonitorFrequency, kube.PeriodicalEnqueueSourceOption{ Predicates: []predicate.Predicate{gp}, }) return ctrl.NewControllerManagedBy(mgr). For(&velerov2alpha1api.DataDownload{}). WatchesRawSource(s). Watches(&corev1api.Pod{}, kube.EnqueueRequestsFromMapUpdateFunc(r.findSnapshotRestoreForPod), builder.WithPredicates(predicate.Funcs{ UpdateFunc: func(ue event.UpdateEvent) bool { newObj := ue.ObjectNew.(*corev1api.Pod) if _, ok := newObj.Labels[velerov1api.DataDownloadLabel]; !ok { return false } if newObj.Spec.NodeName == "" { return false } return true }, CreateFunc: func(event.CreateEvent) bool { return false }, DeleteFunc: func(de event.DeleteEvent) bool { return false }, GenericFunc: func(ge event.GenericEvent) bool { return false }, })). Complete(r) } func (r *DataDownloadReconciler) findSnapshotRestoreForPod(ctx context.Context, podObj client.Object) []reconcile.Request { pod := podObj.(*corev1api.Pod) dd, err := findDataDownloadByPod(r.client, *pod) log := r.logger.WithField("pod", pod.Name) if err != nil { log.WithError(err).Error("unable to get DataDownload") return []reconcile.Request{} } else if dd == nil { log.Error("get empty DataDownload") return []reconcile.Request{} } log = log.WithFields(logrus.Fields{ "Dataddownload": dd.Name, }) if dd.Status.Phase != velerov2alpha1api.DataDownloadPhaseAccepted { return []reconcile.Request{} } if pod.Status.Phase == corev1api.PodRunning { log.Info("Preparing data download") if err = UpdateDataDownloadWithRetry(context.Background(), r.client, types.NamespacedName{Namespace: dd.Namespace, Name: dd.Name}, log, func(dd *velerov2alpha1api.DataDownload) bool { if isDataDownloadInFinalState(dd) { log.Warnf("datadownload %s is terminated, abort setting it to prepared", dd.Name) return false } r.prepareDataDownload(dd) return true }); err != nil { log.WithError(err).Warn("failed to update dataudownload, prepare will halt for this dataudownload") return []reconcile.Request{} } } else if unrecoverable, reason := kube.IsPodUnrecoverable(pod, log); unrecoverable { err := UpdateDataDownloadWithRetry(context.Background(), r.client, types.NamespacedName{Namespace: dd.Namespace, Name: dd.Name}, r.logger.WithField("datadownlad", dd.Name), func(dataDownload *velerov2alpha1api.DataDownload) bool { if dataDownload.Spec.Cancel { return false } dataDownload.Spec.Cancel = true dataDownload.Status.Message = fmt.Sprintf("Cancel datadownload because the exposing pod %s/%s is in abnormal status for reason %s", pod.Namespace, pod.Name, reason) return true }) if err != nil { log.WithError(err).Warn("failed to cancel datadownload, and it will wait for prepare timeout") return []reconcile.Request{} } log.Infof("Exposed pod is in abnormal status(reason %s) and datadownload is marked as cancel", reason) } else { return []reconcile.Request{} } request := reconcile.Request{ NamespacedName: types.NamespacedName{ Namespace: dd.Namespace, Name: dd.Name, }, } return []reconcile.Request{request} } func (r *DataDownloadReconciler) prepareDataDownload(ssb *velerov2alpha1api.DataDownload) { ssb.Status.Phase = velerov2alpha1api.DataDownloadPhasePrepared ssb.Status.Node = r.nodeName } func (r *DataDownloadReconciler) errorOut(ctx context.Context, dd *velerov2alpha1api.DataDownload, err error, msg string, log logrus.FieldLogger) (ctrl.Result, error) { if r.restoreExposer != nil { r.restoreExposer.CleanUp(ctx, getDataDownloadOwnerObject(dd)) } return ctrl.Result{}, r.updateStatusToFailed(ctx, dd, err, msg, log) } func (r *DataDownloadReconciler) updateStatusToFailed(ctx context.Context, dd *velerov2alpha1api.DataDownload, err error, msg string, log logrus.FieldLogger) error { log.Info("update data download status to Failed") if patchErr := UpdateDataDownloadWithRetry(ctx, r.client, types.NamespacedName{Namespace: dd.Namespace, Name: dd.Name}, log, func(dd *velerov2alpha1api.DataDownload) bool { if isDataDownloadInFinalState(dd) { return false } dd.Status.Phase = velerov2alpha1api.DataDownloadPhaseFailed dd.Status.Message = errors.WithMessage(err, msg).Error() dd.Status.CompletionTimestamp = &metav1.Time{Time: r.Clock.Now()} delete(dd.Labels, exposer.ExposeOnGoingLabel) return true }); patchErr != nil { log.WithError(patchErr).Error("error updating DataDownload status") } else { r.metrics.RegisterDataDownloadFailure(r.nodeName) } return err } func (r *DataDownloadReconciler) acceptDataDownload(ctx context.Context, dd *velerov2alpha1api.DataDownload) (bool, error) { r.logger.Infof("Accepting data download %s", dd.Name) // For all data download controller in each node-agent will try to update download CR, and only one controller will success, // and the success one could handle later logic updated := dd.DeepCopy() updateFunc := func(datadownload *velerov2alpha1api.DataDownload) { datadownload.Status.Phase = velerov2alpha1api.DataDownloadPhaseAccepted datadownload.Status.AcceptedByNode = r.nodeName datadownload.Status.AcceptedTimestamp = &metav1.Time{Time: r.Clock.Now()} if datadownload.Labels == nil { datadownload.Labels = make(map[string]string) } datadownload.Labels[exposer.ExposeOnGoingLabel] = "true" } succeeded, err := funcExclusiveUpdateDataDownload(ctx, r.client, updated, updateFunc) if err != nil { return false, err } if succeeded { updateFunc(dd) // If update success, it's need to update du values in memory r.logger.WithField("DataDownload", dd.Name).Infof("This datadownload has been accepted by %s", r.nodeName) return true, nil } r.logger.WithField("DataDownload", dd.Name).Info("This datadownload has been accepted by others") return false, nil } func (r *DataDownloadReconciler) onPrepareTimeout(ctx context.Context, dd *velerov2alpha1api.DataDownload) { log := r.logger.WithField("DataDownload", dd.Name) log.Info("Timeout happened for preparing datadownload") succeeded, err := funcExclusiveUpdateDataDownload(ctx, r.client, dd, func(dd *velerov2alpha1api.DataDownload) { dd.Status.Phase = velerov2alpha1api.DataDownloadPhaseFailed dd.Status.Message = "timeout on preparing data download" delete(dd.Labels, exposer.ExposeOnGoingLabel) }) if err != nil { log.WithError(err).Warn("Failed to update datadownload") return } if !succeeded { log.Warn("Datadownload has been updated by others") return } diags := strings.Split(r.restoreExposer.DiagnoseExpose(ctx, getDataDownloadOwnerObject(dd)), "\n") for _, diag := range diags { log.Warnf("[Diagnose DD expose]%s", diag) } r.restoreExposer.CleanUp(ctx, getDataDownloadOwnerObject(dd)) log.Info("Datadownload has been cleaned up") r.metrics.RegisterDataDownloadFailure(r.nodeName) } var funcExclusiveUpdateDataDownload = exclusiveUpdateDataDownload func exclusiveUpdateDataDownload(ctx context.Context, cli client.Client, dd *velerov2alpha1api.DataDownload, updateFunc func(*velerov2alpha1api.DataDownload)) (bool, error) { updateFunc(dd) err := cli.Update(ctx, dd) if err == nil { return true, nil } // it won't rollback dd in memory when error if apierrors.IsConflict(err) { return false, nil } else { return false, err } } func (r *DataDownloadReconciler) getTargetPVC(ctx context.Context, dd *velerov2alpha1api.DataDownload) (*corev1api.PersistentVolumeClaim, error) { return r.kubeClient.CoreV1().PersistentVolumeClaims(dd.Spec.TargetVolume.Namespace).Get(ctx, dd.Spec.TargetVolume.PVC, metav1.GetOptions{}) } func (r *DataDownloadReconciler) closeDataPath(ctx context.Context, ddName string) { asyncBR := r.dataPathMgr.GetAsyncBR(ddName) if asyncBR != nil { asyncBR.Close(ctx) } r.dataPathMgr.RemoveAsyncBR(ddName) } func (r *DataDownloadReconciler) setupExposeParam(dd *velerov2alpha1api.DataDownload) (exposer.GenericRestoreExposeParam, error) { log := r.logger.WithField("datadownload", dd.Name) nodeOS := string(dd.Spec.NodeOS) if nodeOS == "" { log.Info("nodeOS is empty in DD, fallback to linux") nodeOS = kube.NodeOSLinux } if err := kube.HasNodeWithOS(context.Background(), nodeOS, r.kubeClient.CoreV1()); err != nil { return exposer.GenericRestoreExposeParam{}, errors.Wrapf(err, "no appropriate node to run datadownload %s/%s", dd.Namespace, dd.Name) } hostingPodLabels := map[string]string{velerov1api.DataDownloadLabel: dd.Name} if len(r.podLabels) > 0 { for k, v := range r.podLabels { hostingPodLabels[k] = v } } else { for _, k := range util.ThirdPartyLabels { if v, err := nodeagent.GetLabelValue(context.Background(), r.kubeClient, dd.Namespace, k, nodeOS); err != nil { if err != nodeagent.ErrNodeAgentLabelNotFound { log.WithError(err).Warnf("Failed to check node-agent label, skip adding host pod label %s", k) } } else { hostingPodLabels[k] = v } } } hostingPodAnnotation := map[string]string{} if len(r.podAnnotations) > 0 { for k, v := range r.podAnnotations { hostingPodAnnotation[k] = v } } else { for _, k := range util.ThirdPartyAnnotations { if v, err := nodeagent.GetAnnotationValue(context.Background(), r.kubeClient, dd.Namespace, k, nodeOS); err != nil { if err != nodeagent.ErrNodeAgentAnnotationNotFound { log.WithError(err).Warnf("Failed to check node-agent annotation, skip adding host pod annotation %s", k) } } else { hostingPodAnnotation[k] = v } } } hostingPodTolerations := []corev1api.Toleration{} for _, k := range util.ThirdPartyTolerations { if v, err := nodeagent.GetToleration(context.Background(), r.kubeClient, dd.Namespace, k, nodeOS); err != nil { if err != nodeagent.ErrNodeAgentTolerationNotFound { log.WithError(err).Warnf("Failed to check node-agent toleration, skip adding host pod toleration %s", k) } } else { hostingPodTolerations = append(hostingPodTolerations, *v) } } var cacheVolume *exposer.CacheConfigs if r.cacheVolumeConfigs != nil { if limit, err := r.repoConfigMgr.ClientSideCacheLimit(velerov1api.BackupRepositoryTypeKopia, r.backupRepoConfigs); err != nil { log.WithError(err).Warnf("Failed to get client side cache limit for repo type %s from configs %v", velerov1api.BackupRepositoryTypeKopia, r.backupRepoConfigs) } else { cacheVolume = &exposer.CacheConfigs{ Limit: limit, StorageClass: r.cacheVolumeConfigs.StorageClass, ResidentThreshold: r.cacheVolumeConfigs.ResidentThresholdInMB << 20, } } } return exposer.GenericRestoreExposeParam{ TargetPVCName: dd.Spec.TargetVolume.PVC, TargetNamespace: dd.Spec.TargetVolume.Namespace, HostingPodLabels: hostingPodLabels, HostingPodAnnotations: hostingPodAnnotation, HostingPodTolerations: hostingPodTolerations, Resources: r.podResources, OperationTimeout: dd.Spec.OperationTimeout.Duration, ExposeTimeout: r.preparingTimeout, NodeOS: nodeOS, RestorePVCConfig: r.restorePVCConfig, LoadAffinity: r.loadAffinity, PriorityClassName: r.dataMovePriorityClass, RestoreSize: dd.Spec.SnapshotSize, CacheVolume: cacheVolume, }, nil } func getDataDownloadOwnerObject(dd *velerov2alpha1api.DataDownload) corev1api.ObjectReference { return corev1api.ObjectReference{ Kind: dd.Kind, Namespace: dd.Namespace, Name: dd.Name, UID: dd.UID, APIVersion: dd.APIVersion, } } func findDataDownloadByPod(client client.Client, pod corev1api.Pod) (*velerov2alpha1api.DataDownload, error) { if label, exist := pod.Labels[velerov1api.DataDownloadLabel]; exist { dd := &velerov2alpha1api.DataDownload{} err := client.Get(context.Background(), types.NamespacedName{ Namespace: pod.Namespace, Name: label, }, dd) if err != nil { return nil, errors.Wrapf(err, "error to find DataDownload by pod %s/%s", pod.Namespace, pod.Name) } return dd, nil } return nil, nil } func isDataDownloadInFinalState(dd *velerov2alpha1api.DataDownload) bool { return dd.Status.Phase == velerov2alpha1api.DataDownloadPhaseFailed || dd.Status.Phase == velerov2alpha1api.DataDownloadPhaseCanceled || dd.Status.Phase == velerov2alpha1api.DataDownloadPhaseCompleted } func UpdateDataDownloadWithRetry(ctx context.Context, client client.Client, namespacedName types.NamespacedName, log logrus.FieldLogger, updateFunc func(*velerov2alpha1api.DataDownload) bool) error { return wait.PollUntilContextCancel(ctx, time.Second, true, func(ctx context.Context) (bool, error) { dd := &velerov2alpha1api.DataDownload{} if err := client.Get(ctx, namespacedName, dd); err != nil { return false, errors.Wrap(err, "getting DataDownload") } if updateFunc(dd) { err := client.Update(ctx, dd) if err != nil { if apierrors.IsConflict(err) { log.Debugf("failed to update datadownload for %s/%s and will retry it", dd.Namespace, dd.Name) return false, nil } else { return false, errors.Wrapf(err, "error updating datadownload %s/%s", dd.Namespace, dd.Name) } } } return true, nil }) } var funcResumeCancellableDataRestore = (*DataDownloadReconciler).resumeCancellableDataPath func (r *DataDownloadReconciler) AttemptDataDownloadResume(ctx context.Context, logger *logrus.Entry, ns string) error { dataDownloads := &velerov2alpha1api.DataDownloadList{} if err := r.client.List(ctx, dataDownloads, &client.ListOptions{Namespace: ns}); err != nil { r.logger.WithError(errors.WithStack(err)).Error("failed to list datadownloads") return errors.Wrapf(err, "error to list datadownloads") } for i := range dataDownloads.Items { dd := &dataDownloads.Items[i] if dd.Status.Phase == velerov2alpha1api.DataDownloadPhaseInProgress { if dd.Status.Node != r.nodeName { logger.WithField("dd", dd.Name).WithField("current node", r.nodeName).Infof("DD should be resumed by another node %s", dd.Status.Node) continue } err := funcResumeCancellableDataRestore(r, ctx, dd, logger) if err == nil { logger.WithField("dd", dd.Name).WithField("current node", r.nodeName).Info("Completed to resume in progress DD") continue } logger.WithField("datadownload", dd.GetName()).WithError(err).Warn("Failed to resume data path for dd, have to cancel it") resumeErr := err err = UpdateDataDownloadWithRetry(ctx, r.client, types.NamespacedName{Namespace: dd.Namespace, Name: dd.Name}, logger.WithField("datadownload", dd.Name), func(dataDownload *velerov2alpha1api.DataDownload) bool { if dataDownload.Spec.Cancel { return false } dataDownload.Spec.Cancel = true dataDownload.Status.Message = fmt.Sprintf("Resume InProgress datadownload failed with error %v, mark it as cancel", resumeErr) return true }) if err != nil { logger.WithError(errors.WithStack(err)).WithError(errors.WithStack(err)).Error("Failed to trigger datadownload cancel") } } else if !isDataDownloadInFinalState(dd) { // the Prepared CR could be still handled by datadownload controller after node-agent restart // the accepted CR may also suvived from node-agent restart as long as the intermediate objects are all done logger.WithField("datadownload", dd.GetName()).Infof("find a datadownload with status %s", dd.Status.Phase) } } return nil } func (r *DataDownloadReconciler) resumeCancellableDataPath(ctx context.Context, dd *velerov2alpha1api.DataDownload, log logrus.FieldLogger) error { log.Info("Resume cancelable dataDownload") res, err := r.restoreExposer.GetExposed(ctx, getDataDownloadOwnerObject(dd), r.client, r.nodeName, dd.Spec.OperationTimeout.Duration) if err != nil { return errors.Wrapf(err, "error to get exposed volume for dd %s", dd.Name) } if res == nil { return errors.Errorf("expose info missed for dd %s", dd.Name) } callbacks := datapath.Callbacks{ OnCompleted: r.OnDataDownloadCompleted, OnFailed: r.OnDataDownloadFailed, OnCancelled: r.OnDataDownloadCancelled, OnProgress: r.OnDataDownloadProgress, } asyncBR, err := r.dataPathMgr.CreateMicroServiceBRWatcher(ctx, r.client, r.kubeClient, r.mgr, datapath.TaskTypeRestore, dd.Name, dd.Namespace, res.ByPod.HostingPod.Name, res.ByPod.HostingContainer, dd.Name, callbacks, true, log) if err != nil { return errors.Wrapf(err, "error to create asyncBR watcher for dd %s", dd.Name) } resumeComplete := false defer func() { if !resumeComplete { r.closeDataPath(ctx, dd.Name) } }() if err := asyncBR.Init(ctx, nil); err != nil { return errors.Wrapf(err, "error to init asyncBR watcher for dd %s", dd.Name) } if err := asyncBR.StartRestore(dd.Spec.SnapshotID, datapath.AccessPoint{ ByPath: res.ByPod.VolumeName, }, nil); err != nil { return errors.Wrapf(err, "error to resume asyncBR watcher for dd %s", dd.Name) } resumeComplete = true log.Infof("asyncBR is resumed for dd %s", dd.Name) return nil } ================================================ FILE: pkg/controller/data_download_controller_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package controller import ( "context" "fmt" "testing" "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" appsv1api "k8s.io/api/apps/v1" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" clientgofake "k8s.io/client-go/kubernetes/fake" ctrl "sigs.k8s.io/controller-runtime" kbclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/datapath" datapathmockes "github.com/vmware-tanzu/velero/pkg/datapath/mocks" "github.com/vmware-tanzu/velero/pkg/exposer" exposermockes "github.com/vmware-tanzu/velero/pkg/exposer/mocks" "github.com/vmware-tanzu/velero/pkg/metrics" velerotest "github.com/vmware-tanzu/velero/pkg/test" velerotypes "github.com/vmware-tanzu/velero/pkg/types" "github.com/vmware-tanzu/velero/pkg/uploader" "github.com/vmware-tanzu/velero/pkg/util/kube" ) const dataDownloadName string = "datadownload-1" func dataDownloadBuilder() *builder.DataDownloadBuilder { return builder.ForDataDownload(velerov1api.DefaultNamespace, dataDownloadName). BackupStorageLocation("bsl-loc"). DataMover("velero"). SnapshotID("test-snapshot-id").TargetVolume(velerov2alpha1api.TargetVolumeSpec{ PV: "test-pv", PVC: "test-pvc", Namespace: "test-ns", }) } func initDataDownloadReconciler(t *testing.T, objects []any, needError ...bool) (*DataDownloadReconciler, error) { t.Helper() var errs = make([]error, 6) for k, isError := range needError { if k == 0 && isError { errs[0] = fmt.Errorf("Get error") } else if k == 1 && isError { errs[1] = fmt.Errorf("Create error") } else if k == 2 && isError { errs[2] = fmt.Errorf("Update error") } else if k == 3 && isError { errs[3] = fmt.Errorf("Patch error") } else if k == 4 && isError { errs[4] = apierrors.NewConflict(velerov2alpha1api.Resource("datadownload"), dataDownloadName, errors.New("conflict")) } else if k == 5 && isError { errs[5] = fmt.Errorf("List error") } } return initDataDownloadReconcilerWithError(t, objects, errs...) } func initDataDownloadReconcilerWithError(t *testing.T, objects []any, needError ...error) (*DataDownloadReconciler, error) { t.Helper() runtimeObjects := make([]runtime.Object, 0) for _, obj := range objects { runtimeObjects = append(runtimeObjects, obj.(runtime.Object)) } fakeClient := FakeClient{ Client: velerotest.NewFakeControllerRuntimeClient(t, runtimeObjects...), } fakeKubeClient := clientgofake.NewSimpleClientset(runtimeObjects...) for k := range needError { if k == 0 { fakeClient.getError = needError[0] } else if k == 1 { fakeClient.createError = needError[1] } else if k == 2 { fakeClient.updateError = needError[2] } else if k == 3 { fakeClient.patchError = needError[3] } else if k == 4 { fakeClient.updateConflict = needError[4] } else if k == 5 { fakeClient.listError = needError[5] } } fakeFS := velerotest.NewFakeFileSystem() pathGlob := fmt.Sprintf("/host_pods/%s/volumes/*/%s", "test-uid", "test-pvc") _, err := fakeFS.Create(pathGlob) if err != nil { return nil, err } dataPathMgr := datapath.NewManager(1) return NewDataDownloadReconciler( &fakeClient, nil, fakeKubeClient, dataPathMgr, nil, nil, velerotypes.RestorePVC{}, nil, nil, corev1api.ResourceRequirements{}, "test-node", time.Minute*5, velerotest.NewLogger(), metrics.NewServerMetrics(), "", nil, nil, // podLabels nil, // podAnnotations ), nil } func TestDataDownloadReconcile(t *testing.T) { sc := builder.ForStorageClass("sc").Result() daemonSet := &appsv1api.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", Name: "node-agent", }, TypeMeta: metav1.TypeMeta{ Kind: "DaemonSet", APIVersion: appsv1api.SchemeGroupVersion.String(), }, Spec: appsv1api.DaemonSetSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ Containers: []corev1api.Container{ { Image: "fake-image", }, }, }, }, }, } node := builder.ForNode("fake-node").Labels(map[string]string{kube.NodeOSLabel: kube.NodeOSLinux}).Result() tests := []struct { name string dd *velerov2alpha1api.DataDownload notCreateDD bool targetPVC *corev1api.PersistentVolumeClaim dataMgr *datapath.Manager needErrs []bool needCreateFSBR bool needDelete bool sportTime *metav1.Time isExposeErr bool isGetExposeErr bool isGetExposeNil bool isPeekExposeErr bool isNilExposer bool notNilExpose bool notMockCleanUp bool mockInit bool mockInitErr error mockStart bool mockStartErr error mockCancel bool mockClose bool needExclusiveUpdateError error constrained bool expected *velerov2alpha1api.DataDownload expectDeleted bool expectCancelRecord bool expectedResult *ctrl.Result expectedErr string expectDataPath bool }{ { name: "dd not found", dd: dataDownloadBuilder().Result(), notCreateDD: true, }, { name: "dd not created in velero default namespace", dd: builder.ForDataDownload("test-ns", dataDownloadName).Result(), }, { name: "get dd fail", dd: dataDownloadBuilder().Result(), needErrs: []bool{true, false, false, false}, expectedErr: "Get error", }, { name: "dd is not for built-in dm", dd: dataDownloadBuilder().DataMover("other").Result(), }, { name: "add finalizer to dd", dd: dataDownloadBuilder().Result(), expected: dataDownloadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Result(), }, { name: "add finalizer to dd failed", dd: dataDownloadBuilder().Result(), needErrs: []bool{false, false, true, false}, expectedErr: "error updating datadownload velero/datadownload-1: Update error", }, { name: "dd is under deletion", dd: dataDownloadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Result(), needDelete: true, expected: dataDownloadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Cancel(true).Result(), }, { name: "dd is under deletion but cancel failed", dd: dataDownloadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Result(), needErrs: []bool{false, false, true, false}, needDelete: true, expectedErr: "error updating datadownload velero/datadownload-1: Update error", }, { name: "dd is under deletion and in terminal state", dd: dataDownloadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Phase(velerov2alpha1api.DataDownloadPhaseFailed).Result(), sportTime: &metav1.Time{Time: time.Now()}, needDelete: true, expectDeleted: true, }, { name: "dd is under deletion and in terminal state, but remove finalizer failed", dd: dataDownloadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Phase(velerov2alpha1api.DataDownloadPhaseFailed).Result(), needErrs: []bool{false, false, true, false}, needDelete: true, expectedErr: "error updating datadownload velero/datadownload-1: Update error", }, { name: "delay cancel negative for others", dd: dataDownloadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Cancel(true).Phase(velerov2alpha1api.DataDownloadPhasePrepared).Result(), sportTime: &metav1.Time{Time: time.Now()}, expectCancelRecord: true, }, { name: "delay cancel negative for inProgress", dd: dataDownloadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Cancel(true).Phase(velerov2alpha1api.DataDownloadPhaseInProgress).Result(), sportTime: &metav1.Time{Time: time.Now().Add(-time.Minute * 58)}, expectCancelRecord: true, }, { name: "delay cancel affirmative for others", dd: dataDownloadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Cancel(true).Phase(velerov2alpha1api.DataDownloadPhasePrepared).Result(), sportTime: &metav1.Time{Time: time.Now().Add(-time.Minute * 5)}, expected: dataDownloadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Cancel(true).Phase(velerov2alpha1api.DataDownloadPhaseCanceled).Result(), }, { name: "delay cancel affirmative for inProgress", dd: dataDownloadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Cancel(true).Phase(velerov2alpha1api.DataDownloadPhaseInProgress).Result(), sportTime: &metav1.Time{Time: time.Now().Add(-time.Hour)}, expected: dataDownloadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Cancel(true).Phase(velerov2alpha1api.DataDownloadPhaseCanceled).Result(), }, { name: "delay cancel failed", dd: dataDownloadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Cancel(true).Phase(velerov2alpha1api.DataDownloadPhaseInProgress).Result(), needErrs: []bool{false, false, true, false}, sportTime: &metav1.Time{Time: time.Now().Add(-time.Hour)}, expected: dataDownloadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Cancel(true).Phase(velerov2alpha1api.DataDownloadPhaseInProgress).Result(), expectCancelRecord: true, }, { name: "Unknown data download status", dd: dataDownloadBuilder().Phase("Unknown").Finalizers([]string{DataUploadDownloadFinalizer}).Result(), }, { name: "dd is cancel on new", dd: dataDownloadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Cancel(true).Result(), targetPVC: builder.ForPersistentVolumeClaim("test-ns", "test-pvc").Result(), expectCancelRecord: true, expected: dataDownloadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Cancel(true).Phase(velerov2alpha1api.DataDownloadPhaseCanceled).Result(), }, { name: "new dd but constrained", dd: dataDownloadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Result(), constrained: true, expected: dataDownloadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Result(), expectedResult: &ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, }, { name: "new dd but no target PVC", dd: dataDownloadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Result(), expectedResult: &ctrl.Result{Requeue: true}, }, { name: "new dd but accept failed", dd: dataDownloadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Result(), targetPVC: builder.ForPersistentVolumeClaim("test-ns", "test-pvc").Result(), needExclusiveUpdateError: errors.New("exclusive-update-error"), expected: dataDownloadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Result(), expectedErr: "error accepting the data download datadownload-1: exclusive-update-error", }, { name: "dd is accepted but setup expose param failed", dd: dataDownloadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).NodeOS("xxx").Result(), targetPVC: builder.ForPersistentVolumeClaim("test-ns", "test-pvc").Result(), expected: dataDownloadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).NodeOS("xxx").Phase(velerov2alpha1api.DataDownloadPhaseFailed).Message("failed to set exposer parameters").Result(), expectedErr: "no appropriate node to run datadownload velero/datadownload-1: node with OS xxx doesn't exist", }, { name: "dd expose failed", dd: dataDownloadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Result(), targetPVC: builder.ForPersistentVolumeClaim("test-ns", "test-pvc").StorageClass("test-sc").Result(), isExposeErr: true, expected: dataDownloadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Phase(velerov2alpha1api.DataDownloadPhaseFailed).Message("error to expose snapshot").Result(), expectedErr: "Error to expose restore exposer", }, { name: "dd succeeds for accepted", dd: dataDownloadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Result(), targetPVC: builder.ForPersistentVolumeClaim("test-ns", "test-pvc").StorageClass("sc").Result(), expected: dataDownloadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Phase(velerov2alpha1api.DataDownloadPhaseAccepted).Result(), }, { name: "prepare timeout on accepted", dd: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhaseAccepted).Finalizers([]string{DataUploadDownloadFinalizer}).AcceptedTimestamp(&metav1.Time{Time: time.Now().Add(-time.Minute * 30)}).Result(), expected: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhaseFailed).Finalizers([]string{DataUploadDownloadFinalizer}).Phase(velerov2alpha1api.DataDownloadPhaseFailed).Message("timeout on preparing data download").Result(), }, { name: "peek error on accepted", dd: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhaseAccepted).Finalizers([]string{DataUploadDownloadFinalizer}).Result(), isPeekExposeErr: true, expected: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhaseCanceled).Finalizers([]string{DataUploadDownloadFinalizer}).Phase(velerov2alpha1api.DataDownloadPhaseCanceled).Message("found a datadownload velero/datadownload-1 with expose error: fake-peek-error. mark it as cancel").Result(), }, { name: "cancel on prepared", dd: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhasePrepared).Finalizers([]string{DataUploadDownloadFinalizer}).Node("test-node").Cancel(true).Result(), expected: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhaseCanceled).Finalizers([]string{DataUploadDownloadFinalizer}).Cancel(true).Phase(velerov2alpha1api.DataDownloadPhaseCanceled).Result(), }, { name: "Failed to get restore expose on prepared", dd: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhasePrepared).Finalizers([]string{DataUploadDownloadFinalizer}).Node("test-node").Result(), targetPVC: builder.ForPersistentVolumeClaim("test-ns", "test-pvc").Result(), isGetExposeErr: true, expected: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhaseFailed).Finalizers([]string{DataUploadDownloadFinalizer}).Message("restore exposer is not ready").Result(), expectedErr: "Error to get restore exposer", }, { name: "Get nil restore expose on prepared", dd: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhasePrepared).Finalizers([]string{DataUploadDownloadFinalizer}).Node("test-node").Result(), targetPVC: builder.ForPersistentVolumeClaim("test-ns", "test-pvc").Result(), isGetExposeNil: true, expected: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhaseFailed).Finalizers([]string{DataUploadDownloadFinalizer}).Message("exposed snapshot is not ready").Result(), expectedErr: "no expose result is available for the current node", }, { name: "Error in data path is concurrent limited", dd: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhasePrepared).Finalizers([]string{DataUploadDownloadFinalizer}).Node("test-node").Result(), targetPVC: builder.ForPersistentVolumeClaim("test-ns", "test-pvc").Result(), dataMgr: datapath.NewManager(0), notNilExpose: true, notMockCleanUp: true, expectedResult: &ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, }, { name: "data path init error", dd: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhasePrepared).Finalizers([]string{DataUploadDownloadFinalizer}).Node("test-node").Result(), targetPVC: builder.ForPersistentVolumeClaim("test-ns", "test-pvc").Result(), mockInit: true, mockInitErr: errors.New("fake-data-path-init-error"), mockClose: true, notNilExpose: true, expected: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhaseFailed).Finalizers([]string{DataUploadDownloadFinalizer}).Message("error initializing data path").Result(), expectedErr: "error initializing asyncBR: fake-data-path-init-error", }, { name: "Unable to update status to in progress for data download", dd: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhasePrepared).Finalizers([]string{DataUploadDownloadFinalizer}).Node("test-node").Result(), targetPVC: builder.ForPersistentVolumeClaim("test-ns", "test-pvc").Result(), needErrs: []bool{false, false, true, false}, mockInit: true, mockClose: true, notNilExpose: true, notMockCleanUp: true, expected: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhasePrepared).Finalizers([]string{DataUploadDownloadFinalizer}).Result(), }, { name: "data path start error", dd: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhasePrepared).Finalizers([]string{DataUploadDownloadFinalizer}).Node("test-node").Result(), targetPVC: builder.ForPersistentVolumeClaim("test-ns", "test-pvc").Result(), mockInit: true, mockStart: true, mockStartErr: errors.New("fake-data-path-start-error"), mockClose: true, notNilExpose: true, expected: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhaseFailed).Finalizers([]string{DataUploadDownloadFinalizer}).Message("error starting data path").Result(), expectedErr: "error starting async restore for pod test-name, volume test-pvc: fake-data-path-start-error", }, { name: "Prepare succeeds", dd: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhasePrepared).Finalizers([]string{DataUploadDownloadFinalizer}).Node("test-node").Result(), targetPVC: builder.ForPersistentVolumeClaim("test-ns", "test-pvc").Result(), mockInit: true, mockStart: true, notNilExpose: true, notMockCleanUp: true, expectDataPath: true, expected: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhaseInProgress).Finalizers([]string{DataUploadDownloadFinalizer}).Result(), }, { name: "In progress dd is not handled by the current node", dd: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhaseInProgress).Finalizers([]string{DataUploadDownloadFinalizer}).Result(), expected: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhaseInProgress).Finalizers([]string{DataUploadDownloadFinalizer}).Result(), }, { name: "In progress dd is not set as cancel", dd: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhaseInProgress).Finalizers([]string{DataUploadDownloadFinalizer}).Node("test-node").Result(), expected: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhaseInProgress).Finalizers([]string{DataUploadDownloadFinalizer}).Result(), }, { name: "Cancel data downloand in progress with empty FSBR", dd: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhaseInProgress).Cancel(true).Finalizers([]string{DataUploadDownloadFinalizer}).Node("test-node").Result(), expected: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhaseCanceled).Cancel(true).Finalizers([]string{DataUploadDownloadFinalizer}).Result(), }, { name: "Cancel data downloand in progress and patch data download error", dd: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhaseInProgress).Cancel(true).Finalizers([]string{DataUploadDownloadFinalizer}).Node("test-node").Result(), needErrs: []bool{false, false, true, false}, needCreateFSBR: true, expected: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhaseInProgress).Cancel(true).Finalizers([]string{DataUploadDownloadFinalizer}).Result(), expectedErr: "error updating datadownload velero/datadownload-1: Update error", expectCancelRecord: true, expectDataPath: true, }, { name: "Cancel data downloand in progress succeeds", dd: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhaseInProgress).Cancel(true).Finalizers([]string{DataUploadDownloadFinalizer}).Node("test-node").Result(), needCreateFSBR: true, mockCancel: true, expected: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhaseCanceling).Cancel(true).Finalizers([]string{DataUploadDownloadFinalizer}).Result(), expectDataPath: true, expectCancelRecord: true, }, { name: "pvc StorageClass is nil", dd: dataDownloadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Result(), targetPVC: builder.ForPersistentVolumeClaim("test-ns", "test-pvc").Result(), expected: dataDownloadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Phase(velerov2alpha1api.DataDownloadPhaseAccepted).Result(), }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { objects := []any{daemonSet, node, sc} if test.targetPVC != nil { objects = append(objects, test.targetPVC) } r, err := initDataDownloadReconciler(t, objects, test.needErrs...) require.NoError(t, err) if !test.notCreateDD { err = r.client.Create(t.Context(), test.dd) require.NoError(t, err) } if test.needDelete { err = r.client.Delete(t.Context(), test.dd) require.NoError(t, err) } if test.dataMgr != nil { r.dataPathMgr = test.dataMgr } else { r.dataPathMgr = datapath.NewManager(1) } if test.sportTime != nil { r.cancelledDataDownload[test.dd.Name] = test.sportTime.Time } if test.constrained { r.vgdpCounter = &exposer.VgdpCounter{} } funcExclusiveUpdateDataDownload = exclusiveUpdateDataDownload if test.needExclusiveUpdateError != nil { funcExclusiveUpdateDataDownload = func(context.Context, kbclient.Client, *velerov2alpha1api.DataDownload, func(*velerov2alpha1api.DataDownload)) (bool, error) { return false, test.needExclusiveUpdateError } } datapath.MicroServiceBRWatcherCreator = func(kbclient.Client, kubernetes.Interface, manager.Manager, string, string, string, string, string, string, datapath.Callbacks, logrus.FieldLogger) datapath.AsyncBR { asyncBR := datapathmockes.NewAsyncBR(t) if test.mockInit { asyncBR.On("Init", mock.Anything, mock.Anything).Return(test.mockInitErr) } if test.mockStart { asyncBR.On("StartRestore", mock.Anything, mock.Anything, mock.Anything).Return(test.mockStartErr) } if test.mockCancel { asyncBR.On("Cancel").Return() } if test.mockClose { asyncBR.On("Close", mock.Anything).Return() } return asyncBR } if test.isExposeErr || test.isGetExposeErr || test.isGetExposeNil || test.isPeekExposeErr || test.isNilExposer || test.notNilExpose { if test.isNilExposer { r.restoreExposer = nil } else { r.restoreExposer = func() exposer.GenericRestoreExposer { ep := exposermockes.NewMockGenericRestoreExposer(t) if test.isExposeErr { ep.On("Expose", mock.Anything, mock.Anything, mock.Anything).Return(errors.New("Error to expose restore exposer")) } else if test.notNilExpose { hostingPod := builder.ForPod("test-ns", "test-name").Volumes(&corev1api.Volume{Name: "test-pvc"}).Result() hostingPod.ObjectMeta.SetUID("test-uid") ep.On("GetExposed", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&exposer.ExposeResult{ByPod: exposer.ExposeByPod{HostingPod: hostingPod, VolumeName: "test-pvc"}}, nil) } else if test.isGetExposeErr { ep.On("GetExposed", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, errors.New("Error to get restore exposer")) } else if test.isGetExposeNil { ep.On("GetExposed", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, nil) } else if test.isPeekExposeErr { ep.On("PeekExposed", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(errors.New("fake-peek-error")) ep.On("DiagnoseExpose", mock.Anything, mock.Anything).Return("") } if !test.notMockCleanUp { ep.On("CleanUp", mock.Anything, mock.Anything).Return() } return ep }() } } if test.needCreateFSBR { if fsBR := r.dataPathMgr.GetAsyncBR(test.dd.Name); fsBR == nil { _, err := r.dataPathMgr.CreateMicroServiceBRWatcher(ctx, r.client, nil, nil, datapath.TaskTypeRestore, test.dd.Name, pVBRRequestor, velerov1api.DefaultNamespace, "", "", datapath.Callbacks{OnCancelled: r.OnDataDownloadCancelled}, false, velerotest.NewLogger()) require.NoError(t, err) } } actualResult, err := r.Reconcile(ctx, ctrl.Request{ NamespacedName: types.NamespacedName{ Namespace: velerov1api.DefaultNamespace, Name: test.dd.Name, }, }) if test.expectedErr != "" { require.EqualError(t, err, test.expectedErr) } else { require.NoError(t, err) } if test.expectedResult != nil { assert.Equal(t, test.expectedResult.Requeue, actualResult.Requeue) assert.Equal(t, test.expectedResult.RequeueAfter, actualResult.RequeueAfter) } dd := velerov2alpha1api.DataDownload{} err = r.client.Get(ctx, kbclient.ObjectKey{ Name: test.dd.Name, Namespace: test.dd.Namespace, }, &dd) if test.expected != nil || test.expectDeleted { if test.expectDeleted { assert.True(t, apierrors.IsNotFound(err)) } else { require.NoError(t, err) assert.Equal(t, test.expected.Status.Phase, dd.Status.Phase) assert.Contains(t, dd.Status.Message, test.expected.Status.Message) assert.Equal(t, dd.Finalizers, test.expected.Finalizers) assert.Equal(t, dd.Spec.Cancel, test.expected.Spec.Cancel) } } if !test.expectDataPath { assert.Nil(t, r.dataPathMgr.GetAsyncBR(test.dd.Name)) } else { assert.NotNil(t, r.dataPathMgr.GetAsyncBR(test.dd.Name)) } if test.expectCancelRecord { assert.Contains(t, r.cancelledDataDownload, test.dd.Name) } else { assert.Empty(t, r.cancelledDataDownload) } if isDataDownloadInFinalState(&dd) || dd.Status.Phase == velerov2alpha1api.DataDownloadPhaseInProgress { assert.NotContains(t, dd.Labels, exposer.ExposeOnGoingLabel) } else if dd.Status.Phase == velerov2alpha1api.DataDownloadPhaseAccepted { assert.Contains(t, dd.Labels, exposer.ExposeOnGoingLabel) } }) } } func TestOnDataDownloadFailed(t *testing.T) { for _, getErr := range []bool{true, false} { ctx := t.Context() needErrs := []bool{getErr, false, false, false} r, err := initDataDownloadReconciler(t, nil, needErrs...) require.NoError(t, err) dd := dataDownloadBuilder().Result() namespace := dd.Namespace ddName := dd.Name // Add the DataDownload object to the fake client require.NoError(t, r.client.Create(ctx, dd)) r.OnDataDownloadFailed(ctx, namespace, ddName, fmt.Errorf("Failed to handle %v", ddName)) updatedDD := &velerov2alpha1api.DataDownload{} if getErr { require.Error(t, r.client.Get(ctx, types.NamespacedName{Name: ddName, Namespace: namespace}, updatedDD)) assert.NotEqual(t, velerov2alpha1api.DataDownloadPhaseFailed, updatedDD.Status.Phase) assert.True(t, updatedDD.Status.StartTimestamp.IsZero()) } else { require.NoError(t, r.client.Get(ctx, types.NamespacedName{Name: ddName, Namespace: namespace}, updatedDD)) assert.Equal(t, velerov2alpha1api.DataDownloadPhaseFailed, updatedDD.Status.Phase) assert.True(t, updatedDD.Status.StartTimestamp.IsZero()) } } } func TestOnDataDownloadCancelled(t *testing.T) { for _, getErr := range []bool{true, false} { ctx := t.Context() needErrs := []bool{getErr, false, false, false} r, err := initDataDownloadReconciler(t, nil, needErrs...) require.NoError(t, err) dd := dataDownloadBuilder().Result() namespace := dd.Namespace ddName := dd.Name // Add the DataDownload object to the fake client require.NoError(t, r.client.Create(ctx, dd)) r.OnDataDownloadCancelled(ctx, namespace, ddName) updatedDD := &velerov2alpha1api.DataDownload{} if getErr { require.Error(t, r.client.Get(ctx, types.NamespacedName{Name: ddName, Namespace: namespace}, updatedDD)) assert.NotEqual(t, velerov2alpha1api.DataDownloadPhaseFailed, updatedDD.Status.Phase) assert.True(t, updatedDD.Status.StartTimestamp.IsZero()) } else { require.NoError(t, r.client.Get(ctx, types.NamespacedName{Name: ddName, Namespace: namespace}, updatedDD)) assert.Equal(t, velerov2alpha1api.DataDownloadPhaseCanceled, updatedDD.Status.Phase) assert.False(t, updatedDD.Status.StartTimestamp.IsZero()) assert.False(t, updatedDD.Status.CompletionTimestamp.IsZero()) } } } func TestOnDataDownloadCompleted(t *testing.T) { tests := []struct { name string emptyFSBR bool isGetErr bool rebindVolumeErr bool }{ { name: "Data download complete", emptyFSBR: false, isGetErr: false, rebindVolumeErr: false, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { ctx := t.Context() needErrs := []bool{test.isGetErr, false, false, false} r, err := initDataDownloadReconciler(t, nil, needErrs...) r.restoreExposer = func() exposer.GenericRestoreExposer { ep := exposermockes.NewMockGenericRestoreExposer(t) if test.rebindVolumeErr { ep.On("RebindVolume", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(errors.New("Error to rebind volume")) } else { ep.On("RebindVolume", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) } ep.On("CleanUp", mock.Anything, mock.Anything).Return() return ep }() require.NoError(t, err) dd := dataDownloadBuilder().Result() namespace := dd.Namespace ddName := dd.Name // Add the DataDownload object to the fake client require.NoError(t, r.client.Create(ctx, dd)) r.OnDataDownloadCompleted(ctx, namespace, ddName, datapath.Result{}) updatedDD := &velerov2alpha1api.DataDownload{} if test.isGetErr { require.Error(t, r.client.Get(ctx, types.NamespacedName{Name: ddName, Namespace: namespace}, updatedDD)) assert.Equal(t, velerov2alpha1api.DataDownloadPhase(""), updatedDD.Status.Phase) assert.True(t, updatedDD.Status.CompletionTimestamp.IsZero()) } else { require.NoError(t, r.client.Get(ctx, types.NamespacedName{Name: ddName, Namespace: namespace}, updatedDD)) assert.Equal(t, velerov2alpha1api.DataDownloadPhaseCompleted, updatedDD.Status.Phase) assert.False(t, updatedDD.Status.CompletionTimestamp.IsZero()) } }) } } func TestOnDataDownloadProgress(t *testing.T) { totalBytes := int64(1024) bytesDone := int64(512) tests := []struct { name string dd *velerov2alpha1api.DataDownload progress uploader.Progress needErrs []bool }{ { name: "patch in progress phase success", dd: dataDownloadBuilder().Result(), progress: uploader.Progress{ TotalBytes: totalBytes, BytesDone: bytesDone, }, }, { name: "failed to get datadownload", dd: dataDownloadBuilder().Result(), needErrs: []bool{true, false, false, false}, }, { name: "failed to patch datadownload", dd: dataDownloadBuilder().Result(), needErrs: []bool{false, false, true, false}, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { ctx := t.Context() r, err := initDataDownloadReconciler(t, nil, test.needErrs...) require.NoError(t, err) defer func() { r.client.Delete(ctx, test.dd, &kbclient.DeleteOptions{}) }() // Create a DataDownload object dd := dataDownloadBuilder().Result() namespace := dd.Namespace duName := dd.Name // Add the DataDownload object to the fake client require.NoError(t, r.client.Create(t.Context(), dd)) // Create a Progress object progress := &uploader.Progress{ TotalBytes: totalBytes, BytesDone: bytesDone, } // Call the OnDataDownloadProgress function r.OnDataDownloadProgress(ctx, namespace, duName, progress) if len(test.needErrs) != 0 && !test.needErrs[0] { // Get the updated DataDownload object from the fake client updatedDu := &velerov2alpha1api.DataDownload{} require.NoError(t, r.client.Get(ctx, types.NamespacedName{Name: duName, Namespace: namespace}, updatedDu)) // Assert that the DataDownload object has been updated with the progress assert.Equal(t, test.progress.TotalBytes, updatedDu.Status.Progress.TotalBytes) assert.Equal(t, test.progress.BytesDone, updatedDu.Status.Progress.BytesDone) } }) } } func TestFindDataDownloadForPod(t *testing.T) { needErrs := []bool{false, false, false, false} r, err := initDataDownloadReconciler(t, nil, needErrs...) require.NoError(t, err) tests := []struct { name string du *velerov2alpha1api.DataDownload pod *corev1api.Pod checkFunc func(*velerov2alpha1api.DataDownload, []reconcile.Request) }{ { name: "find dataDownload for pod", du: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhaseAccepted).Result(), pod: builder.ForPod(velerov1api.DefaultNamespace, dataDownloadName).Labels(map[string]string{velerov1api.DataDownloadLabel: dataDownloadName}).Status(corev1api.PodStatus{Phase: corev1api.PodRunning}).Result(), checkFunc: func(du *velerov2alpha1api.DataDownload, requests []reconcile.Request) { // Assert that the function returns a single request assert.Len(t, requests, 1) // Assert that the request contains the correct namespaced name assert.Equal(t, du.Namespace, requests[0].Namespace) assert.Equal(t, du.Name, requests[0].Name) }, }, { name: "no selected label found for pod", du: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhaseAccepted).Result(), pod: builder.ForPod(velerov1api.DefaultNamespace, dataDownloadName).Result(), checkFunc: func(du *velerov2alpha1api.DataDownload, requests []reconcile.Request) { // Assert that the function returns a single request assert.Empty(t, requests) }, }, { name: "no matched pod", du: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhaseAccepted).Result(), pod: builder.ForPod(velerov1api.DefaultNamespace, dataDownloadName).Labels(map[string]string{velerov1api.DataDownloadLabel: "non-existing-datadownload"}).Result(), checkFunc: func(du *velerov2alpha1api.DataDownload, requests []reconcile.Request) { assert.Empty(t, requests) }, }, { name: "dataDownload not accept", du: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhaseInProgress).Result(), pod: builder.ForPod(velerov1api.DefaultNamespace, dataDownloadName).Labels(map[string]string{velerov1api.DataDownloadLabel: dataDownloadName}).Result(), checkFunc: func(du *velerov2alpha1api.DataDownload, requests []reconcile.Request) { assert.Empty(t, requests) }, }, } for _, test := range tests { ctx := t.Context() assert.NoError(t, r.client.Create(ctx, test.pod)) assert.NoError(t, r.client.Create(ctx, test.du)) // Call the findSnapshotRestoreForPod function requests := r.findSnapshotRestoreForPod(t.Context(), test.pod) test.checkFunc(test.du, requests) r.client.Delete(ctx, test.du, &kbclient.DeleteOptions{}) if test.pod != nil { r.client.Delete(ctx, test.pod, &kbclient.DeleteOptions{}) } } } func TestAcceptDataDownload(t *testing.T) { tests := []struct { name string dd *velerov2alpha1api.DataDownload needErrs []error succeeded bool expectedErr string }{ { name: "update fail", dd: dataDownloadBuilder().Result(), needErrs: []error{nil, nil, fmt.Errorf("fake-update-error"), nil}, expectedErr: "fake-update-error", }, { name: "accepted by others", dd: dataDownloadBuilder().Result(), needErrs: []error{nil, nil, &fakeAPIStatus{metav1.StatusReasonConflict}, nil}, }, { name: "succeed", dd: dataDownloadBuilder().Result(), needErrs: []error{nil, nil, nil, nil}, succeeded: true, }, } for _, test := range tests { ctx := t.Context() r, err := initDataDownloadReconcilerWithError(t, nil, test.needErrs...) require.NoError(t, err) err = r.client.Create(ctx, test.dd) require.NoError(t, err) succeeded, err := r.acceptDataDownload(ctx, test.dd) assert.Equal(t, test.succeeded, succeeded) if test.expectedErr == "" { assert.NoError(t, err) } else { assert.EqualError(t, err, test.expectedErr) } } } func TestOnDdPrepareTimeout(t *testing.T) { tests := []struct { name string dd *velerov2alpha1api.DataDownload needErrs []error expected *velerov2alpha1api.DataDownload }{ { name: "update fail", dd: dataDownloadBuilder().Result(), needErrs: []error{nil, nil, fmt.Errorf("fake-update-error"), nil}, expected: dataDownloadBuilder().Result(), }, { name: "update interrupted", dd: dataDownloadBuilder().Result(), needErrs: []error{nil, nil, &fakeAPIStatus{metav1.StatusReasonConflict}, nil}, expected: dataDownloadBuilder().Result(), }, { name: "succeed", dd: dataDownloadBuilder().Result(), needErrs: []error{nil, nil, nil, nil}, expected: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhaseFailed).Result(), }, } for _, test := range tests { ctx := t.Context() r, err := initDataDownloadReconcilerWithError(t, nil, test.needErrs...) require.NoError(t, err) err = r.client.Create(ctx, test.dd) require.NoError(t, err) r.onPrepareTimeout(ctx, test.dd) dd := velerov2alpha1api.DataDownload{} _ = r.client.Get(ctx, kbclient.ObjectKey{ Name: test.dd.Name, Namespace: test.dd.Namespace, }, &dd) assert.Equal(t, test.expected.Status.Phase, dd.Status.Phase) } } func TestTryCancelDataDownload(t *testing.T) { tests := []struct { name string dd *velerov2alpha1api.DataDownload needErrs []error succeeded bool expectedErr string }{ { name: "update fail", dd: dataDownloadBuilder().Result(), needErrs: []error{nil, nil, fmt.Errorf("fake-update-error"), nil}, }, { name: "cancel by others", dd: dataDownloadBuilder().Result(), needErrs: []error{nil, nil, &fakeAPIStatus{metav1.StatusReasonConflict}, nil}, }, { name: "succeed", dd: dataDownloadBuilder().Result(), needErrs: []error{nil, nil, nil, nil}, succeeded: true, }, } for _, test := range tests { ctx := t.Context() r, err := initDataDownloadReconcilerWithError(t, nil, test.needErrs...) require.NoError(t, err) err = r.client.Create(ctx, test.dd) require.NoError(t, err) r.tryCancelDataDownload(ctx, test.dd, "") if test.expectedErr == "" { assert.NoError(t, err) } else { assert.EqualError(t, err, test.expectedErr) } } } func TestUpdateDataDownloadWithRetry(t *testing.T) { namespacedName := types.NamespacedName{ Name: dataDownloadName, Namespace: "velero", } // Define test cases testCases := []struct { Name string needErrs []bool noChange bool ExpectErr bool }{ { Name: "SuccessOnFirstAttempt", }, { Name: "Error get", needErrs: []bool{true, false, false, false, false}, ExpectErr: true, }, { Name: "Error update", needErrs: []bool{false, false, true, false, false}, ExpectErr: true, }, { Name: "no change", noChange: true, needErrs: []bool{false, false, true, false, false}, }, { Name: "Conflict with error timeout", needErrs: []bool{false, false, false, false, true}, ExpectErr: true, }, } for _, tc := range testCases { t.Run(tc.Name, func(t *testing.T) { ctx, cancelFunc := context.WithTimeout(t.Context(), time.Second*5) defer cancelFunc() r, err := initDataDownloadReconciler(t, nil, tc.needErrs...) require.NoError(t, err) err = r.client.Create(ctx, dataDownloadBuilder().Result()) require.NoError(t, err) updateFunc := func(dataDownload *velerov2alpha1api.DataDownload) bool { if tc.noChange { return false } dataDownload.Spec.Cancel = true return true } err = UpdateDataDownloadWithRetry(ctx, r.client, namespacedName, velerotest.NewLogger().WithField("name", tc.Name), updateFunc) if tc.ExpectErr { assert.Error(t, err) } else { assert.NoError(t, err) } }) } } type ddResumeTestHelper struct { resumeErr error getExposeErr error exposeResult *exposer.ExposeResult asyncBR datapath.AsyncBR } func (dt *ddResumeTestHelper) resumeCancellableDataPath(_ *DataUploadReconciler, _ context.Context, _ *velerov2alpha1api.DataUpload, _ logrus.FieldLogger) error { return dt.resumeErr } func (dt *ddResumeTestHelper) Expose(context.Context, corev1api.ObjectReference, exposer.GenericRestoreExposeParam) error { return nil } func (dt *ddResumeTestHelper) GetExposed(context.Context, corev1api.ObjectReference, kbclient.Client, string, time.Duration) (*exposer.ExposeResult, error) { return dt.exposeResult, dt.getExposeErr } func (dt *ddResumeTestHelper) PeekExposed(context.Context, corev1api.ObjectReference) error { return nil } func (dt *ddResumeTestHelper) DiagnoseExpose(context.Context, corev1api.ObjectReference) string { return "" } func (dt *ddResumeTestHelper) RebindVolume(context.Context, corev1api.ObjectReference, string, string, time.Duration) error { return nil } func (dt *ddResumeTestHelper) CleanUp(context.Context, corev1api.ObjectReference) {} func (dt *ddResumeTestHelper) newMicroServiceBRWatcher(kbclient.Client, kubernetes.Interface, manager.Manager, string, string, string, string, string, string, datapath.Callbacks, logrus.FieldLogger) datapath.AsyncBR { return dt.asyncBR } func TestAttemptDataDownloadResume(t *testing.T) { tests := []struct { name string dataUploads []velerov2alpha1api.DataDownload dd *velerov2alpha1api.DataDownload needErrs []bool resumeErr error acceptedDataDownloads []string prepareddDataDownloads []string cancelledDataDownloads []string inProgressDataDownloads []string expectedError string }{ { name: "Other DataDownload", dd: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhasePrepared).Result(), }, { name: "Other DataDownload", dd: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhaseAccepted).Result(), }, { name: "InProgress DataDownload, not the current node", dd: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhaseInProgress).Result(), inProgressDataDownloads: []string{dataDownloadName}, }, { name: "InProgress DataDownload, no resume error", dd: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhaseInProgress).Node("node-1").Result(), inProgressDataDownloads: []string{dataDownloadName}, }, { name: "InProgress DataDownload, resume error, cancel error", dd: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhaseInProgress).Node("node-1").Result(), resumeErr: errors.New("fake-resume-error"), needErrs: []bool{false, false, true, false, false, false}, inProgressDataDownloads: []string{dataDownloadName}, }, { name: "InProgress DataDownload, resume error, cancel succeed", dd: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhaseInProgress).Node("node-1").Result(), resumeErr: errors.New("fake-resume-error"), cancelledDataDownloads: []string{dataDownloadName}, inProgressDataDownloads: []string{dataDownloadName}, }, { name: "Error", needErrs: []bool{false, false, false, false, false, true}, dd: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhasePrepared).Result(), expectedError: "error to list datadownloads: List error", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { ctx := t.Context() r, err := initDataDownloadReconciler(t, nil, test.needErrs...) r.nodeName = "node-1" require.NoError(t, err) defer func() { r.client.Delete(ctx, test.dd, &kbclient.DeleteOptions{}) }() require.NoError(t, r.client.Create(ctx, test.dd)) dt := &duResumeTestHelper{ resumeErr: test.resumeErr, } funcResumeCancellableDataBackup = dt.resumeCancellableDataPath // Run the test err = r.AttemptDataDownloadResume(ctx, r.logger.WithField("name", test.name), test.dd.Namespace) if test.expectedError != "" { assert.EqualError(t, err, test.expectedError) } else { assert.NoError(t, err) // Verify DataDownload marked as Canceled for _, duName := range test.cancelledDataDownloads { dataDownload := &velerov2alpha1api.DataDownload{} err := r.client.Get(t.Context(), types.NamespacedName{Namespace: "velero", Name: duName}, dataDownload) require.NoError(t, err) assert.True(t, dataDownload.Spec.Cancel) } // Verify DataDownload marked as Accepted for _, duName := range test.acceptedDataDownloads { dataUpload := &velerov2alpha1api.DataDownload{} err := r.client.Get(t.Context(), types.NamespacedName{Namespace: "velero", Name: duName}, dataUpload) require.NoError(t, err) assert.Equal(t, velerov2alpha1api.DataDownloadPhaseAccepted, dataUpload.Status.Phase) } // Verify DataDownload marked as Prepared for _, duName := range test.prepareddDataDownloads { dataUpload := &velerov2alpha1api.DataDownload{} err := r.client.Get(t.Context(), types.NamespacedName{Namespace: "velero", Name: duName}, dataUpload) require.NoError(t, err) assert.Equal(t, velerov2alpha1api.DataDownloadPhasePrepared, dataUpload.Status.Phase) } } }) } } func TestResumeCancellableRestore(t *testing.T) { tests := []struct { name string dataDownloads []velerov2alpha1api.DataDownload dd *velerov2alpha1api.DataDownload getExposeErr error exposeResult *exposer.ExposeResult createWatcherErr error initWatcherErr error startWatcherErr error mockInit bool mockStart bool mockClose bool expectedError string }{ { name: "get expose failed", dd: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhaseInProgress).Result(), getExposeErr: errors.New("fake-expose-error"), expectedError: fmt.Sprintf("error to get exposed volume for dd %s: fake-expose-error", dataDownloadName), }, { name: "no expose", dd: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhaseAccepted).Node("node-1").Result(), expectedError: fmt.Sprintf("expose info missed for dd %s", dataDownloadName), }, { name: "watcher init error", dd: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhaseAccepted).Node("node-1").Result(), exposeResult: &exposer.ExposeResult{ ByPod: exposer.ExposeByPod{ HostingPod: &corev1api.Pod{}, }, }, mockInit: true, mockClose: true, initWatcherErr: errors.New("fake-init-watcher-error"), expectedError: fmt.Sprintf("error to init asyncBR watcher for dd %s: fake-init-watcher-error", dataDownloadName), }, { name: "start watcher error", dd: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhaseAccepted).Node("node-1").Result(), exposeResult: &exposer.ExposeResult{ ByPod: exposer.ExposeByPod{ HostingPod: &corev1api.Pod{}, }, }, mockInit: true, mockStart: true, mockClose: true, startWatcherErr: errors.New("fake-start-watcher-error"), expectedError: fmt.Sprintf("error to resume asyncBR watcher for dd %s: fake-start-watcher-error", dataDownloadName), }, { name: "succeed", dd: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhaseAccepted).Node("node-1").Result(), exposeResult: &exposer.ExposeResult{ ByPod: exposer.ExposeByPod{ HostingPod: &corev1api.Pod{}, }, }, mockInit: true, mockStart: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { ctx := t.Context() r, err := initDataDownloadReconciler(t, nil, false) r.nodeName = "node-1" require.NoError(t, err) mockAsyncBR := datapathmockes.NewAsyncBR(t) if test.mockInit { mockAsyncBR.On("Init", mock.Anything, mock.Anything).Return(test.initWatcherErr) } if test.mockStart { mockAsyncBR.On("StartRestore", mock.Anything, mock.Anything, mock.Anything).Return(test.startWatcherErr) } if test.mockClose { mockAsyncBR.On("Close", mock.Anything).Return() } dt := &ddResumeTestHelper{ getExposeErr: test.getExposeErr, exposeResult: test.exposeResult, asyncBR: mockAsyncBR, } r.restoreExposer = dt datapath.MicroServiceBRWatcherCreator = dt.newMicroServiceBRWatcher err = r.resumeCancellableDataPath(ctx, test.dd, velerotest.NewLogger()) if test.expectedError != "" { assert.EqualError(t, err, test.expectedError) } }) } } func TestDataDownloadSetupExposeParam(t *testing.T) { // Common objects for all cases node := builder.ForNode("worker-1").Labels(map[string]string{kube.NodeOSLabel: kube.NodeOSLinux}).Result() baseDataDownload := dataDownloadBuilder().Result() baseDataDownload.Namespace = velerov1api.DefaultNamespace baseDataDownload.Spec.OperationTimeout = metav1.Duration{Duration: time.Minute * 10} baseDataDownload.Spec.SnapshotSize = 5368709120 // 5Gi type args struct { customLabels map[string]string customAnnotations map[string]string } type want struct { labels map[string]string annotations map[string]string } tests := []struct { name string args args want want }{ { name: "label has customize values", args: args{ customLabels: map[string]string{"custom-label": "label-value"}, customAnnotations: nil, }, want: want{ labels: map[string]string{ velerov1api.DataDownloadLabel: baseDataDownload.Name, "custom-label": "label-value", }, annotations: map[string]string{}, }, }, { name: "label has no customize values", args: args{ customLabels: nil, customAnnotations: nil, }, want: want{ labels: map[string]string{velerov1api.DataDownloadLabel: baseDataDownload.Name}, annotations: map[string]string{}, }, }, { name: "annotation has customize values", args: args{ customLabels: nil, customAnnotations: map[string]string{"custom-annotation": "annotation-value"}, }, want: want{ labels: map[string]string{velerov1api.DataDownloadLabel: baseDataDownload.Name}, annotations: map[string]string{"custom-annotation": "annotation-value"}, }, }, { name: "both label and annotation have customize values", args: args{ customLabels: map[string]string{"custom-label": "label-value"}, customAnnotations: map[string]string{"custom-annotation": "annotation-value"}, }, want: want{ labels: map[string]string{ velerov1api.DataDownloadLabel: baseDataDownload.Name, "custom-label": "label-value", }, annotations: map[string]string{"custom-annotation": "annotation-value"}, }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { // Fake clients per case fakeClient := FakeClient{ Client: velerotest.NewFakeControllerRuntimeClient(t, node, baseDataDownload.DeepCopy()), } fakeKubeClient := clientgofake.NewSimpleClientset(node) // Reconciler config per case preparingTimeout := time.Minute * 3 podRes := corev1api.ResourceRequirements{} r := NewDataDownloadReconciler( &fakeClient, nil, fakeKubeClient, datapath.NewManager(1), nil, nil, velerotypes.RestorePVC{}, nil, nil, podRes, "test-node", preparingTimeout, velerotest.NewLogger(), metrics.NewServerMetrics(), "download-priority", nil, // repoConfigMgr (unused when cacheVolumeConfigs is nil) tt.args.customLabels, tt.args.customAnnotations, ) // Act got, err := r.setupExposeParam(baseDataDownload) // Assert no error require.NoError(t, err) // Core fields assert.Equal(t, baseDataDownload.Spec.TargetVolume.PVC, got.TargetPVCName) assert.Equal(t, baseDataDownload.Spec.TargetVolume.Namespace, got.TargetNamespace) // Labels and Annotations assert.Equal(t, tt.want.labels, got.HostingPodLabels) assert.Equal(t, tt.want.annotations, got.HostingPodAnnotations) }) } } ================================================ FILE: pkg/controller/data_upload_controller.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package controller import ( "context" "fmt" "strings" "time" snapshotter "github.com/kubernetes-csi/external-snapshotter/client/v8/clientset/versioned/typed/volumesnapshot/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" clocks "k8s.io/utils/clock" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" "github.com/vmware-tanzu/velero/pkg/apis/velero/shared" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" "github.com/vmware-tanzu/velero/pkg/constant" "github.com/vmware-tanzu/velero/pkg/datamover" "github.com/vmware-tanzu/velero/pkg/datapath" "github.com/vmware-tanzu/velero/pkg/exposer" "github.com/vmware-tanzu/velero/pkg/metrics" "github.com/vmware-tanzu/velero/pkg/nodeagent" velerotypes "github.com/vmware-tanzu/velero/pkg/types" "github.com/vmware-tanzu/velero/pkg/uploader" "github.com/vmware-tanzu/velero/pkg/util" "github.com/vmware-tanzu/velero/pkg/util/kube" ) const ( dataUploadDownloadRequestor = "snapshot-data-upload-download" DataUploadDownloadFinalizer = "velero.io/data-upload-download-finalizer" preparingMonitorFrequency = time.Minute cancelDelayInProgress = time.Hour cancelDelayOthers = time.Minute * 5 ) // DataUploadReconciler reconciles a DataUpload object type DataUploadReconciler struct { client client.Client kubeClient kubernetes.Interface csiSnapshotClient snapshotter.SnapshotV1Interface mgr manager.Manager Clock clocks.WithTickerAndDelayedExecution nodeName string logger logrus.FieldLogger snapshotExposerList map[velerov2alpha1api.SnapshotType]exposer.SnapshotExposer dataPathMgr *datapath.Manager vgdpCounter *exposer.VgdpCounter loadAffinity []*kube.LoadAffinity backupPVCConfig map[string]velerotypes.BackupPVC podResources corev1api.ResourceRequirements preparingTimeout time.Duration metrics *metrics.ServerMetrics cancelledDataUpload map[string]time.Time dataMovePriorityClass string podLabels map[string]string podAnnotations map[string]string } func NewDataUploadReconciler( client client.Client, mgr manager.Manager, kubeClient kubernetes.Interface, csiSnapshotClient snapshotter.SnapshotV1Interface, dataPathMgr *datapath.Manager, counter *exposer.VgdpCounter, loadAffinity []*kube.LoadAffinity, backupPVCConfig map[string]velerotypes.BackupPVC, podResources corev1api.ResourceRequirements, clock clocks.WithTickerAndDelayedExecution, nodeName string, preparingTimeout time.Duration, log logrus.FieldLogger, metrics *metrics.ServerMetrics, dataMovePriorityClass string, podLabels map[string]string, podAnnotations map[string]string, ) *DataUploadReconciler { return &DataUploadReconciler{ client: client, mgr: mgr, kubeClient: kubeClient, csiSnapshotClient: csiSnapshotClient, Clock: clock, nodeName: nodeName, logger: log, snapshotExposerList: map[velerov2alpha1api.SnapshotType]exposer.SnapshotExposer{ velerov2alpha1api.SnapshotTypeCSI: exposer.NewCSISnapshotExposer( kubeClient, csiSnapshotClient, log, ), }, dataPathMgr: dataPathMgr, vgdpCounter: counter, loadAffinity: loadAffinity, backupPVCConfig: backupPVCConfig, podResources: podResources, preparingTimeout: preparingTimeout, metrics: metrics, cancelledDataUpload: make(map[string]time.Time), dataMovePriorityClass: dataMovePriorityClass, podLabels: podLabels, podAnnotations: podAnnotations, } } // +kubebuilder:rbac:groups=velero.io,resources=datauploads,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=velero.io,resources=datauploads/status,verbs=get;update;patch // +kubebuilder:rbac:groups="",resources=pods,verbs=get // +kubebuilder:rbac:groups="",resources=persistentvolumes,verbs=get // +kubebuilder:rbac:groups="",resources=persistentvolumerclaims,verbs=get func (r *DataUploadReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := r.logger.WithFields(logrus.Fields{ "controller": "dataupload", "dataupload": req.NamespacedName, }) log.Infof("Reconcile %s", req.Name) du := &velerov2alpha1api.DataUpload{} if err := r.client.Get(ctx, req.NamespacedName, du); err != nil { if apierrors.IsNotFound(err) { log.Debug("Unable to find DataUpload") return ctrl.Result{}, nil } return ctrl.Result{}, errors.Wrap(err, "getting DataUpload") } if !datamover.IsBuiltInUploader(du.Spec.DataMover) { log.WithField("Data mover", du.Spec.DataMover).Debug("it is not one built-in data mover which is not supported by Velero") return ctrl.Result{}, nil } // Logic for clear resources when dataupload been deleted if !isDataUploadInFinalState(du) { if !controllerutil.ContainsFinalizer(du, DataUploadDownloadFinalizer) { if err := UpdateDataUploadWithRetry(ctx, r.client, req.NamespacedName, log, func(dataUpload *velerov2alpha1api.DataUpload) bool { if controllerutil.ContainsFinalizer(dataUpload, DataUploadDownloadFinalizer) { return false } controllerutil.AddFinalizer(dataUpload, DataUploadDownloadFinalizer) return true }); err != nil { log.WithError(err).Errorf("failed to add finalizer for du %s/%s", du.Namespace, du.Name) return ctrl.Result{}, err } return ctrl.Result{}, nil } if !du.DeletionTimestamp.IsZero() { if !du.Spec.Cancel { // when delete cr we need to clear up internal resources created by Velero, here we use the cancel mechanism // to help clear up resources instead of clear them directly in case of some conflict with Expose action log.Warnf("Cancel du under phase %s because it is being deleted", du.Status.Phase) if err := UpdateDataUploadWithRetry(ctx, r.client, req.NamespacedName, log, func(dataUpload *velerov2alpha1api.DataUpload) bool { if dataUpload.Spec.Cancel { return false } dataUpload.Spec.Cancel = true dataUpload.Status.Message = "Cancel dataupload because it is being deleted" return true }); err != nil { log.WithError(err).Errorf("failed to set cancel flag for du %s/%s", du.Namespace, du.Name) return ctrl.Result{}, err } return ctrl.Result{}, nil } } } else { delete(r.cancelledDataUpload, du.Name) // put the finalizer remove action here for all cr will goes to the final status, we could check finalizer and do remove action in final status // instead of intermediate state. // remove finalizer no matter whether the cr is being deleted or not for it is no longer needed when internal resources are all cleaned up // also in final status cr won't block the direct delete of the velero namespace if controllerutil.ContainsFinalizer(du, DataUploadDownloadFinalizer) { if err := UpdateDataUploadWithRetry(ctx, r.client, req.NamespacedName, log, func(du *velerov2alpha1api.DataUpload) bool { if !controllerutil.ContainsFinalizer(du, DataUploadDownloadFinalizer) { return false } controllerutil.RemoveFinalizer(du, DataUploadDownloadFinalizer) return true }); err != nil { log.WithError(err).Error("error to remove finalizer") return ctrl.Result{}, err } return ctrl.Result{}, nil } } if du.Spec.Cancel { if spotted, found := r.cancelledDataUpload[du.Name]; !found { r.cancelledDataUpload[du.Name] = r.Clock.Now() } else { delay := cancelDelayOthers if du.Status.Phase == velerov2alpha1api.DataUploadPhaseInProgress { delay = cancelDelayInProgress } if time.Since(spotted) > delay { log.Infof("Data upload %s is canceled in Phase %s but not handled in reasonable time", du.GetName(), du.Status.Phase) if r.tryCancelDataUpload(ctx, du, "") { delete(r.cancelledDataUpload, du.Name) } return ctrl.Result{}, nil } } } ep, ok := r.snapshotExposerList[du.Spec.SnapshotType] if !ok { return r.errorOut(ctx, du, errors.Errorf("%s type of snapshot exposer is not exist", du.Spec.SnapshotType), "not exist type of exposer", log) } if du.Status.Phase == "" || du.Status.Phase == velerov2alpha1api.DataUploadPhaseNew { if du.Spec.Cancel { log.Debugf("Data upload is canceled in Phase %s", du.Status.Phase) r.tryCancelDataUpload(ctx, du, "") return ctrl.Result{}, nil } if r.vgdpCounter != nil && r.vgdpCounter.IsConstrained(ctx, r.logger) { log.Debug("Data path initiation is constrained, requeue later") return ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, nil } log.Info("Data upload starting") accepted, err := r.acceptDataUpload(ctx, du) if err != nil { return ctrl.Result{}, errors.Wrapf(err, "error accepting the data upload %s", du.Name) } if !accepted { log.Debug("Data upload is not accepted") return ctrl.Result{}, nil } log.Info("Data upload is accepted") exposeParam, err := r.setupExposeParam(du) if err != nil { return r.errorOut(ctx, du, err, "failed to set exposer parameters", log) } // Expose() will trigger to create one pod whose volume is restored by a given volume snapshot, // but the pod maybe is not in the same node of the current controller, so we need to return it here. // And then only the controller who is in the same node could do the rest work. if err := ep.Expose(ctx, getOwnerObject(du), exposeParam); err != nil { return r.errorOut(ctx, du, err, "error exposing snapshot", log) } log.Info("Snapshot is exposed") return ctrl.Result{}, nil } else if du.Status.Phase == velerov2alpha1api.DataUploadPhaseAccepted { if peekErr := ep.PeekExposed(ctx, getOwnerObject(du)); peekErr != nil { log.Errorf("Cancel du %s/%s because of expose error %s", du.Namespace, du.Name, peekErr) diags := strings.Split(ep.DiagnoseExpose(ctx, getOwnerObject(du)), "\n") for _, diag := range diags { log.Warnf("[Diagnose DU expose]%s", diag) } r.tryCancelDataUpload(ctx, du, fmt.Sprintf("found a du %s/%s with expose error: %s. mark it as cancel", du.Namespace, du.Name, peekErr)) } else if du.Status.AcceptedTimestamp != nil { if time.Since(du.Status.AcceptedTimestamp.Time) >= r.preparingTimeout { r.onPrepareTimeout(ctx, du) } } return ctrl.Result{}, nil } else if du.Status.Phase == velerov2alpha1api.DataUploadPhasePrepared { log.Infof("Data upload is prepared and should be processed by %s (%s)", du.Status.Node, r.nodeName) if du.Status.Node != r.nodeName { return ctrl.Result{}, nil } if du.Spec.Cancel { log.Info("Prepared data upload is being canceled") r.OnDataUploadCancelled(ctx, du.GetNamespace(), du.GetName()) return ctrl.Result{}, nil } asyncBR := r.dataPathMgr.GetAsyncBR(du.Name) if asyncBR != nil { log.Info("Cancellable data path is already started") return ctrl.Result{}, nil } waitExposePara := r.setupWaitExposePara(du) res, err := ep.GetExposed(ctx, getOwnerObject(du), du.Spec.OperationTimeout.Duration, waitExposePara) if err != nil { return r.errorOut(ctx, du, err, "exposed snapshot is not ready", log) } else if res == nil { return r.errorOut(ctx, du, errors.New("no expose result is available for the current node"), "exposed snapshot is not ready", log) } if res.ByPod.NodeOS == nil { return r.errorOut(ctx, du, errors.New("unsupported ambiguous node OS"), "invalid expose result", log) } log.Info("Exposed snapshot is ready and creating data path routine") // Need to first create file system BR and get data path instance then update data upload status callbacks := datapath.Callbacks{ OnCompleted: r.OnDataUploadCompleted, OnFailed: r.OnDataUploadFailed, OnCancelled: r.OnDataUploadCancelled, OnProgress: r.OnDataUploadProgress, } asyncBR, err = r.dataPathMgr.CreateMicroServiceBRWatcher(ctx, r.client, r.kubeClient, r.mgr, datapath.TaskTypeBackup, du.Name, du.Namespace, res.ByPod.HostingPod.Name, res.ByPod.HostingContainer, du.Name, callbacks, false, log) if err != nil { if err == datapath.ConcurrentLimitExceed { log.Debug("Data path instance is concurrent limited requeue later") return ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, nil } else { return r.errorOut(ctx, du, err, "error to create data path", log) } } if err := r.initCancelableDataPath(ctx, asyncBR, res, log); err != nil { log.WithError(err).Errorf("Failed to init cancelable data path for %s", du.Name) r.closeDataPath(ctx, du.Name) return r.errorOut(ctx, du, err, "error initializing data path", log) } // Update status to InProgress terminated := false if err := UpdateDataUploadWithRetry(ctx, r.client, types.NamespacedName{Namespace: du.Namespace, Name: du.Name}, log, func(du *velerov2alpha1api.DataUpload) bool { if isDataUploadInFinalState(du) { terminated = true return false } du.Status.Phase = velerov2alpha1api.DataUploadPhaseInProgress du.Status.StartTimestamp = &metav1.Time{Time: r.Clock.Now()} du.Status.NodeOS = velerov2alpha1api.NodeOS(*res.ByPod.NodeOS) delete(du.Labels, exposer.ExposeOnGoingLabel) return true }); err != nil { log.WithError(err).Warnf("Failed to update dataupload %s to InProgress, will data path close and retry", du.Name) r.closeDataPath(ctx, du.Name) return ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, nil } if terminated { log.Warnf("dataupload %s is terminated during transition from prepared", du.Name) r.closeDataPath(ctx, du.Name) return ctrl.Result{}, nil } log.Info("Data upload is marked as in progress") if err := r.startCancelableDataPath(asyncBR, du, res, log); err != nil { log.WithError(err).Errorf("Failed to start cancelable data path for %s", du.Name) r.closeDataPath(ctx, du.Name) return r.errorOut(ctx, du, err, "error starting data path", log) } return ctrl.Result{}, nil } else if du.Status.Phase == velerov2alpha1api.DataUploadPhaseInProgress { if du.Spec.Cancel { if du.Status.Node != r.nodeName { return ctrl.Result{}, nil } log.Info("In progress data upload is being canceled") asyncBR := r.dataPathMgr.GetAsyncBR(du.Name) if asyncBR == nil { r.OnDataUploadCancelled(ctx, du.GetNamespace(), du.GetName()) return ctrl.Result{}, nil } // Update status to Canceling if err := UpdateDataUploadWithRetry(ctx, r.client, types.NamespacedName{Namespace: du.Namespace, Name: du.Name}, log, func(du *velerov2alpha1api.DataUpload) bool { if isDataUploadInFinalState(du) { log.Warnf("dataupload %s is terminated, abort setting it to canceling", du.Name) return false } du.Status.Phase = velerov2alpha1api.DataUploadPhaseCanceling return true }); err != nil { log.WithError(err).Error("error updating data upload into canceling status") return ctrl.Result{}, err } asyncBR.Cancel() return ctrl.Result{}, nil } } return ctrl.Result{}, nil } func (r *DataUploadReconciler) initCancelableDataPath(ctx context.Context, asyncBR datapath.AsyncBR, res *exposer.ExposeResult, log logrus.FieldLogger) error { log.Info("Init cancelable dataUpload") if err := asyncBR.Init(ctx, nil); err != nil { return errors.Wrap(err, "error initializing asyncBR") } log.Infof("async backup init for pod %s, volume %s", res.ByPod.HostingPod.Name, res.ByPod.VolumeName) return nil } func (r *DataUploadReconciler) startCancelableDataPath(asyncBR datapath.AsyncBR, du *velerov2alpha1api.DataUpload, res *exposer.ExposeResult, log logrus.FieldLogger) error { log.Info("Start cancelable dataUpload") if err := asyncBR.StartBackup(datapath.AccessPoint{ ByPath: res.ByPod.VolumeName, }, du.Spec.DataMoverConfig, nil); err != nil { return errors.Wrapf(err, "error starting async backup for pod %s, volume %s", res.ByPod.HostingPod.Name, res.ByPod.VolumeName) } log.Infof("Async backup started for pod %s, volume %s", res.ByPod.HostingPod.Name, res.ByPod.VolumeName) return nil } func (r *DataUploadReconciler) OnDataUploadCompleted(ctx context.Context, namespace string, duName string, result datapath.Result) { defer r.dataPathMgr.RemoveAsyncBR(duName) log := r.logger.WithField("dataupload", duName) log.Info("Async fs backup data path completed") var du velerov2alpha1api.DataUpload if err := r.client.Get(ctx, types.NamespacedName{Name: duName, Namespace: namespace}, &du); err != nil { log.WithError(err).Warn("Failed to get dataupload on completion") return } // cleans up any objects generated during the snapshot expose ep, ok := r.snapshotExposerList[du.Spec.SnapshotType] if !ok { log.WithError(fmt.Errorf("%v type of snapshot exposer is not exist", du.Spec.SnapshotType)). Warn("Failed to clean up resources on completion") } else { var volumeSnapshotName string if du.Spec.SnapshotType == velerov2alpha1api.SnapshotTypeCSI { // Other exposer should have another condition volumeSnapshotName = du.Spec.CSISnapshot.VolumeSnapshot } ep.CleanUp(ctx, getOwnerObject(&du), volumeSnapshotName, du.Spec.SourceNamespace) } // Update status to Completed with path & snapshot ID. if err := UpdateDataUploadWithRetry(ctx, r.client, types.NamespacedName{Namespace: du.Namespace, Name: du.Name}, log, func(du *velerov2alpha1api.DataUpload) bool { if isDataUploadInFinalState(du) { return false } du.Status.Path = result.Backup.Source.ByPath du.Status.Phase = velerov2alpha1api.DataUploadPhaseCompleted du.Status.SnapshotID = result.Backup.SnapshotID du.Status.IncrementalBytes = result.Backup.IncrementalBytes du.Status.CompletionTimestamp = &metav1.Time{Time: r.Clock.Now()} if result.Backup.EmptySnapshot { du.Status.Message = "volume was empty so no data was upload" } delete(du.Labels, exposer.ExposeOnGoingLabel) return true }); err != nil { log.WithError(err).Error("error updating DataUpload status") } else { log.Info("Data upload completed") r.metrics.RegisterDataUploadSuccess(r.nodeName) } } func (r *DataUploadReconciler) OnDataUploadFailed(ctx context.Context, namespace, duName string, err error) { defer r.dataPathMgr.RemoveAsyncBR(duName) log := r.logger.WithField("dataupload", duName) log.WithError(err).Error("Async fs backup data path failed") var du velerov2alpha1api.DataUpload if getErr := r.client.Get(ctx, types.NamespacedName{Name: duName, Namespace: namespace}, &du); getErr != nil { log.WithError(getErr).Warn("Failed to get dataupload on failure") } else { _, _ = r.errorOut(ctx, &du, err, "data path backup failed", log) } } func (r *DataUploadReconciler) OnDataUploadCancelled(ctx context.Context, namespace string, duName string) { defer r.dataPathMgr.RemoveAsyncBR(duName) log := r.logger.WithField("dataupload", duName) log.Warn("Async fs backup data path canceled") du := &velerov2alpha1api.DataUpload{} if getErr := r.client.Get(ctx, types.NamespacedName{Name: duName, Namespace: namespace}, du); getErr != nil { log.WithError(getErr).Warn("Failed to get dataupload on cancel") return } // cleans up any objects generated during the snapshot expose r.cleanUp(ctx, du, log) if err := UpdateDataUploadWithRetry(ctx, r.client, types.NamespacedName{Namespace: du.Namespace, Name: du.Name}, log, func(du *velerov2alpha1api.DataUpload) bool { if isDataUploadInFinalState(du) { return false } du.Status.Phase = velerov2alpha1api.DataUploadPhaseCanceled if du.Status.StartTimestamp.IsZero() { du.Status.StartTimestamp = &metav1.Time{Time: r.Clock.Now()} } du.Status.CompletionTimestamp = &metav1.Time{Time: r.Clock.Now()} delete(du.Labels, exposer.ExposeOnGoingLabel) return true }); err != nil { log.WithError(err).Error("error updating DataUpload status") } else { r.metrics.RegisterDataUploadCancel(r.nodeName) delete(r.cancelledDataUpload, du.Name) } } func (r *DataUploadReconciler) tryCancelDataUpload(ctx context.Context, du *velerov2alpha1api.DataUpload, message string) bool { log := r.logger.WithField("dataupload", du.Name) succeeded, err := funcExclusiveUpdateDataUpload(ctx, r.client, du, func(dataUpload *velerov2alpha1api.DataUpload) { dataUpload.Status.Phase = velerov2alpha1api.DataUploadPhaseCanceled if dataUpload.Status.StartTimestamp.IsZero() { dataUpload.Status.StartTimestamp = &metav1.Time{Time: r.Clock.Now()} } dataUpload.Status.CompletionTimestamp = &metav1.Time{Time: r.Clock.Now()} if message != "" { dataUpload.Status.Message = message } delete(dataUpload.Labels, exposer.ExposeOnGoingLabel) }) if err != nil { log.WithError(err).Error("error updating dataupload status") return false } else if !succeeded { log.Warn("conflict in updating dataupload status and will try it again later") return false } // success update r.metrics.RegisterDataUploadCancel(r.nodeName) // cleans up any objects generated during the snapshot expose r.cleanUp(ctx, du, log) log.Warn("data upload is canceled") return true } func (r *DataUploadReconciler) cleanUp(ctx context.Context, du *velerov2alpha1api.DataUpload, log logrus.FieldLogger) { ep, ok := r.snapshotExposerList[du.Spec.SnapshotType] if !ok { log.WithError(fmt.Errorf("%v type of snapshot exposer is not exist", du.Spec.SnapshotType)). Warn("Failed to clean up resources on canceled") } else { var volumeSnapshotName string if du.Spec.SnapshotType == velerov2alpha1api.SnapshotTypeCSI { // Other exposer should have another condition volumeSnapshotName = du.Spec.CSISnapshot.VolumeSnapshot } ep.CleanUp(ctx, getOwnerObject(du), volumeSnapshotName, du.Spec.SourceNamespace) } } func (r *DataUploadReconciler) OnDataUploadProgress(ctx context.Context, namespace string, duName string, progress *uploader.Progress) { log := r.logger.WithField("dataupload", duName) if err := UpdateDataUploadWithRetry(ctx, r.client, types.NamespacedName{Namespace: namespace, Name: duName}, log, func(du *velerov2alpha1api.DataUpload) bool { du.Status.Progress = shared.DataMoveOperationProgress{TotalBytes: progress.TotalBytes, BytesDone: progress.BytesDone} return true }); err != nil { log.WithError(err).Error("Failed to update progress") } } // SetupWithManager registers the DataUpload controller. // The fresh new DataUpload CR first created will trigger to create one pod (long time, maybe failure or unknown status) by one of the dataupload controllers // then the request will get out of the Reconcile queue immediately by not blocking others' CR handling, in order to finish the rest data upload process we need to // re-enqueue the previous related request once the related pod is in running status to keep going on the rest logic. and below logic will avoid handling the unwanted // pod status and also avoid block others CR handling func (r *DataUploadReconciler) SetupWithManager(mgr ctrl.Manager) error { gp := kube.NewGenericEventPredicate(func(object client.Object) bool { du := object.(*velerov2alpha1api.DataUpload) if du.Status.Phase == velerov2alpha1api.DataUploadPhaseAccepted { return true } if du.Spec.Cancel && !isDataUploadInFinalState(du) { return true } if isDataUploadInFinalState(du) && !du.DeletionTimestamp.IsZero() { return true } return false }) s := kube.NewPeriodicalEnqueueSource(r.logger.WithField("controller", constant.ControllerDataUpload), r.client, &velerov2alpha1api.DataUploadList{}, preparingMonitorFrequency, kube.PeriodicalEnqueueSourceOption{ Predicates: []predicate.Predicate{gp}, }) return ctrl.NewControllerManagedBy(mgr). For(&velerov2alpha1api.DataUpload{}). WatchesRawSource(s). Watches(&corev1api.Pod{}, kube.EnqueueRequestsFromMapUpdateFunc(r.findDataUploadForPod), builder.WithPredicates(predicate.Funcs{ UpdateFunc: func(ue event.UpdateEvent) bool { newObj := ue.ObjectNew.(*corev1api.Pod) if _, ok := newObj.Labels[velerov1api.DataUploadLabel]; !ok { return false } if newObj.Spec.NodeName != r.nodeName { return false } return true }, CreateFunc: func(event.CreateEvent) bool { return false }, DeleteFunc: func(de event.DeleteEvent) bool { return false }, GenericFunc: func(ge event.GenericEvent) bool { return false }, })). Complete(r) } func (r *DataUploadReconciler) findDataUploadForPod(ctx context.Context, podObj client.Object) []reconcile.Request { pod := podObj.(*corev1api.Pod) du, err := findDataUploadByPod(r.client, *pod) log := r.logger.WithFields(logrus.Fields{ "Backup pod": pod.Name, }) if err != nil { log.WithError(err).Error("unable to get dataupload") return []reconcile.Request{} } else if du == nil { log.Error("get empty DataUpload") return []reconcile.Request{} } log = log.WithFields(logrus.Fields{ "Datadupload": du.Name, }) if du.Status.Phase != velerov2alpha1api.DataUploadPhaseAccepted { return []reconcile.Request{} } if pod.Status.Phase == corev1api.PodRunning { log.Info("Preparing dataupload") if err = UpdateDataUploadWithRetry(context.Background(), r.client, types.NamespacedName{Namespace: du.Namespace, Name: du.Name}, log, func(du *velerov2alpha1api.DataUpload) bool { if isDataUploadInFinalState(du) { log.Warnf("dataupload %s is terminated, abort setting it to prepared", du.Name) return false } r.prepareDataUpload(du) return true }); err != nil { log.WithError(err).Warn("failed to update dataupload, prepare will halt for this dataupload") return []reconcile.Request{} } } else if unrecoverable, reason := kube.IsPodUnrecoverable(pod, log); unrecoverable { // let the abnormal backup pod failed early err := UpdateDataUploadWithRetry(context.Background(), r.client, types.NamespacedName{Namespace: du.Namespace, Name: du.Name}, r.logger.WithField("dataupload", du.Name), func(dataUpload *velerov2alpha1api.DataUpload) bool { if dataUpload.Spec.Cancel { return false } dataUpload.Spec.Cancel = true dataUpload.Status.Message = fmt.Sprintf("Cancel dataupload because the exposing pod %s/%s is in abnormal status for reason %s", pod.Namespace, pod.Name, reason) return true }) if err != nil { log.WithError(err).Warn("failed to cancel dataupload, and it will wait for prepare timeout") return []reconcile.Request{} } log.Infof("Exposed pod is in abnormal status(reason %s) and dataupload is marked as cancel", reason) } else { return []reconcile.Request{} } request := reconcile.Request{ NamespacedName: types.NamespacedName{ Namespace: du.Namespace, Name: du.Name, }, } return []reconcile.Request{request} } func (r *DataUploadReconciler) prepareDataUpload(du *velerov2alpha1api.DataUpload) { du.Status.Phase = velerov2alpha1api.DataUploadPhasePrepared du.Status.Node = r.nodeName } func (r *DataUploadReconciler) errorOut(ctx context.Context, du *velerov2alpha1api.DataUpload, err error, msg string, log logrus.FieldLogger) (ctrl.Result, error) { if se, ok := r.snapshotExposerList[du.Spec.SnapshotType]; ok { var volumeSnapshotName string if du.Spec.SnapshotType == velerov2alpha1api.SnapshotTypeCSI { // Other exposer should have another condition volumeSnapshotName = du.Spec.CSISnapshot.VolumeSnapshot } se.CleanUp(ctx, getOwnerObject(du), volumeSnapshotName, du.Spec.SourceNamespace) } else { log.Errorf("failed to clean up exposed snapshot could not find %s snapshot exposer", du.Spec.SnapshotType) } return ctrl.Result{}, r.updateStatusToFailed(ctx, du, err, msg, log) } func (r *DataUploadReconciler) updateStatusToFailed(ctx context.Context, du *velerov2alpha1api.DataUpload, err error, msg string, log logrus.FieldLogger) error { log.Info("update data upload status to Failed") if patchErr := UpdateDataUploadWithRetry(ctx, r.client, types.NamespacedName{Namespace: du.Namespace, Name: du.Name}, log, func(du *velerov2alpha1api.DataUpload) bool { if isDataUploadInFinalState(du) { return false } du.Status.Phase = velerov2alpha1api.DataUploadPhaseFailed du.Status.Message = errors.WithMessage(err, msg).Error() if du.Status.StartTimestamp.IsZero() { du.Status.StartTimestamp = &metav1.Time{Time: r.Clock.Now()} } if dataPathError, ok := err.(datapath.DataPathError); ok { du.Status.SnapshotID = dataPathError.GetSnapshotID() } du.Status.CompletionTimestamp = &metav1.Time{Time: r.Clock.Now()} delete(du.Labels, exposer.ExposeOnGoingLabel) return true }); patchErr != nil { log.WithError(patchErr).Error("error updating DataUpload status") } else { r.metrics.RegisterDataUploadFailure(r.nodeName) } return err } func (r *DataUploadReconciler) acceptDataUpload(ctx context.Context, du *velerov2alpha1api.DataUpload) (bool, error) { r.logger.Infof("Accepting data upload %s", du.Name) // For all data upload controller in each node-agent will try to update dataupload CR, and only one controller will success, // and the success one could handle later logic updated := du.DeepCopy() updateFunc := func(dataUpload *velerov2alpha1api.DataUpload) { dataUpload.Status.Phase = velerov2alpha1api.DataUploadPhaseAccepted dataUpload.Status.AcceptedByNode = r.nodeName dataUpload.Status.AcceptedTimestamp = &metav1.Time{Time: r.Clock.Now()} if dataUpload.Labels == nil { dataUpload.Labels = make(map[string]string) } dataUpload.Labels[exposer.ExposeOnGoingLabel] = "true" } succeeded, err := funcExclusiveUpdateDataUpload(ctx, r.client, updated, updateFunc) if err != nil { return false, err } if succeeded { updateFunc(du) // If update success, it's need to update du values in memory r.logger.WithField("Dataupload", du.Name).Infof("This datauplod has been accepted by %s", r.nodeName) return true, nil } r.logger.WithField("Dataupload", du.Name).Info("This datauplod has been accepted by others") return false, nil } func (r *DataUploadReconciler) onPrepareTimeout(ctx context.Context, du *velerov2alpha1api.DataUpload) { log := r.logger.WithField("Dataupload", du.Name) log.Info("Timeout happened for preparing dataupload") succeeded, err := funcExclusiveUpdateDataUpload(ctx, r.client, du, func(du *velerov2alpha1api.DataUpload) { du.Status.Phase = velerov2alpha1api.DataUploadPhaseFailed du.Status.Message = "timeout on preparing data upload" delete(du.Labels, exposer.ExposeOnGoingLabel) }) if err != nil { log.WithError(err).Warn("Failed to update dataupload") return } if !succeeded { log.Warn("Dataupload has been updated by others") return } ep, ok := r.snapshotExposerList[du.Spec.SnapshotType] if !ok { log.WithError(fmt.Errorf("%v type of snapshot exposer is not exist", du.Spec.SnapshotType)). Warn("Failed to clean up resources on canceled") } else { var volumeSnapshotName string if du.Spec.SnapshotType == velerov2alpha1api.SnapshotTypeCSI { // Other exposer should have another condition volumeSnapshotName = du.Spec.CSISnapshot.VolumeSnapshot } diags := strings.Split(ep.DiagnoseExpose(ctx, getOwnerObject(du)), "\n") for _, diag := range diags { log.Warnf("[Diagnose DU expose]%s", diag) } ep.CleanUp(ctx, getOwnerObject(du), volumeSnapshotName, du.Spec.SourceNamespace) log.Info("Dataupload has been cleaned up") } r.metrics.RegisterDataUploadFailure(r.nodeName) } var funcExclusiveUpdateDataUpload = exclusiveUpdateDataUpload func exclusiveUpdateDataUpload(ctx context.Context, cli client.Client, du *velerov2alpha1api.DataUpload, updateFunc func(*velerov2alpha1api.DataUpload)) (bool, error) { updateFunc(du) err := cli.Update(ctx, du) if err == nil { return true, nil } // warn we won't rollback du values in memory when error if apierrors.IsConflict(err) { return false, nil } else { return false, err } } func (r *DataUploadReconciler) closeDataPath(ctx context.Context, duName string) { asyncBR := r.dataPathMgr.GetAsyncBR(duName) if asyncBR != nil { asyncBR.Close(ctx) } r.dataPathMgr.RemoveAsyncBR(duName) } func (r *DataUploadReconciler) setupExposeParam(du *velerov2alpha1api.DataUpload) (any, error) { log := r.logger.WithField("dataupload", du.Name) if du.Spec.SnapshotType == velerov2alpha1api.SnapshotTypeCSI { pvc := &corev1api.PersistentVolumeClaim{} err := r.client.Get(context.Background(), types.NamespacedName{ Namespace: du.Spec.SourceNamespace, Name: du.Spec.SourcePVC, }, pvc) if err != nil { return nil, errors.Wrapf(err, "failed to get PVC %s/%s", du.Spec.SourceNamespace, du.Spec.SourcePVC) } pv := &corev1api.PersistentVolume{} if err := r.client.Get(context.Background(), types.NamespacedName{ Name: pvc.Spec.VolumeName, }, pv); err != nil { return nil, errors.Wrapf(err, "failed to get source PV %s", pvc.Spec.VolumeName) } nodeOS := kube.GetPVCAttachingNodeOS(pvc, r.kubeClient.CoreV1(), r.kubeClient.StorageV1(), log) if err := kube.HasNodeWithOS(context.Background(), nodeOS, r.kubeClient.CoreV1()); err != nil { return nil, errors.Wrapf(err, "no appropriate node to run data upload for PVC %s/%s", du.Spec.SourceNamespace, du.Spec.SourcePVC) } accessMode := exposer.AccessModeFileSystem if pvc.Spec.VolumeMode != nil && *pvc.Spec.VolumeMode == corev1api.PersistentVolumeBlock { accessMode = exposer.AccessModeBlock } hostingPodLabels := map[string]string{velerov1api.DataUploadLabel: du.Name} if len(r.podLabels) > 0 { for k, v := range r.podLabels { hostingPodLabels[k] = v } } else { for _, k := range util.ThirdPartyLabels { if v, err := nodeagent.GetLabelValue(context.Background(), r.kubeClient, du.Namespace, k, nodeOS); err != nil { if err != nodeagent.ErrNodeAgentLabelNotFound { log.WithError(err).Warnf("Failed to check node-agent label, skip adding host pod label %s", k) } } else { hostingPodLabels[k] = v } } } hostingPodAnnotation := map[string]string{} if len(r.podAnnotations) > 0 { for k, v := range r.podAnnotations { hostingPodAnnotation[k] = v } } else { for _, k := range util.ThirdPartyAnnotations { if v, err := nodeagent.GetAnnotationValue(context.Background(), r.kubeClient, du.Namespace, k, nodeOS); err != nil { if err != nodeagent.ErrNodeAgentAnnotationNotFound { log.WithError(err).Warnf("Failed to check node-agent annotation, skip adding host pod annotation %s", k) } } else { hostingPodAnnotation[k] = v } } } hostingPodTolerations := []corev1api.Toleration{} for _, k := range util.ThirdPartyTolerations { if v, err := nodeagent.GetToleration(context.Background(), r.kubeClient, du.Namespace, k, nodeOS); err != nil { if err != nodeagent.ErrNodeAgentTolerationNotFound { log.WithError(err).Warnf("Failed to check node-agent toleration, skip adding host pod toleration %s", k) } } else { hostingPodTolerations = append(hostingPodTolerations, *v) } } return &exposer.CSISnapshotExposeParam{ SnapshotName: du.Spec.CSISnapshot.VolumeSnapshot, SourceNamespace: du.Spec.SourceNamespace, SourcePVCName: pvc.Name, SourcePVName: pv.Name, StorageClass: du.Spec.CSISnapshot.StorageClass, HostingPodLabels: hostingPodLabels, HostingPodAnnotations: hostingPodAnnotation, HostingPodTolerations: hostingPodTolerations, AccessMode: accessMode, OperationTimeout: du.Spec.OperationTimeout.Duration, ExposeTimeout: r.preparingTimeout, VolumeSize: pvc.Spec.Resources.Requests[corev1api.ResourceStorage], Affinity: r.loadAffinity, BackupPVCConfig: r.backupPVCConfig, Resources: r.podResources, NodeOS: nodeOS, PriorityClassName: r.dataMovePriorityClass, }, nil } return nil, nil } func (r *DataUploadReconciler) setupWaitExposePara(du *velerov2alpha1api.DataUpload) any { if du.Spec.SnapshotType == velerov2alpha1api.SnapshotTypeCSI { return &exposer.CSISnapshotExposeWaitParam{ NodeClient: r.client, NodeName: r.nodeName, } } return nil } func getOwnerObject(du *velerov2alpha1api.DataUpload) corev1api.ObjectReference { return corev1api.ObjectReference{ Kind: du.Kind, Namespace: du.Namespace, Name: du.Name, UID: du.UID, APIVersion: du.APIVersion, } } func findDataUploadByPod(client client.Client, pod corev1api.Pod) (*velerov2alpha1api.DataUpload, error) { if label, exist := pod.Labels[velerov1api.DataUploadLabel]; exist { du := &velerov2alpha1api.DataUpload{} err := client.Get(context.Background(), types.NamespacedName{ Namespace: pod.Namespace, Name: label, }, du) if err != nil { return nil, errors.Wrapf(err, "error to find DataUpload by pod %s/%s", pod.Namespace, pod.Name) } return du, nil } return nil, nil } func isDataUploadInFinalState(du *velerov2alpha1api.DataUpload) bool { return du.Status.Phase == velerov2alpha1api.DataUploadPhaseFailed || du.Status.Phase == velerov2alpha1api.DataUploadPhaseCanceled || du.Status.Phase == velerov2alpha1api.DataUploadPhaseCompleted } func UpdateDataUploadWithRetry(ctx context.Context, client client.Client, namespacedName types.NamespacedName, log logrus.FieldLogger, updateFunc func(*velerov2alpha1api.DataUpload) bool) error { return wait.PollUntilContextCancel(ctx, time.Second, true, func(ctx context.Context) (bool, error) { du := &velerov2alpha1api.DataUpload{} if err := client.Get(ctx, namespacedName, du); err != nil { return false, errors.Wrap(err, "getting DataUpload") } if updateFunc(du) { err := client.Update(ctx, du) if err != nil { if apierrors.IsConflict(err) { log.Debugf("failed to update dataupload for %s/%s and will retry it", du.Namespace, du.Name) return false, nil } else { return false, errors.Wrapf(err, "error updating dataupload with error %s/%s", du.Namespace, du.Name) } } } return true, nil }) } var funcResumeCancellableDataBackup = (*DataUploadReconciler).resumeCancellableDataPath func (r *DataUploadReconciler) AttemptDataUploadResume(ctx context.Context, logger *logrus.Entry, ns string) error { dataUploads := &velerov2alpha1api.DataUploadList{} if err := r.client.List(ctx, dataUploads, &client.ListOptions{Namespace: ns}); err != nil { r.logger.WithError(errors.WithStack(err)).Error("failed to list datauploads") return errors.Wrapf(err, "error to list datauploads") } for i := range dataUploads.Items { du := &dataUploads.Items[i] if du.Status.Phase == velerov2alpha1api.DataUploadPhaseInProgress { if du.Status.Node != r.nodeName { logger.WithField("du", du.Name).WithField("current node", r.nodeName).Infof("DU should be resumed by another node %s", du.Status.Node) continue } err := funcResumeCancellableDataBackup(r, ctx, du, logger) if err == nil { logger.WithField("du", du.Name).WithField("current node", r.nodeName).Info("Completed to resume in progress DU") continue } logger.WithField("dataupload", du.GetName()).WithError(err).Warn("Failed to resume data path for du, have to cancel it") resumeErr := err err = UpdateDataUploadWithRetry(ctx, r.client, types.NamespacedName{Namespace: du.Namespace, Name: du.Name}, logger.WithField("dataupload", du.Name), func(dataUpload *velerov2alpha1api.DataUpload) bool { if dataUpload.Spec.Cancel { return false } dataUpload.Spec.Cancel = true dataUpload.Status.Message = fmt.Sprintf("Resume InProgress dataupload failed with error %v, mark it as cancel", resumeErr) return true }) if err != nil { logger.WithField("dataupload", du.GetName()).WithError(errors.WithStack(err)).Error("Failed to trigger dataupload cancel") } } else if !isDataUploadInFinalState(du) { // the Prepared CR could be still handled by dataupload controller after node-agent restart // the accepted CR may also suvived from node-agent restart as long as the intermediate objects are all done logger.WithField("dataupload", du.GetName()).Infof("find a dataupload with status %s", du.Status.Phase) } } return nil } func (r *DataUploadReconciler) resumeCancellableDataPath(ctx context.Context, du *velerov2alpha1api.DataUpload, log logrus.FieldLogger) error { log.Info("Resume cancelable dataUpload") ep, ok := r.snapshotExposerList[du.Spec.SnapshotType] if !ok { return errors.Errorf("error to find exposer for du %s", du.Name) } waitExposePara := r.setupWaitExposePara(du) res, err := ep.GetExposed(ctx, getOwnerObject(du), du.Spec.OperationTimeout.Duration, waitExposePara) if err != nil { return errors.Wrapf(err, "error to get exposed snapshot for du %s", du.Name) } if res == nil { return errors.Errorf("expose info missed for du %s", du.Name) } callbacks := datapath.Callbacks{ OnCompleted: r.OnDataUploadCompleted, OnFailed: r.OnDataUploadFailed, OnCancelled: r.OnDataUploadCancelled, OnProgress: r.OnDataUploadProgress, } asyncBR, err := r.dataPathMgr.CreateMicroServiceBRWatcher(ctx, r.client, r.kubeClient, r.mgr, datapath.TaskTypeBackup, du.Name, du.Namespace, res.ByPod.HostingPod.Name, res.ByPod.HostingContainer, du.Name, callbacks, true, log) if err != nil { return errors.Wrapf(err, "error to create asyncBR watcher for du %s", du.Name) } resumeComplete := false defer func() { if !resumeComplete { r.closeDataPath(ctx, du.Name) } }() if err := asyncBR.Init(ctx, nil); err != nil { return errors.Wrapf(err, "error to init asyncBR watcher for du %s", du.Name) } if err := asyncBR.StartBackup(datapath.AccessPoint{ ByPath: res.ByPod.VolumeName, }, du.Spec.DataMoverConfig, nil); err != nil { return errors.Wrapf(err, "error to resume asyncBR watcher for du %s", du.Name) } resumeComplete = true log.Infof("asyncBR is resumed for du %s", du.Name) return nil } ================================================ FILE: pkg/controller/data_upload_controller_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package controller import ( "context" "fmt" "testing" "time" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" snapshotFake "github.com/kubernetes-csi/external-snapshotter/client/v8/clientset/versioned/fake" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" appsv1api "k8s.io/api/apps/v1" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" clientgofake "k8s.io/client-go/kubernetes/fake" "k8s.io/utils/clock" testclocks "k8s.io/utils/clock/testing" ctrl "sigs.k8s.io/controller-runtime" kbclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/datapath" datapathmocks "github.com/vmware-tanzu/velero/pkg/datapath/mocks" "github.com/vmware-tanzu/velero/pkg/exposer" "github.com/vmware-tanzu/velero/pkg/metrics" velerotest "github.com/vmware-tanzu/velero/pkg/test" velerotypes "github.com/vmware-tanzu/velero/pkg/types" "github.com/vmware-tanzu/velero/pkg/uploader" "github.com/vmware-tanzu/velero/pkg/util/boolptr" "github.com/vmware-tanzu/velero/pkg/util/kube" ) const dataUploadName = "dataupload-1" const fakeSnapshotType velerov2alpha1api.SnapshotType = "fake-snapshot" type FakeClient struct { kbclient.Client getError error createError error updateError error patchError error updateConflict error listError error } func (c *FakeClient) Get(ctx context.Context, key kbclient.ObjectKey, obj kbclient.Object, opts ...kbclient.GetOption) error { if c.getError != nil { return c.getError } return c.Client.Get(ctx, key, obj) } func (c *FakeClient) Create(ctx context.Context, obj kbclient.Object, opts ...kbclient.CreateOption) error { if c.createError != nil { return c.createError } return c.Client.Create(ctx, obj, opts...) } func (c *FakeClient) Update(ctx context.Context, obj kbclient.Object, opts ...kbclient.UpdateOption) error { if c.updateError != nil { return c.updateError } if c.updateConflict != nil { return c.updateConflict } return c.Client.Update(ctx, obj, opts...) } func (c *FakeClient) Patch(ctx context.Context, obj kbclient.Object, patch kbclient.Patch, opts ...kbclient.PatchOption) error { if c.patchError != nil { return c.patchError } return c.Client.Patch(ctx, obj, patch, opts...) } func (c *FakeClient) List(ctx context.Context, list kbclient.ObjectList, opts ...kbclient.ListOption) error { if c.listError != nil { return c.listError } return c.Client.List(ctx, list, opts...) } func initDataUploaderReconciler(needError ...bool) (*DataUploadReconciler, error) { var errs = make([]error, 6) for k, isError := range needError { if k == 0 && isError { errs[0] = fmt.Errorf("Get error") } else if k == 1 && isError { errs[1] = fmt.Errorf("Create error") } else if k == 2 && isError { errs[2] = fmt.Errorf("Update error") } else if k == 3 && isError { errs[3] = fmt.Errorf("Patch error") } else if k == 4 && isError { errs[4] = apierrors.NewConflict(velerov2alpha1api.Resource("dataupload"), dataUploadName, errors.New("conflict")) } else if k == 5 && isError { errs[5] = fmt.Errorf("List error") } } return initDataUploaderReconcilerWithError(errs...) } func initDataUploaderReconcilerWithError(needError ...error) (*DataUploadReconciler, error) { vscName := "fake-vsc" vsObject := &snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-volume-snapshot", Namespace: "fake-ns", }, Status: &snapshotv1api.VolumeSnapshotStatus{ BoundVolumeSnapshotContentName: &vscName, ReadyToUse: boolptr.True(), RestoreSize: &resource.Quantity{}, }, } var restoreSize int64 vscObj := &snapshotv1api.VolumeSnapshotContent{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-vsc", }, Spec: snapshotv1api.VolumeSnapshotContentSpec{ DeletionPolicy: snapshotv1api.VolumeSnapshotContentDelete, }, Status: &snapshotv1api.VolumeSnapshotContentStatus{ RestoreSize: &restoreSize, }, } daemonSet := &appsv1api.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", Name: "node-agent", }, TypeMeta: metav1.TypeMeta{ Kind: "DaemonSet", APIVersion: appsv1api.SchemeGroupVersion.String(), }, Spec: appsv1api.DaemonSetSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ Containers: []corev1api.Container{ { Image: "fake-image", }, }, }, }, }, } node := builder.ForNode("fake-node").Labels(map[string]string{kube.NodeOSLabel: kube.NodeOSLinux}).Result() dataPathMgr := datapath.NewManager(1) now, err := time.Parse(time.RFC1123, time.RFC1123) if err != nil { return nil, err } now = now.Local() scheme := runtime.NewScheme() err = velerov1api.AddToScheme(scheme) if err != nil { return nil, err } err = velerov2alpha1api.AddToScheme(scheme) if err != nil { return nil, err } err = corev1api.AddToScheme(scheme) if err != nil { return nil, err } fakeClient := &FakeClient{ Client: fake.NewClientBuilder().WithScheme(scheme).Build(), } for k := range needError { if k == 0 { fakeClient.getError = needError[0] } else if k == 1 { fakeClient.createError = needError[1] } else if k == 2 { fakeClient.updateError = needError[2] } else if k == 3 { fakeClient.patchError = needError[3] } else if k == 4 { fakeClient.updateConflict = needError[4] } else if k == 5 { fakeClient.listError = needError[5] } } fakeSnapshotClient := snapshotFake.NewSimpleClientset(vsObject, vscObj) fakeKubeClient := clientgofake.NewSimpleClientset(daemonSet, node) return NewDataUploadReconciler( fakeClient, nil, fakeKubeClient, fakeSnapshotClient.SnapshotV1(), dataPathMgr, nil, nil, map[string]velerotypes.BackupPVC{}, corev1api.ResourceRequirements{}, testclocks.NewFakeClock(now), "test-node", time.Minute*5, velerotest.NewLogger(), metrics.NewServerMetrics(), "", // dataMovePriorityClass nil, // podLabels nil, // podAnnotations ), nil } func dataUploadBuilder() *builder.DataUploadBuilder { csi := &velerov2alpha1api.CSISnapshotSpec{ SnapshotClass: "csi-azuredisk-vsc", StorageClass: "default", VolumeSnapshot: "fake-volume-snapshot", } return builder.ForDataUpload(velerov1api.DefaultNamespace, dataUploadName). BackupStorageLocation("bsl-loc"). DataMover("velero"). SnapshotType("CSI").SourceNamespace("fake-ns").SourcePVC("test-pvc").CSISnapshot(csi) } type fakeSnapshotExposer struct { kubeClient kbclient.Client clock clock.WithTickerAndDelayedExecution ambiguousNodeOS bool peekErr error exposeErr error getErr error getNil bool } func (f *fakeSnapshotExposer) Expose(ctx context.Context, ownerObject corev1api.ObjectReference, param any) error { if f.exposeErr != nil { return f.exposeErr } return nil } func (f *fakeSnapshotExposer) GetExposed(ctx context.Context, du corev1api.ObjectReference, tm time.Duration, para any) (*exposer.ExposeResult, error) { if f.getErr != nil { return nil, f.getErr } if f.getNil { return nil, nil } pod := &corev1api.Pod{} nodeOS := "linux" pNodeOS := &nodeOS if f.ambiguousNodeOS { pNodeOS = nil } return &exposer.ExposeResult{ByPod: exposer.ExposeByPod{HostingPod: pod, VolumeName: dataUploadName, NodeOS: pNodeOS}}, nil } func (f *fakeSnapshotExposer) PeekExposed(ctx context.Context, ownerObject corev1api.ObjectReference) error { return f.peekErr } func (f *fakeSnapshotExposer) DiagnoseExpose(context.Context, corev1api.ObjectReference) string { return "" } func (f *fakeSnapshotExposer) CleanUp(context.Context, corev1api.ObjectReference, string, string) { } type fakeFSBR struct { kubeClient kbclient.Client clock clock.WithTickerAndDelayedExecution initErr error startErr error } func (f *fakeFSBR) Init(ctx context.Context, param any) error { return f.initErr } func (f *fakeFSBR) StartBackup(source datapath.AccessPoint, uploaderConfigs map[string]string, param any) error { return f.startErr } func (f *fakeFSBR) StartRestore(snapshotID string, target datapath.AccessPoint, uploaderConfigs map[string]string) error { return nil } func (b *fakeFSBR) Cancel() { } func (b *fakeFSBR) Close(ctx context.Context) { } func TestReconcile(t *testing.T) { tests := []struct { name string du *velerov2alpha1api.DataUpload notCreateDU bool needDelete bool sportTime *metav1.Time pod *corev1api.Pod pvc *corev1api.PersistentVolumeClaim snapshotExposerList map[velerov2alpha1api.SnapshotType]exposer.SnapshotExposer dataMgr *datapath.Manager needCreateFSBR bool needExclusiveUpdateError error expected *velerov2alpha1api.DataUpload expectDeleted bool expectCancelRecord bool needErrs []bool ambiguousNodeOS bool peekErr error exposeErr error getExposeErr error getExposeNil bool fsBRInitErr error fsBRStartErr error constrained bool expectedErr string expectedResult *ctrl.Result expectDataPath bool }{ { name: "du not found", du: dataUploadBuilder().Result(), notCreateDU: true, }, { name: "du not created in velero default namespace", du: builder.ForDataUpload("test-ns", dataUploadName).Result(), }, { name: "get du fail", du: dataUploadBuilder().Result(), needErrs: []bool{true, false, false, false}, expectedErr: "getting DataUpload: Get error", }, { name: "du is not for built-in dm", du: dataUploadBuilder().DataMover("other").Result(), }, { name: "add finalizer to du", du: dataUploadBuilder().Result(), expected: dataUploadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Result(), }, { name: "add finalizer to du failed", du: dataUploadBuilder().Result(), needErrs: []bool{false, false, true, false}, expectedErr: "error updating dataupload with error velero/dataupload-1: Update error", }, { name: "du is under deletion", du: dataUploadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Result(), needDelete: true, expected: dataUploadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Cancel(true).Result(), }, { name: "du is under deletion but cancel failed", du: dataUploadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Result(), needErrs: []bool{false, false, true, false}, needDelete: true, expectedErr: "error updating dataupload with error velero/dataupload-1: Update error", }, { name: "du is under deletion and in terminal state", du: dataUploadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Phase(velerov2alpha1api.DataUploadPhaseFailed).Result(), sportTime: &metav1.Time{Time: time.Now()}, needDelete: true, expectDeleted: true, }, { name: "du is under deletion and in terminal state, but remove finalizer failed", du: dataUploadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Phase(velerov2alpha1api.DataUploadPhaseFailed).Result(), needErrs: []bool{false, false, true, false}, needDelete: true, expectedErr: "error updating dataupload with error velero/dataupload-1: Update error", }, { name: "delay cancel negative for others", du: dataUploadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Cancel(true).Phase(velerov2alpha1api.DataUploadPhasePrepared).Result(), sportTime: &metav1.Time{Time: time.Now()}, expectCancelRecord: true, }, { name: "delay cancel negative for inProgress", du: dataUploadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Cancel(true).Phase(velerov2alpha1api.DataUploadPhaseInProgress).Result(), sportTime: &metav1.Time{Time: time.Now().Add(-time.Minute * 58)}, expectCancelRecord: true, }, { name: "delay cancel affirmative for others", du: dataUploadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Cancel(true).Phase(velerov2alpha1api.DataUploadPhasePrepared).Result(), sportTime: &metav1.Time{Time: time.Now().Add(-time.Minute * 5)}, expected: dataUploadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Cancel(true).Phase(velerov2alpha1api.DataUploadPhaseCanceled).Result(), }, { name: "delay cancel affirmative for inProgress", du: dataUploadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Cancel(true).Phase(velerov2alpha1api.DataUploadPhaseInProgress).Result(), sportTime: &metav1.Time{Time: time.Now().Add(-time.Hour)}, expected: dataUploadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Cancel(true).Phase(velerov2alpha1api.DataUploadPhaseCanceled).Result(), }, { name: "delay cancel failed", du: dataUploadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Cancel(true).Phase(velerov2alpha1api.DataUploadPhaseInProgress).Result(), needErrs: []bool{false, false, true, false}, sportTime: &metav1.Time{Time: time.Now().Add(-time.Hour)}, expected: dataUploadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Cancel(true).Phase(velerov2alpha1api.DataUploadPhaseInProgress).Result(), expectCancelRecord: true, }, { name: "Unknown data upload status", du: dataUploadBuilder().Phase("Unknown").Finalizers([]string{DataUploadDownloadFinalizer}).Result(), }, { name: "Unknown type of snapshot exposer is not initialized", du: dataUploadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).SnapshotType("unknown type").Result(), expected: dataUploadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Phase(velerov2alpha1api.DataUploadPhaseFailed).Result(), expectedErr: "unknown type type of snapshot exposer is not exist", }, { name: "du is cancel on new", du: dataUploadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Cancel(true).Result(), expectCancelRecord: true, expected: dataUploadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Cancel(true).Phase(velerov2alpha1api.DataUploadPhaseCanceled).Result(), }, { name: "new du but constrained", du: dataUploadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Result(), constrained: true, expected: dataUploadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Result(), expectedResult: &ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, }, { name: "new du but accept failed", du: dataUploadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Result(), needExclusiveUpdateError: errors.New("exclusive-update-error"), expected: dataUploadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Result(), expectedErr: "error accepting the data upload dataupload-1: exclusive-update-error", }, { name: "du is accepted but setup expose param failed on getting PVC", du: dataUploadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Result(), expected: dataUploadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Phase(velerov2alpha1api.DataUploadPhaseFailed).Message("failed to set exposer parameters").Result(), expectedErr: "failed to get PVC fake-ns/test-pvc: persistentvolumeclaims \"test-pvc\" not found", }, { name: "du expose failed", du: dataUploadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).SnapshotType(fakeSnapshotType).Result(), pvc: builder.ForPersistentVolumeClaim("fake-ns", "test-pvc").Result(), exposeErr: errors.New("fake-expose-error"), expected: dataUploadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Phase(velerov2alpha1api.DataUploadPhaseFailed).Message("error exposing snapshot").Result(), expectedErr: "fake-expose-error", }, { name: "du succeeds for accepted", du: dataUploadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).SnapshotType(fakeSnapshotType).Result(), pvc: builder.ForPersistentVolumeClaim("fake-ns", "test-pvc").Result(), expected: dataUploadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Phase(velerov2alpha1api.DataUploadPhaseAccepted).Result(), }, { name: "prepare timeout on accepted", du: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseAccepted).Finalizers([]string{DataUploadDownloadFinalizer}).AcceptedTimestamp(&metav1.Time{Time: time.Now().Add(-time.Minute * 30)}).Result(), expected: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseFailed).Finalizers([]string{DataUploadDownloadFinalizer}).Phase(velerov2alpha1api.DataUploadPhaseFailed).Message("timeout on preparing data upload").Result(), }, { name: "peek error on accepted", du: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseAccepted).SnapshotType(fakeSnapshotType).Finalizers([]string{DataUploadDownloadFinalizer}).Result(), peekErr: errors.New("fake-peak-error"), expected: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseCanceled).Finalizers([]string{DataUploadDownloadFinalizer}).Phase(velerov2alpha1api.DataUploadPhaseCanceled).Message("found a du velero/dataupload-1 with expose error: fake-peak-error. mark it as cancel").Result(), }, { name: "cancel on prepared", du: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhasePrepared).Finalizers([]string{DataUploadDownloadFinalizer}).Node("test-node").Cancel(true).Result(), expected: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseCanceled).Finalizers([]string{DataUploadDownloadFinalizer}).Cancel(true).Phase(velerov2alpha1api.DataUploadPhaseCanceled).Result(), }, { name: "Failed to get snapshot expose on prepared", du: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhasePrepared).SnapshotType(fakeSnapshotType).Finalizers([]string{DataUploadDownloadFinalizer}).Node("test-node").Result(), getExposeErr: errors.New("fake-get-error"), expected: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseFailed).Finalizers([]string{DataUploadDownloadFinalizer}).Message("exposed snapshot is not ready: fake-get-error").Result(), expectedErr: "fake-get-error", }, { name: "Get nil restore expose on prepared", du: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhasePrepared).SnapshotType(fakeSnapshotType).Finalizers([]string{DataUploadDownloadFinalizer}).Node("test-node").Result(), getExposeNil: true, expected: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseFailed).Finalizers([]string{DataUploadDownloadFinalizer}).Message("exposed snapshot is not ready").Result(), expectedErr: "no expose result is available for the current node", }, { name: "Dataupload should fail if expose returns ambiguous nodeOS", du: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhasePrepared).SnapshotType(fakeSnapshotType).Finalizers([]string{DataUploadDownloadFinalizer}).Node("test-node").Result(), ambiguousNodeOS: true, expected: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseFailed).Finalizers([]string{DataUploadDownloadFinalizer}).Result(), expectedErr: "unsupported ambiguous node OS", }, { name: "Error in data path is concurrent limited", du: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhasePrepared).SnapshotType(fakeSnapshotType).Finalizers([]string{DataUploadDownloadFinalizer}).Node("test-node").Result(), dataMgr: datapath.NewManager(0), expectedResult: &ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, }, { name: "data path init error", du: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhasePrepared).SnapshotType(fakeSnapshotType).Finalizers([]string{DataUploadDownloadFinalizer}).Node("test-node").Result(), fsBRInitErr: errors.New("fake-data-path-init-error"), expected: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseFailed).Finalizers([]string{DataUploadDownloadFinalizer}).Message("error initializing data path").Result(), expectedErr: "error initializing asyncBR: fake-data-path-init-error", }, { name: "Unable to update status to in progress for data upload", du: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhasePrepared).SnapshotType(fakeSnapshotType).Finalizers([]string{DataUploadDownloadFinalizer}).Node("test-node").Result(), needErrs: []bool{false, false, true, false}, expected: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhasePrepared).Finalizers([]string{DataUploadDownloadFinalizer}).Result(), expectedResult: &ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, }, { name: "data path start error", du: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhasePrepared).SnapshotType(fakeSnapshotType).Finalizers([]string{DataUploadDownloadFinalizer}).Node("test-node").Result(), fsBRStartErr: errors.New("fake-data-path-start-error"), expected: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseFailed).Finalizers([]string{DataUploadDownloadFinalizer}).Message("error starting data path").Result(), expectedErr: "error starting async backup for pod , volume dataupload-1: fake-data-path-start-error", }, { name: "Prepare succeeds", du: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhasePrepared).SnapshotType(fakeSnapshotType).Finalizers([]string{DataUploadDownloadFinalizer}).Node("test-node").Result(), expected: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseInProgress).Finalizers([]string{DataUploadDownloadFinalizer}).Result(), expectDataPath: true, }, { name: "In progress du is not handled by the current node", du: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseInProgress).Finalizers([]string{DataUploadDownloadFinalizer}).Result(), expected: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseInProgress).Finalizers([]string{DataUploadDownloadFinalizer}).Result(), }, { name: "In progress du is not set as cancel", du: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseInProgress).Finalizers([]string{DataUploadDownloadFinalizer}).Node("test-node").Result(), expected: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseInProgress).Finalizers([]string{DataUploadDownloadFinalizer}).Result(), }, { name: "Cancel data upload in progress with empty FSBR", du: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseInProgress).Cancel(true).Finalizers([]string{DataUploadDownloadFinalizer}).Node("test-node").Result(), expected: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseCanceled).Cancel(true).Finalizers([]string{DataUploadDownloadFinalizer}).Result(), }, { name: "Cancel data upload in progress and patch data upload error", du: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseInProgress).Cancel(true).Finalizers([]string{DataUploadDownloadFinalizer}).Node("test-node").Result(), needErrs: []bool{false, false, true, false}, needCreateFSBR: true, expected: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseInProgress).Cancel(true).Finalizers([]string{DataUploadDownloadFinalizer}).Result(), expectedErr: "error updating dataupload with error velero/dataupload-1: Update error", expectCancelRecord: true, expectDataPath: true, }, { name: "Cancel data upload in progress succeeds", du: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseInProgress).Cancel(true).Finalizers([]string{DataUploadDownloadFinalizer}).Node("test-node").Result(), needCreateFSBR: true, expected: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseCanceling).Cancel(true).Finalizers([]string{DataUploadDownloadFinalizer}).Result(), expectDataPath: true, expectCancelRecord: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { r, err := initDataUploaderReconciler(test.needErrs...) require.NoError(t, err) if !test.notCreateDU { err = r.client.Create(t.Context(), test.du) require.NoError(t, err) } if test.needDelete { err = r.client.Delete(t.Context(), test.du) require.NoError(t, err) } if test.pod != nil { err = r.client.Create(ctx, test.pod) require.NoError(t, err) } if test.pvc != nil { err = r.client.Create(ctx, test.pvc) require.NoError(t, err) } if test.dataMgr != nil { r.dataPathMgr = test.dataMgr } else { r.dataPathMgr = datapath.NewManager(1) } if test.sportTime != nil { r.cancelledDataUpload[test.du.Name] = test.sportTime.Time } if test.constrained { r.vgdpCounter = &exposer.VgdpCounter{} } if test.du.Spec.SnapshotType == fakeSnapshotType { r.snapshotExposerList = map[velerov2alpha1api.SnapshotType]exposer.SnapshotExposer{fakeSnapshotType: &fakeSnapshotExposer{r.client, r.Clock, test.ambiguousNodeOS, test.peekErr, test.exposeErr, test.getExposeErr, test.getExposeNil}} } else if test.du.Spec.SnapshotType == velerov2alpha1api.SnapshotTypeCSI { r.snapshotExposerList = map[velerov2alpha1api.SnapshotType]exposer.SnapshotExposer{velerov2alpha1api.SnapshotTypeCSI: exposer.NewCSISnapshotExposer(r.kubeClient, r.csiSnapshotClient, velerotest.NewLogger())} } funcExclusiveUpdateDataUpload = exclusiveUpdateDataUpload if test.needExclusiveUpdateError != nil { funcExclusiveUpdateDataUpload = func(context.Context, kbclient.Client, *velerov2alpha1api.DataUpload, func(*velerov2alpha1api.DataUpload)) (bool, error) { return false, test.needExclusiveUpdateError } } datapath.MicroServiceBRWatcherCreator = func(kbclient.Client, kubernetes.Interface, manager.Manager, string, string, string, string, string, string, datapath.Callbacks, logrus.FieldLogger) datapath.AsyncBR { return &fakeFSBR{ kubeClient: r.client, clock: r.Clock, initErr: test.fsBRInitErr, startErr: test.fsBRStartErr, } } if test.needCreateFSBR { if fsBR := r.dataPathMgr.GetAsyncBR(test.du.Name); fsBR == nil { _, err := r.dataPathMgr.CreateMicroServiceBRWatcher(ctx, r.client, nil, nil, datapath.TaskTypeBackup, test.du.Name, velerov1api.DefaultNamespace, "", "", "", datapath.Callbacks{OnCancelled: r.OnDataUploadCancelled}, false, velerotest.NewLogger()) require.NoError(t, err) } } actualResult, err := r.Reconcile(ctx, ctrl.Request{ NamespacedName: types.NamespacedName{ Namespace: velerov1api.DefaultNamespace, Name: test.du.Name, }, }) if test.expectedErr != "" { require.EqualError(t, err, test.expectedErr) } else { require.NoError(t, err) } if test.expectedResult != nil { assert.Equal(t, test.expectedResult.Requeue, actualResult.Requeue) assert.Equal(t, test.expectedResult.RequeueAfter, actualResult.RequeueAfter) } du := velerov2alpha1api.DataUpload{} err = r.client.Get(ctx, kbclient.ObjectKey{ Name: test.du.Name, Namespace: test.du.Namespace, }, &du) if test.expected != nil || test.expectDeleted { if test.expectDeleted { assert.True(t, apierrors.IsNotFound(err)) } else { require.NoError(t, err) assert.Equal(t, test.expected.Status.Phase, du.Status.Phase) assert.Contains(t, du.Status.Message, test.expected.Status.Message) assert.Equal(t, du.Finalizers, test.expected.Finalizers) assert.Equal(t, du.Spec.Cancel, test.expected.Spec.Cancel) } } if !test.expectDataPath { assert.Nil(t, r.dataPathMgr.GetAsyncBR(test.du.Name)) } else { assert.NotNil(t, r.dataPathMgr.GetAsyncBR(test.du.Name)) } if test.expectCancelRecord { assert.Contains(t, r.cancelledDataUpload, test.du.Name) } else { assert.Empty(t, r.cancelledDataUpload) } if isDataUploadInFinalState(&du) || du.Status.Phase == velerov2alpha1api.DataUploadPhaseInProgress { assert.NotContains(t, du.Labels, exposer.ExposeOnGoingLabel) } else if du.Status.Phase == velerov2alpha1api.DataUploadPhaseAccepted { assert.Contains(t, du.Labels, exposer.ExposeOnGoingLabel) } }) } } func TestOnDataUploadCancelled(t *testing.T) { ctx := t.Context() r, err := initDataUploaderReconciler() require.NoError(t, err) // Create a DataUpload object du := dataUploadBuilder().Result() namespace := du.Namespace duName := du.Name // Add the DataUpload object to the fake client require.NoError(t, r.client.Create(ctx, du)) r.OnDataUploadCancelled(ctx, namespace, duName) updatedDu := &velerov2alpha1api.DataUpload{} require.NoError(t, r.client.Get(ctx, types.NamespacedName{Name: duName, Namespace: namespace}, updatedDu)) assert.Equal(t, velerov2alpha1api.DataUploadPhaseCanceled, updatedDu.Status.Phase) assert.False(t, updatedDu.Status.CompletionTimestamp.IsZero()) assert.False(t, updatedDu.Status.StartTimestamp.IsZero()) } func TestOnDataUploadProgress(t *testing.T) { totalBytes := int64(1024) bytesDone := int64(512) tests := []struct { name string du *velerov2alpha1api.DataUpload progress uploader.Progress needErrs []bool }{ { name: "patch in progress phase success", du: dataUploadBuilder().Result(), progress: uploader.Progress{ TotalBytes: totalBytes, BytesDone: bytesDone, }, }, { name: "failed to get dataupload", du: dataUploadBuilder().Result(), needErrs: []bool{true, false, false, false}, }, { name: "failed to patch dataupload", du: dataUploadBuilder().Result(), needErrs: []bool{false, false, true, false}, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { ctx := t.Context() r, err := initDataUploaderReconciler(test.needErrs...) require.NoError(t, err) defer func() { r.client.Delete(ctx, test.du, &kbclient.DeleteOptions{}) }() // Create a DataUpload object du := dataUploadBuilder().Result() namespace := du.Namespace duName := du.Name // Add the DataUpload object to the fake client require.NoError(t, r.client.Create(t.Context(), du)) // Create a Progress object progress := &uploader.Progress{ TotalBytes: totalBytes, BytesDone: bytesDone, } // Call the OnDataUploadProgress function r.OnDataUploadProgress(ctx, namespace, duName, progress) if len(test.needErrs) != 0 && !test.needErrs[0] { // Get the updated DataUpload object from the fake client updatedDu := &velerov2alpha1api.DataUpload{} require.NoError(t, r.client.Get(ctx, types.NamespacedName{Name: duName, Namespace: namespace}, updatedDu)) // Assert that the DataUpload object has been updated with the progress assert.Equal(t, test.progress.TotalBytes, updatedDu.Status.Progress.TotalBytes) assert.Equal(t, test.progress.BytesDone, updatedDu.Status.Progress.BytesDone) } }) } } func TestOnDataUploadFailed(t *testing.T) { ctx := t.Context() r, err := initDataUploaderReconciler() require.NoError(t, err) // Create a DataUpload object du := dataUploadBuilder().Result() namespace := du.Namespace duName := du.Name // Add the DataUpload object to the fake client require.NoError(t, r.client.Create(ctx, du)) r.snapshotExposerList = map[velerov2alpha1api.SnapshotType]exposer.SnapshotExposer{velerov2alpha1api.SnapshotTypeCSI: exposer.NewCSISnapshotExposer(r.kubeClient, r.csiSnapshotClient, velerotest.NewLogger())} r.OnDataUploadFailed(ctx, namespace, duName, fmt.Errorf("Failed to handle %v", duName)) updatedDu := &velerov2alpha1api.DataUpload{} require.NoError(t, r.client.Get(ctx, types.NamespacedName{Name: duName, Namespace: namespace}, updatedDu)) assert.Equal(t, velerov2alpha1api.DataUploadPhaseFailed, updatedDu.Status.Phase) assert.False(t, updatedDu.Status.CompletionTimestamp.IsZero()) assert.False(t, updatedDu.Status.StartTimestamp.IsZero()) } func TestOnDataUploadCompleted(t *testing.T) { ctx := t.Context() r, err := initDataUploaderReconciler() require.NoError(t, err) // Create a DataUpload object du := dataUploadBuilder().Result() namespace := du.Namespace duName := du.Name // Add the DataUpload object to the fake client require.NoError(t, r.client.Create(ctx, du)) r.snapshotExposerList = map[velerov2alpha1api.SnapshotType]exposer.SnapshotExposer{velerov2alpha1api.SnapshotTypeCSI: exposer.NewCSISnapshotExposer(r.kubeClient, r.csiSnapshotClient, velerotest.NewLogger())} r.OnDataUploadCompleted(ctx, namespace, duName, datapath.Result{ Backup: datapath.BackupResult{ SnapshotID: "fake-id", Source: datapath.AccessPoint{ ByPath: "fake-path", }, }, }) updatedDu := &velerov2alpha1api.DataUpload{} require.NoError(t, r.client.Get(ctx, types.NamespacedName{Name: duName, Namespace: namespace}, updatedDu)) assert.Equal(t, velerov2alpha1api.DataUploadPhaseCompleted, updatedDu.Status.Phase) assert.False(t, updatedDu.Status.CompletionTimestamp.IsZero()) assert.Equal(t, "fake-id", updatedDu.Status.SnapshotID) assert.Equal(t, "fake-path", updatedDu.Status.Path) } func TestFindDataUploadForPod(t *testing.T) { r, err := initDataUploaderReconciler() require.NoError(t, err) tests := []struct { name string du *velerov2alpha1api.DataUpload pod *corev1api.Pod checkFunc func(*velerov2alpha1api.DataUpload, []reconcile.Request) }{ { name: "find dataUpload for pod", du: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseAccepted).Result(), pod: builder.ForPod(velerov1api.DefaultNamespace, dataUploadName).Labels(map[string]string{velerov1api.DataUploadLabel: dataUploadName}).Status(corev1api.PodStatus{Phase: corev1api.PodRunning}).Result(), checkFunc: func(du *velerov2alpha1api.DataUpload, requests []reconcile.Request) { // Assert that the function returns a single request assert.Len(t, requests, 1) // Assert that the request contains the correct namespaced name assert.Equal(t, du.Namespace, requests[0].Namespace) assert.Equal(t, du.Name, requests[0].Name) }, }, { name: "no selected label found for pod", du: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseAccepted).Result(), pod: builder.ForPod(velerov1api.DefaultNamespace, dataUploadName).Result(), checkFunc: func(du *velerov2alpha1api.DataUpload, requests []reconcile.Request) { // Assert that the function returns a single request assert.Empty(t, requests) }, }, { name: "no matched pod", du: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseAccepted).Result(), pod: builder.ForPod(velerov1api.DefaultNamespace, dataUploadName).Labels(map[string]string{velerov1api.DataUploadLabel: "non-existing-dataupload"}).Result(), checkFunc: func(du *velerov2alpha1api.DataUpload, requests []reconcile.Request) { assert.Empty(t, requests) }, }, { name: "dataUpload not accepte", du: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseInProgress).Result(), pod: builder.ForPod(velerov1api.DefaultNamespace, dataUploadName).Labels(map[string]string{velerov1api.DataUploadLabel: dataUploadName}).Result(), checkFunc: func(du *velerov2alpha1api.DataUpload, requests []reconcile.Request) { assert.Empty(t, requests) }, }, } for _, test := range tests { ctx := t.Context() assert.NoError(t, r.client.Create(ctx, test.pod)) assert.NoError(t, r.client.Create(ctx, test.du)) // Call the findDataUploadForPod function requests := r.findDataUploadForPod(t.Context(), test.pod) test.checkFunc(test.du, requests) r.client.Delete(ctx, test.du, &kbclient.DeleteOptions{}) if test.pod != nil { r.client.Delete(ctx, test.pod, &kbclient.DeleteOptions{}) } } } type fakeAPIStatus struct { reason metav1.StatusReason } func (f *fakeAPIStatus) Status() metav1.Status { return metav1.Status{ Reason: f.reason, } } func (f *fakeAPIStatus) Error() string { return string(f.reason) } func TestAcceptDataUpload(t *testing.T) { tests := []struct { name string du *velerov2alpha1api.DataUpload needErrs []error succeeded bool expectedErr string }{ { name: "update fail", du: dataUploadBuilder().Result(), needErrs: []error{nil, nil, fmt.Errorf("fake-update-error"), nil}, expectedErr: "fake-update-error", }, { name: "accepted by others", du: dataUploadBuilder().Result(), needErrs: []error{nil, nil, &fakeAPIStatus{metav1.StatusReasonConflict}, nil}, }, { name: "succeed", du: dataUploadBuilder().Result(), needErrs: []error{nil, nil, nil, nil}, succeeded: true, }, } for _, test := range tests { ctx := t.Context() r, err := initDataUploaderReconcilerWithError(test.needErrs...) require.NoError(t, err) err = r.client.Create(ctx, test.du) require.NoError(t, err) succeeded, err := r.acceptDataUpload(ctx, test.du) assert.Equal(t, test.succeeded, succeeded) if test.expectedErr == "" { assert.NoError(t, err) } else { assert.EqualError(t, err, test.expectedErr) } } } func TestOnDuPrepareTimeout(t *testing.T) { tests := []struct { name string du *velerov2alpha1api.DataUpload needErrs []error expected *velerov2alpha1api.DataUpload }{ { name: "update fail", du: dataUploadBuilder().Result(), needErrs: []error{nil, nil, fmt.Errorf("fake-update-error"), nil}, expected: dataUploadBuilder().Result(), }, { name: "update interrupted", du: dataUploadBuilder().Result(), needErrs: []error{nil, nil, &fakeAPIStatus{metav1.StatusReasonConflict}, nil}, expected: dataUploadBuilder().Result(), }, { name: "succeed", du: dataUploadBuilder().Result(), needErrs: []error{nil, nil, nil, nil}, expected: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseFailed).Result(), }, } for _, test := range tests { ctx := t.Context() r, err := initDataUploaderReconcilerWithError(test.needErrs...) require.NoError(t, err) err = r.client.Create(ctx, test.du) require.NoError(t, err) r.onPrepareTimeout(ctx, test.du) du := velerov2alpha1api.DataUpload{} _ = r.client.Get(ctx, kbclient.ObjectKey{ Name: test.du.Name, Namespace: test.du.Namespace, }, &du) assert.Equal(t, test.expected.Status.Phase, du.Status.Phase) } } func TestTryCancelDataUpload(t *testing.T) { tests := []struct { name string dd *velerov2alpha1api.DataUpload needErrs []error succeeded bool expectedErr string }{ { name: "update fail", dd: dataUploadBuilder().Result(), needErrs: []error{nil, nil, fmt.Errorf("fake-update-error"), nil}, }, { name: "cancel by others", dd: dataUploadBuilder().Result(), needErrs: []error{nil, nil, &fakeAPIStatus{metav1.StatusReasonConflict}, nil}, }, { name: "succeed", dd: dataUploadBuilder().Result(), needErrs: []error{nil, nil, nil, nil}, succeeded: true, }, } for _, test := range tests { ctx := t.Context() r, err := initDataUploaderReconcilerWithError(test.needErrs...) require.NoError(t, err) err = r.client.Create(ctx, test.dd) require.NoError(t, err) r.tryCancelDataUpload(ctx, test.dd, "") if test.expectedErr == "" { assert.NoError(t, err) } else { assert.EqualError(t, err, test.expectedErr) } } } func TestUpdateDataUploadWithRetry(t *testing.T) { namespacedName := types.NamespacedName{ Name: dataUploadName, Namespace: "velero", } // Define test cases testCases := []struct { Name string needErrs []bool noChange bool ExpectErr bool }{ { Name: "SuccessOnFirstAttempt", }, { Name: "Error get", needErrs: []bool{true, false, false, false, false}, ExpectErr: true, }, { Name: "Error update", needErrs: []bool{false, false, true, false, false}, ExpectErr: true, }, { Name: "no change", noChange: true, needErrs: []bool{false, false, true, false, false}, }, { Name: "Conflict with error timeout", needErrs: []bool{false, false, false, false, true}, ExpectErr: true, }, } for _, tc := range testCases { t.Run(tc.Name, func(t *testing.T) { ctx, cancelFunc := context.WithTimeout(t.Context(), time.Second*5) defer cancelFunc() r, err := initDataUploaderReconciler(tc.needErrs...) require.NoError(t, err) err = r.client.Create(ctx, dataUploadBuilder().Result()) require.NoError(t, err) updateFunc := func(dataDownload *velerov2alpha1api.DataUpload) bool { if tc.noChange { return false } dataDownload.Spec.Cancel = true return true } err = UpdateDataUploadWithRetry(ctx, r.client, namespacedName, velerotest.NewLogger().WithField("name", tc.Name), updateFunc) if tc.ExpectErr { assert.Error(t, err) } else { assert.NoError(t, err) } }) } } type duResumeTestHelper struct { resumeErr error getExposeErr error exposeResult *exposer.ExposeResult asyncBR datapath.AsyncBR } func (dt *duResumeTestHelper) resumeCancellableDataPath(_ *DataUploadReconciler, _ context.Context, _ *velerov2alpha1api.DataUpload, _ logrus.FieldLogger) error { return dt.resumeErr } func (dt *duResumeTestHelper) Expose(context.Context, corev1api.ObjectReference, any) error { return nil } func (dt *duResumeTestHelper) GetExposed(context.Context, corev1api.ObjectReference, time.Duration, any) (*exposer.ExposeResult, error) { return dt.exposeResult, dt.getExposeErr } func (dt *duResumeTestHelper) PeekExposed(context.Context, corev1api.ObjectReference) error { return nil } func (dt *duResumeTestHelper) DiagnoseExpose(context.Context, corev1api.ObjectReference) string { return "" } func (dt *duResumeTestHelper) CleanUp(context.Context, corev1api.ObjectReference, string, string) {} func (dt *duResumeTestHelper) newMicroServiceBRWatcher(kbclient.Client, kubernetes.Interface, manager.Manager, string, string, string, string, string, string, datapath.Callbacks, logrus.FieldLogger) datapath.AsyncBR { return dt.asyncBR } func TestAttemptDataUploadResume(t *testing.T) { tests := []struct { name string dataUploads []velerov2alpha1api.DataUpload du *velerov2alpha1api.DataUpload needErrs []bool acceptedDataUploads []string prepareddDataUploads []string cancelledDataUploads []string inProgressDataUploads []string resumeErr error expectedError string }{ { name: "Other DataUpload", du: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhasePrepared).Result(), }, { name: "InProgress DataUpload, not the current node", du: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseInProgress).Result(), inProgressDataUploads: []string{dataUploadName}, }, { name: "InProgress DataUpload, resume error and update error", du: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseInProgress).Node("node-1").Result(), needErrs: []bool{false, false, true, false, false, false}, resumeErr: errors.New("fake-resume-error"), inProgressDataUploads: []string{dataUploadName}, }, { name: "InProgress DataUpload, resume error and update succeed", du: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseInProgress).Node("node-1").Result(), resumeErr: errors.New("fake-resume-error"), cancelledDataUploads: []string{dataUploadName}, inProgressDataUploads: []string{dataUploadName}, }, { name: "InProgress DataUpload and resume succeed", du: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseInProgress).Node("node-1").Result(), inProgressDataUploads: []string{dataUploadName}, }, { name: "Error", needErrs: []bool{false, false, false, false, false, true}, du: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhasePrepared).Result(), expectedError: "error to list datauploads: List error", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { ctx := t.Context() r, err := initDataUploaderReconciler(test.needErrs...) r.nodeName = "node-1" require.NoError(t, err) assert.NoError(t, r.client.Create(ctx, test.du)) dt := &duResumeTestHelper{ resumeErr: test.resumeErr, } funcResumeCancellableDataBackup = dt.resumeCancellableDataPath // Run the test err = r.AttemptDataUploadResume(ctx, r.logger.WithField("name", test.name), test.du.Namespace) if test.expectedError != "" { assert.EqualError(t, err, test.expectedError) } else { assert.NoError(t, err) // Verify DataUploads marked as Canceled for _, duName := range test.cancelledDataUploads { dataUpload := &velerov2alpha1api.DataUpload{} err := r.client.Get(t.Context(), types.NamespacedName{Namespace: "velero", Name: duName}, dataUpload) require.NoError(t, err) assert.True(t, dataUpload.Spec.Cancel) } // Verify DataUploads marked as Accepted for _, duName := range test.acceptedDataUploads { dataUpload := &velerov2alpha1api.DataUpload{} err := r.client.Get(t.Context(), types.NamespacedName{Namespace: "velero", Name: duName}, dataUpload) require.NoError(t, err) assert.Equal(t, velerov2alpha1api.DataUploadPhaseAccepted, dataUpload.Status.Phase) } // Verify DataUploads marked as Prepared for _, duName := range test.prepareddDataUploads { dataUpload := &velerov2alpha1api.DataUpload{} err := r.client.Get(t.Context(), types.NamespacedName{Namespace: "velero", Name: duName}, dataUpload) require.NoError(t, err) assert.Equal(t, velerov2alpha1api.DataUploadPhasePrepared, dataUpload.Status.Phase) } // Verify DataUploads marked as InProgress for _, duName := range test.inProgressDataUploads { dataUpload := &velerov2alpha1api.DataUpload{} err := r.client.Get(t.Context(), types.NamespacedName{Namespace: "velero", Name: duName}, dataUpload) require.NoError(t, err) assert.Equal(t, velerov2alpha1api.DataUploadPhaseInProgress, dataUpload.Status.Phase) } } }) } } func TestResumeCancellableBackup(t *testing.T) { tests := []struct { name string dataUploads []velerov2alpha1api.DataUpload du *velerov2alpha1api.DataUpload getExposeErr error exposeResult *exposer.ExposeResult createWatcherErr error initWatcherErr error startWatcherErr error mockInit bool mockStart bool mockClose bool expectedError string }{ { name: "not find exposer", du: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseInProgress).SnapshotType("").Result(), expectedError: fmt.Sprintf("error to find exposer for du %s", dataUploadName), }, { name: "get expose failed", du: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseInProgress).SnapshotType(velerov2alpha1api.SnapshotTypeCSI).Result(), getExposeErr: errors.New("fake-expose-error"), expectedError: fmt.Sprintf("error to get exposed snapshot for du %s: fake-expose-error", dataUploadName), }, { name: "no expose", du: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseAccepted).Node("node-1").Result(), expectedError: fmt.Sprintf("expose info missed for du %s", dataUploadName), }, { name: "watcher init error", du: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseAccepted).Node("node-1").Result(), exposeResult: &exposer.ExposeResult{ ByPod: exposer.ExposeByPod{ HostingPod: &corev1api.Pod{}, }, }, mockInit: true, mockClose: true, initWatcherErr: errors.New("fake-init-watcher-error"), expectedError: fmt.Sprintf("error to init asyncBR watcher for du %s: fake-init-watcher-error", dataUploadName), }, { name: "start watcher error", du: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseAccepted).Node("node-1").Result(), exposeResult: &exposer.ExposeResult{ ByPod: exposer.ExposeByPod{ HostingPod: &corev1api.Pod{}, }, }, mockInit: true, mockStart: true, mockClose: true, startWatcherErr: errors.New("fake-start-watcher-error"), expectedError: fmt.Sprintf("error to resume asyncBR watcher for du %s: fake-start-watcher-error", dataUploadName), }, { name: "succeed", du: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseAccepted).Node("node-1").Result(), exposeResult: &exposer.ExposeResult{ ByPod: exposer.ExposeByPod{ HostingPod: &corev1api.Pod{}, }, }, mockInit: true, mockStart: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { ctx := t.Context() r, err := initDataUploaderReconciler() r.nodeName = "node-1" require.NoError(t, err) mockAsyncBR := datapathmocks.NewAsyncBR(t) if test.mockInit { mockAsyncBR.On("Init", mock.Anything, mock.Anything).Return(test.initWatcherErr) } if test.mockStart { mockAsyncBR.On("StartBackup", mock.Anything, mock.Anything, mock.Anything).Return(test.startWatcherErr) } if test.mockClose { mockAsyncBR.On("Close", mock.Anything).Return() } dt := &duResumeTestHelper{ getExposeErr: test.getExposeErr, exposeResult: test.exposeResult, asyncBR: mockAsyncBR, } r.snapshotExposerList[velerov2alpha1api.SnapshotTypeCSI] = dt datapath.MicroServiceBRWatcherCreator = dt.newMicroServiceBRWatcher err = r.resumeCancellableDataPath(ctx, test.du, velerotest.NewLogger()) if test.expectedError != "" { assert.EqualError(t, err, test.expectedError) } }) } } func TestDataUploadSetupExposeParam(t *testing.T) { // Common objects for all cases fileMode := corev1api.PersistentVolumeFilesystem node := builder.ForNode("worker-1").Labels(map[string]string{kube.NodeOSLabel: kube.NodeOSLinux}).Result() pvc := &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: "app-ns", Name: "test-pvc", }, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "test-pv", VolumeMode: &fileMode, Resources: corev1api.VolumeResourceRequirements{ Requests: corev1api.ResourceList{ corev1api.ResourceStorage: resource.MustParse("10Gi"), }, }, }, } pv := &corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "test-pv", }, } baseDataUpload := dataUploadBuilder().Result() baseDataUpload.Spec.SourceNamespace = "app-ns" baseDataUpload.Spec.SourcePVC = "test-pvc" baseDataUpload.Namespace = velerov1api.DefaultNamespace baseDataUpload.Spec.OperationTimeout = metav1.Duration{Duration: time.Minute * 10} type args struct { customLabels map[string]string customAnnotations map[string]string } type want struct { labels map[string]string annotations map[string]string } tests := []struct { name string args args want want }{ { name: "label has customize values", args: args{ customLabels: map[string]string{"custom-label": "label-value"}, customAnnotations: nil, }, want: want{ labels: map[string]string{ velerov1api.DataUploadLabel: baseDataUpload.Name, "custom-label": "label-value", }, annotations: map[string]string{}, }, }, { name: "label has no customize values", args: args{ customLabels: nil, customAnnotations: nil, }, want: want{ labels: map[string]string{velerov1api.DataUploadLabel: baseDataUpload.Name}, annotations: map[string]string{}, }, }, { name: "annotation has customize values", args: args{ customLabels: nil, customAnnotations: map[string]string{"custom-annotation": "annotation-value"}, }, want: want{ labels: map[string]string{velerov1api.DataUploadLabel: baseDataUpload.Name}, annotations: map[string]string{"custom-annotation": "annotation-value"}, }, }, { name: "both label and annotation have customize values", args: args{ customLabels: map[string]string{"custom-label": "label-value"}, customAnnotations: map[string]string{"custom-annotation": "annotation-value"}, }, want: want{ labels: map[string]string{ velerov1api.DataUploadLabel: baseDataUpload.Name, "custom-label": "label-value", }, annotations: map[string]string{"custom-annotation": "annotation-value"}, }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { // Fake clients per case fakeCRClient := velerotest.NewFakeControllerRuntimeClient(t, pvc, pv, node, baseDataUpload.DeepCopy()) fakeKubeClient := clientgofake.NewSimpleClientset(node) // Reconciler config per case preparingTimeout := time.Minute * 3 podRes := corev1api.ResourceRequirements{} r := NewDataUploadReconciler( fakeCRClient, nil, fakeKubeClient, nil, // snapshotClient (unused in setupExposeParam) datapath.NewManager(1), nil, // dataPathMgr nil, // exposer (unused in setupExposeParam) map[string]velerotypes.BackupPVC{}, podRes, testclocks.NewFakeClock(time.Now()), "test-node", preparingTimeout, velerotest.NewLogger(), metrics.NewServerMetrics(), "upload-priority", tt.args.customLabels, tt.args.customAnnotations, ) // Act got, err := r.setupExposeParam(baseDataUpload) // Assert no error require.NoError(t, err) require.NotNil(t, got) // Type assertion to CSISnapshotExposeParam csiParam, ok := got.(*exposer.CSISnapshotExposeParam) require.True(t, ok, "expected CSISnapshotExposeParam type") // Labels and Annotations assert.Equal(t, tt.want.labels, csiParam.HostingPodLabels) assert.Equal(t, tt.want.annotations, csiParam.HostingPodAnnotations) }) } } ================================================ FILE: pkg/controller/download_request_controller.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package controller import ( "context" "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clocks "k8s.io/utils/clock" ctrl "sigs.k8s.io/controller-runtime" kbclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/predicate" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/constant" "github.com/vmware-tanzu/velero/pkg/itemoperationmap" "github.com/vmware-tanzu/velero/pkg/persistence" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" "github.com/vmware-tanzu/velero/pkg/util/kube" ) const ( defaultDownloadRequestSyncPeriod = time.Minute ) // downloadRequestReconciler reconciles a DownloadRequest object type downloadRequestReconciler struct { client kbclient.Client clock clocks.Clock // use variables to refer to these functions so they can be // replaced with fakes for testing. newPluginManager func(logrus.FieldLogger) clientmgmt.Manager backupStoreGetter persistence.ObjectBackupStoreGetter // used to force update of async backup item operations before processing download request backupItemOperationsMap *itemoperationmap.BackupItemOperationsMap // used to force update of async restore item operations before processing download request restoreItemOperationsMap *itemoperationmap.RestoreItemOperationsMap log logrus.FieldLogger } // NewDownloadRequestReconciler initializes and returns downloadRequestReconciler struct. func NewDownloadRequestReconciler( client kbclient.Client, clock clocks.Clock, newPluginManager func(logrus.FieldLogger) clientmgmt.Manager, backupStoreGetter persistence.ObjectBackupStoreGetter, log logrus.FieldLogger, backupItemOperationsMap *itemoperationmap.BackupItemOperationsMap, restoreItemOperationsMap *itemoperationmap.RestoreItemOperationsMap, ) *downloadRequestReconciler { return &downloadRequestReconciler{ client: client, clock: clock, newPluginManager: newPluginManager, backupStoreGetter: backupStoreGetter, backupItemOperationsMap: backupItemOperationsMap, restoreItemOperationsMap: restoreItemOperationsMap, log: log, } } // +kubebuilder:rbac:groups=velero.io,resources=downloadrequests,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=velero.io,resources=downloadrequests/status,verbs=get;update;patch func (r *downloadRequestReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := r.log.WithFields(logrus.Fields{ "controller": "download-request", "downloadRequest": req.NamespacedName, }) // Fetch the DownloadRequest instance. log.Debug("Getting DownloadRequest") downloadRequest := &velerov1api.DownloadRequest{} if err := r.client.Get(ctx, req.NamespacedName, downloadRequest); err != nil { if apierrors.IsNotFound(err) { log.Debug("Unable to find DownloadRequest") return ctrl.Result{}, nil } log.WithError(err).Error("Error getting DownloadRequest") return ctrl.Result{}, errors.WithStack(err) } if downloadRequest.Status != (velerov1api.DownloadRequestStatus{}) && downloadRequest.Status.Expiration != nil { if downloadRequest.Status.Expiration.Time.Before(r.clock.Now()) { // Delete any request that is expired, regardless of the phase: it is not // worth proceeding and trying/retrying to find it. log.Debug("DownloadRequest has expired - deleting") if err := r.client.Delete(ctx, downloadRequest); err != nil { log.WithError(err).Error("Error deleting an expired download request") return ctrl.Result{}, errors.WithStack(err) } return ctrl.Result{}, nil } else if downloadRequest.Status.Phase == velerov1api.DownloadRequestPhaseProcessed { log.Debug("DownloadRequest has not yet expired.") return ctrl.Result{}, nil } } // Process a brand new request. if downloadRequest.Status.Phase == "" || downloadRequest.Status.Phase == velerov1api.DownloadRequestPhaseNew { backupName := downloadRequest.Spec.Target.Name original := downloadRequest.DeepCopy() defer func() { // Always attempt to Patch the downloadRequest object and status for new DownloadRequest. if err := r.client.Patch(ctx, downloadRequest, kbclient.MergeFrom(original)); err != nil { log.WithError(err).Error("Error updating download request") return } }() // Update the expiration. downloadRequest.Status.Expiration = &metav1.Time{Time: r.clock.Now().Add(persistence.DownloadURLTTL)} if downloadRequest.Spec.Target.Kind == velerov1api.DownloadTargetKindRestoreLog || downloadRequest.Spec.Target.Kind == velerov1api.DownloadTargetKindRestoreResults || downloadRequest.Spec.Target.Kind == velerov1api.DownloadTargetKindRestoreResourceList || downloadRequest.Spec.Target.Kind == velerov1api.DownloadTargetKindRestoreItemOperations || downloadRequest.Spec.Target.Kind == velerov1api.DownloadTargetKindRestoreVolumeInfo { restore := &velerov1api.Restore{} if err := r.client.Get(ctx, kbclient.ObjectKey{ Namespace: downloadRequest.Namespace, Name: downloadRequest.Spec.Target.Name, }, restore); err != nil { if apierrors.IsNotFound(err) { log.WithError(err).Error("fail to get restore for DownloadRequest") return ctrl.Result{}, nil } log.Warnf("fail to get restore for DownloadRequest %s. Retry later.", err.Error()) return ctrl.Result{}, errors.WithStack(err) } backupName = restore.Spec.BackupName } backup := &velerov1api.Backup{} if err := r.client.Get(ctx, kbclient.ObjectKey{ Namespace: downloadRequest.Namespace, Name: backupName, }, backup); err != nil { if apierrors.IsNotFound(err) { log.WithError(err).Error("fail to get backup for DownloadRequest") return ctrl.Result{}, nil } log.Warnf("fail to get backup for DownloadRequest %s. Retry later.", err.Error()) return ctrl.Result{}, errors.WithStack(err) } location := &velerov1api.BackupStorageLocation{} if err := r.client.Get(ctx, kbclient.ObjectKey{ Namespace: backup.Namespace, Name: backup.Spec.StorageLocation, }, location); err != nil { if apierrors.IsNotFound(err) { log.Errorf("BSL for DownloadRequest cannot be found") return ctrl.Result{}, nil } log.Warnf("fail to get BSL for DownloadRequest: %s", err.Error()) return ctrl.Result{}, errors.WithStack(err) } pluginManager := r.newPluginManager(log) defer pluginManager.CleanupClients() backupStore, err := r.backupStoreGetter.Get(location, pluginManager, log) if err != nil { log.WithError(err).Error("Error getting a backup store") // Fail to get backup store is due to BSL setting issue or credential issue. // It cannot be recovered. No need to retry. return ctrl.Result{}, nil } // If this is a request for backup item operations, force upload of in-memory operations that // are not yet uploaded (if there are any) if downloadRequest.Spec.Target.Kind == velerov1api.DownloadTargetKindBackupItemOperations && r.backupItemOperationsMap != nil { // ignore errors here. If we can't upload anything here, process the download as usual _ = r.backupItemOperationsMap.UpdateForBackup(backupStore, backupName) } // If this is a request for restore item operations, force upload of in-memory operations that // are not yet uploaded (if there are any) if downloadRequest.Spec.Target.Kind == velerov1api.DownloadTargetKindRestoreItemOperations && r.restoreItemOperationsMap != nil { // ignore errors here. If we can't upload anything here, process the download as usual _ = r.restoreItemOperationsMap.UpdateForRestore(backupStore, downloadRequest.Spec.Target.Name) } if downloadRequest.Status.DownloadURL, err = backupStore.GetDownloadURL(downloadRequest.Spec.Target); err != nil { log.Warnf("fail to get Backup metadata file's download URL %s, retry later: %s", downloadRequest.Spec.Target, err) return ctrl.Result{}, errors.WithStack(err) } downloadRequest.Status.Phase = velerov1api.DownloadRequestPhaseProcessed // Update the expiration again to extend the time we wait (the TTL) to start after successfully processing the URL. downloadRequest.Status.Expiration = &metav1.Time{Time: r.clock.Now().Add(persistence.DownloadURLTTL)} } return ctrl.Result{}, nil } func (r *downloadRequestReconciler) SetupWithManager(mgr ctrl.Manager) error { downloadRequestPredicate := kube.NewGenericEventPredicate(func(object kbclient.Object) bool { downloadRequest := object.(*velerov1api.DownloadRequest) if downloadRequest.Status != (velerov1api.DownloadRequestStatus{}) && downloadRequest.Status.Expiration != nil { return downloadRequest.Status.Expiration.Time.Before(r.clock.Now()) } return true }) downloadRequestSource := kube.NewPeriodicalEnqueueSource(r.log.WithField("controller", constant.ControllerDownloadRequest), mgr.GetClient(), &velerov1api.DownloadRequestList{}, defaultDownloadRequestSyncPeriod, kube.PeriodicalEnqueueSourceOption{ Predicates: []predicate.Predicate{downloadRequestPredicate}, }) return ctrl.NewControllerManagedBy(mgr). For(&velerov1api.DownloadRequest{}). WatchesRawSource(downloadRequestSource). Complete(r) } ================================================ FILE: pkg/controller/download_request_controller_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package controller import ( "context" "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/sirupsen/logrus" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" testclocks "k8s.io/utils/clock/testing" ctrl "sigs.k8s.io/controller-runtime" kbclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" persistencemocks "github.com/vmware-tanzu/velero/pkg/persistence/mocks" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" pluginmocks "github.com/vmware-tanzu/velero/pkg/plugin/mocks" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) var _ = Describe("Download Request Reconciler", func() { type request struct { downloadRequest *velerov1api.DownloadRequest backup *velerov1api.Backup restore *velerov1api.Restore backupLocation *velerov1api.BackupStorageLocation expired bool expectedReconcileErr string expectGetsURL bool expectedRequeue ctrl.Result } defaultBackup := func() *velerov1api.Backup { return builder.ForBackup(velerov1api.DefaultNamespace, "a-backup").StorageLocation("a-location").Result() } DescribeTable("a Download request", func(test request) { // now will be used to set the fake clock's time; capture // it here so it can be referenced in the test case defs. now, err := time.Parse(time.RFC1123, time.RFC1123) Expect(err).ToNot(HaveOccurred()) now = now.Local() rClock := testclocks.NewFakeClock(now) const signedURLTTL = 10 * time.Minute var ( pluginManager = &pluginmocks.Manager{} backupStores = make(map[string]*persistencemocks.BackupStore) ) pluginManager.On("CleanupClients").Return(nil) Expect(test.downloadRequest).ToNot(BeNil()) // Set .status.expiration properly for all requests test cases that are // meant to be expired. Since "expired" is relative to the controller's // clock time, it's easier to do this here than as part of the test case definitions. if test.expired { test.downloadRequest.Status.Expiration = &metav1.Time{Time: rClock.Now().Add(-1 * time.Minute)} } fakeClient := fake.NewClientBuilder().WithScheme(scheme.Scheme).Build() err = fakeClient.Create(context.TODO(), test.downloadRequest) Expect(err).ToNot(HaveOccurred()) if test.backup != nil { err := fakeClient.Create(context.TODO(), test.backup) Expect(err).ToNot(HaveOccurred()) } if test.backupLocation != nil { err := fakeClient.Create(context.TODO(), test.backupLocation) Expect(err).ToNot(HaveOccurred()) backupStores[test.backupLocation.Name] = &persistencemocks.BackupStore{} } if test.restore != nil { err := fakeClient.Create(context.TODO(), test.restore) Expect(err).ToNot(HaveOccurred()) } // Setup reconciler Expect(velerov1api.AddToScheme(scheme.Scheme)).To(Succeed()) r := NewDownloadRequestReconciler( fakeClient, rClock, func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager }, NewFakeObjectBackupStoreGetter(backupStores), velerotest.NewLogger(), nil, nil, ) if test.backupLocation != nil && test.expectGetsURL { backupStores[test.backupLocation.Name].On("GetDownloadURL", test.downloadRequest.Spec.Target).Return("a-url", nil) } actualResult, err := r.Reconcile(context.Background(), ctrl.Request{ NamespacedName: types.NamespacedName{ Namespace: velerov1api.DefaultNamespace, Name: test.downloadRequest.Name, }, }) Expect(actualResult).To(BeEquivalentTo(test.expectedRequeue)) if test.expectedReconcileErr == "" { Expect(err).ToNot(HaveOccurred()) } else { Expect(err.Error()).To(Equal(test.expectedReconcileErr)) } instance := &velerov1api.DownloadRequest{} err = r.client.Get(ctx, kbclient.ObjectKey{Name: test.downloadRequest.Name, Namespace: test.downloadRequest.Namespace}, instance) if test.expired { Expect(instance).ToNot(Equal(test.downloadRequest)) Expect(apierrors.IsNotFound(err)).To(BeTrue()) } else { if test.downloadRequest.Status.Phase == velerov1api.DownloadRequestPhaseProcessed { Expect(instance.Status).To(Equal(test.downloadRequest.Status)) } else { Expect(instance.Status).ToNot(Equal(test.downloadRequest.Status)) } Expect(err).ToNot(HaveOccurred()) } if test.expectGetsURL { Expect(string(instance.Status.Phase)).To(Equal(string(velerov1api.DownloadRequestPhaseProcessed))) Expect(instance.Status.DownloadURL).To(Equal("a-url")) Expect(velerotest.TimesAreEqual(instance.Status.Expiration.Time, r.clock.Now().Add(signedURLTTL))).To(BeTrue()) } }, Entry("backup contents request for nonexistent backup returns nil", request{ downloadRequest: builder.ForDownloadRequest(velerov1api.DefaultNamespace, "a-download-request").Phase("").Target(velerov1api.DownloadTargetKindBackupContents, "a1-backup").Result(), backup: builder.ForBackup(velerov1api.DefaultNamespace, "non-matching-backup").StorageLocation("a-location").Result(), backupLocation: builder.ForBackupStorageLocation(velerov1api.DefaultNamespace, "a-location").Provider("a-provider").Bucket("a-bucket").Result(), expectedReconcileErr: "", expectedRequeue: ctrl.Result{}, }), Entry("restore log request for nonexistent restore returns nil", request{ downloadRequest: builder.ForDownloadRequest(velerov1api.DefaultNamespace, "a-download-request").Phase("").Target(velerov1api.DownloadTargetKindRestoreLog, "a-backup-20170912150214").Result(), restore: builder.ForRestore(velerov1api.DefaultNamespace, "non-matching-restore").Phase(velerov1api.RestorePhaseCompleted).Backup("a-backup").Result(), backup: defaultBackup(), backupLocation: builder.ForBackupStorageLocation(velerov1api.DefaultNamespace, "a-location").Provider("a-provider").Bucket("a-bucket").Result(), expectedReconcileErr: "", expectedRequeue: ctrl.Result{}, }), Entry("backup contents request for backup with nonexistent location returns nil", request{ downloadRequest: builder.ForDownloadRequest(velerov1api.DefaultNamespace, "a-download-request").Phase("").Target(velerov1api.DownloadTargetKindBackupContents, "a-backup").Result(), backup: defaultBackup(), backupLocation: builder.ForBackupStorageLocation(velerov1api.DefaultNamespace, "non-matching-location").Provider("a-provider").Bucket("a-bucket").Result(), expectedReconcileErr: "", expectedRequeue: ctrl.Result{}, }), Entry("backup contents request with phase '' gets a url", request{ downloadRequest: builder.ForDownloadRequest(velerov1api.DefaultNamespace, "a-download-request").Phase("").Target(velerov1api.DownloadTargetKindBackupContents, "a-backup").Result(), backup: defaultBackup(), backupLocation: builder.ForBackupStorageLocation(velerov1api.DefaultNamespace, "a-location").Provider("a-provider").Bucket("a-bucket").Result(), expectGetsURL: true, expectedRequeue: ctrl.Result{}, }), Entry("backup contents request with phase 'New' gets a url", request{ downloadRequest: builder.ForDownloadRequest(velerov1api.DefaultNamespace, "a-download-request").Phase(velerov1api.DownloadRequestPhaseNew).Target(velerov1api.DownloadTargetKindBackupContents, "a-backup").Result(), backup: defaultBackup(), backupLocation: builder.ForBackupStorageLocation(velerov1api.DefaultNamespace, "a-location").Provider("a-provider").Bucket("a-bucket").Result(), expectGetsURL: true, expectedRequeue: ctrl.Result{}, }), Entry("backup log request with phase '' gets a url", request{ downloadRequest: builder.ForDownloadRequest(velerov1api.DefaultNamespace, "a-download-request").Phase("").Target(velerov1api.DownloadTargetKindBackupLog, "a-backup").Result(), backup: defaultBackup(), backupLocation: builder.ForBackupStorageLocation(velerov1api.DefaultNamespace, "a-location").Provider("a-provider").Bucket("a-bucket").Result(), expectGetsURL: true, expectedRequeue: ctrl.Result{}, }), Entry("backup log request with phase 'New' gets a url", request{ downloadRequest: builder.ForDownloadRequest(velerov1api.DefaultNamespace, "a-download-request").Phase(velerov1api.DownloadRequestPhaseNew).Target(velerov1api.DownloadTargetKindBackupLog, "a-backup").Result(), backup: defaultBackup(), backupLocation: builder.ForBackupStorageLocation(velerov1api.DefaultNamespace, "a-location").Provider("a-provider").Bucket("a-bucket").Result(), expectGetsURL: true, expectedRequeue: ctrl.Result{}, }), Entry("restore log request with phase '' gets a url", request{ downloadRequest: builder.ForDownloadRequest(velerov1api.DefaultNamespace, "a-download-request").Phase("").Target(velerov1api.DownloadTargetKindRestoreLog, "a-backup-20170912150214").Result(), restore: builder.ForRestore(velerov1api.DefaultNamespace, "a-backup-20170912150214").Phase(velerov1api.RestorePhaseCompleted).Backup("a-backup").Result(), backup: defaultBackup(), backupLocation: builder.ForBackupStorageLocation(velerov1api.DefaultNamespace, "a-location").Provider("a-provider").Bucket("a-bucket").Result(), expectGetsURL: true, expectedRequeue: ctrl.Result{}, }), Entry("restore log request with phase 'New' gets a url", request{ downloadRequest: builder.ForDownloadRequest(velerov1api.DefaultNamespace, "a-download-request").Phase(velerov1api.DownloadRequestPhaseNew).Target(velerov1api.DownloadTargetKindRestoreLog, "a-backup-20170912150214").Result(), backup: defaultBackup(), restore: builder.ForRestore(velerov1api.DefaultNamespace, "a-backup-20170912150214").Phase(velerov1api.RestorePhaseCompleted).Backup("a-backup").Result(), backupLocation: builder.ForBackupStorageLocation(velerov1api.DefaultNamespace, "a-location").Provider("a-provider").Bucket("a-bucket").Result(), expectGetsURL: true, expectedRequeue: ctrl.Result{}, }), Entry("restore results request with phase '' gets a url", request{ downloadRequest: builder.ForDownloadRequest(velerov1api.DefaultNamespace, "a-download-request").Phase("").Target(velerov1api.DownloadTargetKindRestoreResults, "a-backup-20170912150214").Result(), restore: builder.ForRestore(velerov1api.DefaultNamespace, "a-backup-20170912150214").Phase(velerov1api.RestorePhaseCompleted).Backup("a-backup").Result(), backup: defaultBackup(), backupLocation: builder.ForBackupStorageLocation(velerov1api.DefaultNamespace, "a-location").Provider("a-provider").Bucket("a-bucket").Result(), expectGetsURL: true, expectedRequeue: ctrl.Result{}, }), Entry("restore results request with phase 'New' gets a url", request{ downloadRequest: builder.ForDownloadRequest(velerov1api.DefaultNamespace, "a-download-request").Phase(velerov1api.DownloadRequestPhaseNew).Target(velerov1api.DownloadTargetKindRestoreResults, "a-backup-20170912150214").Result(), restore: builder.ForRestore(velerov1api.DefaultNamespace, "a-backup-20170912150214").Phase(velerov1api.RestorePhaseCompleted).Backup("a-backup").Result(), backup: defaultBackup(), backupLocation: builder.ForBackupStorageLocation(velerov1api.DefaultNamespace, "a-location").Provider("a-provider").Bucket("a-bucket").Result(), expectGetsURL: true, expectedRequeue: ctrl.Result{}, }), Entry("request with phase 'Processed' and not expired is not deleted", request{ downloadRequest: builder.ForDownloadRequest(velerov1api.DefaultNamespace, "a-download-request").Phase(velerov1api.DownloadRequestPhaseProcessed).Target(velerov1api.DownloadTargetKindBackupLog, "a-backup-20170912150214").Result(), backup: defaultBackup(), expectedRequeue: ctrl.Result{}, }), Entry("request with phase 'Processed' and expired is deleted", request{ downloadRequest: builder.ForDownloadRequest(velerov1api.DefaultNamespace, "a-download-request").Phase(velerov1api.DownloadRequestPhaseProcessed).Target(velerov1api.DownloadTargetKindBackupLog, "a-backup-20170912150214").Result(), backup: defaultBackup(), expired: true, expectedRequeue: ctrl.Result{}, }), Entry("request with phase '' and expired is deleted", request{ downloadRequest: builder.ForDownloadRequest(velerov1api.DefaultNamespace, "a-download-request").Phase("").Target(velerov1api.DownloadTargetKindBackupLog, "a-backup-20170912150214").Result(), backup: defaultBackup(), expired: true, expectedRequeue: ctrl.Result{}, }), Entry("request with phase 'New' and expired is deleted", request{ downloadRequest: builder.ForDownloadRequest(velerov1api.DefaultNamespace, "a-download-request").Phase(velerov1api.DownloadRequestPhaseNew).Target(velerov1api.DownloadTargetKindBackupLog, "a-backup-20170912150214").Result(), backup: defaultBackup(), expired: true, expectedRequeue: ctrl.Result{}, }), ) }) ================================================ FILE: pkg/controller/gc_controller.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package controller import ( "context" "fmt" "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" apierrors "k8s.io/apimachinery/pkg/api/errors" clocks "k8s.io/utils/clock" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/predicate" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" pkgbackup "github.com/vmware-tanzu/velero/pkg/backup" veleroclient "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/constant" "github.com/vmware-tanzu/velero/pkg/label" "github.com/vmware-tanzu/velero/pkg/util/kube" veleroutil "github.com/vmware-tanzu/velero/pkg/util/velero" ) const ( defaultGCFrequency = 60 * time.Minute garbageCollectionFailure = "velero.io/gc-failure" gcFailureBSLNotFound = "BSLNotFound" gcFailureBSLCannotGet = "BSLCannotGet" gcFailureBSLReadOnly = "BSLReadOnly" gcFailureBSLUnavailable = "BSLUnavailable" ) // gcReconciler creates DeleteBackupRequests for expired backups. type gcReconciler struct { client.Client logger logrus.FieldLogger clock clocks.WithTickerAndDelayedExecution frequency time.Duration } // NewGCReconciler constructs a new gcReconciler. func NewGCReconciler( logger logrus.FieldLogger, client client.Client, frequency time.Duration, ) *gcReconciler { gcr := &gcReconciler{ Client: client, logger: logger, clock: clocks.RealClock{}, frequency: frequency, } if gcr.frequency <= 0 { gcr.frequency = defaultGCFrequency } return gcr } // GCController only watches on CreateEvent for ensuring every new backup will be taken care of. // Other Events will be filtered to decrease the number of reconcile call. Especially UpdateEvent must be filtered since we removed // the backup status as the sub-resource of backup in v1.9, every change on it will be treated as UpdateEvent and trigger reconcile call. func (c *gcReconciler) SetupWithManager(mgr ctrl.Manager) error { s := kube.NewPeriodicalEnqueueSource(c.logger.WithField("controller", constant.ControllerGarbageCollection), mgr.GetClient(), &velerov1api.BackupList{}, c.frequency, kube.PeriodicalEnqueueSourceOption{}) return ctrl.NewControllerManagedBy(mgr). For(&velerov1api.Backup{}, builder.WithPredicates(predicate.Funcs{ UpdateFunc: func(ue event.UpdateEvent) bool { return false }, DeleteFunc: func(de event.DeleteEvent) bool { return false }, GenericFunc: func(ge event.GenericEvent) bool { return false }, })). WatchesRawSource(s). Named(constant.ControllerGarbageCollection). Complete(c) } // +kubebuilder:rbac:groups=velero.io,resources=backups,verbs=get;list;watch;update // +kubebuilder:rbac:groups=velero.io,resources=backups/status,verbs=get // +kubebuilder:rbac:groups=velero.io,resources=deletebackuprequests,verbs=get;list;watch;create; // +kubebuilder:rbac:groups=velero.io,resources=deletebackuprequests/status,verbs=get // +kubebuilder:rbac:groups=velero.io,resources=backupstoragelocations,verbs=get func (c *gcReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := c.logger.WithField("gc backup", req.String()) log.Debug("gcController getting backup") backup := &velerov1api.Backup{} if err := c.Get(ctx, req.NamespacedName, backup); err != nil { if apierrors.IsNotFound(err) { log.WithError(err).Error("backup not found") return ctrl.Result{}, nil } return ctrl.Result{}, errors.Wrapf(err, "error getting backup %s", req.String()) } log.Debugf("backup: %s", backup.Name) log = c.logger.WithFields( logrus.Fields{ "backup": req.String(), "expiration": backup.Status.Expiration, }, ) now := c.clock.Now() if backup.Status.Expiration == nil || backup.Status.Expiration.After(now) { log.Debug("Backup has not expired yet, skipping") return ctrl.Result{}, nil } log.Infof("Backup:%s has expired", backup.Name) if backup.Labels == nil { backup.Labels = make(map[string]string) } loc := &velerov1api.BackupStorageLocation{} if err := c.Get(ctx, client.ObjectKey{ Namespace: req.Namespace, Name: backup.Spec.StorageLocation, }, loc); err != nil { if apierrors.IsNotFound(err) { log.Warnf("Backup cannot be garbage-collected because backup storage location %s does not exist", backup.Spec.StorageLocation) backup.Labels[garbageCollectionFailure] = gcFailureBSLNotFound } else { backup.Labels[garbageCollectionFailure] = gcFailureBSLCannotGet } if err := c.Update(ctx, backup); err != nil { log.WithError(err).Error("error updating backup labels") } return ctrl.Result{}, errors.Wrap(err, "error getting backup storage location") } if !veleroutil.BSLIsAvailable(*loc) { log.Infof("BSL %s is unavailable, cannot gc backup", loc.Name) return ctrl.Result{}, fmt.Errorf("bsl %s is unavailable, cannot gc backup", loc.Name) } if loc.Spec.AccessMode == velerov1api.BackupStorageLocationAccessModeReadOnly { log.Infof("Backup cannot be garbage-collected because backup storage location %s is currently in read-only mode", loc.Name) backup.Labels[garbageCollectionFailure] = gcFailureBSLReadOnly if err := c.Update(ctx, backup); err != nil { log.WithError(err).Error("error updating backup labels") } return ctrl.Result{}, nil } // remove gc fail error label after this point delete(backup.Labels, garbageCollectionFailure) if err := c.Update(ctx, backup); err != nil { log.WithError(err).Error("error updating backup labels") } selector := client.MatchingLabels{ velerov1api.BackupNameLabel: label.GetValidName(backup.Name), velerov1api.BackupUIDLabel: string(backup.UID), } dbrs := &velerov1api.DeleteBackupRequestList{} if err := c.List(ctx, dbrs, selector); err != nil { log.WithError(err).Error("error listing DeleteBackupRequests") return ctrl.Result{}, errors.Wrap(err, "error listing existing DeleteBackupRequests for backup") } log.Debugf("length of dbrs:%d", len(dbrs.Items)) // if there's an existing unprocessed deletion request for this backup, don't create // another one for _, dbr := range dbrs.Items { switch dbr.Status.Phase { case "", velerov1api.DeleteBackupRequestPhaseNew, velerov1api.DeleteBackupRequestPhaseInProgress: log.Info("Backup already has a pending deletion request") return ctrl.Result{}, nil } } log.Info("Creating a new deletion request") ndbr := pkgbackup.NewDeleteBackupRequest(backup.Name, string(backup.UID)) ndbr.SetNamespace(backup.Namespace) if err := veleroclient.CreateRetryGenerateName(c, ctx, ndbr); err != nil { log.WithError(err).Error("error creating DeleteBackupRequests") return ctrl.Result{}, errors.Wrap(err, "error creating DeleteBackupRequest") } return ctrl.Result{}, nil } ================================================ FILE: pkg/controller/gc_controller_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package controller import ( "testing" "time" "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" testclocks "k8s.io/utils/clock/testing" ctrl "sigs.k8s.io/controller-runtime" kbclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func mockGCReconciler(fakeClient kbclient.Client, fakeClock *testclocks.FakeClock, freq time.Duration) *gcReconciler { gcr := NewGCReconciler( velerotest.NewLogger(), fakeClient, freq, ) gcr.clock = fakeClock return gcr } func TestGCReconcile(t *testing.T) { fakeClock := testclocks.NewFakeClock(time.Now()) defaultBackupLocation := builder.ForBackupStorageLocation(velerov1api.DefaultNamespace, "default").Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result() tests := []struct { name string backup *velerov1api.Backup deleteBackupRequests []*velerov1api.DeleteBackupRequest backupLocation *velerov1api.BackupStorageLocation expectError bool }{ { name: "can't find backup - no error", }, { name: "unexpired backup is not deleted", backup: defaultBackup().Expiration(fakeClock.Now().Add(time.Minute)).StorageLocation("default").Result(), backupLocation: defaultBackupLocation, }, { name: "expired backup in read-only storage location is not deleted", backup: defaultBackup().Expiration(fakeClock.Now().Add(-time.Minute)).StorageLocation("read-only").Result(), backupLocation: builder.ForBackupStorageLocation("velero", "read-only").AccessMode(velerov1api.BackupStorageLocationAccessModeReadOnly).Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result(), }, { name: "expired backup in read-write storage location is deleted", backup: defaultBackup().Expiration(fakeClock.Now().Add(-time.Minute)).StorageLocation("read-write").Result(), backupLocation: builder.ForBackupStorageLocation("velero", "read-write").AccessMode(velerov1api.BackupStorageLocationAccessModeReadWrite).Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result(), }, { name: "expired backup with no pending deletion requests is deleted", backup: defaultBackup().Expiration(fakeClock.Now().Add(-time.Second)).StorageLocation("default").Result(), backupLocation: defaultBackupLocation, }, { name: "expired backup with a pending deletion request is not deleted", backup: defaultBackup().Expiration(fakeClock.Now().Add(-time.Second)).StorageLocation("default").Result(), backupLocation: defaultBackupLocation, deleteBackupRequests: []*velerov1api.DeleteBackupRequest{ { ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "foo", Labels: map[string]string{ velerov1api.BackupNameLabel: "backup-1", velerov1api.BackupUIDLabel: "", }, }, Status: velerov1api.DeleteBackupRequestStatus{ Phase: velerov1api.DeleteBackupRequestPhaseInProgress, }, }, }, }, { name: "expired backup with only processed deletion requests is deleted", backup: defaultBackup().Expiration(fakeClock.Now().Add(-time.Second)).StorageLocation("default").Result(), backupLocation: defaultBackupLocation, deleteBackupRequests: []*velerov1api.DeleteBackupRequest{ { ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "foo", Labels: map[string]string{ velerov1api.BackupNameLabel: "backup-1", velerov1api.BackupUIDLabel: "", }, }, Status: velerov1api.DeleteBackupRequestStatus{ Phase: velerov1api.DeleteBackupRequestPhaseProcessed, }, }, }, }, { name: "BSL is unavailable", backup: defaultBackup().Expiration(fakeClock.Now().Add(-time.Second)).StorageLocation("default").Result(), backupLocation: builder.ForBackupStorageLocation(velerov1api.DefaultNamespace, "default").Phase(velerov1api.BackupStorageLocationPhaseUnavailable).Result(), expectError: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { if test.backup == nil { return } initObjs := []runtime.Object{} initObjs = append(initObjs, test.backup) if test.backupLocation != nil { initObjs = append(initObjs, test.backupLocation) } for _, dbr := range test.deleteBackupRequests { initObjs = append(initObjs, dbr) } fakeClient := velerotest.NewFakeControllerRuntimeClient(t, initObjs...) reconciler := mockGCReconciler(fakeClient, fakeClock, defaultGCFrequency) _, err := reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: test.backup.Namespace, Name: test.backup.Name}}) gotErr := err != nil assert.Equal(t, test.expectError, gotErr) }) } } ================================================ FILE: pkg/controller/interface.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package controller import "context" // Interface represents a runnable component. type Interface interface { // Run runs the component. Run(ctx context.Context, workers int) error } ================================================ FILE: pkg/controller/pod_volume_backup_controller.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package controller import ( "context" "fmt" "strings" "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" clocks "k8s.io/utils/clock" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" veleroapishared "github.com/vmware-tanzu/velero/pkg/apis/velero/shared" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/constant" "github.com/vmware-tanzu/velero/pkg/datapath" "github.com/vmware-tanzu/velero/pkg/exposer" "github.com/vmware-tanzu/velero/pkg/metrics" "github.com/vmware-tanzu/velero/pkg/nodeagent" "github.com/vmware-tanzu/velero/pkg/uploader" "github.com/vmware-tanzu/velero/pkg/util" "github.com/vmware-tanzu/velero/pkg/util/kube" ) const ( pVBRRequestor = "pod-volume-backup-restore" PodVolumeFinalizer = "velero.io/pod-volume-finalizer" ) // NewPodVolumeBackupReconciler creates the PodVolumeBackupReconciler instance func NewPodVolumeBackupReconciler( client client.Client, mgr manager.Manager, kubeClient kubernetes.Interface, dataPathMgr *datapath.Manager, counter *exposer.VgdpCounter, nodeName string, preparingTimeout time.Duration, resourceTimeout time.Duration, podResources corev1api.ResourceRequirements, metrics *metrics.ServerMetrics, logger logrus.FieldLogger, dataMovePriorityClass string, privileged bool, podLabels map[string]string, podAnnotations map[string]string, ) *PodVolumeBackupReconciler { return &PodVolumeBackupReconciler{ client: client, mgr: mgr, kubeClient: kubeClient, logger: logger.WithField("controller", "PodVolumeBackup"), nodeName: nodeName, clock: &clocks.RealClock{}, metrics: metrics, podResources: podResources, dataPathMgr: dataPathMgr, vgdpCounter: counter, preparingTimeout: preparingTimeout, resourceTimeout: resourceTimeout, exposer: exposer.NewPodVolumeExposer(kubeClient, logger), cancelledPVB: make(map[string]time.Time), dataMovePriorityClass: dataMovePriorityClass, privileged: privileged, podLabels: podLabels, podAnnotations: podAnnotations, } } // PodVolumeBackupReconciler reconciles a PodVolumeBackup object type PodVolumeBackupReconciler struct { client client.Client mgr manager.Manager kubeClient kubernetes.Interface clock clocks.WithTickerAndDelayedExecution exposer exposer.PodVolumeExposer metrics *metrics.ServerMetrics nodeName string logger logrus.FieldLogger podResources corev1api.ResourceRequirements dataPathMgr *datapath.Manager vgdpCounter *exposer.VgdpCounter preparingTimeout time.Duration resourceTimeout time.Duration cancelledPVB map[string]time.Time dataMovePriorityClass string privileged bool podLabels map[string]string podAnnotations map[string]string } // +kubebuilder:rbac:groups=velero.io,resources=podvolumebackups,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=velero.io,resources=podvolumebackups/status,verbs=get;update;patch func (r *PodVolumeBackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := r.logger.WithFields(logrus.Fields{ "controller": "podvolumebackup", "podvolumebackup": req.NamespacedName, }) var pvb = &velerov1api.PodVolumeBackup{} if err := r.client.Get(ctx, req.NamespacedName, pvb); err != nil { if apierrors.IsNotFound(err) { log.Warn("Unable to find PVB, skip") return ctrl.Result{}, nil } return ctrl.Result{}, errors.Wrap(err, "getting PVB") } if len(pvb.OwnerReferences) == 1 { log = log.WithField( "backup", fmt.Sprintf("%s/%s", req.Namespace, pvb.OwnerReferences[0].Name), ) } if !isPVBInFinalState(pvb) { if !controllerutil.ContainsFinalizer(pvb, PodVolumeFinalizer) { if err := UpdatePVBWithRetry(ctx, r.client, req.NamespacedName, log, func(pvb *velerov1api.PodVolumeBackup) bool { if controllerutil.ContainsFinalizer(pvb, PodVolumeFinalizer) { return false } controllerutil.AddFinalizer(pvb, PodVolumeFinalizer) return true }); err != nil { log.WithError(err).Errorf("Failed to add finalizer for PVB %s/%s", pvb.Namespace, pvb.Name) return ctrl.Result{}, err } return ctrl.Result{}, nil } if !pvb.DeletionTimestamp.IsZero() { if !pvb.Spec.Cancel { log.Warnf("Cancel PVB under phase %s because it is being deleted", pvb.Status.Phase) if err := UpdatePVBWithRetry(ctx, r.client, req.NamespacedName, log, func(pvb *velerov1api.PodVolumeBackup) bool { if pvb.Spec.Cancel { return false } pvb.Spec.Cancel = true pvb.Status.Message = "Cancel PVB because it is being deleted" return true }); err != nil { log.WithError(err).Errorf("Failed to set cancel flag for PVB %s/%s", pvb.Namespace, pvb.Name) return ctrl.Result{}, err } return ctrl.Result{}, nil } } } else { delete(r.cancelledPVB, pvb.Name) if controllerutil.ContainsFinalizer(pvb, PodVolumeFinalizer) { if err := UpdatePVBWithRetry(ctx, r.client, req.NamespacedName, log, func(pvb *velerov1api.PodVolumeBackup) bool { if !controllerutil.ContainsFinalizer(pvb, PodVolumeFinalizer) { return false } controllerutil.RemoveFinalizer(pvb, PodVolumeFinalizer) return true }); err != nil { log.WithError(err).Error("error to remove finalizer") return ctrl.Result{}, err } return ctrl.Result{}, nil } } if pvb.Spec.Cancel { if spotted, found := r.cancelledPVB[pvb.Name]; !found { r.cancelledPVB[pvb.Name] = r.clock.Now() } else { delay := cancelDelayOthers if pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseInProgress { delay = cancelDelayInProgress } if time.Since(spotted) > delay { log.Infof("PVB %s is canceled in Phase %s but not handled in reasonable time", pvb.GetName(), pvb.Status.Phase) if r.tryCancelPodVolumeBackup(ctx, pvb, "") { delete(r.cancelledPVB, pvb.Name) } return ctrl.Result{}, nil } } } if pvb.Status.Phase == "" || pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseNew { if pvb.Spec.Cancel { log.Infof("PVB %s is canceled in Phase %s", pvb.GetName(), pvb.Status.Phase) r.tryCancelPodVolumeBackup(ctx, pvb, "") return ctrl.Result{}, nil } // Only process items for this node. if pvb.Spec.Node != r.nodeName { return ctrl.Result{}, nil } if r.vgdpCounter != nil && r.vgdpCounter.IsConstrained(ctx, r.logger) { log.Debug("Data path initiation is constrained, requeue later") return ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, nil } log.Info("Accepting PVB") if err := r.acceptPodVolumeBackup(ctx, pvb); err != nil { return ctrl.Result{}, errors.Wrapf(err, "error accepting PVB %s", pvb.Name) } log.Info("Exposing PVB") exposeParam := r.setupExposeParam(pvb) if err := r.exposer.Expose(ctx, getPVBOwnerObject(pvb), exposeParam); err != nil { return r.errorOut(ctx, pvb, err, "error to expose PVB", log) } log.Info("PVB is exposed") return ctrl.Result{}, nil } else if pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseAccepted { if peekErr := r.exposer.PeekExposed(ctx, getPVBOwnerObject(pvb)); peekErr != nil { log.Errorf("Cancel PVB %s/%s because of expose error %s", pvb.Namespace, pvb.Name, peekErr) diags := strings.Split(r.exposer.DiagnoseExpose(ctx, getPVBOwnerObject(pvb)), "\n") for _, diag := range diags { log.Warnf("[Diagnose PVB expose]%s", diag) } r.tryCancelPodVolumeBackup(ctx, pvb, fmt.Sprintf("found a PVB %s/%s with expose error: %s. mark it as cancel", pvb.Namespace, pvb.Name, peekErr)) } else if pvb.Status.AcceptedTimestamp != nil { if time.Since(pvb.Status.AcceptedTimestamp.Time) >= r.preparingTimeout { r.onPrepareTimeout(ctx, pvb) } } return ctrl.Result{}, nil } else if pvb.Status.Phase == velerov1api.PodVolumeBackupPhasePrepared { log.Infof("PVB is prepared and should be processed by %s (%s)", pvb.Spec.Node, r.nodeName) if pvb.Spec.Node != r.nodeName { return ctrl.Result{}, nil } if pvb.Spec.Cancel { log.Info("Prepared PVB is being canceled") r.OnDataPathCancelled(ctx, pvb.GetNamespace(), pvb.GetName()) return ctrl.Result{}, nil } asyncBR := r.dataPathMgr.GetAsyncBR(pvb.Name) if asyncBR != nil { log.Info("Cancellable data path is already started") return ctrl.Result{}, nil } res, err := r.exposer.GetExposed(ctx, getPVBOwnerObject(pvb), r.client, r.nodeName, r.resourceTimeout) if err != nil { return r.errorOut(ctx, pvb, err, "exposed PVB is not ready", log) } else if res == nil { return r.errorOut(ctx, pvb, errors.New("no expose result is available for the current node"), "exposed PVB is not ready", log) } log.Info("Exposed PVB is ready and creating data path routine") callbacks := datapath.Callbacks{ OnCompleted: r.OnDataPathCompleted, OnFailed: r.OnDataPathFailed, OnCancelled: r.OnDataPathCancelled, OnProgress: r.OnDataPathProgress, } asyncBR, err = r.dataPathMgr.CreateMicroServiceBRWatcher(ctx, r.client, r.kubeClient, r.mgr, datapath.TaskTypeBackup, pvb.Name, pvb.Namespace, res.ByPod.HostingPod.Name, res.ByPod.HostingContainer, pvb.Name, callbacks, false, log) if err != nil { if err == datapath.ConcurrentLimitExceed { log.Debug("Data path instance is concurrent limited requeue later") return ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, nil } else { return r.errorOut(ctx, pvb, err, "error to create data path", log) } } r.metrics.RegisterPodVolumeBackupEnqueue(r.nodeName) if err := r.initCancelableDataPath(ctx, asyncBR, res, log); err != nil { log.WithError(err).Errorf("Failed to init cancelable data path for %s", pvb.Name) r.closeDataPath(ctx, pvb.Name) return r.errorOut(ctx, pvb, err, "error initializing data path", log) } terminated := false if err := UpdatePVBWithRetry(ctx, r.client, types.NamespacedName{Namespace: pvb.Namespace, Name: pvb.Name}, log, func(pvb *velerov1api.PodVolumeBackup) bool { if isPVBInFinalState(pvb) { terminated = true return false } pvb.Status.Phase = velerov1api.PodVolumeBackupPhaseInProgress pvb.Status.StartTimestamp = &metav1.Time{Time: r.clock.Now()} delete(pvb.Labels, exposer.ExposeOnGoingLabel) return true }); err != nil { log.WithError(err).Warnf("Failed to update PVB %s to InProgress, will data path close and retry", pvb.Name) r.closeDataPath(ctx, pvb.Name) return ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, nil } if terminated { log.Warnf("PVB %s is terminated during transition from prepared", pvb.Name) r.closeDataPath(ctx, pvb.Name) return ctrl.Result{}, nil } log.Info("PVB is marked as in progress") if err := r.startCancelableDataPath(asyncBR, pvb, res, log); err != nil { log.WithError(err).Errorf("Failed to start cancelable data path for %s", pvb.Name) r.closeDataPath(ctx, pvb.Name) return r.errorOut(ctx, pvb, err, "error starting data path", log) } return ctrl.Result{}, nil } else if pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseInProgress { if pvb.Spec.Cancel { if pvb.Spec.Node != r.nodeName { return ctrl.Result{}, nil } log.Info("In progress PVB is being canceled") asyncBR := r.dataPathMgr.GetAsyncBR(pvb.Name) if asyncBR == nil { r.OnDataPathCancelled(ctx, pvb.GetNamespace(), pvb.GetName()) return ctrl.Result{}, nil } // Update status to Canceling if err := UpdatePVBWithRetry(ctx, r.client, types.NamespacedName{Namespace: pvb.Namespace, Name: pvb.Name}, log, func(pvb *velerov1api.PodVolumeBackup) bool { if isPVBInFinalState(pvb) { log.Warnf("PVB %s is terminated, abort setting it to canceling", pvb.Name) return false } pvb.Status.Phase = velerov1api.PodVolumeBackupPhaseCanceling return true }); err != nil { log.WithError(err).Error("error updating PVB into canceling status") return ctrl.Result{}, err } asyncBR.Cancel() return ctrl.Result{}, nil } return ctrl.Result{}, nil } return ctrl.Result{}, nil } func (r *PodVolumeBackupReconciler) acceptPodVolumeBackup(ctx context.Context, pvb *velerov1api.PodVolumeBackup) error { return UpdatePVBWithRetry(ctx, r.client, types.NamespacedName{Namespace: pvb.Namespace, Name: pvb.Name}, r.logger, func(pvb *velerov1api.PodVolumeBackup) bool { pvb.Status.AcceptedTimestamp = &metav1.Time{Time: r.clock.Now()} pvb.Status.Phase = velerov1api.PodVolumeBackupPhaseAccepted if pvb.Labels == nil { pvb.Labels = make(map[string]string) } pvb.Labels[exposer.ExposeOnGoingLabel] = "true" return true }) } func (r *PodVolumeBackupReconciler) tryCancelPodVolumeBackup(ctx context.Context, pvb *velerov1api.PodVolumeBackup, message string) bool { log := r.logger.WithField("PVB", pvb.Name) succeeded, err := funcExclusiveUpdatePodVolumeBackup(ctx, r.client, pvb, func(pvb *velerov1api.PodVolumeBackup) { pvb.Status.Phase = velerov1api.PodVolumeBackupPhaseCanceled if pvb.Status.StartTimestamp.IsZero() { pvb.Status.StartTimestamp = &metav1.Time{Time: r.clock.Now()} } pvb.Status.CompletionTimestamp = &metav1.Time{Time: r.clock.Now()} if message != "" { pvb.Status.Message = message } delete(pvb.Labels, exposer.ExposeOnGoingLabel) }) if err != nil { log.WithError(err).Error("error updating PVB status") return false } else if !succeeded { log.Warn("conflict in updating PVB status and will try it again later") return false } r.exposer.CleanUp(ctx, getPVBOwnerObject(pvb)) log.Warn("PVB is canceled") return true } var funcExclusiveUpdatePodVolumeBackup = exclusiveUpdatePodVolumeBackup func exclusiveUpdatePodVolumeBackup(ctx context.Context, cli client.Client, pvb *velerov1api.PodVolumeBackup, updateFunc func(*velerov1api.PodVolumeBackup)) (bool, error) { updateFunc(pvb) err := cli.Update(ctx, pvb) if err == nil { return true, nil } if apierrors.IsConflict(err) { return false, nil } else { return false, err } } func (r *PodVolumeBackupReconciler) onPrepareTimeout(ctx context.Context, pvb *velerov1api.PodVolumeBackup) { log := r.logger.WithField("PVB", pvb.Name) log.Info("Timeout happened for preparing PVB") succeeded, err := funcExclusiveUpdatePodVolumeBackup(ctx, r.client, pvb, func(pvb *velerov1api.PodVolumeBackup) { pvb.Status.Phase = velerov1api.PodVolumeBackupPhaseFailed pvb.Status.Message = "timeout on preparing PVB" delete(pvb.Labels, exposer.ExposeOnGoingLabel) }) if err != nil { log.WithError(err).Warn("Failed to update PVB") return } if !succeeded { log.Warn("PVB has been updated by others") return } diags := strings.Split(r.exposer.DiagnoseExpose(ctx, getPVBOwnerObject(pvb)), "\n") for _, diag := range diags { log.Warnf("[Diagnose PVB expose]%s", diag) } r.exposer.CleanUp(ctx, getPVBOwnerObject(pvb)) log.Info("PVB has been cleaned up") } func (r *PodVolumeBackupReconciler) initCancelableDataPath(ctx context.Context, asyncBR datapath.AsyncBR, res *exposer.ExposeResult, log logrus.FieldLogger) error { log.Info("Init cancelable PVB") if err := asyncBR.Init(ctx, nil); err != nil { return errors.Wrap(err, "error initializing asyncBR") } log.Infof("async data path init for pod %s, volume %s", res.ByPod.HostingPod.Name, res.ByPod.VolumeName) return nil } func (r *PodVolumeBackupReconciler) startCancelableDataPath(asyncBR datapath.AsyncBR, pvb *velerov1api.PodVolumeBackup, res *exposer.ExposeResult, log logrus.FieldLogger) error { log.Info("Start cancelable PVB") if err := asyncBR.StartBackup(datapath.AccessPoint{ ByPath: res.ByPod.VolumeName, }, pvb.Spec.UploaderSettings, nil); err != nil { return errors.Wrapf(err, "error starting async backup for pod %s, volume %s", res.ByPod.HostingPod.Name, res.ByPod.VolumeName) } log.Infof("Async backup started for pod %s, volume %s", res.ByPod.HostingPod.Name, res.ByPod.VolumeName) return nil } func (r *PodVolumeBackupReconciler) OnDataPathCompleted(ctx context.Context, namespace string, pvbName string, result datapath.Result) { defer r.dataPathMgr.RemoveAsyncBR(pvbName) log := r.logger.WithField("PVB", pvbName) log.WithField("PVB", pvbName).Info("Async fs backup data path completed") pvb := &velerov1api.PodVolumeBackup{} if err := r.client.Get(ctx, types.NamespacedName{Name: pvbName, Namespace: namespace}, pvb); err != nil { log.WithError(err).Warn("Failed to get PVB on completion") return } log.Info("Cleaning up exposed environment") r.exposer.CleanUp(ctx, getPVBOwnerObject(pvb)) // Update status to Completed with path & snapshot ID. var completionTime metav1.Time if err := UpdatePVBWithRetry(ctx, r.client, types.NamespacedName{Namespace: pvb.Namespace, Name: pvb.Name}, log, func(pvb *velerov1api.PodVolumeBackup) bool { completionTime = metav1.Time{Time: r.clock.Now()} if isPVBInFinalState(pvb) { return false } pvb.Status.Path = result.Backup.Source.ByPath pvb.Status.Phase = velerov1api.PodVolumeBackupPhaseCompleted pvb.Status.SnapshotID = result.Backup.SnapshotID pvb.Status.CompletionTimestamp = &completionTime pvb.Status.IncrementalBytes = result.Backup.IncrementalBytes if result.Backup.EmptySnapshot { pvb.Status.Message = "volume was empty so no snapshot was taken" } delete(pvb.Labels, exposer.ExposeOnGoingLabel) return true }); err != nil { log.WithError(err).Error("error updating PVB status") } else { latencyDuration := completionTime.Time.Sub(pvb.Status.StartTimestamp.Time) latencySeconds := float64(latencyDuration / time.Second) backupName := fmt.Sprintf("%s/%s", pvb.Namespace, pvb.OwnerReferences[0].Name) generateOpName := fmt.Sprintf("%s-%s-%s-%s-backup", pvb.Name, pvb.Spec.BackupStorageLocation, pvb.Spec.Pod.Namespace, pvb.Spec.UploaderType) r.metrics.ObservePodVolumeOpLatency(r.nodeName, pvb.Name, generateOpName, backupName, latencySeconds) r.metrics.RegisterPodVolumeOpLatencyGauge(r.nodeName, pvb.Name, generateOpName, backupName, latencySeconds) r.metrics.RegisterPodVolumeBackupDequeue(r.nodeName) log.Info("PVB completed") } } func (r *PodVolumeBackupReconciler) OnDataPathFailed(ctx context.Context, namespace, pvbName string, err error) { defer r.dataPathMgr.RemoveAsyncBR(pvbName) log := r.logger.WithField("PVB", pvbName) log.WithError(err).Error("Async fs backup data path failed") var pvb velerov1api.PodVolumeBackup if getErr := r.client.Get(ctx, types.NamespacedName{Name: pvbName, Namespace: namespace}, &pvb); getErr != nil { log.WithError(getErr).Warn("Failed to get PVB on failure") } else { _, _ = r.errorOut(ctx, &pvb, err, "data path backup failed", log) } } func (r *PodVolumeBackupReconciler) OnDataPathCancelled(ctx context.Context, namespace string, pvbName string) { defer r.dataPathMgr.RemoveAsyncBR(pvbName) log := r.logger.WithField("PVB", pvbName) log.Warn("Async fs backup data path canceled") var pvb velerov1api.PodVolumeBackup if getErr := r.client.Get(ctx, types.NamespacedName{Name: pvbName, Namespace: namespace}, &pvb); getErr != nil { log.WithError(getErr).Warn("Failed to get PVB on cancel") return } // cleans up any objects generated during the snapshot expose r.exposer.CleanUp(ctx, getPVBOwnerObject(&pvb)) if err := UpdatePVBWithRetry(ctx, r.client, types.NamespacedName{Namespace: pvb.Namespace, Name: pvb.Name}, log, func(pvb *velerov1api.PodVolumeBackup) bool { if isPVBInFinalState(pvb) { return false } pvb.Status.Phase = velerov1api.PodVolumeBackupPhaseCanceled if pvb.Status.StartTimestamp.IsZero() { pvb.Status.StartTimestamp = &metav1.Time{Time: r.clock.Now()} } pvb.Status.CompletionTimestamp = &metav1.Time{Time: r.clock.Now()} delete(pvb.Labels, exposer.ExposeOnGoingLabel) return true }); err != nil { log.WithError(err).Error("error updating PVB status on cancel") } else { delete(r.cancelledPVB, pvb.Name) } } func (r *PodVolumeBackupReconciler) OnDataPathProgress(ctx context.Context, namespace string, pvbName string, progress *uploader.Progress) { log := r.logger.WithField("pvb", pvbName) if err := UpdatePVBWithRetry(ctx, r.client, types.NamespacedName{Namespace: namespace, Name: pvbName}, log, func(pvb *velerov1api.PodVolumeBackup) bool { pvb.Status.Progress = veleroapishared.DataMoveOperationProgress{TotalBytes: progress.TotalBytes, BytesDone: progress.BytesDone} return true }); err != nil { log.WithError(err).Error("Failed to update progress") } } // SetupWithManager registers the PVB controller. func (r *PodVolumeBackupReconciler) SetupWithManager(mgr ctrl.Manager) error { gp := kube.NewGenericEventPredicate(func(object client.Object) bool { pvb := object.(*velerov1api.PodVolumeBackup) if _, err := uploader.ValidateUploaderType(pvb.Spec.UploaderType); err != nil { return false } if pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseAccepted { return true } if pvb.Spec.Cancel && !isPVBInFinalState(pvb) { return true } if isPVBInFinalState(pvb) && !pvb.DeletionTimestamp.IsZero() { return true } return false }) s := kube.NewPeriodicalEnqueueSource(r.logger.WithField("controller", constant.ControllerPodVolumeBackup), r.client, &velerov1api.PodVolumeBackupList{}, preparingMonitorFrequency, kube.PeriodicalEnqueueSourceOption{ Predicates: []predicate.Predicate{gp}, }) return ctrl.NewControllerManagedBy(mgr). For(&velerov1api.PodVolumeBackup{}). WatchesRawSource(s). Watches(&corev1api.Pod{}, kube.EnqueueRequestsFromMapUpdateFunc(r.findPVBForPod), builder.WithPredicates(predicate.Funcs{ UpdateFunc: func(ue event.UpdateEvent) bool { newObj := ue.ObjectNew.(*corev1api.Pod) if _, ok := newObj.Labels[velerov1api.PVBLabel]; !ok { return false } if newObj.Spec.NodeName == "" { return false } return true }, CreateFunc: func(event.CreateEvent) bool { return false }, DeleteFunc: func(de event.DeleteEvent) bool { return false }, GenericFunc: func(ge event.GenericEvent) bool { return false }, })). Complete(r) } func (r *PodVolumeBackupReconciler) findPVBForPod(ctx context.Context, podObj client.Object) []reconcile.Request { pod := podObj.(*corev1api.Pod) pvb, err := findPVBByPod(r.client, *pod) log := r.logger.WithField("pod", pod.Name) if err != nil { log.WithError(err).Error("unable to get PVB") return []reconcile.Request{} } else if pvb == nil { log.Error("get empty PVB") return []reconcile.Request{} } log = log.WithFields(logrus.Fields{ "PVB": pvb.Name, }) if pvb.Status.Phase != velerov1api.PodVolumeBackupPhaseAccepted { return []reconcile.Request{} } if pod.Status.Phase == corev1api.PodRunning { log.Info("Preparing PVB") if err = UpdatePVBWithRetry(context.Background(), r.client, types.NamespacedName{Namespace: pvb.Namespace, Name: pvb.Name}, log, func(pvb *velerov1api.PodVolumeBackup) bool { if isPVBInFinalState(pvb) { log.Warnf("PVB %s is terminated, abort setting it to prepared", pvb.Name) return false } pvb.Status.Phase = velerov1api.PodVolumeBackupPhasePrepared return true }); err != nil { log.WithError(err).Warn("Failed to update PVB, prepare will halt for this PVB") return []reconcile.Request{} } } else if unrecoverable, reason := kube.IsPodUnrecoverable(pod, log); unrecoverable { err := UpdatePVBWithRetry(context.Background(), r.client, types.NamespacedName{Namespace: pvb.Namespace, Name: pvb.Name}, log, func(pvb *velerov1api.PodVolumeBackup) bool { if pvb.Spec.Cancel { return false } pvb.Spec.Cancel = true pvb.Status.Message = fmt.Sprintf("Cancel PVB because the exposing pod %s/%s is in abnormal status for reason %s", pod.Namespace, pod.Name, reason) return true }) if err != nil { log.WithError(err).Warn("failed to cancel PVB, and it will wait for prepare timeout") return []reconcile.Request{} } log.Infof("Exposed pod is in abnormal status(reason %s) and PVB is marked as cancel", reason) } else { return []reconcile.Request{} } request := reconcile.Request{ NamespacedName: types.NamespacedName{ Namespace: pvb.Namespace, Name: pvb.Name, }, } return []reconcile.Request{request} } func (r *PodVolumeBackupReconciler) errorOut(ctx context.Context, pvb *velerov1api.PodVolumeBackup, err error, msg string, log logrus.FieldLogger) (ctrl.Result, error) { r.exposer.CleanUp(ctx, getPVBOwnerObject(pvb)) _ = UpdatePVBStatusToFailed(ctx, r.client, pvb, err, msg, r.clock.Now(), log) return ctrl.Result{}, err } func UpdatePVBStatusToFailed(ctx context.Context, c client.Client, pvb *velerov1api.PodVolumeBackup, errOut error, msg string, time time.Time, log logrus.FieldLogger) error { log.Info("update PVB status to Failed") if patchErr := UpdatePVBWithRetry(context.Background(), c, types.NamespacedName{Namespace: pvb.Namespace, Name: pvb.Name}, log, func(pvb *velerov1api.PodVolumeBackup) bool { if isPVBInFinalState(pvb) { return false } pvb.Status.Phase = velerov1api.PodVolumeBackupPhaseFailed pvb.Status.CompletionTimestamp = &metav1.Time{Time: time} if dataPathError, ok := errOut.(datapath.DataPathError); ok { pvb.Status.SnapshotID = dataPathError.GetSnapshotID() } if len(strings.TrimSpace(msg)) == 0 { pvb.Status.Message = errOut.Error() } else { pvb.Status.Message = errors.WithMessage(errOut, msg).Error() } if pvb.Status.StartTimestamp.IsZero() { pvb.Status.StartTimestamp = &metav1.Time{Time: time} } delete(pvb.Labels, exposer.ExposeOnGoingLabel) return true }); patchErr != nil { log.WithError(patchErr).Warn("error updating PVB status") } return errOut } func (r *PodVolumeBackupReconciler) closeDataPath(ctx context.Context, pvbName string) { asyncBR := r.dataPathMgr.GetAsyncBR(pvbName) if asyncBR != nil { asyncBR.Close(ctx) } r.dataPathMgr.RemoveAsyncBR(pvbName) } func (r *PodVolumeBackupReconciler) setupExposeParam(pvb *velerov1api.PodVolumeBackup) exposer.PodVolumeExposeParam { log := r.logger.WithField("PVB", pvb.Name) nodeOS, err := kube.GetNodeOS(context.Background(), pvb.Spec.Node, r.kubeClient.CoreV1()) if err != nil { log.WithError(err).Warnf("Failed to get nodeOS for node %s, use linux node-agent for hosting pod labels, annotations and tolerations", pvb.Spec.Node) } hostingPodLabels := map[string]string{velerov1api.PVBLabel: pvb.Name} if len(r.podLabels) > 0 { for k, v := range r.podLabels { hostingPodLabels[k] = v } } else { for _, k := range util.ThirdPartyLabels { if v, err := nodeagent.GetLabelValue(context.Background(), r.kubeClient, pvb.Namespace, k, nodeOS); err != nil { if err != nodeagent.ErrNodeAgentLabelNotFound { log.WithError(err).Warnf("Failed to check node-agent label, skip adding host pod label %s", k) } } else { hostingPodLabels[k] = v } } } hostingPodAnnotation := map[string]string{} if len(r.podAnnotations) > 0 { for k, v := range r.podAnnotations { hostingPodAnnotation[k] = v } } else { for _, k := range util.ThirdPartyAnnotations { if v, err := nodeagent.GetAnnotationValue(context.Background(), r.kubeClient, pvb.Namespace, k, nodeOS); err != nil { if err != nodeagent.ErrNodeAgentAnnotationNotFound { log.WithError(err).Warnf("Failed to check node-agent annotation, skip adding host pod annotation %s", k) } } else { hostingPodAnnotation[k] = v } } } hostingPodTolerations := []corev1api.Toleration{} for _, k := range util.ThirdPartyTolerations { if v, err := nodeagent.GetToleration(context.Background(), r.kubeClient, pvb.Namespace, k, nodeOS); err != nil { if err != nodeagent.ErrNodeAgentTolerationNotFound { log.WithError(err).Warnf("Failed to check node-agent toleration, skip adding host pod toleration %s", k) } } else { hostingPodTolerations = append(hostingPodTolerations, *v) } } return exposer.PodVolumeExposeParam{ Type: exposer.PodVolumeExposeTypeBackup, ClientNamespace: pvb.Spec.Pod.Namespace, ClientPodName: pvb.Spec.Pod.Name, ClientPodVolume: pvb.Spec.Volume, HostingPodLabels: hostingPodLabels, HostingPodAnnotations: hostingPodAnnotation, HostingPodTolerations: hostingPodTolerations, OperationTimeout: r.resourceTimeout, Resources: r.podResources, // Priority class name for the data mover pod, retrieved from node-agent-configmap PriorityClassName: r.dataMovePriorityClass, Privileged: r.privileged, } } func getPVBOwnerObject(pvb *velerov1api.PodVolumeBackup) corev1api.ObjectReference { return corev1api.ObjectReference{ Kind: pvb.Kind, Namespace: pvb.Namespace, Name: pvb.Name, UID: pvb.UID, APIVersion: pvb.APIVersion, } } func findPVBByPod(client client.Client, pod corev1api.Pod) (*velerov1api.PodVolumeBackup, error) { if label, exist := pod.Labels[velerov1api.PVBLabel]; exist { pvb := &velerov1api.PodVolumeBackup{} err := client.Get(context.Background(), types.NamespacedName{ Namespace: pod.Namespace, Name: label, }, pvb) if err != nil { return nil, errors.Wrapf(err, "error to find PVB by pod %s/%s", pod.Namespace, pod.Name) } return pvb, nil } return nil, nil } func isPVBInFinalState(pvb *velerov1api.PodVolumeBackup) bool { return pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseFailed || pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseCanceled || pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseCompleted } func UpdatePVBWithRetry(ctx context.Context, client client.Client, namespacedName types.NamespacedName, log logrus.FieldLogger, updateFunc func(*velerov1api.PodVolumeBackup) bool) error { return wait.PollUntilContextCancel(ctx, time.Millisecond*100, true, func(ctx context.Context) (bool, error) { pvb := &velerov1api.PodVolumeBackup{} if err := client.Get(ctx, namespacedName, pvb); err != nil { return false, errors.Wrap(err, "getting PVB") } if updateFunc(pvb) { err := client.Update(ctx, pvb) if err != nil { if apierrors.IsConflict(err) { log.Debugf("failed to update PVB for %s/%s and will retry it", pvb.Namespace, pvb.Name) return false, nil } else { return false, errors.Wrapf(err, "error updating PVB with error %s/%s", pvb.Namespace, pvb.Name) } } } return true, nil }) } var funcResumeCancellablePVB = (*PodVolumeBackupReconciler).resumeCancellableDataPath func (r *PodVolumeBackupReconciler) AttemptPVBResume(ctx context.Context, logger *logrus.Entry, ns string) error { pvbs := &velerov1api.PodVolumeBackupList{} if err := r.client.List(ctx, pvbs, &client.ListOptions{Namespace: ns}); err != nil { r.logger.WithError(errors.WithStack(err)).Error("failed to list PVBs") return errors.Wrapf(err, "error to list PVBs") } for i := range pvbs.Items { pvb := &pvbs.Items[i] if pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseInProgress { if pvb.Spec.Node != r.nodeName { logger.WithField("PVB", pvb.Name).WithField("current node", r.nodeName).Infof("PVB should be resumed by another node %s", pvb.Spec.Node) continue } err := funcResumeCancellablePVB(r, ctx, pvb, logger) if err == nil { logger.WithField("PVB", pvb.Name).WithField("current node", r.nodeName).Info("Completed to resume in progress PVB") continue } logger.WithField("PVB", pvb.GetName()).WithError(err).Warn("Failed to resume data path for PVB, have to cancel it") resumeErr := err err = UpdatePVBWithRetry(ctx, r.client, types.NamespacedName{Namespace: pvb.Namespace, Name: pvb.Name}, logger.WithField("PVB", pvb.Name), func(pvb *velerov1api.PodVolumeBackup) bool { if pvb.Spec.Cancel { return false } pvb.Spec.Cancel = true pvb.Status.Message = fmt.Sprintf("Resume InProgress PVB failed with error %v, mark it as cancel", resumeErr) return true }) if err != nil { logger.WithField("PVB", pvb.GetName()).WithError(errors.WithStack(err)).Error("Failed to trigger PVB cancel") } } else if !isPVBInFinalState(pvb) { logger.WithField("PVB", pvb.GetName()).Infof("find a PVB with status %s", pvb.Status.Phase) } } return nil } func (r *PodVolumeBackupReconciler) resumeCancellableDataPath(ctx context.Context, pvb *velerov1api.PodVolumeBackup, log logrus.FieldLogger) error { log.Info("Resume cancelable PVB") res, err := r.exposer.GetExposed(ctx, getPVBOwnerObject(pvb), r.client, r.nodeName, r.resourceTimeout) if err != nil { return errors.Wrapf(err, "error to get exposed PVB %s", pvb.Name) } if res == nil { return errors.Errorf("no expose result is available for the current node for PVB %s", pvb.Name) } callbacks := datapath.Callbacks{ OnCompleted: r.OnDataPathCompleted, OnFailed: r.OnDataPathFailed, OnCancelled: r.OnDataPathCancelled, OnProgress: r.OnDataPathProgress, } asyncBR, err := r.dataPathMgr.CreateMicroServiceBRWatcher(ctx, r.client, r.kubeClient, r.mgr, datapath.TaskTypeBackup, pvb.Name, pvb.Namespace, res.ByPod.HostingPod.Name, res.ByPod.HostingContainer, pvb.Name, callbacks, true, log) if err != nil { return errors.Wrapf(err, "error to create asyncBR watcher for PVB %s", pvb.Name) } resumeComplete := false defer func() { if !resumeComplete { r.closeDataPath(ctx, pvb.Name) } }() if err := asyncBR.Init(ctx, nil); err != nil { return errors.Wrapf(err, "error to init asyncBR watcher for PVB %s", pvb.Name) } if err := asyncBR.StartBackup(datapath.AccessPoint{ ByPath: res.ByPod.VolumeName, }, pvb.Spec.UploaderSettings, nil); err != nil { return errors.Wrapf(err, "error to resume asyncBR watcher for PVB %s", pvb.Name) } resumeComplete = true log.Infof("asyncBR is resumed for PVB %s", pvb.Name) return nil } ================================================ FILE: pkg/controller/pod_volume_backup_controller_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package controller import ( "context" "fmt" "testing" "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" appsv1api "k8s.io/api/apps/v1" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" clientgofake "k8s.io/client-go/kubernetes/fake" "k8s.io/utils/clock" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" kbclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/datapath" datapathmocks "github.com/vmware-tanzu/velero/pkg/datapath/mocks" "github.com/vmware-tanzu/velero/pkg/exposer" "github.com/vmware-tanzu/velero/pkg/metrics" velerotest "github.com/vmware-tanzu/velero/pkg/test" "github.com/vmware-tanzu/velero/pkg/uploader" "github.com/vmware-tanzu/velero/pkg/util/kube" ) const pvbName = "pvb-1" func initPVBReconciler(needError ...bool) (*PodVolumeBackupReconciler, error) { var errs = make([]error, 6) for k, isError := range needError { if k == 0 && isError { errs[0] = fmt.Errorf("Get error") } else if k == 1 && isError { errs[1] = fmt.Errorf("Create error") } else if k == 2 && isError { errs[2] = fmt.Errorf("Update error") } else if k == 3 && isError { errs[3] = fmt.Errorf("Patch error") } else if k == 4 && isError { errs[4] = apierrors.NewConflict(velerov1api.Resource("podvolumebackup"), pvbName, errors.New("conflict")) } else if k == 5 && isError { errs[5] = fmt.Errorf("List error") } } return initPVBReconcilerWithError(errs...) } func initPVBReconcilerWithError(needError ...error) (*PodVolumeBackupReconciler, error) { daemonSet := &appsv1api.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", Name: "node-agent", }, TypeMeta: metav1.TypeMeta{ Kind: "DaemonSet", APIVersion: appsv1api.SchemeGroupVersion.String(), }, Spec: appsv1api.DaemonSetSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ Containers: []corev1api.Container{ { Image: "fake-image", }, }, }, }, }, } node := builder.ForNode("fake-node").Labels(map[string]string{kube.NodeOSLabel: kube.NodeOSLinux}).Result() dataPathMgr := datapath.NewManager(1) scheme := runtime.NewScheme() err := velerov1api.AddToScheme(scheme) if err != nil { return nil, err } err = corev1api.AddToScheme(scheme) if err != nil { return nil, err } fakeClient := &FakeClient{ Client: fake.NewClientBuilder().WithScheme(scheme).Build(), } for k := range needError { if k == 0 { fakeClient.getError = needError[0] } else if k == 1 { fakeClient.createError = needError[1] } else if k == 2 { fakeClient.updateError = needError[2] } else if k == 3 { fakeClient.patchError = needError[3] } else if k == 4 { fakeClient.updateConflict = needError[4] } else if k == 5 { fakeClient.listError = needError[5] } } fakeKubeClient := clientgofake.NewSimpleClientset(daemonSet, node) return NewPodVolumeBackupReconciler( fakeClient, nil, fakeKubeClient, dataPathMgr, nil, "test-node", time.Minute*5, time.Minute, corev1api.ResourceRequirements{}, metrics.NewServerMetrics(), velerotest.NewLogger(), "", // dataMovePriorityClass false, // privileged nil, // podLabels nil, // podAnnotations ), nil } func pvbBuilder() *builder.PodVolumeBackupBuilder { return builder.ForPodVolumeBackup(velerov1api.DefaultNamespace, pvbName).BackupStorageLocation("bsl-loc") } type fakePvbExposer struct { kubeClient client.Client clock clock.WithTickerAndDelayedExecution peekErr error exposeErr error getErr error getNil bool } func (f *fakePvbExposer) Expose(ctx context.Context, ownerObject corev1api.ObjectReference, param exposer.PodVolumeExposeParam) error { if f.exposeErr != nil { return f.exposeErr } return nil } func (f *fakePvbExposer) GetExposed(context.Context, corev1api.ObjectReference, client.Client, string, time.Duration) (*exposer.ExposeResult, error) { if f.getErr != nil { return nil, f.getErr } if f.getNil { return nil, nil } pod := &corev1api.Pod{} nodeOS := "linux" pNodeOS := &nodeOS return &exposer.ExposeResult{ByPod: exposer.ExposeByPod{HostingPod: pod, VolumeName: pvbName, NodeOS: pNodeOS}}, nil } func (f *fakePvbExposer) PeekExposed(ctx context.Context, ownerObject corev1api.ObjectReference) error { return f.peekErr } func (f *fakePvbExposer) DiagnoseExpose(context.Context, corev1api.ObjectReference) string { return "" } func (f *fakePvbExposer) CleanUp(context.Context, corev1api.ObjectReference) { } func TestPVBReconcile(t *testing.T) { tests := []struct { name string pvb *velerov1api.PodVolumeBackup notCreatePvb bool needDelete bool sportTime *metav1.Time pod *corev1api.Pod dataMgr *datapath.Manager needCreateFSBR bool needExclusiveUpdateError error needMockExposer bool expected *velerov1api.PodVolumeBackup expectDeleted bool expectCancelRecord bool needErrs []bool peekErr error exposeErr error getExposeErr error getExposeNil bool fsBRInitErr error fsBRStartErr error constrained bool expectedErr string expectedResult *ctrl.Result expectDataPath bool }{ { name: "pvb not found", pvb: pvbBuilder().Result(), notCreatePvb: true, }, { name: "pvb not created in velero default namespace", pvb: builder.ForPodVolumeBackup("test-ns", pvbName).Result(), }, { name: "get pvb fail", pvb: pvbBuilder().Result(), needErrs: []bool{true, false, false, false}, expectedErr: "getting PVB: Get error", }, { name: "add finalizer to pvb", pvb: pvbBuilder().Result(), expected: pvbBuilder().Finalizers([]string{PodVolumeFinalizer}).Result(), }, { name: "add finalizer to pvb failed", pvb: pvbBuilder().Result(), needErrs: []bool{false, false, true, false}, expectedErr: "error updating PVB with error velero/pvb-1: Update error", }, { name: "pvb is under deletion", pvb: pvbBuilder().Finalizers([]string{PodVolumeFinalizer}).Result(), needDelete: true, expected: pvbBuilder().Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Result(), }, { name: "pvb is under deletion but cancel failed", pvb: pvbBuilder().Finalizers([]string{PodVolumeFinalizer}).Result(), needErrs: []bool{false, false, true, false}, needDelete: true, expectedErr: "error updating PVB with error velero/pvb-1: Update error", }, { name: "pvb is under deletion and in terminal state", pvb: pvbBuilder().Finalizers([]string{PodVolumeFinalizer}).Phase(velerov1api.PodVolumeBackupPhaseFailed).Result(), sportTime: &metav1.Time{Time: time.Now()}, needDelete: true, expectDeleted: true, }, { name: "pvb is under deletion and in terminal state, but remove finalizer failed", pvb: pvbBuilder().Finalizers([]string{PodVolumeFinalizer}).Phase(velerov1api.PodVolumeBackupPhaseFailed).Result(), needErrs: []bool{false, false, true, false}, needDelete: true, expectedErr: "error updating PVB with error velero/pvb-1: Update error", }, { name: "delay cancel negative for others", pvb: pvbBuilder().Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeBackupPhasePrepared).Result(), sportTime: &metav1.Time{Time: time.Now()}, expectCancelRecord: true, }, { name: "delay cancel negative for inProgress", pvb: pvbBuilder().Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeBackupPhaseInProgress).Result(), sportTime: &metav1.Time{Time: time.Now().Add(-time.Minute * 58)}, expectCancelRecord: true, }, { name: "delay cancel affirmative for others", pvb: pvbBuilder().Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeBackupPhasePrepared).Result(), sportTime: &metav1.Time{Time: time.Now().Add(-time.Minute * 5)}, expected: pvbBuilder().Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeBackupPhaseCanceled).Result(), }, { name: "delay cancel affirmative for inProgress", pvb: pvbBuilder().Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeBackupPhaseInProgress).Result(), sportTime: &metav1.Time{Time: time.Now().Add(-time.Hour)}, expected: pvbBuilder().Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeBackupPhaseCanceled).Result(), }, { name: "delay cancel failed", pvb: pvbBuilder().Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeBackupPhaseInProgress).Result(), needErrs: []bool{false, false, true, false}, sportTime: &metav1.Time{Time: time.Now().Add(-time.Hour)}, expected: pvbBuilder().Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeBackupPhaseInProgress).Result(), expectCancelRecord: true, }, { name: "Unknown pvb status", pvb: pvbBuilder().Phase("Unknown").Finalizers([]string{PodVolumeFinalizer}).Result(), }, { name: "new pvb but constrained", pvb: pvbBuilder().Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), constrained: true, expected: pvbBuilder().Finalizers([]string{PodVolumeFinalizer}).Result(), expectedResult: &ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, }, { name: "new pvb but accept failed", pvb: pvbBuilder().Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), needErrs: []bool{false, false, true, false}, expected: pvbBuilder().Finalizers([]string{PodVolumeFinalizer}).Result(), expectedErr: "error accepting PVB pvb-1: error updating PVB with error velero/pvb-1: Update error", }, { name: "pvb is cancel on accepted", pvb: pvbBuilder().Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Cancel(true).Result(), expected: pvbBuilder().Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeBackupPhaseCanceled).Result(), expectCancelRecord: true, }, { name: "pvb expose failed", pvb: pvbBuilder().Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), needMockExposer: true, exposeErr: errors.New("fake-expose-error"), expected: pvbBuilder().Finalizers([]string{PodVolumeFinalizer}).Phase(velerov1api.PodVolumeBackupPhaseFailed).Message("error to expose PVB").Result(), expectedErr: "fake-expose-error", }, { name: "pvb succeeds for accepted", pvb: pvbBuilder().Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), needMockExposer: true, expected: pvbBuilder().Finalizers([]string{PodVolumeFinalizer}).Phase(velerov1api.PodVolumeBackupPhaseAccepted).Result(), }, { name: "prepare timeout on accepted", pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseAccepted).Finalizers([]string{PodVolumeFinalizer}).AcceptedTimestamp(&metav1.Time{Time: time.Now().Add(-time.Minute * 30)}).Result(), expected: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseFailed).Finalizers([]string{PodVolumeFinalizer}).Phase(velerov1api.PodVolumeBackupPhaseFailed).Message("timeout on preparing PVB").Result(), }, { name: "peek error on accepted", pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseAccepted).Finalizers([]string{PodVolumeFinalizer}).Result(), needMockExposer: true, peekErr: errors.New("fake-peak-error"), expected: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseCanceled).Finalizers([]string{PodVolumeFinalizer}).Phase(velerov1api.PodVolumeBackupPhaseCanceled).Message("found a PVB velero/pvb-1 with expose error: fake-peak-error. mark it as cancel").Result(), }, { name: "cancel on prepared", pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhasePrepared).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Cancel(true).Result(), expected: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseCanceled).Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeBackupPhaseCanceled).Result(), }, { name: "Failed to get pvb expose on prepared", pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhasePrepared).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), needMockExposer: true, getExposeErr: errors.New("fake-get-error"), expected: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseFailed).Finalizers([]string{PodVolumeFinalizer}).Message("exposed PVB is not ready: fake-get-error").Result(), expectedErr: "fake-get-error", }, { name: "Get nil restore expose on prepared", pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhasePrepared).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), needMockExposer: true, getExposeNil: true, expected: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseFailed).Finalizers([]string{PodVolumeFinalizer}).Message("exposed PVB is not ready").Result(), expectedErr: "no expose result is available for the current node", }, { name: "Error in data path is concurrent limited", pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhasePrepared).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), needMockExposer: true, dataMgr: datapath.NewManager(0), expectedResult: &ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, }, { name: "data path init error", pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhasePrepared).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), needMockExposer: true, fsBRInitErr: errors.New("fake-data-path-init-error"), expected: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseFailed).Finalizers([]string{PodVolumeFinalizer}).Message("error initializing data path").Result(), expectedErr: "error initializing asyncBR: fake-data-path-init-error", }, { name: "Unable to update status to in progress for data upload", pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhasePrepared).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), needMockExposer: true, needErrs: []bool{false, false, true, false}, expected: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhasePrepared).Finalizers([]string{PodVolumeFinalizer}).Result(), expectedResult: &ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, }, { name: "data path start error", pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhasePrepared).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), needMockExposer: true, fsBRStartErr: errors.New("fake-data-path-start-error"), expected: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseFailed).Finalizers([]string{PodVolumeFinalizer}).Message("error starting data path").Result(), expectedErr: "error starting async backup for pod , volume pvb-1: fake-data-path-start-error", }, { name: "Prepare succeeds", pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhasePrepared).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), needMockExposer: true, expected: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseInProgress).Finalizers([]string{PodVolumeFinalizer}).Result(), expectDataPath: true, }, { name: "In progress pvb is not handled by the current node", pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseInProgress).Finalizers([]string{PodVolumeFinalizer}).Result(), expected: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseInProgress).Finalizers([]string{PodVolumeFinalizer}).Result(), }, { name: "In progress pvb is not set as cancel", pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseInProgress).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), expected: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseInProgress).Finalizers([]string{PodVolumeFinalizer}).Result(), }, { name: "Cancel pvb in progress with empty FSBR", pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseInProgress).Cancel(true).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), expected: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseCanceled).Cancel(true).Finalizers([]string{PodVolumeFinalizer}).Result(), }, { name: "Cancel pvb in progress and patch pvb error", pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseInProgress).Cancel(true).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), needErrs: []bool{false, false, true, false}, needCreateFSBR: true, expected: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseInProgress).Cancel(true).Finalizers([]string{PodVolumeFinalizer}).Result(), expectedErr: "error updating PVB with error velero/pvb-1: Update error", expectCancelRecord: true, expectDataPath: true, }, { name: "Cancel pvb in progress succeeds", pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseInProgress).Cancel(true).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), needCreateFSBR: true, expected: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseCanceling).Cancel(true).Finalizers([]string{PodVolumeFinalizer}).Result(), expectDataPath: true, expectCancelRecord: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { r, err := initPVBReconciler(test.needErrs...) require.NoError(t, err) if !test.notCreatePvb { err = r.client.Create(t.Context(), test.pvb) require.NoError(t, err) } if test.needDelete { err = r.client.Delete(t.Context(), test.pvb) require.NoError(t, err) } if test.pod != nil { err = r.client.Create(ctx, test.pod) require.NoError(t, err) } if test.dataMgr != nil { r.dataPathMgr = test.dataMgr } else { r.dataPathMgr = datapath.NewManager(1) } if test.sportTime != nil { r.cancelledPVB[test.pvb.Name] = test.sportTime.Time } if test.constrained { r.vgdpCounter = &exposer.VgdpCounter{} } if test.needMockExposer { r.exposer = &fakePvbExposer{r.client, r.clock, test.peekErr, test.exposeErr, test.getExposeErr, test.getExposeNil} } funcExclusiveUpdatePodVolumeBackup = exclusiveUpdatePodVolumeBackup if test.needExclusiveUpdateError != nil { funcExclusiveUpdatePodVolumeBackup = func(context.Context, client.Client, *velerov1api.PodVolumeBackup, func(*velerov1api.PodVolumeBackup)) (bool, error) { return false, test.needExclusiveUpdateError } } datapath.MicroServiceBRWatcherCreator = func(client.Client, kubernetes.Interface, manager.Manager, string, string, string, string, string, string, datapath.Callbacks, logrus.FieldLogger) datapath.AsyncBR { return &fakeFSBR{ kubeClient: r.client, clock: r.clock, initErr: test.fsBRInitErr, startErr: test.fsBRStartErr, } } if test.needCreateFSBR { if fsBR := r.dataPathMgr.GetAsyncBR(test.pvb.Name); fsBR == nil { _, err := r.dataPathMgr.CreateMicroServiceBRWatcher(ctx, r.client, nil, nil, datapath.TaskTypeBackup, test.pvb.Name, velerov1api.DefaultNamespace, "", "", "", datapath.Callbacks{OnCancelled: r.OnDataPathCancelled}, false, velerotest.NewLogger()) require.NoError(t, err) } } actualResult, err := r.Reconcile(ctx, ctrl.Request{ NamespacedName: types.NamespacedName{ Namespace: velerov1api.DefaultNamespace, Name: test.pvb.Name, }, }) if test.expectedErr != "" { require.EqualError(t, err, test.expectedErr) } else { require.NoError(t, err) } if test.expectedResult != nil { assert.Equal(t, test.expectedResult.Requeue, actualResult.Requeue) assert.Equal(t, test.expectedResult.RequeueAfter, actualResult.RequeueAfter) } pvb := velerov1api.PodVolumeBackup{} err = r.client.Get(ctx, client.ObjectKey{ Name: test.pvb.Name, Namespace: test.pvb.Namespace, }, &pvb) if test.expected != nil || test.expectDeleted { if test.expectDeleted { assert.True(t, apierrors.IsNotFound(err)) } else { require.NoError(t, err) assert.Equal(t, test.expected.Status.Phase, pvb.Status.Phase) assert.Contains(t, pvb.Status.Message, test.expected.Status.Message) assert.Equal(t, pvb.Finalizers, test.expected.Finalizers) assert.Equal(t, pvb.Spec.Cancel, test.expected.Spec.Cancel) } } if !test.expectDataPath { assert.Nil(t, r.dataPathMgr.GetAsyncBR(test.pvb.Name)) } else { assert.NotNil(t, r.dataPathMgr.GetAsyncBR(test.pvb.Name)) } if test.expectCancelRecord { assert.Contains(t, r.cancelledPVB, test.pvb.Name) } else { assert.Empty(t, r.cancelledPVB) } if isPVBInFinalState(&pvb) || pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseInProgress { assert.NotContains(t, pvb.Labels, exposer.ExposeOnGoingLabel) } else if pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseAccepted { assert.Contains(t, pvb.Labels, exposer.ExposeOnGoingLabel) } }) } } func TestOnPVBCancelled(t *testing.T) { ctx := t.Context() r, err := initPVBReconciler() require.NoError(t, err) pvb := pvbBuilder().Result() namespace := pvb.Namespace pvbName := pvb.Name require.NoError(t, r.client.Create(ctx, pvb)) r.OnDataPathCancelled(ctx, namespace, pvbName) updatedPvb := &velerov1api.PodVolumeBackup{} require.NoError(t, r.client.Get(ctx, types.NamespacedName{Name: pvbName, Namespace: namespace}, updatedPvb)) assert.Equal(t, velerov1api.PodVolumeBackupPhaseCanceled, updatedPvb.Status.Phase) assert.False(t, updatedPvb.Status.CompletionTimestamp.IsZero()) assert.False(t, updatedPvb.Status.StartTimestamp.IsZero()) } func TestOnPVBProgress(t *testing.T) { totalBytes := int64(1024) bytesDone := int64(512) tests := []struct { name string pvb *velerov1api.PodVolumeBackup progress uploader.Progress needErrs []bool }{ { name: "patch in progress phase success", pvb: pvbBuilder().Result(), progress: uploader.Progress{ TotalBytes: totalBytes, BytesDone: bytesDone, }, }, { name: "failed to get pvb", pvb: pvbBuilder().Result(), needErrs: []bool{true, false, false, false}, }, { name: "failed to patch pvb", pvb: pvbBuilder().Result(), needErrs: []bool{false, false, true, false}, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { ctx := t.Context() r, err := initPVBReconciler(test.needErrs...) require.NoError(t, err) defer func() { r.client.Delete(ctx, test.pvb, &client.DeleteOptions{}) }() pvb := pvbBuilder().Result() namespace := pvb.Namespace pvbName := pvb.Name require.NoError(t, r.client.Create(t.Context(), pvb)) // Create a Progress object progress := &uploader.Progress{ TotalBytes: totalBytes, BytesDone: bytesDone, } r.OnDataPathProgress(ctx, namespace, pvbName, progress) if len(test.needErrs) != 0 && !test.needErrs[0] { updatedPvb := &velerov1api.PodVolumeBackup{} require.NoError(t, r.client.Get(ctx, types.NamespacedName{Name: pvbName, Namespace: namespace}, updatedPvb)) assert.Equal(t, test.progress.TotalBytes, updatedPvb.Status.Progress.TotalBytes) assert.Equal(t, test.progress.BytesDone, updatedPvb.Status.Progress.BytesDone) } }) } } func TestOnPvbFailed(t *testing.T) { ctx := t.Context() r, err := initPVBReconciler() require.NoError(t, err) pvb := pvbBuilder().Result() namespace := pvb.Namespace pvbName := pvb.Name require.NoError(t, r.client.Create(ctx, pvb)) r.OnDataPathFailed(ctx, namespace, pvbName, fmt.Errorf("Failed to handle %v", pvbName)) updatedPvb := &velerov1api.PodVolumeBackup{} require.NoError(t, r.client.Get(ctx, types.NamespacedName{Name: pvbName, Namespace: namespace}, updatedPvb)) assert.Equal(t, velerov1api.PodVolumeBackupPhaseFailed, updatedPvb.Status.Phase) assert.False(t, updatedPvb.Status.CompletionTimestamp.IsZero()) assert.False(t, updatedPvb.Status.StartTimestamp.IsZero()) } func TestOnPvbCompleted(t *testing.T) { ctx := t.Context() r, err := initPVBReconciler() require.NoError(t, err) now := time.Now() pvb := pvbBuilder().StartTimestamp(&metav1.Time{Time: now.Add(-time.Minute)}).CompletionTimestamp(&metav1.Time{Time: now}).OwnerReference(metav1.OwnerReference{Name: "test-backup"}).Result() namespace := pvb.Namespace pvbName := pvb.Name require.NoError(t, r.client.Create(ctx, pvb)) r.OnDataPathCompleted(ctx, namespace, pvbName, datapath.Result{}) updatedPvb := &velerov1api.PodVolumeBackup{} require.NoError(t, r.client.Get(ctx, types.NamespacedName{Name: pvbName, Namespace: namespace}, updatedPvb)) assert.Equal(t, velerov1api.PodVolumeBackupPhaseCompleted, updatedPvb.Status.Phase) assert.False(t, updatedPvb.Status.CompletionTimestamp.IsZero()) } func TestFindPvbForPod(t *testing.T) { r, err := initPVBReconciler() require.NoError(t, err) tests := []struct { name string pvb *velerov1api.PodVolumeBackup pod *corev1api.Pod checkFunc func(*velerov1api.PodVolumeBackup, []reconcile.Request) }{ { name: "find pvb for pod", pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseAccepted).Result(), pod: builder.ForPod(velerov1api.DefaultNamespace, pvbName).Labels(map[string]string{velerov1api.PVBLabel: pvbName}).Status(corev1api.PodStatus{Phase: corev1api.PodRunning}).Result(), checkFunc: func(pvb *velerov1api.PodVolumeBackup, requests []reconcile.Request) { // Assert that the function returns a single request assert.Len(t, requests, 1) // Assert that the request contains the correct namespaced name assert.Equal(t, pvb.Namespace, requests[0].Namespace) assert.Equal(t, pvb.Name, requests[0].Name) }, }, { name: "no selected label found for pod", pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseAccepted).Result(), pod: builder.ForPod(velerov1api.DefaultNamespace, pvbName).Result(), checkFunc: func(pvb *velerov1api.PodVolumeBackup, requests []reconcile.Request) { // Assert that the function returns a single request assert.Empty(t, requests) }, }, { name: "no matched pod", pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseAccepted).Result(), pod: builder.ForPod(velerov1api.DefaultNamespace, pvbName).Labels(map[string]string{velerov1api.PVBLabel: "non-existing-pvb"}).Result(), checkFunc: func(pvb *velerov1api.PodVolumeBackup, requests []reconcile.Request) { assert.Empty(t, requests) }, }, { name: "pvb not accepte", pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseInProgress).Result(), pod: builder.ForPod(velerov1api.DefaultNamespace, pvbName).Labels(map[string]string{velerov1api.PVBLabel: pvbName}).Result(), checkFunc: func(pvb *velerov1api.PodVolumeBackup, requests []reconcile.Request) { assert.Empty(t, requests) }, }, } for _, test := range tests { ctx := t.Context() assert.NoError(t, r.client.Create(ctx, test.pod)) assert.NoError(t, r.client.Create(ctx, test.pvb)) requests := r.findPVBForPod(t.Context(), test.pod) test.checkFunc(test.pvb, requests) r.client.Delete(ctx, test.pvb, &client.DeleteOptions{}) if test.pod != nil { r.client.Delete(ctx, test.pod, &client.DeleteOptions{}) } } } func TestAcceptPvb(t *testing.T) { tests := []struct { name string pvb *velerov1api.PodVolumeBackup needErrs []error expectedErr string }{ { name: "update fail", pvb: pvbBuilder().Node("test-node").Result(), needErrs: []error{nil, nil, fmt.Errorf("fake-update-error"), nil}, expectedErr: "error updating PVB with error velero/pvb-1: fake-update-error", }, { name: "succeed", pvb: pvbBuilder().Node("test-node").Result(), needErrs: []error{nil, nil, nil, nil}, }, } for _, test := range tests { ctx := t.Context() r, err := initPVBReconcilerWithError(test.needErrs...) require.NoError(t, err) err = r.client.Create(ctx, test.pvb) require.NoError(t, err) err = r.acceptPodVolumeBackup(ctx, test.pvb) if test.expectedErr == "" { assert.NoError(t, err) } else { assert.EqualError(t, err, test.expectedErr) } } } func TestOnPvbPrepareTimeout(t *testing.T) { tests := []struct { name string pvb *velerov1api.PodVolumeBackup needErrs []error expected *velerov1api.PodVolumeBackup }{ { name: "update fail", pvb: pvbBuilder().Result(), needErrs: []error{nil, nil, fmt.Errorf("fake-update-error"), nil}, expected: pvbBuilder().Result(), }, { name: "update interrupted", pvb: pvbBuilder().Result(), needErrs: []error{nil, nil, &fakeAPIStatus{metav1.StatusReasonConflict}, nil}, expected: pvbBuilder().Result(), }, { name: "succeed", pvb: pvbBuilder().Result(), needErrs: []error{nil, nil, nil, nil}, expected: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseFailed).Result(), }, } for _, test := range tests { ctx := t.Context() r, err := initPVBReconcilerWithError(test.needErrs...) require.NoError(t, err) err = r.client.Create(ctx, test.pvb) require.NoError(t, err) r.onPrepareTimeout(ctx, test.pvb) pvb := velerov1api.PodVolumeBackup{} _ = r.client.Get(ctx, client.ObjectKey{ Name: test.pvb.Name, Namespace: test.pvb.Namespace, }, &pvb) assert.Equal(t, test.expected.Status.Phase, pvb.Status.Phase) } } func TestTryCancelPvb(t *testing.T) { tests := []struct { name string pvb *velerov1api.PodVolumeBackup needErrs []error succeeded bool expectedErr string }{ { name: "update fail", pvb: pvbBuilder().Result(), needErrs: []error{nil, nil, fmt.Errorf("fake-update-error"), nil}, }, { name: "cancel by others", pvb: pvbBuilder().Result(), needErrs: []error{nil, nil, &fakeAPIStatus{metav1.StatusReasonConflict}, nil}, }, { name: "succeed", pvb: pvbBuilder().Result(), needErrs: []error{nil, nil, nil, nil}, succeeded: true, }, } for _, test := range tests { ctx := t.Context() r, err := initPVBReconcilerWithError(test.needErrs...) require.NoError(t, err) err = r.client.Create(ctx, test.pvb) require.NoError(t, err) r.tryCancelPodVolumeBackup(ctx, test.pvb, "") if test.expectedErr == "" { assert.NoError(t, err) } else { assert.EqualError(t, err, test.expectedErr) } } } func TestUpdatePvbWithRetry(t *testing.T) { namespacedName := types.NamespacedName{ Name: pvbName, Namespace: "velero", } // Define test cases testCases := []struct { Name string needErrs []bool noChange bool ExpectErr bool }{ { Name: "SuccessOnFirstAttempt", }, { Name: "Error get", needErrs: []bool{true, false, false, false, false}, ExpectErr: true, }, { Name: "Error update", needErrs: []bool{false, false, true, false, false}, ExpectErr: true, }, { Name: "no change", noChange: true, needErrs: []bool{false, false, true, false, false}, }, { Name: "Conflict with error timeout", needErrs: []bool{false, false, false, false, true}, ExpectErr: true, }, } for _, tc := range testCases { t.Run(tc.Name, func(t *testing.T) { ctx, cancelFunc := context.WithTimeout(t.Context(), time.Second*5) defer cancelFunc() r, err := initPVBReconciler(tc.needErrs...) require.NoError(t, err) err = r.client.Create(ctx, pvbBuilder().Result()) require.NoError(t, err) updateFunc := func(pvb *velerov1api.PodVolumeBackup) bool { if tc.noChange { return false } pvb.Spec.Cancel = true return true } err = UpdatePVBWithRetry(ctx, r.client, namespacedName, velerotest.NewLogger().WithField("name", tc.Name), updateFunc) if tc.ExpectErr { assert.Error(t, err) } else { assert.NoError(t, err) } }) } } type pvbResumeTestHelper struct { resumeErr error getExposeErr error exposeResult *exposer.ExposeResult asyncBR datapath.AsyncBR } func (dt *pvbResumeTestHelper) resumeCancellableDataPath(_ *DataUploadReconciler, _ context.Context, _ *velerov2alpha1api.DataUpload, _ logrus.FieldLogger) error { return dt.resumeErr } func (dt *pvbResumeTestHelper) Expose(context.Context, corev1api.ObjectReference, exposer.PodVolumeExposeParam) error { return nil } func (dt *pvbResumeTestHelper) GetExposed(context.Context, corev1api.ObjectReference, kbclient.Client, string, time.Duration) (*exposer.ExposeResult, error) { return dt.exposeResult, dt.getExposeErr } func (dt *pvbResumeTestHelper) PeekExposed(context.Context, corev1api.ObjectReference) error { return nil } func (dt *pvbResumeTestHelper) DiagnoseExpose(context.Context, corev1api.ObjectReference) string { return "" } func (dt *pvbResumeTestHelper) CleanUp(context.Context, corev1api.ObjectReference) {} func (dt *pvbResumeTestHelper) newMicroServiceBRWatcher(kbclient.Client, kubernetes.Interface, manager.Manager, string, string, string, string, string, string, datapath.Callbacks, logrus.FieldLogger) datapath.AsyncBR { return dt.asyncBR } func TestAttemptPVBResume(t *testing.T) { tests := []struct { name string pvbs []*velerov1api.PodVolumeBackup pvb *velerov1api.PodVolumeBackup needErrs []bool acceptedPvbs []string preparedPvbs []string cancelledPvbs []string inProgressPvbs []string resumeErr error expectedError string }{ { name: "Other pvb", pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhasePrepared).Result(), }, { name: "InProgress pvb, not the current node", pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseInProgress).Result(), inProgressPvbs: []string{pvbName}, }, { name: "InProgress pvb, resume error and update error", pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseInProgress).Node("node-1").Result(), needErrs: []bool{false, false, true, false, false, false}, resumeErr: errors.New("fake-resume-error"), inProgressPvbs: []string{pvbName}, }, { name: "InProgress pvb, resume error and update succeed", pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseInProgress).Node("node-1").Result(), resumeErr: errors.New("fake-resume-error"), cancelledPvbs: []string{pvbName}, inProgressPvbs: []string{pvbName}, }, { name: "InProgress pvb and resume succeed", pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseInProgress).Node("node-1").Result(), inProgressPvbs: []string{pvbName}, }, { name: "Error", needErrs: []bool{false, false, false, false, false, true}, pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhasePrepared).Result(), expectedError: "error to list PVBs: List error", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { ctx := t.Context() r, err := initPVBReconciler(test.needErrs...) r.nodeName = "node-1" require.NoError(t, err) assert.NoError(t, r.client.Create(ctx, test.pvb)) dt := &pvbResumeTestHelper{ resumeErr: test.resumeErr, } funcResumeCancellableDataBackup = dt.resumeCancellableDataPath // Run the test err = r.AttemptPVBResume(ctx, r.logger.WithField("name", test.name), test.pvb.Namespace) if test.expectedError != "" { assert.EqualError(t, err, test.expectedError) } else { assert.NoError(t, err) for _, pvbName := range test.cancelledPvbs { pvb := &velerov1api.PodVolumeBackup{} err := r.client.Get(t.Context(), types.NamespacedName{Namespace: "velero", Name: pvbName}, pvb) require.NoError(t, err) assert.True(t, pvb.Spec.Cancel) } for _, pvbName := range test.acceptedPvbs { pvb := &velerov1api.PodVolumeBackup{} err := r.client.Get(t.Context(), types.NamespacedName{Namespace: "velero", Name: pvbName}, pvb) require.NoError(t, err) assert.Equal(t, velerov1api.PodVolumeBackupPhaseAccepted, pvb.Status.Phase) } for _, pvbName := range test.preparedPvbs { pvb := &velerov1api.PodVolumeBackup{} err := r.client.Get(t.Context(), types.NamespacedName{Namespace: "velero", Name: pvbName}, pvb) require.NoError(t, err) assert.Equal(t, velerov1api.PodVolumeBackupPhasePrepared, pvb.Status.Phase) } for _, pvbName := range test.inProgressPvbs { pvb := &velerov1api.PodVolumeBackup{} err := r.client.Get(t.Context(), types.NamespacedName{Namespace: "velero", Name: pvbName}, pvb) require.NoError(t, err) assert.Equal(t, velerov1api.PodVolumeBackupPhaseInProgress, pvb.Status.Phase) } } }) } } func TestResumeCancellablePodVolumeBackup(t *testing.T) { tests := []struct { name string pvbs []velerov1api.PodVolumeBackup pvb *velerov1api.PodVolumeBackup getExposeErr error exposeResult *exposer.ExposeResult createWatcherErr error initWatcherErr error startWatcherErr error mockInit bool mockStart bool mockClose bool expectedError string }{ { name: "get expose failed", pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseInProgress).Result(), getExposeErr: errors.New("fake-expose-error"), expectedError: fmt.Sprintf("error to get exposed PVB %s: fake-expose-error", pvbName), }, { name: "no expose", pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseAccepted).Node("node-1").Result(), expectedError: fmt.Sprintf("no expose result is available for the current node for PVB %s", pvbName), }, { name: "watcher init error", pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseAccepted).Node("node-1").Result(), exposeResult: &exposer.ExposeResult{ ByPod: exposer.ExposeByPod{ HostingPod: &corev1api.Pod{}, }, }, mockInit: true, mockClose: true, initWatcherErr: errors.New("fake-init-watcher-error"), expectedError: fmt.Sprintf("error to init asyncBR watcher for PVB %s: fake-init-watcher-error", pvbName), }, { name: "start watcher error", pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseAccepted).Node("node-1").Result(), exposeResult: &exposer.ExposeResult{ ByPod: exposer.ExposeByPod{ HostingPod: &corev1api.Pod{}, }, }, mockInit: true, mockStart: true, mockClose: true, startWatcherErr: errors.New("fake-start-watcher-error"), expectedError: fmt.Sprintf("error to resume asyncBR watcher for PVB %s: fake-start-watcher-error", pvbName), }, { name: "succeed", pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseAccepted).Node("node-1").Result(), exposeResult: &exposer.ExposeResult{ ByPod: exposer.ExposeByPod{ HostingPod: &corev1api.Pod{}, }, }, mockInit: true, mockStart: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { ctx := t.Context() r, err := initPVBReconciler() r.nodeName = "node-1" require.NoError(t, err) mockAsyncBR := datapathmocks.NewAsyncBR(t) if test.mockInit { mockAsyncBR.On("Init", mock.Anything, mock.Anything).Return(test.initWatcherErr) } if test.mockStart { mockAsyncBR.On("StartBackup", mock.Anything, mock.Anything, mock.Anything).Return(test.startWatcherErr) } if test.mockClose { mockAsyncBR.On("Close", mock.Anything).Return() } dt := &pvbResumeTestHelper{ getExposeErr: test.getExposeErr, exposeResult: test.exposeResult, asyncBR: mockAsyncBR, } r.exposer = dt datapath.MicroServiceBRWatcherCreator = dt.newMicroServiceBRWatcher err = r.resumeCancellableDataPath(ctx, test.pvb, velerotest.NewLogger()) if test.expectedError != "" { assert.EqualError(t, err, test.expectedError) } }) } } func TestPodVolumeBackupSetupExposeParam(t *testing.T) { // common objects for all cases node := builder.ForNode("worker-1").Labels(map[string]string{kube.NodeOSLabel: kube.NodeOSLinux}).Result() basePVB := pvbBuilder().Result() basePVB.Spec.Node = "worker-1" basePVB.Spec.Pod.Namespace = "app-ns" basePVB.Spec.Pod.Name = "app-pod" basePVB.Spec.Volume = "data-vol" type args struct { customLabels map[string]string customAnnotations map[string]string } type want struct { labels map[string]string annotations map[string]string } tests := []struct { name string args args want want }{ { name: "label has customize values", args: args{ customLabels: map[string]string{"custom-label": "label-value"}, customAnnotations: nil, }, want: want{ labels: map[string]string{ velerov1api.PVBLabel: basePVB.Name, "custom-label": "label-value", }, annotations: map[string]string{}, }, }, { name: "label has no customize values", args: args{ customLabels: nil, customAnnotations: nil, }, want: want{ labels: map[string]string{velerov1api.PVBLabel: basePVB.Name}, annotations: map[string]string{}, }, }, { name: "annotation has customize values", args: args{ customLabels: nil, customAnnotations: map[string]string{"custom-annotation": "annotation-value"}, }, want: want{ labels: map[string]string{velerov1api.PVBLabel: basePVB.Name}, annotations: map[string]string{"custom-annotation": "annotation-value"}, }, }, { name: "annotation has no customize values", args: args{ customLabels: map[string]string{"another-label": "lval"}, customAnnotations: nil, }, want: want{ labels: map[string]string{ velerov1api.PVBLabel: basePVB.Name, "another-label": "lval", }, annotations: map[string]string{}, }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { // Fake clients per case fakeCRClient := velerotest.NewFakeControllerRuntimeClient(t, node, basePVB.DeepCopy()) fakeKubeClient := clientgofake.NewSimpleClientset(node) // Reconciler config per case preparingTimeout := time.Minute * 3 resourceTimeout := time.Minute * 10 podRes := corev1api.ResourceRequirements{} r := NewPodVolumeBackupReconciler( fakeCRClient, nil, fakeKubeClient, datapath.NewManager(1), nil, "test-node", preparingTimeout, resourceTimeout, podRes, metrics.NewServerMetrics(), velerotest.NewLogger(), "backup-priority", true, tt.args.customLabels, tt.args.customAnnotations, ) // Act got := r.setupExposeParam(basePVB) // Core fields assert.Equal(t, exposer.PodVolumeExposeTypeBackup, got.Type) assert.Equal(t, basePVB.Spec.Pod.Namespace, got.ClientNamespace) assert.Equal(t, basePVB.Spec.Pod.Name, got.ClientPodName) assert.Equal(t, basePVB.Spec.Volume, got.ClientPodVolume) // Labels/Annotations assert.Equal(t, tt.want.labels, got.HostingPodLabels) assert.Equal(t, tt.want.annotations, got.HostingPodAnnotations) }) } } ================================================ FILE: pkg/controller/pod_volume_restore_controller.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package controller import ( "context" "fmt" "strings" "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" clocks "k8s.io/utils/clock" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" veleroapishared "github.com/vmware-tanzu/velero/pkg/apis/velero/shared" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/constant" "github.com/vmware-tanzu/velero/pkg/datapath" "github.com/vmware-tanzu/velero/pkg/exposer" "github.com/vmware-tanzu/velero/pkg/nodeagent" repository "github.com/vmware-tanzu/velero/pkg/repository/manager" "github.com/vmware-tanzu/velero/pkg/restorehelper" velerotypes "github.com/vmware-tanzu/velero/pkg/types" "github.com/vmware-tanzu/velero/pkg/uploader" "github.com/vmware-tanzu/velero/pkg/util" "github.com/vmware-tanzu/velero/pkg/util/kube" ) func NewPodVolumeRestoreReconciler( client client.Client, mgr manager.Manager, kubeClient kubernetes.Interface, dataPathMgr *datapath.Manager, counter *exposer.VgdpCounter, nodeName string, preparingTimeout time.Duration, resourceTimeout time.Duration, backupRepoConfigs map[string]string, cacheVolumeConfigs *velerotypes.CachePVC, podResources corev1api.ResourceRequirements, logger logrus.FieldLogger, dataMovePriorityClass string, privileged bool, repoConfigMgr repository.ConfigManager, podLabels map[string]string, podAnnotations map[string]string, ) *PodVolumeRestoreReconciler { return &PodVolumeRestoreReconciler{ client: client, mgr: mgr, kubeClient: kubeClient, logger: logger.WithField("controller", "PodVolumeRestore"), nodeName: nodeName, clock: &clocks.RealClock{}, podResources: podResources, backupRepoConfigs: backupRepoConfigs, cacheVolumeConfigs: cacheVolumeConfigs, dataPathMgr: dataPathMgr, vgdpCounter: counter, preparingTimeout: preparingTimeout, resourceTimeout: resourceTimeout, exposer: exposer.NewPodVolumeExposer(kubeClient, logger), cancelledPVR: make(map[string]time.Time), dataMovePriorityClass: dataMovePriorityClass, privileged: privileged, repoConfigMgr: repoConfigMgr, podLabels: podLabels, podAnnotations: podAnnotations, } } type PodVolumeRestoreReconciler struct { client client.Client mgr manager.Manager kubeClient kubernetes.Interface logger logrus.FieldLogger nodeName string clock clocks.WithTickerAndDelayedExecution podResources corev1api.ResourceRequirements backupRepoConfigs map[string]string cacheVolumeConfigs *velerotypes.CachePVC exposer exposer.PodVolumeExposer dataPathMgr *datapath.Manager vgdpCounter *exposer.VgdpCounter preparingTimeout time.Duration resourceTimeout time.Duration cancelledPVR map[string]time.Time dataMovePriorityClass string privileged bool repoConfigMgr repository.ConfigManager podLabels map[string]string podAnnotations map[string]string } // +kubebuilder:rbac:groups=velero.io,resources=podvolumerestores,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=velero.io,resources=podvolumerestores/status,verbs=get;update;patch // +kubebuilder:rbac:groups="",resources=pods,verbs=get // +kubebuilder:rbac:groups="",resources=persistentvolumes,verbs=get // +kubebuilder:rbac:groups="",resources=persistentvolumerclaims,verbs=get func (r *PodVolumeRestoreReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := r.logger.WithField("PodVolumeRestore", req.NamespacedName.String()) log.Info("Reconciling PVR by advanced controller") pvr := &velerov1api.PodVolumeRestore{} if err := r.client.Get(ctx, types.NamespacedName{Namespace: req.Namespace, Name: req.Name}, pvr); err != nil { if apierrors.IsNotFound(err) { log.Warn("PVR not found, skip") return ctrl.Result{}, nil } log.WithError(err).Error("Unable to get the PVR") return ctrl.Result{}, err } log = log.WithField("pod", fmt.Sprintf("%s/%s", pvr.Spec.Pod.Namespace, pvr.Spec.Pod.Name)) if len(pvr.OwnerReferences) == 1 { log = log.WithField("restore", fmt.Sprintf("%s/%s", pvr.Namespace, pvr.OwnerReferences[0].Name)) } // Logic for clear resources when pvr been deleted if !isPVRInFinalState(pvr) { if !controllerutil.ContainsFinalizer(pvr, PodVolumeFinalizer) { if err := UpdatePVRWithRetry(ctx, r.client, req.NamespacedName, log, func(pvr *velerov1api.PodVolumeRestore) bool { if controllerutil.ContainsFinalizer(pvr, PodVolumeFinalizer) { return false } controllerutil.AddFinalizer(pvr, PodVolumeFinalizer) return true }); err != nil { log.WithError(err).Errorf("failed to add finalizer for PVR %s/%s", pvr.Namespace, pvr.Name) return ctrl.Result{}, err } return ctrl.Result{}, nil } if !pvr.DeletionTimestamp.IsZero() { if !pvr.Spec.Cancel { log.Warnf("Cancel PVR under phase %s because it is being deleted", pvr.Status.Phase) if err := UpdatePVRWithRetry(ctx, r.client, req.NamespacedName, log, func(pvr *velerov1api.PodVolumeRestore) bool { if pvr.Spec.Cancel { return false } pvr.Spec.Cancel = true pvr.Status.Message = "Cancel PVR because it is being deleted" return true }); err != nil { log.WithError(err).Errorf("failed to set cancel flag for PVR %s/%s", pvr.Namespace, pvr.Name) return ctrl.Result{}, err } return ctrl.Result{}, nil } } } else { delete(r.cancelledPVR, pvr.Name) if controllerutil.ContainsFinalizer(pvr, PodVolumeFinalizer) { if err := UpdatePVRWithRetry(ctx, r.client, req.NamespacedName, log, func(pvr *velerov1api.PodVolumeRestore) bool { if !controllerutil.ContainsFinalizer(pvr, PodVolumeFinalizer) { return false } controllerutil.RemoveFinalizer(pvr, PodVolumeFinalizer) return true }); err != nil { log.WithError(err).Error("error to remove finalizer") return ctrl.Result{}, err } return ctrl.Result{}, nil } } if pvr.Spec.Cancel { if spotted, found := r.cancelledPVR[pvr.Name]; !found { r.cancelledPVR[pvr.Name] = r.clock.Now() } else { delay := cancelDelayOthers if pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseInProgress { delay = cancelDelayInProgress } if time.Since(spotted) > delay { log.Infof("PVR %s is canceled in Phase %s but not handled in rasonable time", pvr.GetName(), pvr.Status.Phase) if r.tryCancelPodVolumeRestore(ctx, pvr, "") { delete(r.cancelledPVR, pvr.Name) } return ctrl.Result{}, nil } } } if pvr.Status.Phase == "" || pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseNew { if pvr.Spec.Cancel { log.Infof("PVR %s is canceled in Phase %s", pvr.GetName(), pvr.Status.Phase) _ = r.tryCancelPodVolumeRestore(ctx, pvr, "") return ctrl.Result{}, nil } shouldProcess, pod, err := shouldProcess(ctx, r.client, log, pvr) if err != nil { return ctrl.Result{}, err } if !shouldProcess { return ctrl.Result{}, nil } if r.vgdpCounter != nil && r.vgdpCounter.IsConstrained(ctx, r.logger) { log.Debug("Data path initiation is constrained, requeue later") return ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, nil } log.Info("Accepting PVR") if err := r.acceptPodVolumeRestore(ctx, pvr); err != nil { return ctrl.Result{}, errors.Wrapf(err, "error accepting PVR %s", pvr.Name) } initContainerIndex := getInitContainerIndex(pod) if initContainerIndex > 0 { log.Warnf(`Init containers before the %s container may cause issues if they interfere with volumes being restored: %s index %d`, restorehelper.WaitInitContainer, restorehelper.WaitInitContainer, initContainerIndex) } log.Info("Exposing PVR") exposeParam := r.setupExposeParam(pvr) if err := r.exposer.Expose(ctx, getPVROwnerObject(pvr), exposeParam); err != nil { return r.errorOut(ctx, pvr, err, "error to expose PVR", log) } log.Info("PVR is exposed") return ctrl.Result{}, nil } else if pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseAccepted { if peekErr := r.exposer.PeekExposed(ctx, getPVROwnerObject(pvr)); peekErr != nil { log.Errorf("Cancel PVR %s/%s because of expose error %s", pvr.Namespace, pvr.Name, peekErr) diags := strings.Split(r.exposer.DiagnoseExpose(ctx, getPVROwnerObject(pvr)), "\n") for _, diag := range diags { log.Warnf("[Diagnose PVR expose]%s", diag) } _ = r.tryCancelPodVolumeRestore(ctx, pvr, fmt.Sprintf("found a PVR %s/%s with expose error: %s. mark it as cancel", pvr.Namespace, pvr.Name, peekErr)) } else if pvr.Status.AcceptedTimestamp != nil { if time.Since(pvr.Status.AcceptedTimestamp.Time) >= r.preparingTimeout { r.onPrepareTimeout(ctx, pvr) } } return ctrl.Result{}, nil } else if pvr.Status.Phase == velerov1api.PodVolumeRestorePhasePrepared { log.Infof("PVR is prepared and should be processed by %s (%s)", pvr.Status.Node, r.nodeName) if pvr.Status.Node != r.nodeName { return ctrl.Result{}, nil } if pvr.Spec.Cancel { log.Info("Prepared PVR is being canceled") r.OnDataPathCancelled(ctx, pvr.GetNamespace(), pvr.GetName()) return ctrl.Result{}, nil } asyncBR := r.dataPathMgr.GetAsyncBR(pvr.Name) if asyncBR != nil { log.Info("Cancellable data path is already started") return ctrl.Result{}, nil } res, err := r.exposer.GetExposed(ctx, getPVROwnerObject(pvr), r.client, r.nodeName, r.resourceTimeout) if err != nil { return r.errorOut(ctx, pvr, err, "exposed PVR is not ready", log) } else if res == nil { return r.errorOut(ctx, pvr, errors.New("no expose result is available for the current node"), "exposed PVR is not ready", log) } log.Info("Exposed PVR is ready and creating data path routine") callbacks := datapath.Callbacks{ OnCompleted: r.OnDataPathCompleted, OnFailed: r.OnDataPathFailed, OnCancelled: r.OnDataPathCancelled, OnProgress: r.OnDataPathProgress, } asyncBR, err = r.dataPathMgr.CreateMicroServiceBRWatcher(ctx, r.client, r.kubeClient, r.mgr, datapath.TaskTypeRestore, pvr.Name, pvr.Namespace, res.ByPod.HostingPod.Name, res.ByPod.HostingContainer, pvr.Name, callbacks, false, log) if err != nil { if err == datapath.ConcurrentLimitExceed { log.Debug("Data path instance is concurrent limited requeue later") return ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, nil } else { return r.errorOut(ctx, pvr, err, "error to create data path", log) } } if err := r.initCancelableDataPath(ctx, asyncBR, res, log); err != nil { log.WithError(err).Errorf("Failed to init cancelable data path for %s", pvr.Name) r.closeDataPath(ctx, pvr.Name) return r.errorOut(ctx, pvr, err, "error initializing data path", log) } terminated := false if err := UpdatePVRWithRetry(ctx, r.client, types.NamespacedName{Namespace: pvr.Namespace, Name: pvr.Name}, log, func(pvr *velerov1api.PodVolumeRestore) bool { if isPVRInFinalState(pvr) { terminated = true return false } pvr.Status.Phase = velerov1api.PodVolumeRestorePhaseInProgress pvr.Status.StartTimestamp = &metav1.Time{Time: r.clock.Now()} delete(pvr.Labels, exposer.ExposeOnGoingLabel) return true }); err != nil { log.WithError(err).Warnf("Failed to update PVR %s to InProgress, will data path close and retry", pvr.Name) r.closeDataPath(ctx, pvr.Name) return ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, nil } if terminated { log.Warnf("PVR %s is terminated during transition from prepared", pvr.Name) r.closeDataPath(ctx, pvr.Name) return ctrl.Result{}, nil } log.Info("PVR is marked as in progress") if err := r.startCancelableDataPath(asyncBR, pvr, res, log); err != nil { log.WithError(err).Errorf("Failed to start cancelable data path for %s", pvr.Name) r.closeDataPath(ctx, pvr.Name) return r.errorOut(ctx, pvr, err, "error starting data path", log) } return ctrl.Result{}, nil } else if pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseInProgress { if pvr.Spec.Cancel { if pvr.Status.Node != r.nodeName { return ctrl.Result{}, nil } log.Info("PVR is being canceled") asyncBR := r.dataPathMgr.GetAsyncBR(pvr.Name) if asyncBR == nil { r.OnDataPathCancelled(ctx, pvr.GetNamespace(), pvr.GetName()) return ctrl.Result{}, nil } // Update status to Canceling if err := UpdatePVRWithRetry(ctx, r.client, types.NamespacedName{Namespace: pvr.Namespace, Name: pvr.Name}, log, func(pvr *velerov1api.PodVolumeRestore) bool { if isPVRInFinalState(pvr) { log.Warnf("PVR %s is terminated, abort setting it to canceling", pvr.Name) return false } pvr.Status.Phase = velerov1api.PodVolumeRestorePhaseCanceling return true }); err != nil { log.WithError(err).Error("error updating PVR into canceling status") return ctrl.Result{}, err } asyncBR.Cancel() return ctrl.Result{}, nil } return ctrl.Result{}, nil } return ctrl.Result{}, nil } func (r *PodVolumeRestoreReconciler) acceptPodVolumeRestore(ctx context.Context, pvr *velerov1api.PodVolumeRestore) error { return UpdatePVRWithRetry(ctx, r.client, types.NamespacedName{Namespace: pvr.Namespace, Name: pvr.Name}, r.logger, func(pvr *velerov1api.PodVolumeRestore) bool { pvr.Status.AcceptedTimestamp = &metav1.Time{Time: r.clock.Now()} pvr.Status.Phase = velerov1api.PodVolumeRestorePhaseAccepted pvr.Status.Node = r.nodeName if pvr.Labels == nil { pvr.Labels = make(map[string]string) } pvr.Labels[exposer.ExposeOnGoingLabel] = "true" return true }) } func (r *PodVolumeRestoreReconciler) tryCancelPodVolumeRestore(ctx context.Context, pvr *velerov1api.PodVolumeRestore, message string) bool { log := r.logger.WithField("PVR", pvr.Name) succeeded, err := funcExclusiveUpdatePodVolumeRestore(ctx, r.client, pvr, func(pvr *velerov1api.PodVolumeRestore) { pvr.Status.Phase = velerov1api.PodVolumeRestorePhaseCanceled if pvr.Status.StartTimestamp.IsZero() { pvr.Status.StartTimestamp = &metav1.Time{Time: r.clock.Now()} } pvr.Status.CompletionTimestamp = &metav1.Time{Time: r.clock.Now()} if message != "" { pvr.Status.Message = message } delete(pvr.Labels, exposer.ExposeOnGoingLabel) }) if err != nil { log.WithError(err).Error("error updating PVR status") return false } else if !succeeded { log.Warn("conflict in updating PVR status and will try it again later") return false } r.exposer.CleanUp(ctx, getPVROwnerObject(pvr)) log.Warn("PVR is canceled") return true } var funcExclusiveUpdatePodVolumeRestore = exclusiveUpdatePodVolumeRestore func exclusiveUpdatePodVolumeRestore(ctx context.Context, cli client.Client, pvr *velerov1api.PodVolumeRestore, updateFunc func(*velerov1api.PodVolumeRestore)) (bool, error) { updateFunc(pvr) err := cli.Update(ctx, pvr) if err == nil { return true, nil } // warn we won't rollback pvr values in memory when error if apierrors.IsConflict(err) { return false, nil } else { return false, err } } func (r *PodVolumeRestoreReconciler) onPrepareTimeout(ctx context.Context, pvr *velerov1api.PodVolumeRestore) { log := r.logger.WithField("PVR", pvr.Name) log.Info("Timeout happened for preparing PVR") succeeded, err := funcExclusiveUpdatePodVolumeRestore(ctx, r.client, pvr, func(pvr *velerov1api.PodVolumeRestore) { pvr.Status.Phase = velerov1api.PodVolumeRestorePhaseFailed pvr.Status.Message = "timeout on preparing PVR" delete(pvr.Labels, exposer.ExposeOnGoingLabel) }) if err != nil { log.WithError(err).Warn("Failed to update PVR") return } if !succeeded { log.Warn("PVR has been updated by others") return } diags := strings.Split(r.exposer.DiagnoseExpose(ctx, getPVROwnerObject(pvr)), "\n") for _, diag := range diags { log.Warnf("[Diagnose PVR expose]%s", diag) } r.exposer.CleanUp(ctx, getPVROwnerObject(pvr)) log.Info("PVR has been cleaned up") } func (r *PodVolumeRestoreReconciler) initCancelableDataPath(ctx context.Context, asyncBR datapath.AsyncBR, res *exposer.ExposeResult, log logrus.FieldLogger) error { log.Info("Init cancelable PVR") if err := asyncBR.Init(ctx, nil); err != nil { return errors.Wrap(err, "error initializing asyncBR") } log.Infof("async data path init for pod %s, volume %s", res.ByPod.HostingPod.Name, res.ByPod.VolumeName) return nil } func (r *PodVolumeRestoreReconciler) startCancelableDataPath(asyncBR datapath.AsyncBR, pvr *velerov1api.PodVolumeRestore, res *exposer.ExposeResult, log logrus.FieldLogger) error { log.Info("Start cancelable PVR") if err := asyncBR.StartRestore(pvr.Spec.SnapshotID, datapath.AccessPoint{ ByPath: res.ByPod.VolumeName, }, pvr.Spec.UploaderSettings); err != nil { return errors.Wrapf(err, "error starting async restore for pod %s, volume %s", res.ByPod.HostingPod.Name, res.ByPod.VolumeName) } log.Infof("Async restore started for pod %s, volume %s", res.ByPod.HostingPod.Name, res.ByPod.VolumeName) return nil } func (r *PodVolumeRestoreReconciler) errorOut(ctx context.Context, pvr *velerov1api.PodVolumeRestore, err error, msg string, log logrus.FieldLogger) (ctrl.Result, error) { r.exposer.CleanUp(ctx, getPVROwnerObject(pvr)) return ctrl.Result{}, UpdatePVRStatusToFailed(ctx, r.client, pvr, err, msg, r.clock.Now(), log) } func UpdatePVRStatusToFailed(ctx context.Context, c client.Client, pvr *velerov1api.PodVolumeRestore, err error, msg string, time time.Time, log logrus.FieldLogger) error { log.Info("update PVR status to Failed") if patchErr := UpdatePVRWithRetry(context.Background(), c, types.NamespacedName{Namespace: pvr.Namespace, Name: pvr.Name}, log, func(pvr *velerov1api.PodVolumeRestore) bool { if isPVRInFinalState(pvr) { return false } pvr.Status.Phase = velerov1api.PodVolumeRestorePhaseFailed pvr.Status.Message = errors.WithMessage(err, msg).Error() pvr.Status.CompletionTimestamp = &metav1.Time{Time: time} delete(pvr.Labels, exposer.ExposeOnGoingLabel) return true }); patchErr != nil { log.WithError(patchErr).Warn("error updating PVR status") } return err } func shouldProcess(ctx context.Context, client client.Client, log logrus.FieldLogger, pvr *velerov1api.PodVolumeRestore) (bool, *corev1api.Pod, error) { if !isPVRNew(pvr) { log.Debug("PVR is not new, skip") return false, nil, nil } // we filter the pods during the initialization of cache, if we can get a pod here, the pod must be in the same node with the controller // so we don't need to compare the node anymore pod := &corev1api.Pod{} if err := client.Get(ctx, types.NamespacedName{Namespace: pvr.Spec.Pod.Namespace, Name: pvr.Spec.Pod.Name}, pod); err != nil { if apierrors.IsNotFound(err) { log.WithError(err).Debug("Pod not found on this node, skip") return false, nil, nil } log.WithError(err).Error("Unable to get pod") return false, nil, err } if !isInitContainerRunning(pod) { log.Debug("Pod is not running restore-wait init container, skip") return false, nil, nil } return true, pod, nil } func (r *PodVolumeRestoreReconciler) closeDataPath(ctx context.Context, pvrName string) { asyncBR := r.dataPathMgr.GetAsyncBR(pvrName) if asyncBR != nil { asyncBR.Close(ctx) } r.dataPathMgr.RemoveAsyncBR(pvrName) } func (r *PodVolumeRestoreReconciler) SetupWithManager(mgr ctrl.Manager) error { gp := kube.NewGenericEventPredicate(func(object client.Object) bool { pvr := object.(*velerov1api.PodVolumeRestore) if IsLegacyPVR(pvr) { return false } if pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseAccepted { return true } if pvr.Spec.Cancel && !isPVRInFinalState(pvr) { return true } if isPVRInFinalState(pvr) && !pvr.DeletionTimestamp.IsZero() { return true } return false }) s := kube.NewPeriodicalEnqueueSource(r.logger.WithField("controller", constant.ControllerPodVolumeRestore), r.client, &velerov1api.PodVolumeRestoreList{}, preparingMonitorFrequency, kube.PeriodicalEnqueueSourceOption{ Predicates: []predicate.Predicate{gp}, }) pred := kube.NewAllEventPredicate(func(obj client.Object) bool { pvr := obj.(*velerov1api.PodVolumeRestore) return !IsLegacyPVR(pvr) }) return ctrl.NewControllerManagedBy(mgr). For(&velerov1api.PodVolumeRestore{}, builder.WithPredicates(pred)). WatchesRawSource(s). Watches(&corev1api.Pod{}, handler.EnqueueRequestsFromMapFunc(r.findPVRForTargetPod)). Watches(&corev1api.Pod{}, kube.EnqueueRequestsFromMapUpdateFunc(r.findPVRForRestorePod), builder.WithPredicates(predicate.Funcs{ UpdateFunc: func(ue event.UpdateEvent) bool { newObj := ue.ObjectNew.(*corev1api.Pod) if _, ok := newObj.Labels[velerov1api.PVRLabel]; !ok { return false } if newObj.Spec.NodeName == "" { return false } return true }, CreateFunc: func(event.CreateEvent) bool { return false }, DeleteFunc: func(de event.DeleteEvent) bool { return false }, GenericFunc: func(ge event.GenericEvent) bool { return false }, })). Complete(r) } func (r *PodVolumeRestoreReconciler) findPVRForTargetPod(ctx context.Context, pod client.Object) []reconcile.Request { list := &velerov1api.PodVolumeRestoreList{} options := &client.ListOptions{ LabelSelector: labels.Set(map[string]string{ velerov1api.PodUIDLabel: string(pod.GetUID()), }).AsSelector(), } if err := r.client.List(context.TODO(), list, options); err != nil { r.logger.WithField("pod", fmt.Sprintf("%s/%s", pod.GetNamespace(), pod.GetName())).WithError(err). Error("unable to list PodVolumeRestores") return []reconcile.Request{} } requests := []reconcile.Request{} for _, item := range list.Items { if IsLegacyPVR(&item) { continue } requests = append(requests, reconcile.Request{ NamespacedName: types.NamespacedName{ Namespace: item.GetNamespace(), Name: item.GetName(), }, }) } return requests } func (r *PodVolumeRestoreReconciler) findPVRForRestorePod(ctx context.Context, podObj client.Object) []reconcile.Request { pod := podObj.(*corev1api.Pod) pvr, err := findPVRByRestorePod(r.client, *pod) log := r.logger.WithField("pod", pod.Name) if err != nil { log.WithError(err).Error("unable to get PVR") return []reconcile.Request{} } else if pvr == nil { log.Error("get empty PVR") return []reconcile.Request{} } log = log.WithFields(logrus.Fields{ "PVR": pvr.Name, }) if pvr.Status.Phase != velerov1api.PodVolumeRestorePhaseAccepted { return []reconcile.Request{} } if pod.Status.Phase == corev1api.PodRunning { log.Info("Preparing PVR") if err = UpdatePVRWithRetry(context.Background(), r.client, types.NamespacedName{Namespace: pvr.Namespace, Name: pvr.Name}, log, func(pvr *velerov1api.PodVolumeRestore) bool { if isPVRInFinalState(pvr) { log.Warnf("PVR %s is terminated, abort setting it to prepared", pvr.Name) return false } pvr.Status.Phase = velerov1api.PodVolumeRestorePhasePrepared return true }); err != nil { log.WithError(err).Warn("failed to update PVR, prepare will halt for this PVR") return []reconcile.Request{} } } else if unrecoverable, reason := kube.IsPodUnrecoverable(pod, log); unrecoverable { err := UpdatePVRWithRetry(context.Background(), r.client, types.NamespacedName{Namespace: pvr.Namespace, Name: pvr.Name}, log, func(pvr *velerov1api.PodVolumeRestore) bool { if pvr.Spec.Cancel { return false } pvr.Spec.Cancel = true pvr.Status.Message = fmt.Sprintf("Cancel PVR because the exposing pod %s/%s is in abnormal status for reason %s", pod.Namespace, pod.Name, reason) return true }) if err != nil { log.WithError(err).Warn("failed to cancel PVR, and it will wait for prepare timeout") return []reconcile.Request{} } log.Infof("Exposed pod is in abnormal status(reason %s) and PVR is marked as cancel", reason) } else { return []reconcile.Request{} } request := reconcile.Request{ NamespacedName: types.NamespacedName{ Namespace: pvr.Namespace, Name: pvr.Name, }, } return []reconcile.Request{request} } func isPVRNew(pvr *velerov1api.PodVolumeRestore) bool { return pvr.Status.Phase == "" || pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseNew } func isInitContainerRunning(pod *corev1api.Pod) bool { // Pod volume wait container can be anywhere in the list of init containers, but must be running. i := getInitContainerIndex(pod) return i >= 0 && len(pod.Status.InitContainerStatuses)-1 >= i && pod.Status.InitContainerStatuses[i].State.Running != nil } func getInitContainerIndex(pod *corev1api.Pod) int { // Pod volume wait container can be anywhere in the list of init containers so locate it. for i, initContainer := range pod.Spec.InitContainers { if initContainer.Name == restorehelper.WaitInitContainer { return i } } return -1 } func (r *PodVolumeRestoreReconciler) OnDataPathCompleted(ctx context.Context, namespace string, pvrName string, result datapath.Result) { defer r.dataPathMgr.RemoveAsyncBR(pvrName) log := r.logger.WithField("PVR", pvrName) log.WithField("PVR", pvrName).WithField("result", result.Restore).Info("Async fs restore data path completed") var pvr velerov1api.PodVolumeRestore if err := r.client.Get(ctx, types.NamespacedName{Name: pvrName, Namespace: namespace}, &pvr); err != nil { log.WithError(err).Warn("Failed to get PVR on completion") return } log.Info("Cleaning up exposed environment") r.exposer.CleanUp(ctx, getPVROwnerObject(&pvr)) if err := UpdatePVRWithRetry(ctx, r.client, types.NamespacedName{Namespace: pvr.Namespace, Name: pvr.Name}, log, func(pvr *velerov1api.PodVolumeRestore) bool { if isPVRInFinalState(pvr) { return false } pvr.Status.Phase = velerov1api.PodVolumeRestorePhaseCompleted pvr.Status.CompletionTimestamp = &metav1.Time{Time: r.clock.Now()} delete(pvr.Labels, exposer.ExposeOnGoingLabel) return true }); err != nil { log.WithError(err).Error("error updating PVR status") } else { log.Info("Restore completed") } } func (r *PodVolumeRestoreReconciler) OnDataPathFailed(ctx context.Context, namespace string, pvrName string, err error) { defer r.dataPathMgr.RemoveAsyncBR(pvrName) log := r.logger.WithField("PVR", pvrName) log.WithError(err).Error("Async fs restore data path failed") var pvr velerov1api.PodVolumeRestore if getErr := r.client.Get(ctx, types.NamespacedName{Name: pvrName, Namespace: namespace}, &pvr); getErr != nil { log.WithError(getErr).Warn("Failed to get PVR on failure") } else { _, _ = r.errorOut(ctx, &pvr, err, "data path restore failed", log) } } func (r *PodVolumeRestoreReconciler) OnDataPathCancelled(ctx context.Context, namespace string, pvrName string) { defer r.dataPathMgr.RemoveAsyncBR(pvrName) log := r.logger.WithField("PVR", pvrName) log.Warn("Async fs restore data path canceled") var pvr velerov1api.PodVolumeRestore if getErr := r.client.Get(ctx, types.NamespacedName{Name: pvrName, Namespace: namespace}, &pvr); getErr != nil { log.WithError(getErr).Warn("Failed to get PVR on cancel") return } // cleans up any objects generated during the snapshot expose r.exposer.CleanUp(ctx, getPVROwnerObject(&pvr)) if err := UpdatePVRWithRetry(ctx, r.client, types.NamespacedName{Namespace: pvr.Namespace, Name: pvr.Name}, log, func(pvr *velerov1api.PodVolumeRestore) bool { if isPVRInFinalState(pvr) { return false } pvr.Status.Phase = velerov1api.PodVolumeRestorePhaseCanceled if pvr.Status.StartTimestamp.IsZero() { pvr.Status.StartTimestamp = &metav1.Time{Time: r.clock.Now()} } pvr.Status.CompletionTimestamp = &metav1.Time{Time: r.clock.Now()} delete(pvr.Labels, exposer.ExposeOnGoingLabel) return true }); err != nil { log.WithError(err).Error("error updating PVR status on cancel") } else { delete(r.cancelledPVR, pvr.Name) } } func (r *PodVolumeRestoreReconciler) OnDataPathProgress(ctx context.Context, namespace string, pvrName string, progress *uploader.Progress) { log := r.logger.WithField("PVR", pvrName) if err := UpdatePVRWithRetry(ctx, r.client, types.NamespacedName{Namespace: namespace, Name: pvrName}, log, func(pvr *velerov1api.PodVolumeRestore) bool { pvr.Status.Progress = veleroapishared.DataMoveOperationProgress{TotalBytes: progress.TotalBytes, BytesDone: progress.BytesDone} return true }); err != nil { log.WithError(err).Error("Failed to update progress") } } func (r *PodVolumeRestoreReconciler) setupExposeParam(pvr *velerov1api.PodVolumeRestore) exposer.PodVolumeExposeParam { log := r.logger.WithField("PVR", pvr.Name) nodeOS, err := kube.GetNodeOS(context.Background(), pvr.Status.Node, r.kubeClient.CoreV1()) if err != nil { log.WithError(err).Warnf("Failed to get nodeOS for node %s, use linux node-agent for hosting pod labels, annotations and tolerations", pvr.Status.Node) } hostingPodLabels := map[string]string{velerov1api.PVRLabel: pvr.Name} if len(r.podLabels) > 0 { for k, v := range r.podLabels { hostingPodLabels[k] = v } } else { for _, k := range util.ThirdPartyLabels { if v, err := nodeagent.GetLabelValue(context.Background(), r.kubeClient, pvr.Namespace, k, nodeOS); err != nil { if err != nodeagent.ErrNodeAgentLabelNotFound { log.WithError(err).Warnf("Failed to check node-agent label, skip adding host pod label %s", k) } } else { hostingPodLabels[k] = v } } } hostingPodAnnotation := map[string]string{} if len(r.podAnnotations) > 0 { for k, v := range r.podAnnotations { hostingPodAnnotation[k] = v } } else { for _, k := range util.ThirdPartyAnnotations { if v, err := nodeagent.GetAnnotationValue(context.Background(), r.kubeClient, pvr.Namespace, k, nodeOS); err != nil { if err != nodeagent.ErrNodeAgentAnnotationNotFound { log.WithError(err).Warnf("Failed to check node-agent annotation, skip adding host pod annotation %s", k) } } else { hostingPodAnnotation[k] = v } } } hostingPodTolerations := []corev1api.Toleration{} for _, k := range util.ThirdPartyTolerations { if v, err := nodeagent.GetToleration(context.Background(), r.kubeClient, pvr.Namespace, k, nodeOS); err != nil { if err != nodeagent.ErrNodeAgentTolerationNotFound { log.WithError(err).Warnf("Failed to check node-agent toleration, skip adding host pod toleration %s", k) } } else { hostingPodTolerations = append(hostingPodTolerations, *v) } } var cacheVolume *exposer.CacheConfigs if r.cacheVolumeConfigs != nil { if limit, err := r.repoConfigMgr.ClientSideCacheLimit(velerov1api.BackupRepositoryTypeKopia, r.backupRepoConfigs); err != nil { log.WithError(err).Warnf("Failed to get client side cache limit for repo type %s from configs %v", velerov1api.BackupRepositoryTypeKopia, r.backupRepoConfigs) } else { cacheVolume = &exposer.CacheConfigs{ Limit: limit, StorageClass: r.cacheVolumeConfigs.StorageClass, ResidentThreshold: r.cacheVolumeConfigs.ResidentThresholdInMB << 20, } } } return exposer.PodVolumeExposeParam{ Type: exposer.PodVolumeExposeTypeRestore, ClientNamespace: pvr.Spec.Pod.Namespace, ClientPodName: pvr.Spec.Pod.Name, ClientPodVolume: pvr.Spec.Volume, HostingPodLabels: hostingPodLabels, HostingPodAnnotations: hostingPodAnnotation, HostingPodTolerations: hostingPodTolerations, OperationTimeout: r.resourceTimeout, Resources: r.podResources, RestoreSize: pvr.Spec.SnapshotSize, CacheVolume: cacheVolume, // Priority class name for the data mover pod, retrieved from node-agent-configmap PriorityClassName: r.dataMovePriorityClass, Privileged: r.privileged, } } func getPVROwnerObject(pvr *velerov1api.PodVolumeRestore) corev1api.ObjectReference { return corev1api.ObjectReference{ Kind: pvr.Kind, Namespace: pvr.Namespace, Name: pvr.Name, UID: pvr.UID, APIVersion: pvr.APIVersion, } } func findPVRByRestorePod(client client.Client, pod corev1api.Pod) (*velerov1api.PodVolumeRestore, error) { if label, exist := pod.Labels[velerov1api.PVRLabel]; exist { pvr := &velerov1api.PodVolumeRestore{} err := client.Get(context.Background(), types.NamespacedName{ Namespace: pod.Namespace, Name: label, }, pvr) if err != nil { return nil, errors.Wrapf(err, "error to find PVR by pod %s/%s", pod.Namespace, pod.Name) } return pvr, nil } return nil, nil } func isPVRInFinalState(pvr *velerov1api.PodVolumeRestore) bool { return pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseFailed || pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseCanceled || pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseCompleted } func UpdatePVRWithRetry(ctx context.Context, client client.Client, namespacedName types.NamespacedName, log logrus.FieldLogger, updateFunc func(*velerov1api.PodVolumeRestore) bool) error { return wait.PollUntilContextCancel(ctx, time.Millisecond*100, true, func(ctx context.Context) (bool, error) { pvr := &velerov1api.PodVolumeRestore{} if err := client.Get(ctx, namespacedName, pvr); err != nil { return false, errors.Wrap(err, "getting PVR") } if updateFunc(pvr) { err := client.Update(ctx, pvr) if err != nil { if apierrors.IsConflict(err) { log.Debugf("failed to update PVR for %s/%s and will retry it", pvr.Namespace, pvr.Name) return false, nil } else { return false, errors.Wrapf(err, "error updating PVR %s/%s", pvr.Namespace, pvr.Name) } } } return true, nil }) } var funcResumeCancellablePVR = (*PodVolumeRestoreReconciler).resumeCancellableDataPath func (r *PodVolumeRestoreReconciler) AttemptPVRResume(ctx context.Context, logger *logrus.Entry, ns string) error { pvrs := &velerov1api.PodVolumeRestoreList{} if err := r.client.List(ctx, pvrs, &client.ListOptions{Namespace: ns}); err != nil { r.logger.WithError(errors.WithStack(err)).Error("failed to list PVRs") return errors.Wrapf(err, "error to list PVRs") } for i := range pvrs.Items { pvr := &pvrs.Items[i] if IsLegacyPVR(pvr) { continue } if pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseInProgress { if pvr.Status.Node != r.nodeName { logger.WithField("PVR", pvr.Name).WithField("current node", r.nodeName).Infof("PVR should be resumed by another node %s", pvr.Status.Node) continue } err := funcResumeCancellablePVR(r, ctx, pvr, logger) if err == nil { logger.WithField("PVR", pvr.Name).WithField("current node", r.nodeName).Info("Completed to resume in progress PVR") continue } logger.WithField("PVR", pvr.GetName()).WithError(err).Warn("Failed to resume data path for PVR, have to cancel it") resumeErr := err err = UpdatePVRWithRetry(ctx, r.client, types.NamespacedName{Namespace: pvr.Namespace, Name: pvr.Name}, logger.WithField("PVR", pvr.Name), func(pvr *velerov1api.PodVolumeRestore) bool { if pvr.Spec.Cancel { return false } pvr.Spec.Cancel = true pvr.Status.Message = fmt.Sprintf("Resume InProgress PVR failed with error %v, mark it as cancel", resumeErr) return true }) if err != nil { logger.WithField("PVR", pvr.GetName()).WithError(errors.WithStack(err)).Error("Failed to trigger PVR cancel") } } else if !isPVRInFinalState(pvr) { logger.WithField("PVR", pvr.GetName()).Infof("find a PVR with status %s", pvr.Status.Phase) } } return nil } func (r *PodVolumeRestoreReconciler) resumeCancellableDataPath(ctx context.Context, pvr *velerov1api.PodVolumeRestore, log logrus.FieldLogger) error { log.Info("Resume cancelable PVR") res, err := r.exposer.GetExposed(ctx, getPVROwnerObject(pvr), r.client, r.nodeName, r.resourceTimeout) if err != nil { return errors.Wrapf(err, "error to get exposed PVR %s", pvr.Name) } if res == nil { return errors.Errorf("no expose result is available for the current node for PVR %s", pvr.Name) } callbacks := datapath.Callbacks{ OnCompleted: r.OnDataPathCompleted, OnFailed: r.OnDataPathFailed, OnCancelled: r.OnDataPathCancelled, OnProgress: r.OnDataPathProgress, } asyncBR, err := r.dataPathMgr.CreateMicroServiceBRWatcher(ctx, r.client, r.kubeClient, r.mgr, datapath.TaskTypeRestore, pvr.Name, pvr.Namespace, res.ByPod.HostingPod.Name, res.ByPod.HostingContainer, pvr.Name, callbacks, true, log) if err != nil { return errors.Wrapf(err, "error to create asyncBR watcher for PVR %s", pvr.Name) } resumeComplete := false defer func() { if !resumeComplete { r.closeDataPath(ctx, pvr.Name) } }() if err := asyncBR.Init(ctx, nil); err != nil { return errors.Wrapf(err, "error to init asyncBR watcher for PVR %s", pvr.Name) } if err := asyncBR.StartRestore(pvr.Spec.SnapshotID, datapath.AccessPoint{ ByPath: res.ByPod.VolumeName, }, pvr.Spec.UploaderSettings); err != nil { return errors.Wrapf(err, "error to resume asyncBR watcher for PVR %s", pvr.Name) } resumeComplete = true log.Infof("asyncBR is resumed for PVR %s", pvr.Name) return nil } ================================================ FILE: pkg/controller/pod_volume_restore_controller_legacy.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package controller import ( "context" "fmt" "os" "path/filepath" "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" clocks "k8s.io/utils/clock" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" "github.com/vmware-tanzu/velero/internal/credentials" veleroapishared "github.com/vmware-tanzu/velero/pkg/apis/velero/shared" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/datapath" "github.com/vmware-tanzu/velero/pkg/exposer" "github.com/vmware-tanzu/velero/pkg/podvolume" "github.com/vmware-tanzu/velero/pkg/repository" "github.com/vmware-tanzu/velero/pkg/restorehelper" "github.com/vmware-tanzu/velero/pkg/uploader" "github.com/vmware-tanzu/velero/pkg/util/boolptr" "github.com/vmware-tanzu/velero/pkg/util/filesystem" "github.com/vmware-tanzu/velero/pkg/util/kube" ) func InitLegacyPodVolumeRestoreReconciler(client client.Client, mgr manager.Manager, kubeClient kubernetes.Interface, dataPathMgr *datapath.Manager, namespace string, resourceTimeout time.Duration, logger logrus.FieldLogger) error { log := logger.WithField("controller", "PodVolumeRestoreLegacy") credentialFileStore, err := credentials.NewNamespacedFileStore(client, namespace, credentials.DefaultStoreDirectory(), filesystem.NewFileSystem()) if err != nil { return errors.Wrapf(err, "error creating credentials file store") } credSecretStore, err := credentials.NewNamespacedSecretStore(client, namespace) if err != nil { return errors.Wrapf(err, "error creating secret file store") } credentialGetter := &credentials.CredentialGetter{FromFile: credentialFileStore, FromSecret: credSecretStore} ensurer := repository.NewEnsurer(client, log, resourceTimeout) reconciler := &PodVolumeRestoreReconcilerLegacy{ Client: client, kubeClient: kubeClient, logger: log, repositoryEnsurer: ensurer, credentialGetter: credentialGetter, fileSystem: filesystem.NewFileSystem(), clock: &clocks.RealClock{}, dataPathMgr: dataPathMgr, } if err = reconciler.SetupWithManager(mgr); err != nil { return errors.Wrapf(err, "error setup controller manager") } return nil } type PodVolumeRestoreReconcilerLegacy struct { client.Client kubeClient kubernetes.Interface logger logrus.FieldLogger repositoryEnsurer *repository.Ensurer credentialGetter *credentials.CredentialGetter fileSystem filesystem.Interface clock clocks.WithTickerAndDelayedExecution dataPathMgr *datapath.Manager } // +kubebuilder:rbac:groups=velero.io,resources=podvolumerestores,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=velero.io,resources=podvolumerestores/status,verbs=get;update;patch // +kubebuilder:rbac:groups="",resources=pods,verbs=get // +kubebuilder:rbac:groups="",resources=persistentvolumes,verbs=get // +kubebuilder:rbac:groups="",resources=persistentvolumerclaims,verbs=get func (c *PodVolumeRestoreReconcilerLegacy) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := c.logger.WithField("PodVolumeRestore", req.NamespacedName.String()) log.Info("Reconciling PVR by legacy controller") pvr := &velerov1api.PodVolumeRestore{} if err := c.Get(ctx, types.NamespacedName{Namespace: req.Namespace, Name: req.Name}, pvr); err != nil { if apierrors.IsNotFound(err) { log.Warn("PodVolumeRestore not found, skip") return ctrl.Result{}, nil } log.WithError(err).Error("Unable to get the PodVolumeRestore") return ctrl.Result{}, err } log = log.WithField("pod", fmt.Sprintf("%s/%s", pvr.Spec.Pod.Namespace, pvr.Spec.Pod.Name)) if len(pvr.OwnerReferences) == 1 { log = log.WithField("restore", fmt.Sprintf("%s/%s", pvr.Namespace, pvr.OwnerReferences[0].Name)) } shouldProcess, pod, err := shouldProcess(ctx, c.Client, log, pvr) if err != nil { return ctrl.Result{}, err } if !shouldProcess { return ctrl.Result{}, nil } initContainerIndex := getInitContainerIndex(pod) if initContainerIndex > 0 { log.Warnf(`Init containers before the %s container may cause issues if they interfere with volumes being restored: %s index %d`, restorehelper.WaitInitContainer, restorehelper.WaitInitContainer, initContainerIndex) } log.Info("Restore starting") callbacks := datapath.Callbacks{ OnCompleted: c.OnDataPathCompleted, OnFailed: c.OnDataPathFailed, OnCancelled: c.OnDataPathCancelled, OnProgress: c.OnDataPathProgress, } fsRestore, err := c.dataPathMgr.CreateFileSystemBR(pvr.Name, pVBRRequestor, ctx, c.Client, pvr.Namespace, callbacks, log) if err != nil { if err == datapath.ConcurrentLimitExceed { return ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, nil } else { return c.errorOut(ctx, pvr, err, "error to create data path", log) } } original := pvr.DeepCopy() pvr.Status.Phase = velerov1api.PodVolumeRestorePhaseInProgress pvr.Status.StartTimestamp = &metav1.Time{Time: c.clock.Now()} if err = c.Patch(ctx, pvr, client.MergeFrom(original)); err != nil { c.closeDataPath(ctx, pvr.Name) return c.errorOut(ctx, pvr, err, "error to update status to in progress", log) } volumePath, err := exposer.GetPodVolumeHostPath(ctx, pod, pvr.Spec.Volume, c.kubeClient, c.fileSystem, log) if err != nil { c.closeDataPath(ctx, pvr.Name) return c.errorOut(ctx, pvr, err, "error exposing host path for pod volume", log) } log.WithField("path", volumePath.ByPath).Debugf("Found host path") if err := fsRestore.Init(ctx, &datapath.FSBRInitParam{ BSLName: pvr.Spec.BackupStorageLocation, SourceNamespace: pvr.Spec.SourceNamespace, UploaderType: pvr.Spec.UploaderType, RepositoryType: podvolume.GetPvrRepositoryType(pvr), RepoIdentifier: pvr.Spec.RepoIdentifier, RepositoryEnsurer: c.repositoryEnsurer, CredentialGetter: c.credentialGetter, }); err != nil { c.closeDataPath(ctx, pvr.Name) return c.errorOut(ctx, pvr, err, "error to initialize data path", log) } if err := fsRestore.StartRestore(pvr.Spec.SnapshotID, volumePath, pvr.Spec.UploaderSettings); err != nil { c.closeDataPath(ctx, pvr.Name) return c.errorOut(ctx, pvr, err, "error starting data path restore", log) } log.WithField("path", volumePath.ByPath).Info("Async fs restore data path started") return ctrl.Result{}, nil } func (c *PodVolumeRestoreReconcilerLegacy) errorOut(ctx context.Context, pvr *velerov1api.PodVolumeRestore, err error, msg string, log logrus.FieldLogger) (ctrl.Result, error) { _ = UpdatePVRStatusToFailed(ctx, c.Client, pvr, err, msg, c.clock.Now(), log) return ctrl.Result{}, err } func (c *PodVolumeRestoreReconcilerLegacy) SetupWithManager(mgr ctrl.Manager) error { // The pod may not being scheduled at the point when its PVRs are initially reconciled. // By watching the pods, we can trigger the PVR reconciliation again once the pod is finally scheduled on the node. pred := kube.NewAllEventPredicate(func(obj client.Object) bool { pvr := obj.(*velerov1api.PodVolumeRestore) return IsLegacyPVR(pvr) }) return ctrl.NewControllerManagedBy(mgr).Named("podvolumerestorelegacy"). For(&velerov1api.PodVolumeRestore{}, builder.WithPredicates(pred)). Watches(&corev1api.Pod{}, handler.EnqueueRequestsFromMapFunc(c.findVolumeRestoresForPod)). Complete(c) } func (c *PodVolumeRestoreReconcilerLegacy) findVolumeRestoresForPod(ctx context.Context, pod client.Object) []reconcile.Request { list := &velerov1api.PodVolumeRestoreList{} options := &client.ListOptions{ LabelSelector: labels.Set(map[string]string{ velerov1api.PodUIDLabel: string(pod.GetUID()), }).AsSelector(), } if err := c.Client.List(context.TODO(), list, options); err != nil { c.logger.WithField("pod", fmt.Sprintf("%s/%s", pod.GetNamespace(), pod.GetName())).WithError(err). Error("unable to list PodVolumeRestores") return []reconcile.Request{} } requests := []reconcile.Request{} for _, item := range list.Items { if !IsLegacyPVR(&item) { continue } requests = append(requests, reconcile.Request{ NamespacedName: types.NamespacedName{ Namespace: item.GetNamespace(), Name: item.GetName(), }, }) } return requests } func (c *PodVolumeRestoreReconcilerLegacy) OnDataPathCompleted(ctx context.Context, namespace string, pvrName string, result datapath.Result) { defer c.dataPathMgr.RemoveAsyncBR(pvrName) log := c.logger.WithField("pvr", pvrName) log.WithField("PVR", pvrName).Info("Async fs restore data path completed") var pvr velerov1api.PodVolumeRestore if err := c.Client.Get(ctx, types.NamespacedName{Name: pvrName, Namespace: namespace}, &pvr); err != nil { log.WithError(err).Warn("Failed to get PVR on completion") return } volumePath := result.Restore.Target.ByPath if volumePath == "" { _, _ = c.errorOut(ctx, &pvr, errors.New("path is empty"), "invalid restore target", log) return } // Remove the .velero directory from the restored volume (it may contain done files from previous restores // of this volume, which we don't want to carry over). If this fails for any reason, log and continue, since // this is non-essential cleanup (the done files are named based on restore UID and the init container looks // for the one specific to the restore being executed). if err := os.RemoveAll(filepath.Join(volumePath, ".velero")); err != nil { log.WithError(err).Warnf("error removing .velero directory from directory %s", volumePath) } var restoreUID types.UID for _, owner := range pvr.OwnerReferences { if boolptr.IsSetToTrue(owner.Controller) { restoreUID = owner.UID break } } // Create the .velero directory within the volume dir so we can write a done file // for this restore. if err := os.MkdirAll(filepath.Join(volumePath, ".velero"), 0755); err != nil { _, _ = c.errorOut(ctx, &pvr, err, "error creating .velero directory for done file", log) return } // Write a done file with name= into the just-created .velero dir // within the volume. The velero init container on the pod is waiting // for this file to exist in each restored volume before completing. if err := os.WriteFile(filepath.Join(volumePath, ".velero", string(restoreUID)), nil, 0644); err != nil { //nolint:gosec // Internal usage. No need to check. _, _ = c.errorOut(ctx, &pvr, err, "error writing done file", log) return } original := pvr.DeepCopy() pvr.Status.Phase = velerov1api.PodVolumeRestorePhaseCompleted pvr.Status.CompletionTimestamp = &metav1.Time{Time: c.clock.Now()} if err := c.Patch(ctx, &pvr, client.MergeFrom(original)); err != nil { log.WithError(err).Error("error updating PodVolumeRestore status") } log.Info("Restore completed") } func (c *PodVolumeRestoreReconcilerLegacy) OnDataPathFailed(ctx context.Context, namespace string, pvrName string, err error) { defer c.dataPathMgr.RemoveAsyncBR(pvrName) log := c.logger.WithField("pvr", pvrName) log.WithError(err).Error("Async fs restore data path failed") var pvr velerov1api.PodVolumeRestore if getErr := c.Client.Get(ctx, types.NamespacedName{Name: pvrName, Namespace: namespace}, &pvr); getErr != nil { log.WithError(getErr).Warn("Failed to get PVR on failure") } else { _, _ = c.errorOut(ctx, &pvr, err, "data path restore failed", log) } } func (c *PodVolumeRestoreReconcilerLegacy) OnDataPathCancelled(ctx context.Context, namespace string, pvrName string) { defer c.dataPathMgr.RemoveAsyncBR(pvrName) log := c.logger.WithField("pvr", pvrName) log.Warn("Async fs restore data path canceled") var pvr velerov1api.PodVolumeRestore if getErr := c.Client.Get(ctx, types.NamespacedName{Name: pvrName, Namespace: namespace}, &pvr); getErr != nil { log.WithError(getErr).Warn("Failed to get PVR on cancel") } else { _, _ = c.errorOut(ctx, &pvr, errors.New("PVR is canceled"), "data path restore canceled", log) } } func (c *PodVolumeRestoreReconcilerLegacy) OnDataPathProgress(ctx context.Context, namespace string, pvrName string, progress *uploader.Progress) { log := c.logger.WithField("pvr", pvrName) var pvr velerov1api.PodVolumeRestore if err := c.Client.Get(ctx, types.NamespacedName{Name: pvrName, Namespace: namespace}, &pvr); err != nil { log.WithError(err).Warn("Failed to get PVB on progress") return } original := pvr.DeepCopy() pvr.Status.Progress = veleroapishared.DataMoveOperationProgress{TotalBytes: progress.TotalBytes, BytesDone: progress.BytesDone} if err := c.Client.Patch(ctx, &pvr, client.MergeFrom(original)); err != nil { log.WithError(err).Error("Failed to update progress") } } func (c *PodVolumeRestoreReconcilerLegacy) closeDataPath(ctx context.Context, pvbName string) { fsRestore := c.dataPathMgr.GetAsyncBR(pvbName) if fsRestore != nil { fsRestore.Close(ctx) } c.dataPathMgr.RemoveAsyncBR(pvbName) } func IsLegacyPVR(pvr *velerov1api.PodVolumeRestore) bool { return pvr.Spec.UploaderType == uploader.ResticType } ================================================ FILE: pkg/controller/pod_volume_restore_controller_legacy_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package controller import ( "testing" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client/fake" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) func TestFindVolumeRestoresForPodLegacy(t *testing.T) { pod := &corev1api.Pod{} pod.UID = "uid" scheme := runtime.NewScheme() scheme.AddKnownTypes(velerov1api.SchemeGroupVersion, &velerov1api.PodVolumeRestore{}, &velerov1api.PodVolumeRestoreList{}) clientBuilder := fake.NewClientBuilder().WithScheme(scheme) // no matching PVR reconciler := &PodVolumeRestoreReconcilerLegacy{ Client: clientBuilder.Build(), logger: logrus.New(), } requests := reconciler.findVolumeRestoresForPod(t.Context(), pod) assert.Empty(t, requests) // contain one matching PVR reconciler.Client = clientBuilder.WithLists(&velerov1api.PodVolumeRestoreList{ Items: []velerov1api.PodVolumeRestore{ { ObjectMeta: metav1.ObjectMeta{ Name: "pvr1", Labels: map[string]string{ velerov1api.PodUIDLabel: string(pod.GetUID()), }, }, }, { ObjectMeta: metav1.ObjectMeta{ Name: "pvr2", Labels: map[string]string{ velerov1api.PodUIDLabel: "non-matching-uid", }, }, }, { ObjectMeta: metav1.ObjectMeta{ Name: "pvr3", Labels: map[string]string{ velerov1api.PodUIDLabel: string(pod.GetUID()), }, }, Spec: velerov1api.PodVolumeRestoreSpec{ UploaderType: "kopia", }, }, { ObjectMeta: metav1.ObjectMeta{ Name: "pvr4", Labels: map[string]string{ velerov1api.PodUIDLabel: string(pod.GetUID()), }, }, Spec: velerov1api.PodVolumeRestoreSpec{ UploaderType: "restic", }, }, }, }).Build() requests = reconciler.findVolumeRestoresForPod(t.Context(), pod) assert.Len(t, requests, 1) } ================================================ FILE: pkg/controller/pod_volume_restore_controller_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package controller import ( "context" "fmt" "testing" "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" appsv1api "k8s.io/api/apps/v1" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" clientgofake "k8s.io/client-go/kubernetes/fake" clocks "k8s.io/utils/clock" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" kbclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/datapath" datapathmockes "github.com/vmware-tanzu/velero/pkg/datapath/mocks" "github.com/vmware-tanzu/velero/pkg/exposer" exposermockes "github.com/vmware-tanzu/velero/pkg/exposer/mocks" "github.com/vmware-tanzu/velero/pkg/restorehelper" "github.com/vmware-tanzu/velero/pkg/test" velerotest "github.com/vmware-tanzu/velero/pkg/test" "github.com/vmware-tanzu/velero/pkg/uploader" "github.com/vmware-tanzu/velero/pkg/util/boolptr" "github.com/vmware-tanzu/velero/pkg/util/kube" ) func TestShouldProcess(t *testing.T) { controllerNode := "foo" tests := []struct { name string obj *velerov1api.PodVolumeRestore pod *corev1api.Pod shouldProcessed bool }{ { name: "InProgress phase pvr should not be processed", obj: &velerov1api.PodVolumeRestore{ Status: velerov1api.PodVolumeRestoreStatus{ Phase: velerov1api.PodVolumeRestorePhaseInProgress, }, }, shouldProcessed: false, }, { name: "Completed phase pvr should not be processed", obj: &velerov1api.PodVolumeRestore{ ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", Name: "pvr-1", }, Status: velerov1api.PodVolumeRestoreStatus{ Phase: velerov1api.PodVolumeRestorePhaseCompleted, }, }, shouldProcessed: false, }, { name: "Failed phase pvr should not be processed", obj: &velerov1api.PodVolumeRestore{ ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", Name: "pvr-1", }, Status: velerov1api.PodVolumeRestoreStatus{ Phase: velerov1api.PodVolumeRestorePhaseFailed, }, }, shouldProcessed: false, }, { name: "Unable to get pvr's pod should not be processed", obj: &velerov1api.PodVolumeRestore{ Spec: velerov1api.PodVolumeRestoreSpec{ Pod: corev1api.ObjectReference{ Namespace: "ns-1", Name: "pod-1", }, }, Status: velerov1api.PodVolumeRestoreStatus{ Phase: "", }, }, shouldProcessed: false, }, { name: "Empty phase pvr with pod on node not running init container should not be processed", obj: &velerov1api.PodVolumeRestore{ ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", Name: "pvr-1", }, Spec: velerov1api.PodVolumeRestoreSpec{ Pod: corev1api.ObjectReference{ Namespace: "ns-1", Name: "pod-1", }, }, Status: velerov1api.PodVolumeRestoreStatus{ Phase: "", }, }, pod: &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: "ns-1", Name: "pod-1", }, Spec: corev1api.PodSpec{ NodeName: controllerNode, InitContainers: []corev1api.Container{ { Name: restorehelper.WaitInitContainer, }, }, }, Status: corev1api.PodStatus{ InitContainerStatuses: []corev1api.ContainerStatus{ { State: corev1api.ContainerState{}, }, }, }, }, shouldProcessed: false, }, { name: "Empty phase pvr with pod on node running init container should be enqueued", obj: &velerov1api.PodVolumeRestore{ ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", Name: "pvr-1", }, Spec: velerov1api.PodVolumeRestoreSpec{ Pod: corev1api.ObjectReference{ Namespace: "ns-1", Name: "pod-1", }, }, Status: velerov1api.PodVolumeRestoreStatus{ Phase: "", }, }, pod: &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: "ns-1", Name: "pod-1", }, Spec: corev1api.PodSpec{ NodeName: controllerNode, InitContainers: []corev1api.Container{ { Name: restorehelper.WaitInitContainer, }, }, }, Status: corev1api.PodStatus{ InitContainerStatuses: []corev1api.ContainerStatus{ { State: corev1api.ContainerState{ Running: &corev1api.ContainerStateRunning{ StartedAt: metav1.Time{Time: time.Now()}, }, }, }, }, }, }, shouldProcessed: true, }, } for _, ts := range tests { t.Run(ts.name, func(t *testing.T) { ctx := t.Context() var objs []runtime.Object if ts.obj != nil { objs = append(objs, ts.obj) } if ts.pod != nil { objs = append(objs, ts.pod) } cli := test.NewFakeControllerRuntimeClient(t, objs...) c := &PodVolumeRestoreReconciler{ logger: logrus.New(), client: cli, clock: &clocks.RealClock{}, } shouldProcess, _, _ := shouldProcess(ctx, c.client, c.logger, ts.obj) require.Equal(t, ts.shouldProcessed, shouldProcess) }) } } func TestIsInitContainerRunning(t *testing.T) { tests := []struct { name string pod *corev1api.Pod expected bool }{ { name: "pod with no init containers should return false", pod: &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: "ns-1", Name: "pod-1", }, }, expected: false, }, { name: "pod with running init container that's not restore init should return false", pod: &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: "ns-1", Name: "pod-1", }, Spec: corev1api.PodSpec{ InitContainers: []corev1api.Container{ { Name: "non-restore-init", }, }, }, Status: corev1api.PodStatus{ InitContainerStatuses: []corev1api.ContainerStatus{ { State: corev1api.ContainerState{ Running: &corev1api.ContainerStateRunning{StartedAt: metav1.Time{Time: time.Now()}}, }, }, }, }, }, expected: false, }, { name: "pod with running init container that's not first should still work", pod: &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: "ns-1", Name: "pod-1", }, Spec: corev1api.PodSpec{ InitContainers: []corev1api.Container{ { Name: "non-restore-init", }, { Name: restorehelper.WaitInitContainer, }, }, }, Status: corev1api.PodStatus{ InitContainerStatuses: []corev1api.ContainerStatus{ { State: corev1api.ContainerState{ Running: &corev1api.ContainerStateRunning{StartedAt: metav1.Time{Time: time.Now()}}, }, }, { State: corev1api.ContainerState{ Running: &corev1api.ContainerStateRunning{StartedAt: metav1.Time{Time: time.Now()}}, }, }, }, }, }, expected: true, }, { name: "pod with init container as first initContainer that's not running should return false", pod: &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: "ns-1", Name: "pod-1", }, Spec: corev1api.PodSpec{ InitContainers: []corev1api.Container{ { Name: restorehelper.WaitInitContainer, }, { Name: "non-restore-init", }, }, }, Status: corev1api.PodStatus{ InitContainerStatuses: []corev1api.ContainerStatus{ { State: corev1api.ContainerState{}, }, { State: corev1api.ContainerState{ Running: &corev1api.ContainerStateRunning{StartedAt: metav1.Time{Time: time.Now()}}, }, }, }, }, }, expected: false, }, { name: "pod with running init container as first initContainer should return true", pod: &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: "ns-1", Name: "pod-1", }, Spec: corev1api.PodSpec{ InitContainers: []corev1api.Container{ { Name: restorehelper.WaitInitContainer, }, { Name: "non-restore-init", }, }, }, Status: corev1api.PodStatus{ InitContainerStatuses: []corev1api.ContainerStatus{ { State: corev1api.ContainerState{ Running: &corev1api.ContainerStateRunning{StartedAt: metav1.Time{Time: time.Now()}}, }, }, { State: corev1api.ContainerState{ Running: &corev1api.ContainerStateRunning{StartedAt: metav1.Time{Time: time.Now()}}, }, }, }, }, }, expected: true, }, { name: "pod with init container with empty InitContainerStatuses should return 0", pod: &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: "ns-1", Name: "pod-1", }, Spec: corev1api.PodSpec{ InitContainers: []corev1api.Container{ { Name: restorehelper.WaitInitContainer, }, }, }, Status: corev1api.PodStatus{ InitContainerStatuses: []corev1api.ContainerStatus{}, }, }, expected: false, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { assert.Equal(t, test.expected, isInitContainerRunning(test.pod)) }) } } func TestGetInitContainerIndex(t *testing.T) { tests := []struct { name string pod *corev1api.Pod expected int }{ { name: "init container is not present return -1", pod: &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: "ns-1", Name: "pod-1", }, }, expected: -1, }, { name: "pod with no init container return -1", pod: &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: "ns-1", Name: "pod-1", }, Spec: corev1api.PodSpec{ InitContainers: []corev1api.Container{ { Name: "non-restore-init", }, }, }, }, expected: -1, }, { name: "pod with container as second initContainern should return 1", pod: &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: "ns-1", Name: "pod-1", }, Spec: corev1api.PodSpec{ InitContainers: []corev1api.Container{ { Name: "non-restore-init", }, { Name: restorehelper.WaitInitContainer, }, }, }, }, expected: 1, }, { name: "pod with init container as first initContainer should return 0", pod: &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: "ns-1", Name: "pod-1", }, Spec: corev1api.PodSpec{ InitContainers: []corev1api.Container{ { Name: restorehelper.WaitInitContainer, }, { Name: "non-restore-init", }, }, }, }, expected: 0, }, { name: "pod with init container as first initContainer should return 0", pod: &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: "ns-1", Name: "pod-1", }, Spec: corev1api.PodSpec{ InitContainers: []corev1api.Container{ { Name: restorehelper.WaitInitContainer, }, { Name: "non-restore-init", }, }, }, }, expected: 0, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { assert.Equal(t, test.expected, getInitContainerIndex(test.pod)) }) } } func TestFindPVRForTargetPod(t *testing.T) { pod := &corev1api.Pod{} pod.UID = "uid" scheme := runtime.NewScheme() scheme.AddKnownTypes(velerov1api.SchemeGroupVersion, &velerov1api.PodVolumeRestore{}, &velerov1api.PodVolumeRestoreList{}) clientBuilder := fake.NewClientBuilder().WithScheme(scheme) // no matching PVR reconciler := &PodVolumeRestoreReconciler{ client: clientBuilder.Build(), logger: logrus.New(), } requests := reconciler.findPVRForTargetPod(t.Context(), pod) assert.Empty(t, requests) // contain one matching PVR reconciler.client = clientBuilder.WithLists(&velerov1api.PodVolumeRestoreList{ Items: []velerov1api.PodVolumeRestore{ { ObjectMeta: metav1.ObjectMeta{ Name: "pvr1", Labels: map[string]string{ velerov1api.PodUIDLabel: string(pod.GetUID()), }, }, }, { ObjectMeta: metav1.ObjectMeta{ Name: "pvr2", Labels: map[string]string{ velerov1api.PodUIDLabel: "non-matching-uid", }, }, }, }, }).Build() requests = reconciler.findPVRForTargetPod(t.Context(), pod) assert.Len(t, requests, 1) } const pvrName string = "pvr-1" func pvrBuilder() *builder.PodVolumeRestoreBuilder { return builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName). BackupStorageLocation("bsl-loc"). SnapshotID("test-snapshot-id") } func initPodVolumeRestoreReconciler(objects []runtime.Object, cliObj []client.Object, needError ...bool) (*PodVolumeRestoreReconciler, error) { var errs = make([]error, 6) for k, isError := range needError { if k == 0 && isError { errs[0] = fmt.Errorf("Get error") } else if k == 1 && isError { errs[1] = fmt.Errorf("Create error") } else if k == 2 && isError { errs[2] = fmt.Errorf("Update error") } else if k == 3 && isError { errs[3] = fmt.Errorf("Patch error") } else if k == 4 && isError { errs[4] = apierrors.NewConflict(velerov1api.Resource("podvolumerestore"), pvrName, errors.New("conflict")) } else if k == 5 && isError { errs[5] = fmt.Errorf("List error") } } return initPodVolumeRestoreReconcilerWithError(objects, cliObj, errs...) } func initPodVolumeRestoreReconcilerWithError(objects []runtime.Object, cliObj []client.Object, needError ...error) (*PodVolumeRestoreReconciler, error) { scheme := runtime.NewScheme() err := velerov1api.AddToScheme(scheme) if err != nil { return nil, err } err = corev1api.AddToScheme(scheme) if err != nil { return nil, err } fakeClient := &FakeClient{ Client: fake.NewClientBuilder().WithScheme(scheme).WithObjects(cliObj...).Build(), } for k := range needError { if k == 0 { fakeClient.getError = needError[0] } else if k == 1 { fakeClient.createError = needError[1] } else if k == 2 { fakeClient.updateError = needError[2] } else if k == 3 { fakeClient.patchError = needError[3] } else if k == 4 { fakeClient.updateConflict = needError[4] } else if k == 5 { fakeClient.listError = needError[5] } } var fakeKubeClient *clientgofake.Clientset if len(objects) != 0 { fakeKubeClient = clientgofake.NewSimpleClientset(objects...) } else { fakeKubeClient = clientgofake.NewSimpleClientset() } fakeFS := velerotest.NewFakeFileSystem() pathGlob := fmt.Sprintf("/host_pods/%s/volumes/*/%s", "test-uid", "test-pvc") _, err = fakeFS.Create(pathGlob) if err != nil { return nil, err } dataPathMgr := datapath.NewManager(1) return NewPodVolumeRestoreReconciler( fakeClient, nil, fakeKubeClient, dataPathMgr, nil, "test-node", time.Minute*5, time.Minute, nil, nil, corev1api.ResourceRequirements{}, velerotest.NewLogger(), "", false, nil, nil, // podLabels nil, // podAnnotations ), nil } func TestPodVolumeRestoreReconcile(t *testing.T) { daemonSet := &appsv1api.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", Name: "node-agent", }, TypeMeta: metav1.TypeMeta{ Kind: "DaemonSet", APIVersion: appsv1api.SchemeGroupVersion.String(), }, Spec: appsv1api.DaemonSetSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ Containers: []corev1api.Container{ { Image: "fake-image", }, }, }, }, }, } node := builder.ForNode("fake-node").Labels(map[string]string{kube.NodeOSLabel: kube.NodeOSLinux}).Result() tests := []struct { name string pvr *velerov1api.PodVolumeRestore notCreatePVR bool targetPod *corev1api.Pod dataMgr *datapath.Manager needErrs []bool needCreateFSBR bool needDelete bool sportTime *metav1.Time mockExposeErr *bool isGetExposeErr bool isGetExposeNil bool isPeekExposeErr bool isNilExposer bool notNilExpose bool notMockCleanUp bool mockInit bool mockInitErr error mockStart bool mockStartErr error mockCancel bool mockClose bool needExclusiveUpdateError error constrained bool expected *velerov1api.PodVolumeRestore expectDeleted bool expectCancelRecord bool expectedResult *ctrl.Result expectedErr string expectDataPath bool }{ { name: "pvr not found", pvr: pvrBuilder().Result(), notCreatePVR: true, }, { name: "pvr not created in velero default namespace", pvr: builder.ForPodVolumeRestore("test-ns", pvrName).Result(), }, { name: "get dd fail", pvr: builder.ForPodVolumeRestore("test-ns", pvrName).Result(), needErrs: []bool{true, false, false, false}, expectedErr: "Get error", }, { name: "add finalizer to pvr", pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Result(), expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Result(), }, { name: "add finalizer to pvr failed", pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Result(), needErrs: []bool{false, false, true, false}, expectedErr: "error updating PVR velero/pvr-1: Update error", }, { name: "pvr is under deletion", pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Result(), needDelete: true, expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Result(), }, { name: "pvr is under deletion but cancel failed", pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Result(), needErrs: []bool{false, false, true, false}, needDelete: true, expectedErr: "error updating PVR velero/pvr-1: Update error", }, { name: "pvr is under deletion and in terminal state", pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Phase(velerov1api.PodVolumeRestorePhaseFailed).Result(), sportTime: &metav1.Time{Time: time.Now()}, needDelete: true, expectDeleted: true, }, { name: "pvr is under deletion and in terminal state, but remove finalizer failed", pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Phase(velerov1api.PodVolumeRestorePhaseFailed).Result(), needErrs: []bool{false, false, true, false}, needDelete: true, expectedErr: "error updating PVR velero/pvr-1: Update error", }, { name: "delay cancel negative for others", pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeRestorePhasePrepared).Result(), sportTime: &metav1.Time{Time: time.Now()}, expectCancelRecord: true, }, { name: "delay cancel negative for inProgress", pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeRestorePhaseInProgress).Result(), sportTime: &metav1.Time{Time: time.Now().Add(-time.Minute * 58)}, expectCancelRecord: true, }, { name: "delay cancel affirmative for others", pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeRestorePhasePrepared).Result(), sportTime: &metav1.Time{Time: time.Now().Add(-time.Minute * 5)}, expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeRestorePhaseCanceled).Result(), }, { name: "delay cancel affirmative for inProgress", pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeRestorePhaseInProgress).Result(), sportTime: &metav1.Time{Time: time.Now().Add(-time.Hour)}, expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeRestorePhaseCanceled).Result(), }, { name: "delay cancel failed", pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeRestorePhaseInProgress).Result(), needErrs: []bool{false, false, true, false}, sportTime: &metav1.Time{Time: time.Now().Add(-time.Hour)}, expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeRestorePhaseInProgress).Result(), expectCancelRecord: true, }, { name: "Unknown pvr status", pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase("Unknown").Finalizers([]string{PodVolumeFinalizer}).Result(), }, { name: "new pvb but constrained", pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).PodNamespace("test-ns").PodName("test-pod").Result(), targetPod: builder.ForPod("test-ns", "test-pod").InitContainers(&corev1api.Container{Name: restorehelper.WaitInitContainer}).InitContainerState(corev1api.ContainerState{Running: &corev1api.ContainerStateRunning{}}).Result(), constrained: true, expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Result(), expectedResult: &ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, }, { name: "new pvr but accept failed", pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).PodNamespace("test-ns").PodName("test-pod").Result(), targetPod: builder.ForPod("test-ns", "test-pod").InitContainers(&corev1api.Container{Name: restorehelper.WaitInitContainer}).InitContainerState(corev1api.ContainerState{Running: &corev1api.ContainerStateRunning{}}).Result(), needErrs: []bool{false, false, true, false}, expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Result(), expectedErr: "error accepting PVR pvr-1: error updating PVR velero/pvr-1: Update error", }, { name: "pvr is cancel on accepted", pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Result(), expectCancelRecord: true, expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeRestorePhaseCanceled).Result(), }, { name: "pvr expose failed", pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).PodNamespace("test-ns").PodName("test-pod").Finalizers([]string{PodVolumeFinalizer}).Result(), targetPod: builder.ForPod("test-ns", "test-pod").InitContainers(&corev1api.Container{Name: restorehelper.WaitInitContainer}).InitContainerState(corev1api.ContainerState{Running: &corev1api.ContainerStateRunning{}}).Result(), mockExposeErr: boolptr.True(), expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Phase(velerov1api.PodVolumeRestorePhaseFailed).Message("error to expose PVR").Result(), expectedErr: "Error to expose restore exposer", }, { name: "pvr succeeds for accepted", pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).PodNamespace("test-ns").PodName("test-pod").Finalizers([]string{PodVolumeFinalizer}).Result(), mockExposeErr: boolptr.False(), notMockCleanUp: true, targetPod: builder.ForPod("test-ns", "test-pod").InitContainers(&corev1api.Container{Name: restorehelper.WaitInitContainer}).InitContainerState(corev1api.ContainerState{Running: &corev1api.ContainerStateRunning{}}).Result(), expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Phase(velerov1api.PodVolumeRestorePhaseAccepted).Result(), }, { name: "prepare timeout on accepted", pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseAccepted).Finalizers([]string{PodVolumeFinalizer}).AcceptedTimestamp(&metav1.Time{Time: time.Now().Add(-time.Minute * 30)}).Result(), expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseFailed).Finalizers([]string{PodVolumeFinalizer}).Phase(velerov1api.PodVolumeRestorePhaseFailed).Message("timeout on preparing PVR").Result(), }, { name: "peek error on accepted", pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseAccepted).Finalizers([]string{PodVolumeFinalizer}).Result(), isPeekExposeErr: true, expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseCanceled).Finalizers([]string{PodVolumeFinalizer}).Phase(velerov1api.PodVolumeRestorePhaseCanceled).Message("found a PVR velero/pvr-1 with expose error: fake-peek-error. mark it as cancel").Result(), }, { name: "cancel on pvr", pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhasePrepared).Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Node("test-node").Result(), expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseCanceled).Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeRestorePhaseCanceled).Result(), }, { name: "Failed to get restore expose on prepared", pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhasePrepared).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), isGetExposeErr: true, expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseFailed).Finalizers([]string{PodVolumeFinalizer}).Message("exposed PVR is not ready").Result(), expectedErr: "Error to get PVR exposer", }, { name: "Get nil restore expose on prepared", pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhasePrepared).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), isGetExposeNil: true, expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseFailed).Finalizers([]string{PodVolumeFinalizer}).Message("exposed PVR is not ready").Result(), expectedErr: "no expose result is available for the current node", }, { name: "Error in data path is concurrent limited", pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhasePrepared).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), dataMgr: datapath.NewManager(0), notNilExpose: true, notMockCleanUp: true, expectedResult: &ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, }, { name: "data path init error", pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhasePrepared).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), mockInit: true, mockInitErr: errors.New("fake-data-path-init-error"), mockClose: true, notNilExpose: true, expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseFailed).Finalizers([]string{PodVolumeFinalizer}).Message("error initializing data path").Result(), expectedErr: "error initializing asyncBR: fake-data-path-init-error", }, { name: "Unable to update status to in progress for pvr", pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhasePrepared).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), needErrs: []bool{false, false, true, false}, mockInit: true, mockClose: true, notNilExpose: true, notMockCleanUp: true, expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhasePrepared).Finalizers([]string{PodVolumeFinalizer}).Result(), }, { name: "data path start error", pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhasePrepared).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), mockInit: true, mockStart: true, mockStartErr: errors.New("fake-data-path-start-error"), mockClose: true, notNilExpose: true, expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseFailed).Finalizers([]string{PodVolumeFinalizer}).Message("error starting data path").Result(), expectedErr: "error starting async restore for pod test-name, volume test-pvc: fake-data-path-start-error", }, { name: "Prepare succeeds", pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhasePrepared).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), mockInit: true, mockStart: true, notNilExpose: true, notMockCleanUp: true, expectDataPath: true, expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseInProgress).Finalizers([]string{PodVolumeFinalizer}).Result(), }, { name: "In progress pvr is not handled by the current node", pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseInProgress).Finalizers([]string{PodVolumeFinalizer}).Result(), expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseInProgress).Finalizers([]string{PodVolumeFinalizer}).Result(), }, { name: "In progress pvr is not set as cancel", pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseInProgress).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseInProgress).Finalizers([]string{PodVolumeFinalizer}).Result(), }, { name: "Cancel pvr in progress with empty FSBR", pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseInProgress).Cancel(true).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseCanceled).Cancel(true).Finalizers([]string{PodVolumeFinalizer}).Result(), }, { name: "Cancel pvr in progress and patch pvr error", pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseInProgress).Cancel(true).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), needErrs: []bool{false, false, true, false}, needCreateFSBR: true, expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseInProgress).Cancel(true).Finalizers([]string{PodVolumeFinalizer}).Result(), expectedErr: "error updating PVR velero/pvr-1: Update error", expectCancelRecord: true, expectDataPath: true, }, { name: "Cancel pvr in progress succeeds", pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseInProgress).Cancel(true).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), needCreateFSBR: true, mockCancel: true, expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseCanceling).Cancel(true).Finalizers([]string{PodVolumeFinalizer}).Result(), expectDataPath: true, expectCancelRecord: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { objs := []runtime.Object{daemonSet, node} ctlObj := []client.Object{} if test.targetPod != nil { ctlObj = append(ctlObj, test.targetPod) } r, err := initPodVolumeRestoreReconciler(objs, ctlObj, test.needErrs...) require.NoError(t, err) if !test.notCreatePVR { err = r.client.Create(t.Context(), test.pvr) require.NoError(t, err) } if test.needDelete { err = r.client.Delete(t.Context(), test.pvr) require.NoError(t, err) } if test.dataMgr != nil { r.dataPathMgr = test.dataMgr } else { r.dataPathMgr = datapath.NewManager(1) } if test.sportTime != nil { r.cancelledPVR[test.pvr.Name] = test.sportTime.Time } if test.constrained { r.vgdpCounter = &exposer.VgdpCounter{} } funcExclusiveUpdatePodVolumeRestore = exclusiveUpdatePodVolumeRestore if test.needExclusiveUpdateError != nil { funcExclusiveUpdatePodVolumeRestore = func(context.Context, kbclient.Client, *velerov1api.PodVolumeRestore, func(*velerov1api.PodVolumeRestore)) (bool, error) { return false, test.needExclusiveUpdateError } } datapath.MicroServiceBRWatcherCreator = func(kbclient.Client, kubernetes.Interface, manager.Manager, string, string, string, string, string, string, datapath.Callbacks, logrus.FieldLogger) datapath.AsyncBR { asyncBR := datapathmockes.NewAsyncBR(t) if test.mockInit { asyncBR.On("Init", mock.Anything, mock.Anything).Return(test.mockInitErr) } if test.mockStart { asyncBR.On("StartRestore", mock.Anything, mock.Anything, mock.Anything).Return(test.mockStartErr) } if test.mockCancel { asyncBR.On("Cancel").Return() } if test.mockClose { asyncBR.On("Close", mock.Anything).Return() } return asyncBR } if test.mockExposeErr != nil || test.isGetExposeErr || test.isGetExposeNil || test.isPeekExposeErr || test.isNilExposer || test.notNilExpose { if test.isNilExposer { r.exposer = nil } else { r.exposer = func() exposer.PodVolumeExposer { ep := exposermockes.NewMockPodVolumeExposer(t) if test.mockExposeErr != nil { if boolptr.IsSetToTrue(test.mockExposeErr) { ep.On("Expose", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(errors.New("Error to expose restore exposer")) } else { ep.On("Expose", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) } } else if test.notNilExpose { hostingPod := builder.ForPod("test-ns", "test-name").Volumes(&corev1api.Volume{Name: "test-pvc"}).Result() hostingPod.ObjectMeta.SetUID("test-uid") ep.On("GetExposed", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&exposer.ExposeResult{ByPod: exposer.ExposeByPod{HostingPod: hostingPod, VolumeName: "test-pvc"}}, nil) } else if test.isGetExposeErr { ep.On("GetExposed", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, errors.New("Error to get PVR exposer")) } else if test.isGetExposeNil { ep.On("GetExposed", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, nil) } else if test.isPeekExposeErr { ep.On("PeekExposed", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(errors.New("fake-peek-error")) ep.On("DiagnoseExpose", mock.Anything, mock.Anything).Return("") } if !test.notMockCleanUp { ep.On("CleanUp", mock.Anything, mock.Anything).Return() } return ep }() } } if test.needCreateFSBR { if fsBR := r.dataPathMgr.GetAsyncBR(test.pvr.Name); fsBR == nil { _, err := r.dataPathMgr.CreateMicroServiceBRWatcher(ctx, r.client, nil, nil, datapath.TaskTypeRestore, test.pvr.Name, pVBRRequestor, velerov1api.DefaultNamespace, "", "", datapath.Callbacks{OnCancelled: r.OnDataPathCancelled}, false, velerotest.NewLogger()) require.NoError(t, err) } } actualResult, err := r.Reconcile(ctx, ctrl.Request{ NamespacedName: types.NamespacedName{ Namespace: velerov1api.DefaultNamespace, Name: test.pvr.Name, }, }) if test.expectedErr != "" { require.EqualError(t, err, test.expectedErr) } else { require.NoError(t, err) } if test.expectedResult != nil { assert.Equal(t, test.expectedResult.Requeue, actualResult.Requeue) assert.Equal(t, test.expectedResult.RequeueAfter, actualResult.RequeueAfter) } pvr := velerov1api.PodVolumeRestore{} err = r.client.Get(ctx, kbclient.ObjectKey{ Name: test.pvr.Name, Namespace: test.pvr.Namespace, }, &pvr) if test.expected != nil || test.expectDeleted { if test.expectDeleted { assert.True(t, apierrors.IsNotFound(err)) } else { require.NoError(t, err) assert.Equal(t, test.expected.Status.Phase, pvr.Status.Phase) assert.Contains(t, pvr.Status.Message, test.expected.Status.Message) assert.Equal(t, test.expected.Finalizers, pvr.Finalizers) assert.Equal(t, test.expected.Spec.Cancel, pvr.Spec.Cancel) } } if !test.expectDataPath { assert.Nil(t, r.dataPathMgr.GetAsyncBR(test.pvr.Name)) } else { assert.NotNil(t, r.dataPathMgr.GetAsyncBR(test.pvr.Name)) } if test.expectCancelRecord { assert.Contains(t, r.cancelledPVR, test.pvr.Name) } else { assert.Empty(t, r.cancelledPVR) } if isPVRInFinalState(&pvr) || pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseInProgress { assert.NotContains(t, pvr.Labels, exposer.ExposeOnGoingLabel) } else if pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseAccepted { assert.Contains(t, pvr.Labels, exposer.ExposeOnGoingLabel) } }) } } func TestPodVolumeRestoreSetupExposeParam(t *testing.T) { // common objects for all cases node := builder.ForNode("worker-1").Labels(map[string]string{kube.NodeOSLabel: kube.NodeOSLinux}).Result() basePVR := pvrBuilder().Result() basePVR.Status.Node = "worker-1" basePVR.Spec.Pod.Namespace = "app-ns" basePVR.Spec.Pod.Name = "app-pod" basePVR.Spec.Volume = "data-vol" type args struct { customLabels map[string]string customAnnotations map[string]string } type want struct { labels map[string]string annotations map[string]string } tests := []struct { name string args args want want }{ { name: "label has customize values", args: args{ customLabels: map[string]string{"custom-label": "label-value"}, customAnnotations: nil, }, want: want{ labels: map[string]string{ velerov1api.PVRLabel: basePVR.Name, "custom-label": "label-value", }, annotations: map[string]string{}, }, }, { name: "label has no customize values", args: args{ customLabels: nil, customAnnotations: nil, }, want: want{ labels: map[string]string{velerov1api.PVRLabel: basePVR.Name}, annotations: map[string]string{}, }, }, { name: "annotation has customize values", args: args{ customLabels: nil, customAnnotations: map[string]string{"custom-annotation": "annotation-value"}, }, want: want{ labels: map[string]string{velerov1api.PVRLabel: basePVR.Name}, annotations: map[string]string{"custom-annotation": "annotation-value"}, }, }, { name: "annotation has no customize values", args: args{ customLabels: map[string]string{"another-label": "lval"}, customAnnotations: nil, }, want: want{ labels: map[string]string{ velerov1api.PVRLabel: basePVR.Name, "another-label": "lval", }, annotations: map[string]string{}, }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { // Fake clients per case fakeCRClient := velerotest.NewFakeControllerRuntimeClient(t, node, basePVR.DeepCopy()) fakeKubeClient := clientgofake.NewSimpleClientset(node) // Reconciler config per case preparingTimeout := time.Minute * 3 resourceTimeout := time.Minute * 10 podRes := corev1api.ResourceRequirements{} r := NewPodVolumeRestoreReconciler( fakeCRClient, nil, fakeKubeClient, datapath.NewManager(1), nil, "test-node", preparingTimeout, resourceTimeout, nil, // backupRepoConfigs nil, // cacheVolumeConfigs -> keep nil so CacheVolume is nil podRes, velerotest.NewLogger(), "restore-priority", true, nil, // repoConfigMgr (unused when cacheVolumeConfigs is nil) tt.args.customLabels, tt.args.customAnnotations, ) // Act got := r.setupExposeParam(basePVR) // Core fields assert.Equal(t, exposer.PodVolumeExposeTypeRestore, got.Type) assert.Equal(t, basePVR.Spec.Pod.Namespace, got.ClientNamespace) assert.Equal(t, basePVR.Spec.Pod.Name, got.ClientPodName) assert.Equal(t, basePVR.Spec.Volume, got.ClientPodVolume) // Labels/Annotations assert.Equal(t, tt.want.labels, got.HostingPodLabels) assert.Equal(t, tt.want.annotations, got.HostingPodAnnotations) }) } } func TestOnPodVolumeRestoreFailed(t *testing.T) { for _, getErr := range []bool{true, false} { ctx := t.Context() needErrs := []bool{getErr, false, false, false} r, err := initPodVolumeRestoreReconciler(nil, []client.Object{}, needErrs...) require.NoError(t, err) pvr := pvrBuilder().Result() namespace := pvr.Namespace pvrName := pvr.Name require.NoError(t, r.client.Create(ctx, pvr)) r.OnDataPathFailed(ctx, namespace, pvrName, fmt.Errorf("Failed to handle %v", pvrName)) updatedPVR := &velerov1api.PodVolumeRestore{} if getErr { require.Error(t, r.client.Get(ctx, types.NamespacedName{Name: pvrName, Namespace: namespace}, updatedPVR)) assert.NotEqual(t, velerov1api.PodVolumeRestorePhaseFailed, updatedPVR.Status.Phase) assert.True(t, updatedPVR.Status.StartTimestamp.IsZero()) } else { require.NoError(t, r.client.Get(ctx, types.NamespacedName{Name: pvrName, Namespace: namespace}, updatedPVR)) assert.Equal(t, velerov1api.PodVolumeRestorePhaseFailed, updatedPVR.Status.Phase) assert.True(t, updatedPVR.Status.StartTimestamp.IsZero()) } } } func TestOnPodVolumeRestoreCancelled(t *testing.T) { for _, getErr := range []bool{true, false} { ctx := t.Context() needErrs := []bool{getErr, false, false, false} r, err := initPodVolumeRestoreReconciler(nil, nil, needErrs...) require.NoError(t, err) pvr := pvrBuilder().Result() namespace := pvr.Namespace pvrName := pvr.Name require.NoError(t, r.client.Create(ctx, pvr)) r.OnDataPathCancelled(ctx, namespace, pvrName) updatedPVR := &velerov1api.PodVolumeRestore{} if getErr { require.Error(t, r.client.Get(ctx, types.NamespacedName{Name: pvrName, Namespace: namespace}, updatedPVR)) assert.NotEqual(t, velerov1api.PodVolumeRestorePhaseFailed, updatedPVR.Status.Phase) assert.True(t, updatedPVR.Status.StartTimestamp.IsZero()) } else { require.NoError(t, r.client.Get(ctx, types.NamespacedName{Name: pvrName, Namespace: namespace}, updatedPVR)) assert.Equal(t, velerov1api.PodVolumeRestorePhaseCanceled, updatedPVR.Status.Phase) assert.False(t, updatedPVR.Status.StartTimestamp.IsZero()) assert.False(t, updatedPVR.Status.CompletionTimestamp.IsZero()) } } } func TestOnPodVolumeRestoreCompleted(t *testing.T) { tests := []struct { name string emptyFSBR bool isGetErr bool rebindVolumeErr bool }{ { name: "PVR complete", emptyFSBR: false, isGetErr: false, rebindVolumeErr: false, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { ctx := t.Context() needErrs := []bool{test.isGetErr, false, false, false} r, err := initPodVolumeRestoreReconciler(nil, []client.Object{}, needErrs...) r.exposer = func() exposer.PodVolumeExposer { ep := exposermockes.NewMockPodVolumeExposer(t) ep.On("CleanUp", mock.Anything, mock.Anything).Return() return ep }() require.NoError(t, err) pvr := builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Result() namespace := pvr.Namespace ddName := pvr.Name require.NoError(t, r.client.Create(ctx, pvr)) r.OnDataPathCompleted(ctx, namespace, ddName, datapath.Result{}) updatedDD := &velerov1api.PodVolumeRestore{} if test.isGetErr { require.Error(t, r.client.Get(ctx, types.NamespacedName{Name: ddName, Namespace: namespace}, updatedDD)) assert.Equal(t, velerov1api.PodVolumeRestorePhase(""), updatedDD.Status.Phase) assert.True(t, updatedDD.Status.CompletionTimestamp.IsZero()) } else { require.NoError(t, r.client.Get(ctx, types.NamespacedName{Name: ddName, Namespace: namespace}, updatedDD)) assert.Equal(t, velerov1api.PodVolumeRestorePhaseCompleted, updatedDD.Status.Phase) assert.False(t, updatedDD.Status.CompletionTimestamp.IsZero()) } }) } } func TestOnPodVolumeRestoreProgress(t *testing.T) { totalBytes := int64(1024) bytesDone := int64(512) tests := []struct { name string pvr *velerov1api.PodVolumeRestore progress uploader.Progress needErrs []bool }{ { name: "patch in progress phase success", pvr: pvrBuilder().Result(), progress: uploader.Progress{ TotalBytes: totalBytes, BytesDone: bytesDone, }, }, { name: "failed to get pvr", pvr: pvrBuilder().Result(), needErrs: []bool{true, false, false, false}, }, { name: "failed to patch pvr", pvr: pvrBuilder().Result(), needErrs: []bool{false, false, true, false}, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { ctx := t.Context() r, err := initPodVolumeRestoreReconciler(nil, []client.Object{}, test.needErrs...) require.NoError(t, err) defer func() { r.client.Delete(ctx, test.pvr, &kbclient.DeleteOptions{}) }() pvr := pvrBuilder().Result() namespace := pvr.Namespace pvrName := pvr.Name require.NoError(t, r.client.Create(t.Context(), pvr)) // Create a Progress object progress := &uploader.Progress{ TotalBytes: totalBytes, BytesDone: bytesDone, } r.OnDataPathProgress(ctx, namespace, pvrName, progress) if len(test.needErrs) != 0 && !test.needErrs[0] { updatedPVR := &velerov1api.PodVolumeRestore{} require.NoError(t, r.client.Get(ctx, types.NamespacedName{Name: pvrName, Namespace: namespace}, updatedPVR)) assert.Equal(t, test.progress.TotalBytes, updatedPVR.Status.Progress.TotalBytes) assert.Equal(t, test.progress.BytesDone, updatedPVR.Status.Progress.BytesDone) } }) } } func TestFindPVBForRestorePod(t *testing.T) { needErrs := []bool{false, false, false, false} r, err := initPodVolumeRestoreReconciler(nil, []client.Object{}, needErrs...) require.NoError(t, err) tests := []struct { name string pvr *velerov1api.PodVolumeRestore pod *corev1api.Pod checkFunc func(*velerov1api.PodVolumeRestore, []reconcile.Request) }{ { name: "find pvr for pod", pvr: pvrBuilder().Phase(velerov1api.PodVolumeRestorePhaseAccepted).Result(), pod: builder.ForPod(velerov1api.DefaultNamespace, pvrName).Labels(map[string]string{velerov1api.PVRLabel: pvrName}).Status(corev1api.PodStatus{Phase: corev1api.PodRunning}).Result(), checkFunc: func(pvr *velerov1api.PodVolumeRestore, requests []reconcile.Request) { // Assert that the function returns a single request assert.Len(t, requests, 1) // Assert that the request contains the correct namespaced name assert.Equal(t, pvr.Namespace, requests[0].Namespace) assert.Equal(t, pvr.Name, requests[0].Name) }, }, { name: "no selected label found for pod", pvr: pvrBuilder().Phase(velerov1api.PodVolumeRestorePhaseAccepted).Result(), pod: builder.ForPod(velerov1api.DefaultNamespace, pvrName).Result(), checkFunc: func(pvr *velerov1api.PodVolumeRestore, requests []reconcile.Request) { // Assert that the function returns a single request assert.Empty(t, requests) }, }, { name: "no matched pod", pvr: pvrBuilder().Phase(velerov1api.PodVolumeRestorePhaseAccepted).Result(), pod: builder.ForPod(velerov1api.DefaultNamespace, pvrName).Labels(map[string]string{velerov1api.PVRLabel: "non-existing-pvr"}).Result(), checkFunc: func(pvr *velerov1api.PodVolumeRestore, requests []reconcile.Request) { assert.Empty(t, requests) }, }, { name: "pvr not accept", pvr: pvrBuilder().Phase(velerov1api.PodVolumeRestorePhaseInProgress).Result(), pod: builder.ForPod(velerov1api.DefaultNamespace, pvrName).Labels(map[string]string{velerov1api.PVRLabel: pvrName}).Result(), checkFunc: func(pvr *velerov1api.PodVolumeRestore, requests []reconcile.Request) { assert.Empty(t, requests) }, }, } for _, test := range tests { ctx := t.Context() assert.NoError(t, r.client.Create(ctx, test.pod)) assert.NoError(t, r.client.Create(ctx, test.pvr)) // Call the findSnapshotRestoreForPod function requests := r.findPVRForRestorePod(t.Context(), test.pod) test.checkFunc(test.pvr, requests) r.client.Delete(ctx, test.pvr, &kbclient.DeleteOptions{}) if test.pod != nil { r.client.Delete(ctx, test.pod, &kbclient.DeleteOptions{}) } } } func TestOnPVRPrepareTimeout(t *testing.T) { tests := []struct { name string pvr *velerov1api.PodVolumeRestore needErrs []error expected *velerov1api.PodVolumeRestore }{ { name: "update fail", pvr: pvrBuilder().Result(), needErrs: []error{nil, nil, fmt.Errorf("fake-update-error"), nil}, expected: pvrBuilder().Result(), }, { name: "update interrupted", pvr: pvrBuilder().Result(), needErrs: []error{nil, nil, &fakeAPIStatus{metav1.StatusReasonConflict}, nil}, expected: pvrBuilder().Result(), }, { name: "succeed", pvr: pvrBuilder().Result(), needErrs: []error{nil, nil, nil, nil}, expected: pvrBuilder().Phase(velerov1api.PodVolumeRestorePhaseFailed).Result(), }, } for _, test := range tests { ctx := t.Context() r, err := initPodVolumeRestoreReconcilerWithError(nil, []client.Object{}, test.needErrs...) require.NoError(t, err) err = r.client.Create(ctx, test.pvr) require.NoError(t, err) r.onPrepareTimeout(ctx, test.pvr) pvr := velerov1api.PodVolumeRestore{} _ = r.client.Get(ctx, kbclient.ObjectKey{ Name: test.pvr.Name, Namespace: test.pvr.Namespace, }, &pvr) assert.Equal(t, test.expected.Status.Phase, pvr.Status.Phase) } } func TestTryCancelPVR(t *testing.T) { tests := []struct { name string pvr *velerov1api.PodVolumeRestore needErrs []error succeeded bool expectedErr string }{ { name: "update fail", pvr: pvrBuilder().Result(), needErrs: []error{nil, nil, fmt.Errorf("fake-update-error"), nil}, }, { name: "cancel by others", pvr: pvrBuilder().Result(), needErrs: []error{nil, nil, &fakeAPIStatus{metav1.StatusReasonConflict}, nil}, }, { name: "succeed", pvr: pvrBuilder().Result(), needErrs: []error{nil, nil, nil, nil}, succeeded: true, }, } for _, test := range tests { ctx := t.Context() r, err := initPodVolumeRestoreReconcilerWithError(nil, []client.Object{}, test.needErrs...) require.NoError(t, err) err = r.client.Create(ctx, test.pvr) require.NoError(t, err) r.tryCancelPodVolumeRestore(ctx, test.pvr, "") if test.expectedErr == "" { assert.NoError(t, err) } else { assert.EqualError(t, err, test.expectedErr) } } } func TestUpdatePVRWithRetry(t *testing.T) { namespacedName := types.NamespacedName{ Name: pvrName, Namespace: "velero", } // Define test cases testCases := []struct { Name string needErrs []bool noChange bool ExpectErr bool }{ { Name: "SuccessOnFirstAttempt", }, { Name: "Error get", needErrs: []bool{true, false, false, false, false}, ExpectErr: true, }, { Name: "Error update", needErrs: []bool{false, false, true, false, false}, ExpectErr: true, }, { Name: "no change", noChange: true, needErrs: []bool{false, false, true, false, false}, }, { Name: "Conflict with error timeout", needErrs: []bool{false, false, false, false, true}, ExpectErr: true, }, } for _, tc := range testCases { t.Run(tc.Name, func(t *testing.T) { ctx, cancelFunc := context.WithTimeout(t.Context(), time.Second*5) defer cancelFunc() r, err := initPodVolumeRestoreReconciler(nil, []client.Object{}, tc.needErrs...) require.NoError(t, err) err = r.client.Create(ctx, pvrBuilder().Result()) require.NoError(t, err) updateFunc := func(pvr *velerov1api.PodVolumeRestore) bool { if tc.noChange { return false } pvr.Spec.Cancel = true return true } err = UpdatePVRWithRetry(ctx, r.client, namespacedName, velerotest.NewLogger().WithField("name", tc.Name), updateFunc) if tc.ExpectErr { assert.Error(t, err) } else { assert.NoError(t, err) } }) } } func TestAttemptPVRResume(t *testing.T) { tests := []struct { name string pvrs []velerov1api.PodVolumeRestore pvr *velerov1api.PodVolumeRestore needErrs []bool resumeErr error acceptedPvrs []string preparedPvrs []string cancelledPvrs []string inProgressPvrs []string expectedError string }{ { name: "Other pvr", pvr: pvrBuilder().Phase(velerov1api.PodVolumeRestorePhasePrepared).Result(), }, { name: "Other pvr", pvr: pvrBuilder().Phase(velerov1api.PodVolumeRestorePhaseAccepted).Result(), }, { name: "InProgress pvr, not the current node", pvr: pvrBuilder().Phase(velerov1api.PodVolumeRestorePhaseInProgress).Result(), inProgressPvrs: []string{pvrName}, }, { name: "InProgress pvr, no resume error", pvr: pvrBuilder().Phase(velerov1api.PodVolumeRestorePhaseInProgress).Node("node-1").Result(), inProgressPvrs: []string{pvrName}, }, { name: "InProgress pvr, resume error, cancel error", pvr: pvrBuilder().Phase(velerov1api.PodVolumeRestorePhaseInProgress).Node("node-1").Result(), resumeErr: errors.New("fake-resume-error"), needErrs: []bool{false, false, true, false, false, false}, inProgressPvrs: []string{pvrName}, }, { name: "InProgress pvr, resume error, cancel succeed", pvr: pvrBuilder().Phase(velerov1api.PodVolumeRestorePhaseInProgress).Node("node-1").Result(), resumeErr: errors.New("fake-resume-error"), cancelledPvrs: []string{pvrName}, inProgressPvrs: []string{pvrName}, }, { name: "Error", needErrs: []bool{false, false, false, false, false, true}, pvr: pvrBuilder().Phase(velerov1api.PodVolumeRestorePhasePrepared).Result(), expectedError: "error to list PVRs: List error", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { ctx := t.Context() r, err := initPodVolumeRestoreReconciler(nil, []client.Object{}, test.needErrs...) r.nodeName = "node-1" require.NoError(t, err) defer func() { r.client.Delete(ctx, test.pvr, &kbclient.DeleteOptions{}) }() require.NoError(t, r.client.Create(ctx, test.pvr)) dt := &pvbResumeTestHelper{ resumeErr: test.resumeErr, } funcResumeCancellableDataBackup = dt.resumeCancellableDataPath // Run the test err = r.AttemptPVRResume(ctx, r.logger.WithField("name", test.name), test.pvr.Namespace) if test.expectedError != "" { assert.EqualError(t, err, test.expectedError) } else { assert.NoError(t, err) for _, pvrName := range test.cancelledPvrs { pvr := &velerov1api.PodVolumeRestore{} err := r.client.Get(t.Context(), types.NamespacedName{Namespace: "velero", Name: pvrName}, pvr) require.NoError(t, err) assert.True(t, pvr.Spec.Cancel) } for _, pvrName := range test.acceptedPvrs { pvr := &velerov1api.PodVolumeRestore{} err := r.client.Get(t.Context(), types.NamespacedName{Namespace: "velero", Name: pvrName}, pvr) require.NoError(t, err) assert.Equal(t, velerov1api.PodVolumeRestorePhaseAccepted, pvr.Status.Phase) } for _, pvrName := range test.preparedPvrs { pvr := &velerov1api.PodVolumeRestore{} err := r.client.Get(t.Context(), types.NamespacedName{Namespace: "velero", Name: pvrName}, pvr) require.NoError(t, err) assert.Equal(t, velerov1api.PodVolumeRestorePhasePrepared, pvr.Status.Phase) } } }) } } func TestResumeCancellablePodVolumeRestore(t *testing.T) { tests := []struct { name string pvrs []velerov1api.PodVolumeRestore pvr *velerov1api.PodVolumeRestore getExposeErr error exposeResult *exposer.ExposeResult createWatcherErr error initWatcherErr error startWatcherErr error mockInit bool mockStart bool mockClose bool expectedError string }{ { name: "get expose failed", pvr: pvrBuilder().Phase(velerov1api.PodVolumeRestorePhaseInProgress).Result(), getExposeErr: errors.New("fake-expose-error"), expectedError: fmt.Sprintf("error to get exposed PVR %s: fake-expose-error", pvrName), }, { name: "no expose", pvr: pvrBuilder().Phase(velerov1api.PodVolumeRestorePhaseAccepted).Node("node-1").Result(), expectedError: fmt.Sprintf("no expose result is available for the current node for PVR %s", pvrName), }, { name: "watcher init error", pvr: pvrBuilder().Phase(velerov1api.PodVolumeRestorePhaseAccepted).Node("node-1").Result(), exposeResult: &exposer.ExposeResult{ ByPod: exposer.ExposeByPod{ HostingPod: &corev1api.Pod{}, }, }, mockInit: true, mockClose: true, initWatcherErr: errors.New("fake-init-watcher-error"), expectedError: fmt.Sprintf("error to init asyncBR watcher for PVR %s: fake-init-watcher-error", pvrName), }, { name: "start watcher error", pvr: pvrBuilder().Phase(velerov1api.PodVolumeRestorePhaseAccepted).Node("node-1").Result(), exposeResult: &exposer.ExposeResult{ ByPod: exposer.ExposeByPod{ HostingPod: &corev1api.Pod{}, }, }, mockInit: true, mockStart: true, mockClose: true, startWatcherErr: errors.New("fake-start-watcher-error"), expectedError: fmt.Sprintf("error to resume asyncBR watcher for PVR %s: fake-start-watcher-error", pvrName), }, { name: "succeed", pvr: pvrBuilder().Phase(velerov1api.PodVolumeRestorePhaseAccepted).Node("node-1").Result(), exposeResult: &exposer.ExposeResult{ ByPod: exposer.ExposeByPod{ HostingPod: &corev1api.Pod{}, }, }, mockInit: true, mockStart: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { ctx := t.Context() r, err := initPodVolumeRestoreReconciler(nil, []client.Object{}) r.nodeName = "node-1" require.NoError(t, err) mockAsyncBR := datapathmockes.NewAsyncBR(t) if test.mockInit { mockAsyncBR.On("Init", mock.Anything, mock.Anything).Return(test.initWatcherErr) } if test.mockStart { mockAsyncBR.On("StartRestore", mock.Anything, mock.Anything, mock.Anything).Return(test.startWatcherErr) } if test.mockClose { mockAsyncBR.On("Close", mock.Anything).Return() } dt := &pvbResumeTestHelper{ getExposeErr: test.getExposeErr, exposeResult: test.exposeResult, asyncBR: mockAsyncBR, } r.exposer = dt datapath.MicroServiceBRWatcherCreator = dt.newMicroServiceBRWatcher err = r.resumeCancellableDataPath(ctx, test.pvr, velerotest.NewLogger()) if test.expectedError != "" { assert.EqualError(t, err, test.expectedError) } }) } } ================================================ FILE: pkg/controller/restore_controller.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package controller import ( "bytes" "compress/gzip" "context" "encoding/json" "fmt" "io" "os" "sort" "strings" "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/utils/clock" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "github.com/vmware-tanzu/velero/internal/hook" "github.com/vmware-tanzu/velero/internal/resourcemodifiers" "github.com/vmware-tanzu/velero/internal/volume" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/constant" "github.com/vmware-tanzu/velero/pkg/itemoperation" "github.com/vmware-tanzu/velero/pkg/label" "github.com/vmware-tanzu/velero/pkg/metrics" "github.com/vmware-tanzu/velero/pkg/persistence" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" "github.com/vmware-tanzu/velero/pkg/plugin/framework" pkgrestore "github.com/vmware-tanzu/velero/pkg/restore" "github.com/vmware-tanzu/velero/pkg/util/collections" kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube" "github.com/vmware-tanzu/velero/pkg/util/logging" "github.com/vmware-tanzu/velero/pkg/util/results" veleroutil "github.com/vmware-tanzu/velero/pkg/util/velero" pkgrestoreUtil "github.com/vmware-tanzu/velero/pkg/util/velero/restore" ) // nonRestorableResources is an exclusion list for the restoration process. Any resources // included here are explicitly excluded from the restoration process. var nonRestorableResources = []string{ "nodes", "events", "events.events.k8s.io", // Don't ever restore backups - if appropriate, they'll be synced in from object storage. // https://github.com/vmware-tanzu/velero/issues/622 "backups.velero.io", // Restores are cluster-specific, and don't have value moving across clusters. // https://github.com/vmware-tanzu/velero/issues/622 "restores.velero.io", // TODO: Remove this in v1.11 or v1.12 // Restic repositories are automatically managed by Velero and will be automatically // created as needed if they don't exist. // https://github.com/vmware-tanzu/velero/issues/1113 "resticrepositories.velero.io", // CSINode delegates cluster node for CSI operation. // VolumeAttachement records PV mounts to which node. // https://github.com/vmware-tanzu/velero/issues/4823 "csinodes.storage.k8s.io", "volumeattachments.storage.k8s.io", // Backup repositories were renamed from Restic repositories "backuprepositories.velero.io", } var ExternalResourcesFinalizer = "restores.velero.io/external-resources-finalizer" type restoreReconciler struct { ctx context.Context namespace string restorer pkgrestore.Restorer kbClient client.Client restoreLogLevel logrus.Level logger logrus.FieldLogger metrics *metrics.ServerMetrics logFormat logging.Format clock clock.WithTickerAndDelayedExecution defaultItemOperationTimeout time.Duration disableInformerCache bool newPluginManager func(logger logrus.FieldLogger) clientmgmt.Manager backupStoreGetter persistence.ObjectBackupStoreGetter globalCrClient client.Client resourceTimeout time.Duration } type backupInfo struct { backup *api.Backup location *api.BackupStorageLocation } func NewRestoreReconciler( ctx context.Context, namespace string, restorer pkgrestore.Restorer, kbClient client.Client, logger logrus.FieldLogger, restoreLogLevel logrus.Level, newPluginManager func(logrus.FieldLogger) clientmgmt.Manager, backupStoreGetter persistence.ObjectBackupStoreGetter, metrics *metrics.ServerMetrics, logFormat logging.Format, defaultItemOperationTimeout time.Duration, disableInformerCache bool, globalCrClient client.Client, resourceTimeout time.Duration, ) *restoreReconciler { r := &restoreReconciler{ ctx: ctx, namespace: namespace, restorer: restorer, kbClient: kbClient, logger: logger, restoreLogLevel: restoreLogLevel, metrics: metrics, logFormat: logFormat, clock: &clock.RealClock{}, defaultItemOperationTimeout: defaultItemOperationTimeout, disableInformerCache: disableInformerCache, // use variables to refer to these functions so they can be // replaced with fakes for testing. newPluginManager: newPluginManager, backupStoreGetter: backupStoreGetter, globalCrClient: globalCrClient, resourceTimeout: resourceTimeout, } // Move the periodical backup and restore metrics computing logic from controllers to here. // This is due to, after controllers using controller-runtime, controllers doesn't have a // timer as the before generic-controller, and the backup and restore controller only have // one length queue, furthermore the backup and restore process could last for a long time. // Compute the metric here is a better choice. r.updateTotalRestoreMetric() return r } func (r *restoreReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { // Developer note: any error returned by this method will // cause the restore to be re-enqueued and re-processed by // the controller. log := r.logger.WithField("Restore", req.NamespacedName.String()) restore := &api.Restore{} err := r.kbClient.Get(ctx, client.ObjectKey{Namespace: req.Namespace, Name: req.Name}, restore) if err != nil { if apierrors.IsNotFound(err) { log.Debugf("restore[%s] not found", req.Name) return ctrl.Result{}, nil } log.Errorf("Fail to get restore %s: %s", req.NamespacedName.String(), err.Error()) return ctrl.Result{}, err } // deal with finalizer if !restore.DeletionTimestamp.IsZero() { // check the finalizer and run clean-up if controllerutil.ContainsFinalizer(restore, ExternalResourcesFinalizer) { if err := r.deleteExternalResources(restore); err != nil { log.Errorf("fail to delete external resources: %s", err.Error()) return ctrl.Result{}, err } // once finish clean-up, remove the finalizer from the restore so that the restore will be unlocked and deleted. original := restore.DeepCopy() controllerutil.RemoveFinalizer(restore, ExternalResourcesFinalizer) if err := kubeutil.PatchResource(original, restore, r.kbClient); err != nil { log.Errorf("fail to remove finalizer: %s", err.Error()) return ctrl.Result{}, err } return ctrl.Result{}, nil } else { log.Error("DeletionTimestamp is marked but can't find the expected finalizer") return ctrl.Result{}, nil } } // add finalizer if restore.DeletionTimestamp.IsZero() && !controllerutil.ContainsFinalizer(restore, ExternalResourcesFinalizer) { original := restore.DeepCopy() controllerutil.AddFinalizer(restore, ExternalResourcesFinalizer) if err := kubeutil.PatchResource(original, restore, r.kbClient); err != nil { log.Errorf("fail to add finalizer: %s", err.Error()) return ctrl.Result{}, err } } switch restore.Status.Phase { case "", api.RestorePhaseNew: // only process new restores default: r.logger.WithFields(logrus.Fields{ "restore": kubeutil.NamespaceAndName(restore), "phase": restore.Status.Phase, }).Debug("Restore is not handled") return ctrl.Result{}, nil } // store a copy of the original restore for creating patch original := restore.DeepCopy() // Validate the restore and fetch the backup info, resourceModifiers := r.validateAndComplete(restore) // Register attempts after validation so we don't have to fetch the backup multiple times backupScheduleName := restore.Spec.ScheduleName r.metrics.RegisterRestoreAttempt(backupScheduleName) if len(restore.Status.ValidationErrors) > 0 { restore.Status.Phase = api.RestorePhaseFailedValidation r.metrics.RegisterRestoreValidationFailed(backupScheduleName) } else { restore.Status.StartTimestamp = &metav1.Time{Time: r.clock.Now()} restore.Status.Phase = api.RestorePhaseInProgress } if restore.Spec.ItemOperationTimeout.Duration == 0 { // set default item operation timeout restore.Spec.ItemOperationTimeout.Duration = r.defaultItemOperationTimeout } // patch to update status and persist to API // This is patching from "" or New, no retry needed err = kubeutil.PatchResource(original, restore, r.kbClient) if err != nil { // return the error so the restore can be re-processed; it's currently // still in phase = New. log.Errorf("fail to update restore %s status to %s: %s", req.NamespacedName.String(), restore.Status.Phase, err.Error()) return ctrl.Result{}, errors.Wrapf(err, "error updating Restore phase to %s", restore.Status.Phase) } // store ref to just-updated item for creating patch original = restore.DeepCopy() if restore.Status.Phase == api.RestorePhaseFailedValidation { return ctrl.Result{}, nil } if err := r.runValidatedRestore(restore, info, resourceModifiers); err != nil { log.WithError(err).Debug("Restore failed") restore.Status.Phase = api.RestorePhaseFailed restore.Status.FailureReason = err.Error() r.metrics.RegisterRestoreFailed(backupScheduleName) } // mark completion if in terminal phase if restore.Status.Phase == api.RestorePhaseFailed || restore.Status.Phase == api.RestorePhasePartiallyFailed || restore.Status.Phase == api.RestorePhaseCompleted { restore.Status.CompletionTimestamp = &metav1.Time{Time: r.clock.Now()} } log.Debug("Updating restore's status") // Phases were updated in runValidatedRestore // This patch with retry update Phase from InProgress to // WaitingForPluginOperations // WaitingForPluginOperationsPartiallyFailed // Finalizing // FinalizingPartiallyFailed if err = kubeutil.PatchResourceWithRetriesOnErrors(r.resourceTimeout, original, restore, r.kbClient); err != nil { log.WithError(errors.WithStack(err)).Infof("Error updating restore's status from %v to %v", original.Status.Phase, restore.Status.Phase) // No need to re-enqueue here, because restore's already set to InProgress before. // Controller only handle New restore. } return ctrl.Result{}, nil } func (r *restoreReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&api.Restore{}). Named(constant.ControllerRestore). Complete(r) } func (r *restoreReconciler) validateAndComplete(restore *api.Restore) (backupInfo, *resourcemodifiers.ResourceModifiers) { // add non-restorable resources to restore's excluded resources excludedResources := sets.NewString(restore.Spec.ExcludedResources...) for _, nonrestorable := range nonRestorableResources { if !excludedResources.Has(nonrestorable) { restore.Spec.ExcludedResources = append(restore.Spec.ExcludedResources, nonrestorable) } } // validate that included resources don't contain any non-restorable resources includedResources := sets.NewString(restore.Spec.IncludedResources...) for _, nonRestorableResource := range nonRestorableResources { if includedResources.Has(nonRestorableResource) { restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, fmt.Sprintf("%v are non-restorable resources", nonRestorableResource)) } } // validate included/excluded resources for _, err := range collections.ValidateIncludesExcludes(restore.Spec.IncludedResources, restore.Spec.ExcludedResources) { restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, fmt.Sprintf("Invalid included/excluded resource lists: %v", err)) } // validate included/excluded namespaces for _, err := range collections.ValidateIncludesExcludes(restore.Spec.IncludedNamespaces, restore.Spec.ExcludedNamespaces) { restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, fmt.Sprintf("Invalid included/excluded namespace lists: %v", err)) } // validate that only one exists orLabelSelector or just labelSelector (singular) if restore.Spec.OrLabelSelectors != nil && restore.Spec.LabelSelector != nil { restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, "encountered labelSelector as well as orLabelSelectors in restore spec, only one can be specified") } // validate that exactly one of BackupName and ScheduleName have been specified if !backupXorScheduleProvided(restore) { restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, "Either a backup or schedule must be specified as a source for the restore, but not both") return backupInfo{}, nil } // validate Restore Init Hook's InitContainers restoreHooks, err := hook.GetRestoreHooksFromSpec(&restore.Spec.Hooks) if err != nil { restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, err.Error()) } for _, resource := range restoreHooks { for _, h := range resource.RestoreHooks { if h.Init != nil { for _, container := range h.Init.InitContainers { err = hook.ValidateContainer(container.Raw) if err != nil { restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, err.Error()) } } } } } // validate ExistingResourcePolicy if restore.Spec.ExistingResourcePolicy != "" && !pkgrestoreUtil.IsResourcePolicyValid(string(restore.Spec.ExistingResourcePolicy)) { restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, fmt.Sprintf("Invalid ExistingResourcePolicy: %s", restore.Spec.ExistingResourcePolicy)) } // if ScheduleName is specified, fill in BackupName with the most recent successful backup from // the schedule if restore.Spec.ScheduleName != "" { selector := labels.SelectorFromSet(labels.Set(map[string]string{ api.ScheduleNameLabel: restore.Spec.ScheduleName, })) backupList := &api.BackupList{} if err := r.kbClient.List(context.Background(), backupList, &client.ListOptions{LabelSelector: selector}); err != nil { restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, "Unable to list backups for schedule") return backupInfo{}, nil } if len(backupList.Items) == 0 { restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, "No backups found for schedule") } if backup := mostRecentCompletedBackup(backupList.Items); backup.Name != "" { restore.Spec.BackupName = backup.Name } else { restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, "No completed backups found for schedule") return backupInfo{}, nil } } info, err := r.fetchBackupInfo(restore.Spec.BackupName) if err != nil { restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, fmt.Sprintf("Error retrieving backup: %v", err)) return backupInfo{}, nil } if !veleroutil.BSLIsAvailable(*info.location) { restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, fmt.Sprintf("The BSL %s is unavailable, cannot retrieve the backup", info.location.Name)) return backupInfo{}, nil } // Fill in the ScheduleName so it's easier to consume for metrics. if restore.Spec.ScheduleName == "" { restore.Spec.ScheduleName = info.backup.GetLabels()[api.ScheduleNameLabel] } var resourceModifiers *resourcemodifiers.ResourceModifiers if restore.Spec.ResourceModifier != nil && strings.EqualFold(restore.Spec.ResourceModifier.Kind, resourcemodifiers.ConfigmapRefType) { ResourceModifierConfigMap := &corev1api.ConfigMap{} err := r.kbClient.Get(context.Background(), client.ObjectKey{Namespace: restore.Namespace, Name: restore.Spec.ResourceModifier.Name}, ResourceModifierConfigMap) if err != nil { restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, fmt.Sprintf("failed to get resource modifiers configmap %s/%s", restore.Namespace, restore.Spec.ResourceModifier.Name)) return backupInfo{}, nil } resourceModifiers, err = resourcemodifiers.GetResourceModifiersFromConfig(ResourceModifierConfigMap) if err != nil { restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, errors.Wrapf(err, "Error in parsing resource modifiers provided in configmap %s/%s", restore.Namespace, restore.Spec.ResourceModifier.Name).Error()) return backupInfo{}, nil } else if err = resourceModifiers.Validate(); err != nil { restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, errors.Wrapf(err, "Validation error in resource modifiers provided in configmap %s/%s", restore.Namespace, restore.Spec.ResourceModifier.Name).Error()) return backupInfo{}, nil } r.logger.Infof("Retrieved Resource modifiers provided in configmap %s/%s", restore.Namespace, restore.Spec.ResourceModifier.Name) } return info, resourceModifiers } // backupXorScheduleProvided returns true if exactly one of BackupName and // ScheduleName are non-empty for the restore, or false otherwise. func backupXorScheduleProvided(restore *api.Restore) bool { if restore.Spec.BackupName != "" && restore.Spec.ScheduleName != "" { return false } if restore.Spec.BackupName == "" && restore.Spec.ScheduleName == "" { return false } return true } // mostRecentCompletedBackup returns the most recent backup that's // completed from a list of backups. func mostRecentCompletedBackup(backups []api.Backup) api.Backup { sort.Slice(backups, func(i, j int) bool { // Use .After() because we want descending sort. var iStartTime, jStartTime time.Time if backups[i].Status.StartTimestamp != nil { iStartTime = backups[i].Status.StartTimestamp.Time } if backups[j].Status.StartTimestamp != nil { jStartTime = backups[j].Status.StartTimestamp.Time } return iStartTime.After(jStartTime) }) for _, backup := range backups { if backup.Status.Phase == api.BackupPhaseCompleted { return backup } } return api.Backup{} } // fetchBackupInfo checks the backup lister for a backup that matches the given name. If it doesn't // find it, it returns an error. func (r *restoreReconciler) fetchBackupInfo(backupName string) (backupInfo, error) { return fetchBackupInfoInternal(r.kbClient, r.namespace, backupName) } func fetchBackupInfoInternal(kbClient client.Client, namespace, backupName string) (backupInfo, error) { backup := &api.Backup{} err := kbClient.Get(context.Background(), types.NamespacedName{Namespace: namespace, Name: backupName}, backup) if err != nil { return backupInfo{}, errors.Wrap(err, fmt.Sprintf("can't find backup %s/%s", namespace, backupName)) } location := &api.BackupStorageLocation{} if err := kbClient.Get(context.Background(), client.ObjectKey{ Namespace: namespace, Name: backup.Spec.StorageLocation, }, location); err != nil { return backupInfo{}, errors.WithStack(err) } return backupInfo{ backup: backup, location: location, }, nil } // runValidatedRestore takes a validated restore API object and executes the restore process. // The log and results files are uploaded to backup storage. Any error returned from this function // means that the restore failed. This function updates the restore API object with warning and error // counts, but *does not* update its phase or patch it via the API. func (r *restoreReconciler) runValidatedRestore(restore *api.Restore, info backupInfo, resourceModifiers *resourcemodifiers.ResourceModifiers) error { // instantiate the per-restore logger that will output both to a temp file // (for upload to object storage) and to stdout. restoreLog, err := logging.NewTempFileLogger(r.restoreLogLevel, r.logFormat, nil, logrus.Fields{"restore": kubeutil.NamespaceAndName(restore)}) if err != nil { return err } defer restoreLog.Dispose(r.logger) pluginManager := r.newPluginManager(restoreLog) defer pluginManager.CleanupClients() backupStore, err := r.backupStoreGetter.Get(info.location, pluginManager, r.logger) if err != nil { return err } actions, err := pluginManager.GetRestoreItemActionsV2() if err != nil { return errors.Wrap(err, "error getting restore item actions") } actionsResolver := framework.NewRestoreItemActionResolverV2(actions) backupFile, err := downloadToTempFile(restore.Spec.BackupName, backupStore, restoreLog) if err != nil { return errors.Wrap(err, "error downloading backup") } defer closeAndRemoveFile(backupFile, r.logger) listOpts := &client.ListOptions{ LabelSelector: labels.Set(map[string]string{ api.BackupNameLabel: label.GetValidName(restore.Spec.BackupName), }).AsSelector(), } podVolumeBackupList := &api.PodVolumeBackupList{} err = r.kbClient.List(context.TODO(), podVolumeBackupList, listOpts) if err != nil { restoreLog.Errorf("Fail to list PodVolumeBackup :%s", err.Error()) return errors.WithStack(err) } volumeSnapshots, err := backupStore.GetBackupVolumeSnapshots(restore.Spec.BackupName) if err != nil { return errors.Wrap(err, "error fetching volume snapshots metadata") } csiVolumeSnapshots, err := backupStore.GetCSIVolumeSnapshots(restore.Spec.BackupName) if err != nil { return errors.Wrap(err, "fail to fetch CSI VolumeSnapshots metadata") } backupVolumeInfoMap := make(map[string]volume.BackupVolumeInfo) volumeInfos, err := backupStore.GetBackupVolumeInfos(restore.Spec.BackupName) if err != nil { restoreLog.WithError(err).Errorf("fail to get VolumeInfos metadata file for backup %s", restore.Spec.BackupName) return errors.WithStack(err) } else { for _, volumeInfo := range volumeInfos { backupVolumeInfoMap[volumeInfo.PVName] = *volumeInfo } } restoreLog.Info("starting restore") var podVolumeBackups []*api.PodVolumeBackup for i := range podVolumeBackupList.Items { podVolumeBackups = append(podVolumeBackups, &podVolumeBackupList.Items[i]) } restoreReq := &pkgrestore.Request{ Log: restoreLog, Restore: restore, Backup: info.backup, PodVolumeBackups: podVolumeBackups, VolumeSnapshots: volumeSnapshots, BackupReader: backupFile, ResourceModifiers: resourceModifiers, DisableInformerCache: r.disableInformerCache, CSIVolumeSnapshots: csiVolumeSnapshots, BackupVolumeInfoMap: backupVolumeInfoMap, RestoreVolumeInfoTracker: volume.NewRestoreVolInfoTracker(restore, restoreLog, r.globalCrClient), ResourceDeletionStatusTracker: kubeutil.NewResourceDeletionStatusTracker(), } restoreWarnings, restoreErrors := r.restorer.RestoreWithResolvers(restoreReq, actionsResolver, pluginManager) // Iterate over restore item operations and update progress. // Any errors on operations at this point should be added to restore errors. // If any operations are still not complete, then restore will not be set to // Completed yet. inProgressOperations, _, opsCompleted, opsFailed, errs := getRestoreItemOperationProgress(restoreReq.Restore, pluginManager, *restoreReq.GetItemOperationsList()) if len(errs) > 0 { for _, err := range errs { restoreErrors.Velero = append(restoreErrors.Velero, fmt.Sprintf("error from restore item operation: %v", err)) } } restore.Status.RestoreItemOperationsAttempted = len(*restoreReq.GetItemOperationsList()) restore.Status.RestoreItemOperationsCompleted = opsCompleted restore.Status.RestoreItemOperationsFailed = opsFailed // log errors and warnings to the restore log for _, msg := range restoreErrors.Velero { restoreLog.Errorf("Velero restore error: %v", msg) } for _, msg := range restoreErrors.Cluster { restoreLog.Errorf("Cluster resource restore error: %v", msg) } for ns, errs := range restoreErrors.Namespaces { for _, msg := range errs { restoreLog.Errorf("Namespace %v, resource restore error: %v", ns, msg) } } for _, msg := range restoreWarnings.Velero { restoreLog.Warnf("Velero restore warning: %v", msg) } for _, msg := range restoreWarnings.Cluster { restoreLog.Warnf("Cluster resource restore warning: %v", msg) } for ns, errs := range restoreWarnings.Namespaces { for _, msg := range errs { restoreLog.Warnf("Namespace %v, resource restore warning: %v", ns, msg) } } restoreLog.Info("restore completed") restoreLog.DoneForPersist(r.logger) // re-instantiate the backup store because credentials could have changed since the original // instantiation, if this was a long-running restore backupStore, err = r.backupStoreGetter.Get(info.location, pluginManager, r.logger) if err != nil { return errors.Wrap(err, "error setting up backup store to persist log and results files") } if logReader, err := restoreLog.GetPersistFile(); err != nil { restoreErrors.Velero = append(restoreErrors.Velero, fmt.Sprintf("error getting restore log reader: %v", err)) } else { if err := backupStore.PutRestoreLog(restore.Spec.BackupName, restore.Name, logReader); err != nil { restoreErrors.Velero = append(restoreErrors.Velero, fmt.Sprintf("error uploading log file to backup storage: %v", err)) } } // At this point, no further logs should be written to restoreLog since it's been uploaded // to object storage. restore.Status.Warnings = len(restoreWarnings.Velero) + len(restoreWarnings.Cluster) for _, w := range restoreWarnings.Namespaces { restore.Status.Warnings += len(w) } restore.Status.Errors = len(restoreErrors.Velero) + len(restoreErrors.Cluster) for _, e := range restoreErrors.Namespaces { restore.Status.Errors += len(e) } m := map[string]results.Result{ "warnings": restoreWarnings, "errors": restoreErrors, } if err := putResults(restore, m, backupStore); err != nil { r.logger.WithError(err).Error("Error uploading restore results to backup storage") } if err := putRestoredResourceList(restore, restoreReq.RestoredResourceList(), backupStore); err != nil { r.logger.WithError(err).Error("Error uploading restored resource list to backup storage") } if err := putOperationsForRestore(restore, *restoreReq.GetItemOperationsList(), backupStore); err != nil { r.logger.WithError(err).Error("Error uploading restore item action operation resource list to backup storage") } restoreReq.RestoreVolumeInfoTracker.Populate(context.TODO(), restoreReq.RestoredResourceList()) if err := putRestoreVolumeInfoList(restore, restoreReq.RestoreVolumeInfoTracker.Result(), backupStore); err != nil { r.logger.WithError(err).Error("Error uploading restored volume info to backup storage") } if restore.Status.Errors > 0 { if inProgressOperations { r.logger.Debug("Restore WaitingForPluginOperationsPartiallyFailed") restore.Status.Phase = api.RestorePhaseWaitingForPluginOperationsPartiallyFailed } else { r.logger.Debug("Restore FinalizingPartiallyFailed") restore.Status.Phase = api.RestorePhaseFinalizingPartiallyFailed } } else { if inProgressOperations { r.logger.Debug("Restore WaitingForPluginOperations") restore.Status.Phase = api.RestorePhaseWaitingForPluginOperations } else { r.logger.Debug("Restore Finalizing") restore.Status.Phase = api.RestorePhaseFinalizing } } return nil } // updateTotalRestoreMetric update the velero_restore_total metric every minute. func (r *restoreReconciler) updateTotalRestoreMetric() { go func() { // Wait for 5 seconds to let controller-runtime to setup k8s clients. time.Sleep(5 * time.Second) wait.Until( func() { // recompute restore_total metric restoreList := &api.RestoreList{} err := r.kbClient.List(context.Background(), restoreList, &client.ListOptions{}) if err != nil { r.logger.Error(err, "Error computing restore_total metric") } else { r.metrics.SetRestoreTotal(int64(len(restoreList.Items))) } }, 1*time.Minute, r.ctx.Done(), ) }() } // deleteExternalResources deletes all the external resources related to the restore func (r *restoreReconciler) deleteExternalResources(restore *api.Restore) error { r.logger.Infof("Finalizer is deleting external resources, backup: %s", restore.Spec.BackupName) if restore.Spec.BackupName == "" { return nil } backupInfo, err := r.fetchBackupInfo(restore.Spec.BackupName) if err != nil { if apierrors.IsNotFound(err) { r.logger.Errorf("got not found error: %v, skip deleting the restore files in object storage", err) return nil } return errors.Wrap(err, fmt.Sprintf("can't get backup info, backup: %s", restore.Spec.BackupName)) } if !veleroutil.BSLIsAvailable(*backupInfo.location) { return fmt.Errorf("bsl %s is unavailable, cannot get the backup info", backupInfo.location.Name) } // delete restore files in object storage pluginManager := r.newPluginManager(r.logger) defer pluginManager.CleanupClients() backupStore, err := r.backupStoreGetter.Get(backupInfo.location, pluginManager, r.logger) if err != nil { return errors.Wrap(err, fmt.Sprintf("can't get backupStore, backup: %s", restore.Spec.BackupName)) } if err = backupStore.DeleteRestore(restore.Name); err != nil { return errors.Wrap(err, fmt.Sprintf("can't delete restore files in object storage, backup: %s", restore.Spec.BackupName)) } return nil } func putResults(restore *api.Restore, results map[string]results.Result, backupStore persistence.BackupStore) error { buf := new(bytes.Buffer) gzw := gzip.NewWriter(buf) defer gzw.Close() if err := json.NewEncoder(gzw).Encode(results); err != nil { return errors.Wrap(err, "error encoding restore results to JSON") } if err := gzw.Close(); err != nil { return errors.Wrap(err, "error closing gzip writer") } if err := backupStore.PutRestoreResults(restore.Spec.BackupName, restore.Name, buf); err != nil { return err } return nil } func putRestoredResourceList(restore *api.Restore, list map[string][]string, backupStore persistence.BackupStore) error { buf := new(bytes.Buffer) gzw := gzip.NewWriter(buf) defer gzw.Close() if err := json.NewEncoder(gzw).Encode(list); err != nil { return errors.Wrap(err, "error encoding restored resource list to JSON") } if err := gzw.Close(); err != nil { return errors.Wrap(err, "error closing gzip writer") } if err := backupStore.PutRestoredResourceList(restore.Name, buf); err != nil { return err } return nil } func putOperationsForRestore(restore *api.Restore, operations []*itemoperation.RestoreOperation, backupStore persistence.BackupStore) error { buf := new(bytes.Buffer) gzw := gzip.NewWriter(buf) defer gzw.Close() if err := json.NewEncoder(gzw).Encode(operations); err != nil { return errors.Wrap(err, "error encoding restore item operations list to JSON") } if err := gzw.Close(); err != nil { return errors.Wrap(err, "error closing gzip writer") } if err := backupStore.PutRestoreItemOperations(restore.Name, buf); err != nil { return err } return nil } func putRestoreVolumeInfoList(restore *api.Restore, volInfoList []*volume.RestoreVolumeInfo, store persistence.BackupStore) error { buf := new(bytes.Buffer) gzw := gzip.NewWriter(buf) defer gzw.Close() if err := json.NewEncoder(gzw).Encode(volInfoList); err != nil { return errors.Wrap(err, "error encoding restore volume info list to JSON") } if err := gzw.Close(); err != nil { return errors.Wrap(err, "error closing gzip writer") } return store.PutRestoreVolumeInfo(restore.Name, buf) } func downloadToTempFile(backupName string, backupStore persistence.BackupStore, logger logrus.FieldLogger) (*os.File, error) { readCloser, err := backupStore.GetBackupContents(backupName) if err != nil { return nil, err } defer readCloser.Close() file, err := os.CreateTemp("", backupName) if err != nil { return nil, errors.Wrap(err, "error creating Backup temp file") } n, err := io.Copy(file, readCloser) if err != nil { // Temporary file has been created if we go here. And some problems occurs such as network interruption and // so on. So we close and remove temporary file first to prevent residual file. closeAndRemoveFile(file, logger) return nil, errors.Wrap(err, "error copying Backup to temp file") } log := logger.WithField("backup", backupName) log.WithFields(logrus.Fields{ "fileName": file.Name(), "bytes": n, }).Debug("Copied Backup to file") if _, err := file.Seek(0, 0); err != nil { closeAndRemoveFile(file, logger) return nil, errors.Wrap(err, "error resetting Backup file offset") } return file, nil } ================================================ FILE: pkg/controller/restore_controller_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package controller import ( "bytes" "io" "testing" "time" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" clocktesting "k8s.io/utils/clock/testing" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/internal/resourcemodifiers" "github.com/vmware-tanzu/velero/internal/volume" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/metrics" persistencemocks "github.com/vmware-tanzu/velero/pkg/persistence/mocks" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" "github.com/vmware-tanzu/velero/pkg/plugin/framework" pluginmocks "github.com/vmware-tanzu/velero/pkg/plugin/mocks" riav2 "github.com/vmware-tanzu/velero/pkg/plugin/velero/restoreitemaction/v2" pkgrestore "github.com/vmware-tanzu/velero/pkg/restore" velerotest "github.com/vmware-tanzu/velero/pkg/test" "github.com/vmware-tanzu/velero/pkg/util/logging" "github.com/vmware-tanzu/velero/pkg/util/results" ) func TestFetchBackupInfo(t *testing.T) { tests := []struct { name string backupName string informerLocations []*velerov1api.BackupStorageLocation informerBackups []*velerov1api.Backup backupStoreBackup *velerov1api.Backup backupStoreError error expectedRes *velerov1api.Backup expectedErr bool }{ { name: "lister has backup", backupName: "backup-1", informerLocations: []*velerov1api.BackupStorageLocation{builder.ForBackupStorageLocation("velero", "default").Provider("myCloud").Bucket("bucket").Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result()}, informerBackups: []*velerov1api.Backup{defaultBackup().StorageLocation("default").Result()}, expectedRes: defaultBackup().StorageLocation("default").Result(), }, { name: "lister does not have a backup, but backupSvc does", backupName: "backup-1", backupStoreBackup: defaultBackup().StorageLocation("default").Result(), informerLocations: []*velerov1api.BackupStorageLocation{builder.ForBackupStorageLocation("velero", "default").Provider("myCloud").Bucket("bucket").Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result()}, informerBackups: []*velerov1api.Backup{defaultBackup().StorageLocation("default").Result()}, expectedRes: defaultBackup().StorageLocation("default").Result(), }, { name: "no backup", backupName: "backup-1", backupStoreError: errors.New("no backup here"), expectedErr: true, }, } formatFlag := logging.FormatText for _, test := range tests { t.Run(test.name, func(t *testing.T) { var ( fakeClient = velerotest.NewFakeControllerRuntimeClient(t) fakeGlobalClient = velerotest.NewFakeControllerRuntimeClient(t) restorer = &fakeRestorer{kbClient: fakeClient} logger = velerotest.NewLogger() pluginManager = &pluginmocks.Manager{} backupStore = &persistencemocks.BackupStore{} ) defer restorer.AssertExpectations(t) defer backupStore.AssertExpectations(t) r := NewRestoreReconciler( t.Context(), velerov1api.DefaultNamespace, restorer, fakeClient, logger, logrus.InfoLevel, func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager }, NewFakeSingleObjectBackupStoreGetter(backupStore), metrics.NewServerMetrics(), formatFlag, 60*time.Minute, false, fakeGlobalClient, 10*time.Minute, ) if test.backupStoreError == nil { for _, itm := range test.informerLocations { require.NoError(t, r.kbClient.Create(t.Context(), itm)) } for _, itm := range test.informerBackups { require.NoError(t, r.kbClient.Create(t.Context(), itm)) } } if test.backupStoreBackup != nil && test.backupStoreError != nil { panic("developer error - only one of backupStoreBackup, backupStoreError can be non-nil") } if test.backupStoreError != nil { // TODO why do I need .Maybe() here? backupStore.On("GetBackupMetadata", test.backupName).Return(nil, test.backupStoreError).Maybe() } if test.backupStoreBackup != nil { // TODO why do I need .Maybe() here? backupStore.On("GetBackupMetadata", test.backupName).Return(test.backupStoreBackup, nil).Maybe() } info, err := r.fetchBackupInfo(test.backupName) require.Equal(t, test.expectedErr, err != nil) if test.expectedRes != nil { assert.Equal(t, test.expectedRes.Spec, info.backup.Spec) } }) } } func TestProcessQueueItemSkips(t *testing.T) { tests := []struct { name string namespace string restoreName string restore *velerov1api.Restore expectError bool }{ { name: "missing restore returns nil", namespace: "foo", restoreName: "bar", expectError: false, }, } formatFlag := logging.FormatText for _, test := range tests { t.Run(test.name, func(t *testing.T) { var ( fakeClient = velerotest.NewFakeControllerRuntimeClient(t) fakeGlobalClient = velerotest.NewFakeControllerRuntimeClient(t) restorer = &fakeRestorer{kbClient: fakeClient} logger = velerotest.NewLogger() ) if test.restore != nil { require.NoError(t, fakeClient.Create(t.Context(), test.restore)) } r := NewRestoreReconciler( t.Context(), velerov1api.DefaultNamespace, restorer, fakeClient, logger, logrus.InfoLevel, nil, nil, // backupStoreGetter metrics.NewServerMetrics(), formatFlag, 60*time.Minute, false, fakeGlobalClient, 10*time.Minute, ) _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: types.NamespacedName{ Namespace: test.namespace, Name: test.restoreName, }}) assert.Equal(t, test.expectError, err != nil) }) } } func TestRestoreReconcile(t *testing.T) { defaultStorageLocation := builder.ForBackupStorageLocation("velero", "default").Provider("myCloud").Bucket("bucket").Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result() now, err := time.Parse(time.RFC1123Z, time.RFC1123Z) require.NoError(t, err) now = now.Local() timestamp := metav1.NewTime(now) assert.NotNil(t, timestamp) tests := []struct { name string restoreKey string location *velerov1api.BackupStorageLocation restore *velerov1api.Restore backup *velerov1api.Backup restorerError error expectedErr bool expectedPhase string expectedStartTime *metav1.Time expectedCompletedTime *metav1.Time expectedValidationErrors []string expectedRestoreErrors int expectedRestorerCall *velerov1api.Restore backupStoreGetBackupMetadataErr error backupStoreGetBackupContentsErr error putRestoreLogErr error expectedFinalPhase string addValidFinalizer bool emptyVolumeInfo bool }{ { name: "restore with both namespace in both includedNamespaces and excludedNamespaces fails validation", location: defaultStorageLocation, restore: NewRestore("foo", "bar", "backup-1", "another-1", "*", velerov1api.RestorePhaseNew).ExcludedNamespaces("another-1").Result(), backup: defaultBackup().StorageLocation("default").Result(), expectedErr: false, expectedPhase: string(velerov1api.RestorePhaseFailedValidation), expectedValidationErrors: []string{"Invalid included/excluded namespace lists: excludes list cannot contain an item in the includes list: another-1"}, }, { name: "restore with resource in both includedResources and excludedResources fails validation", location: defaultStorageLocation, restore: NewRestore("foo", "bar", "backup-1", "*", "a-resource", velerov1api.RestorePhaseNew).ExcludedResources("a-resource").Result(), backup: defaultBackup().StorageLocation("default").Result(), expectedErr: false, expectedPhase: string(velerov1api.RestorePhaseFailedValidation), expectedValidationErrors: []string{"Invalid included/excluded resource lists: excludes list cannot contain an item in the includes list: a-resource"}, }, { name: "new restore with empty backup and schedule names fails validation", restore: NewRestore("foo", "bar", "", "ns-1", "", velerov1api.RestorePhaseNew).Result(), expectedErr: false, expectedPhase: string(velerov1api.RestorePhaseFailedValidation), expectedValidationErrors: []string{"Either a backup or schedule must be specified as a source for the restore, but not both"}, }, { name: "new restore with backup and schedule names provided fails validation", restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseNew).Schedule("sched-1").Result(), expectedErr: false, expectedPhase: string(velerov1api.RestorePhaseFailedValidation), expectedValidationErrors: []string{"Either a backup or schedule must be specified as a source for the restore, but not both"}, }, { name: "new restore with labelSelector as well as orLabelSelector fails validation", location: defaultStorageLocation, restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseNew).LabelSelector(&metav1.LabelSelector{MatchLabels: map[string]string{"a": "b"}}).OrLabelSelector([]*metav1.LabelSelector{{MatchLabels: map[string]string{"a1": "b1"}}, {MatchLabels: map[string]string{"a2": "b2"}}, {MatchLabels: map[string]string{"a3": "b3"}}, {MatchLabels: map[string]string{"a4": "b4"}}}).Result(), backup: defaultBackup().StorageLocation("default").Result(), expectedErr: false, expectedValidationErrors: []string{"encountered labelSelector as well as orLabelSelectors in restore spec, only one can be specified"}, expectedPhase: string(velerov1api.RestorePhaseFailedValidation), }, { name: "valid restore with schedule name gets executed", location: defaultStorageLocation, restore: NewRestore("foo", "bar", "", "ns-1", "", velerov1api.RestorePhaseNew).Schedule("sched-1").Result(), backup: defaultBackup().StorageLocation("default").ObjectMeta(builder.WithLabels(velerov1api.ScheduleNameLabel, "sched-1")).Phase(velerov1api.BackupPhaseCompleted).Result(), expectedErr: false, expectedPhase: string(velerov1api.RestorePhaseInProgress), expectedStartTime: ×tamp, expectedCompletedTime: ×tamp, expectedRestorerCall: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseInProgress).Schedule("sched-1").Result(), }, { name: "restore with non-existent backup name fails", restore: NewRestore("foo", "bar", "backup-1", "ns-1", "*", velerov1api.RestorePhaseNew).Result(), expectedErr: false, expectedPhase: string(velerov1api.RestorePhaseFailedValidation), expectedValidationErrors: []string{"Error retrieving backup: backup.velero.io \"backup-1\" not found"}, backupStoreGetBackupMetadataErr: errors.New("no backup here"), }, { name: "restorer throwing an error causes the restore to fail", location: defaultStorageLocation, restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseNew).Result(), backup: defaultBackup().StorageLocation("default").Result(), restorerError: errors.New("blarg"), expectedErr: false, expectedPhase: string(velerov1api.RestorePhaseInProgress), expectedFinalPhase: string(velerov1api.RestorePhasePartiallyFailed), expectedStartTime: ×tamp, expectedCompletedTime: ×tamp, expectedRestoreErrors: 1, expectedRestorerCall: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseInProgress).Result(), }, { name: "valid restore with none existingresourcepolicy gets executed", location: defaultStorageLocation, restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseNew).ExistingResourcePolicy("none").Result(), backup: defaultBackup().StorageLocation("default").Result(), expectedErr: false, expectedPhase: string(velerov1api.RestorePhaseInProgress), expectedStartTime: ×tamp, expectedCompletedTime: ×tamp, expectedRestorerCall: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseInProgress).ExistingResourcePolicy("none").Result(), }, { name: "valid restore with update existingresourcepolicy gets executed", location: defaultStorageLocation, restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseNew).ExistingResourcePolicy("update").Result(), backup: defaultBackup().StorageLocation("default").Result(), expectedErr: false, expectedPhase: string(velerov1api.RestorePhaseInProgress), expectedStartTime: ×tamp, expectedCompletedTime: ×tamp, expectedRestorerCall: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseInProgress).ExistingResourcePolicy("update").Result(), }, { name: "invalid restore with invalid existingresourcepolicy errors", location: defaultStorageLocation, restore: NewRestore("foo", "invalidexistingresourcepolicy", "backup-1", "ns-1", "", velerov1api.RestorePhaseNew).ExistingResourcePolicy("invalid").Result(), backup: defaultBackup().StorageLocation("default").Result(), expectedErr: false, expectedPhase: string(velerov1api.RestorePhaseFailedValidation), expectedStartTime: ×tamp, expectedCompletedTime: ×tamp, expectedRestorerCall: nil, // this restore should fail validation and not be passed to the restorer }, { name: "valid restore gets executed", location: defaultStorageLocation, restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseNew).Result(), backup: defaultBackup().StorageLocation("default").Result(), expectedErr: false, expectedPhase: string(velerov1api.RestorePhaseInProgress), expectedStartTime: ×tamp, expectedCompletedTime: ×tamp, expectedRestorerCall: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseInProgress).Result(), }, { name: "restoration of nodes is not supported", location: defaultStorageLocation, restore: NewRestore("foo", "bar", "backup-1", "ns-1", "nodes", velerov1api.RestorePhaseNew).Result(), backup: defaultBackup().StorageLocation("default").Result(), expectedErr: false, expectedPhase: string(velerov1api.RestorePhaseFailedValidation), expectedValidationErrors: []string{ "nodes are non-restorable resources", "Invalid included/excluded resource lists: excludes list cannot contain an item in the includes list: nodes", }, }, { name: "restoration of events is not supported", location: defaultStorageLocation, restore: NewRestore("foo", "bar", "backup-1", "ns-1", "events", velerov1api.RestorePhaseNew).Result(), backup: defaultBackup().StorageLocation("default").Result(), expectedErr: false, expectedPhase: string(velerov1api.RestorePhaseFailedValidation), expectedValidationErrors: []string{ "events are non-restorable resources", "Invalid included/excluded resource lists: excludes list cannot contain an item in the includes list: events", }, }, { name: "restoration of events.events.k8s.io is not supported", location: defaultStorageLocation, restore: NewRestore("foo", "bar", "backup-1", "ns-1", "events.events.k8s.io", velerov1api.RestorePhaseNew).Result(), backup: defaultBackup().StorageLocation("default").Result(), expectedErr: false, expectedPhase: string(velerov1api.RestorePhaseFailedValidation), expectedValidationErrors: []string{ "events.events.k8s.io are non-restorable resources", "Invalid included/excluded resource lists: excludes list cannot contain an item in the includes list: events.events.k8s.io", }, }, { name: "restoration of backups.velero.io is not supported", location: defaultStorageLocation, restore: NewRestore("foo", "bar", "backup-1", "ns-1", "backups.velero.io", velerov1api.RestorePhaseNew).Result(), backup: defaultBackup().StorageLocation("default").Result(), expectedErr: false, expectedPhase: string(velerov1api.RestorePhaseFailedValidation), expectedValidationErrors: []string{ "backups.velero.io are non-restorable resources", "Invalid included/excluded resource lists: excludes list cannot contain an item in the includes list: backups.velero.io", }, }, { name: "restoration of restores.velero.io is not supported", location: defaultStorageLocation, restore: NewRestore("foo", "bar", "backup-1", "ns-1", "restores.velero.io", velerov1api.RestorePhaseNew).Result(), backup: defaultBackup().StorageLocation("default").Result(), expectedErr: false, expectedPhase: string(velerov1api.RestorePhaseFailedValidation), expectedValidationErrors: []string{ "restores.velero.io are non-restorable resources", "Invalid included/excluded resource lists: excludes list cannot contain an item in the includes list: restores.velero.io", }, }, { name: "backup download error results in failed restore", location: defaultStorageLocation, restore: NewRestore(velerov1api.DefaultNamespace, "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseNew).Result(), expectedPhase: string(velerov1api.RestorePhaseInProgress), expectedFinalPhase: string(velerov1api.RestorePhaseFailed), expectedStartTime: ×tamp, expectedCompletedTime: ×tamp, backupStoreGetBackupContentsErr: errors.New("Couldn't download backup"), backup: defaultBackup().StorageLocation("default").Result(), }, { name: "restore attached with an expected finalizer gets cleaned up successfully", location: defaultStorageLocation, restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseCompleted).ObjectMeta(builder.WithFinalizers(ExternalResourcesFinalizer), builder.WithDeletionTimestamp(timestamp.Time)).Result(), backup: defaultBackup().StorageLocation("default").Result(), expectedErr: false, addValidFinalizer: true, }, { name: "restore attached with an unknown finalizer will be skipped", location: defaultStorageLocation, restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseCompleted).ObjectMeta(builder.WithFinalizers("restores.velero.io/unknown-finalizer"), builder.WithDeletionTimestamp(timestamp.Time)).Result(), backup: defaultBackup().StorageLocation("default").Result(), expectedErr: false, addValidFinalizer: false, }, { name: "completed restore will be skipped", location: defaultStorageLocation, restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseCompleted).Result(), backup: defaultBackup().StorageLocation("default").Result(), expectedErr: false, }, { name: "valid restore with empty VolumeInfos", location: defaultStorageLocation, restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseNew).Result(), backup: defaultBackup().StorageLocation("default").Result(), emptyVolumeInfo: true, expectedErr: false, expectedPhase: string(velerov1api.RestorePhaseInProgress), expectedStartTime: ×tamp, expectedCompletedTime: ×tamp, expectedRestorerCall: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseInProgress).Result(), }, { name: "Restore creation is rejected when BSL is unavailable", location: builder.ForBackupStorageLocation("velero", "default").Provider("myCloud").Bucket("bucket").Phase(velerov1api.BackupStorageLocationPhaseUnavailable).Result(), restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseNew).Result(), backup: defaultBackup().StorageLocation("default").Result(), expectedErr: false, expectedPhase: string(velerov1api.RestorePhaseNew), expectedValidationErrors: []string{"The BSL default is unavailable, cannot retrieve the backup"}, }, { name: "Restore deletion is rejected when BSL is unavailable.", location: builder.ForBackupStorageLocation("velero", "default").Provider("myCloud").Bucket("bucket").Phase(velerov1api.BackupStorageLocationPhaseUnavailable).Result(), restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseCompleted).ObjectMeta(builder.WithFinalizers(ExternalResourcesFinalizer), builder.WithDeletionTimestamp(timestamp.Time)).Result(), backup: defaultBackup().StorageLocation("default").Result(), expectedErr: true, }, } formatFlag := logging.FormatText for _, test := range tests { t.Run(test.name, func(t *testing.T) { var ( fakeClient = velerotest.NewFakeControllerRuntimeClientBuilder(t).Build() fakeGlobalClient = velerotest.NewFakeControllerRuntimeClient(t) restorer = &fakeRestorer{kbClient: fakeClient} logger = velerotest.NewLogger() pluginManager = &pluginmocks.Manager{} backupStore = &persistencemocks.BackupStore{} ) defer restorer.AssertExpectations(t) defer backupStore.AssertExpectations(t) defer func() { // reset defaultStorageLocation resourceVersion defaultStorageLocation.ObjectMeta.ResourceVersion = "" }() r := NewRestoreReconciler( t.Context(), velerov1api.DefaultNamespace, restorer, fakeClient, logger, logrus.InfoLevel, func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager }, NewFakeSingleObjectBackupStoreGetter(backupStore), metrics.NewServerMetrics(), formatFlag, 60*time.Minute, false, fakeGlobalClient, 10*time.Minute, ) r.clock = clocktesting.NewFakeClock(now) if test.location != nil { require.NoError(t, r.kbClient.Create(t.Context(), test.location)) } if test.backup != nil { require.NoError(t, r.kbClient.Create(t.Context(), test.backup)) } if test.restore != nil { isDeletionTimestampSet := test.restore.DeletionTimestamp != nil require.NoError(t, r.kbClient.Create(t.Context(), test.restore)) // because of the changes introduced by https://github.com/kubernetes-sigs/controller-runtime/commit/7a66d580c0c53504f5b509b45e9300cc18a1cc30 // the fake client ignores the DeletionTimestamp when calling the Create(), // so call Delete() here if isDeletionTimestampSet { err = r.kbClient.Delete(ctx, test.restore) require.NoError(t, err) } } var warnings, errors results.Result if test.restorerError != nil { errors.Namespaces = map[string][]string{"ns-1": {test.restorerError.Error()}} } if test.putRestoreLogErr != nil { errors.Velero = append(errors.Velero, "error uploading log file to object storage: "+test.putRestoreLogErr.Error()) } if test.expectedRestorerCall != nil { backupStore.On("GetBackupContents", test.backup.Name).Return(io.NopCloser(bytes.NewReader([]byte("hello world"))), nil) backupStore.On("GetCSIVolumeSnapshots", test.backup.Name).Return([]*snapshotv1api.VolumeSnapshot{}, nil) restorer.On("RestoreWithResolvers", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(warnings, errors) backupStore.On("PutRestoreLog", test.backup.Name, test.restore.Name, mock.Anything).Return(test.putRestoreLogErr) backupStore.On("PutRestoreResults", test.backup.Name, test.restore.Name, mock.Anything).Return(nil) backupStore.On("PutRestoredResourceList", test.restore.Name, mock.Anything).Return(nil) backupStore.On("PutRestoreItemOperations", mock.Anything, mock.Anything).Return(nil) backupStore.On("PutRestoreVolumeInfo", test.restore.Name, mock.Anything).Return(nil) if test.emptyVolumeInfo == true { backupStore.On("GetBackupVolumeInfos", test.backup.Name).Return(nil, nil) } else { backupStore.On("GetBackupVolumeInfos", test.backup.Name).Return([]*volume.BackupVolumeInfo{}, nil) } volumeSnapshots := []*volume.Snapshot{ { Spec: volume.SnapshotSpec{ PersistentVolumeName: "test-pv", BackupName: test.backup.Name, }, }, } backupStore.On("GetBackupVolumeSnapshots", test.backup.Name).Return(volumeSnapshots, nil) } if test.backupStoreGetBackupMetadataErr != nil { // TODO why do I need .Maybe() here? backupStore.On("GetBackupMetadata", test.restore.Spec.BackupName).Return(nil, test.backupStoreGetBackupMetadataErr).Maybe() } if test.backupStoreGetBackupContentsErr != nil { // TODO why do I need .Maybe() here? backupStore.On("GetBackupContents", test.restore.Spec.BackupName).Return(nil, test.backupStoreGetBackupContentsErr).Maybe() } if test.restore != nil { pluginManager.On("GetRestoreItemActionsV2").Return(nil, nil) pluginManager.On("CleanupClients") } if test.addValidFinalizer { backupStore.On("DeleteRestore", test.restore.Name).Return(nil) } //err = r.processQueueItem(key) _, err = r.Reconcile(t.Context(), ctrl.Request{NamespacedName: types.NamespacedName{ Namespace: test.restore.Namespace, Name: test.restore.Name, }}) assert.Equal(t, test.expectedErr, err != nil, "got error %v", err) if test.expectedPhase == "" { return } // struct and func for decoding patch content type SpecPatch struct { BackupName string `json:"backupName"` } type StatusPatch struct { Phase velerov1api.RestorePhase `json:"phase"` ValidationErrors []string `json:"validationErrors"` Errors int `json:"errors"` StartTimestamp *metav1.Time `json:"startTimestamp"` CompletionTimestamp *metav1.Time `json:"completionTimestamp"` } type Patch struct { Spec SpecPatch `json:"spec,omitempty"` Status StatusPatch `json:"status"` } expected := Patch{ Status: StatusPatch{ Phase: velerov1api.RestorePhase(test.expectedPhase), ValidationErrors: test.expectedValidationErrors, }, } if test.expectedStartTime != nil { expected.Status.StartTimestamp = test.expectedStartTime } // if we don't expect a restore, validate it wasn't called and exit the test if test.expectedRestorerCall == nil { assert.Empty(t, restorer.Calls) assert.Zero(t, restorer.calledWithArg) return } if !test.addValidFinalizer { assert.Len(t, restorer.Calls, 1) } // validate Patch call 2 (setting phase) expected = Patch{ Status: StatusPatch{ Phase: velerov1api.RestorePhaseCompleted, Errors: test.expectedRestoreErrors, CompletionTimestamp: test.expectedCompletedTime, }, } // Override our default expectations if the case requires it if test.expectedFinalPhase != "" { expected = Patch{ Status: StatusPatch{ Phase: velerov1api.RestorePhase(test.expectedFinalPhase), Errors: test.expectedRestoreErrors, CompletionTimestamp: test.expectedCompletedTime, }, } } // explicitly capturing the argument passed to Restore myself because // I want to validate the called arg as of the time of calling, but // the mock stores the pointer, which gets modified after assert.Equal(t, test.expectedRestorerCall.Spec, restorer.calledWithArg.Spec) assert.Equal(t, test.expectedRestorerCall.Status.Phase, restorer.calledWithArg.Status.Phase) }) } } func TestValidateAndCompleteWhenScheduleNameSpecified(t *testing.T) { formatFlag := logging.FormatText var ( logger = velerotest.NewLogger() pluginManager = &pluginmocks.Manager{} fakeClient = velerotest.NewFakeControllerRuntimeClient(t) fakeGlobalClient = velerotest.NewFakeControllerRuntimeClient(t) backupStore = &persistencemocks.BackupStore{} ) r := NewRestoreReconciler( t.Context(), velerov1api.DefaultNamespace, nil, fakeClient, logger, logrus.DebugLevel, func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager }, NewFakeSingleObjectBackupStoreGetter(backupStore), metrics.NewServerMetrics(), formatFlag, 60*time.Minute, false, fakeGlobalClient, 10*time.Minute, ) restore := &velerov1api.Restore{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "restore-1", }, Spec: velerov1api.RestoreSpec{ ScheduleName: "schedule-1", }, } // no backups created from the schedule: fail validation require.NoError(t, r.kbClient.Create(t.Context(), defaultBackup(). ObjectMeta(builder.WithLabels(velerov1api.ScheduleNameLabel, "non-matching-schedule")). Phase(velerov1api.BackupPhaseCompleted). Result())) r.validateAndComplete(restore) assert.Contains(t, restore.Status.ValidationErrors, "No backups found for schedule") assert.Empty(t, restore.Spec.BackupName) // no completed backups created from the schedule: fail validation require.NoError(t, r.kbClient.Create( t.Context(), defaultBackup(). ObjectMeta( builder.WithName("backup-2"), builder.WithLabels(velerov1api.ScheduleNameLabel, "schedule-1"), ). Phase(velerov1api.BackupPhaseInProgress). Result(), )) r.validateAndComplete(restore) assert.Contains(t, restore.Status.ValidationErrors, "No completed backups found for schedule") assert.Empty(t, restore.Spec.BackupName) // multiple completed backups created from the schedule: use most recent now := time.Now() require.NoError(t, r.kbClient.Create(t.Context(), defaultBackup(). ObjectMeta( builder.WithName("foo"), builder.WithLabels(velerov1api.ScheduleNameLabel, "schedule-1"), ). StorageLocation("default"). Phase(velerov1api.BackupPhaseCompleted). StartTimestamp(now). Result(), )) location := builder.ForBackupStorageLocation("velero", "default").Provider("myCloud").Bucket("bucket").Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result() require.NoError(t, r.kbClient.Create(t.Context(), location)) restore = &velerov1api.Restore{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "restore-1", }, Spec: velerov1api.RestoreSpec{ ScheduleName: "schedule-1", }, } r.validateAndComplete(restore) assert.Nil(t, restore.Status.ValidationErrors) assert.Equal(t, "foo", restore.Spec.BackupName) } func TestValidateAndCompleteWithResourceModifierSpecified(t *testing.T) { formatFlag := logging.FormatText var ( logger = velerotest.NewLogger() pluginManager = &pluginmocks.Manager{} fakeClient = velerotest.NewFakeControllerRuntimeClient(t) fakeGlobalClient = velerotest.NewFakeControllerRuntimeClient(t) backupStore = &persistencemocks.BackupStore{} ) r := NewRestoreReconciler( t.Context(), velerov1api.DefaultNamespace, nil, fakeClient, logger, logrus.DebugLevel, func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager }, NewFakeSingleObjectBackupStoreGetter(backupStore), metrics.NewServerMetrics(), formatFlag, 60*time.Minute, false, fakeGlobalClient, 10*time.Minute, ) restore := &velerov1api.Restore{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "restore-1", }, Spec: velerov1api.RestoreSpec{ BackupName: "backup-1", ResourceModifier: &corev1api.TypedLocalObjectReference{ Kind: resourcemodifiers.ConfigmapRefType, Name: "test-configmap", }, }, } location := builder.ForBackupStorageLocation("velero", "default").Provider("myCloud").Bucket("bucket").Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result() require.NoError(t, r.kbClient.Create(t.Context(), location)) require.NoError(t, r.kbClient.Create( t.Context(), defaultBackup(). ObjectMeta( builder.WithName("backup-1"), ).StorageLocation("default"). Phase(velerov1api.BackupPhaseCompleted). Result(), )) r.validateAndComplete(restore) assert.Contains(t, restore.Status.ValidationErrors[0], "failed to get resource modifiers configmap") restore1 := &velerov1api.Restore{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "restore-1", }, Spec: velerov1api.RestoreSpec{ BackupName: "backup-1", ResourceModifier: &corev1api.TypedLocalObjectReference{ Kind: resourcemodifiers.ConfigmapRefType, Name: "test-configmap", }, }, } cm1 := &corev1api.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "test-configmap", Namespace: velerov1api.DefaultNamespace, }, Data: map[string]string{ "sub.yml": "version: v1\nresourceModifierRules:\n- conditions:\n groupResource: persistentvolumeclaims\n resourceNameRegex: \".*\"\n namespaces:\n - bar\n - foo\n patches:\n - operation: replace\n path: \"/spec/storageClassName\"\n value: \"premium\"\n - operation: remove\n path: \"/metadata/labels/test\"\n\n\n", }, } require.NoError(t, r.kbClient.Create(t.Context(), cm1)) r.validateAndComplete(restore1) assert.Nil(t, restore1.Status.ValidationErrors) restore2 := &velerov1api.Restore{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "restore-1", }, Spec: velerov1api.RestoreSpec{ BackupName: "backup-1", ResourceModifier: &corev1api.TypedLocalObjectReference{ // intentional to ensure case insensitivity works as expected Kind: "confIGMaP", Name: "test-configmap-invalid", }, }, } invalidVersionCm := &corev1api.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "test-configmap-invalid", Namespace: velerov1api.DefaultNamespace, }, Data: map[string]string{ "sub.yml": "version1: v1\nresourceModifierRules:\n- conditions:\n groupResource: persistentvolumeclaims\n resourceNameRegex: \".*\"\n namespaces:\n - bar\n - foo\n patches:\n - operation: replace\n path: \"/spec/storageClassName\"\n value: \"premium\"\n - operation: remove\n path: \"/metadata/labels/test\"\n\n\n", }, } require.NoError(t, r.kbClient.Create(t.Context(), invalidVersionCm)) r.validateAndComplete(restore2) assert.Contains(t, restore2.Status.ValidationErrors[0], "Error in parsing resource modifiers provided in configmap") restore3 := &velerov1api.Restore{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "restore-1", }, Spec: velerov1api.RestoreSpec{ BackupName: "backup-1", ResourceModifier: &corev1api.TypedLocalObjectReference{ Kind: resourcemodifiers.ConfigmapRefType, Name: "test-configmap-invalid-operator", }, }, } invalidOperatorCm := &corev1api.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "test-configmap-invalid-operator", Namespace: velerov1api.DefaultNamespace, }, Data: map[string]string{ "sub.yml": "version: v1\nresourceModifierRules:\n- conditions:\n groupResource: persistentvolumeclaims\n resourceNameRegex: \".*\"\n namespaces:\n - bar\n - foo\n patches:\n - operation: invalid\n path: \"/spec/storageClassName\"\n value: \"premium\"\n - operation: remove\n path: \"/metadata/labels/test\"\n\n\n", }, } require.NoError(t, r.kbClient.Create(t.Context(), invalidOperatorCm)) r.validateAndComplete(restore3) assert.Contains(t, restore3.Status.ValidationErrors[0], "Validation error in resource modifiers provided in configmap") } func TestBackupXorScheduleProvided(t *testing.T) { r := &velerov1api.Restore{} assert.False(t, backupXorScheduleProvided(r)) r.Spec.BackupName = "backup-1" r.Spec.ScheduleName = "schedule-1" assert.False(t, backupXorScheduleProvided(r)) r.Spec.BackupName = "backup-1" r.Spec.ScheduleName = "" assert.True(t, backupXorScheduleProvided(r)) r.Spec.BackupName = "" r.Spec.ScheduleName = "schedule-1" assert.True(t, backupXorScheduleProvided(r)) } func TestMostRecentCompletedBackup(t *testing.T) { backups := []velerov1api.Backup{ { ObjectMeta: metav1.ObjectMeta{ Name: "a", }, Status: velerov1api.BackupStatus{ Phase: "", }, }, { ObjectMeta: metav1.ObjectMeta{ Name: "b", }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseNew, }, }, { ObjectMeta: metav1.ObjectMeta{ Name: "c", }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseInProgress, }, }, { ObjectMeta: metav1.ObjectMeta{ Name: "d", }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFailedValidation, }, }, { ObjectMeta: metav1.ObjectMeta{ Name: "e", }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFailed, }, }, } assert.Empty(t, mostRecentCompletedBackup(backups).Name) now := time.Now() backups = append(backups, velerov1api.Backup{ ObjectMeta: metav1.ObjectMeta{ Name: "foo", }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseCompleted, StartTimestamp: &metav1.Time{Time: now}, }, }) expected := velerov1api.Backup{ ObjectMeta: metav1.ObjectMeta{ Name: "bar", }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseCompleted, StartTimestamp: &metav1.Time{Time: now.Add(time.Second)}, }, } backups = append(backups, expected) assert.Equal(t, expected, mostRecentCompletedBackup(backups)) } func NewRestore(ns, name, backup, includeNS, includeResource string, phase velerov1api.RestorePhase) *builder.RestoreBuilder { restore := builder.ForRestore(ns, name).Phase(phase).Backup(backup).ItemOperationTimeout(60 * time.Minute) if includeNS != "" { restore = restore.IncludedNamespaces(includeNS) } if includeResource != "" { restore = restore.IncludedResources(includeResource) } restore.ExcludedResources(nonRestorableResources...) return restore } type fakeRestorer struct { mock.Mock calledWithArg velerov1api.Restore kbClient client.Client } func (r *fakeRestorer) Restore( info *pkgrestore.Request, actions []riav2.RestoreItemAction, volumeSnapshotterGetter pkgrestore.VolumeSnapshotterGetter, ) (results.Result, results.Result) { res := r.Called(info.Log, info.Restore, info.Backup, info.BackupReader, actions) r.calledWithArg = *info.Restore return res.Get(0).(results.Result), res.Get(1).(results.Result) } func (r *fakeRestorer) RestoreWithResolvers(req *pkgrestore.Request, resolver framework.RestoreItemActionResolverV2, volumeSnapshotterGetter pkgrestore.VolumeSnapshotterGetter, ) (results.Result, results.Result) { res := r.Called(req.Log, req.Restore, req.Backup, req.BackupReader, resolver, r.kbClient, volumeSnapshotterGetter) r.calledWithArg = *req.Restore return res.Get(0).(results.Result), res.Get(1).(results.Result) } ================================================ FILE: pkg/controller/restore_finalizer_controller.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package controller import ( "context" "fmt" "sync" "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" storagev1api "k8s.io/api/storage/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/utils/clock" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/internal/hook" "github.com/vmware-tanzu/velero/internal/volume" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/constant" "github.com/vmware-tanzu/velero/pkg/itemoperation" "github.com/vmware-tanzu/velero/pkg/metrics" "github.com/vmware-tanzu/velero/pkg/persistence" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" "github.com/vmware-tanzu/velero/pkg/plugin/velero" kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube" "github.com/vmware-tanzu/velero/pkg/util/results" ) type restoreFinalizerReconciler struct { client.Client namespace string logger logrus.FieldLogger newPluginManager func(logger logrus.FieldLogger) clientmgmt.Manager backupStoreGetter persistence.ObjectBackupStoreGetter metrics *metrics.ServerMetrics clock clock.WithTickerAndDelayedExecution crClient client.Client multiHookTracker *hook.MultiHookTracker resourceTimeout time.Duration } func NewRestoreFinalizerReconciler( logger logrus.FieldLogger, namespace string, client client.Client, newPluginManager func(logrus.FieldLogger) clientmgmt.Manager, backupStoreGetter persistence.ObjectBackupStoreGetter, metrics *metrics.ServerMetrics, crClient client.Client, multiHookTracker *hook.MultiHookTracker, resourceTimeout time.Duration, ) *restoreFinalizerReconciler { return &restoreFinalizerReconciler{ Client: client, logger: logger, namespace: namespace, newPluginManager: newPluginManager, backupStoreGetter: backupStoreGetter, metrics: metrics, clock: &clock.RealClock{}, crClient: crClient, multiHookTracker: multiHookTracker, resourceTimeout: resourceTimeout, } } func (r *restoreFinalizerReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&velerov1api.Restore{}). Named(constant.ControllerRestoreFinalizer). Complete(r) } // +kubebuilder:rbac:groups=velero.io,resources=restores,verbs=get;list;watch;update // +kubebuilder:rbac:groups=velero.io,resources=restores/status,verbs=get func (r *restoreFinalizerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := r.logger.WithField("restore finalizer", req.String()) log.Debug("restoreFinalizerReconciler getting restore") original := &velerov1api.Restore{} if err := r.Get(ctx, req.NamespacedName, original); err != nil { if apierrors.IsNotFound(err) { log.WithError(err).Error("restore not found") return ctrl.Result{}, nil } return ctrl.Result{}, errors.Wrapf(err, "error getting restore %s", req.String()) } restore := original.DeepCopy() log.Debugf("restore: %s", restore.Name) log = r.logger.WithFields( logrus.Fields{ "restore": req.String(), }, ) switch restore.Status.Phase { case velerov1api.RestorePhaseFinalizing, velerov1api.RestorePhaseFinalizingPartiallyFailed: default: log.Debug("Restore is not awaiting finalization, skipping") return ctrl.Result{}, nil } info, err := fetchBackupInfoInternal(r.Client, r.namespace, restore.Spec.BackupName) if err != nil { if apierrors.IsNotFound(err) { log.WithError(err).Error("not found backup, skip") if err2 := r.finishProcessing(velerov1api.RestorePhasePartiallyFailed, restore, original); err2 != nil { log.WithError(err2).Error("error updating restore's final status") return ctrl.Result{}, errors.Wrap(err2, "error updating restore's final status") } return ctrl.Result{}, nil } log.WithError(err).Error("error getting backup info") return ctrl.Result{}, errors.Wrap(err, "error getting backup info") } pluginManager := r.newPluginManager(r.logger) defer pluginManager.CleanupClients() backupStore, err := r.backupStoreGetter.Get(info.location, pluginManager, r.logger) if err != nil { log.WithError(err).Error("error getting backup store") return ctrl.Result{}, errors.Wrap(err, "error getting backup store") } volumeInfo, err := backupStore.GetBackupVolumeInfos(restore.Spec.BackupName) if err != nil { log.WithError(err).Errorf("error getting volumeInfo for backup %s", restore.Spec.BackupName) return ctrl.Result{}, errors.Wrap(err, "error getting volumeInfo") } restoredResourceList, err := backupStore.GetRestoredResourceList(restore.Name) if err != nil { log.WithError(err).Error("error getting restoredResourceList") return ctrl.Result{}, errors.Wrap(err, "error getting restoredResourceList") } restoredPVCList := volume.RestoredPVCFromRestoredResourceList(restoredResourceList) restoreItemOperations, err := backupStore.GetRestoreItemOperations(restore.Name) if err != nil { log.WithError(err).Error("error getting itemOperationList") return ctrl.Result{}, errors.Wrap(err, "error getting itemOperationList") } finalizerCtx := &finalizerContext{ logger: log, restore: restore, crClient: r.crClient, volumeInfo: volumeInfo, restoredPVCList: restoredPVCList, multiHookTracker: r.multiHookTracker, resourceTimeout: r.resourceTimeout, restoreItemOperationList: restoreItemOperationList{ items: restoreItemOperations, }, } warnings, errs := finalizerCtx.execute() warningCnt := len(warnings.Velero) + len(warnings.Cluster) for _, w := range warnings.Namespaces { warningCnt += len(w) } errCnt := len(errs.Velero) + len(errs.Cluster) for _, e := range errs.Namespaces { errCnt += len(e) } restore.Status.Warnings += warningCnt restore.Status.Errors += errCnt if !errs.IsEmpty() { restore.Status.Phase = velerov1api.RestorePhaseFinalizingPartiallyFailed } if warningCnt > 0 || errCnt > 0 { err := r.updateResults(backupStore, restore, &warnings, &errs) if err != nil { log.WithError(err).Error("error updating results") return ctrl.Result{}, errors.Wrap(err, "error updating results") } } finalPhase := velerov1api.RestorePhaseCompleted if restore.Status.Phase == velerov1api.RestorePhaseFinalizingPartiallyFailed { finalPhase = velerov1api.RestorePhasePartiallyFailed } log.Infof("Marking restore %s", finalPhase) if err := r.finishProcessing(finalPhase, restore, original); err != nil { log.WithError(err).Error("error updating restore's final status") return ctrl.Result{}, errors.Wrap(err, "error updating restore's final status") } return ctrl.Result{}, nil } func (r *restoreFinalizerReconciler) updateResults(backupStore persistence.BackupStore, restore *velerov1api.Restore, newWarnings *results.Result, newErrs *results.Result) error { originResults, err := backupStore.GetRestoreResults(restore.Name) if err != nil { return errors.Wrap(err, "error getting restore results") } warnings := originResults["warnings"] errs := originResults["errors"] warnings.Merge(newWarnings) errs.Merge(newErrs) m := map[string]results.Result{ "warnings": warnings, "errors": errs, } if err := putResults(restore, m, backupStore); err != nil { return errors.Wrap(err, "error putting restore results") } return nil } func (r *restoreFinalizerReconciler) finishProcessing(restorePhase velerov1api.RestorePhase, restore *velerov1api.Restore, original *velerov1api.Restore) error { if restorePhase == velerov1api.RestorePhasePartiallyFailed { restore.Status.Phase = velerov1api.RestorePhasePartiallyFailed r.metrics.RegisterRestorePartialFailure(restore.Spec.ScheduleName) } else { restore.Status.Phase = velerov1api.RestorePhaseCompleted r.metrics.RegisterRestoreSuccess(restore.Spec.ScheduleName) } restore.Status.CompletionTimestamp = &metav1.Time{Time: r.clock.Now()} // retry `Finalizing`/`FinalizingPartiallyFailed` to // - `Completed` // - `PartiallyFailed` return kubeutil.PatchResourceWithRetriesOnErrors(r.resourceTimeout, original, restore, r.Client) } type restoreItemOperationList struct { items []*itemoperation.RestoreOperation } func (r *restoreItemOperationList) selectByResource(group, resource, ns, name string) []*itemoperation.RestoreOperation { var res []*itemoperation.RestoreOperation rid := velero.ResourceIdentifier{ GroupResource: schema.GroupResource{ Group: group, Resource: resource, }, Namespace: ns, Name: name, } for _, item := range r.items { if item != nil && item.Spec.ResourceIdentifier == rid { res = append(res, item) } } return res } // SelectByPVC filters the restore item operation list by PVC namespace and name. func (r *restoreItemOperationList) SelectByPVC(ns, name string) []*itemoperation.RestoreOperation { return r.selectByResource("", "persistentvolumeclaims", ns, name) } // finalizerContext includes all the dependencies required by finalization tasks and // a function execute() to orderly implement task logic. type finalizerContext struct { logger logrus.FieldLogger restore *velerov1api.Restore crClient client.Client volumeInfo []*volume.BackupVolumeInfo restoredPVCList map[string]struct{} restoreItemOperationList restoreItemOperationList multiHookTracker *hook.MultiHookTracker resourceTimeout time.Duration } func (ctx *finalizerContext) execute() (results.Result, results.Result) { //nolint:unparam //temporarily ignore the lint report: result 0 is always nil (unparam) warnings, errs := results.Result{}, results.Result{} // implement finalization tasks pdpErrs := ctx.patchDynamicPVWithVolumeInfo() errs.Merge(&pdpErrs) rehErrs := ctx.WaitRestoreExecHook() errs.Merge(&rehErrs) return warnings, errs } // patchDynamicPV patches newly dynamically provisioned PV using volume info // in order to restore custom settings that would otherwise be lost during dynamic PV recreation. func (ctx *finalizerContext) patchDynamicPVWithVolumeInfo() (errs results.Result) { ctx.logger.Info("patching newly dynamically provisioned PV starts") var pvWaitGroup sync.WaitGroup var resultLock sync.Mutex maxConcurrency := 3 semaphore := make(chan struct{}, maxConcurrency) for _, volumeItem := range ctx.volumeInfo { if (volumeItem.BackupMethod == volume.PodVolumeBackup || volumeItem.BackupMethod == volume.CSISnapshot) && volumeItem.PVInfo != nil { // Determine restored PVC namespace restoredNamespace := volumeItem.PVCNamespace if remapped, ok := ctx.restore.Spec.NamespaceMapping[restoredNamespace]; ok { restoredNamespace = remapped } // Check if PVC was restored in previous phase pvcKey := fmt.Sprintf("%s/%s", restoredNamespace, volumeItem.PVCName) if _, restored := ctx.restoredPVCList[pvcKey]; !restored { continue } pvWaitGroup.Add(1) go func(volInfo volume.BackupVolumeInfo, restoredNamespace string) { defer pvWaitGroup.Done() semaphore <- struct{}{} log := ctx.logger.WithField("PVC", volInfo.PVCName).WithField("PVCNamespace", restoredNamespace) log.Debug("patching dynamic PV is in progress") err := wait.PollUntilContextTimeout(context.Background(), 10*time.Second, ctx.resourceTimeout, true, func(context.Context) (bool, error) { // wait for PVC to be bound pvc := &corev1api.PersistentVolumeClaim{} err := ctx.crClient.Get(context.Background(), client.ObjectKey{Name: volInfo.PVCName, Namespace: restoredNamespace}, pvc) if apierrors.IsNotFound(err) { log.Debug("error not finding PVC") return false, nil } if err != nil { return false, err } // Check whether the async operation to populate the PVC is successful. If it's not, will skip patching the PV, instead of waiting. operations := ctx.restoreItemOperationList.SelectByPVC(pvc.Namespace, pvc.Name) for _, op := range operations { if op.Spec.RestoreItemAction == constant.PluginCSIPVCRestoreRIA && op.Status.Phase != itemoperation.OperationPhaseCompleted { log.Warnf("skipping PV patch, because the operation to restore the PVC is not completed, "+ "operation: %s, phase: %s", op.Spec.OperationID, op.Status.Phase) return true, nil } } // We are handling a common but specific scenario where a PVC is in a pending state and uses a storage class with // VolumeBindingMode set to WaitForFirstConsumer. In this case, the PV patch step is skipped to avoid // failures due to the PVC not being bound, which could cause a timeout and result in a failed restore. if pvc.Status.Phase == corev1api.ClaimPending { // check if storage class used has VolumeBindingMode as WaitForFirstConsumer scName := *pvc.Spec.StorageClassName sc := &storagev1api.StorageClass{} err = ctx.crClient.Get(context.Background(), client.ObjectKey{Name: scName}, sc) if err != nil { errs.Add(restoredNamespace, err) return false, err } // skip PV patch step for this scenario // because pvc would not be bound and the PV patch step would fail due to timeout thus failing the restore if *sc.VolumeBindingMode == storagev1api.VolumeBindingWaitForFirstConsumer { log.Warnf("skipping PV patch to restore custom reclaim policy, if any: StorageClass %s used by PVC %s has VolumeBindingMode set to WaitForFirstConsumer, and the PVC is also in a pending state", scName, pvc.Name) return true, nil } } if pvc.Status.Phase != corev1api.ClaimBound || pvc.Spec.VolumeName == "" { log.Debugf("PVC: %s not ready", pvc.Name) return false, nil } // wait for PV to be bound pvName := pvc.Spec.VolumeName pv := &corev1api.PersistentVolume{} err = ctx.crClient.Get(context.Background(), client.ObjectKey{Name: pvName}, pv) if apierrors.IsNotFound(err) { log.Debugf("error not finding PV: %s", pvName) return false, nil } if err != nil { return false, err } if pv.Spec.ClaimRef == nil || pv.Status.Phase != corev1api.VolumeBound { log.Debugf("PV: %s not ready", pvName) return false, nil } // validate PV if pv.Spec.ClaimRef.Name != pvc.Name || pv.Spec.ClaimRef.Namespace != restoredNamespace { return false, fmt.Errorf("PV was bound by unexpected PVC, unexpected PVC: %s/%s, expected PVC: %s/%s", pv.Spec.ClaimRef.Namespace, pv.Spec.ClaimRef.Name, restoredNamespace, pvc.Name) } // patch PV's reclaim policy and label using the corresponding data stored in volume info if needPatch(pv, volInfo.PVInfo) { updatedPV := pv.DeepCopy() updatedPV.Labels = volInfo.PVInfo.Labels updatedPV.Spec.PersistentVolumeReclaimPolicy = corev1api.PersistentVolumeReclaimPolicy(volInfo.PVInfo.ReclaimPolicy) if err := kubeutil.PatchResource(pv, updatedPV, ctx.crClient); err != nil { return false, err } log.Infof("newly dynamically provisioned PV:%s has been patched using volume info", pvName) } return true, nil }) if err != nil { err = fmt.Errorf("fail to patch dynamic PV, err: %s, PVC: %s, PV: %s", err, volInfo.PVCName, volInfo.PVName) ctx.logger.WithError(errors.WithStack((err))).Error("err patching dynamic PV using volume info") resultLock.Lock() defer resultLock.Unlock() errs.Add(restoredNamespace, err) } <-semaphore }(*volumeItem, restoredNamespace) } } pvWaitGroup.Wait() ctx.logger.Info("patching newly dynamically provisioned PV ends") return errs } func needPatch(newPV *corev1api.PersistentVolume, pvInfo *volume.PVInfo) bool { if newPV.Spec.PersistentVolumeReclaimPolicy != corev1api.PersistentVolumeReclaimPolicy(pvInfo.ReclaimPolicy) { return true } newPVLabels, pvLabels := newPV.Labels, pvInfo.Labels for k, v := range pvLabels { if _, ok := newPVLabels[k]; !ok { return true } if newPVLabels[k] != v { return true } } return false } // WaitRestoreExecHook waits for restore exec hooks to finish then update the hook execution results func (ctx *finalizerContext) WaitRestoreExecHook() (errs results.Result) { log := ctx.logger.WithField("restore", ctx.restore.Name) log.Info("Waiting for restore exec hooks starts") // wait for restore exec hooks to finish err := wait.PollUntilContextCancel(context.Background(), 1*time.Second, true, func(context.Context) (bool, error) { log.Debug("Checking the progress of hooks execution") if ctx.multiHookTracker.IsComplete(ctx.restore.Name) { return true, nil } return false, nil }) if err != nil { errs.Add(ctx.restore.Namespace, err) return errs } log.Info("Done waiting for restore exec hooks starts") for _, ei := range ctx.multiHookTracker.HookErrs(ctx.restore.Name) { errs.Add(ei.Namespace, ei.Err) } // update hooks execution status updated := ctx.restore.DeepCopy() if updated.Status.HookStatus == nil { updated.Status.HookStatus = &velerov1api.HookStatus{} } updated.Status.HookStatus.HooksAttempted, updated.Status.HookStatus.HooksFailed = ctx.multiHookTracker.Stat(ctx.restore.Name) log.Debugf("hookAttempted: %d, hookFailed: %d", updated.Status.HookStatus.HooksAttempted, updated.Status.HookStatus.HooksFailed) if err := kubeutil.PatchResource(ctx.restore, updated, ctx.crClient); err != nil { log.WithError(errors.WithStack((err))).Error("Updating restore status") errs.Add(ctx.restore.Namespace, err) } // delete the hook data for this restore ctx.multiHookTracker.Delete(ctx.restore.Name) return errs } ================================================ FILE: pkg/controller/restore_finalizer_controller_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package controller import ( "fmt" "syscall" "testing" "time" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" testclocks "k8s.io/utils/clock/testing" ctrl "sigs.k8s.io/controller-runtime" crclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/internal/hook" "github.com/vmware-tanzu/velero/internal/volume" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/itemoperation" "github.com/vmware-tanzu/velero/pkg/metrics" persistencemocks "github.com/vmware-tanzu/velero/pkg/persistence/mocks" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" pluginmocks "github.com/vmware-tanzu/velero/pkg/plugin/mocks" "github.com/vmware-tanzu/velero/pkg/plugin/velero" velerotest "github.com/vmware-tanzu/velero/pkg/test" pkgUtilKubeMocks "github.com/vmware-tanzu/velero/pkg/util/kube/mocks" "github.com/vmware-tanzu/velero/pkg/util/results" ) func TestRestoreFinalizerReconcile(t *testing.T) { defaultStorageLocation := builder.ForBackupStorageLocation("velero", "default").Provider("myCloud").Bucket("bucket").Result() now, err := time.Parse(time.RFC1123Z, time.RFC1123Z) require.NoError(t, err) now = now.Local() timestamp := metav1.NewTime(now) assert.NotNil(t, timestamp) rfrTests := []struct { name string restore *velerov1api.Restore backup *velerov1api.Backup location *velerov1api.BackupStorageLocation expectError bool expectPhase velerov1api.RestorePhase expectWarningsCnt int expectErrsCnt int statusCompare bool expectedCompletedTime *metav1.Time }{ { name: "Restore is not awaiting finalization, skip", restore: builder.ForRestore(velerov1api.DefaultNamespace, "restore-1").Phase(velerov1api.RestorePhaseInProgress).Result(), expectError: false, expectPhase: velerov1api.RestorePhaseInProgress, statusCompare: false, }, { name: "Upon completion of all finalization tasks in the 'FinalizingPartiallyFailed' phase, the restore process transit to the 'PartiallyFailed' phase.", restore: builder.ForRestore(velerov1api.DefaultNamespace, "restore-1").Phase(velerov1api.RestorePhaseFinalizingPartiallyFailed).Backup("backup-1").Result(), backup: defaultBackup().StorageLocation("default").Result(), location: defaultStorageLocation, expectError: false, expectPhase: velerov1api.RestorePhasePartiallyFailed, statusCompare: true, expectedCompletedTime: ×tamp, expectWarningsCnt: 0, expectErrsCnt: 0, }, { name: "Upon completion of all finalization tasks in the 'Finalizing' phase, the restore process transit to the 'Completed' phase.", restore: builder.ForRestore(velerov1api.DefaultNamespace, "restore-1").Phase(velerov1api.RestorePhaseFinalizing).Backup("backup-1").Result(), backup: defaultBackup().StorageLocation("default").Result(), location: defaultStorageLocation, expectError: false, expectPhase: velerov1api.RestorePhaseCompleted, statusCompare: true, expectedCompletedTime: ×tamp, expectWarningsCnt: 0, expectErrsCnt: 0, }, { name: "Backup not exist", restore: builder.ForRestore(velerov1api.DefaultNamespace, "restore-1").Phase(velerov1api.RestorePhaseFinalizing).Backup("backup-2").Result(), expectError: false, }, { name: "Restore not exist", restore: builder.ForRestore("unknown", "restore-1").Phase(velerov1api.RestorePhaseFinalizing).Result(), expectError: false, statusCompare: false, }, } for _, test := range rfrTests { t.Run(test.name, func(t *testing.T) { if test.restore == nil { return } var ( fakeClient = velerotest.NewFakeControllerRuntimeClientBuilder(t).Build() logger = velerotest.NewLogger() pluginManager = &pluginmocks.Manager{} backupStore = &persistencemocks.BackupStore{} ) defer func() { // reset defaultStorageLocation resourceVersion defaultStorageLocation.ObjectMeta.ResourceVersion = "" }() r := NewRestoreFinalizerReconciler( logger, velerov1api.DefaultNamespace, fakeClient, func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager }, NewFakeSingleObjectBackupStoreGetter(backupStore), metrics.NewServerMetrics(), fakeClient, hook.NewMultiHookTracker(), 10*time.Minute, ) r.clock = testclocks.NewFakeClock(now) if test.restore != nil && test.restore.Namespace == velerov1api.DefaultNamespace { require.NoError(t, r.Client.Create(t.Context(), test.restore)) backupStore.On("GetRestoredResourceList", test.restore.Name).Return(map[string][]string{}, nil) backupStore.On("GetRestoreItemOperations", test.restore.Name).Return([]*itemoperation.RestoreOperation{}, nil) } if test.backup != nil { require.NoError(t, r.Client.Create(t.Context(), test.backup)) backupStore.On("GetBackupVolumeInfos", test.backup.Name).Return(nil, nil) pluginManager.On("GetRestoreItemActionsV2").Return(nil, nil) pluginManager.On("CleanupClients") } if test.location != nil { require.NoError(t, r.Client.Create(t.Context(), test.location)) } _, err = r.Reconcile(t.Context(), ctrl.Request{NamespacedName: types.NamespacedName{ Namespace: test.restore.Namespace, Name: test.restore.Name, }}) assert.Equal(t, test.expectError, err != nil) if test.expectError { return } if test.statusCompare { restoreAfter := velerov1api.Restore{} err = fakeClient.Get(t.Context(), types.NamespacedName{ Namespace: test.restore.Namespace, Name: test.restore.Name, }, &restoreAfter) require.NoError(t, err) assert.Equal(t, test.expectPhase, restoreAfter.Status.Phase) assert.Equal(t, test.expectErrsCnt, restoreAfter.Status.Errors) assert.Equal(t, test.expectWarningsCnt, restoreAfter.Status.Warnings) require.True(t, test.expectedCompletedTime.Equal(restoreAfter.Status.CompletionTimestamp)) } }) } } func TestUpdateResult(t *testing.T) { var ( fakeClient = velerotest.NewFakeControllerRuntimeClientBuilder(t).Build() logger = velerotest.NewLogger() pluginManager = &pluginmocks.Manager{} backupStore = &persistencemocks.BackupStore{} ) r := NewRestoreFinalizerReconciler( logger, velerov1api.DefaultNamespace, fakeClient, func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager }, NewFakeSingleObjectBackupStoreGetter(backupStore), metrics.NewServerMetrics(), fakeClient, hook.NewMultiHookTracker(), 10*time.Minute, ) restore := builder.ForRestore(velerov1api.DefaultNamespace, "restore-1").Result() res := map[string]results.Result{"warnings": {}, "errors": {}} backupStore.On("GetRestoreResults", restore.Name).Return(res, nil) backupStore.On("PutRestoreResults", mock.Anything, mock.Anything, mock.Anything).Return(nil) err := r.updateResults(backupStore, restore, &results.Result{}, &results.Result{}) require.NoError(t, err) } func TestPatchDynamicPVWithVolumeInfo(t *testing.T) { tests := []struct { name string volumeInfo []*volume.BackupVolumeInfo restoredPVCNames map[string]struct{} restore *velerov1api.Restore restoredPVC []*corev1api.PersistentVolumeClaim restoredPV []*corev1api.PersistentVolume expectedPatch map[string]volume.PVInfo expectedErrNum int }{ { name: "no applicable volumeInfo", volumeInfo: []*volume.BackupVolumeInfo{{BackupMethod: "VeleroNativeSnapshot", PVCName: "pvc1"}}, restore: builder.ForRestore(velerov1api.DefaultNamespace, "restore").Result(), expectedPatch: nil, expectedErrNum: 0, }, { name: "no restored PVC", volumeInfo: []*volume.BackupVolumeInfo{{BackupMethod: "PodVolumeBackup", PVCName: "pvc1"}}, restore: builder.ForRestore(velerov1api.DefaultNamespace, "restore").Result(), expectedPatch: nil, expectedErrNum: 0, }, { name: "no applicable pv patch", volumeInfo: []*volume.BackupVolumeInfo{{ BackupMethod: "PodVolumeBackup", PVCName: "pvc1", PVName: "pv1", PVCNamespace: "ns1", PVInfo: &volume.PVInfo{ ReclaimPolicy: string(corev1api.PersistentVolumeReclaimDelete), Labels: map[string]string{"label1": "label1-val"}, }, }}, restore: builder.ForRestore(velerov1api.DefaultNamespace, "restore").Result(), restoredPVCNames: map[string]struct{}{"ns1/pvc1": {}}, restoredPV: []*corev1api.PersistentVolume{ builder.ForPersistentVolume("new-pv1").ObjectMeta(builder.WithLabels("label1", "label1-val")).ClaimRef("ns1", "pvc1").Phase(corev1api.VolumeBound).ReclaimPolicy(corev1api.PersistentVolumeReclaimDelete).Result()}, restoredPVC: []*corev1api.PersistentVolumeClaim{ builder.ForPersistentVolumeClaim("ns1", "pvc1").VolumeName("new-pv1").Phase(corev1api.ClaimBound).Result(), }, expectedPatch: nil, expectedErrNum: 0, }, { name: "an applicable pv patch", volumeInfo: []*volume.BackupVolumeInfo{{ BackupMethod: "PodVolumeBackup", PVCName: "pvc1", PVName: "pv1", PVCNamespace: "ns1", PVInfo: &volume.PVInfo{ ReclaimPolicy: string(corev1api.PersistentVolumeReclaimDelete), Labels: map[string]string{"label1": "label1-val"}, }, }}, restore: builder.ForRestore(velerov1api.DefaultNamespace, "restore").Result(), restoredPVCNames: map[string]struct{}{"ns1/pvc1": {}}, restoredPV: []*corev1api.PersistentVolume{ builder.ForPersistentVolume("new-pv1").ClaimRef("ns1", "pvc1").Phase(corev1api.VolumeBound).ReclaimPolicy(corev1api.PersistentVolumeReclaimRetain).Result()}, restoredPVC: []*corev1api.PersistentVolumeClaim{ builder.ForPersistentVolumeClaim("ns1", "pvc1").VolumeName("new-pv1").Phase(corev1api.ClaimBound).Result(), }, expectedPatch: map[string]volume.PVInfo{"new-pv1": { ReclaimPolicy: string(corev1api.PersistentVolumeReclaimDelete), Labels: map[string]string{"label1": "label1-val"}, }}, expectedErrNum: 0, }, { name: "a mapped namespace restore", volumeInfo: []*volume.BackupVolumeInfo{{ BackupMethod: "PodVolumeBackup", PVCName: "pvc1", PVName: "pv1", PVCNamespace: "ns2", PVInfo: &volume.PVInfo{ ReclaimPolicy: string(corev1api.PersistentVolumeReclaimDelete), Labels: map[string]string{"label1": "label1-val"}, }, }}, restore: builder.ForRestore(velerov1api.DefaultNamespace, "restore").NamespaceMappings("ns2", "ns1").Result(), restoredPVCNames: map[string]struct{}{"ns1/pvc1": {}}, restoredPV: []*corev1api.PersistentVolume{ builder.ForPersistentVolume("new-pv1").ClaimRef("ns1", "pvc1").Phase(corev1api.VolumeBound).ReclaimPolicy(corev1api.PersistentVolumeReclaimRetain).Result()}, restoredPVC: []*corev1api.PersistentVolumeClaim{ builder.ForPersistentVolumeClaim("ns1", "pvc1").VolumeName("new-pv1").Phase(corev1api.ClaimBound).Result(), }, expectedPatch: map[string]volume.PVInfo{"new-pv1": { ReclaimPolicy: string(corev1api.PersistentVolumeReclaimDelete), Labels: map[string]string{"label1": "label1-val"}, }}, expectedErrNum: 0, }, { name: "two applicable pv patches", volumeInfo: []*volume.BackupVolumeInfo{{ BackupMethod: "PodVolumeBackup", PVCName: "pvc1", PVName: "pv1", PVCNamespace: "ns1", PVInfo: &volume.PVInfo{ ReclaimPolicy: string(corev1api.PersistentVolumeReclaimDelete), Labels: map[string]string{"label1": "label1-val"}, }, }, { BackupMethod: "CSISnapshot", PVCName: "pvc2", PVName: "pv2", PVCNamespace: "ns2", PVInfo: &volume.PVInfo{ ReclaimPolicy: string(corev1api.PersistentVolumeReclaimDelete), Labels: map[string]string{"label2": "label2-val"}, }, }, }, restore: builder.ForRestore(velerov1api.DefaultNamespace, "restore").Result(), restoredPVCNames: map[string]struct{}{ "ns1/pvc1": {}, "ns2/pvc2": {}, }, restoredPV: []*corev1api.PersistentVolume{ builder.ForPersistentVolume("new-pv1").ClaimRef("ns1", "pvc1").Phase(corev1api.VolumeBound).ReclaimPolicy(corev1api.PersistentVolumeReclaimRetain).Result(), builder.ForPersistentVolume("new-pv2").ClaimRef("ns2", "pvc2").Phase(corev1api.VolumeBound).ReclaimPolicy(corev1api.PersistentVolumeReclaimRetain).Result(), }, restoredPVC: []*corev1api.PersistentVolumeClaim{ builder.ForPersistentVolumeClaim("ns1", "pvc1").VolumeName("new-pv1").Phase(corev1api.ClaimBound).Result(), builder.ForPersistentVolumeClaim("ns2", "pvc2").VolumeName("new-pv2").Phase(corev1api.ClaimBound).Result(), }, expectedPatch: map[string]volume.PVInfo{ "new-pv1": { ReclaimPolicy: string(corev1api.PersistentVolumeReclaimDelete), Labels: map[string]string{"label1": "label1-val"}, }, "new-pv2": { ReclaimPolicy: string(corev1api.PersistentVolumeReclaimDelete), Labels: map[string]string{"label2": "label2-val"}, }, }, expectedErrNum: 0, }, { name: "an applicable pv patch with bound error", volumeInfo: []*volume.BackupVolumeInfo{{ BackupMethod: "PodVolumeBackup", PVCName: "pvc1", PVName: "pv1", PVCNamespace: "ns1", PVInfo: &volume.PVInfo{ ReclaimPolicy: string(corev1api.PersistentVolumeReclaimDelete), Labels: map[string]string{"label1": "label1-val"}, }, }}, restore: builder.ForRestore(velerov1api.DefaultNamespace, "restore").Result(), restoredPVCNames: map[string]struct{}{"ns1/pvc1": {}}, restoredPV: []*corev1api.PersistentVolume{ builder.ForPersistentVolume("new-pv1").ClaimRef("ns2", "pvc2").Phase(corev1api.VolumeBound).ReclaimPolicy(corev1api.PersistentVolumeReclaimRetain).Result()}, restoredPVC: []*corev1api.PersistentVolumeClaim{ builder.ForPersistentVolumeClaim("ns1", "pvc1").VolumeName("new-pv1").Phase(corev1api.ClaimBound).Result(), }, expectedErrNum: 1, }, { name: "two applicable pv patches with an error", volumeInfo: []*volume.BackupVolumeInfo{{ BackupMethod: "PodVolumeBackup", PVCName: "pvc1", PVName: "pv1", PVCNamespace: "ns1", PVInfo: &volume.PVInfo{ ReclaimPolicy: string(corev1api.PersistentVolumeReclaimDelete), Labels: map[string]string{"label1": "label1-val"}, }, }, { BackupMethod: "CSISnapshot", PVCName: "pvc2", PVName: "pv2", PVCNamespace: "ns2", PVInfo: &volume.PVInfo{ ReclaimPolicy: string(corev1api.PersistentVolumeReclaimDelete), Labels: map[string]string{"label2": "label2-val"}, }, }, }, restore: builder.ForRestore(velerov1api.DefaultNamespace, "restore").Result(), restoredPVCNames: map[string]struct{}{ "ns1/pvc1": {}, "ns2/pvc2": {}, }, restoredPV: []*corev1api.PersistentVolume{ builder.ForPersistentVolume("new-pv1").ClaimRef("ns1", "pvc1").Phase(corev1api.VolumeBound).ReclaimPolicy(corev1api.PersistentVolumeReclaimRetain).Result(), builder.ForPersistentVolume("new-pv2").ClaimRef("ns3", "pvc3").Phase(corev1api.VolumeBound).ReclaimPolicy(corev1api.PersistentVolumeReclaimRetain).Result(), }, restoredPVC: []*corev1api.PersistentVolumeClaim{ builder.ForPersistentVolumeClaim("ns1", "pvc1").VolumeName("new-pv1").Phase(corev1api.ClaimBound).Result(), builder.ForPersistentVolumeClaim("ns2", "pvc2").VolumeName("new-pv2").Phase(corev1api.ClaimBound).Result(), }, expectedPatch: map[string]volume.PVInfo{ "new-pv1": { ReclaimPolicy: string(corev1api.PersistentVolumeReclaimDelete), Labels: map[string]string{"label1": "label1-val"}, }, }, expectedErrNum: 1, }, } for _, tc := range tests { var ( fakeClient = velerotest.NewFakeControllerRuntimeClientBuilder(t).Build() logger = velerotest.NewLogger() ) ctx := &finalizerContext{ logger: logger, crClient: fakeClient, restore: tc.restore, restoredPVCList: tc.restoredPVCNames, volumeInfo: tc.volumeInfo, } for _, pv := range tc.restoredPV { require.NoError(t, ctx.crClient.Create(t.Context(), pv)) } for _, pvc := range tc.restoredPVC { require.NoError(t, ctx.crClient.Create(t.Context(), pvc)) } errs := ctx.patchDynamicPVWithVolumeInfo() if tc.expectedErrNum > 0 { assert.Len(t, errs.Namespaces, tc.expectedErrNum) } for pvName, expectedPVInfo := range tc.expectedPatch { pv := &corev1api.PersistentVolume{} err := ctx.crClient.Get(t.Context(), crclient.ObjectKey{Name: pvName}, pv) require.NoError(t, err) assert.Equal(t, expectedPVInfo.ReclaimPolicy, string(pv.Spec.PersistentVolumeReclaimPolicy)) assert.Equal(t, expectedPVInfo.Labels, pv.Labels) } } } func TestWaitRestoreExecHook(t *testing.T) { hookTracker1 := hook.NewMultiHookTracker() restoreName1 := "restore1" hookTracker2 := hook.NewMultiHookTracker() restoreName2 := "restore2" hookTracker2.Add(restoreName2, "ns", "pod", "con1", "s1", "h1", "", 0) hookTracker2.Record(restoreName2, "ns", "pod", "con1", "s1", "h1", "", 0, false, nil) hookTracker3 := hook.NewMultiHookTracker() restoreName3 := "restore3" podNs, podName, container, source, hookName := "ns", "pod", "con1", "s1", "h1" hookFailed, hookErr := true, fmt.Errorf("hook failed") hookTracker3.Add(restoreName3, podNs, podName, container, source, hookName, hook.PhasePre, 0) tests := []struct { name string hookTracker *hook.MultiHookTracker restore *velerov1api.Restore expectedHooksAttempted int expectedHooksFailed int expectedHookErrs int waitSec int podName string podNs string Container string Source string hookName string hookFailed bool hookErr error }{ { name: "no restore exec hooks", hookTracker: hookTracker1, restore: builder.ForRestore(velerov1api.DefaultNamespace, restoreName1).Result(), expectedHooksAttempted: 0, expectedHooksFailed: 0, expectedHookErrs: 0, }, { name: "1 restore exec hook having been executed", hookTracker: hookTracker2, restore: builder.ForRestore(velerov1api.DefaultNamespace, restoreName2).Result(), expectedHooksAttempted: 1, expectedHooksFailed: 0, expectedHookErrs: 0, }, { name: "1 restore exec hook to be executed", hookTracker: hookTracker3, restore: builder.ForRestore(velerov1api.DefaultNamespace, restoreName3).Result(), waitSec: 2, expectedHooksAttempted: 1, expectedHooksFailed: 1, expectedHookErrs: 1, podName: podName, podNs: podNs, Container: container, Source: source, hookName: hookName, hookFailed: hookFailed, hookErr: hookErr, }, } for _, tc := range tests { var ( fakeClient = velerotest.NewFakeControllerRuntimeClientBuilder(t).Build() logger = velerotest.NewLogger() ) ctx := &finalizerContext{ logger: logger, crClient: fakeClient, restore: tc.restore, multiHookTracker: tc.hookTracker, } require.NoError(t, ctx.crClient.Create(t.Context(), tc.restore)) if tc.waitSec > 0 { go func() { time.Sleep(time.Second * time.Duration(tc.waitSec)) tc.hookTracker.Record(tc.restore.Name, tc.podNs, tc.podName, tc.Container, tc.Source, tc.hookName, hook.PhasePre, 0, tc.hookFailed, tc.hookErr) }() } errs := ctx.WaitRestoreExecHook() assert.Len(t, errs.Namespaces, tc.expectedHookErrs) updated := &velerov1api.Restore{} err := ctx.crClient.Get(t.Context(), crclient.ObjectKey{Namespace: velerov1api.DefaultNamespace, Name: tc.restore.Name}, updated) require.NoError(t, err) assert.Equal(t, tc.expectedHooksAttempted, updated.Status.HookStatus.HooksAttempted) assert.Equal(t, tc.expectedHooksFailed, updated.Status.HookStatus.HooksFailed) } } // test finishprocessing with mocks of kube client to simulate connection refused func Test_restoreFinalizerReconciler_finishProcessing(t *testing.T) { type args struct { // mockClientActions simulate different client errors mockClientActions func(*pkgUtilKubeMocks.Client) // return bool indicating if the client method was called as expected mockClientAsserts func(*pkgUtilKubeMocks.Client) bool } tests := []struct { name string args args wantErr bool }{ { name: "restore failed to patch status, should retry on connection refused", args: args{ mockClientActions: func(client *pkgUtilKubeMocks.Client) { client.On("Patch", mock.Anything, mock.Anything, mock.Anything).Return(syscall.ECONNREFUSED).Once() client.On("Patch", mock.Anything, mock.Anything, mock.Anything).Return(nil) }, mockClientAsserts: func(client *pkgUtilKubeMocks.Client) bool { return client.AssertNumberOfCalls(t, "Patch", 2) }, }, }, { name: "restore failed to patch status, retry on connection refused until max retries", args: args{ mockClientActions: func(client *pkgUtilKubeMocks.Client) { client.On("Patch", mock.Anything, mock.Anything, mock.Anything).Return(syscall.ECONNREFUSED) }, mockClientAsserts: func(client *pkgUtilKubeMocks.Client) bool { return len(client.Calls) > 2 }, }, wantErr: true, }, { name: "restore patch status ok, should not retry", args: args{ mockClientActions: func(client *pkgUtilKubeMocks.Client) { client.On("Patch", mock.Anything, mock.Anything, mock.Anything).Return(nil) }, mockClientAsserts: func(client *pkgUtilKubeMocks.Client) bool { return client.AssertNumberOfCalls(t, "Patch", 1) }, }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { client := pkgUtilKubeMocks.NewClient(t) // mock client actions tt.args.mockClientActions(client) r := &restoreFinalizerReconciler{ Client: client, metrics: metrics.NewServerMetrics(), clock: testclocks.NewFakeClock(time.Now()), resourceTimeout: 1 * time.Second, } restore := builder.ForRestore(velerov1api.DefaultNamespace, "restoreName").Result() if err := r.finishProcessing(velerov1api.RestorePhaseInProgress, restore, restore); (err != nil) != tt.wantErr { t.Errorf("restoreFinalizerReconciler.finishProcessing() error = %v, wantErr %v", err, tt.wantErr) } if !tt.args.mockClientAsserts(client) { t.Errorf("mockClientAsserts() failed") } }) } } func TestRestoreOperationList(t *testing.T) { var empty []*itemoperation.RestoreOperation tests := []struct { name string items []*itemoperation.RestoreOperation inputPVCNS string inputPVCName string expected []*itemoperation.RestoreOperation }{ { name: "no restore operations", items: []*itemoperation.RestoreOperation{}, inputPVCNS: "ns-1", inputPVCName: "pvc-1", expected: empty, }, { name: "one operation with matched info and a nil element", items: []*itemoperation.RestoreOperation{ nil, { Spec: itemoperation.RestoreOperationSpec{ RestoreName: "restore-1", RestoreUID: "uid-1", RestoreItemAction: "velero.io/csi-pvc-restorer", OperationID: "dd-abbb048d-7036-4855-bf50-ebba978b59a6.2426dd0e-b863-4222b5b2b", ResourceIdentifier: velero.ResourceIdentifier{ GroupResource: schema.GroupResource{ Group: "", Resource: "persistentvolumeclaims", }, Namespace: "ns-1", Name: "pvc-1", }, }, Status: itemoperation.OperationStatus{ Phase: itemoperation.OperationPhaseCompleted, OperationUnits: "Byte", Description: "Completed", }, }, }, inputPVCNS: "ns-1", inputPVCName: "pvc-1", expected: []*itemoperation.RestoreOperation{ { Spec: itemoperation.RestoreOperationSpec{ RestoreName: "restore-1", RestoreUID: "uid-1", RestoreItemAction: "velero.io/csi-pvc-restorer", OperationID: "dd-abbb048d-7036-4855-bf50-ebba978b59a6.2426dd0e-b863-4222b5b2b", ResourceIdentifier: velero.ResourceIdentifier{ GroupResource: schema.GroupResource{ Group: "", Resource: "persistentvolumeclaims", }, Namespace: "ns-1", Name: "pvc-1", }, }, Status: itemoperation.OperationStatus{ Phase: itemoperation.OperationPhaseCompleted, OperationUnits: "Byte", Description: "Completed", }, }, }, }, { name: "one operation with incorrect resource type", items: []*itemoperation.RestoreOperation{ { Spec: itemoperation.RestoreOperationSpec{ RestoreName: "restore-1", RestoreUID: "uid-1", RestoreItemAction: "velero.io/csi-pvc-restorer", OperationID: "dd-abbb048d-7036-4855-bf50-ebba978b59a6.2426dd0e-b863-4222b5b2b", ResourceIdentifier: velero.ResourceIdentifier{ GroupResource: schema.GroupResource{ Group: "", Resource: "configmaps", }, Namespace: "ns-1", Name: "pvc-1", }, }, Status: itemoperation.OperationStatus{ Phase: itemoperation.OperationPhaseCompleted, OperationUnits: "Byte", Description: "Completed", }, }, }, inputPVCNS: "ns-1", inputPVCName: "pvc-1", expected: empty, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { l := restoreItemOperationList{ items: tt.items, } assert.Equal(t, tt.expected, l.SelectByPVC(tt.inputPVCNS, tt.inputPVCName)) }) } } ================================================ FILE: pkg/controller/restore_operations_controller.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package controller import ( "context" "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clocks "k8s.io/utils/clock" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/predicate" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/constant" "github.com/vmware-tanzu/velero/pkg/itemoperation" "github.com/vmware-tanzu/velero/pkg/itemoperationmap" "github.com/vmware-tanzu/velero/pkg/metrics" "github.com/vmware-tanzu/velero/pkg/persistence" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" "github.com/vmware-tanzu/velero/pkg/util/kube" ) const ( defaultRestoreOperationsFrequency = 10 * time.Second ) type restoreOperationsReconciler struct { client.Client namespace string logger logrus.FieldLogger clock clocks.WithTickerAndDelayedExecution frequency time.Duration itemOperationsMap *itemoperationmap.RestoreItemOperationsMap newPluginManager func(logger logrus.FieldLogger) clientmgmt.Manager backupStoreGetter persistence.ObjectBackupStoreGetter metrics *metrics.ServerMetrics } func NewRestoreOperationsReconciler( logger logrus.FieldLogger, namespace string, client client.Client, frequency time.Duration, newPluginManager func(logrus.FieldLogger) clientmgmt.Manager, backupStoreGetter persistence.ObjectBackupStoreGetter, metrics *metrics.ServerMetrics, itemOperationsMap *itemoperationmap.RestoreItemOperationsMap, ) *restoreOperationsReconciler { abor := &restoreOperationsReconciler{ Client: client, logger: logger, namespace: namespace, clock: clocks.RealClock{}, frequency: frequency, itemOperationsMap: itemOperationsMap, newPluginManager: newPluginManager, backupStoreGetter: backupStoreGetter, metrics: metrics, } if abor.frequency <= 0 { abor.frequency = defaultRestoreOperationsFrequency } return abor } func (r *restoreOperationsReconciler) SetupWithManager(mgr ctrl.Manager) error { gp := kube.NewGenericEventPredicate(func(object client.Object) bool { restore := object.(*velerov1api.Restore) return (restore.Status.Phase == velerov1api.RestorePhaseWaitingForPluginOperations || restore.Status.Phase == velerov1api.RestorePhaseWaitingForPluginOperationsPartiallyFailed) }) s := kube.NewPeriodicalEnqueueSource(r.logger.WithField("controller", constant.ControllerRestoreOperations), mgr.GetClient(), &velerov1api.RestoreList{}, r.frequency, kube.PeriodicalEnqueueSourceOption{ Predicates: []predicate.Predicate{gp}, }) return ctrl.NewControllerManagedBy(mgr). For(&velerov1api.Restore{}, builder.WithPredicates(kube.FalsePredicate{})). WatchesRawSource(s). Named(constant.ControllerRestoreOperations). Complete(r) } // +kubebuilder:rbac:groups=velero.io,resources=restores,verbs=get;list;watch;update // +kubebuilder:rbac:groups=velero.io,resources=restores/status,verbs=get func (r *restoreOperationsReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := r.logger.WithField("restore operations for restore", req.String()) log.Debug("restoreOperationsReconciler getting restore") original := &velerov1api.Restore{} if err := r.Get(ctx, req.NamespacedName, original); err != nil { if apierrors.IsNotFound(err) { log.WithError(err).Error("restore not found") return ctrl.Result{}, nil } return ctrl.Result{}, errors.Wrapf(err, "error getting restore %s", req.String()) } restore := original.DeepCopy() log.Debugf("restore: %s", restore.Name) log = r.logger.WithFields( logrus.Fields{ "restore": req.String(), }, ) switch restore.Status.Phase { case velerov1api.RestorePhaseWaitingForPluginOperations, velerov1api.RestorePhaseWaitingForPluginOperationsPartiallyFailed: // only process restores waiting for plugin operations to complete default: log.Debug("Restore has no ongoing plugin operations, skipping") return ctrl.Result{}, nil } info, err := r.fetchBackupInfo(restore.Spec.BackupName) if err != nil { log.Warnf("Cannot check progress on Restore operations because backup info is unavailable %s; marking restore FinalizingPartiallyFailed", err.Error()) restore.Status.Phase = velerov1api.RestorePhaseFinalizingPartiallyFailed err2 := r.updateRestoreAndOperationsJSON(ctx, original, restore, nil, &itemoperationmap.OperationsForRestore{ErrsSinceUpdate: []string{err.Error()}}, false, false) if err2 != nil { log.WithError(err2).Error("error updating Restore") } return ctrl.Result{}, errors.Wrap(err, "error getting backup info") } pluginManager := r.newPluginManager(r.logger) defer pluginManager.CleanupClients() backupStore, err := r.backupStoreGetter.Get(info.location, pluginManager, r.logger) if err != nil { return ctrl.Result{}, errors.Wrap(err, "error getting backup store") } operations, err := r.itemOperationsMap.GetOperationsForRestore(backupStore, restore.Name) if err != nil { err2 := r.updateRestoreAndOperationsJSON(ctx, original, restore, backupStore, &itemoperationmap.OperationsForRestore{ErrsSinceUpdate: []string{err.Error()}}, false, false) if err2 != nil { return ctrl.Result{}, errors.Wrap(err2, "error updating Restore") } return ctrl.Result{}, errors.Wrap(err, "error getting restore operations") } stillInProgress, changes, opsCompleted, opsFailed, errs := getRestoreItemOperationProgress(restore, pluginManager, operations.Operations) // if len(errs)>0, need to update restore errors and error log operations.ErrsSinceUpdate = append(operations.ErrsSinceUpdate, errs...) restore.Status.Errors += len(operations.ErrsSinceUpdate) completionChanges := false if restore.Status.RestoreItemOperationsCompleted != opsCompleted || restore.Status.RestoreItemOperationsFailed != opsFailed { completionChanges = true restore.Status.RestoreItemOperationsCompleted = opsCompleted restore.Status.RestoreItemOperationsFailed = opsFailed } if changes { operations.ChangesSinceUpdate = true } if len(operations.ErrsSinceUpdate) > 0 { restore.Status.Phase = velerov1api.RestorePhaseWaitingForPluginOperationsPartiallyFailed } // if stillInProgress is false, restore moves to terminal phase and needs update // if operations.ErrsSinceUpdate is not empty, then restore phase needs to change to // RestorePhaseWaitingForPluginOperationsPartiallyFailed and needs update // If the only changes are incremental progress, then no write is necessary, progress can remain in memory if !stillInProgress { if restore.Status.Phase == velerov1api.RestorePhaseWaitingForPluginOperations { log.Infof("Marking restore %s Finalizing", restore.Name) restore.Status.Phase = velerov1api.RestorePhaseFinalizing } else { log.Infof("Marking restore %s FinalizingPartiallyFailed", restore.Name) restore.Status.Phase = velerov1api.RestorePhaseFinalizingPartiallyFailed } } err = r.updateRestoreAndOperationsJSON(ctx, original, restore, backupStore, operations, changes, completionChanges) if err != nil { return ctrl.Result{}, errors.Wrap(err, "error updating Restore") } return ctrl.Result{}, nil } // fetchBackupInfo checks the backup lister for a backup that matches the given name. If it doesn't // find it, it returns an error. func (r *restoreOperationsReconciler) fetchBackupInfo(backupName string) (backupInfo, error) { return fetchBackupInfoInternal(r.Client, r.namespace, backupName) } func (r *restoreOperationsReconciler) updateRestoreAndOperationsJSON( ctx context.Context, original, restore *velerov1api.Restore, backupStore persistence.BackupStore, operations *itemoperationmap.OperationsForRestore, changes bool, completionChanges bool) error { if len(operations.ErrsSinceUpdate) > 0 { // FIXME: download/upload results r.logger.WithField("restore", restore.Name).Infof("Restore has %d errors", len(operations.ErrsSinceUpdate)) } removeIfComplete := true defer func() { // remove local operations list if complete if removeIfComplete && (restore.Status.Phase == velerov1api.RestorePhaseFinalizing || restore.Status.Phase == velerov1api.RestorePhaseFinalizingPartiallyFailed) { r.itemOperationsMap.DeleteOperationsForRestore(restore.Name) } else if changes { r.itemOperationsMap.PutOperationsForRestore(operations, restore.Name) } }() // update restore and upload progress if errs or complete if len(operations.ErrsSinceUpdate) > 0 || restore.Status.Phase == velerov1api.RestorePhaseFinalizing || restore.Status.Phase == velerov1api.RestorePhaseFinalizingPartiallyFailed { // update file store if backupStore != nil { if err := r.itemOperationsMap.UploadProgressAndPutOperationsForRestore(backupStore, operations, restore.Name); err != nil { removeIfComplete = false return err } } // update restore err := r.Client.Patch(ctx, restore, client.MergeFrom(original)) if err != nil { removeIfComplete = false return errors.Wrapf(err, "error updating Restore %s", restore.Name) } } else if completionChanges { // If restore is still incomplete and no new errors are found but there are some new operations // completed, patch restore to reflect new completion numbers, but don't upload detailed json file err := r.Client.Patch(ctx, restore, client.MergeFrom(original)) if err != nil { return errors.Wrapf(err, "error updating Restore %s", restore.Name) } } return nil } func getRestoreItemOperationProgress( restore *velerov1api.Restore, pluginManager clientmgmt.Manager, operationsList []*itemoperation.RestoreOperation) (bool, bool, int, int, []string) { inProgressOperations := false changes := false var errs []string var completedCount, failedCount int for _, operation := range operationsList { if operation.Status.Phase == itemoperation.OperationPhaseNew || operation.Status.Phase == itemoperation.OperationPhaseInProgress { ria, err := pluginManager.GetRestoreItemActionV2(operation.Spec.RestoreItemAction) if err != nil { operation.Status.Phase = itemoperation.OperationPhaseFailed operation.Status.Error = err.Error() errs = append(errs, err.Error()) changes = true failedCount++ continue } operationProgress, err := ria.Progress(operation.Spec.OperationID, restore) if err != nil { operation.Status.Phase = itemoperation.OperationPhaseFailed operation.Status.Error = err.Error() errs = append(errs, err.Error()) changes = true failedCount++ continue } if operation.Status.NCompleted != operationProgress.NCompleted { operation.Status.NCompleted = operationProgress.NCompleted changes = true } if operation.Status.NTotal != operationProgress.NTotal { operation.Status.NTotal = operationProgress.NTotal changes = true } if operation.Status.OperationUnits != operationProgress.OperationUnits { operation.Status.OperationUnits = operationProgress.OperationUnits changes = true } if operation.Status.Description != operationProgress.Description { operation.Status.Description = operationProgress.Description changes = true } started := metav1.NewTime(operationProgress.Started) if operation.Status.Started == nil && !operationProgress.Started.IsZero() || operation.Status.Started != nil && *(operation.Status.Started) != started { operation.Status.Started = &started changes = true } updated := metav1.NewTime(operationProgress.Updated) if operation.Status.Updated == nil && !operationProgress.Updated.IsZero() || operation.Status.Updated != nil && *(operation.Status.Updated) != updated { operation.Status.Updated = &updated changes = true } if operationProgress.Completed { if operationProgress.Err != "" { operation.Status.Phase = itemoperation.OperationPhaseFailed operation.Status.Error = operationProgress.Err errs = append(errs, operationProgress.Err) changes = true failedCount++ continue } operation.Status.Phase = itemoperation.OperationPhaseCompleted changes = true completedCount++ continue } // cancel operation if past timeout period if operation.Status.Created.Time.Add(restore.Spec.ItemOperationTimeout.Duration).Before(time.Now()) { _ = ria.Cancel(operation.Spec.OperationID, restore) operation.Status.Phase = itemoperation.OperationPhaseFailed operation.Status.Error = "Asynchronous action timed out" errs = append(errs, operation.Status.Error) changes = true failedCount++ continue } if operation.Status.Phase == itemoperation.OperationPhaseNew && operation.Status.Started != nil { operation.Status.Phase = itemoperation.OperationPhaseInProgress changes = true } // if we reach this point, the operation is still running inProgressOperations = true } else if operation.Status.Phase == itemoperation.OperationPhaseCompleted { completedCount++ } else if operation.Status.Phase == itemoperation.OperationPhaseFailed { failedCount++ } } return inProgressOperations, changes, completedCount, failedCount, errs } ================================================ FILE: pkg/controller/restore_operations_controller_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package controller import ( "testing" "time" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" testclocks "k8s.io/utils/clock/testing" ctrl "sigs.k8s.io/controller-runtime" kbclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/itemoperation" "github.com/vmware-tanzu/velero/pkg/itemoperationmap" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/metrics" persistencemocks "github.com/vmware-tanzu/velero/pkg/persistence/mocks" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" pluginmocks "github.com/vmware-tanzu/velero/pkg/plugin/mocks" "github.com/vmware-tanzu/velero/pkg/plugin/velero" riav2mocks "github.com/vmware-tanzu/velero/pkg/plugin/velero/mocks/restoreitemaction/v2" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) var ( restorePluginManager = &pluginmocks.Manager{} restoreBackupStore = &persistencemocks.BackupStore{} ria = &riav2mocks.RestoreItemAction{} ) func mockRestoreOperationsReconciler(fakeClient kbclient.Client, fakeClock *testclocks.FakeClock, freq time.Duration) *restoreOperationsReconciler { abor := NewRestoreOperationsReconciler( logrus.StandardLogger(), velerov1api.DefaultNamespace, fakeClient, freq, func(logrus.FieldLogger) clientmgmt.Manager { return restorePluginManager }, NewFakeSingleObjectBackupStoreGetter(restoreBackupStore), metrics.NewServerMetrics(), itemoperationmap.NewRestoreItemOperationsMap(), ) abor.clock = fakeClock return abor } func TestRestoreOperationsReconcile(t *testing.T) { fakeClock := testclocks.NewFakeClock(time.Now()) metav1Now := metav1.NewTime(fakeClock.Now()) defaultBackupLocation := builder.ForBackupStorageLocation(velerov1api.DefaultNamespace, "default").Result() tests := []struct { name string restore *velerov1api.Restore restoreOperations []*itemoperation.RestoreOperation backup *velerov1api.Backup backupLocation *velerov1api.BackupStorageLocation operationComplete bool operationErr string expectError bool expectPhase velerov1api.RestorePhase }{ { name: "WaitingForPluginOperations restore with completed operations is Completed", restore: builder.ForRestore(velerov1api.DefaultNamespace, "restore-11"). Backup("backup-1"). ItemOperationTimeout(60 * time.Minute). ObjectMeta(builder.WithUID("foo-11")). Phase(velerov1api.RestorePhaseWaitingForPluginOperations).Result(), backup: defaultBackup().StorageLocation("default").Result(), backupLocation: defaultBackupLocation, operationComplete: true, expectPhase: velerov1api.RestorePhaseFinalizing, restoreOperations: []*itemoperation.RestoreOperation{ { Spec: itemoperation.RestoreOperationSpec{ RestoreName: "restore-11", RestoreUID: "foo-11", RestoreItemAction: "foo-11", ResourceIdentifier: velero.ResourceIdentifier{ GroupResource: kuberesource.Pods, Namespace: "ns-1", Name: "pod-1", }, OperationID: "operation-11", }, Status: itemoperation.OperationStatus{ Phase: itemoperation.OperationPhaseInProgress, Created: &metav1Now, }, }, }, }, { name: "WaitingForPluginOperations restore with incomplete operations is still incomplete", restore: builder.ForRestore(velerov1api.DefaultNamespace, "restore-12"). Backup("backup-1"). ItemOperationTimeout(60 * time.Minute). ObjectMeta(builder.WithUID("foo-12")). Phase(velerov1api.RestorePhaseWaitingForPluginOperations).Result(), backup: defaultBackup().StorageLocation("default").Result(), backupLocation: defaultBackupLocation, operationComplete: false, expectPhase: velerov1api.RestorePhaseWaitingForPluginOperations, restoreOperations: []*itemoperation.RestoreOperation{ { Spec: itemoperation.RestoreOperationSpec{ RestoreName: "restore-12", RestoreUID: "foo-12", RestoreItemAction: "foo-12", ResourceIdentifier: velero.ResourceIdentifier{ GroupResource: kuberesource.Pods, Namespace: "ns-1", Name: "pod-1", }, OperationID: "operation-12", }, Status: itemoperation.OperationStatus{ Phase: itemoperation.OperationPhaseInProgress, Created: &metav1Now, }, }, }, }, { name: "WaitingForPluginOperations restore with completed failed operations is PartiallyFailed", restore: builder.ForRestore(velerov1api.DefaultNamespace, "restore-13"). Backup("backup-1"). ItemOperationTimeout(60 * time.Minute). ObjectMeta(builder.WithUID("foo-13")). Phase(velerov1api.RestorePhaseWaitingForPluginOperations).Result(), backup: defaultBackup().StorageLocation("default").Result(), backupLocation: defaultBackupLocation, operationComplete: true, operationErr: "failed", expectPhase: velerov1api.RestorePhaseFinalizingPartiallyFailed, restoreOperations: []*itemoperation.RestoreOperation{ { Spec: itemoperation.RestoreOperationSpec{ RestoreName: "restore-13", RestoreUID: "foo-13", RestoreItemAction: "foo-13", ResourceIdentifier: velero.ResourceIdentifier{ GroupResource: kuberesource.Pods, Namespace: "ns-1", Name: "pod-1", }, OperationID: "operation-13", }, Status: itemoperation.OperationStatus{ Phase: itemoperation.OperationPhaseInProgress, Created: &metav1Now, }, }, }, }, { name: "WaitingForPluginOperationsPartiallyFailed restore with completed operations is PartiallyFailed", restore: builder.ForRestore(velerov1api.DefaultNamespace, "restore-14"). Backup("backup-1"). ItemOperationTimeout(60 * time.Minute). ObjectMeta(builder.WithUID("foo-14")). Phase(velerov1api.RestorePhaseWaitingForPluginOperationsPartiallyFailed).Result(), backup: defaultBackup().StorageLocation("default").Result(), backupLocation: defaultBackupLocation, operationComplete: true, expectPhase: velerov1api.RestorePhaseFinalizingPartiallyFailed, restoreOperations: []*itemoperation.RestoreOperation{ { Spec: itemoperation.RestoreOperationSpec{ RestoreName: "restore-14", RestoreUID: "foo-14", RestoreItemAction: "foo-14", ResourceIdentifier: velero.ResourceIdentifier{ GroupResource: kuberesource.Pods, Namespace: "ns-1", Name: "pod-1", }, OperationID: "operation-14", }, Status: itemoperation.OperationStatus{ Phase: itemoperation.OperationPhaseInProgress, Created: &metav1Now, }, }, }, }, { name: "WaitingForPluginOperationsPartiallyFailed restore with incomplete operations is still incomplete", restore: builder.ForRestore(velerov1api.DefaultNamespace, "restore-15"). Backup("backup-1"). ItemOperationTimeout(60 * time.Minute). ObjectMeta(builder.WithUID("foo-15")). Phase(velerov1api.RestorePhaseWaitingForPluginOperationsPartiallyFailed).Result(), backup: defaultBackup().StorageLocation("default").Result(), backupLocation: defaultBackupLocation, operationComplete: false, expectPhase: velerov1api.RestorePhaseWaitingForPluginOperationsPartiallyFailed, restoreOperations: []*itemoperation.RestoreOperation{ { Spec: itemoperation.RestoreOperationSpec{ RestoreName: "restore-15", RestoreUID: "foo-15", RestoreItemAction: "foo-15", ResourceIdentifier: velero.ResourceIdentifier{ GroupResource: kuberesource.Pods, Namespace: "ns-1", Name: "pod-1", }, OperationID: "operation-15", }, Status: itemoperation.OperationStatus{ Phase: itemoperation.OperationPhaseInProgress, Created: &metav1Now, }, }, }, }, { name: "WaitingForPluginOperationsPartiallyFailed restore with completed failed operations is PartiallyFailed", restore: builder.ForRestore(velerov1api.DefaultNamespace, "restore-16"). Backup("backup-1"). ItemOperationTimeout(60 * time.Minute). ObjectMeta(builder.WithUID("foo-16")). Phase(velerov1api.RestorePhaseWaitingForPluginOperationsPartiallyFailed).Result(), backup: defaultBackup().StorageLocation("default").Result(), backupLocation: defaultBackupLocation, operationComplete: true, operationErr: "failed", expectPhase: velerov1api.RestorePhaseFinalizingPartiallyFailed, restoreOperations: []*itemoperation.RestoreOperation{ { Spec: itemoperation.RestoreOperationSpec{ RestoreName: "restore-16", RestoreUID: "foo-16", RestoreItemAction: "foo-16", ResourceIdentifier: velero.ResourceIdentifier{ GroupResource: kuberesource.Pods, Namespace: "ns-1", Name: "pod-1", }, OperationID: "operation-16", }, Status: itemoperation.OperationStatus{ Phase: itemoperation.OperationPhaseInProgress, Created: &metav1Now, }, }, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { if test.restore == nil { return } initObjs := []runtime.Object{} initObjs = append(initObjs, test.restore) initObjs = append(initObjs, test.backup) if test.backupLocation != nil { initObjs = append(initObjs, test.backupLocation) } fakeClient := velerotest.NewFakeControllerRuntimeClient(t, initObjs...) reconciler := mockRestoreOperationsReconciler(fakeClient, fakeClock, defaultRestoreOperationsFrequency) restorePluginManager.On("CleanupClients").Return(nil) restoreBackupStore.On("GetRestoreItemOperations", test.restore.Name).Return(test.restoreOperations, nil) restoreBackupStore.On("PutRestoreItemOperations", mock.Anything, mock.Anything).Return(nil) restoreBackupStore.On("PutRestoreMetadata", mock.Anything, mock.Anything).Return(nil) for _, operation := range test.restoreOperations { ria.On("Progress", operation.Spec.OperationID, mock.Anything). Return(velero.OperationProgress{ Completed: test.operationComplete, Err: test.operationErr, }, nil) restorePluginManager.On("GetRestoreItemActionV2", operation.Spec.RestoreItemAction).Return(ria, nil) } _, err := reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: test.restore.Namespace, Name: test.restore.Name}}) gotErr := err != nil assert.Equal(t, test.expectError, gotErr) restoreAfter := velerov1api.Restore{} err = fakeClient.Get(t.Context(), types.NamespacedName{ Namespace: test.restore.Namespace, Name: test.restore.Name, }, &restoreAfter) require.NoError(t, err) assert.Equal(t, test.expectPhase, restoreAfter.Status.Phase) }) } } ================================================ FILE: pkg/controller/schedule_controller.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package controller import ( "context" "fmt" "time" "github.com/pkg/errors" cron "github.com/robfig/cron/v3" "github.com/sirupsen/logrus" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" clocks "k8s.io/utils/clock" ctrl "sigs.k8s.io/controller-runtime" bld "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/predicate" velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/constant" "github.com/vmware-tanzu/velero/pkg/metrics" "github.com/vmware-tanzu/velero/pkg/util/kube" ) const ( scheduleSyncPeriod = time.Minute ) type scheduleReconciler struct { client.Client namespace string logger logrus.FieldLogger clock clocks.WithTickerAndDelayedExecution metrics *metrics.ServerMetrics skipImmediately bool } func NewScheduleReconciler( namespace string, logger logrus.FieldLogger, client client.Client, metrics *metrics.ServerMetrics, skipImmediately bool, ) *scheduleReconciler { return &scheduleReconciler{ Client: client, namespace: namespace, logger: logger, clock: clocks.RealClock{}, metrics: metrics, skipImmediately: skipImmediately, } } func (c *scheduleReconciler) SetupWithManager(mgr ctrl.Manager) error { pred := kube.NewAllEventPredicate(func(obj client.Object) bool { schedule := obj.(*velerov1.Schedule) if pause := schedule.Spec.Paused; pause { c.logger.Infof("schedule %s is paused, skip", schedule.Name) return false } return true }) s := kube.NewPeriodicalEnqueueSource(c.logger.WithField("controller", constant.ControllerSchedule), mgr.GetClient(), &velerov1.ScheduleList{}, scheduleSyncPeriod, kube.PeriodicalEnqueueSourceOption{ Predicates: []predicate.Predicate{pred}, }) return ctrl.NewControllerManagedBy(mgr). For(&velerov1.Schedule{}, bld.WithPredicates(kube.SpecChangePredicate{}, pred)). WatchesRawSource(s). Complete(c) } // +kubebuilder:rbac:groups=velero.io,resources=schedules,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=velero.io,resources=schedules/status,verbs=get;update;patch // +kubebuilder:rbac:groups=velero.io,resources=backups,verbs=create func (c *scheduleReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := c.logger.WithField("schedule", req.String()) log.Debug("Getting schedule") schedule := &velerov1.Schedule{} if err := c.Get(ctx, req.NamespacedName, schedule); err != nil { if apierrors.IsNotFound(err) { log.WithError(err).Error("schedule not found") c.metrics.RemoveSchedule(req.Name) return ctrl.Result{}, nil } return ctrl.Result{}, errors.Wrapf(err, "error getting schedule %s", req.String()) } c.metrics.InitSchedule(schedule.Name) original := schedule.DeepCopy() if schedule.Spec.SkipImmediately == nil { schedule.Spec.SkipImmediately = &c.skipImmediately } if schedule.Spec.SkipImmediately != nil && *schedule.Spec.SkipImmediately { *schedule.Spec.SkipImmediately = false schedule.Status.LastSkipped = &metav1.Time{Time: c.clock.Now()} } // validation - even if the item is Enabled, we can't trust it // so re-validate currentPhase := schedule.Status.Phase cronSchedule, errs := parseCronSchedule(schedule, c.logger) if len(errs) > 0 { schedule.Status.Phase = velerov1.SchedulePhaseFailedValidation schedule.Status.ValidationErrors = errs } else { schedule.Status.Phase = velerov1.SchedulePhaseEnabled schedule.Status.ValidationErrors = nil // Compute expected interval between consecutive scheduled backup runs. // Only meaningful when the cron expression is valid. now := c.clock.Now() nextRun := cronSchedule.Next(now) nextNextRun := cronSchedule.Next(nextRun) c.metrics.SetScheduleExpectedIntervalSeconds(schedule.Name, nextNextRun.Sub(nextRun).Seconds()) } scheduleNeedsPatch := false errStringArr := make([]string, 0) if currentPhase != schedule.Status.Phase { scheduleNeedsPatch = true errStringArr = append(errStringArr, fmt.Sprintf("phase to %s", schedule.Status.Phase)) } // update spec.SkipImmediately if it's changed if original.Spec.SkipImmediately != schedule.Spec.SkipImmediately { scheduleNeedsPatch = true errStringArr = append(errStringArr, fmt.Sprintf("spec.skipImmediately to %v", schedule.Spec.SkipImmediately)) } // update status if it's changed if original.Status.LastSkipped != schedule.Status.LastSkipped { scheduleNeedsPatch = true errStringArr = append(errStringArr, fmt.Sprintf("last skipped to %v", schedule.Status.LastSkipped)) } if scheduleNeedsPatch { if err := c.Patch(ctx, schedule, client.MergeFrom(original)); err != nil { return ctrl.Result{}, errors.Wrapf(err, "error updating %v for schedule %s", errStringArr, req.String()) } } if schedule.Status.Phase != velerov1.SchedulePhaseEnabled { log.Debugf("the schedule's phase is %s, isn't %s, skip", schedule.Status.Phase, velerov1.SchedulePhaseEnabled) return ctrl.Result{}, nil } // Check for the schedule being due to run. // If there are backup created by this schedule still in New or InProgress state, // skip current backup creation to avoid running overlap backups. // As the schedule must be validated before checking whether it's due, we cannot put the checking log in Predicate if c.ifDue(schedule, cronSchedule) && !c.checkIfBackupInNewOrProgress(schedule) { if err := c.submitBackup(ctx, schedule); err != nil { return ctrl.Result{}, errors.Wrapf(err, "error submit backup for schedule %s", req.String()) } } return ctrl.Result{}, nil } func parseCronSchedule(itm *velerov1.Schedule, logger logrus.FieldLogger) (cron.Schedule, []string) { var validationErrors []string var schedule cron.Schedule // cron.Parse panics if schedule is empty if len(itm.Spec.Schedule) == 0 { validationErrors = append(validationErrors, "Schedule must be a non-empty valid Cron expression") return nil, validationErrors } log := logger.WithField("schedule", kube.NamespaceAndName(itm)) // adding a recover() around cron.Parse because it panics on empty string and is possible // that it panics under other scenarios as well. func() { defer func() { if r := recover(); r != nil { log.WithFields(logrus.Fields{ "schedule": itm.Spec.Schedule, "recover": r, }).Debug("Panic parsing schedule") validationErrors = append(validationErrors, fmt.Sprintf("invalid schedule: %v", r)) } }() if res, err := cron.ParseStandard(itm.Spec.Schedule); err != nil { log.WithError(errors.WithStack(err)).WithField("schedule", itm.Spec.Schedule).Debug("Error parsing schedule") validationErrors = append(validationErrors, fmt.Sprintf("invalid schedule: %v", err)) } else { schedule = res } }() if len(validationErrors) > 0 { return nil, validationErrors } return schedule, nil } // checkIfBackupInNewOrProgress check whether there are backups created by this schedule still in New or InProgress state func (c *scheduleReconciler) checkIfBackupInNewOrProgress(schedule *velerov1.Schedule) bool { log := c.logger.WithField("schedule", kube.NamespaceAndName(schedule)) backupList := &velerov1.BackupList{} options := &client.ListOptions{ Namespace: schedule.Namespace, LabelSelector: labels.Set(map[string]string{ velerov1.ScheduleNameLabel: schedule.Name, }).AsSelector(), } err := c.List(context.Background(), backupList, options) if err != nil { log.Errorf("fail to list backup for schedule %s/%s: %s", schedule.Namespace, schedule.Name, err.Error()) return true } for _, backup := range backupList.Items { if backup.Status.Phase == "" || backup.Status.Phase == velerov1.BackupPhaseNew || backup.Status.Phase == velerov1.BackupPhaseInProgress { log.Debugf("%s/%s still has backups that are in InProgress or New...", schedule.Namespace, schedule.Name) return true } } return false } // ifDue check whether schedule is due to create a new backup. func (c *scheduleReconciler) ifDue(schedule *velerov1.Schedule, cronSchedule cron.Schedule) bool { isDue, nextRunTime := getNextRunTime(schedule, cronSchedule, c.clock.Now()) log := c.logger.WithField("schedule", kube.NamespaceAndName(schedule)) if !isDue { log.WithField("nextRunTime", nextRunTime).Debug("Schedule is not due, skipping") return false } return true } // submitBackup create a backup from schedule. func (c *scheduleReconciler) submitBackup(ctx context.Context, schedule *velerov1.Schedule) error { c.logger.WithField("schedule", schedule.Namespace+"/"+schedule.Name).Info("Schedule is due, going to submit backup.") now := c.clock.Now() // Don't attempt to "catch up" if there are any missed or failed runs - simply // trigger a Backup if it's time. backup := getBackup(schedule, now) if err := c.Create(ctx, backup); err != nil { return errors.Wrap(err, "error creating Backup") } original := schedule.DeepCopy() schedule.Status.LastBackup = &metav1.Time{Time: now} if err := c.Patch(ctx, schedule, client.MergeFrom(original)); err != nil { return errors.Wrapf(err, "error updating Schedule's LastBackup time to %v", schedule.Status.LastBackup) } return nil } func getNextRunTime(schedule *velerov1.Schedule, cronSchedule cron.Schedule, asOf time.Time) (bool, time.Time) { var lastBackupTime time.Time if schedule.Status.LastBackup != nil { lastBackupTime = schedule.Status.LastBackup.Time } else { lastBackupTime = schedule.CreationTimestamp.Time } if schedule.Status.LastSkipped != nil && schedule.Status.LastSkipped.After(lastBackupTime) { lastBackupTime = schedule.Status.LastSkipped.Time } nextRunTime := cronSchedule.Next(lastBackupTime) return asOf.After(nextRunTime), nextRunTime } func getBackup(item *velerov1.Schedule, timestamp time.Time) *velerov1.Backup { name := item.TimestampedName(timestamp) return builder. ForBackup(item.Namespace, name). FromSchedule(item). Result() } ================================================ FILE: pkg/controller/schedule_controller_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package controller import ( "testing" "time" cron "github.com/robfig/cron/v3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" testclocks "k8s.io/utils/clock/testing" "k8s.io/utils/pointer" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client/fake" velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/metrics" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) // Test reconcile function of schedule controller. Pause is not covered as event filter will not allow it through func TestReconcileOfSchedule(t *testing.T) { require.NoError(t, velerov1.AddToScheme(scheme.Scheme)) newScheduleBuilder := func(phase velerov1.SchedulePhase) *builder.ScheduleBuilder { return builder.ForSchedule("ns", "name").Phase(phase) } tests := []struct { name string scheduleKey string schedule *velerov1.Schedule fakeClockTime string expectedPhase string expectedValidationErrors []string expectedBackupCreate *velerov1.Backup expectedLastBackup string expectedLastSkipped string backup *velerov1.Backup reconcilerSkipImmediately bool }{ { name: "missing schedule triggers no backup", scheduleKey: "foo/bar", }, { name: "schedule with phase FailedValidation triggers no backup", schedule: newScheduleBuilder(velerov1.SchedulePhaseFailedValidation).Result(), }, { name: "schedule with phase New gets validated and failed if invalid", schedule: newScheduleBuilder(velerov1.SchedulePhaseNew).Result(), expectedPhase: string(velerov1.SchedulePhaseFailedValidation), expectedValidationErrors: []string{"Schedule must be a non-empty valid Cron expression"}, }, { name: "schedule with phase gets validated and failed if invalid", schedule: newScheduleBuilder(velerov1.SchedulePhase("")).Result(), expectedPhase: string(velerov1.SchedulePhaseFailedValidation), expectedValidationErrors: []string{"Schedule must be a non-empty valid Cron expression"}, }, { name: "schedule with phase Enabled gets re-validated and failed if invalid", schedule: newScheduleBuilder(velerov1.SchedulePhaseEnabled).Result(), expectedPhase: string(velerov1.SchedulePhaseFailedValidation), expectedValidationErrors: []string{"Schedule must be a non-empty valid Cron expression"}, }, { name: "schedule with phase New gets validated and triggers a backup", schedule: newScheduleBuilder(velerov1.SchedulePhaseNew).CronSchedule("@every 5m").Result(), fakeClockTime: "2017-01-01 12:00:00", expectedPhase: string(velerov1.SchedulePhaseEnabled), expectedBackupCreate: builder.ForBackup("ns", "name-20170101120000").ObjectMeta(builder.WithLabels(velerov1.ScheduleNameLabel, "name")).Result(), expectedLastBackup: "2017-01-01 12:00:00", }, { name: "schedule with phase New and SkipImmediately gets validated and does not trigger a backup", schedule: newScheduleBuilder(velerov1.SchedulePhaseNew).CronSchedule("@every 5m").SkipImmediately(pointer.Bool(true)).Result(), fakeClockTime: "2017-01-01 12:00:00", expectedPhase: string(velerov1.SchedulePhaseEnabled), expectedLastSkipped: "2017-01-01 12:00:00", }, { name: "schedule with phase Enabled gets re-validated and triggers a backup if valid", schedule: newScheduleBuilder(velerov1.SchedulePhaseEnabled).CronSchedule("@every 5m").Result(), fakeClockTime: "2017-01-01 12:00:00", expectedPhase: string(velerov1.SchedulePhaseEnabled), expectedBackupCreate: builder.ForBackup("ns", "name-20170101120000").ObjectMeta(builder.WithLabels(velerov1.ScheduleNameLabel, "name")).Result(), expectedLastBackup: "2017-01-01 12:00:00", }, { name: "schedule that's already run gets LastBackup updated", schedule: newScheduleBuilder(velerov1.SchedulePhaseEnabled).CronSchedule("@every 5m").LastBackupTime("2000-01-01 00:00:00").Result(), fakeClockTime: "2017-01-01 12:00:00", expectedBackupCreate: builder.ForBackup("ns", "name-20170101120000").ObjectMeta(builder.WithLabels(velerov1.ScheduleNameLabel, "name")).Result(), expectedLastBackup: "2017-01-01 12:00:00", }, { name: "schedule that's already run but has SkippedImmediately=nil gets LastBackup updated", schedule: newScheduleBuilder(velerov1.SchedulePhaseEnabled).CronSchedule("@every 5m").LastBackupTime("2000-01-01 00:00:00").SkipImmediately(nil).Result(), fakeClockTime: "2017-01-01 12:00:00", expectedBackupCreate: builder.ForBackup("ns", "name-20170101120000").ObjectMeta(builder.WithLabels(velerov1.ScheduleNameLabel, "name")).Result(), expectedLastBackup: "2017-01-01 12:00:00", }, { name: "schedule that's already run but has SkippedImmediately=false gets LastBackup updated", schedule: newScheduleBuilder(velerov1.SchedulePhaseEnabled).CronSchedule("@every 5m").LastBackupTime("2000-01-01 00:00:00").SkipImmediately(pointer.Bool(false)).Result(), fakeClockTime: "2017-01-01 12:00:00", expectedBackupCreate: builder.ForBackup("ns", "name-20170101120000").ObjectMeta(builder.WithLabels(velerov1.ScheduleNameLabel, "name")).Result(), expectedLastBackup: "2017-01-01 12:00:00", }, { name: "schedule that's already run, server has skipImmediately set to true, and Schedule has SkippedImmediately=nil do not get LastBackup updated", schedule: newScheduleBuilder(velerov1.SchedulePhaseEnabled).CronSchedule("@every 5m").LastBackupTime("2000-01-01 00:00:00").SkipImmediately(nil).Result(), fakeClockTime: "2017-01-01 12:00:00", expectedLastBackup: "2000-01-01 00:00:00", expectedLastSkipped: "2017-01-01 12:00:00", reconcilerSkipImmediately: true, }, { name: "schedule that's already run but has SkippedImmediately=true do not get LastBackup updated", schedule: newScheduleBuilder(velerov1.SchedulePhaseEnabled).CronSchedule("@every 5m").LastBackupTime("2000-01-01 00:00:00").SkipImmediately(pointer.Bool(true)).Result(), fakeClockTime: "2017-01-01 12:00:00", expectedLastBackup: "2000-01-01 00:00:00", expectedLastSkipped: "2017-01-01 12:00:00", }, { name: "schedule already has backup in New state.", schedule: newScheduleBuilder(velerov1.SchedulePhaseEnabled).CronSchedule("@every 5m").LastBackupTime("2000-01-01 00:00:00").Result(), expectedPhase: string(velerov1.SchedulePhaseEnabled), backup: builder.ForBackup("ns", "name-20220905120000").ObjectMeta(builder.WithLabels(velerov1.ScheduleNameLabel, "name")).Phase(velerov1.BackupPhaseNew).Result(), }, { name: "schedule already has backup with empty phase (not yet reconciled).", schedule: newScheduleBuilder(velerov1.SchedulePhaseEnabled).CronSchedule("@every 5m").LastBackupTime("2000-01-01 00:00:00").Result(), fakeClockTime: "2017-01-01 12:00:00", expectedPhase: string(velerov1.SchedulePhaseEnabled), backup: builder.ForBackup("ns", "name-20220905120000").ObjectMeta(builder.WithLabels(velerov1.ScheduleNameLabel, "name")).Phase("").Result(), }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { var ( client = (&fake.ClientBuilder{}).Build() logger = velerotest.NewLogger() testTime time.Time err error ) reconciler := NewScheduleReconciler("namespace", logger, client, metrics.NewServerMetrics(), test.reconcilerSkipImmediately) if test.fakeClockTime != "" { testTime, err = time.Parse("2006-01-02 15:04:05", test.fakeClockTime) require.NoError(t, err, "unable to parse test.fakeClockTime: %v", err) } reconciler.clock = testclocks.NewFakeClock(testTime) if test.schedule != nil { require.NoError(t, client.Create(ctx, test.schedule)) } if test.backup != nil { require.NoError(t, client.Create(ctx, test.backup)) } scheduleb4reconcile := &velerov1.Schedule{} err = client.Get(ctx, types.NamespacedName{Namespace: "ns", Name: "name"}, scheduleb4reconcile) if test.schedule != nil { require.NoError(t, err) } _, err = reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "ns", Name: "name"}}) require.NoError(t, err) schedule := &velerov1.Schedule{} err = client.Get(ctx, types.NamespacedName{Namespace: "ns", Name: "name"}, schedule) if len(test.expectedPhase) > 0 { require.NoError(t, err) assert.Equal(t, test.expectedPhase, string(schedule.Status.Phase)) } if len(test.expectedValidationErrors) > 0 { require.NoError(t, err) assert.Equal(t, test.expectedValidationErrors, schedule.Status.ValidationErrors) } if len(test.expectedLastBackup) > 0 { require.NoError(t, err) require.NotNil(t, schedule.Status.LastBackup) assert.Equal(t, parseTime(test.expectedLastBackup).Unix(), schedule.Status.LastBackup.Unix()) } if len(test.expectedLastSkipped) > 0 { require.NoError(t, err) require.NotNil(t, schedule.Status.LastSkipped) assert.Equal(t, parseTime(test.expectedLastSkipped).Unix(), schedule.Status.LastSkipped.Unix()) } // we expect reconcile to flip SkipImmediately to false if it's true or the server is configured to skip immediately and the schedule doesn't have it set if scheduleb4reconcile.Spec.SkipImmediately != nil && *scheduleb4reconcile.Spec.SkipImmediately || test.reconcilerSkipImmediately && scheduleb4reconcile.Spec.SkipImmediately == nil { assert.Equal(t, schedule.Spec.SkipImmediately, pointer.Bool(false)) } backups := &velerov1.BackupList{} require.NoError(t, client.List(ctx, backups)) // If backup associated with schedule's status is in New or InProgress or empty phase, // new backup shouldn't be submitted. if test.backup != nil && (test.backup.Status.Phase == "" || test.backup.Status.Phase == velerov1.BackupPhaseNew || test.backup.Status.Phase == velerov1.BackupPhaseInProgress) { assert.Len(t, backups.Items, 1) require.NoError(t, client.Delete(ctx, test.backup)) } require.NoError(t, client.List(ctx, backups)) if test.expectedBackupCreate == nil { assert.Empty(t, backups.Items) } else { assert.Len(t, backups.Items, 1) } }) } } func parseTime(timeString string) time.Time { res, _ := time.Parse("2006-01-02 15:04:05", timeString) return res } func TestGetNextRunTime(t *testing.T) { defaultSchedule := func() *velerov1.Schedule { return builder.ForSchedule("velero", "schedule-1").CronSchedule("@every 5m").Result() } tests := []struct { name string schedule *velerov1.Schedule lastRanOffset string expectedDue bool expectedNextRunTimeOffset string }{ { name: "first run", schedule: defaultSchedule(), expectedDue: false, expectedNextRunTimeOffset: "5m", }, { name: "just ran", schedule: defaultSchedule(), lastRanOffset: "0s", expectedDue: false, expectedNextRunTimeOffset: "5m", }, { name: "almost but not quite time to run", schedule: defaultSchedule(), lastRanOffset: "4m59s", expectedDue: false, expectedNextRunTimeOffset: "5m", }, { name: "time to run again", schedule: defaultSchedule(), lastRanOffset: "5m", expectedDue: true, expectedNextRunTimeOffset: "5m", }, { name: "several runs missed", schedule: defaultSchedule(), lastRanOffset: "5h", expectedDue: true, expectedNextRunTimeOffset: "5m", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { cronSchedule, err := cron.ParseStandard(test.schedule.Spec.Schedule) require.NoError(t, err, "unable to parse test.schedule.Spec.Schedule: %v", err) testClock := testclocks.NewFakeClock(time.Now()) if test.lastRanOffset != "" { offsetDuration, err := time.ParseDuration(test.lastRanOffset) require.NoError(t, err, "unable to parse test.lastRanOffset: %v", err) test.schedule.Status.LastBackup = &metav1.Time{Time: testClock.Now().Add(-offsetDuration)} test.schedule.CreationTimestamp = *test.schedule.Status.LastBackup } else { test.schedule.CreationTimestamp = metav1.Time{Time: testClock.Now()} } nextRunTimeOffset, err := time.ParseDuration(test.expectedNextRunTimeOffset) if err != nil { panic(err) } var baseTime time.Time if test.lastRanOffset != "" { baseTime = test.schedule.Status.LastBackup.Time } else { baseTime = test.schedule.CreationTimestamp.Time } expectedNextRunTime := baseTime.Add(nextRunTimeOffset) due, nextRunTime := getNextRunTime(test.schedule, cronSchedule, testClock.Now()) assert.Equal(t, test.expectedDue, due) // ignore diffs of under a second. the cron library does some rounding. assert.WithinDuration(t, expectedNextRunTime, nextRunTime, time.Second) }) } } func TestParseCronSchedule(t *testing.T) { // From https://github.com/vmware-tanzu/velero/issues/30, where we originally were using cron.Parse(), // which treats the first field as seconds, and not minutes. We want to use cron.ParseStandard() // instead, which has the first field as minutes. now := time.Date(2017, 8, 10, 12, 27, 0, 0, time.UTC) // Start with a Schedule with: // - schedule: once a day at 9am // - last backup: 2017-08-10 12:27:00 (just happened) s := builder.ForSchedule("velero", "schedule-1").CronSchedule("0 9 * * *").LastBackupTime(now.Format("2006-01-02 15:04:05")).Result() logger := velerotest.NewLogger() c, errs := parseCronSchedule(s, logger) require.Empty(t, errs) // make sure we're not due and next backup is tomorrow at 9am due, next := getNextRunTime(s, c, now) assert.False(t, due) assert.Equal(t, time.Date(2017, 8, 11, 9, 0, 0, 0, time.UTC), next) // advance the clock a couple of hours and make sure nothing has changed now = now.Add(2 * time.Hour) due, next = getNextRunTime(s, c, now) assert.False(t, due) assert.Equal(t, time.Date(2017, 8, 11, 9, 0, 0, 0, time.UTC), next) // advance clock to 1 minute after due time, make sure due=true now = time.Date(2017, 8, 11, 9, 1, 0, 0, time.UTC) due, next = getNextRunTime(s, c, now) assert.True(t, due) assert.Equal(t, time.Date(2017, 8, 11, 9, 0, 0, 0, time.UTC), next) // record backup time s.Status.LastBackup = &metav1.Time{Time: now} // advance clock 1 minute, make sure we're not due and next backup is tomorrow at 9am now = time.Date(2017, 8, 11, 9, 2, 0, 0, time.UTC) due, next = getNextRunTime(s, c, now) assert.False(t, due) assert.Equal(t, time.Date(2017, 8, 12, 9, 0, 0, 0, time.UTC), next) } func TestGetBackup(t *testing.T) { tests := []struct { name string schedule *velerov1.Schedule testClockTime string expectedBackup *velerov1.Backup }{ { name: "ensure name is formatted correctly (AM time)", schedule: builder.ForSchedule("foo", "bar").Result(), testClockTime: "2017-07-25 09:15:00", expectedBackup: builder.ForBackup("foo", "bar-20170725091500").ObjectMeta(builder.WithLabels(velerov1.ScheduleNameLabel, "bar")).Result(), }, { name: "ensure name is formatted correctly (PM time)", schedule: builder.ForSchedule("foo", "bar").Result(), testClockTime: "2017-07-25 14:15:00", expectedBackup: builder.ForBackup("foo", "bar-20170725141500").ObjectMeta(builder.WithLabels(velerov1.ScheduleNameLabel, "bar")).Result(), }, { name: "ensure schedule backup template is copied", schedule: builder.ForSchedule("foo", "bar"). Template(builder.ForBackup("", ""). IncludedNamespaces("ns-1", "ns-2"). ExcludedNamespaces("ns-3"). IncludedResources("foo", "bar"). ExcludedResources("baz"). LabelSelector(&metav1.LabelSelector{MatchLabels: map[string]string{"label": "value"}}). TTL(time.Duration(300)). Result(). Spec). Result(), testClockTime: "2017-07-25 09:15:00", expectedBackup: builder.ForBackup("foo", "bar-20170725091500"). ObjectMeta(builder.WithLabels(velerov1.ScheduleNameLabel, "bar")). IncludedNamespaces("ns-1", "ns-2"). ExcludedNamespaces("ns-3"). IncludedResources("foo", "bar"). ExcludedResources("baz"). LabelSelector(&metav1.LabelSelector{MatchLabels: map[string]string{"label": "value"}}). TTL(time.Duration(300)). Result(), }, { name: "ensure schedule labels are copied", schedule: builder.ForSchedule("foo", "bar").ObjectMeta(builder.WithLabels("foo", "bar", "bar", "baz")).Result(), testClockTime: "2017-07-25 14:15:00", expectedBackup: builder.ForBackup("foo", "bar-20170725141500").ObjectMeta(builder.WithLabels(velerov1.ScheduleNameLabel, "bar", "bar", "baz", "foo", "bar")).Result(), }, { name: "ensure schedule annotations are copied", schedule: builder.ForSchedule("foo", "bar").ObjectMeta(builder.WithAnnotations("foo", "bar", "bar", "baz")).Result(), testClockTime: "2017-07-25 14:15:00", expectedBackup: builder.ForBackup("foo", "bar-20170725141500").ObjectMeta(builder.WithLabels(velerov1.ScheduleNameLabel, "bar"), builder.WithAnnotations("bar", "baz", "foo", "bar")).Result(), }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { testTime, err := time.Parse("2006-01-02 15:04:05", test.testClockTime) require.NoError(t, err, "unable to parse test.testClockTime: %v", err) backup := getBackup(test.schedule, testclocks.NewFakeClock(testTime).Now()) assert.Equal(t, test.expectedBackup.Namespace, backup.Namespace) assert.Equal(t, test.expectedBackup.Name, backup.Name) assert.Equal(t, test.expectedBackup.Labels, backup.Labels) assert.Equal(t, test.expectedBackup.Annotations, backup.Annotations) assert.Equal(t, test.expectedBackup.Spec, backup.Spec) }) } } func TestCheckIfBackupInNewOrProgress(t *testing.T) { require.NoError(t, velerov1.AddToScheme(scheme.Scheme)) client := fake.NewClientBuilder().WithScheme(scheme.Scheme).Build() logger := velerotest.NewLogger() // Create testing schedule testSchedule := builder.ForSchedule("ns", "name").Phase(velerov1.SchedulePhaseEnabled).Result() err := client.Create(ctx, testSchedule) require.NoError(t, err, "fail to create schedule in TestCheckIfBackupInNewOrProgress: %v", err) // Create backup in New phase. newBackup := builder.ForBackup("ns", "backup-1"). ObjectMeta(builder.WithLabels(velerov1.ScheduleNameLabel, "name")). Phase(velerov1.BackupPhaseNew).Result() err = client.Create(ctx, newBackup) require.NoError(t, err, "fail to create backup in New phase in TestCheckIfBackupInNewOrProgress: %v", err) reconciler := NewScheduleReconciler("ns", logger, client, metrics.NewServerMetrics(), false) result := reconciler.checkIfBackupInNewOrProgress(testSchedule) assert.True(t, result) // Clean backup in New phase. err = client.Delete(ctx, newBackup) require.NoError(t, err, "fail to delete backup in New phase in TestCheckIfBackupInNewOrProgress: %v", err) // Create backup in InProgress phase. inProgressBackup := builder.ForBackup("ns", "backup-2"). ObjectMeta(builder.WithLabels(velerov1.ScheduleNameLabel, "name")). Phase(velerov1.BackupPhaseInProgress).Result() err = client.Create(ctx, inProgressBackup) require.NoError(t, err, "fail to create backup in InProgress phase in TestCheckIfBackupInNewOrProgress: %v", err) reconciler = NewScheduleReconciler("namespace", logger, client, metrics.NewServerMetrics(), false) result = reconciler.checkIfBackupInNewOrProgress(testSchedule) assert.True(t, result) // Clean backup in InProgress phase. err = client.Delete(ctx, inProgressBackup) require.NoError(t, err, "fail to delete backup in InProgress phase in TestCheckIfBackupInNewOrProgress: %v", err) // Create backup with empty phase (not yet reconciled). emptyPhaseBackup := builder.ForBackup("ns", "backup-3"). ObjectMeta(builder.WithLabels(velerov1.ScheduleNameLabel, "name")). Phase("").Result() err = client.Create(ctx, emptyPhaseBackup) require.NoError(t, err, "fail to create backup with empty phase in TestCheckIfBackupInNewOrProgress: %v", err) reconciler = NewScheduleReconciler("namespace", logger, client, metrics.NewServerMetrics(), false) result = reconciler.checkIfBackupInNewOrProgress(testSchedule) assert.True(t, result) } ================================================ FILE: pkg/controller/server_status_request_controller.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package controller import ( "context" "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clocks "k8s.io/utils/clock" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "github.com/vmware-tanzu/velero/internal/velero" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/buildinfo" "github.com/vmware-tanzu/velero/pkg/constant" "github.com/vmware-tanzu/velero/pkg/plugin/framework" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" ) const ( ttl = time.Minute statusRequestResyncPeriod = 5 * time.Minute ) type PluginLister interface { // List returns all PluginIdentifiers for kind. List(kind common.PluginKind) []framework.PluginIdentifier } // serverStatusRequestReconciler reconciles a ServerStatusRequest object type serverStatusRequestReconciler struct { client client.Client ctx context.Context pluginRegistry PluginLister clock clocks.WithTickerAndDelayedExecution log logrus.FieldLogger } // NewServerStatusRequestReconciler initializes and returns serverStatusRequestReconciler struct. func NewServerStatusRequestReconciler( ctx context.Context, client client.Client, pluginRegistry PluginLister, clock clocks.WithTickerAndDelayedExecution, log logrus.FieldLogger) *serverStatusRequestReconciler { return &serverStatusRequestReconciler{ client: client, ctx: ctx, pluginRegistry: pluginRegistry, clock: clock, log: log, } } // +kubebuilder:rbac:groups=velero.io,resources=serverstatusrequests,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=velero.io,resources=serverstatusrequests/status,verbs=get;update;patch func (r *serverStatusRequestReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := r.log.WithFields(logrus.Fields{ "controller": constant.ControllerServerStatusRequest, "serverStatusRequest": req.NamespacedName, }) // Fetch the ServerStatusRequest instance. log.Debug("Getting ServerStatusRequest") statusRequest := &velerov1api.ServerStatusRequest{} if err := r.client.Get(r.ctx, req.NamespacedName, statusRequest); err != nil { if apierrors.IsNotFound(err) { log.Debug("Unable to find ServerStatusRequest") return ctrl.Result{}, nil } log.WithError(err).Error("Error getting ServerStatusRequest") return ctrl.Result{}, err } log = r.log.WithFields(logrus.Fields{ "controller": constant.ControllerServerStatusRequest, "serverStatusRequest": req.NamespacedName, "phase": statusRequest.Status.Phase, }) switch statusRequest.Status.Phase { case "", velerov1api.ServerStatusRequestPhaseNew: log.Info("Processing new ServerStatusRequest") original := statusRequest.DeepCopy() statusRequest.Status.ServerVersion = buildinfo.Version statusRequest.Status.Phase = velerov1api.ServerStatusRequestPhaseProcessed statusRequest.Status.ProcessedTimestamp = &metav1.Time{Time: r.clock.Now()} statusRequest.Status.Plugins = velero.GetInstalledPluginInfo(r.pluginRegistry) if err := r.client.Patch(r.ctx, statusRequest, client.MergeFrom(original)); err != nil { log.WithError(err).Error("Error updating ServerStatusRequest status") return ctrl.Result{RequeueAfter: statusRequestResyncPeriod}, err } case velerov1api.ServerStatusRequestPhaseProcessed: log.Debug("Checking whether ServerStatusRequest has expired") expiration := statusRequest.Status.ProcessedTimestamp.Add(ttl) if expiration.After(r.clock.Now()) { log.Debug("ServerStatusRequest has not expired") return ctrl.Result{RequeueAfter: statusRequestResyncPeriod}, nil } log.Debug("ServerStatusRequest has expired, deleting it") if err := r.client.Delete(r.ctx, statusRequest); err != nil { log.WithError(err).Error("Unable to delete the request") return ctrl.Result{}, nil } default: return ctrl.Result{}, errors.New("unexpected ServerStatusRequest phase") } // Requeue is mostly to handle deleting any expired status requests that were not // deleted as part of the normal client flow for whatever reason. return ctrl.Result{RequeueAfter: statusRequestResyncPeriod}, nil } func (r *serverStatusRequestReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&velerov1api.ServerStatusRequest{}). WithOptions(controller.Options{ MaxConcurrentReconciles: 10, }). Complete(r) } ================================================ FILE: pkg/controller/server_status_request_controller_test.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package controller import ( "context" "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" testclocks "k8s.io/utils/clock/testing" ctrl "sigs.k8s.io/controller-runtime" kbclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/buildinfo" "github.com/vmware-tanzu/velero/pkg/plugin/framework" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func statusRequestBuilder(resourceVersion string) *builder.ServerStatusRequestBuilder { return builder.ForServerStatusRequest(velerov1api.DefaultNamespace, "sr-1", resourceVersion) } var _ = Describe("Server Status Request Reconciler", func() { type request struct { req *velerov1api.ServerStatusRequest reqPluginLister *fakePluginLister expected *velerov1api.ServerStatusRequest expectedRequeue ctrl.Result expectedErrMsg string } // `now` will be used to set the fake clock's time; capture // it here so it can be referenced in the test case defs. now, err := time.Parse(time.RFC1123, time.RFC1123) Expect(err).ToNot(HaveOccurred()) now = now.Local() DescribeTable("a Server Status request", func(test request) { // Setup reconciler Expect(velerov1api.AddToScheme(scheme.Scheme)).To(Succeed()) r := NewServerStatusRequestReconciler( context.Background(), fake.NewClientBuilder().WithScheme(scheme.Scheme).WithObjects(test.req).Build(), test.reqPluginLister, testclocks.NewFakeClock(now), velerotest.NewLogger(), ) actualResult, err := r.Reconcile(r.ctx, ctrl.Request{ NamespacedName: types.NamespacedName{ Namespace: velerov1api.DefaultNamespace, Name: test.req.Name, }, }) Expect(actualResult).To(BeEquivalentTo(test.expectedRequeue)) if test.expectedErrMsg == "" { Expect(err).ToNot(HaveOccurred()) } else { Expect(err.Error()).To(BeEquivalentTo(test.expectedErrMsg)) return } instance := &velerov1api.ServerStatusRequest{} err = r.client.Get(ctx, kbclient.ObjectKey{Name: test.req.Name, Namespace: test.req.Namespace}, instance) // Assertions if test.expected == nil { Expect(apierrors.IsNotFound(err)).To(BeTrue()) } else { Expect(err).ToNot(HaveOccurred()) Eventually(instance.Status.Phase == test.expected.Status.Phase, timeout).Should(BeTrue()) } }, Entry("with phase=empty will be processed and phased successfully patched", request{ req: statusRequestBuilder("1"). ServerVersion(buildinfo.Version). ProcessedTimestamp(now). Plugins([]velerov1api.PluginInfo{ { Name: "custom.io/myown", Kind: "VolumeSnapshotter", }, }). Result(), reqPluginLister: &fakePluginLister{ plugins: []framework.PluginIdentifier{ { Name: "custom.io/myown", Kind: "VolumeSnapshotter", }, }, }, expected: statusRequestBuilder("1"). ServerVersion(buildinfo.Version). Phase(velerov1api.ServerStatusRequestPhaseProcessed). ProcessedTimestamp(now). Plugins([]velerov1api.PluginInfo{ { Name: "custom.io/myown", Kind: "VolumeSnapshotter", }, }). Result(), expectedRequeue: ctrl.Result{Requeue: false, RequeueAfter: statusRequestResyncPeriod}, }), Entry("with phase=new will be processed and phased successfully patched", request{ req: statusRequestBuilder("1"). ServerVersion(buildinfo.Version). Phase(velerov1api.ServerStatusRequestPhaseNew). ProcessedTimestamp(now). Plugins([]velerov1api.PluginInfo{ { Name: "custom.io/myown", Kind: "VolumeSnapshotter", }, }). Result(), reqPluginLister: &fakePluginLister{ plugins: []framework.PluginIdentifier{ { Name: "custom.io/myown", Kind: "VolumeSnapshotter", }, }, }, expected: statusRequestBuilder("1"). ServerVersion(buildinfo.Version). Phase(velerov1api.ServerStatusRequestPhaseProcessed). ProcessedTimestamp(now). Plugins([]velerov1api.PluginInfo{ { Name: "custom.io/myown", Kind: "VolumeSnapshotter", }, }). Result(), expectedRequeue: ctrl.Result{Requeue: false, RequeueAfter: statusRequestResyncPeriod}, }), Entry("with phase=Processed does not get deleted if not expired", request{ req: statusRequestBuilder("1"). ServerVersion(buildinfo.Version). Phase(velerov1api.ServerStatusRequestPhaseProcessed). ProcessedTimestamp(now). // not yet expired Plugins([]velerov1api.PluginInfo{ { Name: "custom.io/myotherown", Kind: "VolumeSnapshotter", }, }). Result(), reqPluginLister: &fakePluginLister{ plugins: []framework.PluginIdentifier{ { Name: "custom.io/myotherown", Kind: "VolumeSnapshotter", }, }, }, expected: statusRequestBuilder("1"). ServerVersion(buildinfo.Version). Phase(velerov1api.ServerStatusRequestPhaseProcessed). ProcessedTimestamp(now). Plugins([]velerov1api.PluginInfo{ { Name: "custom.io/myown", Kind: "VolumeSnapshotter", }, }). Result(), expectedRequeue: ctrl.Result{Requeue: false, RequeueAfter: statusRequestResyncPeriod}, }), Entry("with phase=Processed gets deleted if expired", request{ req: statusRequestBuilder("1"). ServerVersion(buildinfo.Version). Phase(velerov1api.ServerStatusRequestPhaseProcessed). ProcessedTimestamp(now.Add(-61 * time.Second)). // expired Plugins([]velerov1api.PluginInfo{ { Name: "custom.io/myotherown", Kind: "VolumeSnapshotter", }, }). Result(), reqPluginLister: &fakePluginLister{ plugins: []framework.PluginIdentifier{ { Name: "custom.io/myotherown", Kind: "VolumeSnapshotter", }, }, }, expected: nil, expectedRequeue: ctrl.Result{Requeue: false, RequeueAfter: statusRequestResyncPeriod}, }), Entry("with invalid phase returns an error and does not requeue", request{ req: statusRequestBuilder("1"). ServerVersion(buildinfo.Version). Phase("an-invalid-phase"). ProcessedTimestamp(now). Plugins([]velerov1api.PluginInfo{ { Name: "custom.io/myown", Kind: "VolumeSnapshotter", }, }). Result(), reqPluginLister: &fakePluginLister{ plugins: []framework.PluginIdentifier{ { Name: "custom.io/myown", Kind: "VolumeSnapshotter", }, }, }, expectedErrMsg: "unexpected ServerStatusRequest phase", expectedRequeue: ctrl.Result{Requeue: false, RequeueAfter: 0}, }), ) }) type fakePluginLister struct { plugins []framework.PluginIdentifier } func (l *fakePluginLister) List(kind common.PluginKind) []framework.PluginIdentifier { var plugins []framework.PluginIdentifier for _, plugin := range l.plugins { if plugin.Kind == kind { plugins = append(plugins, plugin) } } return plugins } ================================================ FILE: pkg/controller/suite_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package controller import ( "context" "fmt" "path/filepath" "testing" "time" "github.com/sirupsen/logrus" "github.com/vmware-tanzu/velero/pkg/persistence" persistencemocks "github.com/vmware-tanzu/velero/pkg/persistence/mocks" "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/envtest" "sigs.k8s.io/controller-runtime/pkg/manager" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" // +kubebuilder:scaffold:imports ) // These tests use Ginkgo (BDD-style Go testing framework). Refer to // http://onsi.github.io/ginkgo/ to learn more about Ginkgo. const ( timeout = time.Second * 30 ) var ( env *envtest.Environment testEnv *testEnvironment ctx, cancel = context.WithCancel(context.Background()) ) func TestAPIs(t *testing.T) { RegisterFailHandler(Fail) RunSpecs(t, "Controller Suite") } var _ = BeforeSuite(func(done Done) { By("bootstrapping test environment") testEnv = newTestEnvironment() By("starting the manager") go func() { defer GinkgoRecover() Expect(testEnv.startManager()).To(Succeed()) }() close(done) }) var _ = AfterSuite(func() { By("tearing down the test environment") err := testEnv.stop() Expect(err).ToNot(HaveOccurred()) }) // testEnvironment encapsulates a Kubernetes local test environment. type testEnvironment struct { manager.Manager client.Client Config *rest.Config doneMgr context.Context } // newTestEnvironment creates a new environment spinning up a local api-server. // // This function should be called only once for each package you're running tests within, // usually the environment is initialized in a suite_test.go file within a `BeforeSuite` ginkgo block. func newTestEnvironment() *testEnvironment { // scheme.Scheme is initialized with all native Kubernetes types err := velerov1api.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) err = velerov2alpha1api.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) env = &envtest.Environment{ CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")}, } if _, err := env.Start(); err != nil { panic(err) } mgr, err := manager.New(env.Config, manager.Options{ Scheme: scheme.Scheme, }) if err != nil { klog.Fatalf("Failed to start testenv manager: %v", err) } return &testEnvironment{ Manager: mgr, Client: mgr.GetClient(), Config: mgr.GetConfig(), doneMgr: ctx, } } func (t *testEnvironment) startManager() error { return t.Manager.Start(t.doneMgr) } func (t *testEnvironment) stop() error { cancel() return env.Stop() } type fakeErrorBackupStoreGetter struct { } func (f *fakeErrorBackupStoreGetter) Get(*velerov1api.BackupStorageLocation, persistence.ObjectStoreGetter, logrus.FieldLogger) (persistence.BackupStore, error) { return nil, fmt.Errorf("some error") } type fakeSingleObjectBackupStoreGetter struct { store persistence.BackupStore } func (f *fakeSingleObjectBackupStoreGetter) Get(*velerov1api.BackupStorageLocation, persistence.ObjectStoreGetter, logrus.FieldLogger) (persistence.BackupStore, error) { return f.store, nil } // NewFakeSingleObjectBackupStoreGetter returns an ObjectBackupStoreGetter // that will return only the given BackupStore. func NewFakeSingleObjectBackupStoreGetter(store persistence.BackupStore) persistence.ObjectBackupStoreGetter { return &fakeSingleObjectBackupStoreGetter{store: store} } type fakeObjectBackupStoreGetter struct { stores map[string]*persistencemocks.BackupStore } func (f *fakeObjectBackupStoreGetter) Get(loc *velerov1api.BackupStorageLocation, _ persistence.ObjectStoreGetter, _ logrus.FieldLogger) (persistence.BackupStore, error) { return f.stores[loc.Name], nil } // NewFakeObjectBackupStoreGetter returns an ObjectBackupStoreGetter that will // return the BackupStore for a given BackupStorageLocation name. func NewFakeObjectBackupStoreGetter(stores map[string]*persistencemocks.BackupStore) persistence.ObjectBackupStoreGetter { return &fakeObjectBackupStoreGetter{stores: stores} } ================================================ FILE: pkg/datamover/backup_micro_service.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package datamover import ( "context" "encoding/json" "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" "sigs.k8s.io/controller-runtime/pkg/client" cachetool "k8s.io/client-go/tools/cache" "sigs.k8s.io/controller-runtime/pkg/cache" "github.com/vmware-tanzu/velero/internal/credentials" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" "github.com/vmware-tanzu/velero/pkg/datapath" "github.com/vmware-tanzu/velero/pkg/repository" "github.com/vmware-tanzu/velero/pkg/uploader" "github.com/vmware-tanzu/velero/pkg/util/kube" apierrors "k8s.io/apimachinery/pkg/api/errors" ) const ( dataUploadDownloadRequestor = "snapshot-data-upload-download" ) // BackupMicroService process data mover backups inside the backup pod type BackupMicroService struct { ctx context.Context client client.Client kubeClient kubernetes.Interface repoEnsurer *repository.Ensurer credentialGetter *credentials.CredentialGetter logger logrus.FieldLogger dataPathMgr *datapath.Manager eventRecorder kube.EventRecorder namespace string dataUploadName string dataUpload *velerov2alpha1api.DataUpload sourceTargetPath datapath.AccessPoint resultSignal chan dataPathResult duInformer cache.Informer duHandler cachetool.ResourceEventHandlerRegistration nodeName string } type dataPathResult struct { err error result string } func NewBackupMicroService(ctx context.Context, client client.Client, kubeClient kubernetes.Interface, dataUploadName string, namespace string, nodeName string, sourceTargetPath datapath.AccessPoint, dataPathMgr *datapath.Manager, repoEnsurer *repository.Ensurer, cred *credentials.CredentialGetter, duInformer cache.Informer, log logrus.FieldLogger) *BackupMicroService { return &BackupMicroService{ ctx: ctx, client: client, kubeClient: kubeClient, credentialGetter: cred, logger: log, repoEnsurer: repoEnsurer, dataPathMgr: dataPathMgr, namespace: namespace, dataUploadName: dataUploadName, sourceTargetPath: sourceTargetPath, nodeName: nodeName, resultSignal: make(chan dataPathResult), duInformer: duInformer, } } func (r *BackupMicroService) Init() error { r.eventRecorder = kube.NewEventRecorder(r.kubeClient, r.client.Scheme(), r.dataUploadName, r.nodeName, r.logger) handler, err := r.duInformer.AddEventHandler( cachetool.ResourceEventHandlerFuncs{ UpdateFunc: func(oldObj any, newObj any) { oldDu := oldObj.(*velerov2alpha1api.DataUpload) newDu := newObj.(*velerov2alpha1api.DataUpload) if newDu.Name != r.dataUploadName { return } if newDu.Status.Phase != velerov2alpha1api.DataUploadPhaseInProgress { return } if newDu.Spec.Cancel && !oldDu.Spec.Cancel { r.cancelDataUpload(newDu) } }, }, ) if err != nil { return errors.Wrap(err, "error adding du handler") } r.duHandler = handler return err } func (r *BackupMicroService) RunCancelableDataPath(ctx context.Context) (string, error) { log := r.logger.WithFields(logrus.Fields{ "dataupload": r.dataUploadName, }) du := &velerov2alpha1api.DataUpload{} err := wait.PollUntilContextCancel(ctx, 500*time.Millisecond, true, func(ctx context.Context) (bool, error) { err := r.client.Get(ctx, types.NamespacedName{ Namespace: r.namespace, Name: r.dataUploadName, }, du) if apierrors.IsNotFound(err) { return false, nil } if err != nil { return true, errors.Wrapf(err, "error to get du %s", r.dataUploadName) } if du.Status.Phase == velerov2alpha1api.DataUploadPhaseInProgress { return true, nil } else { return false, nil } }) if err != nil { log.WithError(err).Error("Failed to wait du") return "", errors.Wrap(err, "error waiting for du") } r.dataUpload = du log.Info("Run cancelable dataUpload") callbacks := datapath.Callbacks{ OnCompleted: r.OnDataUploadCompleted, OnFailed: r.OnDataUploadFailed, OnCancelled: r.OnDataUploadCancelled, OnProgress: r.OnDataUploadProgress, } fsBackup, err := r.dataPathMgr.CreateFileSystemBR(du.Name, dataUploadDownloadRequestor, ctx, r.client, du.Namespace, callbacks, log) if err != nil { return "", errors.Wrap(err, "error to create data path") } log.Debug("Async fs br created") if err := fsBackup.Init(ctx, &datapath.FSBRInitParam{ BSLName: du.Spec.BackupStorageLocation, SourceNamespace: du.Spec.SourceNamespace, UploaderType: GetUploaderType(du.Spec.DataMover), RepositoryType: velerov1api.BackupRepositoryTypeKopia, RepoIdentifier: "", RepositoryEnsurer: r.repoEnsurer, CredentialGetter: r.credentialGetter, }); err != nil { return "", errors.Wrap(err, "error to initialize data path") } log.Info("Async fs br init") tags := map[string]string{ velerov1api.AsyncOperationIDLabel: du.Labels[velerov1api.AsyncOperationIDLabel], } if err := fsBackup.StartBackup(r.sourceTargetPath, du.Spec.DataMoverConfig, &datapath.FSBRStartParam{ RealSource: GetRealSource(du.Spec.SourceNamespace, du.Spec.SourcePVC), ParentSnapshot: "", ForceFull: false, Tags: tags, }); err != nil { return "", errors.Wrap(err, "error starting data path backup") } log.Info("Async fs backup data path started") r.eventRecorder.Event(du, false, datapath.EventReasonStarted, "Data path for %s started", du.Name) result := "" select { case <-ctx.Done(): err = errors.New("timed out waiting for fs backup to complete") break case res := <-r.resultSignal: err = res.err result = res.result break } if err != nil { log.WithError(err).Error("Async fs backup was not completed") } r.eventRecorder.EndingEvent(du, false, datapath.EventReasonStopped, "Data path for %s stopped", du.Name) return result, err } func (r *BackupMicroService) Shutdown() { r.eventRecorder.Shutdown() r.closeDataPath(r.ctx, r.dataUploadName) if r.duHandler != nil { if err := r.duInformer.RemoveEventHandler(r.duHandler); err != nil { r.logger.WithError(err).Warn("Failed to remove pod handler") } } } var funcMarshal = json.Marshal func (r *BackupMicroService) OnDataUploadCompleted(ctx context.Context, namespace string, duName string, result datapath.Result) { log := r.logger.WithField("dataupload", duName) backupBytes, err := funcMarshal(result.Backup) if err != nil { log.WithError(err).Errorf("Failed to marshal backup result %v", result.Backup) r.resultSignal <- dataPathResult{ err: errors.Wrapf(err, "Failed to marshal backup result %v", result.Backup), } } else { r.eventRecorder.Event(r.dataUpload, false, datapath.EventReasonCompleted, string(backupBytes)) r.resultSignal <- dataPathResult{ result: string(backupBytes), } } log.Info("Async fs backup completed") } func (r *BackupMicroService) OnDataUploadFailed(ctx context.Context, namespace string, duName string, err error) { log := r.logger.WithField("dataupload", duName) log.WithError(err).Error("Async fs backup data path failed") r.eventRecorder.Event(r.dataUpload, false, datapath.EventReasonFailed, "Data path for data upload %s failed, error %v", r.dataUploadName, err) r.resultSignal <- dataPathResult{ err: errors.Wrapf(err, "Data path for data upload %s failed", r.dataUploadName), } } func (r *BackupMicroService) OnDataUploadCancelled(ctx context.Context, namespace string, duName string) { log := r.logger.WithField("dataupload", duName) log.Warn("Async fs backup data path canceled") r.eventRecorder.Event(r.dataUpload, false, datapath.EventReasonCancelled, "Data path for data upload %s canceled", duName) r.resultSignal <- dataPathResult{ err: errors.New(datapath.ErrCancelled), } } func (r *BackupMicroService) OnDataUploadProgress(ctx context.Context, namespace string, duName string, progress *uploader.Progress) { log := r.logger.WithFields(logrus.Fields{ "dataupload": duName, }) progressBytes, err := funcMarshal(progress) if err != nil { log.WithError(err).Errorf("Failed to marshal progress %v", progress) return } r.eventRecorder.Event(r.dataUpload, false, datapath.EventReasonProgress, string(progressBytes)) } func (r *BackupMicroService) closeDataPath(ctx context.Context, duName string) { fsBackup := r.dataPathMgr.GetAsyncBR(duName) if fsBackup != nil { fsBackup.Close(ctx) } r.dataPathMgr.RemoveAsyncBR(duName) } func (r *BackupMicroService) cancelDataUpload(du *velerov2alpha1api.DataUpload) { r.logger.WithField("DataUpload", du.Name).Info("Data upload is being canceled") r.eventRecorder.Event(du, false, datapath.EventReasonCancelling, "Canceling for data upload %s", du.Name) fsBackup := r.dataPathMgr.GetAsyncBR(du.Name) if fsBackup == nil { r.OnDataUploadCancelled(r.ctx, du.GetNamespace(), du.GetName()) } else { fsBackup.Cancel() } } ================================================ FILE: pkg/datamover/backup_micro_service_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package datamover import ( "context" "fmt" "sync" "testing" "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/runtime" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/datapath" "github.com/vmware-tanzu/velero/pkg/uploader" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" clientFake "sigs.k8s.io/controller-runtime/pkg/client/fake" velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" velerotest "github.com/vmware-tanzu/velero/pkg/test" kbclient "sigs.k8s.io/controller-runtime/pkg/client" datapathmockes "github.com/vmware-tanzu/velero/pkg/datapath/mocks" ) type backupMsTestHelper struct { eventReason string eventMsg string marshalErr error marshalBytes []byte withEvent bool eventLock sync.Mutex } func (bt *backupMsTestHelper) Event(_ runtime.Object, _ bool, reason string, message string, a ...any) { bt.eventLock.Lock() defer bt.eventLock.Unlock() bt.withEvent = true bt.eventReason = reason bt.eventMsg = fmt.Sprintf(message, a...) } func (bt *backupMsTestHelper) EndingEvent(_ runtime.Object, _ bool, reason string, message string, a ...any) { bt.eventLock.Lock() defer bt.eventLock.Unlock() bt.withEvent = true bt.eventReason = reason bt.eventMsg = fmt.Sprintf(message, a...) } func (bt *backupMsTestHelper) Shutdown() {} func (bt *backupMsTestHelper) Marshal(v any) ([]byte, error) { if bt.marshalErr != nil { return nil, bt.marshalErr } return bt.marshalBytes, nil } func (bt *backupMsTestHelper) EventReason() string { bt.eventLock.Lock() defer bt.eventLock.Unlock() return bt.eventReason } func (bt *backupMsTestHelper) EventMessage() string { bt.eventLock.Lock() defer bt.eventLock.Unlock() return bt.eventMsg } func TestOnDataUploadFailed(t *testing.T) { dataUploadName := "fake-data-upload" bt := &backupMsTestHelper{} bs := &BackupMicroService{ dataUploadName: dataUploadName, dataPathMgr: datapath.NewManager(1), eventRecorder: bt, resultSignal: make(chan dataPathResult), logger: velerotest.NewLogger(), } expectedErr := "Data path for data upload fake-data-upload failed: fake-error" expectedEventReason := datapath.EventReasonFailed expectedEventMsg := "Data path for data upload fake-data-upload failed, error fake-error" go bs.OnDataUploadFailed(t.Context(), velerov1api.DefaultNamespace, dataUploadName, errors.New("fake-error")) result := <-bs.resultSignal require.EqualError(t, result.err, expectedErr) assert.Equal(t, expectedEventReason, bt.EventReason()) assert.Equal(t, expectedEventMsg, bt.EventMessage()) } func TestOnDataUploadCancelled(t *testing.T) { dataUploadName := "fake-data-upload" bt := &backupMsTestHelper{} bs := &BackupMicroService{ dataUploadName: dataUploadName, dataPathMgr: datapath.NewManager(1), eventRecorder: bt, resultSignal: make(chan dataPathResult), logger: velerotest.NewLogger(), } expectedErr := datapath.ErrCancelled expectedEventReason := datapath.EventReasonCancelled expectedEventMsg := "Data path for data upload fake-data-upload canceled" go bs.OnDataUploadCancelled(t.Context(), velerov1api.DefaultNamespace, dataUploadName) result := <-bs.resultSignal require.EqualError(t, result.err, expectedErr) assert.Equal(t, expectedEventReason, bt.EventReason()) assert.Equal(t, expectedEventMsg, bt.EventMessage()) } func TestOnDataUploadCompleted(t *testing.T) { tests := []struct { name string expectedErr string expectedEventReason string expectedEventMsg string marshalErr error marshallStr string }{ { name: "marshal fail", marshalErr: errors.New("fake-marshal-error"), expectedErr: "Failed to marshal backup result { false { } 0 0}: fake-marshal-error", }, { name: "succeed", marshallStr: "fake-complete-string", expectedEventReason: datapath.EventReasonCompleted, expectedEventMsg: "fake-complete-string", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { dataUploadName := "fake-data-upload" bt := &backupMsTestHelper{ marshalErr: test.marshalErr, marshalBytes: []byte(test.marshallStr), } bs := &BackupMicroService{ dataPathMgr: datapath.NewManager(1), eventRecorder: bt, resultSignal: make(chan dataPathResult), logger: velerotest.NewLogger(), } funcMarshal = bt.Marshal go bs.OnDataUploadCompleted(t.Context(), velerov1api.DefaultNamespace, dataUploadName, datapath.Result{}) result := <-bs.resultSignal if test.marshalErr != nil { assert.EqualError(t, result.err, test.expectedErr) } else { require.NoError(t, result.err) assert.Equal(t, test.expectedEventReason, bt.EventReason()) assert.Equal(t, test.expectedEventMsg, bt.EventMessage()) } }) } } func TestOnDataUploadProgress(t *testing.T) { tests := []struct { name string expectedErr string expectedEventReason string expectedEventMsg string marshalErr error marshallStr string }{ { name: "marshal fail", marshalErr: errors.New("fake-marshal-error"), expectedErr: "Failed to marshal backup result", }, { name: "succeed", marshallStr: "fake-progress-string", expectedEventReason: datapath.EventReasonProgress, expectedEventMsg: "fake-progress-string", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { dataUploadName := "fake-data-upload" bt := &backupMsTestHelper{ marshalErr: test.marshalErr, marshalBytes: []byte(test.marshallStr), } bs := &BackupMicroService{ dataPathMgr: datapath.NewManager(1), eventRecorder: bt, logger: velerotest.NewLogger(), } funcMarshal = bt.Marshal bs.OnDataUploadProgress(t.Context(), velerov1api.DefaultNamespace, dataUploadName, &uploader.Progress{}) if test.marshalErr != nil { assert.False(t, bt.withEvent) } else { assert.True(t, bt.withEvent) assert.Equal(t, test.expectedEventReason, bt.EventReason()) assert.Equal(t, test.expectedEventMsg, bt.EventMessage()) } }) } } func TestCancelDataUpload(t *testing.T) { tests := []struct { name string expectedEventReason string expectedEventMsg string expectedErr string }{ { name: "no fs backup", expectedEventReason: datapath.EventReasonCancelled, expectedEventMsg: "Data path for data upload fake-data-upload canceled", expectedErr: datapath.ErrCancelled, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { dataUploadName := "fake-data-upload" du := builder.ForDataUpload(velerov1api.DefaultNamespace, dataUploadName).Result() bt := &backupMsTestHelper{} bs := &BackupMicroService{ dataPathMgr: datapath.NewManager(1), eventRecorder: bt, resultSignal: make(chan dataPathResult), logger: velerotest.NewLogger(), } go bs.cancelDataUpload(du) result := <-bs.resultSignal require.EqualError(t, result.err, test.expectedErr) assert.True(t, bt.withEvent) assert.Equal(t, test.expectedEventReason, bt.EventReason()) assert.Equal(t, test.expectedEventMsg, bt.EventMessage()) }) } } func TestRunCancelableDataPath(t *testing.T) { dataUploadName := "fake-data-upload" du := builder.ForDataUpload(velerov1api.DefaultNamespace, dataUploadName).Phase(velerov2alpha1api.DataUploadPhaseNew).Result() duInProgress := builder.ForDataUpload(velerov1api.DefaultNamespace, dataUploadName).Phase(velerov2alpha1api.DataUploadPhaseInProgress).Result() ctxTimeout, cancel := context.WithTimeout(t.Context(), time.Second) tests := []struct { name string ctx context.Context result *dataPathResult dataPathMgr *datapath.Manager kubeClientObj []runtime.Object initErr error startErr error dataPathStarted bool expectedEventMsg string expectedErr string }{ { name: "no du", ctx: ctxTimeout, expectedErr: "error waiting for du: context deadline exceeded", }, { name: "du not in in-progress", ctx: ctxTimeout, kubeClientObj: []runtime.Object{du}, expectedErr: "error waiting for du: context deadline exceeded", }, { name: "create data path fail", ctx: t.Context(), kubeClientObj: []runtime.Object{duInProgress}, dataPathMgr: datapath.NewManager(0), expectedErr: "error to create data path: Concurrent number exceeds", }, { name: "init data path fail", ctx: t.Context(), kubeClientObj: []runtime.Object{duInProgress}, initErr: errors.New("fake-init-error"), expectedErr: "error to initialize data path: fake-init-error", }, { name: "start data path fail", ctx: t.Context(), kubeClientObj: []runtime.Object{duInProgress}, startErr: errors.New("fake-start-error"), expectedErr: "error starting data path backup: fake-start-error", }, { name: "data path timeout", ctx: ctxTimeout, kubeClientObj: []runtime.Object{duInProgress}, dataPathStarted: true, expectedEventMsg: fmt.Sprintf("Data path for %s stopped", dataUploadName), expectedErr: "timed out waiting for fs backup to complete", }, { name: "data path returns error", ctx: t.Context(), kubeClientObj: []runtime.Object{duInProgress}, dataPathStarted: true, result: &dataPathResult{ err: errors.New("fake-data-path-error"), }, expectedEventMsg: fmt.Sprintf("Data path for %s stopped", dataUploadName), expectedErr: "fake-data-path-error", }, { name: "succeed", ctx: t.Context(), kubeClientObj: []runtime.Object{duInProgress}, dataPathStarted: true, result: &dataPathResult{ result: "fake-succeed-result", }, expectedEventMsg: fmt.Sprintf("Data path for %s stopped", dataUploadName), }, } scheme := runtime.NewScheme() velerov2alpha1api.AddToScheme(scheme) for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeClientBuilder := clientFake.NewClientBuilder() fakeClientBuilder = fakeClientBuilder.WithScheme(scheme) fakeClient := fakeClientBuilder.WithRuntimeObjects(test.kubeClientObj...).Build() bt := &backupMsTestHelper{} bs := &BackupMicroService{ namespace: velerov1api.DefaultNamespace, dataUploadName: dataUploadName, ctx: t.Context(), client: fakeClient, dataPathMgr: datapath.NewManager(1), eventRecorder: bt, resultSignal: make(chan dataPathResult), logger: velerotest.NewLogger(), } if test.ctx != nil { bs.ctx = test.ctx } if test.dataPathMgr != nil { bs.dataPathMgr = test.dataPathMgr } datapath.FSBRCreator = func(string, string, kbclient.Client, string, datapath.Callbacks, logrus.FieldLogger) datapath.AsyncBR { fsBR := datapathmockes.NewAsyncBR(t) if test.initErr != nil { fsBR.On("Init", mock.Anything, mock.Anything).Return(test.initErr) } if test.startErr != nil { fsBR.On("Init", mock.Anything, mock.Anything).Return(nil) fsBR.On("StartBackup", mock.Anything, mock.Anything, mock.Anything).Return(test.startErr) } if test.dataPathStarted { fsBR.On("Init", mock.Anything, mock.Anything).Return(nil) fsBR.On("StartBackup", mock.Anything, mock.Anything, mock.Anything).Return(nil) } return fsBR } if test.result != nil { go func() { time.Sleep(time.Millisecond * 500) bs.resultSignal <- *test.result }() } result, err := bs.RunCancelableDataPath(test.ctx) if test.expectedErr != "" { require.EqualError(t, err, test.expectedErr) } else { require.NoError(t, err) assert.Equal(t, test.result.result, result) } if test.expectedEventMsg != "" { assert.True(t, bt.withEvent) assert.Equal(t, test.expectedEventMsg, bt.EventMessage()) } }) } cancel() } ================================================ FILE: pkg/datamover/dataupload_delete_action.go ================================================ package datamover import ( "context" "encoding/json" "fmt" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" "github.com/vmware-tanzu/velero/pkg/plugin/velero" repotypes "github.com/vmware-tanzu/velero/pkg/repository/types" ) type DataUploadDeleteAction struct { logger logrus.FieldLogger client client.Client } func (d *DataUploadDeleteAction) AppliesTo() (velero.ResourceSelector, error) { return velero.ResourceSelector{ IncludedResources: []string{"datauploads.velero.io"}, }, nil } func (d *DataUploadDeleteAction) Execute(input *velero.DeleteItemActionExecuteInput) error { d.logger.Infof("Executing DataUploadDeleteAction") du := &velerov2alpha1.DataUpload{} if err := runtime.DefaultUnstructuredConverter.FromUnstructured(input.Item.UnstructuredContent(), &du); err != nil { return errors.WithStack(errors.Wrapf(err, "failed to convert input.Item from unstructured")) } cm := genConfigmap(input.Backup, *du) if cm == nil { // will not fail the backup deletion return nil } err := d.client.Create(context.Background(), cm) if err != nil { return errors.WithStack(errors.Wrapf(err, "failed to create the configmap for DataUpload %s/%s", du.Namespace, du.Name)) } return nil } // generate the configmap which is to be created and used as a way to communicate the snapshot info to the backup deletion controller func genConfigmap(bak *velerov1.Backup, du velerov2alpha1.DataUpload) *corev1api.ConfigMap { if !IsBuiltInUploader(du.Spec.DataMover) || du.Status.SnapshotID == "" { return nil } snapshot := repotypes.SnapshotIdentifier{ VolumeNamespace: du.Spec.SourceNamespace, BackupStorageLocation: bak.Spec.StorageLocation, SnapshotID: du.Status.SnapshotID, RepositoryType: velerov1.BackupRepositoryTypeKopia, UploaderType: GetUploaderType(du.Spec.DataMover), Source: GetRealSource(du.Spec.SourceNamespace, du.Spec.SourcePVC), } b, err := json.Marshal(snapshot) if err != nil { return nil } data := make(map[string]string) if err := json.Unmarshal(b, &data); err != nil { return nil } return &corev1api.ConfigMap{ TypeMeta: metav1.TypeMeta{ APIVersion: corev1api.SchemeGroupVersion.String(), Kind: "ConfigMap", }, ObjectMeta: metav1.ObjectMeta{ Namespace: bak.Namespace, Name: fmt.Sprintf("%s-info", du.Name), Labels: map[string]string{ velerov1.BackupNameLabel: bak.Name, velerov1.DataUploadSnapshotInfoLabel: "true", }, }, Data: data, } } func NewDataUploadDeleteAction(logger logrus.FieldLogger, client client.Client) *DataUploadDeleteAction { return &DataUploadDeleteAction{ logger: logger, client: client, } } ================================================ FILE: pkg/datamover/restore_micro_service.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package datamover import ( "context" "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/internal/credentials" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" "github.com/vmware-tanzu/velero/pkg/datapath" "github.com/vmware-tanzu/velero/pkg/repository" "github.com/vmware-tanzu/velero/pkg/uploader" "github.com/vmware-tanzu/velero/pkg/util/kube" cachetool "k8s.io/client-go/tools/cache" ) // RestoreMicroService process data mover restores inside the restore pod type RestoreMicroService struct { ctx context.Context client client.Client kubeClient kubernetes.Interface repoEnsurer *repository.Ensurer credentialGetter *credentials.CredentialGetter logger logrus.FieldLogger dataPathMgr *datapath.Manager eventRecorder kube.EventRecorder namespace string dataDownloadName string dataDownload *velerov2alpha1api.DataDownload sourceTargetPath datapath.AccessPoint resultSignal chan dataPathResult ddInformer cache.Informer ddHandler cachetool.ResourceEventHandlerRegistration nodeName string cacheDir string } func NewRestoreMicroService(ctx context.Context, client client.Client, kubeClient kubernetes.Interface, dataDownloadName string, namespace string, nodeName string, sourceTargetPath datapath.AccessPoint, dataPathMgr *datapath.Manager, repoEnsurer *repository.Ensurer, cred *credentials.CredentialGetter, ddInformer cache.Informer, cacheDir string, log logrus.FieldLogger) *RestoreMicroService { return &RestoreMicroService{ ctx: ctx, client: client, kubeClient: kubeClient, credentialGetter: cred, logger: log, repoEnsurer: repoEnsurer, dataPathMgr: dataPathMgr, namespace: namespace, dataDownloadName: dataDownloadName, sourceTargetPath: sourceTargetPath, nodeName: nodeName, resultSignal: make(chan dataPathResult), ddInformer: ddInformer, cacheDir: cacheDir, } } func (r *RestoreMicroService) Init() error { r.eventRecorder = kube.NewEventRecorder(r.kubeClient, r.client.Scheme(), r.dataDownloadName, r.nodeName, r.logger) handler, err := r.ddInformer.AddEventHandler( cachetool.ResourceEventHandlerFuncs{ UpdateFunc: func(oldObj any, newObj any) { oldDd := oldObj.(*velerov2alpha1api.DataDownload) newDd := newObj.(*velerov2alpha1api.DataDownload) if newDd.Name != r.dataDownloadName { return } if newDd.Status.Phase != velerov2alpha1api.DataDownloadPhaseInProgress { return } if newDd.Spec.Cancel && !oldDd.Spec.Cancel { r.cancelDataDownload(newDd) } }, }, ) if err != nil { return errors.Wrap(err, "error adding dd handler") } r.ddHandler = handler return err } func (r *RestoreMicroService) RunCancelableDataPath(ctx context.Context) (string, error) { log := r.logger.WithFields(logrus.Fields{ "datadownload": r.dataDownloadName, }) dd := &velerov2alpha1api.DataDownload{} err := wait.PollUntilContextCancel(ctx, 500*time.Millisecond, true, func(ctx context.Context) (bool, error) { err := r.client.Get(ctx, types.NamespacedName{ Namespace: r.namespace, Name: r.dataDownloadName, }, dd) if apierrors.IsNotFound(err) { return false, nil } if err != nil { return true, errors.Wrapf(err, "error to get dd %s", r.dataDownloadName) } if dd.Status.Phase == velerov2alpha1api.DataDownloadPhaseInProgress { return true, nil } else { return false, nil } }) if err != nil { log.WithError(err).Error("Failed to wait dd") return "", errors.Wrap(err, "error waiting for dd") } r.dataDownload = dd log.Info("Run cancelable dataDownload") callbacks := datapath.Callbacks{ OnCompleted: r.OnDataDownloadCompleted, OnFailed: r.OnDataDownloadFailed, OnCancelled: r.OnDataDownloadCancelled, OnProgress: r.OnDataDownloadProgress, } fsRestore, err := r.dataPathMgr.CreateFileSystemBR(dd.Name, dataUploadDownloadRequestor, ctx, r.client, dd.Namespace, callbacks, log) if err != nil { return "", errors.Wrap(err, "error to create data path") } log.Debug("Found volume path") if err := fsRestore.Init(ctx, &datapath.FSBRInitParam{ BSLName: dd.Spec.BackupStorageLocation, SourceNamespace: dd.Spec.SourceNamespace, UploaderType: GetUploaderType(dd.Spec.DataMover), RepositoryType: velerov1api.BackupRepositoryTypeKopia, RepoIdentifier: "", RepositoryEnsurer: r.repoEnsurer, CredentialGetter: r.credentialGetter, CacheDir: r.cacheDir, }); err != nil { return "", errors.Wrap(err, "error to initialize data path") } log.Info("fs init") if err := fsRestore.StartRestore(dd.Spec.SnapshotID, r.sourceTargetPath, dd.Spec.DataMoverConfig); err != nil { return "", errors.Wrap(err, "error starting data path restore") } log.Info("Async fs restore data path started") r.eventRecorder.Event(dd, false, datapath.EventReasonStarted, "Data path for %s started", dd.Name) result := "" select { case <-ctx.Done(): err = errors.New("timed out waiting for fs restore to complete") break case res := <-r.resultSignal: err = res.err result = res.result break } if err != nil { log.WithError(err).Error("Async fs restore was not completed") } r.eventRecorder.EndingEvent(dd, false, datapath.EventReasonStopped, "Data path for %s stopped", dd.Name) return result, err } func (r *RestoreMicroService) Shutdown() { r.eventRecorder.Shutdown() r.closeDataPath(r.ctx, r.dataDownloadName) if r.ddHandler != nil { if err := r.ddInformer.RemoveEventHandler(r.ddHandler); err != nil { r.logger.WithError(err).Warn("Failed to remove pod handler") } } } func (r *RestoreMicroService) OnDataDownloadCompleted(ctx context.Context, namespace string, ddName string, result datapath.Result) { log := r.logger.WithField("datadownload", ddName) restoreBytes, err := funcMarshal(result.Restore) if err != nil { log.WithError(err).Errorf("Failed to marshal restore result %v", result.Restore) r.resultSignal <- dataPathResult{ err: errors.Wrapf(err, "Failed to marshal restore result %v", result.Restore), } } else { r.eventRecorder.Event(r.dataDownload, false, datapath.EventReasonCompleted, string(restoreBytes)) r.resultSignal <- dataPathResult{ result: string(restoreBytes), } } log.Info("Async fs restore data path completed") } func (r *RestoreMicroService) OnDataDownloadFailed(ctx context.Context, namespace string, ddName string, err error) { log := r.logger.WithField("datadownload", ddName) log.WithError(err).Error("Async fs restore data path failed") r.eventRecorder.Event(r.dataDownload, false, datapath.EventReasonFailed, "Data path for data download %s failed, error %v", r.dataDownloadName, err) r.resultSignal <- dataPathResult{ err: errors.Wrapf(err, "Data path for data download %s failed", r.dataDownloadName), } } func (r *RestoreMicroService) OnDataDownloadCancelled(ctx context.Context, namespace string, ddName string) { log := r.logger.WithField("datadownload", ddName) log.Warn("Async fs restore data path canceled") r.eventRecorder.Event(r.dataDownload, false, datapath.EventReasonCancelled, "Data path for data download %s canceled", ddName) r.resultSignal <- dataPathResult{ err: errors.New(datapath.ErrCancelled), } } func (r *RestoreMicroService) OnDataDownloadProgress(ctx context.Context, namespace string, ddName string, progress *uploader.Progress) { log := r.logger.WithFields(logrus.Fields{ "datadownload": ddName, }) progressBytes, err := funcMarshal(progress) if err != nil { log.WithError(err).Errorf("Failed to marshal progress %v", progress) return } r.eventRecorder.Event(r.dataDownload, false, datapath.EventReasonProgress, string(progressBytes)) } func (r *RestoreMicroService) closeDataPath(ctx context.Context, ddName string) { fsRestore := r.dataPathMgr.GetAsyncBR(ddName) if fsRestore != nil { fsRestore.Close(ctx) } r.dataPathMgr.RemoveAsyncBR(ddName) } func (r *RestoreMicroService) cancelDataDownload(dd *velerov2alpha1api.DataDownload) { r.logger.WithField("DataDownload", dd.Name).Info("Data download is being canceled") r.eventRecorder.Event(dd, false, datapath.EventReasonCancelling, "Canceling for data download %s", dd.Name) fsBackup := r.dataPathMgr.GetAsyncBR(dd.Name) if fsBackup == nil { r.OnDataDownloadCancelled(r.ctx, dd.GetNamespace(), dd.GetName()) } else { fsBackup.Cancel() } } ================================================ FILE: pkg/datamover/restore_micro_service_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package datamover import ( "context" "fmt" "testing" "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/runtime" kbclient "sigs.k8s.io/controller-runtime/pkg/client" clientFake "sigs.k8s.io/controller-runtime/pkg/client/fake" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/datapath" datapathmockes "github.com/vmware-tanzu/velero/pkg/datapath/mocks" "github.com/vmware-tanzu/velero/pkg/uploader" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func TestOnDataDownloadFailed(t *testing.T) { dataDownloadName := "fake-data-download" bt := &backupMsTestHelper{} bs := &RestoreMicroService{ dataDownloadName: dataDownloadName, dataPathMgr: datapath.NewManager(1), eventRecorder: bt, resultSignal: make(chan dataPathResult), logger: velerotest.NewLogger(), } expectedErr := "Data path for data download fake-data-download failed: fake-error" expectedEventReason := datapath.EventReasonFailed expectedEventMsg := "Data path for data download fake-data-download failed, error fake-error" go bs.OnDataDownloadFailed(t.Context(), velerov1api.DefaultNamespace, dataDownloadName, errors.New("fake-error")) result := <-bs.resultSignal require.EqualError(t, result.err, expectedErr) assert.Equal(t, expectedEventReason, bt.EventReason()) assert.Equal(t, expectedEventMsg, bt.EventMessage()) } func TestOnDataDownloadCancelled(t *testing.T) { dataDownloadName := "fake-data-download" bt := &backupMsTestHelper{} bs := &RestoreMicroService{ dataDownloadName: dataDownloadName, dataPathMgr: datapath.NewManager(1), eventRecorder: bt, resultSignal: make(chan dataPathResult), logger: velerotest.NewLogger(), } expectedErr := datapath.ErrCancelled expectedEventReason := datapath.EventReasonCancelled expectedEventMsg := "Data path for data download fake-data-download canceled" go bs.OnDataDownloadCancelled(t.Context(), velerov1api.DefaultNamespace, dataDownloadName) result := <-bs.resultSignal require.EqualError(t, result.err, expectedErr) assert.Equal(t, expectedEventReason, bt.EventReason()) assert.Equal(t, expectedEventMsg, bt.EventMessage()) } func TestOnDataDownloadCompleted(t *testing.T) { tests := []struct { name string expectedErr string expectedEventReason string expectedEventMsg string marshalErr error marshallStr string }{ { name: "marshal fail", marshalErr: errors.New("fake-marshal-error"), expectedErr: "Failed to marshal restore result {{ } 0}: fake-marshal-error", }, { name: "succeed", marshallStr: "fake-complete-string", expectedEventReason: datapath.EventReasonCompleted, expectedEventMsg: "fake-complete-string", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { dataDownloadName := "fake-data-download" bt := &backupMsTestHelper{ marshalErr: test.marshalErr, marshalBytes: []byte(test.marshallStr), } bs := &RestoreMicroService{ dataPathMgr: datapath.NewManager(1), eventRecorder: bt, resultSignal: make(chan dataPathResult), logger: velerotest.NewLogger(), } funcMarshal = bt.Marshal go bs.OnDataDownloadCompleted(t.Context(), velerov1api.DefaultNamespace, dataDownloadName, datapath.Result{}) result := <-bs.resultSignal if test.marshalErr != nil { assert.EqualError(t, result.err, test.expectedErr) } else { require.NoError(t, result.err) assert.Equal(t, test.expectedEventReason, bt.EventReason()) assert.Equal(t, test.expectedEventMsg, bt.EventMessage()) } }) } } func TestOnDataDownloadProgress(t *testing.T) { tests := []struct { name string expectedEventReason string expectedEventMsg string marshalErr error marshallStr string }{ { name: "marshal fail", marshalErr: errors.New("fake-marshal-error"), }, { name: "succeed", marshallStr: "fake-progress-string", expectedEventReason: datapath.EventReasonProgress, expectedEventMsg: "fake-progress-string", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { dataDownloadName := "fake-data-download" bt := &backupMsTestHelper{ marshalErr: test.marshalErr, marshalBytes: []byte(test.marshallStr), } bs := &RestoreMicroService{ dataPathMgr: datapath.NewManager(1), eventRecorder: bt, logger: velerotest.NewLogger(), } funcMarshal = bt.Marshal bs.OnDataDownloadProgress(t.Context(), velerov1api.DefaultNamespace, dataDownloadName, &uploader.Progress{}) if test.marshalErr != nil { assert.False(t, bt.withEvent) } else { assert.True(t, bt.withEvent) assert.Equal(t, test.expectedEventReason, bt.EventReason()) assert.Equal(t, test.expectedEventMsg, bt.EventMessage()) } }) } } func TestCancelDataDownload(t *testing.T) { tests := []struct { name string expectedEventReason string expectedEventMsg string expectedErr string }{ { name: "no fs restore", expectedEventReason: datapath.EventReasonCancelled, expectedEventMsg: "Data path for data download fake-data-download canceled", expectedErr: datapath.ErrCancelled, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { dataDownloadName := "fake-data-download" dd := builder.ForDataDownload(velerov1api.DefaultNamespace, dataDownloadName).Result() bt := &backupMsTestHelper{} bs := &RestoreMicroService{ dataPathMgr: datapath.NewManager(1), eventRecorder: bt, resultSignal: make(chan dataPathResult), logger: velerotest.NewLogger(), } go bs.cancelDataDownload(dd) result := <-bs.resultSignal require.EqualError(t, result.err, test.expectedErr) assert.True(t, bt.withEvent) assert.Equal(t, test.expectedEventReason, bt.EventReason()) assert.Equal(t, test.expectedEventMsg, bt.EventMessage()) }) } } func TestRunCancelableRestore(t *testing.T) { dataDownloadName := "fake-data-download" dd := builder.ForDataDownload(velerov1api.DefaultNamespace, dataDownloadName).Phase(velerov2alpha1api.DataDownloadPhaseNew).Result() ddInProgress := builder.ForDataDownload(velerov1api.DefaultNamespace, dataDownloadName).Phase(velerov2alpha1api.DataDownloadPhaseInProgress).Result() ctxTimeout, cancel := context.WithTimeout(t.Context(), time.Second) tests := []struct { name string ctx context.Context result *dataPathResult dataPathMgr *datapath.Manager kubeClientObj []runtime.Object initErr error startErr error dataPathStarted bool expectedEventMsg string expectedErr string }{ { name: "no dd", ctx: ctxTimeout, expectedErr: "error waiting for dd: context deadline exceeded", }, { name: "dd not in in-progress", ctx: ctxTimeout, kubeClientObj: []runtime.Object{dd}, expectedErr: "error waiting for dd: context deadline exceeded", }, { name: "create data path fail", ctx: t.Context(), kubeClientObj: []runtime.Object{ddInProgress}, dataPathMgr: datapath.NewManager(0), expectedErr: "error to create data path: Concurrent number exceeds", }, { name: "init data path fail", ctx: t.Context(), kubeClientObj: []runtime.Object{ddInProgress}, initErr: errors.New("fake-init-error"), expectedErr: "error to initialize data path: fake-init-error", }, { name: "start data path fail", ctx: t.Context(), kubeClientObj: []runtime.Object{ddInProgress}, startErr: errors.New("fake-start-error"), expectedErr: "error starting data path restore: fake-start-error", }, { name: "data path timeout", ctx: ctxTimeout, kubeClientObj: []runtime.Object{ddInProgress}, dataPathStarted: true, expectedEventMsg: fmt.Sprintf("Data path for %s stopped", dataDownloadName), expectedErr: "timed out waiting for fs restore to complete", }, { name: "data path returns error", ctx: t.Context(), kubeClientObj: []runtime.Object{ddInProgress}, dataPathStarted: true, result: &dataPathResult{ err: errors.New("fake-data-path-error"), }, expectedEventMsg: fmt.Sprintf("Data path for %s stopped", dataDownloadName), expectedErr: "fake-data-path-error", }, { name: "succeed", ctx: t.Context(), kubeClientObj: []runtime.Object{ddInProgress}, dataPathStarted: true, result: &dataPathResult{ result: "fake-succeed-result", }, expectedEventMsg: fmt.Sprintf("Data path for %s stopped", dataDownloadName), }, } scheme := runtime.NewScheme() velerov2alpha1api.AddToScheme(scheme) for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeClientBuilder := clientFake.NewClientBuilder() fakeClientBuilder = fakeClientBuilder.WithScheme(scheme) fakeClient := fakeClientBuilder.WithRuntimeObjects(test.kubeClientObj...).Build() bt := &backupMsTestHelper{} rs := &RestoreMicroService{ namespace: velerov1api.DefaultNamespace, dataDownloadName: dataDownloadName, ctx: t.Context(), client: fakeClient, dataPathMgr: datapath.NewManager(1), eventRecorder: bt, resultSignal: make(chan dataPathResult), logger: velerotest.NewLogger(), } if test.ctx != nil { rs.ctx = test.ctx } if test.dataPathMgr != nil { rs.dataPathMgr = test.dataPathMgr } datapath.FSBRCreator = func(string, string, kbclient.Client, string, datapath.Callbacks, logrus.FieldLogger) datapath.AsyncBR { fsBR := datapathmockes.NewAsyncBR(t) if test.initErr != nil { fsBR.On("Init", mock.Anything, mock.Anything).Return(test.initErr) } if test.startErr != nil { fsBR.On("Init", mock.Anything, mock.Anything).Return(nil) fsBR.On("StartRestore", mock.Anything, mock.Anything, mock.Anything).Return(test.startErr) } if test.dataPathStarted { fsBR.On("Init", mock.Anything, mock.Anything).Return(nil) fsBR.On("StartRestore", mock.Anything, mock.Anything, mock.Anything).Return(nil) } return fsBR } if test.result != nil { go func() { time.Sleep(time.Millisecond * 500) rs.resultSignal <- *test.result }() } result, err := rs.RunCancelableDataPath(test.ctx) if test.expectedErr != "" { require.EqualError(t, err, test.expectedErr) } else { require.NoError(t, err) assert.Equal(t, test.result.result, result) } if test.expectedEventMsg != "" { assert.True(t, bt.withEvent) assert.Equal(t, test.expectedEventMsg, bt.EventMessage()) } }) } cancel() } ================================================ FILE: pkg/datamover/util.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package datamover import "fmt" func GetUploaderType(dataMover string) string { if dataMover == "" || dataMover == "velero" { return "kopia" } else { return dataMover } } func IsBuiltInUploader(dataMover string) bool { return dataMover == "" || dataMover == "velero" } func GetRealSource(sourceNamespace string, pvcName string) string { return fmt.Sprintf("%s/%s", sourceNamespace, pvcName) } ================================================ FILE: pkg/datamover/util_test.go ================================================ package datamover import ( "testing" "github.com/stretchr/testify/assert" ) func TestIsBuiltInUploader(t *testing.T) { testcases := []struct { name string dataMover string want bool }{ { name: "empty dataMover is builtin", dataMover: "", want: true, }, { name: "velero dataMover is builtin", dataMover: "velero", want: true, }, { name: "kopia dataMover is not builtin", dataMover: "kopia", want: false, }, } for _, tc := range testcases { t.Run(tc.name, func(tt *testing.T) { assert.Equal(tt, tc.want, IsBuiltInUploader(tc.dataMover)) }) } } func TestGetUploaderType(t *testing.T) { testcases := []struct { name string input string want string }{ { name: "empty dataMover is kopia", input: "", want: "kopia", }, { name: "velero dataMover is kopia", input: "velero", want: "kopia", }, { name: "kopia dataMover is kopia", input: "kopia", want: "kopia", }, { name: "restic dataMover is restic", input: "restic", want: "restic", }, } for _, tc := range testcases { t.Run(tc.name, func(tt *testing.T) { assert.Equal(tt, tc.want, GetUploaderType(tc.input)) }) } } ================================================ FILE: pkg/datapath/error.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package datapath // DataPathError represents an error that occurred during a backup or restore operation type DataPathError struct { snapshotID string err error } // Error implements error. func (e DataPathError) Error() string { return e.err.Error() } // GetSnapshotID returns the snapshot ID for the error. func (e DataPathError) GetSnapshotID() string { return e.snapshotID } ================================================ FILE: pkg/datapath/error_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package datapath import ( "errors" "testing" ) func TestGetSnapshotID(t *testing.T) { // Create a DataPathError instance for testing err := DataPathError{snapshotID: "123", err: errors.New("example error")} // Call the GetSnapshotID method to retrieve the snapshot ID snapshotID := err.GetSnapshotID() // Check if the retrieved snapshot ID matches the expected value if snapshotID != "123" { t.Errorf("GetSnapshotID() returned unexpected snapshot ID: got %s, want %s", snapshotID, "123") } } func TestError(t *testing.T) { // Create a DataPathError instance for testing err := DataPathError{snapshotID: "123", err: errors.New("example error")} // Call the Error method to retrieve the error message errMsg := err.Error() // Check if the retrieved error message matches the expected value expectedErrMsg := "example error" if errMsg != expectedErrMsg { t.Errorf("Error() returned unexpected error message: got %s, want %s", errMsg, expectedErrMsg) } } ================================================ FILE: pkg/datapath/file_system.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package datapath import ( "context" "sync" "github.com/pkg/errors" "github.com/sirupsen/logrus" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/internal/credentials" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/repository" repokey "github.com/vmware-tanzu/velero/pkg/repository/keys" repoProvider "github.com/vmware-tanzu/velero/pkg/repository/provider" "github.com/vmware-tanzu/velero/pkg/uploader" "github.com/vmware-tanzu/velero/pkg/uploader/provider" "github.com/vmware-tanzu/velero/pkg/util/filesystem" ) // FSBRInitParam define the input param for FSBR init type FSBRInitParam struct { BSLName string SourceNamespace string UploaderType string RepositoryType string RepoIdentifier string RepositoryEnsurer *repository.Ensurer CredentialGetter *credentials.CredentialGetter Filesystem filesystem.Interface CacheDir string } // FSBRStartParam define the input param for FSBR start type FSBRStartParam struct { RealSource string ParentSnapshot string ForceFull bool Tags map[string]string } type fileSystemBR struct { ctx context.Context cancel context.CancelFunc backupRepo *velerov1api.BackupRepository uploaderProv provider.Provider log logrus.FieldLogger client client.Client backupLocation *velerov1api.BackupStorageLocation namespace string initialized bool callbacks Callbacks jobName string requestorType string wgDataPath sync.WaitGroup dataPathLock sync.Mutex } func newFileSystemBR(jobName string, requestorType string, client client.Client, namespace string, callbacks Callbacks, log logrus.FieldLogger) AsyncBR { fs := &fileSystemBR{ jobName: jobName, requestorType: requestorType, client: client, namespace: namespace, callbacks: callbacks, wgDataPath: sync.WaitGroup{}, log: log, } return fs } func (fs *fileSystemBR) Init(ctx context.Context, param any) error { initParam := param.(*FSBRInitParam) var err error defer func() { if err != nil { fs.Close(ctx) } }() fs.ctx, fs.cancel = context.WithCancel(ctx) backupLocation := &velerov1api.BackupStorageLocation{} if err = fs.client.Get(ctx, client.ObjectKey{ Namespace: fs.namespace, Name: initParam.BSLName, }, backupLocation); err != nil { return errors.Wrapf(err, "error getting backup storage location %s", initParam.BSLName) } fs.backupLocation = backupLocation fs.backupRepo, err = initParam.RepositoryEnsurer.EnsureRepo(ctx, fs.namespace, initParam.SourceNamespace, initParam.BSLName, initParam.RepositoryType) if err != nil { return errors.Wrapf(err, "error to ensure backup repository %s-%s-%s", initParam.BSLName, initParam.SourceNamespace, initParam.RepositoryType) } err = fs.boostRepoConnect(ctx, initParam.RepositoryType, initParam.CredentialGetter, initParam.CacheDir) if err != nil { return errors.Wrapf(err, "error to boost backup repository connection %s-%s-%s", initParam.BSLName, initParam.SourceNamespace, initParam.RepositoryType) } fs.uploaderProv, err = provider.NewUploaderProvider(ctx, fs.client, initParam.UploaderType, fs.requestorType, initParam.RepoIdentifier, fs.backupLocation, fs.backupRepo, initParam.CredentialGetter, repokey.RepoKeySelector(), fs.log) if err != nil { return errors.Wrapf(err, "error creating uploader %s", initParam.UploaderType) } fs.initialized = true fs.log.WithFields( logrus.Fields{ "jobName": fs.jobName, "bsl": initParam.BSLName, "source namespace": initParam.SourceNamespace, "uploader": initParam.UploaderType, "repository": initParam.RepositoryType, }).Info("FileSystemBR is initialized") return nil } func (fs *fileSystemBR) Close(ctx context.Context) { if fs.cancel != nil { fs.cancel() } fs.log.WithField("user", fs.jobName).Info("Closing FileSystemBR") fs.wgDataPath.Wait() fs.close(ctx) fs.log.WithField("user", fs.jobName).Info("FileSystemBR is closed") } func (fs *fileSystemBR) close(ctx context.Context) { fs.dataPathLock.Lock() defer fs.dataPathLock.Unlock() if fs.uploaderProv != nil { if err := fs.uploaderProv.Close(ctx); err != nil { fs.log.Errorf("failed to close uploader provider with error %v", err) } fs.uploaderProv = nil } } func (fs *fileSystemBR) StartBackup(source AccessPoint, uploaderConfig map[string]string, param any) error { if !fs.initialized { return errors.New("file system data path is not initialized") } fs.wgDataPath.Add(1) backupParam := param.(*FSBRStartParam) go func() { fs.log.Info("Start data path backup") defer func() { fs.close(context.Background()) fs.wgDataPath.Done() }() snapshotID, emptySnapshot, totalBytes, incrementalBytes, err := fs.uploaderProv.RunBackup(fs.ctx, source.ByPath, backupParam.RealSource, backupParam.Tags, backupParam.ForceFull, backupParam.ParentSnapshot, source.VolMode, uploaderConfig, fs) if err == provider.ErrorCanceled { fs.callbacks.OnCancelled(context.Background(), fs.namespace, fs.jobName) } else if err != nil { dataPathErr := DataPathError{ snapshotID: snapshotID, err: err, } fs.callbacks.OnFailed(context.Background(), fs.namespace, fs.jobName, dataPathErr) } else { fs.callbacks.OnCompleted(context.Background(), fs.namespace, fs.jobName, Result{Backup: BackupResult{snapshotID, emptySnapshot, source, totalBytes, incrementalBytes}}) } }() return nil } func (fs *fileSystemBR) StartRestore(snapshotID string, target AccessPoint, uploaderConfigs map[string]string) error { if !fs.initialized { return errors.New("file system data path is not initialized") } fs.wgDataPath.Add(1) go func() { fs.log.Info("Start data path restore") defer func() { fs.close(context.Background()) fs.wgDataPath.Done() }() totalBytes, err := fs.uploaderProv.RunRestore(fs.ctx, snapshotID, target.ByPath, target.VolMode, uploaderConfigs, fs) if err == provider.ErrorCanceled { fs.callbacks.OnCancelled(context.Background(), fs.namespace, fs.jobName) } else if err != nil { dataPathErr := DataPathError{ snapshotID: snapshotID, err: err, } fs.callbacks.OnFailed(context.Background(), fs.namespace, fs.jobName, dataPathErr) } else { fs.callbacks.OnCompleted(context.Background(), fs.namespace, fs.jobName, Result{Restore: RestoreResult{Target: target, TotalBytes: totalBytes}}) } }() return nil } // UpdateProgress which implement ProgressUpdater interface to update progress status func (fs *fileSystemBR) UpdateProgress(p *uploader.Progress) { if fs.callbacks.OnProgress != nil { fs.callbacks.OnProgress(context.Background(), fs.namespace, fs.jobName, &uploader.Progress{TotalBytes: p.TotalBytes, BytesDone: p.BytesDone}) } } func (fs *fileSystemBR) Cancel() { fs.cancel() fs.log.WithField("user", fs.jobName).Info("FileSystemBR is canceled") } func (fs *fileSystemBR) boostRepoConnect(ctx context.Context, repositoryType string, credentialGetter *credentials.CredentialGetter, cacheDir string) error { if repositoryType == velerov1api.BackupRepositoryTypeKopia { if err := repoProvider.NewUnifiedRepoProvider(*credentialGetter, repositoryType, fs.log).BoostRepoConnect(ctx, repoProvider.RepoParam{BackupLocation: fs.backupLocation, BackupRepo: fs.backupRepo, CacheDir: cacheDir}); err != nil { return err } } else { if err := repoProvider.NewResticRepositoryProvider(*credentialGetter, filesystem.NewFileSystem(), fs.log).BoostRepoConnect(ctx, repoProvider.RepoParam{BackupLocation: fs.backupLocation, BackupRepo: fs.backupRepo}); err != nil { return err } } return nil } ================================================ FILE: pkg/datapath/file_system_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package datapath import ( "context" "testing" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" velerotest "github.com/vmware-tanzu/velero/pkg/test" "github.com/vmware-tanzu/velero/pkg/uploader/provider" providerMock "github.com/vmware-tanzu/velero/pkg/uploader/provider/mocks" ) func TestAsyncBackup(t *testing.T) { var asyncErr error var asyncResult Result finish := make(chan struct{}) var failErr = errors.New("fake-fail-error") tests := []struct { name string uploaderProv provider.Provider callbacks Callbacks err error result Result path string }{ { name: "async backup fail", callbacks: Callbacks{ OnCompleted: nil, OnCancelled: nil, OnFailed: func(ctx context.Context, namespace string, job string, err error) { asyncErr = failErr asyncResult = Result{} finish <- struct{}{} }, }, err: failErr, }, { name: "async backup cancel", callbacks: Callbacks{ OnCompleted: nil, OnFailed: nil, OnCancelled: func(ctx context.Context, namespace string, job string) { asyncErr = provider.ErrorCanceled asyncResult = Result{} finish <- struct{}{} }, }, err: provider.ErrorCanceled, }, { name: "async backup complete", callbacks: Callbacks{ OnFailed: nil, OnCancelled: nil, OnCompleted: func(ctx context.Context, namespace string, job string, result Result) { asyncResult = result asyncErr = nil finish <- struct{}{} }, }, result: Result{ Backup: BackupResult{ SnapshotID: "fake-snapshot", EmptySnapshot: false, Source: AccessPoint{ByPath: "fake-path"}, TotalBytes: 1000, }, }, path: "fake-path", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fs := newFileSystemBR("job-1", "test", nil, "velero", Callbacks{}, velerotest.NewLogger()).(*fileSystemBR) mockProvider := providerMock.NewProvider(t) mockProvider.On("RunBackup", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(test.result.Backup.SnapshotID, test.result.Backup.EmptySnapshot, test.result.Backup.TotalBytes, test.result.Backup.IncrementalBytes, test.err) mockProvider.On("Close", mock.Anything).Return(nil) fs.uploaderProv = mockProvider fs.initialized = true fs.callbacks = test.callbacks err := fs.StartBackup(AccessPoint{ByPath: test.path}, map[string]string{}, &FSBRStartParam{}) require.NoError(t, err) <-finish // Ensure the goroutine finishes so deferred fs.close executes, satisfying mock expectations. fs.wgDataPath.Wait() assert.Equal(t, test.err, asyncErr) assert.Equal(t, test.result, asyncResult) }) } close(finish) } func TestAsyncRestore(t *testing.T) { var asyncErr error var asyncResult Result finish := make(chan struct{}) var failErr = errors.New("fake-fail-error") tests := []struct { name string uploaderProv provider.Provider callbacks Callbacks err error result Result path string snapshot string }{ { name: "async restore fail", callbacks: Callbacks{ OnCompleted: nil, OnCancelled: nil, OnFailed: func(ctx context.Context, namespace string, job string, err error) { asyncErr = failErr asyncResult = Result{} finish <- struct{}{} }, }, err: failErr, }, { name: "async restore cancel", callbacks: Callbacks{ OnCompleted: nil, OnFailed: nil, OnCancelled: func(ctx context.Context, namespace string, job string) { asyncErr = provider.ErrorCanceled asyncResult = Result{} finish <- struct{}{} }, }, err: provider.ErrorCanceled, }, { name: "async restore complete", callbacks: Callbacks{ OnFailed: nil, OnCancelled: nil, OnCompleted: func(ctx context.Context, namespace string, job string, result Result) { asyncResult = result asyncErr = nil finish <- struct{}{} }, }, result: Result{ Restore: RestoreResult{ Target: AccessPoint{ByPath: "fake-path"}, TotalBytes: 1000, }, }, path: "fake-path", snapshot: "fake-snapshot", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fs := newFileSystemBR("job-1", "test", nil, "velero", Callbacks{}, velerotest.NewLogger()).(*fileSystemBR) mockProvider := providerMock.NewProvider(t) mockProvider.On("RunRestore", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(test.result.Restore.TotalBytes, test.err) mockProvider.On("Close", mock.Anything).Return(nil) fs.uploaderProv = mockProvider fs.initialized = true fs.callbacks = test.callbacks err := fs.StartRestore(test.snapshot, AccessPoint{ByPath: test.path}, map[string]string{}) require.NoError(t, err) <-finish // Ensure the goroutine finishes so deferred fs.close executes, satisfying mock expectations. fs.wgDataPath.Wait() assert.Equal(t, asyncErr, test.err) assert.Equal(t, asyncResult, test.result) }) } close(finish) } ================================================ FILE: pkg/datapath/manager.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package datapath import ( "context" "sync" "github.com/pkg/errors" "github.com/sirupsen/logrus" "k8s.io/client-go/kubernetes" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" ) var ConcurrentLimitExceed error = errors.New("Concurrent number exceeds") var FSBRCreator = newFileSystemBR var MicroServiceBRWatcherCreator = newMicroServiceBRWatcher type Manager struct { cocurrentNum int trackerLock sync.Mutex tracker map[string]AsyncBR } // NewManager creates the data path manager to manage concurrent data path instances func NewManager(cocurrentNum int) *Manager { return &Manager{ cocurrentNum: cocurrentNum, tracker: map[string]AsyncBR{}, } } // CreateFileSystemBR creates a new file system backup/restore data path instance func (m *Manager) CreateFileSystemBR(jobName string, requestorType string, ctx context.Context, client client.Client, namespace string, callbacks Callbacks, log logrus.FieldLogger) (AsyncBR, error) { m.trackerLock.Lock() defer m.trackerLock.Unlock() if len(m.tracker) >= m.cocurrentNum { return nil, ConcurrentLimitExceed } m.tracker[jobName] = FSBRCreator(jobName, requestorType, client, namespace, callbacks, log) return m.tracker[jobName], nil } // CreateMicroServiceBRWatcher creates a new micro service watcher instance func (m *Manager) CreateMicroServiceBRWatcher(ctx context.Context, client client.Client, kubeClient kubernetes.Interface, mgr manager.Manager, taskType string, taskName string, namespace string, podName string, containerName string, associatedObject string, callbacks Callbacks, resume bool, log logrus.FieldLogger) (AsyncBR, error) { m.trackerLock.Lock() defer m.trackerLock.Unlock() if !resume { if len(m.tracker) >= m.cocurrentNum { return nil, ConcurrentLimitExceed } } m.tracker[taskName] = MicroServiceBRWatcherCreator(client, kubeClient, mgr, taskType, taskName, namespace, podName, containerName, associatedObject, callbacks, log) return m.tracker[taskName], nil } // RemoveAsyncBR removes a file system backup/restore data path instance func (m *Manager) RemoveAsyncBR(jobName string) { m.trackerLock.Lock() defer m.trackerLock.Unlock() delete(m.tracker, jobName) } // GetAsyncBR returns the file system backup/restore data path instance for the specified job name func (m *Manager) GetAsyncBR(jobName string) AsyncBR { m.trackerLock.Lock() defer m.trackerLock.Unlock() if async, exist := m.tracker[jobName]; exist { return async } else { return nil } } ================================================ FILE: pkg/datapath/manager_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package datapath import ( "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestCreateFileSystemBR(t *testing.T) { m := NewManager(2) async_job_1, err := m.CreateFileSystemBR("job-1", "test", t.Context(), nil, "velero", Callbacks{}, nil) require.NoError(t, err) _, err = m.CreateFileSystemBR("job-2", "test", t.Context(), nil, "velero", Callbacks{}, nil) require.NoError(t, err) _, err = m.CreateFileSystemBR("job-3", "test", t.Context(), nil, "velero", Callbacks{}, nil) assert.Equal(t, ConcurrentLimitExceed, err) ret := m.GetAsyncBR("job-0") assert.Nil(t, ret) ret = m.GetAsyncBR("job-1") assert.Equal(t, async_job_1, ret) m.RemoveAsyncBR("job-0") assert.Len(t, m.tracker, 2) m.RemoveAsyncBR("job-1") assert.Len(t, m.tracker, 1) ret = m.GetAsyncBR("job-1") assert.Nil(t, ret) } func TestCreateMicroServiceBRWatcher(t *testing.T) { m := NewManager(2) async_job_1, err := m.CreateMicroServiceBRWatcher(t.Context(), nil, nil, nil, "test", "job-1", "velero", "pod-1", "container", "du-1", Callbacks{}, false, nil) require.NoError(t, err) _, err = m.CreateMicroServiceBRWatcher(t.Context(), nil, nil, nil, "test", "job-2", "velero", "pod-2", "container", "du-2", Callbacks{}, false, nil) require.NoError(t, err) _, err = m.CreateMicroServiceBRWatcher(t.Context(), nil, nil, nil, "test", "job-3", "velero", "pod-3", "container", "du-3", Callbacks{}, false, nil) assert.Equal(t, ConcurrentLimitExceed, err) async_job_4, err := m.CreateMicroServiceBRWatcher(t.Context(), nil, nil, nil, "test", "job-4", "velero", "pod-4", "container", "du-4", Callbacks{}, true, nil) require.NoError(t, err) ret := m.GetAsyncBR("job-0") assert.Nil(t, ret) ret = m.GetAsyncBR("job-1") assert.Equal(t, async_job_1, ret) ret = m.GetAsyncBR("job-4") assert.Equal(t, async_job_4, ret) m.RemoveAsyncBR("job-0") assert.Len(t, m.tracker, 3) m.RemoveAsyncBR("job-1") assert.Len(t, m.tracker, 2) ret = m.GetAsyncBR("job-1") assert.Nil(t, ret) } ================================================ FILE: pkg/datapath/micro_service_watcher.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package datapath import ( "context" "encoding/json" "os" "strings" "sync" "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/pkg/uploader" "github.com/vmware-tanzu/velero/pkg/util/kube" ctrlcache "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/manager" "github.com/vmware-tanzu/velero/pkg/util/logging" ) const ( TaskTypeBackup = "backup" TaskTypeRestore = "restore" ErrCancelled = "data path is canceled" EventReasonStarted = "Data-Path-Started" EventReasonCompleted = "Data-Path-Completed" EventReasonFailed = "Data-Path-Failed" EventReasonCancelled = "Data-Path-Canceled" EventReasonProgress = "Data-Path-Progress" EventReasonCancelling = "Data-Path-Canceling" EventReasonStopped = "Data-Path-Stopped" ) type microServiceBRWatcher struct { ctx context.Context cancel context.CancelFunc log logrus.FieldLogger client client.Client kubeClient kubernetes.Interface mgr manager.Manager namespace string callbacks Callbacks taskName string taskType string thisPod string thisContainer string associatedObject string eventCh chan *corev1api.Event podCh chan *corev1api.Pod startedFromEvent bool terminatedFromEvent bool wgWatcher sync.WaitGroup eventInformer ctrlcache.Informer podInformer ctrlcache.Informer eventHandler cache.ResourceEventHandlerRegistration podHandler cache.ResourceEventHandlerRegistration watcherLock sync.Mutex } func newMicroServiceBRWatcher(client client.Client, kubeClient kubernetes.Interface, mgr manager.Manager, taskType string, taskName string, namespace string, podName string, containerName string, associatedObject string, callbacks Callbacks, log logrus.FieldLogger) AsyncBR { ms := µServiceBRWatcher{ mgr: mgr, client: client, kubeClient: kubeClient, namespace: namespace, callbacks: callbacks, taskType: taskType, taskName: taskName, thisPod: podName, thisContainer: containerName, associatedObject: associatedObject, eventCh: make(chan *corev1api.Event, 10), podCh: make(chan *corev1api.Pod, 2), wgWatcher: sync.WaitGroup{}, log: log, } return ms } func (ms *microServiceBRWatcher) Init(ctx context.Context, param any) error { eventInformer, err := ms.mgr.GetCache().GetInformer(ctx, &corev1api.Event{}) if err != nil { return errors.Wrap(err, "error getting event informer") } podInformer, err := ms.mgr.GetCache().GetInformer(ctx, &corev1api.Pod{}) if err != nil { return errors.Wrap(err, "error getting pod informer") } eventHandler, err := eventInformer.AddEventHandler( cache.ResourceEventHandlerFuncs{ AddFunc: func(obj any) { evt := obj.(*corev1api.Event) if evt.InvolvedObject.Namespace != ms.namespace || evt.InvolvedObject.Name != ms.associatedObject { return } ms.eventCh <- evt }, UpdateFunc: func(_, obj any) { evt := obj.(*corev1api.Event) if evt.InvolvedObject.Namespace != ms.namespace || evt.InvolvedObject.Name != ms.associatedObject { return } ms.eventCh <- evt }, }, ) if err != nil { return errors.Wrap(err, "error registering event handler") } podHandler, err := podInformer.AddEventHandler( cache.ResourceEventHandlerFuncs{ UpdateFunc: func(_, obj any) { pod := obj.(*corev1api.Pod) if pod.Namespace != ms.namespace || pod.Name != ms.thisPod { return } if pod.Status.Phase == corev1api.PodSucceeded || pod.Status.Phase == corev1api.PodFailed { ms.podCh <- pod } }, }, ) if err != nil { return errors.Wrap(err, "error registering pod handler") } if err := ms.reEnsureThisPod(ctx); err != nil { return err } ms.eventInformer = eventInformer ms.podInformer = podInformer ms.eventHandler = eventHandler ms.podHandler = podHandler ms.ctx, ms.cancel = context.WithCancel(ctx) ms.log.WithFields( logrus.Fields{ "taskType": ms.taskType, "taskName": ms.taskName, "thisPod": ms.thisPod, }).Info("MicroServiceBR is initialized") return nil } func (ms *microServiceBRWatcher) Close(ctx context.Context) { if ms.cancel != nil { ms.cancel() } ms.log.WithField("taskType", ms.taskType).WithField("taskName", ms.taskName).Info("Closing MicroServiceBR") ms.wgWatcher.Wait() ms.close() ms.log.WithField("taskType", ms.taskType).WithField("taskName", ms.taskName).Info("MicroServiceBR is closed") } func (ms *microServiceBRWatcher) close() { ms.watcherLock.Lock() defer ms.watcherLock.Unlock() if ms.eventHandler != nil { if err := ms.eventInformer.RemoveEventHandler(ms.eventHandler); err != nil { ms.log.WithError(err).Warn("Failed to remove event handler") } ms.eventHandler = nil } if ms.podHandler != nil { if err := ms.podInformer.RemoveEventHandler(ms.podHandler); err != nil { ms.log.WithError(err).Warn("Failed to remove pod handler") } ms.podHandler = nil } } func (ms *microServiceBRWatcher) StartBackup(source AccessPoint, uploaderConfig map[string]string, param any) error { ms.log.Infof("Start watching backup ms for source %v", source.ByPath) ms.startWatch() return nil } func (ms *microServiceBRWatcher) StartRestore(snapshotID string, target AccessPoint, uploaderConfigs map[string]string) error { ms.log.Infof("Start watching restore ms to target %s, from snapshot %s", target.ByPath, snapshotID) ms.startWatch() return nil } func (ms *microServiceBRWatcher) reEnsureThisPod(ctx context.Context) error { thisPod := &corev1api.Pod{} if err := ms.client.Get(ctx, types.NamespacedName{ Namespace: ms.namespace, Name: ms.thisPod, }, thisPod); err != nil { return errors.Wrapf(err, "error getting this pod %s", ms.thisPod) } if thisPod.Status.Phase == corev1api.PodSucceeded || thisPod.Status.Phase == corev1api.PodFailed { ms.podCh <- thisPod ms.log.WithField("this pod", ms.thisPod).Infof("This pod comes to terminital status %s before watch start", thisPod.Status.Phase) } return nil } var funcGetPodTerminationMessage = kube.GetPodContainerTerminateMessage var funcRedirectLog = redirectDataMoverLogs var funcGetResultFromMessage = getResultFromMessage var funcGetProgressFromMessage = getProgressFromMessage var eventWaitTimeout = time.Minute func (ms *microServiceBRWatcher) startWatch() { ms.wgWatcher.Add(1) go func() { ms.log.Info("Start watching data path pod") defer func() { ms.close() ms.wgWatcher.Done() }() var lastPod *corev1api.Pod watchLoop: for { select { case <-ms.ctx.Done(): break watchLoop case pod := <-ms.podCh: lastPod = pod break watchLoop case evt := <-ms.eventCh: ms.onEvent(evt) } } if lastPod == nil { ms.log.Warn("Watch loop is canceled on waiting data path pod") return } epilogLoop: for !ms.startedFromEvent || !ms.terminatedFromEvent { select { case <-ms.ctx.Done(): ms.log.Warn("Watch loop is canceled on waiting final event") return case <-time.After(eventWaitTimeout): break epilogLoop case evt := <-ms.eventCh: ms.onEvent(evt) } } terminateMessage := funcGetPodTerminationMessage(lastPod, ms.thisContainer) logger := ms.log.WithField("data path pod", lastPod.Name) logger.Infof("Finish waiting data path pod, phase %s, message %s", lastPod.Status.Phase, terminateMessage) if !ms.startedFromEvent { logger.Warn("VGDP seems not started") } if ms.startedFromEvent && !ms.terminatedFromEvent { logger.Warn("VGDP started but termination event is not received") } logger.Info("Recording data path pod logs") if err := funcRedirectLog(ms.ctx, ms.kubeClient, ms.namespace, lastPod.Name, ms.thisContainer, ms.log); err != nil { logger.WithError(err).Warn("Failed to collect data mover logs") } logger.Info("Calling callback on data path pod termination") if lastPod.Status.Phase == corev1api.PodSucceeded { result := funcGetResultFromMessage(ms.taskType, terminateMessage, ms.log) ms.callbacks.OnProgress(ms.ctx, ms.namespace, ms.taskName, getCompletionProgressFromResult(ms.taskType, result)) ms.callbacks.OnCompleted(ms.ctx, ms.namespace, ms.taskName, result) } else { if strings.HasSuffix(terminateMessage, ErrCancelled) { ms.callbacks.OnCancelled(ms.ctx, ms.namespace, ms.taskName) } else { ms.callbacks.OnFailed(ms.ctx, ms.namespace, ms.taskName, errors.New(terminateMessage)) } } logger.Info("Complete callback on data path pod termination") }() } func (ms *microServiceBRWatcher) onEvent(evt *corev1api.Event) { switch evt.Reason { case EventReasonStarted: ms.startedFromEvent = true ms.log.Infof("Received data path start message: %s", evt.Message) case EventReasonProgress: ms.callbacks.OnProgress(ms.ctx, ms.namespace, ms.taskName, funcGetProgressFromMessage(evt.Message, ms.log)) case EventReasonCompleted: ms.log.Infof("Received data path completed message: %v", funcGetResultFromMessage(ms.taskType, evt.Message, ms.log)) case EventReasonCancelled: ms.log.Infof("Received data path canceled message: %s", evt.Message) case EventReasonFailed: ms.log.Infof("Received data path failed message: %s", evt.Message) case EventReasonCancelling: ms.log.Infof("Received data path canceling message: %s", evt.Message) case EventReasonStopped: ms.terminatedFromEvent = true ms.log.Infof("Received data path stop message: %s", evt.Message) default: ms.log.Infof("Received event for data path %s, reason: %s, message: %s", ms.taskName, evt.Reason, evt.Message) } } func getResultFromMessage(taskType string, message string, logger logrus.FieldLogger) Result { result := Result{} if taskType == TaskTypeBackup { backupResult := BackupResult{} err := json.Unmarshal([]byte(message), &backupResult) if err != nil { logger.WithError(err).Errorf("Failed to unmarshal result message %s", message) } else { result.Backup = backupResult } } else { restoreResult := RestoreResult{} err := json.Unmarshal([]byte(message), &restoreResult) if err != nil { logger.WithError(err).Errorf("Failed to unmarshal result message %s", message) } else { result.Restore = restoreResult } } return result } func getProgressFromMessage(message string, logger logrus.FieldLogger) *uploader.Progress { progress := &uploader.Progress{} err := json.Unmarshal([]byte(message), progress) if err != nil { logger.WithError(err).Debugf("Failed to unmarshal progress message %s", message) } return progress } func getCompletionProgressFromResult(taskType string, result Result) *uploader.Progress { progress := &uploader.Progress{} if taskType == TaskTypeBackup { progress.BytesDone = result.Backup.TotalBytes progress.TotalBytes = result.Backup.TotalBytes } else { progress.BytesDone = result.Restore.TotalBytes progress.TotalBytes = result.Restore.TotalBytes } return progress } func (ms *microServiceBRWatcher) Cancel() { ms.log.WithField("taskType", ms.taskType).WithField("taskName", ms.taskName).Info("MicroServiceBR is canceled") } var funcCreateTemp = os.CreateTemp var funcCollectPodLogs = kube.CollectPodLogs func redirectDataMoverLogs(ctx context.Context, kubeClient kubernetes.Interface, namespace string, thisPod string, thisContainer string, logger logrus.FieldLogger) error { logger.Infof("Starting to collect data mover pod log for %s", thisPod) logFile, err := funcCreateTemp("", "") if err != nil { return errors.Wrap(err, "error to create temp file for data mover pod log") } defer logFile.Close() logFileName := logFile.Name() logger.Infof("Created log file %s", logFileName) err = funcCollectPodLogs(ctx, kubeClient.CoreV1(), thisPod, namespace, thisContainer, logFile) if err != nil { return errors.Wrapf(err, "error to collect logs to %s for data mover pod %s", logFileName, thisPod) } logFile.Close() logger.Infof("Redirecting to log file %s", logFileName) hookLogger := logger.WithField(logging.LogSourceKey, logFileName) hookLogger.Logln(logging.ListeningLevel, logging.ListeningMessage) logger.Infof("Completed to collect data mover pod log for %s", thisPod) return nil } ================================================ FILE: pkg/datapath/micro_service_watcher_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package datapath import ( "context" "errors" "fmt" "io" "os" "path" "testing" "time" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes" kubeclientfake "k8s.io/client-go/kubernetes/fake" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" "sigs.k8s.io/controller-runtime/pkg/client/fake" "github.com/vmware-tanzu/velero/pkg/builder" velerotest "github.com/vmware-tanzu/velero/pkg/test" "github.com/vmware-tanzu/velero/pkg/uploader" "github.com/vmware-tanzu/velero/pkg/util/logging" ) func TestReEnsureThisPod(t *testing.T) { tests := []struct { name string namespace string thisPod string kubeClientObj []runtime.Object expectChan bool expectErr string }{ { name: "get pod error", thisPod: "fak-pod-1", expectErr: "error getting this pod fak-pod-1: pods \"fak-pod-1\" not found", }, { name: "get pod not in terminated state", namespace: "velero", thisPod: "fake-pod-1", kubeClientObj: []runtime.Object{ builder.ForPod("velero", "fake-pod-1").Phase(corev1api.PodRunning).Result(), }, }, { name: "get pod succeed state", namespace: "velero", thisPod: "fake-pod-1", kubeClientObj: []runtime.Object{ builder.ForPod("velero", "fake-pod-1").Phase(corev1api.PodSucceeded).Result(), }, expectChan: true, }, { name: "get pod failed state", namespace: "velero", thisPod: "fake-pod-1", kubeClientObj: []runtime.Object{ builder.ForPod("velero", "fake-pod-1").Phase(corev1api.PodFailed).Result(), }, expectChan: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { scheme := runtime.NewScheme() corev1api.AddToScheme(scheme) fakeClientBuilder := fake.NewClientBuilder() fakeClientBuilder = fakeClientBuilder.WithScheme(scheme) fakeClient := fakeClientBuilder.WithRuntimeObjects(test.kubeClientObj...).Build() ms := µServiceBRWatcher{ namespace: test.namespace, thisPod: test.thisPod, client: fakeClient, podCh: make(chan *corev1api.Pod, 2), log: velerotest.NewLogger(), } err := ms.reEnsureThisPod(t.Context()) if test.expectErr != "" { assert.EqualError(t, err, test.expectErr) } else { if test.expectChan { assert.Len(t, ms.podCh, 1) pod := <-ms.podCh assert.Equal(t, pod.Name, test.thisPod) } } }) } } type startWatchFake struct { terminationMessage string redirectErr error complete bool failed bool canceled bool progress int } func (sw *startWatchFake) getPodContainerTerminateMessage(pod *corev1api.Pod, container string) string { return sw.terminationMessage } func (sw *startWatchFake) redirectDataMoverLogs(ctx context.Context, kubeClient kubernetes.Interface, namespace string, thisPod string, thisContainer string, logger logrus.FieldLogger) error { return sw.redirectErr } func (sw *startWatchFake) getResultFromMessage(_ string, _ string, _ logrus.FieldLogger) Result { return Result{} } func (sw *startWatchFake) OnCompleted(ctx context.Context, namespace string, task string, result Result) { sw.complete = true } func (sw *startWatchFake) OnFailed(ctx context.Context, namespace string, task string, err error) { sw.failed = true } func (sw *startWatchFake) OnCancelled(ctx context.Context, namespace string, task string) { sw.canceled = true } func (sw *startWatchFake) OnProgress(ctx context.Context, namespace string, task string, progress *uploader.Progress) { sw.progress++ } type insertEvent struct { event *corev1api.Event after time.Duration delay time.Duration } func TestStartWatch(t *testing.T) { tests := []struct { name string namespace string thisPod string thisContainer string terminationMessage string redirectLogErr error insertPod *corev1api.Pod insertEventsBefore []insertEvent insertEventsAfter []insertEvent ctxCancel bool expectStartEvent bool expectTerminateEvent bool expectComplete bool expectCancel bool expectFail bool expectProgress int }{ { name: "exit from ctx", thisPod: "fak-pod-1", thisContainer: "fake-container-1", ctxCancel: true, }, { name: "completed with rantional sequence", thisPod: "fak-pod-1", thisContainer: "fake-container-1", insertPod: builder.ForPod("velero", "fake-pod-1").Phase(corev1api.PodSucceeded).Result(), insertEventsBefore: []insertEvent{ { event: &corev1api.Event{Reason: EventReasonStarted}, }, { event: &corev1api.Event{Reason: EventReasonCompleted}, }, { event: &corev1api.Event{Reason: EventReasonStopped}, delay: time.Second, }, }, expectStartEvent: true, expectTerminateEvent: true, expectComplete: true, expectProgress: 1, }, { name: "completed", thisPod: "fak-pod-1", thisContainer: "fake-container-1", insertPod: builder.ForPod("velero", "fake-pod-1").Phase(corev1api.PodSucceeded).Result(), insertEventsBefore: []insertEvent{ { event: &corev1api.Event{Reason: EventReasonStarted}, }, { event: &corev1api.Event{Reason: EventReasonCompleted}, }, { event: &corev1api.Event{Reason: EventReasonStopped}, }, }, expectStartEvent: true, expectTerminateEvent: true, expectComplete: true, expectProgress: 1, }, { name: "completed with redirect error", thisPod: "fak-pod-1", thisContainer: "fake-container-1", insertPod: builder.ForPod("velero", "fake-pod-1").Phase(corev1api.PodSucceeded).Result(), insertEventsBefore: []insertEvent{ { event: &corev1api.Event{Reason: EventReasonStarted}, }, { event: &corev1api.Event{Reason: EventReasonCompleted}, }, { event: &corev1api.Event{Reason: EventReasonStopped}, }, }, redirectLogErr: errors.New("fake-error"), expectStartEvent: true, expectTerminateEvent: true, expectComplete: true, expectProgress: 1, }, { name: "complete but terminated event not received in time", thisPod: "fak-pod-1", thisContainer: "fake-container-1", insertPod: builder.ForPod("velero", "fake-pod-1").Phase(corev1api.PodSucceeded).Result(), insertEventsBefore: []insertEvent{ { event: &corev1api.Event{Reason: EventReasonStarted}, }, }, insertEventsAfter: []insertEvent{ { event: &corev1api.Event{Reason: EventReasonStarted}, after: time.Second * 6, }, }, expectStartEvent: true, expectComplete: true, expectProgress: 1, }, { name: "complete but terminated event not received immediately", thisPod: "fak-pod-1", thisContainer: "fake-container-1", insertPod: builder.ForPod("velero", "fake-pod-1").Phase(corev1api.PodSucceeded).Result(), insertEventsBefore: []insertEvent{ { event: &corev1api.Event{Reason: EventReasonStarted}, }, }, insertEventsAfter: []insertEvent{ { event: &corev1api.Event{Reason: EventReasonCompleted}, }, { event: &corev1api.Event{Reason: EventReasonStopped}, delay: time.Second, }, }, expectStartEvent: true, expectTerminateEvent: true, expectComplete: true, expectProgress: 1, }, { name: "completed with progress", thisPod: "fak-pod-1", thisContainer: "fake-container-1", insertPod: builder.ForPod("velero", "fake-pod-1").Phase(corev1api.PodSucceeded).Result(), insertEventsBefore: []insertEvent{ { event: &corev1api.Event{Reason: EventReasonStarted}, }, { event: &corev1api.Event{Reason: EventReasonProgress, Message: "fake-progress-1"}, }, { event: &corev1api.Event{Reason: EventReasonProgress, Message: "fake-progress-2"}, }, { event: &corev1api.Event{Reason: EventReasonCompleted}, }, { event: &corev1api.Event{Reason: EventReasonStopped}, delay: time.Second, }, }, expectStartEvent: true, expectTerminateEvent: true, expectComplete: true, expectProgress: 3, }, { name: "failed", thisPod: "fak-pod-1", thisContainer: "fake-container-1", insertPod: builder.ForPod("velero", "fake-pod-1").Phase(corev1api.PodFailed).Result(), insertEventsBefore: []insertEvent{ { event: &corev1api.Event{Reason: EventReasonStarted}, }, { event: &corev1api.Event{Reason: EventReasonCancelled}, }, { event: &corev1api.Event{Reason: EventReasonStopped}, }, }, terminationMessage: "fake-termination-message-1", expectStartEvent: true, expectTerminateEvent: true, expectFail: true, }, { name: "pod crash", thisPod: "fak-pod-1", thisContainer: "fake-container-1", insertPod: builder.ForPod("velero", "fake-pod-1").Phase(corev1api.PodFailed).Result(), terminationMessage: "fake-termination-message-2", expectFail: true, }, { name: "canceled", thisPod: "fak-pod-1", thisContainer: "fake-container-1", insertPod: builder.ForPod("velero", "fake-pod-1").Phase(corev1api.PodFailed).Result(), insertEventsBefore: []insertEvent{ { event: &corev1api.Event{Reason: EventReasonStarted}, }, { event: &corev1api.Event{Reason: EventReasonCancelled}, }, { event: &corev1api.Event{Reason: EventReasonStopped}, }, }, terminationMessage: fmt.Sprintf("Failed to init data path service for DataUpload %s: %v", "fake-du-name", errors.New(ErrCancelled)), expectStartEvent: true, expectTerminateEvent: true, expectCancel: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { ctx, cancel := context.WithCancel(t.Context()) eventWaitTimeout = time.Second * 5 sw := startWatchFake{ terminationMessage: test.terminationMessage, redirectErr: test.redirectLogErr, } funcGetPodTerminationMessage = sw.getPodContainerTerminateMessage funcRedirectLog = sw.redirectDataMoverLogs funcGetResultFromMessage = sw.getResultFromMessage ms := µServiceBRWatcher{ ctx: ctx, namespace: test.namespace, thisPod: test.thisPod, thisContainer: test.thisContainer, podCh: make(chan *corev1api.Pod, 2), eventCh: make(chan *corev1api.Event, 10), log: velerotest.NewLogger(), callbacks: Callbacks{ OnCompleted: sw.OnCompleted, OnFailed: sw.OnFailed, OnCancelled: sw.OnCancelled, OnProgress: sw.OnProgress, }, } ms.startWatch() if test.ctxCancel { cancel() } for _, ev := range test.insertEventsBefore { if ev.after != 0 { time.Sleep(ev.after) } ms.eventCh <- ev.event if ev.delay != 0 { time.Sleep(ev.delay) } } if test.insertPod != nil { ms.podCh <- test.insertPod } for _, ev := range test.insertEventsAfter { if ev.after != 0 { time.Sleep(ev.after) } ms.eventCh <- ev.event if ev.delay != 0 { time.Sleep(ev.delay) } } ms.wgWatcher.Wait() assert.Equal(t, test.expectStartEvent, ms.startedFromEvent) assert.Equal(t, test.expectTerminateEvent, ms.terminatedFromEvent) assert.Equal(t, test.expectComplete, sw.complete) assert.Equal(t, test.expectCancel, sw.canceled) assert.Equal(t, test.expectFail, sw.failed) assert.Equal(t, test.expectProgress, sw.progress) cancel() }) } } func TestGetResultFromMessage(t *testing.T) { tests := []struct { name string taskType string message string expectResult Result }{ { name: "error to unmarshall backup result", taskType: TaskTypeBackup, message: "fake-message", expectResult: Result{}, }, { name: "error to unmarshall restore result", taskType: TaskTypeRestore, message: "fake-message", expectResult: Result{}, }, { name: "succeed to unmarshall backup result", taskType: TaskTypeBackup, message: "{\"snapshotID\":\"fake-snapshot-id\",\"emptySnapshot\":true,\"source\":{\"byPath\":\"fake-path-1\",\"volumeMode\":\"Block\"}}", expectResult: Result{ Backup: BackupResult{ SnapshotID: "fake-snapshot-id", EmptySnapshot: true, Source: AccessPoint{ ByPath: "fake-path-1", VolMode: uploader.PersistentVolumeBlock, }, }, }, }, { name: "succeed to unmarshall restore result", taskType: TaskTypeRestore, message: "{\"target\":{\"byPath\":\"fake-path-2\",\"volumeMode\":\"Filesystem\"}}", expectResult: Result{ Restore: RestoreResult{ Target: AccessPoint{ ByPath: "fake-path-2", VolMode: uploader.PersistentVolumeFilesystem, }, }, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { result := getResultFromMessage(test.taskType, test.message, velerotest.NewLogger()) assert.Equal(t, test.expectResult, result) }) } } func TestGetProgressFromMessage(t *testing.T) { tests := []struct { name string message string expectProgress uploader.Progress }{ { name: "error to unmarshall progress", message: "fake-message", expectProgress: uploader.Progress{}, }, { name: "succeed to unmarshall progress", message: "{\"totalBytes\":1000,\"doneBytes\":200}", expectProgress: uploader.Progress{ TotalBytes: 1000, BytesDone: 200, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { progress := getProgressFromMessage(test.message, velerotest.NewLogger()) assert.Equal(t, test.expectProgress, *progress) }) } } type redirectFake struct { logFile *os.File createTempErr error getPodLogErr error logMessage string } func (rf *redirectFake) fakeCreateTempFile(_ string, _ string) (*os.File, error) { if rf.createTempErr != nil { return nil, rf.createTempErr } return rf.logFile, nil } func (rf *redirectFake) fakeCollectPodLogs(_ context.Context, _ corev1client.CoreV1Interface, _ string, _ string, _ string, output io.Writer) error { if rf.getPodLogErr != nil { return rf.getPodLogErr } _, err := output.Write([]byte(rf.logMessage)) return err } func TestRedirectDataMoverLogs(t *testing.T) { logFileName := path.Join(os.TempDir(), "test-logger-file.log") var buffer string tests := []struct { name string thisPod string logMessage string logger logrus.FieldLogger createTempErr error collectLogErr error expectErr string }{ { name: "error to create temp file", thisPod: "fake-pod", createTempErr: errors.New("fake-create-temp-error"), logger: velerotest.NewLogger(), expectErr: "error to create temp file for data mover pod log: fake-create-temp-error", }, { name: "error to collect pod log", thisPod: "fake-pod", collectLogErr: errors.New("fake-collect-log-error"), logger: velerotest.NewLogger(), expectErr: fmt.Sprintf("error to collect logs to %s for data mover pod fake-pod: fake-collect-log-error", logFileName), }, { name: "succeed", thisPod: "fake-pod", logMessage: "fake-log-message-01\nfake-log-message-02\nfake-log-message-03\n", logger: velerotest.NewSingleLoggerWithHooks(&buffer, logging.DefaultHooks(true)), }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { buffer = "" logFile, err := os.Create(logFileName) require.NoError(t, err) rf := redirectFake{ logFile: logFile, createTempErr: test.createTempErr, getPodLogErr: test.collectLogErr, logMessage: test.logMessage, } funcCreateTemp = rf.fakeCreateTempFile funcCollectPodLogs = rf.fakeCollectPodLogs fakeKubeClient := kubeclientfake.NewSimpleClientset() err = redirectDataMoverLogs(t.Context(), fakeKubeClient, "", test.thisPod, "", test.logger) if test.expectErr != "" { assert.EqualError(t, err, test.expectErr) } else { require.NoError(t, err) assert.Contains(t, buffer, test.logMessage) } }) } } ================================================ FILE: pkg/datapath/mocks/asyncBR.go ================================================ // Code generated by mockery v2.39.1. DO NOT EDIT. package mocks import ( context "context" mock "github.com/stretchr/testify/mock" datapath "github.com/vmware-tanzu/velero/pkg/datapath" ) // AsyncBR is an autogenerated mock type for the AsyncBR type type AsyncBR struct { mock.Mock } // Cancel provides a mock function with given fields: func (_m *AsyncBR) Cancel() { _m.Called() } // Close provides a mock function with given fields: ctx func (_m *AsyncBR) Close(ctx context.Context) { _m.Called(ctx) } // Init provides a mock function with given fields: ctx, param func (_m *AsyncBR) Init(ctx context.Context, param interface{}) error { ret := _m.Called(ctx, param) if len(ret) == 0 { panic("no return value specified for Init") } var r0 error if rf, ok := ret.Get(0).(func(context.Context, interface{}) error); ok { r0 = rf(ctx, param) } else { r0 = ret.Error(0) } return r0 } // StartBackup provides a mock function with given fields: source, dataMoverConfig, param func (_m *AsyncBR) StartBackup(source datapath.AccessPoint, dataMoverConfig map[string]string, param interface{}) error { ret := _m.Called(source, dataMoverConfig, param) if len(ret) == 0 { panic("no return value specified for StartBackup") } var r0 error if rf, ok := ret.Get(0).(func(datapath.AccessPoint, map[string]string, interface{}) error); ok { r0 = rf(source, dataMoverConfig, param) } else { r0 = ret.Error(0) } return r0 } // StartRestore provides a mock function with given fields: snapshotID, target, dataMoverConfig func (_m *AsyncBR) StartRestore(snapshotID string, target datapath.AccessPoint, dataMoverConfig map[string]string) error { ret := _m.Called(snapshotID, target, dataMoverConfig) if len(ret) == 0 { panic("no return value specified for StartRestore") } var r0 error if rf, ok := ret.Get(0).(func(string, datapath.AccessPoint, map[string]string) error); ok { r0 = rf(snapshotID, target, dataMoverConfig) } else { r0 = ret.Error(0) } return r0 } // NewAsyncBR creates a new instance of AsyncBR. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. func NewAsyncBR(t interface { mock.TestingT Cleanup(func()) }) *AsyncBR { mock := &AsyncBR{} mock.Mock.Test(t) t.Cleanup(func() { mock.AssertExpectations(t) }) return mock } ================================================ FILE: pkg/datapath/types.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package datapath import ( "context" "github.com/vmware-tanzu/velero/pkg/uploader" ) // Result represents the result of a backup/restore type Result struct { Backup BackupResult Restore RestoreResult } // BackupResult represents the result of a backup type BackupResult struct { SnapshotID string `json:"snapshotID"` EmptySnapshot bool `json:"emptySnapshot"` Source AccessPoint `json:"source,omitempty"` TotalBytes int64 `json:"totalBytes,omitempty"` IncrementalBytes int64 `json:"incrementalBytes,omitempty"` } // RestoreResult represents the result of a restore type RestoreResult struct { Target AccessPoint `json:"target,omitempty"` TotalBytes int64 `json:"totalBytes,omitempty"` } // Callbacks defines the collection of callbacks during backup/restore type Callbacks struct { OnCompleted func(context.Context, string, string, Result) OnFailed func(context.Context, string, string, error) OnCancelled func(context.Context, string, string) OnProgress func(context.Context, string, string, *uploader.Progress) } // AccessPoint represents an access point that has been exposed to a data path instance type AccessPoint struct { ByPath string `json:"byPath"` VolMode uploader.PersistentVolumeMode `json:"volumeMode"` } // AsyncBR is the interface for asynchronous data path methods type AsyncBR interface { // Init initializes an asynchronous data path instance Init(ctx context.Context, param any) error // StartBackup starts an asynchronous data path instance for backup StartBackup(source AccessPoint, dataMoverConfig map[string]string, param any) error // StartRestore starts an asynchronous data path instance for restore StartRestore(snapshotID string, target AccessPoint, dataMoverConfig map[string]string) error // Cancel cancels an asynchronous data path instance Cancel() // Close closes an asynchronous data path instance Close(ctx context.Context) } ================================================ FILE: pkg/discovery/helper.go ================================================ /* Copyright 2017, 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package discovery import ( "sort" "strings" "sync" "github.com/pkg/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/version" "k8s.io/client-go/discovery" "k8s.io/client-go/restmapper" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/features" kcmdutil "github.com/vmware-tanzu/velero/third_party/kubernetes/pkg/kubectl/cmd/util" ) //go:generate mockery --name Helper // Helper exposes functions for interacting with the Kubernetes discovery // API. type Helper interface { // Resources gets the current set of resources retrieved from discovery // that are backuppable by Velero. Resources() []*metav1.APIResourceList // ResourceFor gets a fully-resolved GroupVersionResource and an // APIResource for the provided partially-specified GroupVersionResource. ResourceFor(input schema.GroupVersionResource) (schema.GroupVersionResource, metav1.APIResource, error) // KindFor gets a fully-resolved GroupVersionResource and an // APIResource for the provided partially-specified GroupVersionKind. KindFor(input schema.GroupVersionKind) (schema.GroupVersionResource, metav1.APIResource, error) // Refresh pulls an updated set of Velero-backuppable resources from the // discovery API. Refresh() error // APIGroups gets the current set of supported APIGroups // in the cluster. APIGroups() []metav1.APIGroup // ServerVersion retrieves and parses the server's k8s version (git version) // in the cluster. ServerVersion() *version.Info } type serverResourcesInterface interface { // ServerPreferredResources() is used to populate Resources() with only Preferred Versions - this is the default ServerPreferredResources() ([]*metav1.APIResourceList, error) // ServerGroupsAndResources returns supported groups and resources for *all* groups and versions // Used to populate Resources() if feature flag is passed ServerGroupsAndResources() ([]*metav1.APIGroup, []*metav1.APIResourceList, error) } type helper struct { discoveryClient discovery.AggregatedDiscoveryInterface logger logrus.FieldLogger // lock guards mapper, resources and resourcesMap lock sync.RWMutex mapper meta.RESTMapper resources []*metav1.APIResourceList resourcesMap map[schema.GroupVersionResource]metav1.APIResource kindMap map[schema.GroupVersionKind]metav1.APIResource apiGroups []metav1.APIGroup serverVersion *version.Info } var _ Helper = &helper{} func NewHelper(discoveryClient discovery.AggregatedDiscoveryInterface, logger logrus.FieldLogger) (Helper, error) { h := &helper{ discoveryClient: discoveryClient, logger: logger, } if err := h.Refresh(); err != nil { return nil, err } return h, nil } func (h *helper) ResourceFor(input schema.GroupVersionResource) (schema.GroupVersionResource, metav1.APIResource, error) { h.lock.RLock() defer h.lock.RUnlock() gvr, err := h.mapper.ResourceFor(input) if err != nil { return schema.GroupVersionResource{}, metav1.APIResource{}, err } apiResource, found := h.resourcesMap[gvr] if !found { return schema.GroupVersionResource{}, metav1.APIResource{}, errors.Errorf("APIResource not found for GroupVersionResource %s", gvr) } return gvr, apiResource, nil } func (h *helper) KindFor(input schema.GroupVersionKind) (schema.GroupVersionResource, metav1.APIResource, error) { h.lock.RLock() defer h.lock.RUnlock() if resource, ok := h.kindMap[input]; ok { return schema.GroupVersionResource{ Group: resource.Group, Version: resource.Version, Resource: resource.Name, }, resource, nil } m, err := h.mapper.RESTMapping(schema.GroupKind{Group: input.Group, Kind: input.Kind}, input.Version) if err != nil { return schema.GroupVersionResource{}, metav1.APIResource{}, err } if resource, ok := h.kindMap[m.GroupVersionKind]; ok { return schema.GroupVersionResource{ Group: resource.Group, Version: resource.Version, Resource: resource.Name, }, resource, nil } return schema.GroupVersionResource{}, metav1.APIResource{}, errors.Errorf("APIResource not found for GroupVersionKind %v ", input) } func (h *helper) Refresh() error { h.lock.Lock() defer h.lock.Unlock() groupResources, err := restmapper.GetAPIGroupResources(h.discoveryClient) if err != nil { return errors.WithStack(err) } var serverResources []*metav1.APIResourceList if features.IsEnabled(velerov1api.APIGroupVersionsFeatureFlag) { // ServerGroupsAndResources returns all APIGroup and APIResouceList - not only preferred versions _, serverAllResources, err := refreshServerGroupsAndResources(h.discoveryClient, h.logger) if err != nil { return errors.WithStack(err) } h.logger.Infof("The '%s' feature flag was specified, using all API group versions.", velerov1api.APIGroupVersionsFeatureFlag) serverResources = serverAllResources } else { // ServerPreferredResources() returns only preferred APIGroup - this is the default since no feature flag has been passed serverPreferredResources, err := refreshServerPreferredResources(h.discoveryClient, h.logger) if err != nil { return errors.WithStack(err) } serverResources = serverPreferredResources } h.resources = discovery.FilteredBy( And(filterByVerbs, skipSubresource), serverResources, ) sortResources(h.resources) shortcutExpander, err := kcmdutil.NewShortcutExpander(restmapper.NewDiscoveryRESTMapper(groupResources), h.resources, h.logger) if err != nil { return errors.WithStack(err) } h.mapper = shortcutExpander h.resourcesMap = make(map[schema.GroupVersionResource]metav1.APIResource) h.kindMap = make(map[schema.GroupVersionKind]metav1.APIResource) for _, resourceGroup := range h.resources { gv, err := schema.ParseGroupVersion(resourceGroup.GroupVersion) if err != nil { return errors.Wrapf(err, "unable to parse GroupVersion %s", resourceGroup.GroupVersion) } for _, resource := range resourceGroup.APIResources { gvr := gv.WithResource(resource.Name) gvk := gv.WithKind(resource.Kind) resource.Group = gv.Group resource.Version = gv.Version h.resourcesMap[gvr] = resource h.kindMap[gvk] = resource } } apiGroupList, err := h.discoveryClient.ServerGroups() if err != nil { return errors.WithStack(err) } h.apiGroups = apiGroupList.Groups serverVersion, err := h.discoveryClient.ServerVersion() if err != nil { return errors.WithStack(err) } h.serverVersion = serverVersion return nil } func refreshServerPreferredResources(discoveryClient serverResourcesInterface, logger logrus.FieldLogger) ([]*metav1.APIResourceList, error) { preferredResources, err := discoveryClient.ServerPreferredResources() if err != nil { if discoveryErr, ok := err.(*discovery.ErrGroupDiscoveryFailed); ok { for groupVersion, err := range discoveryErr.Groups { logger.WithError(err).Warnf("Failed to discover group: %v", groupVersion) } return preferredResources, nil } } return preferredResources, err } func refreshServerGroupsAndResources(discoveryClient serverResourcesInterface, logger logrus.FieldLogger) ([]*metav1.APIGroup, []*metav1.APIResourceList, error) { serverGroups, serverResources, err := discoveryClient.ServerGroupsAndResources() if err != nil { if discoveryErr, ok := err.(*discovery.ErrGroupDiscoveryFailed); ok { for groupVersion, err := range discoveryErr.Groups { logger.WithError(err).Warnf("Failed to discover group: %v", groupVersion) } return serverGroups, serverResources, nil } } return serverGroups, serverResources, err } // And returns a composite predicate that implements a logical AND of the predicates passed to it. func And(predicates ...discovery.ResourcePredicateFunc) discovery.ResourcePredicate { return and{predicates} } type and struct { predicates []discovery.ResourcePredicateFunc } func (a and) Match(groupVersion string, r *metav1.APIResource) bool { for _, p := range a.predicates { if !p(groupVersion, r) { return false } } return true } func filterByVerbs(groupVersion string, r *metav1.APIResource) bool { return discovery.SupportsAllVerbs{Verbs: []string{"list", "create", "get", "delete"}}.Match(groupVersion, r) } func skipSubresource(_ string, r *metav1.APIResource) bool { // if we have a slash, then this is a subresource and we shouldn't include it. return !strings.Contains(r.Name, "/") } // sortResources sources resources by moving extensions to the end of the slice. The order of all // the other resources is preserved. func sortResources(resources []*metav1.APIResourceList) { sort.SliceStable(resources, func(i, j int) bool { left := resources[i] leftGV, _ := schema.ParseGroupVersion(left.GroupVersion) // not checking error because it should be impossible to fail to parse data coming from the // apiserver if leftGV.Group == "extensions" { // always sort extensions at the bottom by saying left is "greater" return false } right := resources[j] rightGV, _ := schema.ParseGroupVersion(right.GroupVersion) // not checking error because it should be impossible to fail to parse data coming from the // apiserver if rightGV.Group == "extensions" { // always sort extensions at the bottom by saying left is "less" return true } return i < j }) } func (h *helper) Resources() []*metav1.APIResourceList { h.lock.RLock() defer h.lock.RUnlock() return h.resources } func (h *helper) APIGroups() []metav1.APIGroup { h.lock.RLock() defer h.lock.RUnlock() return h.apiGroups } func (h *helper) ServerVersion() *version.Info { h.lock.RLock() defer h.lock.RUnlock() return h.serverVersion } ================================================ FILE: pkg/discovery/helper_test.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package discovery import ( "errors" "sync" "testing" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/version" "k8s.io/client-go/discovery/fake" clientgotesting "k8s.io/client-go/testing" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/features" velerotest "github.com/vmware-tanzu/velero/pkg/test" "github.com/vmware-tanzu/velero/pkg/util/logging" ) func TestSortResources(t *testing.T) { tests := []struct { name string resources []*metav1.APIResourceList expected []*metav1.APIResourceList }{ { name: "no resources", }, { name: "no extensions, order is preserved", resources: []*metav1.APIResourceList{ {GroupVersion: "v1"}, {GroupVersion: "groupC/v1"}, {GroupVersion: "groupA/v1"}, {GroupVersion: "groupB/v1"}, }, expected: []*metav1.APIResourceList{ {GroupVersion: "v1"}, {GroupVersion: "groupC/v1"}, {GroupVersion: "groupA/v1"}, {GroupVersion: "groupB/v1"}, }, }, { name: "extensions moves to end, order is preserved", resources: []*metav1.APIResourceList{ {GroupVersion: "extensions/v1beta1"}, {GroupVersion: "v1"}, {GroupVersion: "groupC/v1"}, {GroupVersion: "groupA/v1"}, {GroupVersion: "groupB/v1"}, {GroupVersion: "apps/v1beta1"}, }, expected: []*metav1.APIResourceList{ {GroupVersion: "v1"}, {GroupVersion: "groupC/v1"}, {GroupVersion: "groupA/v1"}, {GroupVersion: "groupB/v1"}, {GroupVersion: "apps/v1beta1"}, {GroupVersion: "extensions/v1beta1"}, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { t.Log("before") for _, r := range test.resources { t.Log(r.GroupVersion) } sortResources(test.resources) t.Logf("after") for _, r := range test.resources { t.Log(r.GroupVersion) } assert.Equal(t, test.expected, test.resources) }) } } func TestFilteringByVerbs(t *testing.T) { tests := []struct { name string groupVersion string res *metav1.APIResource expected bool }{ { name: "resource that supports list, create, get, delete", groupVersion: "v1", res: &metav1.APIResource{ Verbs: metav1.Verbs{"list", "create", "get", "delete"}, }, expected: true, }, { name: "resource that supports list, create, get, delete in a different order", groupVersion: "v1", res: &metav1.APIResource{ Verbs: metav1.Verbs{"delete", "get", "create", "list"}, }, expected: true, }, { name: "resource that supports list, create, get, delete, and more", groupVersion: "v1", res: &metav1.APIResource{ Verbs: metav1.Verbs{"list", "create", "get", "delete", "update", "patch", "deletecollection"}, }, expected: true, }, { name: "resource that supports only list and create", groupVersion: "v1", res: &metav1.APIResource{ Verbs: metav1.Verbs{"list", "create"}, }, expected: false, }, { name: "resource that supports only get and delete", groupVersion: "v1", res: &metav1.APIResource{ Verbs: metav1.Verbs{"get", "delete"}, }, expected: false, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { out := filterByVerbs(test.groupVersion, test.res) assert.Equal(t, test.expected, out) }) } } func TestRefreshServerPreferredResources(t *testing.T) { tests := []struct { name string resourceList []*metav1.APIResourceList apiGroup []*metav1.APIGroup failedGroups map[schema.GroupVersion]error returnError error }{ { name: "all groups discovered, no error is returned", resourceList: []*metav1.APIResourceList{ {GroupVersion: "groupB/v1"}, {GroupVersion: "apps/v1beta1"}, {GroupVersion: "extensions/v1beta1"}, }, }, { name: "failed to discover some groups, no error is returned", resourceList: []*metav1.APIResourceList{ {GroupVersion: "groupB/v1"}, {GroupVersion: "apps/v1beta1"}, {GroupVersion: "extensions/v1beta1"}, }, failedGroups: map[schema.GroupVersion]error{ {Group: "groupA", Version: "v1"}: errors.New("Fake error"), {Group: "groupC", Version: "v2"}: errors.New("Fake error"), }, }, { name: "non ErrGroupDiscoveryFailed error, returns error", returnError: errors.New("Generic error"), }, } formatFlag := logging.FormatText for _, test := range tests { fakeServer := velerotest.NewFakeServerResourcesInterface(test.resourceList, test.apiGroup, test.failedGroups, test.returnError) t.Run(test.name, func(t *testing.T) { resources, err := refreshServerPreferredResources(fakeServer, logging.DefaultLogger(logrus.DebugLevel, formatFlag)) if test.returnError != nil { require.Error(t, err) } else { require.NoError(t, err) assert.Equal(t, test.returnError, err) } assert.Equal(t, test.resourceList, resources) }) } } func TestHelper_ResourceFor(t *testing.T) { fakeDiscoveryClient := &fake.FakeDiscovery{ Fake: &clientgotesting.Fake{}, } fakeDiscoveryClient.Resources = []*metav1.APIResourceList{ { GroupVersion: "v1", APIResources: []metav1.APIResource{ { Name: "pods", Kind: "Pod", Group: "", Version: "v1", Verbs: []string{"create", "get", "list"}, }, }, }, } h := &helper{ discoveryClient: &velerotest.DiscoveryClient{ FakeDiscovery: fakeDiscoveryClient, }, lock: sync.RWMutex{}, mapper: nil, resources: fakeDiscoveryClient.Resources, resourcesMap: make(map[schema.GroupVersionResource]metav1.APIResource), serverVersion: &version.Info{Major: "1", Minor: "22", GitVersion: "v1.22.1"}, } for _, resourceList := range h.resources { for _, resource := range resourceList.APIResources { gvr := schema.GroupVersionResource{ Group: resource.Group, Version: resource.Version, Resource: resource.Name, } h.resourcesMap[gvr] = resource } } pvGVR := schema.GroupVersionResource{ Group: "", Version: "v1", Resource: "pods", } h.mapper = &velerotest.FakeMapper{Resources: map[schema.GroupVersionResource]schema.GroupVersionResource{pvGVR: pvGVR}} tests := []struct { name string err string input *schema.GroupVersionResource isNotFoundRes bool expectedGVR *schema.GroupVersionResource expectedAPIResource *metav1.APIResource }{ { name: "Found resource", input: &schema.GroupVersionResource{ Group: "", Version: "v1", Resource: "pods", }, expectedAPIResource: &metav1.APIResource{ Name: "pods", Kind: "Pod", Group: "", Version: "v1", Verbs: []string{"create", "get", "list"}, }, expectedGVR: &schema.GroupVersionResource{ Group: "", Version: "v1", Resource: "pods", }, }, { name: "Error to found resource", input: &schema.GroupVersionResource{ Group: "", Version: "v2", Resource: "pods", }, err: "invalid resource", expectedGVR: &schema.GroupVersionResource{}, expectedAPIResource: &metav1.APIResource{}, }, { name: "Error to found api resource", input: &schema.GroupVersionResource{ Group: "", Version: "v1", Resource: "pods", }, isNotFoundRes: true, err: "APIResource not found for GroupVersionResource", expectedGVR: &schema.GroupVersionResource{}, expectedAPIResource: &metav1.APIResource{}, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { if tc.isNotFoundRes { h.resourcesMap = nil } gvr, apiResource, err := h.ResourceFor(*tc.input) if tc.err == "" { require.NoError(t, err) } else { require.ErrorContains(t, err, tc.err) } assert.Equal(t, *tc.expectedGVR, gvr) assert.Equal(t, *tc.expectedAPIResource, apiResource) }) } } func TestHelper_KindFor(t *testing.T) { fakeDiscoveryClient := &fake.FakeDiscovery{ Fake: &clientgotesting.Fake{}, } fakeDiscoveryClient.Resources = []*metav1.APIResourceList{ { GroupVersion: "v1", APIResources: []metav1.APIResource{ { Name: "pods", Kind: "Pod", Group: "", Version: "v1", Verbs: []string{"create", "get", "list"}, }, }, }, } pvGVK := schema.GroupVersionKind{ Group: "", Version: "v1", Kind: "Deployment", } pvAPIRes := metav1.APIResource{ Name: "deployments", Kind: "Deployment", Group: "apps", Version: "v1", Verbs: []string{"create", "get", "list"}, } h := &helper{ discoveryClient: &velerotest.DiscoveryClient{ FakeDiscovery: fakeDiscoveryClient, }, lock: sync.RWMutex{}, resources: fakeDiscoveryClient.Resources, resourcesMap: make(map[schema.GroupVersionResource]metav1.APIResource), serverVersion: &version.Info{Major: "1", Minor: "22", GitVersion: "v1.22.1"}, } h.kindMap = map[schema.GroupVersionKind]metav1.APIResource{pvGVK: pvAPIRes} h.mapper = &velerotest.FakeMapper{KindToPluralResource: map[schema.GroupVersionKind]schema.GroupVersionResource{}} for _, resourceList := range h.resources { for _, resource := range resourceList.APIResources { gvr := schema.GroupVersionResource{ Group: resource.Group, Version: resource.Version, Resource: resource.Name, } h.resourcesMap[gvr] = resource } } tests := []struct { name string err string input *schema.GroupVersionKind isNotFoundRes bool expectedGVR *schema.GroupVersionResource expectedAPIResource *metav1.APIResource }{ { name: "Found resource", input: &schema.GroupVersionKind{ Group: "", Version: "v1", Kind: "Deployment", }, expectedAPIResource: &metav1.APIResource{ Name: "deployments", Kind: "Deployment", Group: "apps", Version: "v1", Verbs: []string{"create", "get", "list"}, }, expectedGVR: &schema.GroupVersionResource{ Group: "apps", Version: "v1", Resource: "deployments", }, }, { name: "Not found resource", input: &schema.GroupVersionKind{ Group: "", Version: "v2", Kind: "Deployment", }, expectedAPIResource: &metav1.APIResource{}, expectedGVR: &schema.GroupVersionResource{}, err: "no matches for kind", }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { gvr, apiResource, err := h.KindFor(*tc.input) if tc.err == "" { require.NoError(t, err) } else { require.ErrorContains(t, err, tc.err) } assert.Equal(t, *tc.expectedGVR, gvr) assert.Equal(t, *tc.expectedAPIResource, apiResource) }) } } func TestHelper_Refresh(t *testing.T) { testCases := []struct { description string features string groupResources []*metav1.APIResourceList serverGroups []*metav1.APIGroup expectedErr error expectedResource metav1.APIResource }{ { description: "Default case - Resource found", groupResources: []*metav1.APIResourceList{ { GroupVersion: "v1", APIResources: []metav1.APIResource{ { Name: "pods", Kind: "Pod", Group: "", Version: "v1", Verbs: []string{"get", "list", "create"}, }, }, }, }, serverGroups: []*metav1.APIGroup{ { Name: "group1", Versions: []metav1.GroupVersionForDiscovery{ { GroupVersion: "v1", Version: "v1", }, }, }, }, expectedErr: nil, expectedResource: metav1.APIResource{ Name: "pods", Kind: "Pod", Group: "", Version: "v1", Verbs: []string{"get", "list", "create"}, }, }, { description: "Feature flag enabled - ServerGroupsAndResources", features: velerov1api.APIGroupVersionsFeatureFlag, groupResources: []*metav1.APIResourceList{}, serverGroups: []*metav1.APIGroup{ { Name: "group1", Versions: []metav1.GroupVersionForDiscovery{ { GroupVersion: "v1", Version: "v1", }, }, }, }, expectedErr: nil, expectedResource: metav1.APIResource{ Name: "pods", Kind: "Pod", Group: "", Version: "v1", Verbs: []string{"get", "list", "create"}, }, }, } fakeDiscoveryClient := &fake.FakeDiscovery{ Fake: &clientgotesting.Fake{}, } fakeDiscoveryClient.Resources = []*metav1.APIResourceList{ { GroupVersion: "v1", APIResources: []metav1.APIResource{ { Name: "pods", Kind: "Pod", Group: "", Version: "v1", Verbs: []string{"create", "get", "list"}, }, }, }, } for _, testCase := range testCases { t.Run(testCase.description, func(t *testing.T) { h := &helper{ lock: sync.RWMutex{}, discoveryClient: &velerotest.DiscoveryClient{ FakeDiscovery: fakeDiscoveryClient, }, logger: logrus.New(), } // Set feature flags if testCase.features != "" { features.Enable(testCase.features) } err := h.Refresh() assert.Equal(t, testCase.expectedErr, err) }) } } func TestHelper_refreshServerPreferredResources(t *testing.T) { apiList := []*metav1.APIResourceList{ { GroupVersion: "v1", APIResources: []metav1.APIResource{ { Name: "pods", Kind: "Pod", Group: "", Version: "v1", Verbs: []string{"create", "get", "list"}, }, }, }, } tests := []struct { name string expectedErr error }{ { name: "success get preferred resources", expectedErr: nil, }, { name: "failed to get preferred resources", expectedErr: errors.New("Failed to discover preferred resources"), }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { fakeClient := velerotest.NewFakeServerResourcesInterface(apiList, []*metav1.APIGroup{}, map[schema.GroupVersion]error{}, tc.expectedErr) resources, err := refreshServerPreferredResources(fakeClient, logrus.New()) if tc.expectedErr != nil { assert.Error(t, err) } else { require.NoError(t, err) assert.NotNil(t, resources) } }) } } func TestHelper_refreshServerGroupsAndResources(t *testing.T) { apiList := []*metav1.APIResourceList{ { GroupVersion: "v1", APIResources: []metav1.APIResource{ { Name: "pods", Kind: "Pod", Group: "", Version: "v1", Verbs: []string{"create", "get", "list"}, }, }, }, } apiGroup := []*metav1.APIGroup{ { Name: "group1", Versions: []metav1.GroupVersionForDiscovery{ { GroupVersion: "v1", Version: "v1", }, }, }, } tests := []struct { name string expectedErr error }{ { name: "success get service groups and resouorces", }, { name: "failed to service groups and resouorces", expectedErr: errors.New("Failed to discover service groups and resouorces"), }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { fakeClient := velerotest.NewFakeServerResourcesInterface(apiList, apiGroup, map[schema.GroupVersion]error{}, tc.expectedErr) serverGroups, serverResources, err := refreshServerGroupsAndResources(fakeClient, logrus.New()) if tc.expectedErr != nil { assert.Error(t, err) } else { require.NoError(t, err) assert.NotNil(t, serverGroups) assert.NotNil(t, serverResources) } }) } } func TestHelper(t *testing.T) { fakeDiscoveryClient := &fake.FakeDiscovery{ Fake: &clientgotesting.Fake{}, } h, err := NewHelper(&velerotest.DiscoveryClient{ FakeDiscovery: fakeDiscoveryClient, }, logrus.New()) require.NoError(t, err) // All below calls put together for the implementation are empty or just very simple, and just want to cover testing // If wanting to write unit tests for some functions could remove it and with writing new function alone h.Resources() h.APIGroups() h.ServerVersion() } ================================================ FILE: pkg/discovery/mocks/Helper.go ================================================ // Code generated by mockery v2.20.0. DO NOT EDIT. package mocks import ( mock "github.com/stretchr/testify/mock" schema "k8s.io/apimachinery/pkg/runtime/schema" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" version "k8s.io/apimachinery/pkg/version" ) // Helper is an autogenerated mock type for the Helper type type Helper struct { mock.Mock } // APIGroups provides a mock function with given fields: func (_m *Helper) APIGroups() []metav1.APIGroup { ret := _m.Called() var r0 []metav1.APIGroup if rf, ok := ret.Get(0).(func() []metav1.APIGroup); ok { r0 = rf() } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]metav1.APIGroup) } } return r0 } // KindFor provides a mock function with given fields: input func (_m *Helper) KindFor(input schema.GroupVersionKind) (schema.GroupVersionResource, metav1.APIResource, error) { ret := _m.Called(input) var r0 schema.GroupVersionResource var r1 metav1.APIResource var r2 error if rf, ok := ret.Get(0).(func(schema.GroupVersionKind) (schema.GroupVersionResource, metav1.APIResource, error)); ok { return rf(input) } if rf, ok := ret.Get(0).(func(schema.GroupVersionKind) schema.GroupVersionResource); ok { r0 = rf(input) } else { r0 = ret.Get(0).(schema.GroupVersionResource) } if rf, ok := ret.Get(1).(func(schema.GroupVersionKind) metav1.APIResource); ok { r1 = rf(input) } else { r1 = ret.Get(1).(metav1.APIResource) } if rf, ok := ret.Get(2).(func(schema.GroupVersionKind) error); ok { r2 = rf(input) } else { r2 = ret.Error(2) } return r0, r1, r2 } // Refresh provides a mock function with given fields: func (_m *Helper) Refresh() error { ret := _m.Called() var r0 error if rf, ok := ret.Get(0).(func() error); ok { r0 = rf() } else { r0 = ret.Error(0) } return r0 } // ResourceFor provides a mock function with given fields: input func (_m *Helper) ResourceFor(input schema.GroupVersionResource) (schema.GroupVersionResource, metav1.APIResource, error) { ret := _m.Called(input) var r0 schema.GroupVersionResource var r1 metav1.APIResource var r2 error if rf, ok := ret.Get(0).(func(schema.GroupVersionResource) (schema.GroupVersionResource, metav1.APIResource, error)); ok { return rf(input) } if rf, ok := ret.Get(0).(func(schema.GroupVersionResource) schema.GroupVersionResource); ok { r0 = rf(input) } else { r0 = ret.Get(0).(schema.GroupVersionResource) } if rf, ok := ret.Get(1).(func(schema.GroupVersionResource) metav1.APIResource); ok { r1 = rf(input) } else { r1 = ret.Get(1).(metav1.APIResource) } if rf, ok := ret.Get(2).(func(schema.GroupVersionResource) error); ok { r2 = rf(input) } else { r2 = ret.Error(2) } return r0, r1, r2 } // Resources provides a mock function with given fields: func (_m *Helper) Resources() []*metav1.APIResourceList { ret := _m.Called() var r0 []*metav1.APIResourceList if rf, ok := ret.Get(0).(func() []*metav1.APIResourceList); ok { r0 = rf() } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]*metav1.APIResourceList) } } return r0 } // ServerVersion provides a mock function with given fields: func (_m *Helper) ServerVersion() *version.Info { ret := _m.Called() var r0 *version.Info if rf, ok := ret.Get(0).(func() *version.Info); ok { r0 = rf() } else { if ret.Get(0) != nil { r0 = ret.Get(0).(*version.Info) } } return r0 } type mockConstructorTestingTNewHelper interface { mock.TestingT Cleanup(func()) } // NewHelper creates a new instance of Helper. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. func NewHelper(t mockConstructorTestingTNewHelper) *Helper { mock := &Helper{} mock.Mock.Test(t) t.Cleanup(func() { mock.AssertExpectations(t) }) return mock } ================================================ FILE: pkg/exposer/cache_volume.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package exposer import ( "context" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" "github.com/vmware-tanzu/velero/pkg/util/boolptr" "github.com/vmware-tanzu/velero/pkg/util/kube" ) type CacheConfigs struct { Limit int64 StorageClass string ResidentThreshold int64 } const ( cacheVolumeName = "cachedir" cacheVolumeDirSuffix = "-cache" ) func createCachePVC(ctx context.Context, pvcClient corev1client.CoreV1Interface, ownerObject corev1api.ObjectReference, sc string, size int64, selectedNode string) (*corev1api.PersistentVolumeClaim, error) { cachePVCName := getCachePVCName(ownerObject) volumeMode := corev1api.PersistentVolumeFilesystem pvcObj := &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: ownerObject.Namespace, Name: cachePVCName, OwnerReferences: []metav1.OwnerReference{ { APIVersion: ownerObject.APIVersion, Kind: ownerObject.Kind, Name: ownerObject.Name, UID: ownerObject.UID, Controller: boolptr.True(), }, }, }, Spec: corev1api.PersistentVolumeClaimSpec{ AccessModes: []corev1api.PersistentVolumeAccessMode{corev1api.ReadWriteOnce}, StorageClassName: &sc, VolumeMode: &volumeMode, Resources: corev1api.VolumeResourceRequirements{ Requests: corev1api.ResourceList{ corev1api.ResourceStorage: *resource.NewQuantity(size, resource.BinarySI), }, }, }, } if selectedNode != "" { pvcObj.Annotations = map[string]string{ kube.KubeAnnSelectedNode: selectedNode, } } return pvcClient.PersistentVolumeClaims(pvcObj.Namespace).Create(ctx, pvcObj, metav1.CreateOptions{}) } func getCachePVCName(ownerObject corev1api.ObjectReference) string { return ownerObject.Name + cacheVolumeDirSuffix } func getCacheVolumeSize(dataSize int64, info *CacheConfigs) int64 { if info == nil { return 0 } if dataSize != 0 && dataSize < info.ResidentThreshold { return 0 } // 20% inflate and round up to GB volumeSize := (info.Limit*12/10 + (1 << 30) - 1) / (1 << 30) * (1 << 30) return volumeSize } ================================================ FILE: pkg/exposer/cache_volume_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package exposer import ( "testing" "github.com/stretchr/testify/require" ) func TestGetCacheVolumeSize(t *testing.T) { tests := []struct { name string dataSize int64 info *CacheConfigs expected int64 }{ { name: "nil info", dataSize: 1024, expected: 0, }, { name: "0 data size", info: &CacheConfigs{Limit: 1 << 30, ResidentThreshold: 5120}, expected: 2 << 30, }, { name: "0 threshold", dataSize: 2048, info: &CacheConfigs{Limit: 1 << 30}, expected: 2 << 30, }, { name: "data size is smaller", dataSize: 2048, info: &CacheConfigs{Limit: 1 << 30, ResidentThreshold: 5120}, expected: 0, }, { name: "data size is lager", dataSize: 2048, info: &CacheConfigs{Limit: 1 << 30, ResidentThreshold: 1024}, expected: 2 << 30, }, { name: "limit smaller than 1G", dataSize: 2048, info: &CacheConfigs{Limit: 5120, ResidentThreshold: 1024}, expected: 1 << 30, }, { name: "larger than 1G after inflate", dataSize: 2048, info: &CacheConfigs{Limit: (1 << 30) - 1024, ResidentThreshold: 1024}, expected: 2 << 30, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { size := getCacheVolumeSize(test.dataSize, test.info) require.Equal(t, test.expected, size) }) } } ================================================ FILE: pkg/exposer/csi_snapshot.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package exposer import ( "context" "fmt" "time" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" snapshotter "github.com/kubernetes-csi/external-snapshotter/client/v8/clientset/versioned/typed/volumesnapshot/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/pkg/nodeagent" velerotypes "github.com/vmware-tanzu/velero/pkg/types" "github.com/vmware-tanzu/velero/pkg/util" "github.com/vmware-tanzu/velero/pkg/util/boolptr" "github.com/vmware-tanzu/velero/pkg/util/csi" "github.com/vmware-tanzu/velero/pkg/util/kube" ) // CSISnapshotExposeParam define the input param for Expose of CSI snapshots type CSISnapshotExposeParam struct { // SnapshotName is the original volume snapshot name SnapshotName string // SourceNamespace is the original namespace of the volume that the snapshot is taken for SourceNamespace string // SourcePVCName is the original name of the PVC that the snapshot is taken for SourcePVCName string // SourcePVName is the name of PV for SourcePVC SourcePVName string // AccessMode defines the mode to access the snapshot AccessMode string // StorageClass is the storage class of the volume that the snapshot is taken for StorageClass string // HostingPodLabels is the labels that are going to apply to the hosting pod HostingPodLabels map[string]string // HostingPodAnnotations is the annotations that are going to apply to the hosting pod HostingPodAnnotations map[string]string // HostingPodTolerations is the tolerations that are going to apply to the hosting pod HostingPodTolerations []corev1api.Toleration // OperationTimeout specifies the time wait for resources operations in Expose OperationTimeout time.Duration // ExposeTimeout specifies the timeout for the entire expose process ExposeTimeout time.Duration // VolumeSize specifies the size of the source volume VolumeSize resource.Quantity // Affinity specifies the node affinity of the backup pod Affinity []*kube.LoadAffinity // BackupPVCConfig is the config for backupPVC (intermediate PVC) of snapshot data movement BackupPVCConfig map[string]velerotypes.BackupPVC // Resources defines the resource requirements of the hosting pod Resources corev1api.ResourceRequirements // NodeOS specifies the OS of node that the source volume is attaching NodeOS string // PriorityClassName is the priority class name for the data mover pod PriorityClassName string } // CSISnapshotExposeWaitParam define the input param for WaitExposed of CSI snapshots type CSISnapshotExposeWaitParam struct { // NodeClient is the client that is used to find the hosting pod NodeClient client.Client NodeName string } // NewCSISnapshotExposer create a new instance of CSI snapshot exposer func NewCSISnapshotExposer(kubeClient kubernetes.Interface, csiSnapshotClient snapshotter.SnapshotV1Interface, log logrus.FieldLogger) SnapshotExposer { return &csiSnapshotExposer{ kubeClient: kubeClient, csiSnapshotClient: csiSnapshotClient, log: log, } } type csiSnapshotExposer struct { kubeClient kubernetes.Interface csiSnapshotClient snapshotter.SnapshotV1Interface log logrus.FieldLogger } func (e *csiSnapshotExposer) Expose(ctx context.Context, ownerObject corev1api.ObjectReference, param any) error { csiExposeParam := param.(*CSISnapshotExposeParam) curLog := e.log.WithFields(logrus.Fields{ "owner": ownerObject.Name, }) volumeTopology, err := kube.GetVolumeTopology(ctx, e.kubeClient.CoreV1(), e.kubeClient.StorageV1(), csiExposeParam.SourcePVName, csiExposeParam.StorageClass) if err != nil { return errors.Wrapf(err, "error getting volume topology for PV %s, storage class %s", csiExposeParam.SourcePVName, csiExposeParam.StorageClass) } if volumeTopology != nil { curLog.Infof("Using volume topology %v", volumeTopology) } curLog.Info("Exposing CSI snapshot") volumeSnapshot, err := csi.WaitVolumeSnapshotReady(ctx, e.csiSnapshotClient, csiExposeParam.SnapshotName, csiExposeParam.SourceNamespace, csiExposeParam.ExposeTimeout, curLog) if err != nil { return errors.Wrapf(err, "error wait volume snapshot ready") } curLog.Info("Volumesnapshot is ready") vsc, err := csi.GetVolumeSnapshotContentForVolumeSnapshot(volumeSnapshot, e.csiSnapshotClient) if err != nil { return errors.Wrap(err, "error to get volume snapshot content") } curLog.WithField("vsc name", vsc.Name).WithField("vs name", volumeSnapshot.Name).Infof("Got VSC from VS in namespace %s", volumeSnapshot.Namespace) backupVS, err := e.createBackupVS(ctx, ownerObject, volumeSnapshot) if err != nil { return errors.Wrap(err, "error to create backup volume snapshot") } curLog.WithField("vs name", backupVS.Name).Infof("Backup VS is created from %s/%s", volumeSnapshot.Namespace, volumeSnapshot.Name) defer func() { if err != nil { csi.DeleteVolumeSnapshotIfAny(ctx, e.csiSnapshotClient, backupVS.Name, backupVS.Namespace, curLog) } }() backupVSC, err := e.createBackupVSC(ctx, ownerObject, vsc, backupVS) if err != nil { return errors.Wrap(err, "error to create backup volume snapshot content") } curLog.WithField("vsc name", backupVSC.Name).Infof("Backup VSC is created from %s", vsc.Name) retained, err := csi.RetainVSC(ctx, e.csiSnapshotClient, vsc) if err != nil { return errors.Wrap(err, "error to retain volume snapshot content") } curLog.WithField("vsc name", vsc.Name).WithField("retained", (retained != nil)).Info("Finished to retain VSC") err = csi.EnsureDeleteVS(ctx, e.csiSnapshotClient, volumeSnapshot.Name, volumeSnapshot.Namespace, csiExposeParam.OperationTimeout) if err != nil { return errors.Wrap(err, "error to delete volume snapshot") } curLog.WithField("vs name", volumeSnapshot.Name).Infof("VS is deleted in namespace %s", volumeSnapshot.Namespace) err = csi.EnsureDeleteVSC(ctx, e.csiSnapshotClient, vsc.Name, csiExposeParam.OperationTimeout) if err != nil { return errors.Wrap(err, "error to delete volume snapshot content") } curLog.WithField("vsc name", vsc.Name).Infof("VSC is deleted") var volumeSize resource.Quantity if volumeSnapshot.Status.RestoreSize != nil && !volumeSnapshot.Status.RestoreSize.IsZero() { volumeSize = *volumeSnapshot.Status.RestoreSize } else { volumeSize = csiExposeParam.VolumeSize curLog.WithField("vs name", volumeSnapshot.Name).Warnf("The snapshot doesn't contain a valid restore size, use source volume's size %v", volumeSize) } // check if there is a mapping for source pvc storage class in backupPVC config // if the mapping exists then use the values(storage class, readOnly accessMode) // for backupPVC (intermediate PVC in snapshot data movement) object creation backupPVCStorageClass := csiExposeParam.StorageClass backupPVCReadOnly := false spcNoRelabeling := false backupPVCAnnotations := map[string]string{} intoleratableNodes := []string{} if value, exists := csiExposeParam.BackupPVCConfig[csiExposeParam.StorageClass]; exists { if value.StorageClass != "" { backupPVCStorageClass = value.StorageClass } backupPVCReadOnly = value.ReadOnly if value.SPCNoRelabeling { if backupPVCReadOnly { spcNoRelabeling = true } else { curLog.WithField("vs name", volumeSnapshot.Name).Warn("Ignoring spcNoRelabling for read-write volume") } } if len(value.Annotations) > 0 { backupPVCAnnotations = value.Annotations } if _, found := backupPVCAnnotations[util.VSphereCNSFastCloneAnno]; found { if n, err := kube.GetPVAttachedNodes(ctx, csiExposeParam.SourcePVName, e.kubeClient.StorageV1()); err != nil { curLog.WithField("source PV", csiExposeParam.SourcePVName).WithError(err).Warnf("Failed to get attached node for source PV, ignore %s annotation", util.VSphereCNSFastCloneAnno) delete(backupPVCAnnotations, util.VSphereCNSFastCloneAnno) } else { intoleratableNodes = n } } } backupPVC, err := e.createBackupPVC(ctx, ownerObject, backupVS.Name, backupPVCStorageClass, csiExposeParam.AccessMode, volumeSize, backupPVCReadOnly, backupPVCAnnotations) if err != nil { return errors.Wrap(err, "error to create backup pvc") } curLog.WithField("pvc name", backupPVC.Name).Info("Backup PVC is created") defer func() { if err != nil { kube.DeletePVAndPVCIfAny(ctx, e.kubeClient.CoreV1(), backupPVC.Name, backupPVC.Namespace, 0, curLog) } }() affinity := kube.GetLoadAffinityByStorageClass(csiExposeParam.Affinity, backupPVCStorageClass, curLog) backupPod, err := e.createBackupPod( ctx, ownerObject, backupPVC, csiExposeParam.OperationTimeout, csiExposeParam.HostingPodLabels, csiExposeParam.HostingPodAnnotations, csiExposeParam.HostingPodTolerations, affinity, csiExposeParam.Resources, backupPVCReadOnly, spcNoRelabeling, csiExposeParam.NodeOS, csiExposeParam.PriorityClassName, intoleratableNodes, volumeTopology, ) if err != nil { return errors.Wrap(err, "error to create backup pod") } curLog.WithField("pod name", backupPod.Name).WithField("affinity", affinity).Info("Backup pod is created") defer func() { if err != nil { kube.DeletePodIfAny(ctx, e.kubeClient.CoreV1(), backupPod.Name, backupPod.Namespace, curLog) } }() return nil } func (e *csiSnapshotExposer) GetExposed(ctx context.Context, ownerObject corev1api.ObjectReference, timeout time.Duration, param any) (*ExposeResult, error) { exposeWaitParam := param.(*CSISnapshotExposeWaitParam) backupPodName := ownerObject.Name backupPVCName := ownerObject.Name containerName := string(ownerObject.UID) volumeName := string(ownerObject.UID) curLog := e.log.WithFields(logrus.Fields{ "owner": ownerObject.Name, }) pod := &corev1api.Pod{} err := exposeWaitParam.NodeClient.Get(ctx, types.NamespacedName{ Namespace: ownerObject.Namespace, Name: backupPodName, }, pod) if err != nil { if apierrors.IsNotFound(err) { curLog.WithField("backup pod", backupPodName).Debugf("Backup pod is not running in the current node %s", exposeWaitParam.NodeName) return nil, nil } else { return nil, errors.Wrapf(err, "error to get backup pod %s", backupPodName) } } curLog.WithField("pod", pod.Name).Infof("Backup pod is in running state in node %s", pod.Spec.NodeName) _, err = kube.WaitPVCBound(ctx, e.kubeClient.CoreV1(), e.kubeClient.CoreV1(), backupPVCName, ownerObject.Namespace, timeout) if err != nil { return nil, errors.Wrapf(err, "error to wait backup PVC bound, %s", backupPVCName) } curLog.WithField("backup pvc", backupPVCName).Info("Backup PVC is bound") i := 0 for i = 0; i < len(pod.Spec.Volumes); i++ { if pod.Spec.Volumes[i].Name == volumeName { break } } if i == len(pod.Spec.Volumes) { return nil, errors.Errorf("backup pod %s doesn't have the expected backup volume", pod.Name) } curLog.WithField("pod", pod.Name).Infof("Backup volume is found in pod at index %v", i) var nodeOS *string if pod.Spec.OS != nil { os := string(pod.Spec.OS.Name) nodeOS = &os } return &ExposeResult{ByPod: ExposeByPod{ HostingPod: pod, HostingContainer: containerName, VolumeName: volumeName, NodeOS: nodeOS, }}, nil } func (e *csiSnapshotExposer) PeekExposed(ctx context.Context, ownerObject corev1api.ObjectReference) error { backupPodName := ownerObject.Name curLog := e.log.WithFields(logrus.Fields{ "owner": ownerObject.Name, }) pod, err := e.kubeClient.CoreV1().Pods(ownerObject.Namespace).Get(ctx, backupPodName, metav1.GetOptions{}) if apierrors.IsNotFound(err) { return nil } if err != nil { curLog.WithError(err).Warnf("error to peek backup pod %s", backupPodName) return nil } if podFailed, message := kube.IsPodUnrecoverable(pod, curLog); podFailed { return errors.New(message) } return nil } func (e *csiSnapshotExposer) DiagnoseExpose(ctx context.Context, ownerObject corev1api.ObjectReference) string { backupPodName := ownerObject.Name backupPVCName := ownerObject.Name backupVSName := ownerObject.Name diag := "begin diagnose CSI exposer\n" pod, err := e.kubeClient.CoreV1().Pods(ownerObject.Namespace).Get(ctx, backupPodName, metav1.GetOptions{}) if err != nil { pod = nil diag += fmt.Sprintf("error getting backup pod %s, err: %v\n", backupPodName, err) } pvc, err := e.kubeClient.CoreV1().PersistentVolumeClaims(ownerObject.Namespace).Get(ctx, backupPVCName, metav1.GetOptions{}) if err != nil { pvc = nil diag += fmt.Sprintf("error getting backup pvc %s, err: %v\n", backupPVCName, err) } vs, err := e.csiSnapshotClient.VolumeSnapshots(ownerObject.Namespace).Get(ctx, backupVSName, metav1.GetOptions{}) if err != nil { vs = nil diag += fmt.Sprintf("error getting backup vs %s, err: %v\n", backupVSName, err) } events, err := e.kubeClient.CoreV1().Events(ownerObject.Namespace).List(ctx, metav1.ListOptions{}) if err != nil { diag += fmt.Sprintf("error listing events, err: %v\n", err) } if pod != nil { diag += kube.DiagnosePod(pod, events) if pod.Spec.NodeName != "" { if err := nodeagent.KbClientIsRunningInNode(ctx, ownerObject.Namespace, pod.Spec.NodeName, e.kubeClient); err != nil { diag += fmt.Sprintf("node-agent is not running in node %s, err: %v\n", pod.Spec.NodeName, err) } } } if pvc != nil { diag += kube.DiagnosePVC(pvc, events) if pvc.Spec.VolumeName != "" { if pv, err := e.kubeClient.CoreV1().PersistentVolumes().Get(ctx, pvc.Spec.VolumeName, metav1.GetOptions{}); err != nil { diag += fmt.Sprintf("error getting backup pv %s, err: %v\n", pvc.Spec.VolumeName, err) } else { diag += kube.DiagnosePV(pv) } } } if vs != nil { diag += csi.DiagnoseVS(vs, events) if vs.Status != nil && vs.Status.BoundVolumeSnapshotContentName != nil && *vs.Status.BoundVolumeSnapshotContentName != "" { if vsc, err := e.csiSnapshotClient.VolumeSnapshotContents().Get(ctx, *vs.Status.BoundVolumeSnapshotContentName, metav1.GetOptions{}); err != nil { diag += fmt.Sprintf("error getting backup vsc %s, err: %v\n", *vs.Status.BoundVolumeSnapshotContentName, err) } else { diag += csi.DiagnoseVSC(vsc) } } } diag += "end diagnose CSI exposer" return diag } const cleanUpTimeout = time.Minute func (e *csiSnapshotExposer) CleanUp(ctx context.Context, ownerObject corev1api.ObjectReference, vsName string, sourceNamespace string) { backupPodName := ownerObject.Name backupPVCName := ownerObject.Name backupVSName := ownerObject.Name kube.DeletePodIfAny(ctx, e.kubeClient.CoreV1(), backupPodName, ownerObject.Namespace, e.log) kube.DeletePVAndPVCIfAny(ctx, e.kubeClient.CoreV1(), backupPVCName, ownerObject.Namespace, cleanUpTimeout, e.log) csi.DeleteVolumeSnapshotIfAny(ctx, e.csiSnapshotClient, backupVSName, ownerObject.Namespace, e.log) csi.DeleteVolumeSnapshotIfAny(ctx, e.csiSnapshotClient, vsName, sourceNamespace, e.log) } func getVolumeModeByAccessMode(accessMode string) (corev1api.PersistentVolumeMode, error) { switch accessMode { case AccessModeFileSystem: return corev1api.PersistentVolumeFilesystem, nil case AccessModeBlock: return corev1api.PersistentVolumeBlock, nil default: return "", errors.Errorf("unsupported access mode %s", accessMode) } } func (e *csiSnapshotExposer) createBackupVS(ctx context.Context, ownerObject corev1api.ObjectReference, snapshotVS *snapshotv1api.VolumeSnapshot) (*snapshotv1api.VolumeSnapshot, error) { backupVSName := ownerObject.Name backupVSCName := ownerObject.Name vs := &snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: backupVSName, Namespace: ownerObject.Namespace, Annotations: snapshotVS.Annotations, // Don't add ownerReference to SnapshotBackup. // The backupPVC should be deleted before backupVS, otherwise, the deletion of backupVS will fail since // backupPVC has its dataSource referring to it }, Spec: snapshotv1api.VolumeSnapshotSpec{ Source: snapshotv1api.VolumeSnapshotSource{ VolumeSnapshotContentName: &backupVSCName, }, VolumeSnapshotClassName: snapshotVS.Spec.VolumeSnapshotClassName, }, } return e.csiSnapshotClient.VolumeSnapshots(vs.Namespace).Create(ctx, vs, metav1.CreateOptions{}) } func (e *csiSnapshotExposer) createBackupVSC(ctx context.Context, ownerObject corev1api.ObjectReference, snapshotVSC *snapshotv1api.VolumeSnapshotContent, vs *snapshotv1api.VolumeSnapshot) (*snapshotv1api.VolumeSnapshotContent, error) { backupVSCName := ownerObject.Name vsc := &snapshotv1api.VolumeSnapshotContent{ ObjectMeta: metav1.ObjectMeta{ Name: backupVSCName, Annotations: snapshotVSC.Annotations, Labels: map[string]string{}, }, Spec: snapshotv1api.VolumeSnapshotContentSpec{ VolumeSnapshotRef: corev1api.ObjectReference{ Name: vs.Name, Namespace: vs.Namespace, UID: vs.UID, ResourceVersion: vs.ResourceVersion, }, Source: snapshotv1api.VolumeSnapshotContentSource{ SnapshotHandle: snapshotVSC.Status.SnapshotHandle, }, DeletionPolicy: snapshotv1api.VolumeSnapshotContentDelete, Driver: snapshotVSC.Spec.Driver, VolumeSnapshotClassName: snapshotVSC.Spec.VolumeSnapshotClassName, }, } /* We need to keep the label of the managing node for distributed snapshots. The external snapshot manager will only manage snapshots matching it's node if that feature is enabled. https://github.com/kubernetes-csi/external-snapshotter/tree/4cedb3f45790ac593ebfa3324c490abedf739477?tab=readme-ov-file#distributed-snapshotting https://github.com/kubernetes-csi/external-snapshotter/blob/4cedb3f45790ac593ebfa3324c490abedf739477/pkg/utils/util.go#L158 */ if manager, ok := snapshotVSC.Labels[kube.VolumeSnapshotContentManagedByLabel]; ok { vsc.ObjectMeta.Labels[kube.VolumeSnapshotContentManagedByLabel] = manager } return e.csiSnapshotClient.VolumeSnapshotContents().Create(ctx, vsc, metav1.CreateOptions{}) } func (e *csiSnapshotExposer) createBackupPVC(ctx context.Context, ownerObject corev1api.ObjectReference, backupVS, storageClass, accessMode string, resource resource.Quantity, readOnly bool, annotations map[string]string) (*corev1api.PersistentVolumeClaim, error) { backupPVCName := ownerObject.Name volumeMode, err := getVolumeModeByAccessMode(accessMode) if err != nil { return nil, err } pvcAccessMode := corev1api.ReadWriteOnce if readOnly { pvcAccessMode = corev1api.ReadOnlyMany } dataSource := &corev1api.TypedLocalObjectReference{ APIGroup: &snapshotv1api.SchemeGroupVersion.Group, Kind: "VolumeSnapshot", Name: backupVS, } pvc := &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: ownerObject.Namespace, Name: backupPVCName, Annotations: annotations, OwnerReferences: []metav1.OwnerReference{ { APIVersion: ownerObject.APIVersion, Kind: ownerObject.Kind, Name: ownerObject.Name, UID: ownerObject.UID, Controller: boolptr.True(), }, }, }, Spec: corev1api.PersistentVolumeClaimSpec{ AccessModes: []corev1api.PersistentVolumeAccessMode{ pvcAccessMode, }, StorageClassName: &storageClass, VolumeMode: &volumeMode, DataSource: dataSource, DataSourceRef: nil, Resources: corev1api.VolumeResourceRequirements{ Requests: corev1api.ResourceList{ corev1api.ResourceStorage: resource, }, }, }, } created, err := e.kubeClient.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(ctx, pvc, metav1.CreateOptions{}) if err != nil { return nil, errors.Wrap(err, "error to create pvc") } return created, err } func (e *csiSnapshotExposer) createBackupPod( ctx context.Context, ownerObject corev1api.ObjectReference, backupPVC *corev1api.PersistentVolumeClaim, operationTimeout time.Duration, label map[string]string, annotation map[string]string, toleration []corev1api.Toleration, affinity *kube.LoadAffinity, resources corev1api.ResourceRequirements, backupPVCReadOnly bool, spcNoRelabeling bool, nodeOS string, priorityClassName string, intoleratableNodes []string, volumeTopology *corev1api.NodeSelector, ) (*corev1api.Pod, error) { podName := ownerObject.Name containerName := string(ownerObject.UID) volumeName := string(ownerObject.UID) podInfo, err := getInheritedPodInfo(ctx, e.kubeClient, ownerObject.Namespace, nodeOS) if err != nil { return nil, errors.Wrap(err, "error to get inherited pod info from node-agent") } // Log the priority class if it's set if priorityClassName != "" { e.log.Debugf("Setting priority class %q for data mover pod %s", priorityClassName, podName) } var gracePeriod int64 volumeMounts, volumeDevices, volumePath := kube.MakePodPVCAttachment(volumeName, backupPVC.Spec.VolumeMode, backupPVCReadOnly) volumeMounts = append(volumeMounts, podInfo.volumeMounts...) volumes := []corev1api.Volume{{ Name: volumeName, VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: backupPVC.Name, }, }, }} if backupPVCReadOnly { volumes[0].VolumeSource.PersistentVolumeClaim.ReadOnly = true } volumes = append(volumes, podInfo.volumes...) if label == nil { label = make(map[string]string) } label[podGroupLabel] = podGroupSnapshot volumeMode := corev1api.PersistentVolumeFilesystem if backupPVC.Spec.VolumeMode != nil { volumeMode = *backupPVC.Spec.VolumeMode } args := []string{ fmt.Sprintf("--volume-path=%s", volumePath), fmt.Sprintf("--volume-mode=%s", volumeMode), fmt.Sprintf("--data-upload=%s", ownerObject.Name), fmt.Sprintf("--resource-timeout=%s", operationTimeout.String()), } args = append(args, podInfo.logFormatArgs...) args = append(args, podInfo.logLevelArgs...) if affinity == nil { affinity = &kube.LoadAffinity{} } var securityCtx *corev1api.PodSecurityContext nodeSelector := map[string]string{} podOS := corev1api.PodOS{} if nodeOS == kube.NodeOSWindows { userID := "ContainerAdministrator" securityCtx = &corev1api.PodSecurityContext{ WindowsOptions: &corev1api.WindowsSecurityContextOptions{ RunAsUserName: &userID, }, } podOS.Name = kube.NodeOSWindows affinity.NodeSelector.MatchExpressions = append(affinity.NodeSelector.MatchExpressions, metav1.LabelSelectorRequirement{ Key: kube.NodeOSLabel, Values: []string{kube.NodeOSWindows}, Operator: metav1.LabelSelectorOpIn, }) toleration = append(toleration, []corev1api.Toleration{ { Key: "os", Operator: "Equal", Effect: "NoSchedule", Value: "windows", }, { Key: "os", Operator: "Equal", Effect: "NoExecute", Value: "windows", }, }...) } else { userID := int64(0) securityCtx = &corev1api.PodSecurityContext{ RunAsUser: &userID, } if spcNoRelabeling { securityCtx.SELinuxOptions = &corev1api.SELinuxOptions{ Type: "spc_t", } } podOS.Name = kube.NodeOSLinux affinity.NodeSelector.MatchExpressions = append(affinity.NodeSelector.MatchExpressions, metav1.LabelSelectorRequirement{ Key: kube.NodeOSLabel, Values: []string{kube.NodeOSWindows}, Operator: metav1.LabelSelectorOpNotIn, }) } if len(intoleratableNodes) > 0 { if affinity == nil { affinity = &kube.LoadAffinity{} } affinity.NodeSelector.MatchExpressions = append(affinity.NodeSelector.MatchExpressions, metav1.LabelSelectorRequirement{ Key: "kubernetes.io/hostname", Values: intoleratableNodes, Operator: metav1.LabelSelectorOpNotIn, }) } podAffinity := kube.ToSystemAffinity(affinity, volumeTopology) pod := &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: podName, Namespace: ownerObject.Namespace, OwnerReferences: []metav1.OwnerReference{ { APIVersion: ownerObject.APIVersion, Kind: ownerObject.Kind, Name: ownerObject.Name, UID: ownerObject.UID, Controller: boolptr.True(), }, }, Labels: label, Annotations: annotation, }, Spec: corev1api.PodSpec{ TopologySpreadConstraints: []corev1api.TopologySpreadConstraint{ { MaxSkew: 1, TopologyKey: "kubernetes.io/hostname", WhenUnsatisfiable: corev1api.ScheduleAnyway, LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ podGroupLabel: podGroupSnapshot, }, }, }, }, NodeSelector: nodeSelector, OS: &podOS, Affinity: podAffinity, Containers: []corev1api.Container{ { Name: containerName, Image: podInfo.image, ImagePullPolicy: corev1api.PullNever, Command: []string{ "/velero", "data-mover", "backup", }, Args: args, VolumeMounts: volumeMounts, VolumeDevices: volumeDevices, Env: podInfo.env, EnvFrom: podInfo.envFrom, Resources: resources, }, }, PriorityClassName: priorityClassName, ServiceAccountName: podInfo.serviceAccount, TerminationGracePeriodSeconds: &gracePeriod, Volumes: volumes, RestartPolicy: corev1api.RestartPolicyNever, SecurityContext: securityCtx, Tolerations: toleration, DNSPolicy: podInfo.dnsPolicy, DNSConfig: podInfo.dnsConfig, ImagePullSecrets: podInfo.imagePullSecrets, }, } return e.kubeClient.CoreV1().Pods(ownerObject.Namespace).Create(ctx, pod, metav1.CreateOptions{}) } ================================================ FILE: pkg/exposer/csi_snapshot_priority_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package exposer import ( "testing" "time" snapshotFake "github.com/kubernetes-csi/external-snapshotter/client/v8/clientset/versioned/fake" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" appsv1api "k8s.io/api/apps/v1" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerotest "github.com/vmware-tanzu/velero/pkg/test" "github.com/vmware-tanzu/velero/pkg/util/kube" ) func TestCreateBackupPodWithPriorityClass(t *testing.T) { testCases := []struct { name string nodeAgentConfigMapData string expectedPriorityClass string description string }{ { name: "with priority class in config map", nodeAgentConfigMapData: `{ "priorityClassName": "high-priority" }`, expectedPriorityClass: "high-priority", description: "Should set priority class from node-agent-configmap", }, { name: "without priority class in config map", nodeAgentConfigMapData: `{ "loadAffinity": [] }`, expectedPriorityClass: "", description: "Should have empty priority class when not specified", }, { name: "empty config map", nodeAgentConfigMapData: `{}`, expectedPriorityClass: "", description: "Should handle empty config map gracefully", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { ctx := t.Context() // Create fake Kubernetes client kubeClient := fake.NewSimpleClientset() // Create node-agent daemonset (required for getInheritedPodInfo) daemonSet := &appsv1api.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Name: "node-agent", Namespace: velerov1api.DefaultNamespace, }, Spec: appsv1api.DaemonSetSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ Containers: []corev1api.Container{ { Name: "node-agent", Image: "velero/velero:latest", }, }, }, }, }, } _, err := kubeClient.AppsV1().DaemonSets(velerov1api.DefaultNamespace).Create(ctx, daemonSet, metav1.CreateOptions{}) require.NoError(t, err) // Create node-agent config map configMap := &corev1api.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "node-agent-config", Namespace: velerov1api.DefaultNamespace, }, Data: map[string]string{ "config": tc.nodeAgentConfigMapData, }, } _, err = kubeClient.CoreV1().ConfigMaps(velerov1api.DefaultNamespace).Create(ctx, configMap, metav1.CreateOptions{}) require.NoError(t, err) // Create owner object for the backup pod ownerObject := corev1api.ObjectReference{ APIVersion: velerov1api.SchemeGroupVersion.String(), Kind: "DataUpload", Name: "test-dataupload", Namespace: velerov1api.DefaultNamespace, UID: "test-uid", } // Create a backup PVC backupPVC := &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: "test-backup-pvc", Namespace: velerov1api.DefaultNamespace, }, Spec: corev1api.PersistentVolumeClaimSpec{ AccessModes: []corev1api.PersistentVolumeAccessMode{ corev1api.ReadWriteOnce, }, }, } // Create fake snapshot client fakeSnapshotClient := snapshotFake.NewSimpleClientset() // Create CSI snapshot exposer exposer := &csiSnapshotExposer{ kubeClient: kubeClient, csiSnapshotClient: fakeSnapshotClient.SnapshotV1(), log: velerotest.NewLogger(), } // Call createBackupPod pod, err := exposer.createBackupPod( ctx, ownerObject, backupPVC, time.Minute*5, nil, // labels nil, // annotations nil, // tolerations nil, // affinity corev1api.ResourceRequirements{}, false, // backupPVCReadOnly false, // spcNoRelabeling kube.NodeOSLinux, tc.expectedPriorityClass, nil, nil, ) require.NoError(t, err, tc.description) assert.NotNil(t, pod) assert.Equal(t, tc.expectedPriorityClass, pod.Spec.PriorityClassName, tc.description) }) } } func TestCreateBackupPodWithMissingConfigMap(t *testing.T) { ctx := t.Context() // Create fake Kubernetes client without config map kubeClient := fake.NewSimpleClientset() // Create node-agent daemonset (required for getInheritedPodInfo) daemonSet := &appsv1api.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Name: "node-agent", Namespace: velerov1api.DefaultNamespace, }, Spec: appsv1api.DaemonSetSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ Containers: []corev1api.Container{ { Name: "node-agent", Image: "velero/velero:latest", }, }, }, }, }, } _, err := kubeClient.AppsV1().DaemonSets(velerov1api.DefaultNamespace).Create(ctx, daemonSet, metav1.CreateOptions{}) require.NoError(t, err) // Create owner object for the backup pod ownerObject := corev1api.ObjectReference{ APIVersion: velerov1api.SchemeGroupVersion.String(), Kind: "DataUpload", Name: "test-dataupload", Namespace: velerov1api.DefaultNamespace, UID: "test-uid", } // Create a backup PVC backupPVC := &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: "test-backup-pvc", Namespace: velerov1api.DefaultNamespace, }, Spec: corev1api.PersistentVolumeClaimSpec{ AccessModes: []corev1api.PersistentVolumeAccessMode{ corev1api.ReadWriteOnce, }, }, } // Create fake snapshot client fakeSnapshotClient := snapshotFake.NewSimpleClientset() // Create CSI snapshot exposer exposer := &csiSnapshotExposer{ kubeClient: kubeClient, csiSnapshotClient: fakeSnapshotClient.SnapshotV1(), log: velerotest.NewLogger(), } // Call createBackupPod pod, err := exposer.createBackupPod( ctx, ownerObject, backupPVC, time.Minute*5, nil, // labels nil, // annotations nil, // tolerations nil, // affinity corev1api.ResourceRequirements{}, false, // backupPVCReadOnly false, // spcNoRelabeling kube.NodeOSLinux, "", // empty priority class since config map is missing nil, nil, ) // Should succeed even when config map is missing require.NoError(t, err, "Should succeed even when config map is missing") assert.NotNil(t, pod) assert.Empty(t, pod.Spec.PriorityClassName, "Should have empty priority class when config map is missing") } ================================================ FILE: pkg/exposer/csi_snapshot_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package exposer import ( "fmt" "testing" "time" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" snapshotFake "github.com/kubernetes-csi/external-snapshotter/client/v8/clientset/versioned/fake" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" appsv1api "k8s.io/api/apps/v1" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes/fake" clientTesting "k8s.io/client-go/testing" "k8s.io/utils/pointer" clientFake "sigs.k8s.io/controller-runtime/pkg/client/fake" velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerotest "github.com/vmware-tanzu/velero/pkg/test" velerotypes "github.com/vmware-tanzu/velero/pkg/types" "github.com/vmware-tanzu/velero/pkg/util" "github.com/vmware-tanzu/velero/pkg/util/boolptr" "github.com/vmware-tanzu/velero/pkg/util/kube" storagev1api "k8s.io/api/storage/v1" ) type reactor struct { verb string resource string reactorFunc clientTesting.ReactionFunc } func TestExpose(t *testing.T) { vscName := "fake-vsc" backup := &velerov1.Backup{ TypeMeta: metav1.TypeMeta{ APIVersion: velerov1.SchemeGroupVersion.String(), Kind: "Backup", }, ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1.DefaultNamespace, Name: "fake-backup", UID: "fake-uid", }, } var restoreSize int64 = 123456 scObj := &storagev1api.StorageClass{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-sc", }, } snapshotClass := "fake-snapshot-class" vsObject := &snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-vs", Namespace: "fake-ns", Annotations: map[string]string{ "fake-key-1": "fake-value-1", "fake-key-2": "fake-value-2", }, }, Spec: snapshotv1api.VolumeSnapshotSpec{ Source: snapshotv1api.VolumeSnapshotSource{ VolumeSnapshotContentName: &vscName, }, VolumeSnapshotClassName: &snapshotClass, }, Status: &snapshotv1api.VolumeSnapshotStatus{ BoundVolumeSnapshotContentName: &vscName, ReadyToUse: boolptr.True(), RestoreSize: resource.NewQuantity(restoreSize, ""), }, } vsObjectWithoutRestoreSize := &snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-vs", Namespace: "fake-ns", Annotations: map[string]string{ "fake-key-1": "fake-value-1", "fake-key-2": "fake-value-2", }, }, Spec: snapshotv1api.VolumeSnapshotSpec{ Source: snapshotv1api.VolumeSnapshotSource{ VolumeSnapshotContentName: &vscName, }, VolumeSnapshotClassName: &snapshotClass, }, Status: &snapshotv1api.VolumeSnapshotStatus{ BoundVolumeSnapshotContentName: &vscName, ReadyToUse: boolptr.True(), }, } snapshotHandle := "fake-handle" vscObj := &snapshotv1api.VolumeSnapshotContent{ ObjectMeta: metav1.ObjectMeta{ Name: vscName, Annotations: map[string]string{ "fake-key-3": "fake-value-3", "fake-key-4": "fake-value-4", }, }, Spec: snapshotv1api.VolumeSnapshotContentSpec{ DeletionPolicy: snapshotv1api.VolumeSnapshotContentDelete, Driver: "fake-driver", VolumeSnapshotClassName: &snapshotClass, }, Status: &snapshotv1api.VolumeSnapshotContentStatus{ RestoreSize: &restoreSize, SnapshotHandle: &snapshotHandle, }, } vscObjWithLabels := vscObj vscObjWithLabels.Labels = map[string]string{ "snapshot.storage.kubernetes.io/managed-by": "worker", } daemonSet := &appsv1api.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", Name: "node-agent", }, TypeMeta: metav1.TypeMeta{ Kind: "DaemonSet", APIVersion: appsv1api.SchemeGroupVersion.String(), }, Spec: appsv1api.DaemonSetSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ Containers: []corev1api.Container{ { Name: "node-agent", }, }, }, }, }, } pvName := "pv-1" volumeAttachement1 := &storagev1api.VolumeAttachment{ ObjectMeta: metav1.ObjectMeta{ Name: "va1", }, Spec: storagev1api.VolumeAttachmentSpec{ Source: storagev1api.VolumeAttachmentSource{ PersistentVolumeName: &pvName, }, NodeName: "node-1", }, } volumeAttachement2 := &storagev1api.VolumeAttachment{ ObjectMeta: metav1.ObjectMeta{ Name: "va2", }, Spec: storagev1api.VolumeAttachmentSpec{ Source: storagev1api.VolumeAttachmentSource{ PersistentVolumeName: &pvName, }, NodeName: "node-2", }, } tests := []struct { name string snapshotClientObj []runtime.Object kubeClientObj []runtime.Object ownerBackup *velerov1.Backup exposeParam CSISnapshotExposeParam snapReactors []reactor kubeReactors []reactor err string expectedVolumeSize *resource.Quantity expectedReadOnlyPVC bool expectedBackupPVCStorageClass string expectedAffinity *corev1api.Affinity expectedPVCAnnotation map[string]string }{ { name: "get volume topology fail", ownerBackup: backup, exposeParam: CSISnapshotExposeParam{ SnapshotName: "fake-vs", OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, StorageClass: "fake-sc", SourcePVName: "fake-pv", }, err: "error getting volume topology for PV fake-pv, storage class fake-sc: error getting storage class fake-sc: storageclasses.storage.k8s.io \"fake-sc\" not found", }, { name: "wait vs ready fail", ownerBackup: backup, exposeParam: CSISnapshotExposeParam{ SnapshotName: "fake-vs", OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, StorageClass: "fake-sc", SourcePVName: "fake-pv", }, kubeClientObj: []runtime.Object{ scObj, }, err: "error wait volume snapshot ready: error to get VolumeSnapshot /fake-vs: volumesnapshots.snapshot.storage.k8s.io \"fake-vs\" not found", }, { name: "get vsc fail", ownerBackup: backup, exposeParam: CSISnapshotExposeParam{ SnapshotName: "fake-vs", SourceNamespace: "fake-ns", OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, StorageClass: "fake-sc", SourcePVName: "fake-pv", }, snapshotClientObj: []runtime.Object{ vsObject, }, kubeClientObj: []runtime.Object{ scObj, }, err: "error to get volume snapshot content: error getting volume snapshot content from API: volumesnapshotcontents.snapshot.storage.k8s.io \"fake-vsc\" not found", }, { name: "delete vs fail", ownerBackup: backup, exposeParam: CSISnapshotExposeParam{ SnapshotName: "fake-vs", SourceNamespace: "fake-ns", OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, StorageClass: "fake-sc", SourcePVName: "fake-pv", }, snapshotClientObj: []runtime.Object{ vsObject, vscObj, }, snapReactors: []reactor{ { verb: "delete", resource: "volumesnapshots", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, errors.New("fake-delete-error") }, }, }, kubeClientObj: []runtime.Object{ scObj, }, err: "error to delete volume snapshot: error to delete volume snapshot: fake-delete-error", }, { name: "delete vsc fail", ownerBackup: backup, exposeParam: CSISnapshotExposeParam{ SnapshotName: "fake-vs", SourceNamespace: "fake-ns", OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, StorageClass: "fake-sc", SourcePVName: "fake-pv", }, snapshotClientObj: []runtime.Object{ vsObject, vscObj, }, snapReactors: []reactor{ { verb: "delete", resource: "volumesnapshotcontents", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, errors.New("fake-delete-error") }, }, }, kubeClientObj: []runtime.Object{ scObj, }, err: "error to delete volume snapshot content: error to delete volume snapshot content: fake-delete-error", }, { name: "create backup vs fail", ownerBackup: backup, exposeParam: CSISnapshotExposeParam{ SnapshotName: "fake-vs", SourceNamespace: "fake-ns", OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, StorageClass: "fake-sc", SourcePVName: "fake-pv", }, snapshotClientObj: []runtime.Object{ vsObject, vscObj, }, snapReactors: []reactor{ { verb: "create", resource: "volumesnapshots", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, errors.New("fake-create-error") }, }, }, kubeClientObj: []runtime.Object{ scObj, }, err: "error to create backup volume snapshot: fake-create-error", }, { name: "create backup vsc fail", ownerBackup: backup, exposeParam: CSISnapshotExposeParam{ SnapshotName: "fake-vs", SourceNamespace: "fake-ns", OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, StorageClass: "fake-sc", SourcePVName: "fake-pv", }, snapshotClientObj: []runtime.Object{ vsObject, vscObj, }, snapReactors: []reactor{ { verb: "create", resource: "volumesnapshotcontents", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, errors.New("fake-create-error") }, }, }, kubeClientObj: []runtime.Object{ scObj, }, err: "error to create backup volume snapshot content: fake-create-error", }, { name: "create backup pvc fail, invalid access mode", ownerBackup: backup, exposeParam: CSISnapshotExposeParam{ SnapshotName: "fake-vs", SourceNamespace: "fake-ns", AccessMode: "fake-mode", StorageClass: "fake-sc", SourcePVName: "fake-pv", }, snapshotClientObj: []runtime.Object{ vsObject, vscObj, }, kubeClientObj: []runtime.Object{ scObj, }, err: "error to create backup pvc: unsupported access mode fake-mode", }, { name: "create backup pvc fail", ownerBackup: backup, exposeParam: CSISnapshotExposeParam{ SnapshotName: "fake-vs", SourceNamespace: "fake-ns", OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, AccessMode: AccessModeFileSystem, StorageClass: "fake-sc", SourcePVName: "fake-pv", }, snapshotClientObj: []runtime.Object{ vsObject, vscObj, }, kubeReactors: []reactor{ { verb: "create", resource: "persistentvolumeclaims", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, errors.New("fake-create-error") }, }, }, kubeClientObj: []runtime.Object{ scObj, }, err: "error to create backup pvc: error to create pvc: fake-create-error", }, { name: "create backup pod fail", ownerBackup: backup, exposeParam: CSISnapshotExposeParam{ SnapshotName: "fake-vs", SourceNamespace: "fake-ns", AccessMode: AccessModeFileSystem, OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, StorageClass: "fake-sc", SourcePVName: "fake-pv", }, snapshotClientObj: []runtime.Object{ vsObject, vscObj, }, kubeClientObj: []runtime.Object{ daemonSet, scObj, }, kubeReactors: []reactor{ { verb: "create", resource: "pods", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, errors.New("fake-create-error") }, }, }, err: "error to create backup pod: fake-create-error", }, { name: "success", ownerBackup: backup, exposeParam: CSISnapshotExposeParam{ SnapshotName: "fake-vs", SourceNamespace: "fake-ns", AccessMode: AccessModeFileSystem, OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, StorageClass: "fake-sc", SourcePVName: "fake-pv", }, snapshotClientObj: []runtime.Object{ vsObject, vscObj, }, kubeClientObj: []runtime.Object{ daemonSet, scObj, }, expectedAffinity: &corev1api.Affinity{ NodeAffinity: &corev1api.NodeAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ NodeSelectorTerms: []corev1api.NodeSelectorTerm{ { MatchExpressions: []corev1api.NodeSelectorRequirement{ { Key: "kubernetes.io/os", Operator: corev1api.NodeSelectorOpNotIn, Values: []string{"windows"}, }, }, }, }, }, }, }, }, { name: "success-with-labels", ownerBackup: backup, exposeParam: CSISnapshotExposeParam{ SnapshotName: "fake-vs", SourceNamespace: "fake-ns", AccessMode: AccessModeFileSystem, OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, StorageClass: "fake-sc", SourcePVName: "fake-pv", }, snapshotClientObj: []runtime.Object{ vsObject, vscObjWithLabels, }, kubeClientObj: []runtime.Object{ daemonSet, scObj, }, expectedAffinity: &corev1api.Affinity{ NodeAffinity: &corev1api.NodeAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ NodeSelectorTerms: []corev1api.NodeSelectorTerm{ { MatchExpressions: []corev1api.NodeSelectorRequirement{ { Key: "kubernetes.io/os", Operator: corev1api.NodeSelectorOpNotIn, Values: []string{"windows"}, }, }, }, }, }, }, }, }, { name: "restore size from exposeParam", ownerBackup: backup, exposeParam: CSISnapshotExposeParam{ SnapshotName: "fake-vs", SourceNamespace: "fake-ns", AccessMode: AccessModeFileSystem, OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, VolumeSize: *resource.NewQuantity(567890, ""), StorageClass: "fake-sc", SourcePVName: "fake-pv", }, snapshotClientObj: []runtime.Object{ vsObjectWithoutRestoreSize, vscObj, }, kubeClientObj: []runtime.Object{ daemonSet, scObj, }, expectedVolumeSize: resource.NewQuantity(567890, ""), expectedAffinity: &corev1api.Affinity{ NodeAffinity: &corev1api.NodeAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ NodeSelectorTerms: []corev1api.NodeSelectorTerm{ { MatchExpressions: []corev1api.NodeSelectorRequirement{ { Key: "kubernetes.io/os", Operator: corev1api.NodeSelectorOpNotIn, Values: []string{"windows"}, }, }, }, }, }, }, }, }, { name: "backupPod mounts read only backupPVC", ownerBackup: backup, exposeParam: CSISnapshotExposeParam{ SnapshotName: "fake-vs", SourceNamespace: "fake-ns", StorageClass: "fake-sc", SourcePVName: "fake-pv", AccessMode: AccessModeFileSystem, OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, BackupPVCConfig: map[string]velerotypes.BackupPVC{ "fake-sc": { StorageClass: "fake-sc-read-only", ReadOnly: true, }, }, }, snapshotClientObj: []runtime.Object{ vsObject, vscObj, }, kubeClientObj: []runtime.Object{ daemonSet, scObj, }, expectedReadOnlyPVC: true, expectedAffinity: &corev1api.Affinity{ NodeAffinity: &corev1api.NodeAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ NodeSelectorTerms: []corev1api.NodeSelectorTerm{ { MatchExpressions: []corev1api.NodeSelectorRequirement{ { Key: "kubernetes.io/os", Operator: corev1api.NodeSelectorOpNotIn, Values: []string{"windows"}, }, }, }, }, }, }, }, }, { name: "backupPod mounts read only backupPVC and storageClass specified in backupPVC config", ownerBackup: backup, exposeParam: CSISnapshotExposeParam{ SnapshotName: "fake-vs", SourceNamespace: "fake-ns", StorageClass: "fake-sc", SourcePVName: "fake-pv", AccessMode: AccessModeFileSystem, OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, BackupPVCConfig: map[string]velerotypes.BackupPVC{ "fake-sc": { StorageClass: "fake-sc-read-only", ReadOnly: true, }, }, }, snapshotClientObj: []runtime.Object{ vsObject, vscObj, }, kubeClientObj: []runtime.Object{ daemonSet, scObj, }, expectedReadOnlyPVC: true, expectedBackupPVCStorageClass: "fake-sc-read-only", expectedAffinity: &corev1api.Affinity{ NodeAffinity: &corev1api.NodeAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ NodeSelectorTerms: []corev1api.NodeSelectorTerm{ { MatchExpressions: []corev1api.NodeSelectorRequirement{ { Key: "kubernetes.io/os", Operator: corev1api.NodeSelectorOpNotIn, Values: []string{"windows"}, }, }, }, }, }, }, }, }, { name: "backupPod mounts backupPVC with storageClass specified in backupPVC config", ownerBackup: backup, exposeParam: CSISnapshotExposeParam{ SnapshotName: "fake-vs", SourceNamespace: "fake-ns", StorageClass: "fake-sc", SourcePVName: "fake-pv", AccessMode: AccessModeFileSystem, OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, BackupPVCConfig: map[string]velerotypes.BackupPVC{ "fake-sc": { StorageClass: "fake-sc-read-only", }, }, }, snapshotClientObj: []runtime.Object{ vsObject, vscObj, }, kubeClientObj: []runtime.Object{ daemonSet, scObj, }, expectedBackupPVCStorageClass: "fake-sc-read-only", expectedAffinity: &corev1api.Affinity{ NodeAffinity: &corev1api.NodeAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ NodeSelectorTerms: []corev1api.NodeSelectorTerm{ { MatchExpressions: []corev1api.NodeSelectorRequirement{ { Key: "kubernetes.io/os", Operator: corev1api.NodeSelectorOpNotIn, Values: []string{"windows"}, }, }, }, }, }, }, }, }, { name: "Affinity per StorageClass", ownerBackup: backup, exposeParam: CSISnapshotExposeParam{ SnapshotName: "fake-vs", SourceNamespace: "fake-ns", StorageClass: "fake-sc", SourcePVName: "fake-pv", AccessMode: AccessModeFileSystem, OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, Affinity: []*kube.LoadAffinity{ { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { Key: "kubernetes.io/os", Operator: metav1.LabelSelectorOpIn, Values: []string{"Linux"}, }, }, }, StorageClass: "fake-sc", }, }, }, snapshotClientObj: []runtime.Object{ vsObject, vscObj, }, kubeClientObj: []runtime.Object{ daemonSet, scObj, }, expectedAffinity: &corev1api.Affinity{ NodeAffinity: &corev1api.NodeAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ NodeSelectorTerms: []corev1api.NodeSelectorTerm{ { MatchExpressions: []corev1api.NodeSelectorRequirement{ { Key: "kubernetes.io/os", Operator: corev1api.NodeSelectorOpIn, Values: []string{"Linux"}, }, { Key: "kubernetes.io/os", Operator: corev1api.NodeSelectorOpNotIn, Values: []string{"windows"}, }, }, }, }, }, }, }, }, { name: "Affinity per StorageClass with expectedBackupPVCStorageClass", ownerBackup: backup, exposeParam: CSISnapshotExposeParam{ SnapshotName: "fake-vs", SourceNamespace: "fake-ns", StorageClass: "fake-sc", SourcePVName: "fake-pv", AccessMode: AccessModeFileSystem, OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, BackupPVCConfig: map[string]velerotypes.BackupPVC{ "fake-sc": { StorageClass: "fake-sc-read-only", }, }, Affinity: []*kube.LoadAffinity{ { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { Key: "kubernetes.io/arch", Operator: metav1.LabelSelectorOpIn, Values: []string{"amd64"}, }, }, }, StorageClass: "fake-sc-read-only", }, }, }, snapshotClientObj: []runtime.Object{ vsObject, vscObj, }, kubeClientObj: []runtime.Object{ daemonSet, scObj, }, expectedBackupPVCStorageClass: "fake-sc-read-only", expectedAffinity: &corev1api.Affinity{ NodeAffinity: &corev1api.NodeAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ NodeSelectorTerms: []corev1api.NodeSelectorTerm{ { MatchExpressions: []corev1api.NodeSelectorRequirement{ { Key: "kubernetes.io/arch", Operator: corev1api.NodeSelectorOpIn, Values: []string{"amd64"}, }, { Key: "kubernetes.io/os", Operator: corev1api.NodeSelectorOpNotIn, Values: []string{"windows"}, }, }, }, }, }, }, }, }, { name: "Affinity in exposeParam is nil", ownerBackup: backup, exposeParam: CSISnapshotExposeParam{ SnapshotName: "fake-vs", SourceNamespace: "fake-ns", StorageClass: "fake-sc", SourcePVName: "fake-pv", AccessMode: AccessModeFileSystem, OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, BackupPVCConfig: map[string]velerotypes.BackupPVC{ "fake-sc": { StorageClass: "fake-sc-read-only", }, }, Affinity: nil, }, snapshotClientObj: []runtime.Object{ vsObject, vscObj, }, kubeClientObj: []runtime.Object{ daemonSet, scObj, }, expectedBackupPVCStorageClass: "fake-sc-read-only", expectedAffinity: &corev1api.Affinity{ NodeAffinity: &corev1api.NodeAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ NodeSelectorTerms: []corev1api.NodeSelectorTerm{ { MatchExpressions: []corev1api.NodeSelectorRequirement{ { Key: "kubernetes.io/os", Operator: corev1api.NodeSelectorOpNotIn, Values: []string{"windows"}, }, }, }, }, }, }, }, }, { name: "IntolerateSourceNode, get source node fail", ownerBackup: backup, exposeParam: CSISnapshotExposeParam{ SnapshotName: "fake-vs", SourceNamespace: "fake-ns", SourcePVName: pvName, StorageClass: "fake-sc", AccessMode: AccessModeFileSystem, OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, BackupPVCConfig: map[string]velerotypes.BackupPVC{ "fake-sc": { Annotations: map[string]string{util.VSphereCNSFastCloneAnno: "true"}, }, }, Affinity: nil, }, snapshotClientObj: []runtime.Object{ vsObject, vscObj, }, kubeClientObj: []runtime.Object{ daemonSet, scObj, }, kubeReactors: []reactor{ { verb: "list", resource: "volumeattachments", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, errors.New("fake-create-error") }, }, }, expectedAffinity: &corev1api.Affinity{ NodeAffinity: &corev1api.NodeAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ NodeSelectorTerms: []corev1api.NodeSelectorTerm{ { MatchExpressions: []corev1api.NodeSelectorRequirement{ { Key: "kubernetes.io/os", Operator: corev1api.NodeSelectorOpNotIn, Values: []string{"windows"}, }, }, }, }, }, }, }, expectedPVCAnnotation: nil, }, { name: "IntolerateSourceNode, get empty source node", ownerBackup: backup, exposeParam: CSISnapshotExposeParam{ SnapshotName: "fake-vs", SourceNamespace: "fake-ns", SourcePVName: pvName, StorageClass: "fake-sc", AccessMode: AccessModeFileSystem, OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, BackupPVCConfig: map[string]velerotypes.BackupPVC{ "fake-sc": { Annotations: map[string]string{util.VSphereCNSFastCloneAnno: "true"}, }, }, Affinity: nil, }, snapshotClientObj: []runtime.Object{ vsObject, vscObj, }, kubeClientObj: []runtime.Object{ daemonSet, scObj, }, expectedAffinity: &corev1api.Affinity{ NodeAffinity: &corev1api.NodeAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ NodeSelectorTerms: []corev1api.NodeSelectorTerm{ { MatchExpressions: []corev1api.NodeSelectorRequirement{ { Key: "kubernetes.io/os", Operator: corev1api.NodeSelectorOpNotIn, Values: []string{"windows"}, }, }, }, }, }, }, }, expectedPVCAnnotation: map[string]string{util.VSphereCNSFastCloneAnno: "true"}, }, { name: "IntolerateSourceNode, get source nodes", ownerBackup: backup, exposeParam: CSISnapshotExposeParam{ SnapshotName: "fake-vs", SourceNamespace: "fake-ns", SourcePVName: pvName, StorageClass: "fake-sc", AccessMode: AccessModeFileSystem, OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, BackupPVCConfig: map[string]velerotypes.BackupPVC{ "fake-sc": { Annotations: map[string]string{util.VSphereCNSFastCloneAnno: "true"}, }, }, Affinity: nil, }, snapshotClientObj: []runtime.Object{ vsObject, vscObj, }, kubeClientObj: []runtime.Object{ daemonSet, volumeAttachement1, volumeAttachement2, scObj, }, expectedAffinity: &corev1api.Affinity{ NodeAffinity: &corev1api.NodeAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ NodeSelectorTerms: []corev1api.NodeSelectorTerm{ { MatchExpressions: []corev1api.NodeSelectorRequirement{ { Key: "kubernetes.io/os", Operator: corev1api.NodeSelectorOpNotIn, Values: []string{"windows"}, }, { Key: "kubernetes.io/hostname", Operator: corev1api.NodeSelectorOpNotIn, Values: []string{"node-1", "node-2"}, }, }, }, }, }, }, }, expectedPVCAnnotation: map[string]string{util.VSphereCNSFastCloneAnno: "true"}, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeSnapshotClient := snapshotFake.NewSimpleClientset(test.snapshotClientObj...) fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...) for _, reactor := range test.snapReactors { fakeSnapshotClient.Fake.PrependReactor(reactor.verb, reactor.resource, reactor.reactorFunc) } for _, reactor := range test.kubeReactors { fakeKubeClient.Fake.PrependReactor(reactor.verb, reactor.resource, reactor.reactorFunc) } exposer := csiSnapshotExposer{ kubeClient: fakeKubeClient, csiSnapshotClient: fakeSnapshotClient.SnapshotV1(), log: velerotest.NewLogger(), } var ownerObject corev1api.ObjectReference if test.ownerBackup != nil { ownerObject = corev1api.ObjectReference{ Kind: test.ownerBackup.Kind, Namespace: test.ownerBackup.Namespace, Name: test.ownerBackup.Name, UID: test.ownerBackup.UID, APIVersion: test.ownerBackup.APIVersion, } } err := exposer.Expose(t.Context(), ownerObject, &test.exposeParam) if err == nil { require.NoError(t, err) backupPod, err := exposer.kubeClient.CoreV1().Pods(ownerObject.Namespace).Get(t.Context(), ownerObject.Name, metav1.GetOptions{}) require.NoError(t, err) backupPVC, err := exposer.kubeClient.CoreV1().PersistentVolumeClaims(ownerObject.Namespace).Get(t.Context(), ownerObject.Name, metav1.GetOptions{}) require.NoError(t, err) expectedVS, err := exposer.csiSnapshotClient.VolumeSnapshots(ownerObject.Namespace).Get(t.Context(), ownerObject.Name, metav1.GetOptions{}) require.NoError(t, err) expectedVSC, err := exposer.csiSnapshotClient.VolumeSnapshotContents().Get(t.Context(), ownerObject.Name, metav1.GetOptions{}) require.NoError(t, err) assert.Equal(t, expectedVS.Annotations, vsObject.Annotations) assert.Equal(t, *expectedVS.Spec.VolumeSnapshotClassName, *vsObject.Spec.VolumeSnapshotClassName) assert.Equal(t, expectedVSC.Name, *expectedVS.Spec.Source.VolumeSnapshotContentName) assert.Equal(t, expectedVSC.Annotations, vscObj.Annotations) assert.Equal(t, expectedVSC.Labels, vscObj.Labels) assert.Equal(t, expectedVSC.Spec.DeletionPolicy, vscObj.Spec.DeletionPolicy) assert.Equal(t, expectedVSC.Spec.Driver, vscObj.Spec.Driver) assert.Equal(t, *expectedVSC.Spec.VolumeSnapshotClassName, *vscObj.Spec.VolumeSnapshotClassName) if test.expectedVolumeSize != nil { assert.Equal(t, *test.expectedVolumeSize, backupPVC.Spec.Resources.Requests[corev1api.ResourceStorage]) } else { assert.Equal(t, *resource.NewQuantity(restoreSize, ""), backupPVC.Spec.Resources.Requests[corev1api.ResourceStorage]) } if test.expectedReadOnlyPVC { gotReadOnlyAccessMode := false for _, accessMode := range backupPVC.Spec.AccessModes { if accessMode == corev1api.ReadOnlyMany { gotReadOnlyAccessMode = true } } assert.Equal(t, test.expectedReadOnlyPVC, gotReadOnlyAccessMode) } if test.expectedBackupPVCStorageClass != "" { assert.Equal(t, test.expectedBackupPVCStorageClass, *backupPVC.Spec.StorageClassName) } if test.expectedAffinity != nil { assert.Equal(t, test.expectedAffinity, backupPod.Spec.Affinity) } else { assert.Nil(t, backupPod.Spec.Affinity) } if test.expectedPVCAnnotation != nil { assert.Equal(t, test.expectedPVCAnnotation, backupPVC.Annotations) } else { assert.Empty(t, backupPVC.Annotations) } } else { assert.EqualError(t, err, test.err) } }) } } func TestGetExpose(t *testing.T) { backup := &velerov1.Backup{ TypeMeta: metav1.TypeMeta{ APIVersion: velerov1.SchemeGroupVersion.String(), Kind: "Backup", }, ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1.DefaultNamespace, Name: "fake-backup", UID: "fake-uid", }, } backupPod := &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: backup.Namespace, Name: backup.Name, }, Spec: corev1api.PodSpec{ Volumes: []corev1api.Volume{ { Name: "fake-volume", }, { Name: "fake-volume-2", }, { Name: string(backup.UID), }, }, }, } backupPodWithoutVolume := &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: backup.Namespace, Name: backup.Name, }, Spec: corev1api.PodSpec{ Volumes: []corev1api.Volume{ { Name: "fake-volume-1", }, { Name: "fake-volume-2", }, }, }, } backupPVC := &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: backup.Namespace, Name: backup.Name, }, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "fake-pv-name", }, } backupPV := &corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-pv-name", }, } scheme := runtime.NewScheme() corev1api.AddToScheme(scheme) tests := []struct { name string kubeClientObj []runtime.Object ownerBackup *velerov1.Backup exposeWaitParam CSISnapshotExposeWaitParam Timeout time.Duration err string expectedResult *ExposeResult }{ { name: "backup pod is not found", ownerBackup: backup, exposeWaitParam: CSISnapshotExposeWaitParam{ NodeName: "fake-node", }, }, { name: "wait pvc bound fail", ownerBackup: backup, exposeWaitParam: CSISnapshotExposeWaitParam{ NodeName: "fake-node", }, kubeClientObj: []runtime.Object{ backupPod, }, Timeout: time.Second, err: "error to wait backup PVC bound, fake-backup: error to wait for rediness of PVC: error to get pvc velero/fake-backup: persistentvolumeclaims \"fake-backup\" not found", }, { name: "backup volume not found in pod", ownerBackup: backup, exposeWaitParam: CSISnapshotExposeWaitParam{ NodeName: "fake-node", }, kubeClientObj: []runtime.Object{ backupPodWithoutVolume, backupPVC, backupPV, }, Timeout: time.Second, err: "backup pod fake-backup doesn't have the expected backup volume", }, { name: "succeed", ownerBackup: backup, exposeWaitParam: CSISnapshotExposeWaitParam{ NodeName: "fake-node", }, kubeClientObj: []runtime.Object{ backupPod, backupPVC, backupPV, }, Timeout: time.Second, expectedResult: &ExposeResult{ ByPod: ExposeByPod{ HostingPod: backupPod, VolumeName: string(backup.UID), }, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...) fakeClientBuilder := clientFake.NewClientBuilder() fakeClientBuilder = fakeClientBuilder.WithScheme(scheme) fakeClient := fakeClientBuilder.WithRuntimeObjects(test.kubeClientObj...).Build() exposer := csiSnapshotExposer{ kubeClient: fakeKubeClient, log: velerotest.NewLogger(), } var ownerObject corev1api.ObjectReference if test.ownerBackup != nil { ownerObject = corev1api.ObjectReference{ Kind: test.ownerBackup.Kind, Namespace: test.ownerBackup.Namespace, Name: test.ownerBackup.Name, UID: test.ownerBackup.UID, APIVersion: test.ownerBackup.APIVersion, } } test.exposeWaitParam.NodeClient = fakeClient result, err := exposer.GetExposed(t.Context(), ownerObject, test.Timeout, &test.exposeWaitParam) if test.err == "" { require.NoError(t, err) if test.expectedResult == nil { assert.Nil(t, result) } else { require.NoError(t, err) assert.Equal(t, test.expectedResult.ByPod.VolumeName, result.ByPod.VolumeName) assert.Equal(t, test.expectedResult.ByPod.HostingPod.Name, result.ByPod.HostingPod.Name) } } else { assert.EqualError(t, err, test.err) } }) } } func TestPeekExpose(t *testing.T) { backup := &velerov1.Backup{ TypeMeta: metav1.TypeMeta{ APIVersion: velerov1.SchemeGroupVersion.String(), Kind: "Backup", }, ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1.DefaultNamespace, Name: "fake-backup", UID: "fake-uid", }, } backupPodUrecoverable := &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: backup.Namespace, Name: backup.Name, }, Status: corev1api.PodStatus{ Phase: corev1api.PodFailed, }, } backupPod := &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: backup.Namespace, Name: backup.Name, }, } scheme := runtime.NewScheme() corev1api.AddToScheme(scheme) tests := []struct { name string kubeClientObj []runtime.Object ownerBackup *velerov1.Backup err string }{ { name: "backup pod is not found", ownerBackup: backup, }, { name: "pod is unrecoverable", ownerBackup: backup, kubeClientObj: []runtime.Object{ backupPodUrecoverable, }, err: "Pod is in abnormal state [Failed], message []", }, { name: "succeed", ownerBackup: backup, kubeClientObj: []runtime.Object{ backupPod, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...) exposer := csiSnapshotExposer{ kubeClient: fakeKubeClient, log: velerotest.NewLogger(), } var ownerObject corev1api.ObjectReference if test.ownerBackup != nil { ownerObject = corev1api.ObjectReference{ Kind: test.ownerBackup.Kind, Namespace: test.ownerBackup.Namespace, Name: test.ownerBackup.Name, UID: test.ownerBackup.UID, APIVersion: test.ownerBackup.APIVersion, } } err := exposer.PeekExposed(t.Context(), ownerObject) if test.err == "" { assert.NoError(t, err) } else { assert.EqualError(t, err, test.err) } }) } } func Test_csiSnapshotExposer_createBackupPVC(t *testing.T) { backup := &velerov1.Backup{ TypeMeta: metav1.TypeMeta{ APIVersion: velerov1.SchemeGroupVersion.String(), Kind: "Backup", }, ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1.DefaultNamespace, Name: "fake-backup", UID: "fake-uid", }, } dataSource := &corev1api.TypedLocalObjectReference{ APIGroup: &snapshotv1api.SchemeGroupVersion.Group, Kind: "VolumeSnapshot", Name: "fake-snapshot", } volumeMode := corev1api.PersistentVolumeFilesystem backupPVC := corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1.DefaultNamespace, Name: "fake-backup", Annotations: map[string]string{}, OwnerReferences: []metav1.OwnerReference{ { APIVersion: backup.APIVersion, Kind: backup.Kind, Name: backup.Name, UID: backup.UID, Controller: pointer.BoolPtr(true), }, }, }, Spec: corev1api.PersistentVolumeClaimSpec{ AccessModes: []corev1api.PersistentVolumeAccessMode{ corev1api.ReadWriteOnce, }, VolumeMode: &volumeMode, DataSource: dataSource, DataSourceRef: nil, StorageClassName: pointer.String("fake-storage-class"), Resources: corev1api.VolumeResourceRequirements{ Requests: corev1api.ResourceList{ corev1api.ResourceStorage: resource.MustParse("1Gi"), }, }, }, } backupPVCReadOnly := corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1.DefaultNamespace, Name: "fake-backup", Annotations: map[string]string{}, OwnerReferences: []metav1.OwnerReference{ { APIVersion: backup.APIVersion, Kind: backup.Kind, Name: backup.Name, UID: backup.UID, Controller: pointer.BoolPtr(true), }, }, }, Spec: corev1api.PersistentVolumeClaimSpec{ AccessModes: []corev1api.PersistentVolumeAccessMode{ corev1api.ReadOnlyMany, }, VolumeMode: &volumeMode, DataSource: dataSource, DataSourceRef: nil, StorageClassName: pointer.String("fake-storage-class"), Resources: corev1api.VolumeResourceRequirements{ Requests: corev1api.ResourceList{ corev1api.ResourceStorage: resource.MustParse("1Gi"), }, }, }, } tests := []struct { name string ownerBackup *velerov1.Backup backupVS string storageClass string accessMode string resource resource.Quantity readOnly bool kubeClientObj []runtime.Object snapshotClientObj []runtime.Object want *corev1api.PersistentVolumeClaim wantErr assert.ErrorAssertionFunc }{ { name: "backupPVC gets created successfully with parameters from source PVC", ownerBackup: backup, backupVS: "fake-snapshot", storageClass: "fake-storage-class", accessMode: AccessModeFileSystem, resource: resource.MustParse("1Gi"), readOnly: false, want: &backupPVC, wantErr: assert.NoError, }, { name: "backupPVC gets created successfully with parameters from source PVC but accessMode from backupPVC Config as read only", ownerBackup: backup, backupVS: "fake-snapshot", storageClass: "fake-storage-class", accessMode: AccessModeFileSystem, resource: resource.MustParse("1Gi"), readOnly: true, want: &backupPVCReadOnly, wantErr: assert.NoError, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { fakeKubeClient := fake.NewSimpleClientset(tt.kubeClientObj...) fakeSnapshotClient := snapshotFake.NewSimpleClientset(tt.snapshotClientObj...) e := &csiSnapshotExposer{ kubeClient: fakeKubeClient, csiSnapshotClient: fakeSnapshotClient.SnapshotV1(), log: velerotest.NewLogger(), } var ownerObject corev1api.ObjectReference if tt.ownerBackup != nil { ownerObject = corev1api.ObjectReference{ Kind: tt.ownerBackup.Kind, Namespace: tt.ownerBackup.Namespace, Name: tt.ownerBackup.Name, UID: tt.ownerBackup.UID, APIVersion: tt.ownerBackup.APIVersion, } } got, err := e.createBackupPVC(t.Context(), ownerObject, tt.backupVS, tt.storageClass, tt.accessMode, tt.resource, tt.readOnly, map[string]string{}) if !tt.wantErr(t, err, fmt.Sprintf("createBackupPVC(%v, %v, %v, %v, %v, %v)", ownerObject, tt.backupVS, tt.storageClass, tt.accessMode, tt.resource, tt.readOnly)) { return } assert.Equalf(t, tt.want, got, "createBackupPVC(%v, %v, %v, %v, %v, %v)", ownerObject, tt.backupVS, tt.storageClass, tt.accessMode, tt.resource, tt.readOnly) }) } } func Test_csiSnapshotExposer_DiagnoseExpose(t *testing.T) { backup := &velerov1.Backup{ TypeMeta: metav1.TypeMeta{ APIVersion: velerov1.SchemeGroupVersion.String(), Kind: "Backup", }, ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1.DefaultNamespace, Name: "fake-backup", UID: "fake-uid", }, } backupPodWithoutNodeName := corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1.DefaultNamespace, Name: "fake-backup", UID: "fake-pod-uid", OwnerReferences: []metav1.OwnerReference{ { APIVersion: backup.APIVersion, Kind: backup.Kind, Name: backup.Name, UID: backup.UID, }, }, }, Status: corev1api.PodStatus{ Phase: corev1api.PodPending, Conditions: []corev1api.PodCondition{ { Type: corev1api.PodInitialized, Status: corev1api.ConditionTrue, Message: "fake-pod-message", }, }, Message: "fake-pod-message-1", }, } backupPodWithNodeName := corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1.DefaultNamespace, Name: "fake-backup", UID: "fake-pod-uid", OwnerReferences: []metav1.OwnerReference{ { APIVersion: backup.APIVersion, Kind: backup.Kind, Name: backup.Name, UID: backup.UID, }, }, }, Spec: corev1api.PodSpec{ NodeName: "fake-node", }, Status: corev1api.PodStatus{ Phase: corev1api.PodPending, Conditions: []corev1api.PodCondition{ { Type: corev1api.PodInitialized, Status: corev1api.ConditionTrue, Message: "fake-pod-message", }, }, }, } backupPVCWithoutVolumeName := corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1.DefaultNamespace, Name: "fake-backup", UID: "fake-pvc-uid", OwnerReferences: []metav1.OwnerReference{ { APIVersion: backup.APIVersion, Kind: backup.Kind, Name: backup.Name, UID: backup.UID, }, }, }, Status: corev1api.PersistentVolumeClaimStatus{ Phase: corev1api.ClaimPending, }, } backupPVCWithVolumeName := corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1.DefaultNamespace, Name: "fake-backup", UID: "fake-pvc-uid", OwnerReferences: []metav1.OwnerReference{ { APIVersion: backup.APIVersion, Kind: backup.Kind, Name: backup.Name, UID: backup.UID, }, }, }, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "fake-pv", }, Status: corev1api.PersistentVolumeClaimStatus{ Phase: corev1api.ClaimPending, }, } backupPV := corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-pv", }, Status: corev1api.PersistentVolumeStatus{ Phase: corev1api.VolumePending, Message: "fake-pv-message", }, } readyToUse := false vscMessage := "fake-vsc-message" backupVSC := snapshotv1api.VolumeSnapshotContent{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-vsc", }, Status: &snapshotv1api.VolumeSnapshotContentStatus{ ReadyToUse: &readyToUse, Error: &snapshotv1api.VolumeSnapshotError{ Message: &vscMessage, }, }, } backupVSWithoutStatus := snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1.DefaultNamespace, Name: "fake-backup", UID: "fake-vs-uid", OwnerReferences: []metav1.OwnerReference{ { APIVersion: backup.APIVersion, Kind: backup.Kind, Name: backup.Name, UID: backup.UID, }, }, }, } backupVSWithoutVSC := snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1.DefaultNamespace, Name: "fake-backup", UID: "fake-vs-uid", OwnerReferences: []metav1.OwnerReference{ { APIVersion: backup.APIVersion, Kind: backup.Kind, Name: backup.Name, UID: backup.UID, }, }, }, Status: &snapshotv1api.VolumeSnapshotStatus{}, } vsMessage := "fake-vs-message" backupVSWithVSC := snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1.DefaultNamespace, Name: "fake-backup", UID: "fake-vs-uid", OwnerReferences: []metav1.OwnerReference{ { APIVersion: backup.APIVersion, Kind: backup.Kind, Name: backup.Name, UID: backup.UID, }, }, }, Status: &snapshotv1api.VolumeSnapshotStatus{ BoundVolumeSnapshotContentName: &backupVSC.Name, Error: &snapshotv1api.VolumeSnapshotError{ Message: &vsMessage, }, }, } nodeAgentPod := corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1.DefaultNamespace, Name: "node-agent-pod-1", Labels: map[string]string{"role": "node-agent"}, }, Spec: corev1api.PodSpec{ NodeName: "fake-node", }, Status: corev1api.PodStatus{ Phase: corev1api.PodRunning, }, } tests := []struct { name string ownerBackup *velerov1.Backup kubeClientObj []runtime.Object snapshotClientObj []runtime.Object expected string }{ { name: "no pod, pvc, vs", ownerBackup: backup, expected: `begin diagnose CSI exposer error getting backup pod fake-backup, err: pods "fake-backup" not found error getting backup pvc fake-backup, err: persistentvolumeclaims "fake-backup" not found error getting backup vs fake-backup, err: volumesnapshots.snapshot.storage.k8s.io "fake-backup" not found end diagnose CSI exposer`, }, { name: "pod without node name, pvc without volume name, vs without status", ownerBackup: backup, kubeClientObj: []runtime.Object{ &backupPodWithoutNodeName, &backupPVCWithoutVolumeName, }, snapshotClientObj: []runtime.Object{ &backupVSWithoutStatus, }, expected: `begin diagnose CSI exposer Pod velero/fake-backup, phase Pending, node name , message fake-pod-message-1 Pod condition Initialized, status True, reason , message fake-pod-message PVC velero/fake-backup, phase Pending, binding to VS velero/fake-backup, bind to , readyToUse false, errMessage end diagnose CSI exposer`, }, { name: "pod without node name, pvc without volume name, vs without VSC", ownerBackup: backup, kubeClientObj: []runtime.Object{ &backupPodWithoutNodeName, &backupPVCWithoutVolumeName, }, snapshotClientObj: []runtime.Object{ &backupVSWithoutVSC, }, expected: `begin diagnose CSI exposer Pod velero/fake-backup, phase Pending, node name , message fake-pod-message-1 Pod condition Initialized, status True, reason , message fake-pod-message PVC velero/fake-backup, phase Pending, binding to VS velero/fake-backup, bind to , readyToUse false, errMessage end diagnose CSI exposer`, }, { name: "pod with node name, no node agent", ownerBackup: backup, kubeClientObj: []runtime.Object{ &backupPodWithNodeName, &backupPVCWithoutVolumeName, }, snapshotClientObj: []runtime.Object{ &backupVSWithoutVSC, }, expected: `begin diagnose CSI exposer Pod velero/fake-backup, phase Pending, node name fake-node, message Pod condition Initialized, status True, reason , message fake-pod-message node-agent is not running in node fake-node, err: daemonset pod not found in running state in node fake-node PVC velero/fake-backup, phase Pending, binding to VS velero/fake-backup, bind to , readyToUse false, errMessage end diagnose CSI exposer`, }, { name: "pod with node name, node agent is running", ownerBackup: backup, kubeClientObj: []runtime.Object{ &backupPodWithNodeName, &backupPVCWithoutVolumeName, &nodeAgentPod, }, snapshotClientObj: []runtime.Object{ &backupVSWithoutVSC, }, expected: `begin diagnose CSI exposer Pod velero/fake-backup, phase Pending, node name fake-node, message Pod condition Initialized, status True, reason , message fake-pod-message PVC velero/fake-backup, phase Pending, binding to VS velero/fake-backup, bind to , readyToUse false, errMessage end diagnose CSI exposer`, }, { name: "pvc with volume name, no pv", ownerBackup: backup, kubeClientObj: []runtime.Object{ &backupPodWithNodeName, &backupPVCWithVolumeName, &nodeAgentPod, }, snapshotClientObj: []runtime.Object{ &backupVSWithoutVSC, }, expected: `begin diagnose CSI exposer Pod velero/fake-backup, phase Pending, node name fake-node, message Pod condition Initialized, status True, reason , message fake-pod-message PVC velero/fake-backup, phase Pending, binding to fake-pv error getting backup pv fake-pv, err: persistentvolumes "fake-pv" not found VS velero/fake-backup, bind to , readyToUse false, errMessage end diagnose CSI exposer`, }, { name: "pvc with volume name, pv exists", ownerBackup: backup, kubeClientObj: []runtime.Object{ &backupPodWithNodeName, &backupPVCWithVolumeName, &backupPV, &nodeAgentPod, }, snapshotClientObj: []runtime.Object{ &backupVSWithoutVSC, }, expected: `begin diagnose CSI exposer Pod velero/fake-backup, phase Pending, node name fake-node, message Pod condition Initialized, status True, reason , message fake-pod-message PVC velero/fake-backup, phase Pending, binding to fake-pv PV fake-pv, phase Pending, reason , message fake-pv-message VS velero/fake-backup, bind to , readyToUse false, errMessage end diagnose CSI exposer`, }, { name: "vs with vsc, vsc doesn't exist", ownerBackup: backup, kubeClientObj: []runtime.Object{ &backupPodWithNodeName, &backupPVCWithVolumeName, &backupPV, &nodeAgentPod, }, snapshotClientObj: []runtime.Object{ &backupVSWithVSC, }, expected: `begin diagnose CSI exposer Pod velero/fake-backup, phase Pending, node name fake-node, message Pod condition Initialized, status True, reason , message fake-pod-message PVC velero/fake-backup, phase Pending, binding to fake-pv PV fake-pv, phase Pending, reason , message fake-pv-message VS velero/fake-backup, bind to fake-vsc, readyToUse false, errMessage fake-vs-message error getting backup vsc fake-vsc, err: volumesnapshotcontents.snapshot.storage.k8s.io "fake-vsc" not found end diagnose CSI exposer`, }, { name: "vs with vsc, vsc exists", ownerBackup: backup, kubeClientObj: []runtime.Object{ &backupPodWithNodeName, &backupPVCWithVolumeName, &backupPV, &nodeAgentPod, }, snapshotClientObj: []runtime.Object{ &backupVSWithVSC, &backupVSC, }, expected: `begin diagnose CSI exposer Pod velero/fake-backup, phase Pending, node name fake-node, message Pod condition Initialized, status True, reason , message fake-pod-message PVC velero/fake-backup, phase Pending, binding to fake-pv PV fake-pv, phase Pending, reason , message fake-pv-message VS velero/fake-backup, bind to fake-vsc, readyToUse false, errMessage fake-vs-message VSC fake-vsc, readyToUse false, errMessage fake-vsc-message, handle end diagnose CSI exposer`, }, { name: "with events", ownerBackup: backup, kubeClientObj: []runtime.Object{ &backupPodWithNodeName, &backupPVCWithVolumeName, &backupPV, &nodeAgentPod, &corev1api.Event{ ObjectMeta: metav1.ObjectMeta{Namespace: velerov1.DefaultNamespace, Name: "event-1"}, Type: corev1api.EventTypeWarning, InvolvedObject: corev1api.ObjectReference{UID: "fake-uid-1"}, Reason: "reason-1", Message: "message-1", }, &corev1api.Event{ ObjectMeta: metav1.ObjectMeta{Namespace: velerov1.DefaultNamespace, Name: "event-2"}, Type: corev1api.EventTypeWarning, InvolvedObject: corev1api.ObjectReference{UID: "fake-pod-uid"}, Reason: "reason-2", Message: "message-2", }, &corev1api.Event{ ObjectMeta: metav1.ObjectMeta{Namespace: velerov1.DefaultNamespace, Name: "event-3"}, Type: corev1api.EventTypeWarning, InvolvedObject: corev1api.ObjectReference{UID: "fake-pvc-uid"}, Reason: "reason-3", Message: "message-3", }, &corev1api.Event{ ObjectMeta: metav1.ObjectMeta{Namespace: velerov1.DefaultNamespace, Name: "event-4"}, Type: corev1api.EventTypeWarning, InvolvedObject: corev1api.ObjectReference{UID: "fake-vs-uid"}, Reason: "reason-4", Message: "message-4", }, &corev1api.Event{ ObjectMeta: metav1.ObjectMeta{Namespace: "other-namespace", Name: "event-5"}, Type: corev1api.EventTypeWarning, InvolvedObject: corev1api.ObjectReference{UID: "fake-pod-uid"}, Reason: "reason-5", Message: "message-5", }, &corev1api.Event{ ObjectMeta: metav1.ObjectMeta{Namespace: velerov1.DefaultNamespace, Name: "event-6"}, Type: corev1api.EventTypeWarning, InvolvedObject: corev1api.ObjectReference{UID: "fake-pod-uid"}, Reason: "reason-6", Message: "message-6", }, }, snapshotClientObj: []runtime.Object{ &backupVSWithVSC, &backupVSC, }, expected: `begin diagnose CSI exposer Pod velero/fake-backup, phase Pending, node name fake-node, message Pod condition Initialized, status True, reason , message fake-pod-message Pod event reason reason-2, message message-2 Pod event reason reason-6, message message-6 PVC velero/fake-backup, phase Pending, binding to fake-pv PVC event reason reason-3, message message-3 PV fake-pv, phase Pending, reason , message fake-pv-message VS velero/fake-backup, bind to fake-vsc, readyToUse false, errMessage fake-vs-message VS event reason reason-4, message message-4 VSC fake-vsc, readyToUse false, errMessage fake-vsc-message, handle end diagnose CSI exposer`, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { fakeKubeClient := fake.NewSimpleClientset(tt.kubeClientObj...) fakeSnapshotClient := snapshotFake.NewSimpleClientset(tt.snapshotClientObj...) e := &csiSnapshotExposer{ kubeClient: fakeKubeClient, csiSnapshotClient: fakeSnapshotClient.SnapshotV1(), log: velerotest.NewLogger(), } var ownerObject corev1api.ObjectReference if tt.ownerBackup != nil { ownerObject = corev1api.ObjectReference{ Kind: tt.ownerBackup.Kind, Namespace: tt.ownerBackup.Namespace, Name: tt.ownerBackup.Name, UID: tt.ownerBackup.UID, APIVersion: tt.ownerBackup.APIVersion, } } diag := e.DiagnoseExpose(t.Context(), ownerObject) assert.Equal(t, tt.expected, diag) }) } } ================================================ FILE: pkg/exposer/generic_restore.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package exposer import ( "context" "fmt" "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/pkg/nodeagent" velerotypes "github.com/vmware-tanzu/velero/pkg/types" "github.com/vmware-tanzu/velero/pkg/util/boolptr" "github.com/vmware-tanzu/velero/pkg/util/kube" ) // GenericRestoreExposeParam define the input param for Generic Restore Expose type GenericRestoreExposeParam struct { // TargetPVCName is the target volume name to be restored TargetPVCName string // TargetNamespace is the namespace of the volume to be restored TargetNamespace string // HostingPodLabels is the labels that are going to apply to the hosting pod HostingPodLabels map[string]string // HostingPodAnnotations is the annotations that are going to apply to the hosting pod HostingPodAnnotations map[string]string // HostingPodTolerations is the tolerations that are going to apply to the hosting pod HostingPodTolerations []corev1api.Toleration // Resources defines the resource requirements of the hosting pod Resources corev1api.ResourceRequirements // ExposeTimeout specifies the timeout for the entire expose process ExposeTimeout time.Duration // OperationTimeout specifies the time wait for resources operations in Expose OperationTimeout time.Duration // NodeOS specifies the OS of node that the volume should be attached NodeOS string // RestorePVCConfig is the config for restorePVC (intermediate PVC) of generic restore RestorePVCConfig velerotypes.RestorePVC // LoadAffinity specifies the node affinity of the backup pod LoadAffinity []*kube.LoadAffinity // PriorityClassName is the priority class name for the data mover pod PriorityClassName string // RestoreSize specifies the data size for the volume to be restored RestoreSize int64 // CacheVolume specifies the info for cache volumes CacheVolume *CacheConfigs } // GenericRestoreExposer is the interfaces for a generic restore exposer type GenericRestoreExposer interface { // Expose starts the process to a restore expose, the expose process may take long time Expose(context.Context, corev1api.ObjectReference, GenericRestoreExposeParam) error // GetExposed polls the status of the expose. // If the expose is accessible by the current caller, it waits the expose ready and returns the expose result. // Otherwise, it returns nil as the expose result without an error. GetExposed(context.Context, corev1api.ObjectReference, client.Client, string, time.Duration) (*ExposeResult, error) // PeekExposed tests the status of the expose. // If the expose is incomplete but not recoverable, it returns an error. // Otherwise, it returns nil immediately. PeekExposed(context.Context, corev1api.ObjectReference) error // DiagnoseExpose generate the diagnostic info when the expose is not finished for a long time. // If it finds any problem, it returns an string about the problem. DiagnoseExpose(context.Context, corev1api.ObjectReference) string // RebindVolume unexposes the restored PV and rebind it to the target PVC RebindVolume(context.Context, corev1api.ObjectReference, string, string, time.Duration) error // CleanUp cleans up any objects generated during the restore expose CleanUp(context.Context, corev1api.ObjectReference) } // NewGenericRestoreExposer creates a new instance of generic restore exposer func NewGenericRestoreExposer(kubeClient kubernetes.Interface, log logrus.FieldLogger) GenericRestoreExposer { return &genericRestoreExposer{ kubeClient: kubeClient, log: log, } } type genericRestoreExposer struct { kubeClient kubernetes.Interface log logrus.FieldLogger } func (e *genericRestoreExposer) Expose(ctx context.Context, ownerObject corev1api.ObjectReference, param GenericRestoreExposeParam) error { curLog := e.log.WithFields(logrus.Fields{ "owner": ownerObject.Name, "target PVC": param.TargetPVCName, "target namespace": param.TargetNamespace, }) selectedNode, targetPVC, err := kube.WaitPVCConsumed( ctx, e.kubeClient.CoreV1(), param.TargetPVCName, param.TargetNamespace, e.kubeClient.StorageV1(), param.ExposeTimeout, param.RestorePVCConfig.IgnoreDelayBinding, ) if err != nil { return errors.Wrapf(err, "error to wait target PVC consumed, %s/%s", param.TargetNamespace, param.TargetPVCName) } curLog.WithField("target PVC", param.TargetPVCName).WithField("selected node", selectedNode).Info("Target PVC is consumed") if kube.IsPVCBound(targetPVC) { return errors.Errorf("Target PVC %s/%s has already been bound, abort", param.TargetNamespace, param.TargetPVCName) } // Data mover allows the StorageClass name not set for PVC. storageClassName := "" if targetPVC.Spec.StorageClassName != nil { storageClassName = *targetPVC.Spec.StorageClassName } affinity := kube.GetLoadAffinityByStorageClass(param.LoadAffinity, storageClassName, curLog) var cachePVC *corev1api.PersistentVolumeClaim if param.CacheVolume != nil { cacheVolumeSize := getCacheVolumeSize(param.RestoreSize, param.CacheVolume) if cacheVolumeSize > 0 { curLog.Infof("Creating cache PVC with size %v", cacheVolumeSize) if pvc, err := createCachePVC(ctx, e.kubeClient.CoreV1(), ownerObject, param.CacheVolume.StorageClass, cacheVolumeSize, selectedNode); err != nil { return errors.Wrap(err, "error to create cache pvc") } else { cachePVC = pvc } defer func() { if err != nil { kube.DeletePVAndPVCIfAny(ctx, e.kubeClient.CoreV1(), cachePVC.Name, cachePVC.Namespace, 0, curLog) } }() } else { curLog.Infof("Don't need to create cache volume, restore size %v, cache info %v", param.RestoreSize, param.CacheVolume) } } restorePod, err := e.createRestorePod( ctx, ownerObject, targetPVC, param.OperationTimeout, param.HostingPodLabels, param.HostingPodAnnotations, param.HostingPodTolerations, selectedNode, param.Resources, param.NodeOS, affinity, param.PriorityClassName, cachePVC, ) if err != nil { return errors.Wrapf(err, "error to create restore pod") } curLog.WithField("pod name", restorePod.Name).Info("Restore pod is created") defer func() { if err != nil { kube.DeletePodIfAny(ctx, e.kubeClient.CoreV1(), restorePod.Name, restorePod.Namespace, curLog) } }() restorePVC, err := e.createRestorePVC(ctx, ownerObject, targetPVC, selectedNode) if err != nil { return errors.Wrap(err, "error to create restore pvc") } curLog.WithField("pvc name", restorePVC.Name).Info("Restore PVC is created") defer func() { if err != nil { kube.DeletePVAndPVCIfAny(ctx, e.kubeClient.CoreV1(), restorePVC.Name, restorePVC.Namespace, 0, curLog) } }() return nil } func (e *genericRestoreExposer) GetExposed(ctx context.Context, ownerObject corev1api.ObjectReference, nodeClient client.Client, nodeName string, timeout time.Duration) (*ExposeResult, error) { restorePodName := ownerObject.Name restorePVCName := ownerObject.Name containerName := string(ownerObject.UID) volumeName := string(ownerObject.UID) curLog := e.log.WithFields(logrus.Fields{ "owner": ownerObject.Name, "node": nodeName, }) pod := &corev1api.Pod{} err := nodeClient.Get(ctx, types.NamespacedName{ Namespace: ownerObject.Namespace, Name: restorePodName, }, pod) if err != nil { if apierrors.IsNotFound(err) { curLog.WithField("restore pod", restorePodName).Debug("Restore pod is not running in the current node") return nil, nil } else { return nil, errors.Wrapf(err, "error to get restore pod %s", restorePodName) } } curLog.WithField("pod", pod.Name).Infof("Restore pod is in running state in node %s", pod.Spec.NodeName) _, err = kube.WaitPVCBound(ctx, e.kubeClient.CoreV1(), e.kubeClient.CoreV1(), restorePVCName, ownerObject.Namespace, timeout) if err != nil { return nil, errors.Wrapf(err, "error to wait restore PVC bound, %s", restorePVCName) } curLog.WithField("restore pvc", restorePVCName).Info("Restore PVC is bound") i := 0 for i = 0; i < len(pod.Spec.Volumes); i++ { if pod.Spec.Volumes[i].Name == volumeName { break } } if i == len(pod.Spec.Volumes) { return nil, errors.Errorf("restore pod %s doesn't have the expected restore volume", pod.Name) } curLog.WithField("pod", pod.Name).Infof("Restore volume is found in pod at index %v", i) return &ExposeResult{ByPod: ExposeByPod{ HostingPod: pod, HostingContainer: containerName, VolumeName: volumeName, }}, nil } func (e *genericRestoreExposer) PeekExposed(ctx context.Context, ownerObject corev1api.ObjectReference) error { restorePodName := ownerObject.Name curLog := e.log.WithFields(logrus.Fields{ "owner": ownerObject.Name, }) pod, err := e.kubeClient.CoreV1().Pods(ownerObject.Namespace).Get(ctx, restorePodName, metav1.GetOptions{}) if apierrors.IsNotFound(err) { return nil } if err != nil { curLog.WithError(err).Warnf("error to peek restore pod %s", restorePodName) return nil } if podFailed, message := kube.IsPodUnrecoverable(pod, curLog); podFailed { return errors.New(message) } return nil } func (e *genericRestoreExposer) DiagnoseExpose(ctx context.Context, ownerObject corev1api.ObjectReference) string { restorePodName := ownerObject.Name restorePVCName := ownerObject.Name diag := "begin diagnose restore exposer\n" pod, err := e.kubeClient.CoreV1().Pods(ownerObject.Namespace).Get(ctx, restorePodName, metav1.GetOptions{}) if err != nil { pod = nil diag += fmt.Sprintf("error getting restore pod %s, err: %v\n", restorePodName, err) } pvc, err := e.kubeClient.CoreV1().PersistentVolumeClaims(ownerObject.Namespace).Get(ctx, restorePVCName, metav1.GetOptions{}) if err != nil { pvc = nil diag += fmt.Sprintf("error getting restore pvc %s, err: %v\n", restorePVCName, err) } cachePVC, err := e.kubeClient.CoreV1().PersistentVolumeClaims(ownerObject.Namespace).Get(ctx, getCachePVCName(ownerObject), metav1.GetOptions{}) if err != nil { cachePVC = nil if !apierrors.IsNotFound(err) { diag += fmt.Sprintf("error getting cache pvc %s, err: %v\n", getCachePVCName(ownerObject), err) } } events, err := e.kubeClient.CoreV1().Events(ownerObject.Namespace).List(ctx, metav1.ListOptions{}) if err != nil { diag += fmt.Sprintf("error listing events, err: %v\n", err) } if pod != nil { diag += kube.DiagnosePod(pod, events) if pod.Spec.NodeName != "" { if err := nodeagent.KbClientIsRunningInNode(ctx, ownerObject.Namespace, pod.Spec.NodeName, e.kubeClient); err != nil { diag += fmt.Sprintf("node-agent is not running in node %s, err: %v\n", pod.Spec.NodeName, err) } } } if pvc != nil { diag += kube.DiagnosePVC(pvc, events) if pvc.Spec.VolumeName != "" { if pv, err := e.kubeClient.CoreV1().PersistentVolumes().Get(ctx, pvc.Spec.VolumeName, metav1.GetOptions{}); err != nil { diag += fmt.Sprintf("error getting restore pv %s, err: %v\n", pvc.Spec.VolumeName, err) } else { diag += kube.DiagnosePV(pv) } } } if cachePVC != nil { diag += kube.DiagnosePVC(cachePVC, events) if cachePVC.Spec.VolumeName != "" { if pv, err := e.kubeClient.CoreV1().PersistentVolumes().Get(ctx, cachePVC.Spec.VolumeName, metav1.GetOptions{}); err != nil { diag += fmt.Sprintf("error getting cache pv %s, err: %v\n", cachePVC.Spec.VolumeName, err) } else { diag += kube.DiagnosePV(pv) } } } diag += "end diagnose restore exposer" return diag } func (e *genericRestoreExposer) CleanUp(ctx context.Context, ownerObject corev1api.ObjectReference) { restorePodName := ownerObject.Name restorePVCName := ownerObject.Name cachePVCName := getCachePVCName(ownerObject) kube.DeletePodIfAny(ctx, e.kubeClient.CoreV1(), restorePodName, ownerObject.Namespace, e.log) kube.DeletePVAndPVCIfAny(ctx, e.kubeClient.CoreV1(), restorePVCName, ownerObject.Namespace, 0, e.log) kube.DeletePVAndPVCIfAny(ctx, e.kubeClient.CoreV1(), cachePVCName, ownerObject.Namespace, 0, e.log) } func (e *genericRestoreExposer) RebindVolume(ctx context.Context, ownerObject corev1api.ObjectReference, targetPVCName string, targetNamespace string, timeout time.Duration) error { restorePodName := ownerObject.Name restorePVCName := ownerObject.Name curLog := e.log.WithFields(logrus.Fields{ "owner": ownerObject.Name, "target PVC": targetPVCName, "target namespace": targetNamespace, }) targetPVC, err := e.kubeClient.CoreV1().PersistentVolumeClaims(targetNamespace).Get(ctx, targetPVCName, metav1.GetOptions{}) if err != nil { return errors.Wrapf(err, "error to get target PVC %s/%s", targetNamespace, targetPVCName) } restorePV, err := kube.WaitPVCBound(ctx, e.kubeClient.CoreV1(), e.kubeClient.CoreV1(), restorePVCName, ownerObject.Namespace, timeout) if err != nil { return errors.Wrapf(err, "error to get PV from restore PVC %s", restorePVCName) } orgReclaim := restorePV.Spec.PersistentVolumeReclaimPolicy curLog.WithField("restore PV", restorePV.Name).Info("Restore PV is retrieved") retained, err := kube.SetPVReclaimPolicy(ctx, e.kubeClient.CoreV1(), restorePV, corev1api.PersistentVolumeReclaimRetain) if err != nil { return errors.Wrapf(err, "error to retain PV %s", restorePV.Name) } curLog.WithField("restore PV", restorePV.Name).WithField("retained", (retained != nil)).Info("Restore PV is retained") defer func() { if retained != nil { curLog.WithField("retained PV", retained.Name).Info("Deleting retained PV on error") kube.DeletePVIfAny(ctx, e.kubeClient.CoreV1(), retained.Name, curLog) } }() if retained != nil { restorePV = retained } err = kube.EnsureDeletePod(ctx, e.kubeClient.CoreV1(), restorePodName, ownerObject.Namespace, timeout) if err != nil { return errors.Wrapf(err, "error to delete restore pod %s", restorePodName) } err = kube.EnsureDeletePVC(ctx, e.kubeClient.CoreV1(), restorePVCName, ownerObject.Namespace, timeout) if err != nil { return errors.Wrapf(err, "error to delete restore PVC %s", restorePVCName) } curLog.WithField("restore PVC", restorePVCName).Info("Restore PVC is deleted") _, err = kube.RebindPVC(ctx, e.kubeClient.CoreV1(), targetPVC, restorePV.Name) if err != nil { return errors.Wrapf(err, "error to rebind target PVC %s/%s to %s", targetPVC.Namespace, targetPVC.Name, restorePV.Name) } curLog.WithField("tartet PVC", fmt.Sprintf("%s/%s", targetPVC.Namespace, targetPVC.Name)).WithField("restore PV", restorePV.Name).Info("Target PVC is rebound to restore PV") var matchLabel map[string]string if targetPVC.Spec.Selector != nil { matchLabel = targetPVC.Spec.Selector.MatchLabels } restorePVName := restorePV.Name restorePV, err = kube.ResetPVBinding(ctx, e.kubeClient.CoreV1(), restorePV, matchLabel, targetPVC) if err != nil { return errors.Wrapf(err, "error to reset binding info for restore PV %s", restorePVName) } curLog.WithField("restore PV", restorePV.Name).Info("Restore PV is rebound") restorePV, err = kube.WaitPVBound(ctx, e.kubeClient.CoreV1(), restorePV.Name, targetPVC.Name, targetPVC.Namespace, timeout) if err != nil { return errors.Wrapf(err, "error to wait restore PV bound, restore PV %s", restorePVName) } curLog.WithField("restore PV", restorePV.Name).Info("Restore PV is ready") retained = nil _, err = kube.SetPVReclaimPolicy(ctx, e.kubeClient.CoreV1(), restorePV, orgReclaim) if err != nil { curLog.WithField("restore PV", restorePV.Name).WithError(err).Warn("Restore PV's reclaim policy is not restored") } else { curLog.WithField("restore PV", restorePV.Name).Info("Restore PV's reclaim policy is restored") } return nil } func (e *genericRestoreExposer) createRestorePod( ctx context.Context, ownerObject corev1api.ObjectReference, targetPVC *corev1api.PersistentVolumeClaim, operationTimeout time.Duration, label map[string]string, annotation map[string]string, toleration []corev1api.Toleration, selectedNode string, resources corev1api.ResourceRequirements, nodeOS string, affinity *kube.LoadAffinity, priorityClassName string, cachePVC *corev1api.PersistentVolumeClaim, ) (*corev1api.Pod, error) { restorePodName := ownerObject.Name restorePVCName := ownerObject.Name containerName := string(ownerObject.UID) volumeName := string(ownerObject.UID) nodeSelector := map[string]string{} if selectedNode != "" { affinity = nil nodeSelector["kubernetes.io/hostname"] = selectedNode e.log.Infof("Selected node for restore pod. Ignore affinity from the node-agent config.") } if affinity == nil { affinity = &kube.LoadAffinity{} } podInfo, err := getInheritedPodInfo(ctx, e.kubeClient, ownerObject.Namespace, nodeOS) if err != nil { return nil, errors.Wrap(err, "error to get inherited pod info from node-agent") } // Log the priority class if it's set if priorityClassName != "" { e.log.Debugf("Setting priority class %q for data mover pod %s", priorityClassName, restorePodName) } var gracePeriod int64 volumeMounts, volumeDevices, volumePath := kube.MakePodPVCAttachment(volumeName, targetPVC.Spec.VolumeMode, false) volumes := []corev1api.Volume{{ Name: volumeName, VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: restorePVCName, }, }, }} cacheVolumePath := "" if cachePVC != nil { mnt, _, path := kube.MakePodPVCAttachment(cacheVolumeName, nil, false) volumeMounts = append(volumeMounts, mnt...) volumes = append(volumes, corev1api.Volume{ Name: cacheVolumeName, VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: cachePVC.Name, }, }, }) cacheVolumePath = path } volumeMounts = append(volumeMounts, podInfo.volumeMounts...) volumes = append(volumes, podInfo.volumes...) if label == nil { label = make(map[string]string) } label[podGroupLabel] = podGroupGenericRestore volumeMode := corev1api.PersistentVolumeFilesystem if targetPVC.Spec.VolumeMode != nil { volumeMode = *targetPVC.Spec.VolumeMode } args := []string{ fmt.Sprintf("--volume-path=%s", volumePath), fmt.Sprintf("--volume-mode=%s", volumeMode), fmt.Sprintf("--data-download=%s", ownerObject.Name), fmt.Sprintf("--resource-timeout=%s", operationTimeout.String()), fmt.Sprintf("--cache-volume-path=%s", cacheVolumePath), } args = append(args, podInfo.logFormatArgs...) args = append(args, podInfo.logLevelArgs...) var securityCtx *corev1api.PodSecurityContext podOS := corev1api.PodOS{} if nodeOS == kube.NodeOSWindows { userID := "ContainerAdministrator" securityCtx = &corev1api.PodSecurityContext{ WindowsOptions: &corev1api.WindowsSecurityContextOptions{ RunAsUserName: &userID, }, } podOS.Name = kube.NodeOSWindows affinity.NodeSelector.MatchExpressions = append(affinity.NodeSelector.MatchExpressions, metav1.LabelSelectorRequirement{ Key: kube.NodeOSLabel, Values: []string{kube.NodeOSWindows}, Operator: metav1.LabelSelectorOpIn, }) toleration = append(toleration, []corev1api.Toleration{ { Key: "os", Operator: "Equal", Effect: "NoSchedule", Value: "windows", }, { Key: "os", Operator: "Equal", Effect: "NoExecute", Value: "windows", }, }...) } else { userID := int64(0) securityCtx = &corev1api.PodSecurityContext{ RunAsUser: &userID, } podOS.Name = kube.NodeOSLinux affinity.NodeSelector.MatchExpressions = append(affinity.NodeSelector.MatchExpressions, metav1.LabelSelectorRequirement{ Key: kube.NodeOSLabel, Values: []string{kube.NodeOSWindows}, Operator: metav1.LabelSelectorOpNotIn, }) } podAffinity := kube.ToSystemAffinity(affinity, nil) pod := &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: restorePodName, Namespace: ownerObject.Namespace, OwnerReferences: []metav1.OwnerReference{ { APIVersion: ownerObject.APIVersion, Kind: ownerObject.Kind, Name: ownerObject.Name, UID: ownerObject.UID, Controller: boolptr.True(), }, }, Labels: label, Annotations: annotation, }, Spec: corev1api.PodSpec{ TopologySpreadConstraints: []corev1api.TopologySpreadConstraint{ { MaxSkew: 1, TopologyKey: "kubernetes.io/hostname", WhenUnsatisfiable: corev1api.ScheduleAnyway, LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ podGroupLabel: podGroupGenericRestore, }, }, }, }, NodeSelector: nodeSelector, OS: &podOS, Containers: []corev1api.Container{ { Name: containerName, Image: podInfo.image, ImagePullPolicy: corev1api.PullNever, Command: []string{ "/velero", "data-mover", "restore", }, Args: args, VolumeMounts: volumeMounts, VolumeDevices: volumeDevices, Env: podInfo.env, EnvFrom: podInfo.envFrom, Resources: resources, }, }, PriorityClassName: priorityClassName, ServiceAccountName: podInfo.serviceAccount, TerminationGracePeriodSeconds: &gracePeriod, Volumes: volumes, RestartPolicy: corev1api.RestartPolicyNever, SecurityContext: securityCtx, Tolerations: toleration, DNSPolicy: podInfo.dnsPolicy, DNSConfig: podInfo.dnsConfig, Affinity: podAffinity, ImagePullSecrets: podInfo.imagePullSecrets, }, } return e.kubeClient.CoreV1().Pods(ownerObject.Namespace).Create(ctx, pod, metav1.CreateOptions{}) } func (e *genericRestoreExposer) createRestorePVC(ctx context.Context, ownerObject corev1api.ObjectReference, targetPVC *corev1api.PersistentVolumeClaim, selectedNode string) (*corev1api.PersistentVolumeClaim, error) { restorePVCName := ownerObject.Name pvcObj := &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: ownerObject.Namespace, Name: restorePVCName, Labels: targetPVC.Labels, Annotations: targetPVC.Annotations, OwnerReferences: []metav1.OwnerReference{ { APIVersion: ownerObject.APIVersion, Kind: ownerObject.Kind, Name: ownerObject.Name, UID: ownerObject.UID, Controller: boolptr.True(), }, }, }, Spec: corev1api.PersistentVolumeClaimSpec{ AccessModes: targetPVC.Spec.AccessModes, StorageClassName: targetPVC.Spec.StorageClassName, VolumeMode: targetPVC.Spec.VolumeMode, Resources: targetPVC.Spec.Resources, }, } if selectedNode != "" { pvcObj.Annotations = map[string]string{ kube.KubeAnnSelectedNode: selectedNode, } } return e.kubeClient.CoreV1().PersistentVolumeClaims(pvcObj.Namespace).Create(ctx, pvcObj, metav1.CreateOptions{}) } ================================================ FILE: pkg/exposer/generic_restore_priority_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package exposer import ( "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" appsv1api "k8s.io/api/apps/v1" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerotest "github.com/vmware-tanzu/velero/pkg/test" "github.com/vmware-tanzu/velero/pkg/util/kube" ) // TestCreateRestorePodWithPriorityClass verifies that the priority class name is properly set in the restore pod func TestCreateRestorePodWithPriorityClass(t *testing.T) { testCases := []struct { name string nodeAgentConfigMapData string expectedPriorityClass string description string }{ { name: "with priority class in config map", nodeAgentConfigMapData: `{ "priorityClassName": "low-priority" }`, expectedPriorityClass: "low-priority", description: "Should set priority class from node-agent-configmap", }, { name: "without priority class in config map", nodeAgentConfigMapData: `{ "loadAffinity": [] }`, expectedPriorityClass: "", description: "Should have empty priority class when not specified", }, { name: "empty config map", nodeAgentConfigMapData: `{}`, expectedPriorityClass: "", description: "Should handle empty config map gracefully", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { ctx := t.Context() // Create fake Kubernetes client kubeClient := fake.NewSimpleClientset() // Create node-agent daemonset (required for getInheritedPodInfo) daemonSet := &appsv1api.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Name: "node-agent", Namespace: velerov1api.DefaultNamespace, }, Spec: appsv1api.DaemonSetSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ Containers: []corev1api.Container{ { Name: "node-agent", Image: "velero/velero:latest", }, }, }, }, }, } _, err := kubeClient.AppsV1().DaemonSets(velerov1api.DefaultNamespace).Create(ctx, daemonSet, metav1.CreateOptions{}) require.NoError(t, err) // Create node-agent config map configMap := &corev1api.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "node-agent-config", Namespace: velerov1api.DefaultNamespace, }, Data: map[string]string{ "config": tc.nodeAgentConfigMapData, }, } _, err = kubeClient.CoreV1().ConfigMaps(velerov1api.DefaultNamespace).Create(ctx, configMap, metav1.CreateOptions{}) require.NoError(t, err) // Create owner object for the restore pod ownerObject := corev1api.ObjectReference{ APIVersion: velerov1api.SchemeGroupVersion.String(), Kind: "DataDownload", Name: "test-datadownload", Namespace: velerov1api.DefaultNamespace, UID: "test-uid", } // Create a target PVC targetPVC := &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: "test-target-pvc", Namespace: velerov1api.DefaultNamespace, }, Spec: corev1api.PersistentVolumeClaimSpec{ AccessModes: []corev1api.PersistentVolumeAccessMode{ corev1api.ReadWriteOnce, }, }, } // Create generic restore exposer exposer := &genericRestoreExposer{ kubeClient: kubeClient, log: velerotest.NewLogger(), } // Call createRestorePod pod, err := exposer.createRestorePod( ctx, ownerObject, targetPVC, time.Minute*5, nil, // labels nil, // annotations nil, // tolerations "", // selectedNode corev1api.ResourceRequirements{}, kube.NodeOSLinux, nil, // affinity tc.expectedPriorityClass, nil, ) require.NoError(t, err, tc.description) assert.NotNil(t, pod) assert.Equal(t, tc.expectedPriorityClass, pod.Spec.PriorityClassName, tc.description) }) } } func TestCreateRestorePodWithMissingConfigMap(t *testing.T) { ctx := t.Context() // Create fake Kubernetes client without config map kubeClient := fake.NewSimpleClientset() // Create node-agent daemonset (required for getInheritedPodInfo) daemonSet := &appsv1api.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Name: "node-agent", Namespace: velerov1api.DefaultNamespace, }, Spec: appsv1api.DaemonSetSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ Containers: []corev1api.Container{ { Name: "node-agent", Image: "velero/velero:latest", }, }, }, }, }, } _, err := kubeClient.AppsV1().DaemonSets(velerov1api.DefaultNamespace).Create(ctx, daemonSet, metav1.CreateOptions{}) require.NoError(t, err) // Create owner object for the restore pod ownerObject := corev1api.ObjectReference{ APIVersion: velerov1api.SchemeGroupVersion.String(), Kind: "DataDownload", Name: "test-datadownload", Namespace: velerov1api.DefaultNamespace, UID: "test-uid", } // Create a target PVC targetPVC := &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: "test-target-pvc", Namespace: velerov1api.DefaultNamespace, }, Spec: corev1api.PersistentVolumeClaimSpec{ AccessModes: []corev1api.PersistentVolumeAccessMode{ corev1api.ReadWriteOnce, }, }, } // Create generic restore exposer exposer := &genericRestoreExposer{ kubeClient: kubeClient, log: velerotest.NewLogger(), } // Call createRestorePod pod, err := exposer.createRestorePod( ctx, ownerObject, targetPVC, time.Minute*5, nil, // labels nil, // annotations nil, // tolerations "", // selectedNode corev1api.ResourceRequirements{}, kube.NodeOSLinux, nil, // affinity "", // empty priority class since config map is missing nil, ) // Should succeed even when config map is missing require.NoError(t, err, "Should succeed even when config map is missing") assert.NotNil(t, pod) assert.Empty(t, pod.Spec.PriorityClassName, "Should have empty priority class when config map is missing") } ================================================ FILE: pkg/exposer/generic_restore_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package exposer import ( "testing" "time" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" appsv1api "k8s.io/api/apps/v1" corev1api "k8s.io/api/core/v1" storagev1api "k8s.io/api/storage/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes/fake" clientTesting "k8s.io/client-go/testing" velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerotest "github.com/vmware-tanzu/velero/pkg/test" "github.com/vmware-tanzu/velero/pkg/util/kube" ) func TestRestoreExpose(t *testing.T) { scName := "fake-sc" restore := &velerov1.Restore{ TypeMeta: metav1.TypeMeta{ APIVersion: velerov1.SchemeGroupVersion.String(), Kind: "Restore", }, ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1.DefaultNamespace, Name: "fake-restore", UID: "fake-uid", }, } targetPVCObj := &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-ns", Name: "fake-target-pvc", }, Spec: corev1api.PersistentVolumeClaimSpec{ StorageClassName: &scName, }, } storageClass := &storagev1api.StorageClass{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-sc", }, } targetPVCObjBound := &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-ns", Name: "fake-target-pvc", }, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "fake-pv", }, } daemonSet := &appsv1api.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", Name: "node-agent", }, TypeMeta: metav1.TypeMeta{ Kind: "DaemonSet", APIVersion: appsv1api.SchemeGroupVersion.String(), }, Spec: appsv1api.DaemonSetSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ Containers: []corev1api.Container{ { Image: "fake-image", }, }, }, }, }, } tests := []struct { name string kubeClientObj []runtime.Object ownerRestore *velerov1.Restore targetPVCName string targetNamespace string kubeReactors []reactor cacheVolume *CacheConfigs expectBackupPod bool expectBackupPVC bool expectCachePVC bool err string }{ { name: "wait target pvc consumed fail", targetPVCName: "fake-target-pvc", targetNamespace: "fake-ns", ownerRestore: restore, err: "error to wait target PVC consumed, fake-ns/fake-target-pvc: error to wait for PVC: error to get pvc fake-ns/fake-target-pvc: persistentvolumeclaims \"fake-target-pvc\" not found", }, { name: "target pvc is already bound", targetPVCName: "fake-target-pvc", targetNamespace: "fake-ns", ownerRestore: restore, kubeClientObj: []runtime.Object{ targetPVCObjBound, storageClass, }, err: "Target PVC fake-ns/fake-target-pvc has already been bound, abort", }, { name: "create restore pod fail", targetPVCName: "fake-target-pvc", targetNamespace: "fake-ns", ownerRestore: restore, kubeClientObj: []runtime.Object{ targetPVCObj, daemonSet, storageClass, }, kubeReactors: []reactor{ { verb: "create", resource: "pods", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, errors.New("fake-create-error") }, }, }, err: "error to create restore pod: fake-create-error", }, { name: "create restore pvc fail", targetPVCName: "fake-target-pvc", targetNamespace: "fake-ns", ownerRestore: restore, kubeClientObj: []runtime.Object{ targetPVCObj, daemonSet, storageClass, }, kubeReactors: []reactor{ { verb: "create", resource: "persistentvolumeclaims", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, errors.New("fake-create-error") }, }, }, err: "error to create restore pvc: fake-create-error", }, { name: "succeed", targetPVCName: "fake-target-pvc", targetNamespace: "fake-ns", ownerRestore: restore, kubeClientObj: []runtime.Object{ targetPVCObj, daemonSet, storageClass, }, expectBackupPod: true, expectBackupPVC: true, }, { name: "succeed, cache config, no cache volume", targetPVCName: "fake-target-pvc", targetNamespace: "fake-ns", ownerRestore: restore, kubeClientObj: []runtime.Object{ targetPVCObj, daemonSet, storageClass, }, cacheVolume: &CacheConfigs{}, expectBackupPod: true, expectBackupPVC: true, }, { name: "create cache volume fail", targetPVCName: "fake-target-pvc", targetNamespace: "fake-ns", ownerRestore: restore, kubeClientObj: []runtime.Object{ targetPVCObj, daemonSet, storageClass, }, cacheVolume: &CacheConfigs{Limit: 1024}, kubeReactors: []reactor{ { verb: "create", resource: "persistentvolumeclaims", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, errors.New("fake-create-error") }, }, }, err: "error to create cache pvc: fake-create-error", }, { name: "succeed with cache volume", targetPVCName: "fake-target-pvc", targetNamespace: "fake-ns", ownerRestore: restore, kubeClientObj: []runtime.Object{ targetPVCObj, daemonSet, storageClass, }, cacheVolume: &CacheConfigs{Limit: 1024}, expectBackupPod: true, expectBackupPVC: true, expectCachePVC: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...) for _, reactor := range test.kubeReactors { fakeKubeClient.Fake.PrependReactor(reactor.verb, reactor.resource, reactor.reactorFunc) } exposer := genericRestoreExposer{ kubeClient: fakeKubeClient, log: velerotest.NewLogger(), } var ownerObject corev1api.ObjectReference if test.ownerRestore != nil { ownerObject = corev1api.ObjectReference{ Kind: test.ownerRestore.Kind, Namespace: test.ownerRestore.Namespace, Name: test.ownerRestore.Name, UID: test.ownerRestore.UID, APIVersion: test.ownerRestore.APIVersion, } } err := exposer.Expose( t.Context(), ownerObject, GenericRestoreExposeParam{ TargetPVCName: test.targetPVCName, TargetNamespace: test.targetNamespace, HostingPodLabels: map[string]string{}, Resources: corev1api.ResourceRequirements{}, ExposeTimeout: time.Millisecond, LoadAffinity: nil, CacheVolume: test.cacheVolume, }, ) if test.err != "" { require.EqualError(t, err, test.err) } else { require.NoError(t, err) } _, err = exposer.kubeClient.CoreV1().Pods(ownerObject.Namespace).Get(t.Context(), ownerObject.Name, metav1.GetOptions{}) if test.expectBackupPod { require.NoError(t, err) } else { require.True(t, apierrors.IsNotFound(err)) } _, err = exposer.kubeClient.CoreV1().PersistentVolumeClaims(ownerObject.Namespace).Get(t.Context(), ownerObject.Name, metav1.GetOptions{}) if test.expectBackupPVC { require.NoError(t, err) } else { require.True(t, apierrors.IsNotFound(err)) } _, err = exposer.kubeClient.CoreV1().PersistentVolumeClaims(ownerObject.Namespace).Get(t.Context(), getCachePVCName(ownerObject), metav1.GetOptions{}) if test.expectCachePVC { require.NoError(t, err) } else { require.True(t, apierrors.IsNotFound(err)) } }) } } func TestRebindVolume(t *testing.T) { restore := &velerov1.Restore{ TypeMeta: metav1.TypeMeta{ APIVersion: velerov1.SchemeGroupVersion.String(), Kind: "Restore", }, ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1.DefaultNamespace, Name: "fake-restore", UID: "fake-uid", }, } targetPVCObj := &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-ns", Name: "fake-target-pvc", }, } restorePVCObj := &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1.DefaultNamespace, Name: "fake-restore", }, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "fake-restore-pv", }, } restorePVObj := &corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-restore-pv", }, Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeReclaimPolicy: corev1api.PersistentVolumeReclaimDelete, }, } restorePod := &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1.DefaultNamespace, Name: "fake-restore", }, } hookCount := 0 tests := []struct { name string kubeClientObj []runtime.Object ownerRestore *velerov1.Restore targetPVCName string targetNamespace string kubeReactors []reactor err string }{ { name: "get target pvc fail", targetPVCName: "fake-target-pvc", targetNamespace: "fake-ns", ownerRestore: restore, err: "error to get target PVC fake-ns/fake-target-pvc: persistentvolumeclaims \"fake-target-pvc\" not found", }, { name: "wait restore pvc bound fail", targetPVCName: "fake-target-pvc", targetNamespace: "fake-ns", ownerRestore: restore, kubeClientObj: []runtime.Object{ targetPVCObj, }, err: "error to get PV from restore PVC fake-restore: error to wait for rediness of PVC: error to get pvc velero/fake-restore: persistentvolumeclaims \"fake-restore\" not found", }, { name: "retain target pv fail", targetPVCName: "fake-target-pvc", targetNamespace: "fake-ns", ownerRestore: restore, kubeClientObj: []runtime.Object{ targetPVCObj, restorePVCObj, restorePVObj, }, kubeReactors: []reactor{ { verb: "patch", resource: "persistentvolumes", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, errors.New("fake-patch-error") }, }, }, err: "error to retain PV fake-restore-pv: error patching PV: fake-patch-error", }, { name: "delete restore pod fail", targetPVCName: "fake-target-pvc", targetNamespace: "fake-ns", ownerRestore: restore, kubeClientObj: []runtime.Object{ targetPVCObj, restorePVCObj, restorePVObj, restorePod, }, kubeReactors: []reactor{ { verb: "delete", resource: "pods", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, errors.New("fake-delete-error") }, }, }, err: "error to delete restore pod fake-restore: error to delete pod fake-restore: fake-delete-error", }, { name: "delete restore pvc fail", targetPVCName: "fake-target-pvc", targetNamespace: "fake-ns", ownerRestore: restore, kubeClientObj: []runtime.Object{ targetPVCObj, restorePVCObj, restorePVObj, restorePod, }, kubeReactors: []reactor{ { verb: "delete", resource: "persistentvolumeclaims", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, errors.New("fake-delete-error") }, }, }, err: "error to delete restore PVC fake-restore: error to delete pvc fake-restore: fake-delete-error", }, { name: "rebind target pvc fail", targetPVCName: "fake-target-pvc", targetNamespace: "fake-ns", ownerRestore: restore, kubeClientObj: []runtime.Object{ targetPVCObj, restorePVCObj, restorePVObj, restorePod, }, kubeReactors: []reactor{ { verb: "patch", resource: "persistentvolumeclaims", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, errors.New("fake-patch-error") }, }, }, err: "error to rebind target PVC fake-ns/fake-target-pvc to fake-restore-pv: error patching PVC: fake-patch-error", }, { name: "reset pv binding fail", targetPVCName: "fake-target-pvc", targetNamespace: "fake-ns", ownerRestore: restore, kubeClientObj: []runtime.Object{ targetPVCObj, restorePVCObj, restorePVObj, restorePod, }, kubeReactors: []reactor{ { verb: "patch", resource: "persistentvolumes", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { if hookCount == 0 { hookCount++ return false, nil, nil } else { return true, nil, errors.New("fake-patch-error") } }, }, }, err: "error to reset binding info for restore PV fake-restore-pv: error patching PV: fake-patch-error", }, { name: "wait restore PV bound fail", targetPVCName: "fake-target-pvc", targetNamespace: "fake-ns", ownerRestore: restore, kubeClientObj: []runtime.Object{ targetPVCObj, restorePVCObj, restorePVObj, restorePod, }, err: "error to wait restore PV bound, restore PV fake-restore-pv: error to wait for bound of PV: context deadline exceeded", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...) for _, reactor := range test.kubeReactors { fakeKubeClient.Fake.PrependReactor(reactor.verb, reactor.resource, reactor.reactorFunc) } exposer := genericRestoreExposer{ kubeClient: fakeKubeClient, log: velerotest.NewLogger(), } var ownerObject corev1api.ObjectReference if test.ownerRestore != nil { ownerObject = corev1api.ObjectReference{ Kind: test.ownerRestore.Kind, Namespace: test.ownerRestore.Namespace, Name: test.ownerRestore.Name, UID: test.ownerRestore.UID, APIVersion: test.ownerRestore.APIVersion, } } hookCount = 0 err := exposer.RebindVolume(t.Context(), ownerObject, test.targetPVCName, test.targetNamespace, time.Millisecond) assert.EqualError(t, err, test.err) }) } } func TestRestorePeekExpose(t *testing.T) { restore := &velerov1.Restore{ TypeMeta: metav1.TypeMeta{ APIVersion: velerov1.SchemeGroupVersion.String(), Kind: "Restore", }, ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1.DefaultNamespace, Name: "fake-restore", UID: "fake-uid", }, } restorePodUrecoverable := &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: restore.Namespace, Name: restore.Name, }, Status: corev1api.PodStatus{ Phase: corev1api.PodFailed, }, } restorePod := &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: restore.Namespace, Name: restore.Name, }, } tests := []struct { name string kubeClientObj []runtime.Object ownerRestore *velerov1.Restore err string }{ { name: "restore pod is not found", ownerRestore: restore, }, { name: "pod is unrecoverable", ownerRestore: restore, kubeClientObj: []runtime.Object{ restorePodUrecoverable, }, err: "Pod is in abnormal state [Failed], message []", }, { name: "succeed", ownerRestore: restore, kubeClientObj: []runtime.Object{ restorePod, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...) exposer := genericRestoreExposer{ kubeClient: fakeKubeClient, log: velerotest.NewLogger(), } var ownerObject corev1api.ObjectReference if test.ownerRestore != nil { ownerObject = corev1api.ObjectReference{ Kind: test.ownerRestore.Kind, Namespace: test.ownerRestore.Namespace, Name: test.ownerRestore.Name, UID: test.ownerRestore.UID, APIVersion: test.ownerRestore.APIVersion, } } err := exposer.PeekExposed(t.Context(), ownerObject) if test.err == "" { assert.NoError(t, err) } else { assert.EqualError(t, err, test.err) } }) } } func Test_ReastoreDiagnoseExpose(t *testing.T) { restore := &velerov1.Restore{ TypeMeta: metav1.TypeMeta{ APIVersion: velerov1.SchemeGroupVersion.String(), Kind: "Restore", }, ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1.DefaultNamespace, Name: "fake-restore", UID: "fake-uid", }, } restorePodWithoutNodeName := corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1.DefaultNamespace, Name: "fake-restore", UID: "fake-pod-uid", OwnerReferences: []metav1.OwnerReference{ { APIVersion: restore.APIVersion, Kind: restore.Kind, Name: restore.Name, UID: restore.UID, }, }, }, Status: corev1api.PodStatus{ Phase: corev1api.PodPending, Conditions: []corev1api.PodCondition{ { Type: corev1api.PodInitialized, Status: corev1api.ConditionTrue, Message: "fake-pod-message", }, }, Message: "fake-pod-message-1", }, } restorePodWithNodeName := corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1.DefaultNamespace, Name: "fake-restore", UID: "fake-pod-uid", OwnerReferences: []metav1.OwnerReference{ { APIVersion: restore.APIVersion, Kind: restore.Kind, Name: restore.Name, UID: restore.UID, }, }, }, Spec: corev1api.PodSpec{ NodeName: "fake-node", }, Status: corev1api.PodStatus{ Phase: corev1api.PodPending, Conditions: []corev1api.PodCondition{ { Type: corev1api.PodInitialized, Status: corev1api.ConditionTrue, Message: "fake-pod-message", }, }, }, } restorePVCWithoutVolumeName := corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1.DefaultNamespace, Name: "fake-restore", UID: "fake-pvc-uid", OwnerReferences: []metav1.OwnerReference{ { APIVersion: restore.APIVersion, Kind: restore.Kind, Name: restore.Name, UID: restore.UID, }, }, }, Status: corev1api.PersistentVolumeClaimStatus{ Phase: corev1api.ClaimPending, }, } restorePVCWithVolumeName := corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1.DefaultNamespace, Name: "fake-restore", UID: "fake-pvc-uid", OwnerReferences: []metav1.OwnerReference{ { APIVersion: restore.APIVersion, Kind: restore.Kind, Name: restore.Name, UID: restore.UID, }, }, }, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "fake-pv", }, Status: corev1api.PersistentVolumeClaimStatus{ Phase: corev1api.ClaimPending, }, } restorePV := corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-pv", }, Status: corev1api.PersistentVolumeStatus{ Phase: corev1api.VolumePending, Message: "fake-pv-message", }, } cachePVCWithVolumeName := corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1.DefaultNamespace, Name: "fake-restore-cache", UID: "fake-cache-pvc-uid", OwnerReferences: []metav1.OwnerReference{ { APIVersion: restore.APIVersion, Kind: restore.Kind, Name: restore.Name, UID: restore.UID, }, }, }, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "fake-pv-cache", }, Status: corev1api.PersistentVolumeClaimStatus{ Phase: corev1api.ClaimPending, }, } cachePV := corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-pv-cache", }, Status: corev1api.PersistentVolumeStatus{ Phase: corev1api.VolumePending, Message: "fake-pv-message", }, } nodeAgentPod := corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1.DefaultNamespace, Name: "node-agent-pod-1", Labels: map[string]string{"role": "node-agent"}, }, Spec: corev1api.PodSpec{ NodeName: "fake-node", }, Status: corev1api.PodStatus{ Phase: corev1api.PodRunning, }, } tests := []struct { name string ownerRestore *velerov1.Restore kubeClientObj []runtime.Object expected string }{ { name: "no pod, pvc", ownerRestore: restore, expected: `begin diagnose restore exposer error getting restore pod fake-restore, err: pods "fake-restore" not found error getting restore pvc fake-restore, err: persistentvolumeclaims "fake-restore" not found end diagnose restore exposer`, }, { name: "pod without node name, pvc without volume name, vs without status", ownerRestore: restore, kubeClientObj: []runtime.Object{ &restorePodWithoutNodeName, &restorePVCWithoutVolumeName, }, expected: `begin diagnose restore exposer Pod velero/fake-restore, phase Pending, node name , message fake-pod-message-1 Pod condition Initialized, status True, reason , message fake-pod-message PVC velero/fake-restore, phase Pending, binding to end diagnose restore exposer`, }, { name: "pod without node name, pvc without volume name", ownerRestore: restore, kubeClientObj: []runtime.Object{ &restorePodWithoutNodeName, &restorePVCWithoutVolumeName, }, expected: `begin diagnose restore exposer Pod velero/fake-restore, phase Pending, node name , message fake-pod-message-1 Pod condition Initialized, status True, reason , message fake-pod-message PVC velero/fake-restore, phase Pending, binding to end diagnose restore exposer`, }, { name: "pod with node name, no node agent", ownerRestore: restore, kubeClientObj: []runtime.Object{ &restorePodWithNodeName, &restorePVCWithoutVolumeName, }, expected: `begin diagnose restore exposer Pod velero/fake-restore, phase Pending, node name fake-node, message Pod condition Initialized, status True, reason , message fake-pod-message node-agent is not running in node fake-node, err: daemonset pod not found in running state in node fake-node PVC velero/fake-restore, phase Pending, binding to end diagnose restore exposer`, }, { name: "pod with node name, node agent is running", ownerRestore: restore, kubeClientObj: []runtime.Object{ &restorePodWithNodeName, &restorePVCWithoutVolumeName, &nodeAgentPod, }, expected: `begin diagnose restore exposer Pod velero/fake-restore, phase Pending, node name fake-node, message Pod condition Initialized, status True, reason , message fake-pod-message PVC velero/fake-restore, phase Pending, binding to end diagnose restore exposer`, }, { name: "pvc with volume name, no pv", ownerRestore: restore, kubeClientObj: []runtime.Object{ &restorePodWithNodeName, &restorePVCWithVolumeName, &nodeAgentPod, }, expected: `begin diagnose restore exposer Pod velero/fake-restore, phase Pending, node name fake-node, message Pod condition Initialized, status True, reason , message fake-pod-message PVC velero/fake-restore, phase Pending, binding to fake-pv error getting restore pv fake-pv, err: persistentvolumes "fake-pv" not found end diagnose restore exposer`, }, { name: "pvc with volume name, pv exists", ownerRestore: restore, kubeClientObj: []runtime.Object{ &restorePodWithNodeName, &restorePVCWithVolumeName, &restorePV, &nodeAgentPod, }, expected: `begin diagnose restore exposer Pod velero/fake-restore, phase Pending, node name fake-node, message Pod condition Initialized, status True, reason , message fake-pod-message PVC velero/fake-restore, phase Pending, binding to fake-pv PV fake-pv, phase Pending, reason , message fake-pv-message end diagnose restore exposer`, }, { name: "cache pvc with volume name, no pv", ownerRestore: restore, kubeClientObj: []runtime.Object{ &restorePodWithNodeName, &restorePVCWithVolumeName, &cachePVCWithVolumeName, &nodeAgentPod, }, expected: `begin diagnose restore exposer Pod velero/fake-restore, phase Pending, node name fake-node, message Pod condition Initialized, status True, reason , message fake-pod-message PVC velero/fake-restore, phase Pending, binding to fake-pv error getting restore pv fake-pv, err: persistentvolumes "fake-pv" not found PVC velero/fake-restore-cache, phase Pending, binding to fake-pv-cache error getting cache pv fake-pv-cache, err: persistentvolumes "fake-pv-cache" not found end diagnose restore exposer`, }, { name: "cache pvc with volume name, pv exists", ownerRestore: restore, kubeClientObj: []runtime.Object{ &restorePodWithNodeName, &restorePVCWithVolumeName, &cachePVCWithVolumeName, &restorePV, &cachePV, &nodeAgentPod, }, expected: `begin diagnose restore exposer Pod velero/fake-restore, phase Pending, node name fake-node, message Pod condition Initialized, status True, reason , message fake-pod-message PVC velero/fake-restore, phase Pending, binding to fake-pv PV fake-pv, phase Pending, reason , message fake-pv-message PVC velero/fake-restore-cache, phase Pending, binding to fake-pv-cache PV fake-pv-cache, phase Pending, reason , message fake-pv-message end diagnose restore exposer`, }, { name: "with events", ownerRestore: restore, kubeClientObj: []runtime.Object{ &restorePodWithNodeName, &restorePVCWithVolumeName, &restorePV, &nodeAgentPod, &corev1api.Event{ ObjectMeta: metav1.ObjectMeta{Namespace: velerov1.DefaultNamespace, Name: "event-1"}, Type: corev1api.EventTypeWarning, InvolvedObject: corev1api.ObjectReference{UID: "fake-uid-1"}, Reason: "reason-1", Message: "message-1", }, &corev1api.Event{ ObjectMeta: metav1.ObjectMeta{Namespace: velerov1.DefaultNamespace, Name: "event-2"}, Type: corev1api.EventTypeWarning, InvolvedObject: corev1api.ObjectReference{UID: "fake-pod-uid"}, Reason: "reason-2", Message: "message-2", }, &corev1api.Event{ ObjectMeta: metav1.ObjectMeta{Namespace: velerov1.DefaultNamespace, Name: "event-3"}, Type: corev1api.EventTypeWarning, InvolvedObject: corev1api.ObjectReference{UID: "fake-pvc-uid"}, Reason: "reason-3", Message: "message-3", }, &corev1api.Event{ ObjectMeta: metav1.ObjectMeta{Namespace: "other-namespace", Name: "event-4"}, Type: corev1api.EventTypeWarning, InvolvedObject: corev1api.ObjectReference{UID: "fake-pod-uid"}, Reason: "reason-4", Message: "message-4", }, &corev1api.Event{ ObjectMeta: metav1.ObjectMeta{Namespace: velerov1.DefaultNamespace, Name: "event-5"}, Type: corev1api.EventTypeWarning, InvolvedObject: corev1api.ObjectReference{UID: "fake-pod-uid"}, Reason: "reason-5", Message: "message-5", }, }, expected: `begin diagnose restore exposer Pod velero/fake-restore, phase Pending, node name fake-node, message Pod condition Initialized, status True, reason , message fake-pod-message Pod event reason reason-2, message message-2 Pod event reason reason-5, message message-5 PVC velero/fake-restore, phase Pending, binding to fake-pv PVC event reason reason-3, message message-3 PV fake-pv, phase Pending, reason , message fake-pv-message end diagnose restore exposer`, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...) e := genericRestoreExposer{ kubeClient: fakeKubeClient, log: velerotest.NewLogger(), } var ownerObject corev1api.ObjectReference if test.ownerRestore != nil { ownerObject = corev1api.ObjectReference{ Kind: test.ownerRestore.Kind, Namespace: test.ownerRestore.Namespace, Name: test.ownerRestore.Name, UID: test.ownerRestore.UID, APIVersion: test.ownerRestore.APIVersion, } } diag := e.DiagnoseExpose(t.Context(), ownerObject) assert.Equal(t, test.expected, diag) }) } } func TestCreateRestorePod(t *testing.T) { scName := "storage-class-01" daemonSet := &appsv1api.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", Name: "node-agent", }, TypeMeta: metav1.TypeMeta{ Kind: "DaemonSet", APIVersion: appsv1api.SchemeGroupVersion.String(), }, Spec: appsv1api.DaemonSetSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ Containers: []corev1api.Container{ { Image: "fake-image", }, }, }, }, }, } daemonSetWin := &appsv1api.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", Name: "node-agent-windows", }, TypeMeta: metav1.TypeMeta{ Kind: "DaemonSet", APIVersion: appsv1api.SchemeGroupVersion.String(), }, Spec: appsv1api.DaemonSetSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ Containers: []corev1api.Container{ { Image: "fake-image", }, }, }, }, }, } targetPVCObj := &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-ns", Name: "fake-target-pvc", }, Spec: corev1api.PersistentVolumeClaimSpec{ StorageClassName: &scName, }, } tests := []struct { name string kubeClientObj []runtime.Object selectedNode string affinity *kube.LoadAffinity nodeOS string expectedPod *corev1api.Pod }{ { name: "linux", kubeClientObj: []runtime.Object{daemonSet, daemonSetWin, targetPVCObj}, selectedNode: "", affinity: &kube.LoadAffinity{ NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { Key: "kubernetes.io/os", Operator: metav1.LabelSelectorOpIn, Values: []string{"linux"}, }, }, }, StorageClass: scName, }, nodeOS: "linux", }, { name: "windows", kubeClientObj: []runtime.Object{daemonSet, daemonSetWin, targetPVCObj}, selectedNode: "", affinity: &kube.LoadAffinity{ NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { Key: "kubernetes.io/os", Operator: metav1.LabelSelectorOpIn, Values: []string{"windows"}, }, }, }, StorageClass: scName, }, nodeOS: "windows", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...) exposer := genericRestoreExposer{ kubeClient: fakeKubeClient, log: velerotest.NewLogger(), } pod, err := exposer.createRestorePod( t.Context(), corev1api.ObjectReference{ Namespace: velerov1.DefaultNamespace, Name: "data-download", }, targetPVCObj, time.Second*3, nil, nil, nil, test.selectedNode, corev1api.ResourceRequirements{}, test.nodeOS, test.affinity, "", // priority class name nil, ) require.NoError(t, err) if test.expectedPod != nil { assert.Equal(t, test.expectedPod, pod) } }) } } ================================================ FILE: pkg/exposer/host_path.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package exposer import ( "context" "fmt" "strings" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" "github.com/vmware-tanzu/velero/pkg/datapath" "github.com/vmware-tanzu/velero/pkg/nodeagent" "github.com/vmware-tanzu/velero/pkg/uploader" "github.com/vmware-tanzu/velero/pkg/util/filesystem" "github.com/vmware-tanzu/velero/pkg/util/kube" ) var getVolumeDirectory = kube.GetVolumeDirectory var getVolumeMode = kube.GetVolumeMode var singlePathMatch = kube.SinglePathMatch // GetPodVolumeHostPath returns a path that can be accessed from the host for a given volume of a pod func GetPodVolumeHostPath(ctx context.Context, pod *corev1api.Pod, volumeName string, kubeClient kubernetes.Interface, fs filesystem.Interface, log logrus.FieldLogger) (datapath.AccessPoint, error) { logger := log.WithField("pod name", pod.Name).WithField("pod UID", pod.GetUID()).WithField("volume", volumeName) volDir, err := getVolumeDirectory(ctx, logger, pod, volumeName, kubeClient) if err != nil { return datapath.AccessPoint{}, errors.Wrapf(err, "error getting volume directory name for volume %s in pod %s", volumeName, pod.Name) } logger.WithField("volDir", volDir).Info("Got volume dir") volMode, err := getVolumeMode(ctx, logger, pod, volumeName, kubeClient) if err != nil { return datapath.AccessPoint{}, errors.Wrapf(err, "error getting volume mode for volume %s in pod %s", volumeName, pod.Name) } volSubDir := "volumes" if volMode == uploader.PersistentVolumeBlock { volSubDir = "volumeDevices" } pathGlob := fmt.Sprintf("%s/%s/%s/*/%s", nodeagent.HostPodVolumeMountPath(), string(pod.GetUID()), volSubDir, volDir) logger.WithField("pathGlob", pathGlob).Debug("Looking for path matching glob") path, err := singlePathMatch(pathGlob, fs, logger) if err != nil { return datapath.AccessPoint{}, errors.Wrapf(err, "error identifying unique volume path on host for volume %s in pod %s", volumeName, pod.Name) } logger.WithField("path", path).Info("Found path matching glob") return datapath.AccessPoint{ ByPath: path, VolMode: volMode, }, nil } var getHostPodPath = nodeagent.GetHostPodPath func ExtractPodVolumeHostPath(ctx context.Context, path string, kubeClient kubernetes.Interface, veleroNamespace string, osType string) (string, error) { podPath, err := getHostPodPath(ctx, kubeClient, veleroNamespace, osType) if err != nil { return "", errors.Wrap(err, "error getting host pod path from node-agent") } if osType == kube.NodeOSWindows { podPath = strings.Replace(podPath, "/", "\\", -1) } if osType == kube.NodeOSWindows { return strings.Replace(path, nodeagent.HostPodVolumeMountPathWin(), podPath, 1), nil } else { return strings.Replace(path, nodeagent.HostPodVolumeMountPath(), podPath, 1), nil } } ================================================ FILE: pkg/exposer/host_path_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package exposer import ( "context" "fmt" "testing" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/nodeagent" velerotest "github.com/vmware-tanzu/velero/pkg/test" "github.com/vmware-tanzu/velero/pkg/uploader" "github.com/vmware-tanzu/velero/pkg/util/filesystem" "github.com/vmware-tanzu/velero/pkg/util/kube" ) func TestGetPodVolumeHostPath(t *testing.T) { tests := []struct { name string getVolumeDirFunc func(context.Context, logrus.FieldLogger, *corev1api.Pod, string, kubernetes.Interface) (string, error) getVolumeModeFunc func(context.Context, logrus.FieldLogger, *corev1api.Pod, string, kubernetes.Interface) (uploader.PersistentVolumeMode, error) pathMatchFunc func(string, filesystem.Interface, logrus.FieldLogger) (string, error) pod *corev1api.Pod pvc string err string }{ { name: "get volume dir fail", getVolumeDirFunc: func(context.Context, logrus.FieldLogger, *corev1api.Pod, string, kubernetes.Interface) (string, error) { return "", errors.New("fake-error-1") }, pod: builder.ForPod(velerov1api.DefaultNamespace, "fake-pod-1").Result(), pvc: "fake-pvc-1", err: "error getting volume directory name for volume fake-pvc-1 in pod fake-pod-1: fake-error-1", }, { name: "single path match fail", getVolumeDirFunc: func(context.Context, logrus.FieldLogger, *corev1api.Pod, string, kubernetes.Interface) (string, error) { return "", nil }, getVolumeModeFunc: func(context.Context, logrus.FieldLogger, *corev1api.Pod, string, kubernetes.Interface) (uploader.PersistentVolumeMode, error) { return uploader.PersistentVolumeFilesystem, nil }, pathMatchFunc: func(string, filesystem.Interface, logrus.FieldLogger) (string, error) { return "", errors.New("fake-error-2") }, pod: builder.ForPod(velerov1api.DefaultNamespace, "fake-pod-2").Result(), pvc: "fake-pvc-1", err: "error identifying unique volume path on host for volume fake-pvc-1 in pod fake-pod-2: fake-error-2", }, { name: "get block volume dir success", getVolumeDirFunc: func(context.Context, logrus.FieldLogger, *corev1api.Pod, string, kubernetes.Interface) ( string, error) { return "fake-pvc-1", nil }, pathMatchFunc: func(string, filesystem.Interface, logrus.FieldLogger) (string, error) { return "/host_pods/fake-pod-1-id/volumeDevices/kubernetes.io~csi/fake-pvc-1-id", nil }, pod: builder.ForPod(velerov1api.DefaultNamespace, "fake-pod-1").Result(), pvc: "fake-pvc-1", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { if test.getVolumeDirFunc != nil { getVolumeDirectory = test.getVolumeDirFunc } if test.getVolumeModeFunc != nil { getVolumeMode = test.getVolumeModeFunc } if test.pathMatchFunc != nil { singlePathMatch = test.pathMatchFunc } _, err := GetPodVolumeHostPath(t.Context(), test.pod, test.pvc, nil, nil, velerotest.NewLogger()) if test.err != "" || err != nil { assert.EqualError(t, err, test.err) } }) } } func TestExtractPodVolumeHostPath(t *testing.T) { tests := []struct { name string getHostPodPathFunc func(context.Context, kubernetes.Interface, string, string) (string, error) path string osType string expectedErr string expected string }{ { name: "get host pod path error", getHostPodPathFunc: func(context.Context, kubernetes.Interface, string, string) (string, error) { return "", errors.New("fake-error-1") }, expectedErr: "error getting host pod path from node-agent: fake-error-1", }, { name: "Windows os", getHostPodPathFunc: func(context.Context, kubernetes.Interface, string, string) (string, error) { return "/var/lib/kubelet/pods", nil }, path: fmt.Sprintf("\\%s\\pod-id-xxx\\volumes\\kubernetes.io~csi\\pvc-id-xxx\\mount", nodeagent.HostPodVolumeMountPoint), osType: kube.NodeOSWindows, expected: "\\var\\lib\\kubelet\\pods\\pod-id-xxx\\volumes\\kubernetes.io~csi\\pvc-id-xxx\\mount", }, { name: "linux OS", getHostPodPathFunc: func(context.Context, kubernetes.Interface, string, string) (string, error) { return "/var/lib/kubelet/pods", nil }, path: fmt.Sprintf("/%s/pod-id-xxx/volumes/kubernetes.io~csi/pvc-id-xxx/mount", nodeagent.HostPodVolumeMountPoint), osType: kube.NodeOSLinux, expected: "/var/lib/kubelet/pods/pod-id-xxx/volumes/kubernetes.io~csi/pvc-id-xxx/mount", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { if test.getHostPodPathFunc != nil { getHostPodPath = test.getHostPodPathFunc } path, err := ExtractPodVolumeHostPath(t.Context(), test.path, nil, "", test.osType) if test.expectedErr != "" { assert.EqualError(t, err, test.expectedErr) } else { require.NoError(t, err) assert.Equal(t, test.expected, path) } }) } } ================================================ FILE: pkg/exposer/image.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package exposer import ( "context" "strings" "github.com/pkg/errors" corev1api "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" "github.com/vmware-tanzu/velero/pkg/nodeagent" ) type inheritedPodInfo struct { image string serviceAccount string env []corev1api.EnvVar envFrom []corev1api.EnvFromSource volumeMounts []corev1api.VolumeMount volumes []corev1api.Volume logLevelArgs []string logFormatArgs []string dnsPolicy corev1api.DNSPolicy dnsConfig *corev1api.PodDNSConfig imagePullSecrets []corev1api.LocalObjectReference } func getInheritedPodInfo(ctx context.Context, client kubernetes.Interface, veleroNamespace string, osType string) (inheritedPodInfo, error) { podInfo := inheritedPodInfo{} podSpec, err := nodeagent.GetPodSpec(ctx, client, veleroNamespace, osType) if err != nil { return podInfo, errors.Wrap(err, "error to get node-agent pod template") } if len(podSpec.Containers) != 1 { return podInfo, errors.New("unexpected pod template from node-agent") } podInfo.image = podSpec.Containers[0].Image podInfo.serviceAccount = podSpec.ServiceAccountName podInfo.env = podSpec.Containers[0].Env podInfo.envFrom = podSpec.Containers[0].EnvFrom podInfo.volumeMounts = podSpec.Containers[0].VolumeMounts podInfo.volumes = podSpec.Volumes podInfo.dnsPolicy = podSpec.DNSPolicy podInfo.dnsConfig = podSpec.DNSConfig args := podSpec.Containers[0].Args for i, arg := range args { if arg == "--log-format" { podInfo.logFormatArgs = append(podInfo.logFormatArgs, args[i:i+2]...) } else if strings.HasPrefix(arg, "--log-format") { podInfo.logFormatArgs = append(podInfo.logFormatArgs, arg) } else if arg == "--log-level" { podInfo.logLevelArgs = append(podInfo.logLevelArgs, args[i:i+2]...) } else if strings.HasPrefix(arg, "--log-level") { podInfo.logLevelArgs = append(podInfo.logLevelArgs, arg) } } podInfo.imagePullSecrets = podSpec.ImagePullSecrets return podInfo, nil } ================================================ FILE: pkg/exposer/image_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package exposer import ( "reflect" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes" "github.com/vmware-tanzu/velero/pkg/util/kube" appsv1api "k8s.io/api/apps/v1" corev1api "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes/fake" ) func TestGetInheritedPodInfo(t *testing.T) { daemonSet := &appsv1api.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-ns", Name: "node-agent", }, TypeMeta: metav1.TypeMeta{ Kind: "DaemonSet", }, } daemonSetWithNoLog := &appsv1api.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-ns", Name: "node-agent", }, TypeMeta: metav1.TypeMeta{ Kind: "DaemonSet", }, Spec: appsv1api.DaemonSetSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ Containers: []corev1api.Container{ { Name: "container-1", Image: "image-1", Env: []corev1api.EnvVar{ { Name: "env-1", Value: "value-1", }, { Name: "env-2", Value: "value-2", }, }, EnvFrom: []corev1api.EnvFromSource{ { ConfigMapRef: &corev1api.ConfigMapEnvSource{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "test-configmap", }, }, }, { SecretRef: &corev1api.SecretEnvSource{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "test-secret", }, }, }, }, VolumeMounts: []corev1api.VolumeMount{ { Name: "volume-1", }, { Name: "volume-2", }, }, }, }, Volumes: []corev1api.Volume{ { Name: "volume-1", }, { Name: "volume-2", }, }, ServiceAccountName: "sa-1", }, }, }, } daemonSetWithLog := &appsv1api.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-ns", Name: "node-agent", }, TypeMeta: metav1.TypeMeta{ Kind: "DaemonSet", }, Spec: appsv1api.DaemonSetSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ Containers: []corev1api.Container{ { Name: "container-1", Image: "image-1", Env: []corev1api.EnvVar{ { Name: "env-1", Value: "value-1", }, { Name: "env-2", Value: "value-2", }, }, EnvFrom: []corev1api.EnvFromSource{ { ConfigMapRef: &corev1api.ConfigMapEnvSource{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "test-configmap", }, }, }, { SecretRef: &corev1api.SecretEnvSource{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "test-secret", }, }, }, }, VolumeMounts: []corev1api.VolumeMount{ { Name: "volume-1", }, { Name: "volume-2", }, }, Args: []string{ "--log-format=json", "--log-level", "debug", }, Command: []string{ "command-1", }, }, }, Volumes: []corev1api.Volume{ { Name: "volume-1", }, { Name: "volume-2", }, }, ServiceAccountName: "sa-1", ImagePullSecrets: []corev1api.LocalObjectReference{ { Name: "imagePullSecret1", }, }, }, }, }, } scheme := runtime.NewScheme() appsv1api.AddToScheme(scheme) tests := []struct { name string namespace string client kubernetes.Interface kubeClientObj []runtime.Object result inheritedPodInfo expectErr string }{ { name: "ds is not found", namespace: "fake-ns", expectErr: "error to get node-agent pod template: error to get node-agent daemonset: daemonsets.apps \"node-agent\" not found", }, { name: "ds pod container number is invalidate", namespace: "fake-ns", kubeClientObj: []runtime.Object{ daemonSet, }, expectErr: "unexpected pod template from node-agent", }, { name: "no log info", namespace: "fake-ns", kubeClientObj: []runtime.Object{ daemonSetWithNoLog, }, result: inheritedPodInfo{ image: "image-1", serviceAccount: "sa-1", env: []corev1api.EnvVar{ { Name: "env-1", Value: "value-1", }, { Name: "env-2", Value: "value-2", }, }, envFrom: []corev1api.EnvFromSource{ { ConfigMapRef: &corev1api.ConfigMapEnvSource{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "test-configmap", }, }, }, { SecretRef: &corev1api.SecretEnvSource{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "test-secret", }, }, }, }, volumeMounts: []corev1api.VolumeMount{ { Name: "volume-1", }, { Name: "volume-2", }, }, volumes: []corev1api.Volume{ { Name: "volume-1", }, { Name: "volume-2", }, }, }, }, { name: "with log info", namespace: "fake-ns", kubeClientObj: []runtime.Object{ daemonSetWithLog, }, result: inheritedPodInfo{ image: "image-1", serviceAccount: "sa-1", env: []corev1api.EnvVar{ { Name: "env-1", Value: "value-1", }, { Name: "env-2", Value: "value-2", }, }, envFrom: []corev1api.EnvFromSource{ { ConfigMapRef: &corev1api.ConfigMapEnvSource{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "test-configmap", }, }, }, { SecretRef: &corev1api.SecretEnvSource{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "test-secret", }, }, }, }, volumeMounts: []corev1api.VolumeMount{ { Name: "volume-1", }, { Name: "volume-2", }, }, volumes: []corev1api.Volume{ { Name: "volume-1", }, { Name: "volume-2", }, }, logFormatArgs: []string{ "--log-format=json", }, logLevelArgs: []string{ "--log-level", "debug", }, imagePullSecrets: []corev1api.LocalObjectReference{ { Name: "imagePullSecret1", }, }, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...) info, err := getInheritedPodInfo(t.Context(), fakeKubeClient, test.namespace, kube.NodeOSLinux) if test.expectErr == "" { require.NoError(t, err) assert.True(t, reflect.DeepEqual(info, test.result)) } else { assert.EqualError(t, err, test.expectErr) } }) } } ================================================ FILE: pkg/exposer/mocks/GenericRestoreExposer.go ================================================ // Code generated by mockery; DO NOT EDIT. // github.com/vektra/mockery // template: testify package mocks import ( "context" "time" mock "github.com/stretchr/testify/mock" "github.com/vmware-tanzu/velero/pkg/exposer" "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) // NewMockGenericRestoreExposer creates a new instance of MockGenericRestoreExposer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. func NewMockGenericRestoreExposer(t interface { mock.TestingT Cleanup(func()) }) *MockGenericRestoreExposer { mock := &MockGenericRestoreExposer{} mock.Mock.Test(t) t.Cleanup(func() { mock.AssertExpectations(t) }) return mock } // MockGenericRestoreExposer is an autogenerated mock type for the GenericRestoreExposer type type MockGenericRestoreExposer struct { mock.Mock } type MockGenericRestoreExposer_Expecter struct { mock *mock.Mock } func (_m *MockGenericRestoreExposer) EXPECT() *MockGenericRestoreExposer_Expecter { return &MockGenericRestoreExposer_Expecter{mock: &_m.Mock} } // CleanUp provides a mock function for the type MockGenericRestoreExposer func (_mock *MockGenericRestoreExposer) CleanUp(context1 context.Context, objectReference v1.ObjectReference) { _mock.Called(context1, objectReference) return } // MockGenericRestoreExposer_CleanUp_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CleanUp' type MockGenericRestoreExposer_CleanUp_Call struct { *mock.Call } // CleanUp is a helper method to define mock.On call // - context1 context.Context // - objectReference v1.ObjectReference func (_e *MockGenericRestoreExposer_Expecter) CleanUp(context1 interface{}, objectReference interface{}) *MockGenericRestoreExposer_CleanUp_Call { return &MockGenericRestoreExposer_CleanUp_Call{Call: _e.mock.On("CleanUp", context1, objectReference)} } func (_c *MockGenericRestoreExposer_CleanUp_Call) Run(run func(context1 context.Context, objectReference v1.ObjectReference)) *MockGenericRestoreExposer_CleanUp_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { arg0 = args[0].(context.Context) } var arg1 v1.ObjectReference if args[1] != nil { arg1 = args[1].(v1.ObjectReference) } run( arg0, arg1, ) }) return _c } func (_c *MockGenericRestoreExposer_CleanUp_Call) Return() *MockGenericRestoreExposer_CleanUp_Call { _c.Call.Return() return _c } func (_c *MockGenericRestoreExposer_CleanUp_Call) RunAndReturn(run func(context1 context.Context, objectReference v1.ObjectReference)) *MockGenericRestoreExposer_CleanUp_Call { _c.Run(run) return _c } // DiagnoseExpose provides a mock function for the type MockGenericRestoreExposer func (_mock *MockGenericRestoreExposer) DiagnoseExpose(context1 context.Context, objectReference v1.ObjectReference) string { ret := _mock.Called(context1, objectReference) if len(ret) == 0 { panic("no return value specified for DiagnoseExpose") } var r0 string if returnFunc, ok := ret.Get(0).(func(context.Context, v1.ObjectReference) string); ok { r0 = returnFunc(context1, objectReference) } else { r0 = ret.Get(0).(string) } return r0 } // MockGenericRestoreExposer_DiagnoseExpose_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DiagnoseExpose' type MockGenericRestoreExposer_DiagnoseExpose_Call struct { *mock.Call } // DiagnoseExpose is a helper method to define mock.On call // - context1 context.Context // - objectReference v1.ObjectReference func (_e *MockGenericRestoreExposer_Expecter) DiagnoseExpose(context1 interface{}, objectReference interface{}) *MockGenericRestoreExposer_DiagnoseExpose_Call { return &MockGenericRestoreExposer_DiagnoseExpose_Call{Call: _e.mock.On("DiagnoseExpose", context1, objectReference)} } func (_c *MockGenericRestoreExposer_DiagnoseExpose_Call) Run(run func(context1 context.Context, objectReference v1.ObjectReference)) *MockGenericRestoreExposer_DiagnoseExpose_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { arg0 = args[0].(context.Context) } var arg1 v1.ObjectReference if args[1] != nil { arg1 = args[1].(v1.ObjectReference) } run( arg0, arg1, ) }) return _c } func (_c *MockGenericRestoreExposer_DiagnoseExpose_Call) Return(s string) *MockGenericRestoreExposer_DiagnoseExpose_Call { _c.Call.Return(s) return _c } func (_c *MockGenericRestoreExposer_DiagnoseExpose_Call) RunAndReturn(run func(context1 context.Context, objectReference v1.ObjectReference) string) *MockGenericRestoreExposer_DiagnoseExpose_Call { _c.Call.Return(run) return _c } // Expose provides a mock function for the type MockGenericRestoreExposer func (_mock *MockGenericRestoreExposer) Expose(context1 context.Context, objectReference v1.ObjectReference, genericRestoreExposeParam exposer.GenericRestoreExposeParam) error { ret := _mock.Called(context1, objectReference, genericRestoreExposeParam) if len(ret) == 0 { panic("no return value specified for Expose") } var r0 error if returnFunc, ok := ret.Get(0).(func(context.Context, v1.ObjectReference, exposer.GenericRestoreExposeParam) error); ok { r0 = returnFunc(context1, objectReference, genericRestoreExposeParam) } else { r0 = ret.Error(0) } return r0 } // MockGenericRestoreExposer_Expose_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Expose' type MockGenericRestoreExposer_Expose_Call struct { *mock.Call } // Expose is a helper method to define mock.On call // - context1 context.Context // - objectReference v1.ObjectReference // - genericRestoreExposeParam exposer.GenericRestoreExposeParam func (_e *MockGenericRestoreExposer_Expecter) Expose(context1 interface{}, objectReference interface{}, genericRestoreExposeParam interface{}) *MockGenericRestoreExposer_Expose_Call { return &MockGenericRestoreExposer_Expose_Call{Call: _e.mock.On("Expose", context1, objectReference, genericRestoreExposeParam)} } func (_c *MockGenericRestoreExposer_Expose_Call) Run(run func(context1 context.Context, objectReference v1.ObjectReference, genericRestoreExposeParam exposer.GenericRestoreExposeParam)) *MockGenericRestoreExposer_Expose_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { arg0 = args[0].(context.Context) } var arg1 v1.ObjectReference if args[1] != nil { arg1 = args[1].(v1.ObjectReference) } var arg2 exposer.GenericRestoreExposeParam if args[2] != nil { arg2 = args[2].(exposer.GenericRestoreExposeParam) } run( arg0, arg1, arg2, ) }) return _c } func (_c *MockGenericRestoreExposer_Expose_Call) Return(err error) *MockGenericRestoreExposer_Expose_Call { _c.Call.Return(err) return _c } func (_c *MockGenericRestoreExposer_Expose_Call) RunAndReturn(run func(context1 context.Context, objectReference v1.ObjectReference, genericRestoreExposeParam exposer.GenericRestoreExposeParam) error) *MockGenericRestoreExposer_Expose_Call { _c.Call.Return(run) return _c } // GetExposed provides a mock function for the type MockGenericRestoreExposer func (_mock *MockGenericRestoreExposer) GetExposed(context1 context.Context, objectReference v1.ObjectReference, client1 client.Client, s string, duration time.Duration) (*exposer.ExposeResult, error) { ret := _mock.Called(context1, objectReference, client1, s, duration) if len(ret) == 0 { panic("no return value specified for GetExposed") } var r0 *exposer.ExposeResult var r1 error if returnFunc, ok := ret.Get(0).(func(context.Context, v1.ObjectReference, client.Client, string, time.Duration) (*exposer.ExposeResult, error)); ok { return returnFunc(context1, objectReference, client1, s, duration) } if returnFunc, ok := ret.Get(0).(func(context.Context, v1.ObjectReference, client.Client, string, time.Duration) *exposer.ExposeResult); ok { r0 = returnFunc(context1, objectReference, client1, s, duration) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(*exposer.ExposeResult) } } if returnFunc, ok := ret.Get(1).(func(context.Context, v1.ObjectReference, client.Client, string, time.Duration) error); ok { r1 = returnFunc(context1, objectReference, client1, s, duration) } else { r1 = ret.Error(1) } return r0, r1 } // MockGenericRestoreExposer_GetExposed_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetExposed' type MockGenericRestoreExposer_GetExposed_Call struct { *mock.Call } // GetExposed is a helper method to define mock.On call // - context1 context.Context // - objectReference v1.ObjectReference // - client1 client.Client // - s string // - duration time.Duration func (_e *MockGenericRestoreExposer_Expecter) GetExposed(context1 interface{}, objectReference interface{}, client1 interface{}, s interface{}, duration interface{}) *MockGenericRestoreExposer_GetExposed_Call { return &MockGenericRestoreExposer_GetExposed_Call{Call: _e.mock.On("GetExposed", context1, objectReference, client1, s, duration)} } func (_c *MockGenericRestoreExposer_GetExposed_Call) Run(run func(context1 context.Context, objectReference v1.ObjectReference, client1 client.Client, s string, duration time.Duration)) *MockGenericRestoreExposer_GetExposed_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { arg0 = args[0].(context.Context) } var arg1 v1.ObjectReference if args[1] != nil { arg1 = args[1].(v1.ObjectReference) } var arg2 client.Client if args[2] != nil { arg2 = args[2].(client.Client) } var arg3 string if args[3] != nil { arg3 = args[3].(string) } var arg4 time.Duration if args[4] != nil { arg4 = args[4].(time.Duration) } run( arg0, arg1, arg2, arg3, arg4, ) }) return _c } func (_c *MockGenericRestoreExposer_GetExposed_Call) Return(exposeResult *exposer.ExposeResult, err error) *MockGenericRestoreExposer_GetExposed_Call { _c.Call.Return(exposeResult, err) return _c } func (_c *MockGenericRestoreExposer_GetExposed_Call) RunAndReturn(run func(context1 context.Context, objectReference v1.ObjectReference, client1 client.Client, s string, duration time.Duration) (*exposer.ExposeResult, error)) *MockGenericRestoreExposer_GetExposed_Call { _c.Call.Return(run) return _c } // PeekExposed provides a mock function for the type MockGenericRestoreExposer func (_mock *MockGenericRestoreExposer) PeekExposed(context1 context.Context, objectReference v1.ObjectReference) error { ret := _mock.Called(context1, objectReference) if len(ret) == 0 { panic("no return value specified for PeekExposed") } var r0 error if returnFunc, ok := ret.Get(0).(func(context.Context, v1.ObjectReference) error); ok { r0 = returnFunc(context1, objectReference) } else { r0 = ret.Error(0) } return r0 } // MockGenericRestoreExposer_PeekExposed_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'PeekExposed' type MockGenericRestoreExposer_PeekExposed_Call struct { *mock.Call } // PeekExposed is a helper method to define mock.On call // - context1 context.Context // - objectReference v1.ObjectReference func (_e *MockGenericRestoreExposer_Expecter) PeekExposed(context1 interface{}, objectReference interface{}) *MockGenericRestoreExposer_PeekExposed_Call { return &MockGenericRestoreExposer_PeekExposed_Call{Call: _e.mock.On("PeekExposed", context1, objectReference)} } func (_c *MockGenericRestoreExposer_PeekExposed_Call) Run(run func(context1 context.Context, objectReference v1.ObjectReference)) *MockGenericRestoreExposer_PeekExposed_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { arg0 = args[0].(context.Context) } var arg1 v1.ObjectReference if args[1] != nil { arg1 = args[1].(v1.ObjectReference) } run( arg0, arg1, ) }) return _c } func (_c *MockGenericRestoreExposer_PeekExposed_Call) Return(err error) *MockGenericRestoreExposer_PeekExposed_Call { _c.Call.Return(err) return _c } func (_c *MockGenericRestoreExposer_PeekExposed_Call) RunAndReturn(run func(context1 context.Context, objectReference v1.ObjectReference) error) *MockGenericRestoreExposer_PeekExposed_Call { _c.Call.Return(run) return _c } // RebindVolume provides a mock function for the type MockGenericRestoreExposer func (_mock *MockGenericRestoreExposer) RebindVolume(context1 context.Context, objectReference v1.ObjectReference, s string, s1 string, duration time.Duration) error { ret := _mock.Called(context1, objectReference, s, s1, duration) if len(ret) == 0 { panic("no return value specified for RebindVolume") } var r0 error if returnFunc, ok := ret.Get(0).(func(context.Context, v1.ObjectReference, string, string, time.Duration) error); ok { r0 = returnFunc(context1, objectReference, s, s1, duration) } else { r0 = ret.Error(0) } return r0 } // MockGenericRestoreExposer_RebindVolume_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RebindVolume' type MockGenericRestoreExposer_RebindVolume_Call struct { *mock.Call } // RebindVolume is a helper method to define mock.On call // - context1 context.Context // - objectReference v1.ObjectReference // - s string // - s1 string // - duration time.Duration func (_e *MockGenericRestoreExposer_Expecter) RebindVolume(context1 interface{}, objectReference interface{}, s interface{}, s1 interface{}, duration interface{}) *MockGenericRestoreExposer_RebindVolume_Call { return &MockGenericRestoreExposer_RebindVolume_Call{Call: _e.mock.On("RebindVolume", context1, objectReference, s, s1, duration)} } func (_c *MockGenericRestoreExposer_RebindVolume_Call) Run(run func(context1 context.Context, objectReference v1.ObjectReference, s string, s1 string, duration time.Duration)) *MockGenericRestoreExposer_RebindVolume_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { arg0 = args[0].(context.Context) } var arg1 v1.ObjectReference if args[1] != nil { arg1 = args[1].(v1.ObjectReference) } var arg2 string if args[2] != nil { arg2 = args[2].(string) } var arg3 string if args[3] != nil { arg3 = args[3].(string) } var arg4 time.Duration if args[4] != nil { arg4 = args[4].(time.Duration) } run( arg0, arg1, arg2, arg3, arg4, ) }) return _c } func (_c *MockGenericRestoreExposer_RebindVolume_Call) Return(err error) *MockGenericRestoreExposer_RebindVolume_Call { _c.Call.Return(err) return _c } func (_c *MockGenericRestoreExposer_RebindVolume_Call) RunAndReturn(run func(context1 context.Context, objectReference v1.ObjectReference, s string, s1 string, duration time.Duration) error) *MockGenericRestoreExposer_RebindVolume_Call { _c.Call.Return(run) return _c } ================================================ FILE: pkg/exposer/mocks/PodVolumeExposer.go ================================================ // Code generated by mockery; DO NOT EDIT. // github.com/vektra/mockery // template: testify package mocks import ( "context" "time" mock "github.com/stretchr/testify/mock" "github.com/vmware-tanzu/velero/pkg/exposer" "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) // NewMockPodVolumeExposer creates a new instance of MockPodVolumeExposer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. func NewMockPodVolumeExposer(t interface { mock.TestingT Cleanup(func()) }) *MockPodVolumeExposer { mock := &MockPodVolumeExposer{} mock.Mock.Test(t) t.Cleanup(func() { mock.AssertExpectations(t) }) return mock } // MockPodVolumeExposer is an autogenerated mock type for the PodVolumeExposer type type MockPodVolumeExposer struct { mock.Mock } type MockPodVolumeExposer_Expecter struct { mock *mock.Mock } func (_m *MockPodVolumeExposer) EXPECT() *MockPodVolumeExposer_Expecter { return &MockPodVolumeExposer_Expecter{mock: &_m.Mock} } // CleanUp provides a mock function for the type MockPodVolumeExposer func (_mock *MockPodVolumeExposer) CleanUp(context1 context.Context, objectReference v1.ObjectReference) { _mock.Called(context1, objectReference) return } // MockPodVolumeExposer_CleanUp_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CleanUp' type MockPodVolumeExposer_CleanUp_Call struct { *mock.Call } // CleanUp is a helper method to define mock.On call // - context1 context.Context // - objectReference v1.ObjectReference func (_e *MockPodVolumeExposer_Expecter) CleanUp(context1 interface{}, objectReference interface{}) *MockPodVolumeExposer_CleanUp_Call { return &MockPodVolumeExposer_CleanUp_Call{Call: _e.mock.On("CleanUp", context1, objectReference)} } func (_c *MockPodVolumeExposer_CleanUp_Call) Run(run func(context1 context.Context, objectReference v1.ObjectReference)) *MockPodVolumeExposer_CleanUp_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { arg0 = args[0].(context.Context) } var arg1 v1.ObjectReference if args[1] != nil { arg1 = args[1].(v1.ObjectReference) } run( arg0, arg1, ) }) return _c } func (_c *MockPodVolumeExposer_CleanUp_Call) Return() *MockPodVolumeExposer_CleanUp_Call { _c.Call.Return() return _c } func (_c *MockPodVolumeExposer_CleanUp_Call) RunAndReturn(run func(context1 context.Context, objectReference v1.ObjectReference)) *MockPodVolumeExposer_CleanUp_Call { _c.Run(run) return _c } // DiagnoseExpose provides a mock function for the type MockPodVolumeExposer func (_mock *MockPodVolumeExposer) DiagnoseExpose(context1 context.Context, objectReference v1.ObjectReference) string { ret := _mock.Called(context1, objectReference) if len(ret) == 0 { panic("no return value specified for DiagnoseExpose") } var r0 string if returnFunc, ok := ret.Get(0).(func(context.Context, v1.ObjectReference) string); ok { r0 = returnFunc(context1, objectReference) } else { r0 = ret.Get(0).(string) } return r0 } // MockPodVolumeExposer_DiagnoseExpose_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DiagnoseExpose' type MockPodVolumeExposer_DiagnoseExpose_Call struct { *mock.Call } // DiagnoseExpose is a helper method to define mock.On call // - context1 context.Context // - objectReference v1.ObjectReference func (_e *MockPodVolumeExposer_Expecter) DiagnoseExpose(context1 interface{}, objectReference interface{}) *MockPodVolumeExposer_DiagnoseExpose_Call { return &MockPodVolumeExposer_DiagnoseExpose_Call{Call: _e.mock.On("DiagnoseExpose", context1, objectReference)} } func (_c *MockPodVolumeExposer_DiagnoseExpose_Call) Run(run func(context1 context.Context, objectReference v1.ObjectReference)) *MockPodVolumeExposer_DiagnoseExpose_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { arg0 = args[0].(context.Context) } var arg1 v1.ObjectReference if args[1] != nil { arg1 = args[1].(v1.ObjectReference) } run( arg0, arg1, ) }) return _c } func (_c *MockPodVolumeExposer_DiagnoseExpose_Call) Return(s string) *MockPodVolumeExposer_DiagnoseExpose_Call { _c.Call.Return(s) return _c } func (_c *MockPodVolumeExposer_DiagnoseExpose_Call) RunAndReturn(run func(context1 context.Context, objectReference v1.ObjectReference) string) *MockPodVolumeExposer_DiagnoseExpose_Call { _c.Call.Return(run) return _c } // Expose provides a mock function for the type MockPodVolumeExposer func (_mock *MockPodVolumeExposer) Expose(context1 context.Context, objectReference v1.ObjectReference, podVolumeExposeParam exposer.PodVolumeExposeParam) error { ret := _mock.Called(context1, objectReference, podVolumeExposeParam) if len(ret) == 0 { panic("no return value specified for Expose") } var r0 error if returnFunc, ok := ret.Get(0).(func(context.Context, v1.ObjectReference, exposer.PodVolumeExposeParam) error); ok { r0 = returnFunc(context1, objectReference, podVolumeExposeParam) } else { r0 = ret.Error(0) } return r0 } // MockPodVolumeExposer_Expose_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Expose' type MockPodVolumeExposer_Expose_Call struct { *mock.Call } // Expose is a helper method to define mock.On call // - context1 context.Context // - objectReference v1.ObjectReference // - podVolumeExposeParam exposer.PodVolumeExposeParam func (_e *MockPodVolumeExposer_Expecter) Expose(context1 interface{}, objectReference interface{}, podVolumeExposeParam interface{}) *MockPodVolumeExposer_Expose_Call { return &MockPodVolumeExposer_Expose_Call{Call: _e.mock.On("Expose", context1, objectReference, podVolumeExposeParam)} } func (_c *MockPodVolumeExposer_Expose_Call) Run(run func(context1 context.Context, objectReference v1.ObjectReference, podVolumeExposeParam exposer.PodVolumeExposeParam)) *MockPodVolumeExposer_Expose_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { arg0 = args[0].(context.Context) } var arg1 v1.ObjectReference if args[1] != nil { arg1 = args[1].(v1.ObjectReference) } var arg2 exposer.PodVolumeExposeParam if args[2] != nil { arg2 = args[2].(exposer.PodVolumeExposeParam) } run( arg0, arg1, arg2, ) }) return _c } func (_c *MockPodVolumeExposer_Expose_Call) Return(err error) *MockPodVolumeExposer_Expose_Call { _c.Call.Return(err) return _c } func (_c *MockPodVolumeExposer_Expose_Call) RunAndReturn(run func(context1 context.Context, objectReference v1.ObjectReference, podVolumeExposeParam exposer.PodVolumeExposeParam) error) *MockPodVolumeExposer_Expose_Call { _c.Call.Return(run) return _c } // GetExposed provides a mock function for the type MockPodVolumeExposer func (_mock *MockPodVolumeExposer) GetExposed(context1 context.Context, objectReference v1.ObjectReference, client1 client.Client, s string, duration time.Duration) (*exposer.ExposeResult, error) { ret := _mock.Called(context1, objectReference, client1, s, duration) if len(ret) == 0 { panic("no return value specified for GetExposed") } var r0 *exposer.ExposeResult var r1 error if returnFunc, ok := ret.Get(0).(func(context.Context, v1.ObjectReference, client.Client, string, time.Duration) (*exposer.ExposeResult, error)); ok { return returnFunc(context1, objectReference, client1, s, duration) } if returnFunc, ok := ret.Get(0).(func(context.Context, v1.ObjectReference, client.Client, string, time.Duration) *exposer.ExposeResult); ok { r0 = returnFunc(context1, objectReference, client1, s, duration) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(*exposer.ExposeResult) } } if returnFunc, ok := ret.Get(1).(func(context.Context, v1.ObjectReference, client.Client, string, time.Duration) error); ok { r1 = returnFunc(context1, objectReference, client1, s, duration) } else { r1 = ret.Error(1) } return r0, r1 } // MockPodVolumeExposer_GetExposed_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetExposed' type MockPodVolumeExposer_GetExposed_Call struct { *mock.Call } // GetExposed is a helper method to define mock.On call // - context1 context.Context // - objectReference v1.ObjectReference // - client1 client.Client // - s string // - duration time.Duration func (_e *MockPodVolumeExposer_Expecter) GetExposed(context1 interface{}, objectReference interface{}, client1 interface{}, s interface{}, duration interface{}) *MockPodVolumeExposer_GetExposed_Call { return &MockPodVolumeExposer_GetExposed_Call{Call: _e.mock.On("GetExposed", context1, objectReference, client1, s, duration)} } func (_c *MockPodVolumeExposer_GetExposed_Call) Run(run func(context1 context.Context, objectReference v1.ObjectReference, client1 client.Client, s string, duration time.Duration)) *MockPodVolumeExposer_GetExposed_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { arg0 = args[0].(context.Context) } var arg1 v1.ObjectReference if args[1] != nil { arg1 = args[1].(v1.ObjectReference) } var arg2 client.Client if args[2] != nil { arg2 = args[2].(client.Client) } var arg3 string if args[3] != nil { arg3 = args[3].(string) } var arg4 time.Duration if args[4] != nil { arg4 = args[4].(time.Duration) } run( arg0, arg1, arg2, arg3, arg4, ) }) return _c } func (_c *MockPodVolumeExposer_GetExposed_Call) Return(exposeResult *exposer.ExposeResult, err error) *MockPodVolumeExposer_GetExposed_Call { _c.Call.Return(exposeResult, err) return _c } func (_c *MockPodVolumeExposer_GetExposed_Call) RunAndReturn(run func(context1 context.Context, objectReference v1.ObjectReference, client1 client.Client, s string, duration time.Duration) (*exposer.ExposeResult, error)) *MockPodVolumeExposer_GetExposed_Call { _c.Call.Return(run) return _c } // PeekExposed provides a mock function for the type MockPodVolumeExposer func (_mock *MockPodVolumeExposer) PeekExposed(context1 context.Context, objectReference v1.ObjectReference) error { ret := _mock.Called(context1, objectReference) if len(ret) == 0 { panic("no return value specified for PeekExposed") } var r0 error if returnFunc, ok := ret.Get(0).(func(context.Context, v1.ObjectReference) error); ok { r0 = returnFunc(context1, objectReference) } else { r0 = ret.Error(0) } return r0 } // MockPodVolumeExposer_PeekExposed_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'PeekExposed' type MockPodVolumeExposer_PeekExposed_Call struct { *mock.Call } // PeekExposed is a helper method to define mock.On call // - context1 context.Context // - objectReference v1.ObjectReference func (_e *MockPodVolumeExposer_Expecter) PeekExposed(context1 interface{}, objectReference interface{}) *MockPodVolumeExposer_PeekExposed_Call { return &MockPodVolumeExposer_PeekExposed_Call{Call: _e.mock.On("PeekExposed", context1, objectReference)} } func (_c *MockPodVolumeExposer_PeekExposed_Call) Run(run func(context1 context.Context, objectReference v1.ObjectReference)) *MockPodVolumeExposer_PeekExposed_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { arg0 = args[0].(context.Context) } var arg1 v1.ObjectReference if args[1] != nil { arg1 = args[1].(v1.ObjectReference) } run( arg0, arg1, ) }) return _c } func (_c *MockPodVolumeExposer_PeekExposed_Call) Return(err error) *MockPodVolumeExposer_PeekExposed_Call { _c.Call.Return(err) return _c } func (_c *MockPodVolumeExposer_PeekExposed_Call) RunAndReturn(run func(context1 context.Context, objectReference v1.ObjectReference) error) *MockPodVolumeExposer_PeekExposed_Call { _c.Call.Return(run) return _c } ================================================ FILE: pkg/exposer/pod_volume.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package exposer import ( "context" "fmt" "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/pkg/nodeagent" "github.com/vmware-tanzu/velero/pkg/util/boolptr" "github.com/vmware-tanzu/velero/pkg/util/filesystem" "github.com/vmware-tanzu/velero/pkg/util/kube" ) const ( PodVolumeExposeTypeBackup = "pod-volume-backup" PodVolumeExposeTypeRestore = "pod-volume-restore" ) // PodVolumeExposeParam define the input param for pod volume Expose type PodVolumeExposeParam struct { // ClientPodName is the name of pod to be backed up or restored ClientPodName string // ClientNamespace is the namespace to be backed up or restored ClientNamespace string // ClientNamespace is the pod volume for the client PVC ClientPodVolume string // HostingPodLabels is the labels that are going to apply to the hosting pod HostingPodLabels map[string]string // HostingPodAnnotations is the annotations that are going to apply to the hosting pod HostingPodAnnotations map[string]string // HostingPodTolerations is the tolerations that are going to apply to the hosting pod HostingPodTolerations []corev1api.Toleration // Resources defines the resource requirements of the hosting pod Resources corev1api.ResourceRequirements // OperationTimeout specifies the time wait for resources operations in Expose OperationTimeout time.Duration // Type specifies the type of the expose, either backup or erstore Type string // PriorityClassName is the priority class name for the data mover pod PriorityClassName string // Privileged indicates whether to create the pod with a privileged container Privileged bool // RestoreSize specifies the data size for the volume to be restored, for restore only RestoreSize int64 // CacheVolume specifies the info for cache volumes, for restore only CacheVolume *CacheConfigs } // PodVolumeExposer is the interfaces for a pod volume exposer type PodVolumeExposer interface { // Expose starts the process to a pod volume expose, the expose process may take long time Expose(context.Context, corev1api.ObjectReference, PodVolumeExposeParam) error // GetExposed polls the status of the expose. // If the expose is accessible by the current caller, it waits the expose ready and returns the expose result. // Otherwise, it returns nil as the expose result without an error. GetExposed(context.Context, corev1api.ObjectReference, client.Client, string, time.Duration) (*ExposeResult, error) // PeekExposed tests the status of the expose. // If the expose is incomplete but not recoverable, it returns an error. // Otherwise, it returns nil immediately. PeekExposed(context.Context, corev1api.ObjectReference) error // DiagnoseExpose generate the diagnostic info when the expose is not finished for a long time. // If it finds any problem, it returns an string about the problem. DiagnoseExpose(context.Context, corev1api.ObjectReference) string // CleanUp cleans up any objects generated during the restore expose CleanUp(context.Context, corev1api.ObjectReference) } // NewPodVolumeExposer creates a new instance of pod volume exposer func NewPodVolumeExposer(kubeClient kubernetes.Interface, log logrus.FieldLogger) PodVolumeExposer { return &podVolumeExposer{ kubeClient: kubeClient, fs: filesystem.NewFileSystem(), log: log, } } type podVolumeExposer struct { kubeClient kubernetes.Interface fs filesystem.Interface log logrus.FieldLogger } var getPodVolumeHostPath = GetPodVolumeHostPath var extractPodVolumeHostPath = ExtractPodVolumeHostPath func (e *podVolumeExposer) Expose(ctx context.Context, ownerObject corev1api.ObjectReference, param PodVolumeExposeParam) error { curLog := e.log.WithFields(logrus.Fields{ "owner": ownerObject.Name, "client pod": param.ClientPodName, "client pod volume": param.ClientPodVolume, "client namespace": param.ClientNamespace, "type": param.Type, }) pod, err := e.kubeClient.CoreV1().Pods(param.ClientNamespace).Get(ctx, param.ClientPodName, metav1.GetOptions{}) if err != nil { return errors.Wrapf(err, "error getting client pod %s", param.ClientPodName) } if pod.Spec.NodeName == "" { return errors.Errorf("client pod %s doesn't have a node name", pod.Name) } nodeOS, err := kube.GetNodeOS(ctx, pod.Spec.NodeName, e.kubeClient.CoreV1()) if err != nil { return errors.Wrapf(err, "error getting OS for node %s", pod.Spec.NodeName) } curLog.Infof("Client pod is running in node %s, os %s", pod.Spec.NodeName, nodeOS) path, err := getPodVolumeHostPath(ctx, pod, param.ClientPodVolume, e.kubeClient, e.fs, e.log) if err != nil { return errors.Wrapf(err, "error to get pod volume path") } path.ByPath, err = extractPodVolumeHostPath(ctx, path.ByPath, e.kubeClient, ownerObject.Namespace, nodeOS) if err != nil { return errors.Wrapf(err, "error to extract pod volume path") } curLog.WithField("path", path).Infof("Host path is retrieved for pod %s, volume %s", param.ClientPodName, param.ClientPodVolume) var cachePVC *corev1api.PersistentVolumeClaim if param.CacheVolume != nil { cacheVolumeSize := getCacheVolumeSize(param.RestoreSize, param.CacheVolume) if cacheVolumeSize > 0 { curLog.Infof("Creating cache PVC with size %v", cacheVolumeSize) if pvc, err := createCachePVC(ctx, e.kubeClient.CoreV1(), ownerObject, param.CacheVolume.StorageClass, cacheVolumeSize, pod.Spec.NodeName); err != nil { return errors.Wrap(err, "error to create cache pvc") } else { cachePVC = pvc } defer func() { if err != nil { kube.DeletePVAndPVCIfAny(ctx, e.kubeClient.CoreV1(), cachePVC.Name, cachePVC.Namespace, 0, curLog) } }() } else { curLog.Infof("Don't need to create cache volume, restore size %v, cache info %v", param.RestoreSize, param.CacheVolume) } } hostingPod, err := e.createHostingPod( ctx, ownerObject, param.Type, path.ByPath, param.OperationTimeout, param.HostingPodLabels, param.HostingPodAnnotations, param.HostingPodTolerations, pod.Spec.NodeName, param.Resources, nodeOS, param.PriorityClassName, param.Privileged, cachePVC, ) if err != nil { return errors.Wrapf(err, "error to create hosting pod") } curLog.WithField("pod name", hostingPod.Name).Info("Hosting pod is created") return nil } func (e *podVolumeExposer) GetExposed(ctx context.Context, ownerObject corev1api.ObjectReference, nodeClient client.Client, nodeName string, timeout time.Duration) (*ExposeResult, error) { hostingPodName := ownerObject.Name containerName := string(ownerObject.UID) volumeName := string(ownerObject.UID) curLog := e.log.WithFields(logrus.Fields{ "owner": ownerObject.Name, "node": nodeName, }) var updated *corev1api.Pod err := wait.PollUntilContextTimeout(ctx, 2*time.Second, timeout, true, func(ctx context.Context) (bool, error) { pod := &corev1api.Pod{} err := nodeClient.Get(ctx, types.NamespacedName{ Namespace: ownerObject.Namespace, Name: hostingPodName, }, pod) if err != nil { return false, errors.Wrapf(err, "error to get pod %s/%s", ownerObject.Namespace, hostingPodName) } if pod.Status.Phase != corev1api.PodRunning { return false, nil } updated = pod return true, nil }) if err != nil { if apierrors.IsNotFound(err) { curLog.WithField("hosting pod", hostingPodName).Debug("Hosting pod is not running in the current node") return nil, nil } else { return nil, errors.Wrapf(err, "error to wait for rediness of pod %s", hostingPodName) } } curLog.WithField("pod", updated.Name).Infof("Hosting pod is in running state in node %s", updated.Spec.NodeName) return &ExposeResult{ByPod: ExposeByPod{ HostingPod: updated, HostingContainer: containerName, VolumeName: volumeName, }}, nil } func (e *podVolumeExposer) PeekExposed(ctx context.Context, ownerObject corev1api.ObjectReference) error { hostingPodName := ownerObject.Name curLog := e.log.WithFields(logrus.Fields{ "owner": ownerObject.Name, }) pod, err := e.kubeClient.CoreV1().Pods(ownerObject.Namespace).Get(ctx, hostingPodName, metav1.GetOptions{}) if apierrors.IsNotFound(err) { return nil } if err != nil { curLog.WithError(err).Warnf("error to peek hosting pod %s", hostingPodName) return nil } if podFailed, message := kube.IsPodUnrecoverable(pod, curLog); podFailed { return errors.New(message) } return nil } func (e *podVolumeExposer) DiagnoseExpose(ctx context.Context, ownerObject corev1api.ObjectReference) string { hostingPodName := ownerObject.Name diag := "begin diagnose pod volume exposer\n" pod, err := e.kubeClient.CoreV1().Pods(ownerObject.Namespace).Get(ctx, hostingPodName, metav1.GetOptions{}) if err != nil { pod = nil diag += fmt.Sprintf("error getting hosting pod %s, err: %v\n", hostingPodName, err) } cachePVC, err := e.kubeClient.CoreV1().PersistentVolumeClaims(ownerObject.Namespace).Get(ctx, getCachePVCName(ownerObject), metav1.GetOptions{}) if err != nil { cachePVC = nil if !apierrors.IsNotFound(err) { diag += fmt.Sprintf("error getting cache pvc %s, err: %v\n", getCachePVCName(ownerObject), err) } } events, err := e.kubeClient.CoreV1().Events(ownerObject.Namespace).List(ctx, metav1.ListOptions{}) if err != nil { diag += fmt.Sprintf("error listing events, err: %v\n", err) } if pod != nil { diag += kube.DiagnosePod(pod, events) if pod.Spec.NodeName != "" { if err := nodeagent.KbClientIsRunningInNode(ctx, ownerObject.Namespace, pod.Spec.NodeName, e.kubeClient); err != nil { diag += fmt.Sprintf("node-agent is not running in node %s, err: %v\n", pod.Spec.NodeName, err) } } } if cachePVC != nil { diag += kube.DiagnosePVC(cachePVC, events) if cachePVC.Spec.VolumeName != "" { if pv, err := e.kubeClient.CoreV1().PersistentVolumes().Get(ctx, cachePVC.Spec.VolumeName, metav1.GetOptions{}); err != nil { diag += fmt.Sprintf("error getting cache pv %s, err: %v\n", cachePVC.Spec.VolumeName, err) } else { diag += kube.DiagnosePV(pv) } } } diag += "end diagnose pod volume exposer" return diag } func (e *podVolumeExposer) CleanUp(ctx context.Context, ownerObject corev1api.ObjectReference) { restorePodName := ownerObject.Name cachePVCName := getCachePVCName(ownerObject) kube.DeletePodIfAny(ctx, e.kubeClient.CoreV1(), restorePodName, ownerObject.Namespace, e.log) kube.DeletePVAndPVCIfAny(ctx, e.kubeClient.CoreV1(), cachePVCName, ownerObject.Namespace, 0, e.log) } func (e *podVolumeExposer) createHostingPod( ctx context.Context, ownerObject corev1api.ObjectReference, exposeType string, hostPath string, operationTimeout time.Duration, label map[string]string, annotation map[string]string, toleration []corev1api.Toleration, selectedNode string, resources corev1api.ResourceRequirements, nodeOS string, priorityClassName string, privileged bool, cachePVC *corev1api.PersistentVolumeClaim, ) (*corev1api.Pod, error) { hostingPodName := ownerObject.Name containerName := string(ownerObject.UID) clientVolumeName := string(ownerObject.UID) clientVolumePath := "/" + clientVolumeName podInfo, err := getInheritedPodInfo(ctx, e.kubeClient, ownerObject.Namespace, nodeOS) if err != nil { return nil, errors.Wrap(err, "error to get inherited pod info from node-agent") } // Log the priority class if it's set if priorityClassName != "" { e.log.Debugf("Setting priority class %q for data mover pod %s", priorityClassName, hostingPodName) } var gracePeriod int64 mountPropagation := corev1api.MountPropagationHostToContainer volumeMounts := []corev1api.VolumeMount{{ Name: clientVolumeName, MountPath: clientVolumePath, MountPropagation: &mountPropagation, }} volumes := []corev1api.Volume{{ Name: clientVolumeName, VolumeSource: corev1api.VolumeSource{ HostPath: &corev1api.HostPathVolumeSource{ Path: hostPath, }, }, }} cacheVolumePath := "" if cachePVC != nil { mnt, _, path := kube.MakePodPVCAttachment(cacheVolumeName, nil, false) volumeMounts = append(volumeMounts, mnt...) volumes = append(volumes, corev1api.Volume{ Name: cacheVolumeName, VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: cachePVC.Name, }, }, }) cacheVolumePath = path } volumeMounts = append(volumeMounts, podInfo.volumeMounts...) volumes = append(volumes, podInfo.volumes...) args := []string{ fmt.Sprintf("--volume-path=%s", clientVolumePath), fmt.Sprintf("--resource-timeout=%s", operationTimeout.String()), } command := []string{ "/velero", "pod-volume", } if exposeType == PodVolumeExposeTypeBackup { args = append(args, fmt.Sprintf("--pod-volume-backup=%s", ownerObject.Name)) command = append(command, "backup") } else { args = append(args, fmt.Sprintf("--pod-volume-restore=%s", ownerObject.Name)) args = append(args, fmt.Sprintf("--cache-volume-path=%s", cacheVolumePath)) command = append(command, "restore") } args = append(args, podInfo.logFormatArgs...) args = append(args, podInfo.logLevelArgs...) affinity := &kube.LoadAffinity{} var securityCtx *corev1api.PodSecurityContext var containerSecurityCtx *corev1api.SecurityContext nodeSelector := map[string]string{} podOS := corev1api.PodOS{} if nodeOS == kube.NodeOSWindows { userID := "ContainerAdministrator" securityCtx = &corev1api.PodSecurityContext{ WindowsOptions: &corev1api.WindowsSecurityContextOptions{ RunAsUserName: &userID, }, } podOS.Name = kube.NodeOSWindows affinity.NodeSelector.MatchExpressions = append(affinity.NodeSelector.MatchExpressions, metav1.LabelSelectorRequirement{ Key: kube.NodeOSLabel, Values: []string{kube.NodeOSWindows}, Operator: metav1.LabelSelectorOpIn, }) toleration = append(toleration, []corev1api.Toleration{ { Key: "os", Operator: "Equal", Effect: "NoSchedule", Value: "windows", }, { Key: "os", Operator: "Equal", Effect: "NoExecute", Value: "windows", }, }...) } else { userID := int64(0) securityCtx = &corev1api.PodSecurityContext{ RunAsUser: &userID, } containerSecurityCtx = &corev1api.SecurityContext{ Privileged: &privileged, } podOS.Name = kube.NodeOSLinux affinity.NodeSelector.MatchExpressions = append(affinity.NodeSelector.MatchExpressions, metav1.LabelSelectorRequirement{ Key: kube.NodeOSLabel, Values: []string{kube.NodeOSWindows}, Operator: metav1.LabelSelectorOpNotIn, }) } podAffinity := kube.ToSystemAffinity(affinity, nil) pod := &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: hostingPodName, Namespace: ownerObject.Namespace, OwnerReferences: []metav1.OwnerReference{ { APIVersion: ownerObject.APIVersion, Kind: ownerObject.Kind, Name: ownerObject.Name, UID: ownerObject.UID, Controller: boolptr.True(), }, }, Labels: label, Annotations: annotation, }, Spec: corev1api.PodSpec{ NodeSelector: nodeSelector, OS: &podOS, Affinity: podAffinity, Containers: []corev1api.Container{ { Name: containerName, Image: podInfo.image, ImagePullPolicy: corev1api.PullNever, Command: command, Args: args, VolumeMounts: volumeMounts, Env: podInfo.env, EnvFrom: podInfo.envFrom, Resources: resources, SecurityContext: containerSecurityCtx, }, }, PriorityClassName: priorityClassName, ServiceAccountName: podInfo.serviceAccount, TerminationGracePeriodSeconds: &gracePeriod, Volumes: volumes, NodeName: selectedNode, RestartPolicy: corev1api.RestartPolicyNever, SecurityContext: securityCtx, Tolerations: toleration, DNSPolicy: podInfo.dnsPolicy, DNSConfig: podInfo.dnsConfig, ImagePullSecrets: podInfo.imagePullSecrets, }, } return e.kubeClient.CoreV1().Pods(ownerObject.Namespace).Create(ctx, pod, metav1.CreateOptions{}) } ================================================ FILE: pkg/exposer/pod_volume_test.go ================================================ package exposer import ( "context" "errors" "testing" "time" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" appsv1api "k8s.io/api/apps/v1" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" clientTesting "k8s.io/client-go/testing" clientFake "sigs.k8s.io/controller-runtime/pkg/client/fake" velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/datapath" velerotest "github.com/vmware-tanzu/velero/pkg/test" "github.com/vmware-tanzu/velero/pkg/util/filesystem" ) func TestPodVolumeExpose(t *testing.T) { backup := &velerov1.Backup{ TypeMeta: metav1.TypeMeta{ APIVersion: velerov1.SchemeGroupVersion.String(), Kind: "Backup", }, ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1.DefaultNamespace, Name: "fake-backup", UID: "fake-uid", }, } podWithNoNode := builder.ForPod("fake-ns", "fake-client-pod").Result() podWithNode := builder.ForPod("fake-ns", "fake-client-pod").NodeName("fake-node").Result() node := builder.ForNode("fake-node").Result() daemonSet := &appsv1api.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", Name: "node-agent", }, TypeMeta: metav1.TypeMeta{ Kind: "DaemonSet", APIVersion: appsv1api.SchemeGroupVersion.String(), }, Spec: appsv1api.DaemonSetSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ Containers: []corev1api.Container{ { Name: "node-agent", }, }, }, }, }, } tests := []struct { name string snapshotClientObj []runtime.Object kubeClientObj []runtime.Object ownerBackup *velerov1.Backup exposeParam PodVolumeExposeParam funcGetPodVolumeHostPath func(context.Context, *corev1api.Pod, string, kubernetes.Interface, filesystem.Interface, logrus.FieldLogger) (datapath.AccessPoint, error) funcExtractPodVolumeHostPath func(context.Context, string, kubernetes.Interface, string, string) (string, error) kubeReactors []reactor expectBackupPod bool expectCachePVC bool err string }{ { name: "get client pod fail", ownerBackup: backup, exposeParam: PodVolumeExposeParam{ ClientNamespace: "fake-ns", ClientPodName: "fake-client-pod", }, err: "error getting client pod fake-client-pod: pods \"fake-client-pod\" not found", }, { name: "client pod with no node name", ownerBackup: backup, exposeParam: PodVolumeExposeParam{ ClientNamespace: "fake-ns", ClientPodName: "fake-client-pod", }, kubeClientObj: []runtime.Object{ podWithNoNode, }, err: "client pod fake-client-pod doesn't have a node name", }, { name: "get node os fail", ownerBackup: backup, exposeParam: PodVolumeExposeParam{ ClientNamespace: "fake-ns", ClientPodName: "fake-client-pod", }, kubeClientObj: []runtime.Object{ podWithNode, }, err: "error getting OS for node fake-node: error getting node fake-node: nodes \"fake-node\" not found", }, { name: "get pod volume path fail", ownerBackup: backup, exposeParam: PodVolumeExposeParam{ ClientNamespace: "fake-ns", ClientPodName: "fake-client-pod", ClientPodVolume: "fake-client-volume", }, kubeClientObj: []runtime.Object{ podWithNode, node, }, funcGetPodVolumeHostPath: func(context.Context, *corev1api.Pod, string, kubernetes.Interface, filesystem.Interface, logrus.FieldLogger) (datapath.AccessPoint, error) { return datapath.AccessPoint{}, errors.New("fake-get-pod-volume-path-error") }, err: "error to get pod volume path: fake-get-pod-volume-path-error", }, { name: "extract pod volume path fail", ownerBackup: backup, exposeParam: PodVolumeExposeParam{ ClientNamespace: "fake-ns", ClientPodName: "fake-client-pod", ClientPodVolume: "fake-client-volume", }, kubeClientObj: []runtime.Object{ podWithNode, node, }, funcGetPodVolumeHostPath: func(context.Context, *corev1api.Pod, string, kubernetes.Interface, filesystem.Interface, logrus.FieldLogger) (datapath.AccessPoint, error) { return datapath.AccessPoint{ ByPath: "/var/lib/kubelet/pods/pod-id-xxx/volumes/kubernetes.io~csi/pvc-id-xxx/mount", }, nil }, funcExtractPodVolumeHostPath: func(context.Context, string, kubernetes.Interface, string, string) (string, error) { return "", errors.New("fake-extract-error") }, err: "error to extract pod volume path: fake-extract-error", }, { name: "create hosting pod fail", ownerBackup: backup, exposeParam: PodVolumeExposeParam{ ClientNamespace: "fake-ns", ClientPodName: "fake-client-pod", ClientPodVolume: "fake-client-volume", }, kubeClientObj: []runtime.Object{ podWithNode, node, }, funcGetPodVolumeHostPath: func(context.Context, *corev1api.Pod, string, kubernetes.Interface, filesystem.Interface, logrus.FieldLogger) (datapath.AccessPoint, error) { return datapath.AccessPoint{ ByPath: "/host_pods/pod-id-xxx/volumes/kubernetes.io~csi/pvc-id-xxx/mount", }, nil }, funcExtractPodVolumeHostPath: func(context.Context, string, kubernetes.Interface, string, string) (string, error) { return "/var/lib/kubelet/pods/pod-id-xxx/volumes/kubernetes.io~csi/pvc-id-xxx/mount", nil }, err: "error to create hosting pod: error to get inherited pod info from node-agent: error to get node-agent pod template: error to get node-agent daemonset: daemonsets.apps \"node-agent\" not found", }, { name: "succeed", ownerBackup: backup, exposeParam: PodVolumeExposeParam{ ClientNamespace: "fake-ns", ClientPodName: "fake-client-pod", ClientPodVolume: "fake-client-volume", }, kubeClientObj: []runtime.Object{ podWithNode, node, daemonSet, }, funcGetPodVolumeHostPath: func(context.Context, *corev1api.Pod, string, kubernetes.Interface, filesystem.Interface, logrus.FieldLogger) (datapath.AccessPoint, error) { return datapath.AccessPoint{ ByPath: "/host_pods/pod-id-xxx/volumes/kubernetes.io~csi/pvc-id-xxx/mount", }, nil }, funcExtractPodVolumeHostPath: func(context.Context, string, kubernetes.Interface, string, string) (string, error) { return "/var/lib/kubelet/pods/pod-id-xxx/volumes/kubernetes.io~csi/pvc-id-xxx/mount", nil }, expectBackupPod: true, }, { name: "succeed with privileged pod", ownerBackup: backup, exposeParam: PodVolumeExposeParam{ ClientNamespace: "fake-ns", ClientPodName: "fake-client-pod", ClientPodVolume: "fake-client-volume", Privileged: true, }, kubeClientObj: []runtime.Object{ podWithNode, node, daemonSet, }, funcGetPodVolumeHostPath: func(context.Context, *corev1api.Pod, string, kubernetes.Interface, filesystem.Interface, logrus.FieldLogger) (datapath.AccessPoint, error) { return datapath.AccessPoint{ ByPath: "/host_pods/pod-id-xxx/volumes/kubernetes.io~csi/pvc-id-xxx/mount", }, nil }, funcExtractPodVolumeHostPath: func(context.Context, string, kubernetes.Interface, string, string) (string, error) { return "/var/lib/kubelet/pods/pod-id-xxx/volumes/kubernetes.io~csi/pvc-id-xxx/mount", nil }, expectBackupPod: true, }, { name: "succeed, cache config, no cache volume", ownerBackup: backup, exposeParam: PodVolumeExposeParam{ ClientNamespace: "fake-ns", ClientPodName: "fake-client-pod", ClientPodVolume: "fake-client-volume", CacheVolume: &CacheConfigs{}, }, kubeClientObj: []runtime.Object{ podWithNode, node, daemonSet, }, funcGetPodVolumeHostPath: func(context.Context, *corev1api.Pod, string, kubernetes.Interface, filesystem.Interface, logrus.FieldLogger) (datapath.AccessPoint, error) { return datapath.AccessPoint{ ByPath: "/host_pods/pod-id-xxx/volumes/kubernetes.io~csi/pvc-id-xxx/mount", }, nil }, funcExtractPodVolumeHostPath: func(context.Context, string, kubernetes.Interface, string, string) (string, error) { return "/var/lib/kubelet/pods/pod-id-xxx/volumes/kubernetes.io~csi/pvc-id-xxx/mount", nil }, expectBackupPod: true, }, { name: "create cache volume fail", ownerBackup: backup, exposeParam: PodVolumeExposeParam{ ClientNamespace: "fake-ns", ClientPodName: "fake-client-pod", ClientPodVolume: "fake-client-volume", CacheVolume: &CacheConfigs{Limit: 1024}, }, kubeClientObj: []runtime.Object{ podWithNode, node, daemonSet, }, funcGetPodVolumeHostPath: func(context.Context, *corev1api.Pod, string, kubernetes.Interface, filesystem.Interface, logrus.FieldLogger) (datapath.AccessPoint, error) { return datapath.AccessPoint{ ByPath: "/host_pods/pod-id-xxx/volumes/kubernetes.io~csi/pvc-id-xxx/mount", }, nil }, funcExtractPodVolumeHostPath: func(context.Context, string, kubernetes.Interface, string, string) (string, error) { return "/var/lib/kubelet/pods/pod-id-xxx/volumes/kubernetes.io~csi/pvc-id-xxx/mount", nil }, kubeReactors: []reactor{ { verb: "create", resource: "persistentvolumeclaims", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, errors.New("fake-create-error") }, }, }, err: "error to create cache pvc: fake-create-error", }, { name: "succeed with cache volume", ownerBackup: backup, exposeParam: PodVolumeExposeParam{ ClientNamespace: "fake-ns", ClientPodName: "fake-client-pod", ClientPodVolume: "fake-client-volume", CacheVolume: &CacheConfigs{Limit: 1024}, }, kubeClientObj: []runtime.Object{ podWithNode, node, daemonSet, }, funcGetPodVolumeHostPath: func(context.Context, *corev1api.Pod, string, kubernetes.Interface, filesystem.Interface, logrus.FieldLogger) (datapath.AccessPoint, error) { return datapath.AccessPoint{ ByPath: "/host_pods/pod-id-xxx/volumes/kubernetes.io~csi/pvc-id-xxx/mount", }, nil }, funcExtractPodVolumeHostPath: func(context.Context, string, kubernetes.Interface, string, string) (string, error) { return "/var/lib/kubelet/pods/pod-id-xxx/volumes/kubernetes.io~csi/pvc-id-xxx/mount", nil }, expectBackupPod: true, expectCachePVC: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...) for _, reactor := range test.kubeReactors { fakeKubeClient.Fake.PrependReactor(reactor.verb, reactor.resource, reactor.reactorFunc) } exposer := podVolumeExposer{ kubeClient: fakeKubeClient, log: velerotest.NewLogger(), } var ownerObject corev1api.ObjectReference if test.ownerBackup != nil { ownerObject = corev1api.ObjectReference{ Kind: test.ownerBackup.Kind, Namespace: test.ownerBackup.Namespace, Name: test.ownerBackup.Name, UID: test.ownerBackup.UID, APIVersion: test.ownerBackup.APIVersion, } } if test.funcGetPodVolumeHostPath != nil { getPodVolumeHostPath = test.funcGetPodVolumeHostPath } if test.funcExtractPodVolumeHostPath != nil { extractPodVolumeHostPath = test.funcExtractPodVolumeHostPath } err := exposer.Expose(t.Context(), ownerObject, test.exposeParam) if err == nil { require.NoError(t, err) _, err = exposer.kubeClient.CoreV1().Pods(ownerObject.Namespace).Get(t.Context(), ownerObject.Name, metav1.GetOptions{}) require.NoError(t, err) } else { require.EqualError(t, err, test.err) } _, err = exposer.kubeClient.CoreV1().Pods(ownerObject.Namespace).Get(t.Context(), ownerObject.Name, metav1.GetOptions{}) if test.expectBackupPod { require.NoError(t, err) } else { require.True(t, apierrors.IsNotFound(err)) } _, err = exposer.kubeClient.CoreV1().PersistentVolumeClaims(ownerObject.Namespace).Get(t.Context(), getCachePVCName(ownerObject), metav1.GetOptions{}) if test.expectCachePVC { require.NoError(t, err) } else { require.True(t, apierrors.IsNotFound(err)) } }) } } func TestGetPodVolumeExpose(t *testing.T) { backup := &velerov1.Backup{ TypeMeta: metav1.TypeMeta{ APIVersion: velerov1.SchemeGroupVersion.String(), Kind: "Backup", }, ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1.DefaultNamespace, Name: "fake-backup", UID: "fake-uid", }, } backupPodNotRunning := builder.ForPod(backup.Namespace, backup.Name).Result() backupPodRunning := builder.ForPod(backup.Namespace, backup.Name).Phase(corev1api.PodRunning).Result() scheme := runtime.NewScheme() corev1api.AddToScheme(scheme) tests := []struct { name string kubeClientObj []runtime.Object ownerBackup *velerov1.Backup nodeName string Timeout time.Duration err string expectedResult *ExposeResult }{ { name: "backup pod is not found", ownerBackup: backup, nodeName: "fake-node", }, { name: "wait backup pod running fail", ownerBackup: backup, nodeName: "fake-node", kubeClientObj: []runtime.Object{ backupPodNotRunning, }, Timeout: time.Second, err: "error to wait for rediness of pod fake-backup: context deadline exceeded", }, { name: "succeed", ownerBackup: backup, nodeName: "fake-node", kubeClientObj: []runtime.Object{ backupPodRunning, }, Timeout: time.Second, expectedResult: &ExposeResult{ ByPod: ExposeByPod{ HostingPod: backupPodRunning, VolumeName: string(backup.UID), }, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...) fakeClientBuilder := clientFake.NewClientBuilder() fakeClientBuilder = fakeClientBuilder.WithScheme(scheme) fakeClient := fakeClientBuilder.WithRuntimeObjects(test.kubeClientObj...).Build() exposer := podVolumeExposer{ kubeClient: fakeKubeClient, log: velerotest.NewLogger(), } var ownerObject corev1api.ObjectReference if test.ownerBackup != nil { ownerObject = corev1api.ObjectReference{ Kind: test.ownerBackup.Kind, Namespace: test.ownerBackup.Namespace, Name: test.ownerBackup.Name, UID: test.ownerBackup.UID, APIVersion: test.ownerBackup.APIVersion, } } result, err := exposer.GetExposed(t.Context(), ownerObject, fakeClient, test.nodeName, test.Timeout) if test.err == "" { require.NoError(t, err) if test.expectedResult == nil { assert.Nil(t, result) } else { require.NoError(t, err) assert.Equal(t, test.expectedResult.ByPod.VolumeName, result.ByPod.VolumeName) assert.Equal(t, test.expectedResult.ByPod.HostingPod.Name, result.ByPod.HostingPod.Name) } } else { assert.EqualError(t, err, test.err) } }) } } func TestPodVolumePeekExpose(t *testing.T) { backup := &velerov1.Backup{ TypeMeta: metav1.TypeMeta{ APIVersion: velerov1.SchemeGroupVersion.String(), Kind: "Backup", }, ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1.DefaultNamespace, Name: "fake-backup", UID: "fake-uid", }, } backupPodUrecoverable := &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: backup.Namespace, Name: backup.Name, }, Status: corev1api.PodStatus{ Phase: corev1api.PodFailed, }, } backupPod := &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: backup.Namespace, Name: backup.Name, }, } scheme := runtime.NewScheme() corev1api.AddToScheme(scheme) tests := []struct { name string kubeClientObj []runtime.Object ownerBackup *velerov1.Backup err string }{ { name: "backup pod is not found", ownerBackup: backup, }, { name: "pod is unrecoverable", ownerBackup: backup, kubeClientObj: []runtime.Object{ backupPodUrecoverable, }, err: "Pod is in abnormal state [Failed], message []", }, { name: "succeed", ownerBackup: backup, kubeClientObj: []runtime.Object{ backupPod, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...) exposer := podVolumeExposer{ kubeClient: fakeKubeClient, log: velerotest.NewLogger(), } var ownerObject corev1api.ObjectReference if test.ownerBackup != nil { ownerObject = corev1api.ObjectReference{ Kind: test.ownerBackup.Kind, Namespace: test.ownerBackup.Namespace, Name: test.ownerBackup.Name, UID: test.ownerBackup.UID, APIVersion: test.ownerBackup.APIVersion, } } err := exposer.PeekExposed(t.Context(), ownerObject) if test.err == "" { assert.NoError(t, err) } else { assert.EqualError(t, err, test.err) } }) } } func TestPodVolumeDiagnoseExpose(t *testing.T) { backup := &velerov1.Backup{ TypeMeta: metav1.TypeMeta{ APIVersion: velerov1.SchemeGroupVersion.String(), Kind: "Backup", }, ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1.DefaultNamespace, Name: "fake-backup", UID: "fake-uid", }, } backupPodWithoutNodeName := corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1.DefaultNamespace, Name: "fake-backup", UID: "fake-pod-uid", OwnerReferences: []metav1.OwnerReference{ { APIVersion: backup.APIVersion, Kind: backup.Kind, Name: backup.Name, UID: backup.UID, }, }, }, Status: corev1api.PodStatus{ Phase: corev1api.PodPending, Conditions: []corev1api.PodCondition{ { Type: corev1api.PodInitialized, Status: corev1api.ConditionTrue, Message: "fake-pod-message", }, }, Message: "fake-pod-message-1", }, } backupPodWithNodeName := corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1.DefaultNamespace, Name: "fake-backup", UID: "fake-pod-uid", OwnerReferences: []metav1.OwnerReference{ { APIVersion: backup.APIVersion, Kind: backup.Kind, Name: backup.Name, UID: backup.UID, }, }, }, Spec: corev1api.PodSpec{ NodeName: "fake-node", }, Status: corev1api.PodStatus{ Phase: corev1api.PodPending, Conditions: []corev1api.PodCondition{ { Type: corev1api.PodInitialized, Status: corev1api.ConditionTrue, Message: "fake-pod-message", }, }, }, } cachePVCWithVolumeName := corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1.DefaultNamespace, Name: "fake-backup-cache", UID: "fake-cache-pvc-uid", OwnerReferences: []metav1.OwnerReference{ { APIVersion: backup.APIVersion, Kind: backup.Kind, Name: backup.Name, UID: backup.UID, }, }, }, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "fake-pv-cache", }, Status: corev1api.PersistentVolumeClaimStatus{ Phase: corev1api.ClaimPending, }, } cachePV := corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-pv-cache", }, Status: corev1api.PersistentVolumeStatus{ Phase: corev1api.VolumePending, Message: "fake-pv-message", }, } nodeAgentPod := corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1.DefaultNamespace, Name: "node-agent-pod-1", Labels: map[string]string{"role": "node-agent"}, }, Spec: corev1api.PodSpec{ NodeName: "fake-node", }, Status: corev1api.PodStatus{ Phase: corev1api.PodRunning, }, } tests := []struct { name string ownerBackup *velerov1.Backup kubeClientObj []runtime.Object snapshotClientObj []runtime.Object expected string }{ { name: "no pod", ownerBackup: backup, expected: `begin diagnose pod volume exposer error getting hosting pod fake-backup, err: pods "fake-backup" not found end diagnose pod volume exposer`, }, { name: "pod without node name, pvc without volume name, vs without status", ownerBackup: backup, kubeClientObj: []runtime.Object{ &backupPodWithoutNodeName, }, expected: `begin diagnose pod volume exposer Pod velero/fake-backup, phase Pending, node name , message fake-pod-message-1 Pod condition Initialized, status True, reason , message fake-pod-message end diagnose pod volume exposer`, }, { name: "pod without node name", ownerBackup: backup, kubeClientObj: []runtime.Object{ &backupPodWithoutNodeName, }, expected: `begin diagnose pod volume exposer Pod velero/fake-backup, phase Pending, node name , message fake-pod-message-1 Pod condition Initialized, status True, reason , message fake-pod-message end diagnose pod volume exposer`, }, { name: "pod with node name, no node agent", ownerBackup: backup, kubeClientObj: []runtime.Object{ &backupPodWithNodeName, }, expected: `begin diagnose pod volume exposer Pod velero/fake-backup, phase Pending, node name fake-node, message Pod condition Initialized, status True, reason , message fake-pod-message node-agent is not running in node fake-node, err: daemonset pod not found in running state in node fake-node end diagnose pod volume exposer`, }, { name: "pod with node name, node agent is running", ownerBackup: backup, kubeClientObj: []runtime.Object{ &backupPodWithNodeName, &nodeAgentPod, }, expected: `begin diagnose pod volume exposer Pod velero/fake-backup, phase Pending, node name fake-node, message Pod condition Initialized, status True, reason , message fake-pod-message end diagnose pod volume exposer`, }, { name: "cache pvc with volume name, no pv", ownerBackup: backup, kubeClientObj: []runtime.Object{ &backupPodWithNodeName, &cachePVCWithVolumeName, &nodeAgentPod, }, expected: `begin diagnose pod volume exposer Pod velero/fake-backup, phase Pending, node name fake-node, message Pod condition Initialized, status True, reason , message fake-pod-message PVC velero/fake-backup-cache, phase Pending, binding to fake-pv-cache error getting cache pv fake-pv-cache, err: persistentvolumes "fake-pv-cache" not found end diagnose pod volume exposer`, }, { name: "cache pvc with volume name, pv exists", ownerBackup: backup, kubeClientObj: []runtime.Object{ &backupPodWithNodeName, &cachePVCWithVolumeName, &cachePV, &nodeAgentPod, }, expected: `begin diagnose pod volume exposer Pod velero/fake-backup, phase Pending, node name fake-node, message Pod condition Initialized, status True, reason , message fake-pod-message PVC velero/fake-backup-cache, phase Pending, binding to fake-pv-cache PV fake-pv-cache, phase Pending, reason , message fake-pv-message end diagnose pod volume exposer`, }, { name: "with events", ownerBackup: backup, kubeClientObj: []runtime.Object{ &backupPodWithNodeName, &nodeAgentPod, &corev1api.Event{ ObjectMeta: metav1.ObjectMeta{Namespace: velerov1.DefaultNamespace, Name: "event-1"}, Type: corev1api.EventTypeWarning, InvolvedObject: corev1api.ObjectReference{UID: "fake-uid-1"}, Reason: "reason-1", Message: "message-1", }, &corev1api.Event{ ObjectMeta: metav1.ObjectMeta{Namespace: velerov1.DefaultNamespace, Name: "event-2"}, Type: corev1api.EventTypeWarning, InvolvedObject: corev1api.ObjectReference{UID: "fake-pod-uid"}, Reason: "reason-2", Message: "message-2", }, &corev1api.Event{ ObjectMeta: metav1.ObjectMeta{Namespace: "other-namespace", Name: "event-3"}, Type: corev1api.EventTypeWarning, InvolvedObject: corev1api.ObjectReference{UID: "fake-pod-uid"}, Reason: "reason-3", Message: "message-3", }, &corev1api.Event{ ObjectMeta: metav1.ObjectMeta{Namespace: velerov1.DefaultNamespace, Name: "event-4"}, Type: corev1api.EventTypeWarning, InvolvedObject: corev1api.ObjectReference{UID: "fake-pod-uid"}, Reason: "reason-4", Message: "message-4", }, }, expected: `begin diagnose pod volume exposer Pod velero/fake-backup, phase Pending, node name fake-node, message Pod condition Initialized, status True, reason , message fake-pod-message Pod event reason reason-2, message message-2 Pod event reason reason-4, message message-4 end diagnose pod volume exposer`, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { fakeKubeClient := fake.NewSimpleClientset(tt.kubeClientObj...) e := &podVolumeExposer{ kubeClient: fakeKubeClient, log: velerotest.NewLogger(), } var ownerObject corev1api.ObjectReference if tt.ownerBackup != nil { ownerObject = corev1api.ObjectReference{ Kind: tt.ownerBackup.Kind, Namespace: tt.ownerBackup.Namespace, Name: tt.ownerBackup.Name, UID: tt.ownerBackup.UID, APIVersion: tt.ownerBackup.APIVersion, } } diag := e.DiagnoseExpose(t.Context(), ownerObject) assert.Equal(t, tt.expected, diag) }) } } ================================================ FILE: pkg/exposer/snapshot.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package exposer import ( "context" "time" corev1api "k8s.io/api/core/v1" ) // SnapshotExposer is the interfaces for a snapshot exposer type SnapshotExposer interface { // Expose starts the process to expose a snapshot, the expose process may take long time Expose(context.Context, corev1api.ObjectReference, any) error // GetExposed polls the status of the expose. // If the expose is accessible by the current caller, it waits the expose ready and returns the expose result. // Otherwise, it returns nil as the expose result without an error. GetExposed(context.Context, corev1api.ObjectReference, time.Duration, any) (*ExposeResult, error) // PeekExposed tests the status of the expose. // If the expose is incomplete but not recoverable, it returns an error. // Otherwise, it returns nil immediately. PeekExposed(context.Context, corev1api.ObjectReference) error // DiagnoseExpose generate the diagnostic info when the expose is not finished for a long time. // If it finds any problem, it returns an string about the problem. DiagnoseExpose(context.Context, corev1api.ObjectReference) string // CleanUp cleans up any objects generated during the snapshot expose CleanUp(context.Context, corev1api.ObjectReference, string, string) } ================================================ FILE: pkg/exposer/types.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package exposer import ( corev1api "k8s.io/api/core/v1" ) const ( AccessModeFileSystem = "by-file-system" AccessModeBlock = "by-block-device" podGroupLabel = "velero.io/exposer-pod-group" podGroupSnapshot = "snapshot-exposer" podGroupGenericRestore = "generic-restore-exposer" ExposeOnGoingLabel = "velero.io/expose-on-going" ) // ExposeResult defines the result of expose. // Varying from the type of the expose, the result may be different. type ExposeResult struct { ByPod ExposeByPod } // ExposeByPod defines the result for the expose method that a hosting pod is created type ExposeByPod struct { HostingPod *corev1api.Pod HostingContainer string VolumeName string NodeOS *string } ================================================ FILE: pkg/exposer/vgdp_counter.go ================================================ package exposer import ( "context" "sync/atomic" "github.com/pkg/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/tools/cache" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" "sigs.k8s.io/controller-runtime/pkg/manager" ctlclient "sigs.k8s.io/controller-runtime/pkg/client" ) type dynamicQueueLength struct { queueLength int changeID uint64 } type VgdpCounter struct { client ctlclient.Client allowedQueueLength int duState dynamicQueueLength ddState dynamicQueueLength pvbState dynamicQueueLength pvrState dynamicQueueLength duCacheState dynamicQueueLength ddCacheState dynamicQueueLength pvbCacheState dynamicQueueLength pvrCacheState dynamicQueueLength } func StartVgdpCounter(ctx context.Context, mgr manager.Manager, queueLength int) (*VgdpCounter, error) { counter := &VgdpCounter{ client: mgr.GetClient(), allowedQueueLength: queueLength, } atomic.StoreUint64(&counter.duState.changeID, 1) atomic.StoreUint64(&counter.ddState.changeID, 1) atomic.StoreUint64(&counter.pvbState.changeID, 1) atomic.StoreUint64(&counter.pvrState.changeID, 1) if err := counter.initListeners(ctx, mgr); err != nil { return nil, err } return counter, nil } func (w *VgdpCounter) initListeners(ctx context.Context, mgr manager.Manager) error { duInformer, err := mgr.GetCache().GetInformer(ctx, &velerov2alpha1api.DataUpload{}) if err != nil { return errors.Wrap(err, "error getting du informer") } if _, err := duInformer.AddEventHandler( cache.ResourceEventHandlerFuncs{ UpdateFunc: func(oldObj, newObj any) { oldDu := oldObj.(*velerov2alpha1api.DataUpload) newDu := newObj.(*velerov2alpha1api.DataUpload) if oldDu.Status.Phase == newDu.Status.Phase { return } if newDu.Status.Phase == velerov2alpha1api.DataUploadPhaseAccepted || oldDu.Status.Phase == velerov2alpha1api.DataUploadPhasePrepared || oldDu.Status.Phase == velerov2alpha1api.DataUploadPhaseAccepted && newDu.Status.Phase != velerov2alpha1api.DataUploadPhasePrepared { atomic.AddUint64(&w.duState.changeID, 1) } }, }, ); err != nil { return errors.Wrap(err, "error registering du handler") } ddInformer, err := mgr.GetCache().GetInformer(ctx, &velerov2alpha1api.DataDownload{}) if err != nil { return errors.Wrap(err, "error getting dd informer") } if _, err := ddInformer.AddEventHandler( cache.ResourceEventHandlerFuncs{ UpdateFunc: func(oldObj, newObj any) { oldDd := oldObj.(*velerov2alpha1api.DataDownload) newDd := newObj.(*velerov2alpha1api.DataDownload) if oldDd.Status.Phase == newDd.Status.Phase { return } if newDd.Status.Phase == velerov2alpha1api.DataDownloadPhaseAccepted || oldDd.Status.Phase == velerov2alpha1api.DataDownloadPhasePrepared || oldDd.Status.Phase == velerov2alpha1api.DataDownloadPhaseAccepted && newDd.Status.Phase != velerov2alpha1api.DataDownloadPhasePrepared { atomic.AddUint64(&w.ddState.changeID, 1) } }, }, ); err != nil { return errors.Wrap(err, "error registering dd handler") } pvbInformer, err := mgr.GetCache().GetInformer(ctx, &velerov1api.PodVolumeBackup{}) if err != nil { return errors.Wrap(err, "error getting PVB informer") } if _, err := pvbInformer.AddEventHandler( cache.ResourceEventHandlerFuncs{ UpdateFunc: func(oldObj, newObj any) { oldPvb := oldObj.(*velerov1api.PodVolumeBackup) newPvb := newObj.(*velerov1api.PodVolumeBackup) if oldPvb.Status.Phase == newPvb.Status.Phase { return } if newPvb.Status.Phase == velerov1api.PodVolumeBackupPhaseAccepted || oldPvb.Status.Phase == velerov1api.PodVolumeBackupPhasePrepared || oldPvb.Status.Phase == velerov1api.PodVolumeBackupPhaseAccepted && newPvb.Status.Phase != velerov1api.PodVolumeBackupPhasePrepared { atomic.AddUint64(&w.pvbState.changeID, 1) } }, }, ); err != nil { return errors.Wrap(err, "error registering PVB handler") } pvrInformer, err := mgr.GetCache().GetInformer(ctx, &velerov1api.PodVolumeRestore{}) if err != nil { return errors.Wrap(err, "error getting PVR informer") } if _, err := pvrInformer.AddEventHandler( cache.ResourceEventHandlerFuncs{ UpdateFunc: func(oldObj, newObj any) { oldPvr := oldObj.(*velerov1api.PodVolumeRestore) newPvr := newObj.(*velerov1api.PodVolumeRestore) if oldPvr.Status.Phase == newPvr.Status.Phase { return } if newPvr.Status.Phase == velerov1api.PodVolumeRestorePhaseAccepted || oldPvr.Status.Phase == velerov1api.PodVolumeRestorePhasePrepared || oldPvr.Status.Phase == velerov1api.PodVolumeRestorePhaseAccepted && newPvr.Status.Phase != velerov1api.PodVolumeRestorePhasePrepared { atomic.AddUint64(&w.pvrState.changeID, 1) } }, }, ); err != nil { return errors.Wrap(err, "error registering PVR handler") } return nil } func (w *VgdpCounter) IsConstrained(ctx context.Context, log logrus.FieldLogger) bool { id := atomic.LoadUint64(&w.duState.changeID) if id != w.duCacheState.changeID { duList := &velerov2alpha1api.DataUploadList{} if err := w.client.List(ctx, duList, &ctlclient.ListOptions{LabelSelector: labels.SelectorFromSet(labels.Set(map[string]string{ExposeOnGoingLabel: "true"}))}); err != nil { log.WithError(err).Warn("Failed to list data uploads, skip counting") } else { w.duCacheState.queueLength = len(duList.Items) w.duCacheState.changeID = id log.Infof("Query queue length for du %d", w.duCacheState.queueLength) } } id = atomic.LoadUint64(&w.ddState.changeID) if id != w.ddCacheState.changeID { ddList := &velerov2alpha1api.DataDownloadList{} if err := w.client.List(ctx, ddList, &ctlclient.ListOptions{LabelSelector: labels.SelectorFromSet(labels.Set(map[string]string{ExposeOnGoingLabel: "true"}))}); err != nil { log.WithError(err).Warn("Failed to list data downloads, skip counting") } else { w.ddCacheState.queueLength = len(ddList.Items) w.ddCacheState.changeID = id log.Infof("Query queue length for dd %d", w.ddCacheState.queueLength) } } id = atomic.LoadUint64(&w.pvbState.changeID) if id != w.pvbCacheState.changeID { pvbList := &velerov1api.PodVolumeBackupList{} if err := w.client.List(ctx, pvbList, &ctlclient.ListOptions{LabelSelector: labels.SelectorFromSet(labels.Set(map[string]string{ExposeOnGoingLabel: "true"}))}); err != nil { log.WithError(err).Warn("Failed to list PVB, skip counting") } else { w.pvbCacheState.queueLength = len(pvbList.Items) w.pvbCacheState.changeID = id log.Infof("Query queue length for pvb %d", w.pvbCacheState.queueLength) } } id = atomic.LoadUint64(&w.pvrState.changeID) if id != w.pvrCacheState.changeID { pvrList := &velerov1api.PodVolumeRestoreList{} if err := w.client.List(ctx, pvrList, &ctlclient.ListOptions{LabelSelector: labels.SelectorFromSet(labels.Set(map[string]string{ExposeOnGoingLabel: "true"}))}); err != nil { log.WithError(err).Warn("Failed to list PVR, skip counting") } else { w.pvrCacheState.queueLength = len(pvrList.Items) w.pvrCacheState.changeID = id log.Infof("Query queue length for pvr %d", w.pvrCacheState.queueLength) } } existing := w.duCacheState.queueLength + w.ddCacheState.queueLength + w.pvbCacheState.queueLength + w.pvrCacheState.queueLength constrained := existing >= w.allowedQueueLength return constrained } ================================================ FILE: pkg/exposer/vgdp_counter_test.go ================================================ package exposer import ( "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "github.com/vmware-tanzu/velero/pkg/builder" velerotest "github.com/vmware-tanzu/velero/pkg/test" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" ) func TestIsConstrained(t *testing.T) { tests := []struct { name string counter VgdpCounter kubeClientObj []client.Object getErr bool expected bool }{ { name: "no change, constrained", counter: VgdpCounter{}, expected: true, }, { name: "no change, not constrained", counter: VgdpCounter{allowedQueueLength: 1}, }, { name: "change in du, get failed", counter: VgdpCounter{ allowedQueueLength: 1, duState: dynamicQueueLength{0, 1}, }, getErr: true, }, { name: "change in du, constrained", counter: VgdpCounter{ allowedQueueLength: 1, duState: dynamicQueueLength{0, 1}, }, kubeClientObj: []client.Object{ builder.ForDataUpload("velero", "test-1").Labels(map[string]string{ExposeOnGoingLabel: "true"}).Result(), }, expected: true, }, { name: "change in dd, get failed", counter: VgdpCounter{ allowedQueueLength: 1, ddState: dynamicQueueLength{0, 1}, }, getErr: true, }, { name: "change in dd, constrained", counter: VgdpCounter{ allowedQueueLength: 1, ddState: dynamicQueueLength{0, 1}, }, kubeClientObj: []client.Object{ builder.ForDataDownload("velero", "test-1").Labels(map[string]string{ExposeOnGoingLabel: "true"}).Result(), }, expected: true, }, { name: "change in pvb, get failed", counter: VgdpCounter{ allowedQueueLength: 1, pvbState: dynamicQueueLength{0, 1}, }, getErr: true, }, { name: "change in pvb, constrained", counter: VgdpCounter{ allowedQueueLength: 1, pvbState: dynamicQueueLength{0, 1}, }, kubeClientObj: []client.Object{ builder.ForPodVolumeBackup("velero", "test-1").Labels(map[string]string{ExposeOnGoingLabel: "true"}).Result(), }, expected: true, }, { name: "change in pvr, get failed", counter: VgdpCounter{ allowedQueueLength: 1, pvrState: dynamicQueueLength{0, 1}, }, getErr: true, }, { name: "change in pvr, constrained", counter: VgdpCounter{ allowedQueueLength: 1, pvrState: dynamicQueueLength{0, 1}, }, kubeClientObj: []client.Object{ builder.ForPodVolumeRestore("velero", "test-1").Labels(map[string]string{ExposeOnGoingLabel: "true"}).Result(), }, expected: true, }, { name: "change in du, pvb, not constrained", counter: VgdpCounter{ allowedQueueLength: 3, duState: dynamicQueueLength{0, 1}, pvbState: dynamicQueueLength{0, 1}, }, kubeClientObj: []client.Object{ builder.ForDataUpload("velero", "test-1").Labels(map[string]string{ExposeOnGoingLabel: "true"}).Result(), builder.ForPodVolumeBackup("velero", "test-1").Labels(map[string]string{ExposeOnGoingLabel: "true"}).Result(), }, }, { name: "change in dd, pvr, constrained", counter: VgdpCounter{ allowedQueueLength: 1, ddState: dynamicQueueLength{0, 1}, pvrState: dynamicQueueLength{0, 1}, }, kubeClientObj: []client.Object{ builder.ForDataDownload("velero", "test-1").Labels(map[string]string{ExposeOnGoingLabel: "true"}).Result(), builder.ForPodVolumeRestore("velero", "test-1").Labels(map[string]string{ExposeOnGoingLabel: "true"}).Result(), }, expected: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { scheme := runtime.NewScheme() if !test.getErr { err := velerov1api.AddToScheme(scheme) require.NoError(t, err) err = velerov2alpha1api.AddToScheme(scheme) require.NoError(t, err) } test.counter.client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(test.kubeClientObj...).Build() result := test.counter.IsConstrained(t.Context(), velerotest.NewLogger()) assert.Equal(t, test.expected, result) if !test.getErr { assert.Equal(t, test.counter.duState.changeID, test.counter.duCacheState.changeID) assert.Equal(t, test.counter.ddState.changeID, test.counter.ddCacheState.changeID) assert.Equal(t, test.counter.pvbState.changeID, test.counter.pvbCacheState.changeID) assert.Equal(t, test.counter.pvrState.changeID, test.counter.pvrCacheState.changeID) } else { or := test.counter.duState.changeID != test.counter.duCacheState.changeID if !or { or = test.counter.ddState.changeID != test.counter.ddCacheState.changeID } if !or { or = test.counter.pvbState.changeID != test.counter.pvbCacheState.changeID } if !or { or = test.counter.pvrState.changeID != test.counter.pvrCacheState.changeID } assert.True(t, or) } }) } } ================================================ FILE: pkg/features/feature_flags.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package features import ( "strings" "k8s.io/apimachinery/pkg/util/sets" ) type featureFlagSet struct { set sets.Set[string] } // featureFlags will store all the flags for this process until NewFeatureFlagSet is called. var featureFlags featureFlagSet // IsEnabled returns True if a specified flag is enabled. func IsEnabled(name string) bool { return featureFlags.set.Has(name) } // Enable adds a given slice of feature names to the current feature list. func Enable(names ...string) { // Initialize the flag set so that users don't have to if featureFlags.set == nil { NewFeatureFlagSet() } featureFlags.set.Insert(names...) } // Disable removes all feature flags in a given slice from the current feature list. func Disable(names ...string) { featureFlags.set.Delete(names...) } // All returns enabled features as a slice of strings. func All() []string { return sets.List[string](featureFlags.set) } // Serialize returns all features as a comma-separated string. func Serialize() string { return strings.Join(All(), ",") } // NewFeatureFlagSet initializes and populates a new FeatureFlagSet. // This must be called to properly initialize the set for tracking flags. // It is also useful for selectively controlling flags during tests. func NewFeatureFlagSet(flags ...string) { featureFlags = featureFlagSet{ set: sets.New[string](flags...), } } ================================================ FILE: pkg/features/feature_flags_test.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package features import ( "testing" "github.com/stretchr/testify/assert" ) func TestFeatureFlags(t *testing.T) { // Prepare a new flag set NewFeatureFlagSet("feature1", "feature2") assert.True(t, IsEnabled("feature1")) assert.False(t, IsEnabled("feature3")) assert.Equal(t, []string{"feature1", "feature2"}, All()) Enable("feature3") assert.True(t, IsEnabled("feature3")) assert.Equal(t, []string{"feature1", "feature2", "feature3"}, All()) Disable("feature3") assert.Equal(t, []string{"feature1", "feature2"}, All()) assert.Equal(t, "feature1,feature2", Serialize()) // Calling NewFeatureFlagSet re-initializes the set of flags NewFeatureFlagSet() assert.Empty(t, All()) } ================================================ FILE: pkg/install/daemonset.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package install import ( "fmt" "path/filepath" "strings" appsv1api "k8s.io/api/apps/v1" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" "github.com/vmware-tanzu/velero/internal/velero" "github.com/vmware-tanzu/velero/pkg/nodeagent" ) func DaemonSet(namespace string, opts ...podTemplateOption) *appsv1api.DaemonSet { c := &podTemplateConfig{ image: velero.DefaultVeleroImage(), } for _, opt := range opts { opt(c) } pullPolicy := corev1api.PullAlways imageParts := strings.Split(c.image, ":") if len(imageParts) == 2 && imageParts[1] != "latest" { pullPolicy = corev1api.PullIfNotPresent } daemonSetArgs := []string{ "node-agent", "server", } if len(c.features) > 0 { daemonSetArgs = append(daemonSetArgs, fmt.Sprintf("--features=%s", strings.Join(c.features, ","))) } if len(c.nodeAgentConfigMap) > 0 { daemonSetArgs = append(daemonSetArgs, fmt.Sprintf("--node-agent-configmap=%s", c.nodeAgentConfigMap)) } if len(c.backupRepoConfigMap) > 0 { daemonSetArgs = append(daemonSetArgs, fmt.Sprintf("--backup-repository-configmap=%s", c.backupRepoConfigMap)) } userID := int64(0) mountPropagationMode := corev1api.MountPropagationHostToContainer dsName := "node-agent" if c.forWindows { dsName = "node-agent-windows" } hostPodsVolumePath := filepath.Join(c.kubeletRootDir, "pods") hostPluginsVolumePath := filepath.Join(c.kubeletRootDir, "plugins") volumes := []corev1api.Volume{} volumeMounts := []corev1api.VolumeMount{} if !c.nodeAgentDisableHostPath { volumes = append(volumes, []corev1api.Volume{ { Name: "host-pods", VolumeSource: corev1api.VolumeSource{ HostPath: &corev1api.HostPathVolumeSource{ Path: hostPodsVolumePath, }, }, }, { Name: "host-plugins", VolumeSource: corev1api.VolumeSource{ HostPath: &corev1api.HostPathVolumeSource{ Path: hostPluginsVolumePath, }, }, }, }...) volumeMounts = append(volumeMounts, []corev1api.VolumeMount{ { Name: nodeagent.HostPodVolumeMount, MountPath: nodeagent.HostPodVolumeMountPath(), MountPropagation: &mountPropagationMode, }, { Name: "host-plugins", MountPath: "/var/lib/kubelet/plugins", MountPropagation: &mountPropagationMode, }, }...) } volumes = append(volumes, corev1api.Volume{ Name: "scratch", VolumeSource: corev1api.VolumeSource{ EmptyDir: new(corev1api.EmptyDirVolumeSource), }, }) volumeMounts = append(volumeMounts, corev1api.VolumeMount{ Name: "scratch", MountPath: "/scratch", }) daemonSet := &appsv1api.DaemonSet{ ObjectMeta: objectMeta(namespace, dsName), TypeMeta: metav1.TypeMeta{ Kind: "DaemonSet", APIVersion: appsv1api.SchemeGroupVersion.String(), }, Spec: appsv1api.DaemonSetSpec{ Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "name": dsName, }, }, Template: corev1api.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: podLabels(c.labels, map[string]string{ "name": dsName, "role": "node-agent", }), Annotations: c.annotations, }, Spec: corev1api.PodSpec{ ServiceAccountName: c.serviceAccountName, SecurityContext: &corev1api.PodSecurityContext{ RunAsUser: &userID, }, Volumes: volumes, Containers: []corev1api.Container{ { Name: dsName, Image: c.image, Ports: containerPorts(), ImagePullPolicy: pullPolicy, Command: []string{ "/velero", }, Args: daemonSetArgs, SecurityContext: &corev1api.SecurityContext{ Privileged: &c.privilegedNodeAgent, }, VolumeMounts: volumeMounts, Env: []corev1api.EnvVar{ { Name: "NODE_NAME", ValueFrom: &corev1api.EnvVarSource{ FieldRef: &corev1api.ObjectFieldSelector{ FieldPath: "spec.nodeName", }, }, }, { Name: "VELERO_NAMESPACE", ValueFrom: &corev1api.EnvVarSource{ FieldRef: &corev1api.ObjectFieldSelector{ FieldPath: "metadata.namespace", }, }, }, { Name: "VELERO_SCRATCH_DIR", Value: "/scratch", }, }, Resources: c.resources, }, }, PriorityClassName: c.priorityClassName, }, }, }, } if c.withSecret { daemonSet.Spec.Template.Spec.Volumes = append( daemonSet.Spec.Template.Spec.Volumes, corev1api.Volume{ Name: "cloud-credentials", VolumeSource: corev1api.VolumeSource{ Secret: &corev1api.SecretVolumeSource{ // read-only for Owner, Group, Public DefaultMode: ptr.To(int32(0444)), SecretName: "cloud-credentials", }, }, }, ) daemonSet.Spec.Template.Spec.Containers[0].VolumeMounts = append( daemonSet.Spec.Template.Spec.Containers[0].VolumeMounts, corev1api.VolumeMount{ Name: "cloud-credentials", MountPath: "/credentials", }, ) daemonSet.Spec.Template.Spec.Containers[0].Env = append(daemonSet.Spec.Template.Spec.Containers[0].Env, []corev1api.EnvVar{ { Name: "GOOGLE_APPLICATION_CREDENTIALS", Value: "/credentials/cloud", }, { Name: "AWS_SHARED_CREDENTIALS_FILE", Value: "/credentials/cloud", }, { Name: "AZURE_CREDENTIALS_FILE", Value: "/credentials/cloud", }, { Name: "ALIBABA_CLOUD_CREDENTIALS_FILE", Value: "/credentials/cloud", }, }...) } if c.forWindows { daemonSet.Spec.Template.Spec.SecurityContext = nil daemonSet.Spec.Template.Spec.Containers[0].SecurityContext = nil daemonSet.Spec.Template.Spec.OS = &corev1api.PodOS{ Name: "windows", } daemonSet.Spec.Template.Spec.Affinity = &corev1api.Affinity{ NodeAffinity: &corev1api.NodeAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ NodeSelectorTerms: []corev1api.NodeSelectorTerm{ { MatchExpressions: []corev1api.NodeSelectorRequirement{ { Key: "kubernetes.io/os", Values: []string{"windows"}, Operator: corev1api.NodeSelectorOpIn, }, }, }, }, }, }, } daemonSet.Spec.Template.Spec.Tolerations = []corev1api.Toleration{ { Key: "os", Operator: "Equal", Effect: "NoSchedule", Value: "windows", }, { Key: "os", Operator: "Equal", Effect: "NoExecute", Value: "windows", }, } } else { daemonSet.Spec.Template.Spec.Affinity = &corev1api.Affinity{ NodeAffinity: &corev1api.NodeAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ NodeSelectorTerms: []corev1api.NodeSelectorTerm{ { MatchExpressions: []corev1api.NodeSelectorRequirement{ { Key: "kubernetes.io/os", Values: []string{"windows"}, Operator: corev1api.NodeSelectorOpNotIn, }, }, }, }, }, }, } } daemonSet.Spec.Template.Spec.Containers[0].Env = append(daemonSet.Spec.Template.Spec.Containers[0].Env, c.envVars...) return daemonSet } ================================================ FILE: pkg/install/daemonset_test.go ================================================ /* Copyright 2019, 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package install import ( "testing" "github.com/stretchr/testify/assert" corev1api "k8s.io/api/core/v1" ) func TestDaemonSet(t *testing.T) { userID := int64(0) boolFalse := false boolTrue := true ds := DaemonSet("velero") assert.Equal(t, "node-agent", ds.Spec.Template.Spec.Containers[0].Name) assert.Equal(t, "velero", ds.ObjectMeta.Namespace) assert.Equal(t, "node-agent", ds.Spec.Template.ObjectMeta.Labels["name"]) assert.Equal(t, "node-agent", ds.Spec.Template.ObjectMeta.Labels["role"]) assert.Equal(t, &corev1api.Affinity{ NodeAffinity: &corev1api.NodeAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ NodeSelectorTerms: []corev1api.NodeSelectorTerm{ { MatchExpressions: []corev1api.NodeSelectorRequirement{ { Key: "kubernetes.io/os", Values: []string{"windows"}, Operator: corev1api.NodeSelectorOpNotIn, }, }, }, }, }, }, }, ds.Spec.Template.Spec.Affinity) assert.Equal(t, corev1api.PodSecurityContext{RunAsUser: &userID}, *ds.Spec.Template.Spec.SecurityContext) assert.Equal(t, corev1api.SecurityContext{Privileged: &boolFalse}, *ds.Spec.Template.Spec.Containers[0].SecurityContext) assert.Len(t, ds.Spec.Template.Spec.Volumes, 3) assert.Len(t, ds.Spec.Template.Spec.Containers[0].VolumeMounts, 3) ds = DaemonSet("velero", WithPrivilegedNodeAgent(true)) assert.Equal(t, corev1api.SecurityContext{Privileged: &boolTrue}, *ds.Spec.Template.Spec.Containers[0].SecurityContext) ds = DaemonSet("velero", WithImage("velero/velero:v0.11")) assert.Equal(t, "velero/velero:v0.11", ds.Spec.Template.Spec.Containers[0].Image) assert.Equal(t, corev1api.PullIfNotPresent, ds.Spec.Template.Spec.Containers[0].ImagePullPolicy) ds = DaemonSet("velero", WithSecret(true)) assert.Len(t, ds.Spec.Template.Spec.Containers[0].Env, 7) assert.Len(t, ds.Spec.Template.Spec.Volumes, 4) ds = DaemonSet("velero", WithFeatures([]string{"foo,bar,baz"})) assert.Len(t, ds.Spec.Template.Spec.Containers[0].Args, 3) assert.Equal(t, "--features=foo,bar,baz", ds.Spec.Template.Spec.Containers[0].Args[2]) ds = DaemonSet("velero", WithNodeAgentConfigMap("node-agent-config-map")) assert.Len(t, ds.Spec.Template.Spec.Containers[0].Args, 3) assert.Equal(t, "--node-agent-configmap=node-agent-config-map", ds.Spec.Template.Spec.Containers[0].Args[2]) ds = DaemonSet("velero", WithBackupRepoConfigMap("backup-repo-config-map")) assert.Len(t, ds.Spec.Template.Spec.Containers[0].Args, 3) assert.Equal(t, "--backup-repository-configmap=backup-repo-config-map", ds.Spec.Template.Spec.Containers[0].Args[2]) ds = DaemonSet("velero", WithServiceAccountName("test-sa")) assert.Equal(t, "test-sa", ds.Spec.Template.Spec.ServiceAccountName) ds = DaemonSet("velero", WithKubeletRootDir("/data/test/kubelet")) assert.Equal(t, "/data/test/kubelet/pods", ds.Spec.Template.Spec.Volumes[0].HostPath.Path) assert.Equal(t, "/data/test/kubelet/plugins", ds.Spec.Template.Spec.Volumes[1].HostPath.Path) ds = DaemonSet("velero", WithNodeAgentDisableHostPath(true)) assert.Len(t, ds.Spec.Template.Spec.Volumes, 1) assert.Len(t, ds.Spec.Template.Spec.Containers[0].VolumeMounts, 1) ds = DaemonSet("velero", WithForWindows()) assert.Equal(t, "node-agent-windows", ds.Spec.Template.Spec.Containers[0].Name) assert.Equal(t, "velero", ds.ObjectMeta.Namespace) assert.Equal(t, "node-agent-windows", ds.Spec.Template.ObjectMeta.Labels["name"]) assert.Equal(t, "node-agent", ds.Spec.Template.ObjectMeta.Labels["role"]) assert.Equal(t, "windows", string(ds.Spec.Template.Spec.OS.Name)) assert.Equal(t, &corev1api.Affinity{ NodeAffinity: &corev1api.NodeAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ NodeSelectorTerms: []corev1api.NodeSelectorTerm{ { MatchExpressions: []corev1api.NodeSelectorRequirement{ { Key: "kubernetes.io/os", Values: []string{"windows"}, Operator: corev1api.NodeSelectorOpIn, }, }, }, }, }, }, }, ds.Spec.Template.Spec.Affinity) assert.Equal(t, (*corev1api.PodSecurityContext)(nil), ds.Spec.Template.Spec.SecurityContext) assert.Equal(t, (*corev1api.SecurityContext)(nil), ds.Spec.Template.Spec.Containers[0].SecurityContext) } func TestDaemonSetWithPriorityClassName(t *testing.T) { testCases := []struct { name string priorityClassName string expectedValue string }{ { name: "with priority class name", priorityClassName: "high-priority", expectedValue: "high-priority", }, { name: "without priority class name", priorityClassName: "", expectedValue: "", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { // Create a daemonset with the priority class name option var opts []podTemplateOption if tc.priorityClassName != "" { opts = append(opts, WithPriorityClassName(tc.priorityClassName)) } daemonset := DaemonSet("velero", opts...) // Verify the priority class name is set correctly assert.Equal(t, tc.expectedValue, daemonset.Spec.Template.Spec.PriorityClassName) }) } } ================================================ FILE: pkg/install/deployment.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package install import ( "fmt" "strings" "time" appsv1api "k8s.io/api/apps/v1" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" "github.com/vmware-tanzu/velero/internal/velero" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/util/kube" ) type podTemplateOption func(*podTemplateConfig) type podTemplateConfig struct { image string envVars []corev1api.EnvVar restoreOnly bool annotations map[string]string labels map[string]string resources corev1api.ResourceRequirements withSecret bool defaultRepoMaintenanceFrequency time.Duration garbageCollectionFrequency time.Duration podVolumeOperationTimeout time.Duration plugins []string features []string defaultVolumesToFsBackup bool serviceAccountName string uploaderType string defaultSnapshotMoveData bool privilegedNodeAgent bool disableInformerCache bool scheduleSkipImmediately bool podResources kube.PodResources keepLatestMaintenanceJobs int backupRepoConfigMap string repoMaintenanceJobConfigMap string nodeAgentConfigMap string itemBlockWorkerCount int concurrentBackups int forWindows bool kubeletRootDir string nodeAgentDisableHostPath bool priorityClassName string } func WithImage(image string) podTemplateOption { return func(c *podTemplateConfig) { c.image = image } } func WithAnnotations(annotations map[string]string) podTemplateOption { return func(c *podTemplateConfig) { c.annotations = annotations } } func WithLabels(labels map[string]string) podTemplateOption { return func(c *podTemplateConfig) { c.labels = labels } } func WithEnvFromSecretKey(varName, secret, key string) podTemplateOption { return func(c *podTemplateConfig) { c.envVars = append(c.envVars, corev1api.EnvVar{ Name: varName, ValueFrom: &corev1api.EnvVarSource{ SecretKeyRef: &corev1api.SecretKeySelector{ LocalObjectReference: corev1api.LocalObjectReference{ Name: secret, }, Key: key, }, }, }) } } func WithSecret(secretPresent bool) podTemplateOption { return func(c *podTemplateConfig) { c.withSecret = secretPresent } } func WithRestoreOnly(b bool) podTemplateOption { return func(c *podTemplateConfig) { c.restoreOnly = b } } func WithResources(resources corev1api.ResourceRequirements) podTemplateOption { return func(c *podTemplateConfig) { c.resources = resources } } func WithDefaultRepoMaintenanceFrequency(val time.Duration) podTemplateOption { return func(c *podTemplateConfig) { c.defaultRepoMaintenanceFrequency = val } } func WithGarbageCollectionFrequency(val time.Duration) podTemplateOption { return func(c *podTemplateConfig) { c.garbageCollectionFrequency = val } } func WithPodVolumeOperationTimeout(val time.Duration) podTemplateOption { return func(c *podTemplateConfig) { c.podVolumeOperationTimeout = val } } func WithPlugins(plugins []string) podTemplateOption { return func(c *podTemplateConfig) { c.plugins = plugins } } func WithFeatures(features []string) podTemplateOption { return func(c *podTemplateConfig) { c.features = features } } func WithUploaderType(t string) podTemplateOption { return func(c *podTemplateConfig) { c.uploaderType = t } } func WithDefaultVolumesToFsBackup(b bool) podTemplateOption { return func(c *podTemplateConfig) { c.defaultVolumesToFsBackup = b } } func WithDefaultSnapshotMoveData(b bool) podTemplateOption { return func(c *podTemplateConfig) { c.defaultSnapshotMoveData = b } } func WithDisableInformerCache(b bool) podTemplateOption { return func(c *podTemplateConfig) { c.disableInformerCache = b } } func WithServiceAccountName(sa string) podTemplateOption { return func(c *podTemplateConfig) { c.serviceAccountName = sa } } func WithPrivilegedNodeAgent(b bool) podTemplateOption { return func(c *podTemplateConfig) { c.privilegedNodeAgent = b } } func WithNodeAgentConfigMap(nodeAgentConfigMap string) podTemplateOption { return func(c *podTemplateConfig) { c.nodeAgentConfigMap = nodeAgentConfigMap } } func WithScheduleSkipImmediately(b bool) podTemplateOption { return func(c *podTemplateConfig) { c.scheduleSkipImmediately = b } } func WithPodResources(podResources kube.PodResources) podTemplateOption { return func(c *podTemplateConfig) { c.podResources = podResources } } func WithKeepLatestMaintenanceJobs(keepLatestMaintenanceJobs int) podTemplateOption { return func(c *podTemplateConfig) { c.keepLatestMaintenanceJobs = keepLatestMaintenanceJobs } } func WithBackupRepoConfigMap(backupRepoConfigMap string) podTemplateOption { return func(c *podTemplateConfig) { c.backupRepoConfigMap = backupRepoConfigMap } } func WithRepoMaintenanceJobConfigMap(repoMaintenanceJobConfigMap string) podTemplateOption { return func(c *podTemplateConfig) { c.repoMaintenanceJobConfigMap = repoMaintenanceJobConfigMap } } func WithItemBlockWorkerCount(itemBlockWorkerCount int) podTemplateOption { return func(c *podTemplateConfig) { c.itemBlockWorkerCount = itemBlockWorkerCount } } func WithConcurrentBackups(concurrentBackups int) podTemplateOption { return func(c *podTemplateConfig) { c.concurrentBackups = concurrentBackups } } func WithPriorityClassName(priorityClassName string) podTemplateOption { return func(c *podTemplateConfig) { c.priorityClassName = priorityClassName } } func WithForWindows() podTemplateOption { return func(c *podTemplateConfig) { c.forWindows = true } } func WithKubeletRootDir(kubeletRootDir string) podTemplateOption { return func(c *podTemplateConfig) { c.kubeletRootDir = kubeletRootDir } } func WithNodeAgentDisableHostPath(disable bool) podTemplateOption { return func(c *podTemplateConfig) { c.nodeAgentDisableHostPath = disable } } func Deployment(namespace string, opts ...podTemplateOption) *appsv1api.Deployment { // TODO: Add support for server args c := &podTemplateConfig{ image: velero.DefaultVeleroImage(), } for _, opt := range opts { opt(c) } pullPolicy := corev1api.PullAlways imageParts := strings.Split(c.image, ":") if len(imageParts) == 2 && imageParts[1] != "latest" { pullPolicy = corev1api.PullIfNotPresent } args := []string{"server"} if len(c.features) > 0 { args = append(args, fmt.Sprintf("--features=%s", strings.Join(c.features, ","))) } if c.defaultVolumesToFsBackup { args = append(args, "--default-volumes-to-fs-backup=true") } if c.defaultSnapshotMoveData { args = append(args, "--default-snapshot-move-data=true") } if c.disableInformerCache { args = append(args, "--disable-informer-cache=true") } if c.scheduleSkipImmediately { args = append(args, "--schedule-skip-immediately=true") } if len(c.uploaderType) > 0 { args = append(args, fmt.Sprintf("--uploader-type=%s", c.uploaderType)) } if c.restoreOnly { args = append(args, "--restore-only") } if c.defaultRepoMaintenanceFrequency > 0 { args = append(args, fmt.Sprintf("--default-repo-maintain-frequency=%v", c.defaultRepoMaintenanceFrequency)) } if c.garbageCollectionFrequency > 0 { args = append(args, fmt.Sprintf("--garbage-collection-frequency=%v", c.garbageCollectionFrequency)) } if c.podVolumeOperationTimeout > 0 { args = append(args, fmt.Sprintf("--fs-backup-timeout=%v", c.podVolumeOperationTimeout)) } if c.keepLatestMaintenanceJobs > 0 { args = append(args, fmt.Sprintf("--keep-latest-maintenance-jobs=%d", c.keepLatestMaintenanceJobs)) } if len(c.podResources.CPULimit) > 0 { args = append(args, fmt.Sprintf("--maintenance-job-cpu-limit=%s", c.podResources.CPULimit)) } if len(c.podResources.CPURequest) > 0 { args = append(args, fmt.Sprintf("--maintenance-job-cpu-request=%s", c.podResources.CPURequest)) } if len(c.podResources.MemoryLimit) > 0 { args = append(args, fmt.Sprintf("--maintenance-job-mem-limit=%s", c.podResources.MemoryLimit)) } if len(c.podResources.MemoryRequest) > 0 { args = append(args, fmt.Sprintf("--maintenance-job-mem-request=%s", c.podResources.MemoryRequest)) } if len(c.backupRepoConfigMap) > 0 { args = append(args, fmt.Sprintf("--backup-repository-configmap=%s", c.backupRepoConfigMap)) } if len(c.repoMaintenanceJobConfigMap) > 0 { args = append(args, fmt.Sprintf("--repo-maintenance-job-configmap=%s", c.repoMaintenanceJobConfigMap)) } if c.itemBlockWorkerCount > 0 { args = append(args, fmt.Sprintf("--item-block-worker-count=%d", c.itemBlockWorkerCount)) } if c.concurrentBackups > 0 { args = append(args, fmt.Sprintf("--concurrent-backups=%d", c.concurrentBackups)) } deployment := &appsv1api.Deployment{ ObjectMeta: objectMeta(namespace, "velero"), TypeMeta: metav1.TypeMeta{ Kind: "Deployment", APIVersion: appsv1api.SchemeGroupVersion.String(), }, Spec: appsv1api.DeploymentSpec{ Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"deploy": "velero"}}, Template: corev1api.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: podLabels(c.labels, map[string]string{"deploy": "velero"}), Annotations: podAnnotations(c.annotations), }, Spec: corev1api.PodSpec{ RestartPolicy: corev1api.RestartPolicyAlways, ServiceAccountName: c.serviceAccountName, OS: &corev1api.PodOS{ Name: "linux", }, Affinity: &corev1api.Affinity{ NodeAffinity: &corev1api.NodeAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ NodeSelectorTerms: []corev1api.NodeSelectorTerm{ { MatchExpressions: []corev1api.NodeSelectorRequirement{ { Key: "kubernetes.io/os", Values: []string{"windows"}, Operator: corev1api.NodeSelectorOpNotIn, }, }, }, }, }, }, }, Containers: []corev1api.Container{ { Name: "velero", Image: c.image, Ports: containerPorts(), ImagePullPolicy: pullPolicy, Command: []string{ "/velero", }, Args: args, VolumeMounts: []corev1api.VolumeMount{ { Name: "plugins", MountPath: "/plugins", }, { Name: "scratch", MountPath: "/scratch", }, }, Env: []corev1api.EnvVar{ { Name: "VELERO_SCRATCH_DIR", Value: "/scratch", }, { Name: "VELERO_NAMESPACE", ValueFrom: &corev1api.EnvVarSource{ FieldRef: &corev1api.ObjectFieldSelector{ FieldPath: "metadata.namespace", }, }, }, { Name: "LD_LIBRARY_PATH", Value: "/plugins", }, }, Resources: c.resources, }, }, Volumes: []corev1api.Volume{ { Name: "plugins", VolumeSource: corev1api.VolumeSource{ EmptyDir: &corev1api.EmptyDirVolumeSource{}, }, }, { Name: "scratch", VolumeSource: corev1api.VolumeSource{ EmptyDir: new(corev1api.EmptyDirVolumeSource), }, }, }, PriorityClassName: c.priorityClassName, }, }, }, } if c.withSecret { deployment.Spec.Template.Spec.Volumes = append( deployment.Spec.Template.Spec.Volumes, corev1api.Volume{ Name: "cloud-credentials", VolumeSource: corev1api.VolumeSource{ Secret: &corev1api.SecretVolumeSource{ // read-only for Owner, Group, Public DefaultMode: ptr.To(int32(0444)), SecretName: "cloud-credentials", }, }, }, ) deployment.Spec.Template.Spec.Containers[0].VolumeMounts = append( deployment.Spec.Template.Spec.Containers[0].VolumeMounts, corev1api.VolumeMount{ Name: "cloud-credentials", MountPath: "/credentials", }, ) deployment.Spec.Template.Spec.Containers[0].Env = append(deployment.Spec.Template.Spec.Containers[0].Env, []corev1api.EnvVar{ { Name: "GOOGLE_APPLICATION_CREDENTIALS", Value: "/credentials/cloud", }, { Name: "AWS_SHARED_CREDENTIALS_FILE", Value: "/credentials/cloud", }, { Name: "AZURE_CREDENTIALS_FILE", Value: "/credentials/cloud", }, { Name: "ALIBABA_CLOUD_CREDENTIALS_FILE", Value: "/credentials/cloud", }, }...) } deployment.Spec.Template.Spec.Containers[0].Env = append(deployment.Spec.Template.Spec.Containers[0].Env, c.envVars...) if len(c.plugins) > 0 { for _, image := range c.plugins { container := *builder.ForPluginContainer(image, pullPolicy).Result() deployment.Spec.Template.Spec.InitContainers = append(deployment.Spec.Template.Spec.InitContainers, container) } } return deployment } ================================================ FILE: pkg/install/deployment_test.go ================================================ /* Copyright 2019, 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package install import ( "testing" "time" "github.com/stretchr/testify/assert" corev1api "k8s.io/api/core/v1" "github.com/vmware-tanzu/velero/pkg/util/kube" ) func TestDeployment(t *testing.T) { deploy := Deployment("velero") assert.Equal(t, "velero", deploy.ObjectMeta.Namespace) deploy = Deployment("velero", WithRestoreOnly(true)) assert.Equal(t, "--restore-only", deploy.Spec.Template.Spec.Containers[0].Args[1]) deploy = Deployment("velero", WithEnvFromSecretKey("my-var", "my-secret", "my-key")) envSecret := deploy.Spec.Template.Spec.Containers[0].Env[3] assert.Equal(t, "my-var", envSecret.Name) assert.Equal(t, "my-secret", envSecret.ValueFrom.SecretKeyRef.LocalObjectReference.Name) assert.Equal(t, "my-key", envSecret.ValueFrom.SecretKeyRef.Key) deploy = Deployment("velero", WithImage("velero/velero:v0.11")) assert.Equal(t, "velero/velero:v0.11", deploy.Spec.Template.Spec.Containers[0].Image) assert.Equal(t, corev1api.PullIfNotPresent, deploy.Spec.Template.Spec.Containers[0].ImagePullPolicy) deploy = Deployment("velero", WithSecret(true)) assert.Len(t, deploy.Spec.Template.Spec.Containers[0].Env, 7) assert.Len(t, deploy.Spec.Template.Spec.Volumes, 3) deploy = Deployment("velero", WithDefaultRepoMaintenanceFrequency(24*time.Hour)) assert.Len(t, deploy.Spec.Template.Spec.Containers[0].Args, 2) assert.Equal(t, "--default-repo-maintain-frequency=24h0m0s", deploy.Spec.Template.Spec.Containers[0].Args[1]) deploy = Deployment("velero", WithGarbageCollectionFrequency(24*time.Hour)) assert.Len(t, deploy.Spec.Template.Spec.Containers[0].Args, 2) assert.Equal(t, "--garbage-collection-frequency=24h0m0s", deploy.Spec.Template.Spec.Containers[0].Args[1]) deploy = Deployment("velero", WithFeatures([]string{"EnableCSI", "foo", "bar", "baz"})) assert.Len(t, deploy.Spec.Template.Spec.Containers[0].Args, 2) assert.Equal(t, "--features=EnableCSI,foo,bar,baz", deploy.Spec.Template.Spec.Containers[0].Args[1]) deploy = Deployment("velero", WithUploaderType("kopia")) assert.Len(t, deploy.Spec.Template.Spec.Containers[0].Args, 2) assert.Equal(t, "--uploader-type=kopia", deploy.Spec.Template.Spec.Containers[0].Args[1]) deploy = Deployment("velero", WithServiceAccountName("test-sa")) assert.Equal(t, "test-sa", deploy.Spec.Template.Spec.ServiceAccountName) deploy = Deployment("velero", WithDisableInformerCache(true)) assert.Len(t, deploy.Spec.Template.Spec.Containers[0].Args, 2) assert.Equal(t, "--disable-informer-cache=true", deploy.Spec.Template.Spec.Containers[0].Args[1]) deploy = Deployment("velero", WithKeepLatestMaintenanceJobs(3)) assert.Len(t, deploy.Spec.Template.Spec.Containers[0].Args, 2) assert.Equal(t, "--keep-latest-maintenance-jobs=3", deploy.Spec.Template.Spec.Containers[0].Args[1]) deploy = Deployment( "velero", WithPodResources( kube.PodResources{ CPURequest: "100m", MemoryRequest: "256Mi", CPULimit: "200m", MemoryLimit: "512Mi", }, ), ) assert.Len(t, deploy.Spec.Template.Spec.Containers[0].Args, 5) assert.Equal(t, "--maintenance-job-cpu-limit=200m", deploy.Spec.Template.Spec.Containers[0].Args[1]) assert.Equal(t, "--maintenance-job-cpu-request=100m", deploy.Spec.Template.Spec.Containers[0].Args[2]) assert.Equal(t, "--maintenance-job-mem-limit=512Mi", deploy.Spec.Template.Spec.Containers[0].Args[3]) assert.Equal(t, "--maintenance-job-mem-request=256Mi", deploy.Spec.Template.Spec.Containers[0].Args[4]) deploy = Deployment("velero", WithBackupRepoConfigMap("test-backup-repo-config")) assert.Len(t, deploy.Spec.Template.Spec.Containers[0].Args, 2) assert.Equal(t, "--backup-repository-configmap=test-backup-repo-config", deploy.Spec.Template.Spec.Containers[0].Args[1]) deploy = Deployment("velero", WithRepoMaintenanceJobConfigMap("test-repo-maintenance-config")) assert.Len(t, deploy.Spec.Template.Spec.Containers[0].Args, 2) assert.Equal(t, "--repo-maintenance-job-configmap=test-repo-maintenance-config", deploy.Spec.Template.Spec.Containers[0].Args[1]) assert.Equal(t, &corev1api.Affinity{ NodeAffinity: &corev1api.NodeAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ NodeSelectorTerms: []corev1api.NodeSelectorTerm{ { MatchExpressions: []corev1api.NodeSelectorRequirement{ { Key: "kubernetes.io/os", Values: []string{"windows"}, Operator: corev1api.NodeSelectorOpNotIn, }, }, }, }, }, }, }, deploy.Spec.Template.Spec.Affinity) } func TestDeploymentWithPriorityClassName(t *testing.T) { testCases := []struct { name string priorityClassName string expectedValue string }{ { name: "with priority class name", priorityClassName: "high-priority", expectedValue: "high-priority", }, { name: "without priority class name", priorityClassName: "", expectedValue: "", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { // Create a deployment with the priority class name option var opts []podTemplateOption if tc.priorityClassName != "" { opts = append(opts, WithPriorityClassName(tc.priorityClassName)) } deployment := Deployment("velero", opts...) // Verify the priority class name is set correctly assert.Equal(t, tc.expectedValue, deployment.Spec.Template.Spec.PriorityClassName) }) } } ================================================ FILE: pkg/install/doc.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ // Package install provides public functions for easily creating and installing // resources necessary for Velero to operate. Some default settings are assumed with these functions. package install ================================================ FILE: pkg/install/import_test.go ================================================ package install import ( "os/exec" "path/filepath" "regexp" "runtime" "strings" "testing" "github.com/stretchr/testify/require" ) // test that this package do not import cloud provider // Prevent https://github.com/vmware-tanzu/velero/issues/8207 and https://github.com/vmware-tanzu/velero/issues/8157 func TestPkgImportNoCloudProvider(t *testing.T) { _, filename, _, ok := runtime.Caller(0) if !ok { t.Fatalf("No caller information") } t.Logf("Current test file path: %s", filename) t.Logf("Current test directory: %s", filepath.Dir(filename)) // should be this package name // go list -f {{.Deps}} ./ cmd := exec.CommandContext( t.Context(), "go", "list", "-f", "{{.Deps}}", ".", ) // set cmd.Dir to this package even if executed from different dir cmd.Dir = filepath.Dir(filename) output, err := cmd.Output() require.NoError(t, err) // split dep by line, replace space with newline deps := strings.ReplaceAll(string(output), " ", "\n") require.NotEmpty(t, deps) // ignore k8s.io k8sio, err := regexp.Compile("^k8s.io") require.NoError(t, err) cloudProvider, err := regexp.Compile("aws|cloud.google.com|azure") require.NoError(t, err) cloudProviderDeps := []string{} for _, dep := range strings.Split(deps, "\n") { if !k8sio.MatchString(dep) { if cloudProvider.MatchString(dep) { cloudProviderDeps = append(cloudProviderDeps, dep) } } } require.Empty(t, cloudProviderDeps) } ================================================ FILE: pkg/install/install.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package install import ( "context" "fmt" "io" "strings" "time" "github.com/pkg/errors" appsv1api "k8s.io/api/apps/v1" corev1api "k8s.io/api/core/v1" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/wait" kbclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/util/kube" ) // kindToResource translates a Kind (mixed case, singular) to a Resource (lowercase, plural) string. // This is to accommodate the dynamic client's need for an APIResource, as the Unstructured objects do not have easy helpers for this information. var kindToResource = map[string]string{ "CustomResourceDefinition": "customresourcedefinitions", "Namespace": "namespaces", "ClusterRoleBinding": "clusterrolebindings", "ServiceAccount": "serviceaccounts", "Deployment": "deployments", "DaemonSet": "daemonsets", "Secret": "secrets", "ConfigMap": "configmaps", "BackupStorageLocation": "backupstoragelocations", "VolumeSnapshotLocation": "volumesnapshotlocations", } // ResourceGroup represents a collection of Kubernetes objects with a common ready condition type ResourceGroup struct { CRDResources []*unstructured.Unstructured OtherResources []*unstructured.Unstructured } // crdV1Beta1ReadinessFn returns a function that can be used for polling to check // if the provided unstructured v1beta1 CRDs are ready for use in the cluster. func crdV1Beta1ReadinessFn(kbClient kbclient.Client, unstructuredCrds []*unstructured.Unstructured) func(context.Context) (bool, error) { // Track all the CRDs that have been found and in ready state. // len should be equal to len(unstructuredCrds) in the happy path. return func(ctx context.Context) (bool, error) { foundCRDs := make([]*apiextv1beta1.CustomResourceDefinition, 0) for _, unstructuredCrd := range unstructuredCrds { crd := &apiextv1beta1.CustomResourceDefinition{} key := kbclient.ObjectKey{Name: unstructuredCrd.GetName()} err := kbClient.Get(ctx, key, crd) if apierrors.IsNotFound(err) { return false, nil } else if err != nil { return false, errors.Wrapf(err, "error waiting for %s to be ready", crd.GetName()) } foundCRDs = append(foundCRDs, crd) } if len(foundCRDs) != len(unstructuredCrds) { return false, nil } for _, crd := range foundCRDs { ready := kube.IsV1Beta1CRDReady(crd) if !ready { return false, nil } } return true, nil } } // crdV1ReadinessFn returns a function that can be used for polling to check // if the provided unstructured v1 CRDs are ready for use in the cluster. func crdV1ReadinessFn(kbClient kbclient.Client, unstructuredCrds []*unstructured.Unstructured) func(context.Context) (bool, error) { return func(ctx context.Context) (bool, error) { foundCRDs := make([]*apiextv1.CustomResourceDefinition, 0) for _, unstructuredCrd := range unstructuredCrds { crd := &apiextv1.CustomResourceDefinition{} key := kbclient.ObjectKey{Name: unstructuredCrd.GetName()} err := kbClient.Get(ctx, key, crd) if apierrors.IsNotFound(err) { return false, nil } else if err != nil { return false, errors.Wrapf(err, "error waiting for %s to be ready", crd.GetName()) } foundCRDs = append(foundCRDs, crd) } if len(foundCRDs) != len(unstructuredCrds) { return false, nil } for _, crd := range foundCRDs { ready := kube.IsV1CRDReady(crd) if !ready { return false, nil } } return true, nil } } // crdsAreReady polls the API server to see if the Velero CRDs are ready to create objects. func crdsAreReady(kbClient kbclient.Client, crds []*unstructured.Unstructured) (bool, error) { if len(crds) == 0 { // no CRDs to check so return return true, nil } // We assume that all Velero CRDs have the same GVK so we can use the GVK of the // first CRD to determine whether to use the v1beta1 or v1 API during polling. gvk := crds[0].GroupVersionKind() var crdReadinessFn func(context.Context) (bool, error) if gvk.Version == "v1beta1" { crdReadinessFn = crdV1Beta1ReadinessFn(kbClient, crds) } else if gvk.Version == "v1" { crdReadinessFn = crdV1ReadinessFn(kbClient, crds) } else { return false, fmt.Errorf("unsupported CRD version %q", gvk.Version) } err := wait.PollUntilContextTimeout(context.Background(), time.Second, time.Minute, true, crdReadinessFn) if err != nil { return false, errors.Wrap(err, "Error polling for CRDs") } return true, nil } func isAvailable(c appsv1api.DeploymentCondition) bool { // Make sure that the deployment has been available for at least 10 seconds. // This is because the deployment can show as Ready momentarily before the pods fall into a CrashLoopBackOff. // See podutils.IsPodAvailable upstream for similar logic with pods if c.Type == appsv1api.DeploymentAvailable && c.Status == corev1api.ConditionTrue { if !c.LastTransitionTime.IsZero() && c.LastTransitionTime.Add(10*time.Second).Before(time.Now()) { return true } } return false } // DeploymentIsReady will poll the Kubernetes API server to see if the velero deployment is ready to service user requests. func DeploymentIsReady(factory client.DynamicFactory, namespace string) (bool, error) { gvk := schema.FromAPIVersionAndKind(appsv1api.SchemeGroupVersion.String(), "Deployment") apiResource := metav1.APIResource{ Name: "deployments", Namespaced: true, } c, err := factory.ClientForGroupVersionResource(gvk.GroupVersion(), apiResource, namespace) if err != nil { return false, errors.Wrapf(err, "Error creating client for deployment polling") } // declare this variable out of scope so we can return it var isReady bool var readyObservations int32 err = wait.PollUntilContextTimeout(context.Background(), time.Second, 3*time.Minute, true, func(ctx context.Context) (bool, error) { unstructuredDeployment, err := c.Get("velero", metav1.GetOptions{}) if apierrors.IsNotFound(err) { return false, nil } else if err != nil { return false, errors.Wrap(err, "error waiting for deployment to be ready") } deploy := new(appsv1api.Deployment) if err := runtime.DefaultUnstructuredConverter.FromUnstructured(unstructuredDeployment.Object, deploy); err != nil { return false, errors.Wrap(err, "error converting deployment from unstructured") } for _, cond := range deploy.Status.Conditions { if isAvailable(cond) { readyObservations++ } } // Make sure we query the deployment enough times to see the state change, provided there is one. if readyObservations > 4 { isReady = true return true, nil } return false, nil }) return isReady, err } // NodeAgentIsReady will poll the Kubernetes API server to ensure the node-agent daemonset is ready, i.e. that // pods are scheduled and available on all of the desired nodes. func NodeAgentIsReady(factory client.DynamicFactory, namespace string) (bool, error) { return daemonSetIsReady(factory, namespace, "node-agent") } // NodeAgentWindowsIsReady will poll the Kubernetes API server to ensure the node-agent-windows daemonset is ready, i.e. that // pods are scheduled and available on all of the desired nodes. func NodeAgentWindowsIsReady(factory client.DynamicFactory, namespace string) (bool, error) { return daemonSetIsReady(factory, namespace, "node-agent-windows") } func daemonSetIsReady(factory client.DynamicFactory, namespace string, name string) (bool, error) { gvk := schema.FromAPIVersionAndKind(appsv1api.SchemeGroupVersion.String(), "DaemonSet") apiResource := metav1.APIResource{ Name: "daemonsets", Namespaced: true, } c, err := factory.ClientForGroupVersionResource(gvk.GroupVersion(), apiResource, namespace) if err != nil { return false, errors.Wrapf(err, "Error creating client for daemonset polling") } // declare this variable out of scope so we can return it var isReady bool var readyObservations int32 err = wait.PollUntilContextTimeout(context.Background(), time.Second, time.Minute, true, func(ctx context.Context) (bool, error) { unstructuredDaemonSet, err := c.Get(name, metav1.GetOptions{}) if apierrors.IsNotFound(err) { return false, nil } else if err != nil { return false, errors.Wrap(err, "error waiting for daemonset to be ready") } daemonSet := new(appsv1api.DaemonSet) if err := runtime.DefaultUnstructuredConverter.FromUnstructured(unstructuredDaemonSet.Object, daemonSet); err != nil { return false, errors.Wrap(err, "error converting daemonset from unstructured") } if daemonSet.Status.NumberAvailable == daemonSet.Status.DesiredNumberScheduled { readyObservations++ } // Wait for 5 observations of the daemonset being "ready" to be consistent with our check for // the deployment being ready. if readyObservations > 4 { isReady = true return true, nil } return false, nil }) return isReady, err } // GroupResources groups resources based on whether the resources are CustomResourceDefinitions or other types of Kubernetes objects // This is useful to wait for readiness before creating CRD objects func GroupResources(resources *unstructured.UnstructuredList) *ResourceGroup { rg := new(ResourceGroup) for i, r := range resources.Items { if r.GetKind() == "CustomResourceDefinition" { rg.CRDResources = append(rg.CRDResources, &resources.Items[i]) continue } rg.OtherResources = append(rg.OtherResources, &resources.Items[i]) } return rg } // createOrApplyResource attempts to create or apply a resource in the cluster. // If apply is true, it uses server-side apply to update existing resources. // If apply is false and the resource already exists in the cluster, it's merely logged. func createOrApplyResource(r *unstructured.Unstructured, factory client.DynamicFactory, w io.Writer, apply bool) error { id := fmt.Sprintf("%s/%s", r.GetKind(), r.GetName()) // Helper to reduce boilerplate message about the same object log := func(f string) { fmt.Fprintf(w, "%s: %s\n", id, f) } c, err := CreateClient(r, factory, w) if err != nil { return err } if apply { log("attempting to apply resource") // Set field manager for server-side apply and force to override conflicts applyOpts := metav1.ApplyOptions{ FieldManager: "velero-cli", Force: true, } if _, err := c.Apply(r.GetName(), r, applyOpts); err != nil { return errors.Wrapf(err, "Error applying resource %s", id) } log("applied") } else { log("attempting to create resource") if _, err := c.Create(r); apierrors.IsAlreadyExists(err) { log("already exists, proceeding") } else if err != nil { return errors.Wrapf(err, "Error creating resource %s", id) } else { log("created") } } return nil } // CreateClient creates a client for an unstructured resource func CreateClient(r *unstructured.Unstructured, factory client.DynamicFactory, w io.Writer) (client.Dynamic, error) { id := fmt.Sprintf("%s/%s", r.GetKind(), r.GetName()) // Helper to reduce boilerplate message about the same object log := func(f string, a ...any) { format := strings.Join([]string{id, ": ", f, "\n"}, "") fmt.Fprintf(w, format, a...) } log("attempting to create resource client") gvk := schema.FromAPIVersionAndKind(r.GetAPIVersion(), r.GetKind()) apiResource := metav1.APIResource{ Name: kindToResource[r.GetKind()], Namespaced: (r.GetNamespace() != ""), } c, err := factory.ClientForGroupVersionResource(gvk.GroupVersion(), apiResource, r.GetNamespace()) if err != nil { return nil, errors.Wrapf(err, "Error creating client for resource %s", id) } return c, nil } // Install creates resources on the Kubernetes cluster. // An unstructured list of resources is sent, one at a time, to the server. These are assumed to be in the preferred order already. // Resources will be sorted into CustomResourceDefinitions and any other resource type, and the function will wait up to 1 minute // for CRDs to be ready before proceeding. // If apply is true, it uses server-side apply to update existing resources. // An io.Writer can be used to output to a log or the console. func Install(dynamicFactory client.DynamicFactory, kbClient kbclient.Client, resources *unstructured.UnstructuredList, w io.Writer, apply bool) error { rg := GroupResources(resources) //Install CRDs first for _, r := range rg.CRDResources { if err := createOrApplyResource(r, dynamicFactory, w, apply); err != nil { return err } } // Wait for CRDs to be ready before proceeding fmt.Fprint(w, "Waiting for resources to be ready in cluster...\n") _, err := crdsAreReady(kbClient, rg.CRDResources) if wait.Interrupted(err) { return errors.Errorf("timeout reached, CRDs not ready") } else if err != nil { return err } // Install all other resources for _, r := range rg.OtherResources { if err = createOrApplyResource(r, dynamicFactory, w, apply); err != nil { return err } } return nil } ================================================ FILE: pkg/install/install_test.go ================================================ package install import ( "bytes" "errors" "os" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" appsv1api "k8s.io/api/apps/v1" corev1api "k8s.io/api/core/v1" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "sigs.k8s.io/controller-runtime/pkg/client/fake" v1crds "github.com/vmware-tanzu/velero/config/crd/v1/crds" "github.com/vmware-tanzu/velero/pkg/test" ) func TestInstall(t *testing.T) { dc := &test.FakeDynamicClient{} dc.On("Create", mock.Anything).Return(&unstructured.Unstructured{}, nil) factory := &test.FakeDynamicFactory{} factory.On("ClientForGroupVersionResource", mock.Anything, mock.Anything, mock.Anything).Return(dc, nil) c := fake.NewClientBuilder().WithObjects( &apiextv1.CustomResourceDefinition{ ObjectMeta: metav1.ObjectMeta{ Name: "backuprepositories.velero.io", }, Status: apiextv1.CustomResourceDefinitionStatus{ Conditions: []apiextv1.CustomResourceDefinitionCondition{ { Type: apiextv1.Established, Status: apiextv1.ConditionTrue, }, { Type: apiextv1.NamesAccepted, Status: apiextv1.ConditionTrue, }, }, }, }, ).Build() resources := &unstructured.UnstructuredList{} require.NoError(t, appendUnstructured(resources, v1crds.CRDs[0])) require.NoError(t, appendUnstructured(resources, Namespace("velero"))) assert.NoError(t, Install(factory, c, resources, os.Stdout, false)) } func Test_crdsAreReady(t *testing.T) { c := fake.NewClientBuilder().WithObjects( &apiextv1beta1.CustomResourceDefinition{ ObjectMeta: metav1.ObjectMeta{ Name: "backuprepositories.velero.io", }, Status: apiextv1beta1.CustomResourceDefinitionStatus{ Conditions: []apiextv1beta1.CustomResourceDefinitionCondition{ { Type: apiextv1beta1.Established, Status: apiextv1beta1.ConditionTrue, }, { Type: apiextv1beta1.NamesAccepted, Status: apiextv1beta1.ConditionTrue, }, }, }, }, ).Build() crd := &apiextv1beta1.CustomResourceDefinition{ TypeMeta: metav1.TypeMeta{ Kind: "CustomResourceDefinition", APIVersion: "v1beta1", }, ObjectMeta: metav1.ObjectMeta{ Name: "backuprepositories.velero.io", }, } obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(crd) require.NoError(t, err) crds := []*unstructured.Unstructured{ { Object: obj, }, } ready, err := crdsAreReady(c, crds) require.NoError(t, err) assert.True(t, ready) } func TestDeploymentIsReady(t *testing.T) { deployment := &appsv1api.Deployment{ Status: appsv1api.DeploymentStatus{ Conditions: []appsv1api.DeploymentCondition{ { Type: appsv1api.DeploymentAvailable, Status: corev1api.ConditionTrue, LastTransitionTime: metav1.NewTime(time.Now().Add(-15 * time.Second)), }, }, }, } obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(deployment) require.NoError(t, err) dc := &test.FakeDynamicClient{} dc.On("Get", mock.Anything, mock.Anything).Return(&unstructured.Unstructured{Object: obj}, nil) factory := &test.FakeDynamicFactory{} factory.On("ClientForGroupVersionResource", mock.Anything, mock.Anything, mock.Anything).Return(dc, nil) ready, err := DeploymentIsReady(factory, "velero") require.NoError(t, err) assert.True(t, ready) } func TestNodeAgentIsReady(t *testing.T) { daemonset := &appsv1api.DaemonSet{ Status: appsv1api.DaemonSetStatus{ NumberAvailable: 1, DesiredNumberScheduled: 1, }, } obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(daemonset) require.NoError(t, err) dc := &test.FakeDynamicClient{} dc.On("Get", mock.Anything, mock.Anything).Return(&unstructured.Unstructured{Object: obj}, nil) factory := &test.FakeDynamicFactory{} factory.On("ClientForGroupVersionResource", mock.Anything, mock.Anything, mock.Anything).Return(dc, nil) ready, err := NodeAgentIsReady(factory, "velero") require.NoError(t, err) assert.True(t, ready) } func TestNodeAgentWindowsIsReady(t *testing.T) { daemonset := &appsv1api.DaemonSet{ Status: appsv1api.DaemonSetStatus{ NumberAvailable: 0, DesiredNumberScheduled: 0, }, } obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(daemonset) require.NoError(t, err) dc := &test.FakeDynamicClient{} dc.On("Get", mock.Anything, mock.Anything).Return(&unstructured.Unstructured{Object: obj}, nil) factory := &test.FakeDynamicFactory{} factory.On("ClientForGroupVersionResource", mock.Anything, mock.Anything, mock.Anything).Return(dc, nil) ready, err := NodeAgentWindowsIsReady(factory, "velero") require.NoError(t, err) assert.True(t, ready) } func TestCreateOrApplyResourceError(t *testing.T) { r := &unstructured.Unstructured{ Object: map[string]any{ "apiVersion": "v1", "kind": "ConfigMap", "metadata": map[string]any{ "name": "test-configmap", "namespace": "velero", }, }, } dc := &test.FakeDynamicClient{} expectedErr := errors.New("create error") dc.On("Create", mock.Anything).Return(&unstructured.Unstructured{}, expectedErr) factory := &test.FakeDynamicFactory{} factory.On("ClientForGroupVersionResource", mock.Anything, mock.Anything, mock.Anything).Return(dc, nil) var buf bytes.Buffer err := createOrApplyResource(r, factory, &buf, false) require.Error(t, err) require.Contains(t, err.Error(), expectedErr.Error()) } func TestCreateOrApplyResourceAlreadyExists(t *testing.T) { r := &unstructured.Unstructured{ Object: map[string]any{ "apiVersion": "v1", "kind": "ConfigMap", "metadata": map[string]any{ "name": "test-configmap", "namespace": "velero", }, }, } dc := &test.FakeDynamicClient{} alreadyExistsErr := apierrors.NewAlreadyExists(schema.GroupResource{Resource: "configmaps"}, "test-configmap") // We need to return a non-nil unstructured object even though it's not used dc.On("Create", mock.Anything).Return(&unstructured.Unstructured{}, alreadyExistsErr) factory := &test.FakeDynamicFactory{} factory.On("ClientForGroupVersionResource", mock.Anything, mock.Anything, mock.Anything).Return(dc, nil) var buf bytes.Buffer err := createOrApplyResource(r, factory, &buf, false) require.NoError(t, err) } func TestCreateOrApplyResourceClientError(t *testing.T) { r := &unstructured.Unstructured{ Object: map[string]any{ "apiVersion": "v1", "kind": "ConfigMap", "metadata": map[string]any{ "name": "test-configmap", "namespace": "velero", }, }, } factory := &test.FakeDynamicFactory{} expectedErr := errors.New("client creation error") // Return error from ClientForGroupVersionResource factory.On("ClientForGroupVersionResource", mock.Anything, mock.Anything, mock.Anything).Return(&test.FakeDynamicClient{}, expectedErr) var buf bytes.Buffer err := createOrApplyResource(r, factory, &buf, false) require.Error(t, err) require.Contains(t, err.Error(), expectedErr.Error()) } func TestCreateOrApplyResourceApplyError(t *testing.T) { r := &unstructured.Unstructured{ Object: map[string]any{ "apiVersion": "v1", "kind": "ConfigMap", "metadata": map[string]any{ "name": "test-configmap", "namespace": "velero", }, }, } dc := &test.FakeDynamicClient{} expectedErr := errors.New("apply error") // Mock Apply to return an error dc.On("Apply", mock.Anything, mock.Anything, mock.Anything).Return(&unstructured.Unstructured{}, expectedErr) factory := &test.FakeDynamicFactory{} factory.On("ClientForGroupVersionResource", mock.Anything, mock.Anything, mock.Anything).Return(dc, nil) var buf bytes.Buffer err := createOrApplyResource(r, factory, &buf, true) // true for apply flag to use Apply require.Error(t, err) require.Contains(t, err.Error(), expectedErr.Error()) } func TestInstallErrorAfterCreateClient(t *testing.T) { // Create a test non-CRD resource nonCRDResource := &unstructured.Unstructured{ Object: map[string]any{ "apiVersion": "v1", "kind": "ConfigMap", "metadata": map[string]any{ "name": "test-configmap", }, }, } resources := &unstructured.UnstructuredList{ Items: []unstructured.Unstructured{*nonCRDResource}, } // Mock the factory to return a client that will succeed on ClientForGroupVersionResource // but fail on Create dc := &test.FakeDynamicClient{} expectedErr := errors.New("create error after successful client creation") dc.On("Create", mock.Anything).Return(&unstructured.Unstructured{}, expectedErr) factory := &test.FakeDynamicFactory{} factory.On("ClientForGroupVersionResource", mock.Anything, mock.Anything, mock.Anything).Return(dc, nil) c := fake.NewClientBuilder().Build() var buf bytes.Buffer err := Install(factory, c, resources, &buf, false) require.Error(t, err) require.Contains(t, err.Error(), expectedErr.Error()) } func TestInstallErrorOnCRDResource(t *testing.T) { crdResource := &unstructured.Unstructured{ Object: map[string]any{ "apiVersion": "apiextensions.k8s.io/v1", "kind": "CustomResourceDefinition", "metadata": map[string]any{ "name": "test-crd", }, }, } resources := &unstructured.UnstructuredList{ Items: []unstructured.Unstructured{*crdResource}, } dc := &test.FakeDynamicClient{} expectedErr := errors.New("error creating CRD resource") // We need to return a non-nil unstructured object even though it's not used dc.On("Create", mock.Anything).Return(&unstructured.Unstructured{}, expectedErr) factory := &test.FakeDynamicFactory{} factory.On("ClientForGroupVersionResource", mock.Anything, mock.Anything, mock.Anything).Return(dc, nil) c := fake.NewClientBuilder().Build() var buf bytes.Buffer err := Install(factory, c, resources, &buf, false) require.Error(t, err) require.Contains(t, err.Error(), expectedErr.Error()) } func TestInstallWithApplyFlag(t *testing.T) { // Create a test resource testResource := &unstructured.Unstructured{ Object: map[string]any{ "apiVersion": "v1", "kind": "ConfigMap", "metadata": map[string]any{ "name": "test-configmap", "namespace": "velero", }, "data": map[string]any{ "key1": "value1", }, }, } resources := &unstructured.UnstructuredList{ Items: []unstructured.Unstructured{*testResource}, } // Test case 1: Without apply flag (create) { dc := &test.FakeDynamicClient{} // Expect Create to be called dc.On("Create", mock.Anything).Return(testResource, nil) // Apply should not be called factory := &test.FakeDynamicFactory{} factory.On("ClientForGroupVersionResource", mock.Anything, mock.Anything, mock.Anything).Return(dc, nil) c := fake.NewClientBuilder().Build() err := Install(factory, c, resources, os.Stdout, false) require.NoError(t, err) // Verify that Create was called and Apply was not dc.AssertCalled(t, "Create", mock.Anything) dc.AssertNotCalled(t, "Apply", mock.Anything, mock.Anything, mock.Anything) } // Test case 2: With apply flag { dc := &test.FakeDynamicClient{} // Create should not be called // Expect Apply to be called dc.On("Apply", mock.Anything, mock.Anything, mock.Anything).Return(testResource, nil) factory := &test.FakeDynamicFactory{} factory.On("ClientForGroupVersionResource", mock.Anything, mock.Anything, mock.Anything).Return(dc, nil) c := fake.NewClientBuilder().Build() err := Install(factory, c, resources, os.Stdout, true) require.NoError(t, err) // Verify that Apply was called and Create was not dc.AssertCalled(t, "Apply", mock.Anything, mock.Anything, mock.Anything) dc.AssertNotCalled(t, "Create", mock.Anything) } } ================================================ FILE: pkg/install/resources.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package install import ( "fmt" "time" corev1api "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" v1crds "github.com/vmware-tanzu/velero/config/crd/v1/crds" v2alpha1crds "github.com/vmware-tanzu/velero/config/crd/v2alpha1/crds" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/util/kube" ) const ( defaultServiceAccountName = "velero" podSecurityLevel = "privileged" podSecurityVersion = "latest" ) var ( // default values for Velero server pod resource request/limit DefaultVeleroPodCPURequest = "500m" DefaultVeleroPodMemRequest = "128Mi" DefaultVeleroPodCPULimit = "1000m" DefaultVeleroPodMemLimit = "512Mi" // default values for node-agent pod resource request/limit, // "0" means no request/limit is set, so as to make the QoS as BestEffort DefaultNodeAgentPodCPURequest = "0" DefaultNodeAgentPodMemRequest = "0" DefaultNodeAgentPodCPULimit = "0" DefaultNodeAgentPodMemLimit = "0" DefaultVeleroNamespace = "velero" DefaultKubeletRootDir = "/var/lib/kubelet" ) func Labels() map[string]string { return map[string]string{ "component": "velero", } } func podLabels(userLabels ...map[string]string) map[string]string { // Use the default labels as a starting point base := Labels() // Merge base labels with user labels to enforce CLI precedence for _, labels := range userLabels { for k, v := range labels { base[k] = v } } return base } func podAnnotations(userAnnotations map[string]string) map[string]string { // Use the default annotations as a starting point base := map[string]string{ "prometheus.io/scrape": "true", "prometheus.io/port": "8085", "prometheus.io/path": "/metrics", } // Merge base annotations with user annotations to enforce CLI precedence for k, v := range userAnnotations { base[k] = v } return base } func containerPorts() []corev1api.ContainerPort { return []corev1api.ContainerPort{ { Name: "metrics", ContainerPort: 8085, }, } } func objectMeta(namespace, name string) metav1.ObjectMeta { return metav1.ObjectMeta{ Name: name, Namespace: namespace, Labels: Labels(), } } func ServiceAccount(namespace string, annotations map[string]string) *corev1api.ServiceAccount { objMeta := objectMeta(namespace, defaultServiceAccountName) objMeta.Annotations = annotations return &corev1api.ServiceAccount{ ObjectMeta: objMeta, TypeMeta: metav1.TypeMeta{ Kind: "ServiceAccount", APIVersion: corev1api.SchemeGroupVersion.String(), }, } } func ClusterRoleBinding(namespace string) *rbacv1.ClusterRoleBinding { crbName := "velero" if namespace != DefaultVeleroNamespace { crbName = "velero-" + namespace } crb := &rbacv1.ClusterRoleBinding{ ObjectMeta: objectMeta("", crbName), TypeMeta: metav1.TypeMeta{ Kind: "ClusterRoleBinding", APIVersion: rbacv1.SchemeGroupVersion.String(), }, Subjects: []rbacv1.Subject{ { Kind: "ServiceAccount", Namespace: namespace, Name: "velero", }, }, RoleRef: rbacv1.RoleRef{ Kind: "ClusterRole", Name: "cluster-admin", APIGroup: "rbac.authorization.k8s.io", }, } return crb } func Namespace(namespace string) *corev1api.Namespace { ns := &corev1api.Namespace{ ObjectMeta: objectMeta("", namespace), TypeMeta: metav1.TypeMeta{ Kind: "Namespace", APIVersion: corev1api.SchemeGroupVersion.String(), }, } ns.Labels["pod-security.kubernetes.io/enforce"] = podSecurityLevel ns.Labels["pod-security.kubernetes.io/enforce-version"] = podSecurityVersion ns.Labels["pod-security.kubernetes.io/audit"] = podSecurityLevel ns.Labels["pod-security.kubernetes.io/audit-version"] = podSecurityVersion ns.Labels["pod-security.kubernetes.io/warn"] = podSecurityLevel ns.Labels["pod-security.kubernetes.io/warn-version"] = podSecurityVersion return ns } func BackupStorageLocation(namespace, provider, bucket, prefix string, config map[string]string, caCert []byte) *velerov1api.BackupStorageLocation { return &velerov1api.BackupStorageLocation{ ObjectMeta: objectMeta(namespace, "default"), TypeMeta: metav1.TypeMeta{ Kind: "BackupStorageLocation", APIVersion: velerov1api.SchemeGroupVersion.String(), }, Spec: velerov1api.BackupStorageLocationSpec{ Provider: provider, StorageType: velerov1api.StorageType{ ObjectStorage: &velerov1api.ObjectStorageLocation{ Bucket: bucket, Prefix: prefix, CACert: caCert, }, }, Config: config, Default: true, }, } } func VolumeSnapshotLocation(namespace, provider string, config map[string]string) *velerov1api.VolumeSnapshotLocation { return &velerov1api.VolumeSnapshotLocation{ ObjectMeta: objectMeta(namespace, "default"), TypeMeta: metav1.TypeMeta{ Kind: "VolumeSnapshotLocation", APIVersion: velerov1api.SchemeGroupVersion.String(), }, Spec: velerov1api.VolumeSnapshotLocationSpec{ Provider: provider, Config: config, }, } } func Secret(namespace string, data []byte) *corev1api.Secret { return &corev1api.Secret{ ObjectMeta: objectMeta(namespace, "cloud-credentials"), TypeMeta: metav1.TypeMeta{ Kind: "Secret", APIVersion: corev1api.SchemeGroupVersion.String(), }, Data: map[string][]byte{ "cloud": data, }, Type: corev1api.SecretTypeOpaque, } } func appendUnstructured(list *unstructured.UnstructuredList, obj runtime.Object) error { u, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&obj) // Remove the status field so we're not sending blank data to the server. // On CRDs, having an empty status is actually a validation error. delete(u, "status") if err != nil { return err } list.Items = append(list.Items, unstructured.Unstructured{Object: u}) return nil } type VeleroOptions struct { Namespace string Image string ProviderName string Bucket string Prefix string PodAnnotations map[string]string PodLabels map[string]string ServiceAccountAnnotations map[string]string ServiceAccountName string VeleroPodResources corev1api.ResourceRequirements NodeAgentPodResources corev1api.ResourceRequirements SecretData []byte RestoreOnly bool UseNodeAgent bool UseNodeAgentWindows bool PrivilegedNodeAgent bool UseVolumeSnapshots bool BSLConfig map[string]string VSLConfig map[string]string DefaultRepoMaintenanceFrequency time.Duration GarbageCollectionFrequency time.Duration PodVolumeOperationTimeout time.Duration Plugins []string NoDefaultBackupLocation bool CACertData []byte Features []string DefaultVolumesToFsBackup bool UploaderType string DefaultSnapshotMoveData bool DisableInformerCache bool ScheduleSkipImmediately bool PodResources kube.PodResources KeepLatestMaintenanceJobs int BackupRepoConfigMap string RepoMaintenanceJobConfigMap string NodeAgentConfigMap string ItemBlockWorkerCount int ConcurrentBackups int KubeletRootDir string NodeAgentDisableHostPath bool ServerPriorityClassName string NodeAgentPriorityClassName string } func AllCRDs() *unstructured.UnstructuredList { resources := new(unstructured.UnstructuredList) // Set the GVK so that the serialization framework outputs the list properly resources.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "List"}) for _, crd := range v1crds.CRDs { crd.SetLabels(Labels()) if err := appendUnstructured(resources, crd); err != nil { fmt.Printf("error appending v1 CRD %s: %s\n", crd.GetName(), err.Error()) } } for _, crd := range v2alpha1crds.CRDs { crd.SetLabels(Labels()) if err := appendUnstructured(resources, crd); err != nil { fmt.Printf("error appending v2alpha1 CRD %s: %s\n", crd.GetName(), err.Error()) } } return resources } // AllResources returns a list of all resources necessary to install Velero, in the appropriate order, into a Kubernetes cluster. // Items are unstructured, since there are different data types returned. func AllResources(o *VeleroOptions) *unstructured.UnstructuredList { resources := AllCRDs() ns := Namespace(o.Namespace) if err := appendUnstructured(resources, ns); err != nil { fmt.Printf("error appending Namespace %s: %s\n", ns.GetName(), err.Error()) } serviceAccountName := defaultServiceAccountName if o.ServiceAccountName == "" { crb := ClusterRoleBinding(o.Namespace) if err := appendUnstructured(resources, crb); err != nil { fmt.Printf("error appending ClusterRoleBinding %s: %s\n", crb.GetName(), err.Error()) } sa := ServiceAccount(o.Namespace, o.ServiceAccountAnnotations) if err := appendUnstructured(resources, sa); err != nil { fmt.Printf("error appending ServiceAccount %s: %s\n", sa.GetName(), err.Error()) } } else { serviceAccountName = o.ServiceAccountName } if o.SecretData != nil { sec := Secret(o.Namespace, o.SecretData) if err := appendUnstructured(resources, sec); err != nil { fmt.Printf("error appending Secret %s: %s\n", sec.GetName(), err.Error()) } } if !o.NoDefaultBackupLocation { bsl := BackupStorageLocation(o.Namespace, o.ProviderName, o.Bucket, o.Prefix, o.BSLConfig, o.CACertData) if err := appendUnstructured(resources, bsl); err != nil { fmt.Printf("error appending BackupStorageLocation %s: %s\n", bsl.GetName(), err.Error()) } } // A snapshot location may not be desirable for users relying on pod volume backup/restore if o.UseVolumeSnapshots { vsl := VolumeSnapshotLocation(o.Namespace, o.ProviderName, o.VSLConfig) if err := appendUnstructured(resources, vsl); err != nil { fmt.Printf("error appending VolumeSnapshotLocation %s: %s\n", vsl.GetName(), err.Error()) } } secretPresent := o.SecretData != nil deployOpts := []podTemplateOption{ WithAnnotations(o.PodAnnotations), WithLabels(o.PodLabels), WithImage(o.Image), WithResources(o.VeleroPodResources), WithSecret(secretPresent), WithDefaultRepoMaintenanceFrequency(o.DefaultRepoMaintenanceFrequency), WithServiceAccountName(serviceAccountName), WithGarbageCollectionFrequency(o.GarbageCollectionFrequency), WithPodVolumeOperationTimeout(o.PodVolumeOperationTimeout), WithUploaderType(o.UploaderType), WithScheduleSkipImmediately(o.ScheduleSkipImmediately), WithPodResources(o.PodResources), WithKeepLatestMaintenanceJobs(o.KeepLatestMaintenanceJobs), WithItemBlockWorkerCount(o.ItemBlockWorkerCount), WithConcurrentBackups(o.ConcurrentBackups), } if o.ServerPriorityClassName != "" { deployOpts = append(deployOpts, WithPriorityClassName(o.ServerPriorityClassName)) } if len(o.Features) > 0 { deployOpts = append(deployOpts, WithFeatures(o.Features)) } if o.RestoreOnly { deployOpts = append(deployOpts, WithRestoreOnly(true)) } if len(o.Plugins) > 0 { deployOpts = append(deployOpts, WithPlugins(o.Plugins)) } if o.DefaultVolumesToFsBackup { deployOpts = append(deployOpts, WithDefaultVolumesToFsBackup(true)) } if o.DefaultSnapshotMoveData { deployOpts = append(deployOpts, WithDefaultSnapshotMoveData(true)) } if o.DisableInformerCache { deployOpts = append(deployOpts, WithDisableInformerCache(true)) } if len(o.BackupRepoConfigMap) > 0 { deployOpts = append(deployOpts, WithBackupRepoConfigMap(o.BackupRepoConfigMap)) } if len(o.RepoMaintenanceJobConfigMap) > 0 { deployOpts = append(deployOpts, WithRepoMaintenanceJobConfigMap(o.RepoMaintenanceJobConfigMap)) } deploy := Deployment(o.Namespace, deployOpts...) if err := appendUnstructured(resources, deploy); err != nil { fmt.Printf("error appending Deployment %s: %s\n", deploy.GetName(), err.Error()) } if o.UseNodeAgent || o.UseNodeAgentWindows { dsOpts := []podTemplateOption{ WithAnnotations(o.PodAnnotations), WithLabels(o.PodLabels), WithImage(o.Image), WithResources(o.NodeAgentPodResources), WithSecret(secretPresent), WithServiceAccountName(serviceAccountName), WithNodeAgentDisableHostPath(o.NodeAgentDisableHostPath), } if len(o.Features) > 0 { dsOpts = append(dsOpts, WithFeatures(o.Features)) } if o.PrivilegedNodeAgent { dsOpts = append(dsOpts, WithPrivilegedNodeAgent(true)) } if len(o.NodeAgentConfigMap) > 0 { dsOpts = append(dsOpts, WithNodeAgentConfigMap(o.NodeAgentConfigMap)) } if len(o.BackupRepoConfigMap) > 0 { dsOpts = append(dsOpts, WithBackupRepoConfigMap(o.BackupRepoConfigMap)) } if len(o.KubeletRootDir) > 0 { dsOpts = append(dsOpts, WithKubeletRootDir(o.KubeletRootDir)) } if o.NodeAgentPriorityClassName != "" { dsOpts = append(dsOpts, WithPriorityClassName(o.NodeAgentPriorityClassName)) } if o.UseNodeAgent { ds := DaemonSet(o.Namespace, dsOpts...) if err := appendUnstructured(resources, ds); err != nil { fmt.Printf("error appending DaemonSet %s: %s\n", ds.GetName(), err.Error()) } } if o.UseNodeAgentWindows { dsOpts = append(dsOpts, WithForWindows()) dsWin := DaemonSet(o.Namespace, dsOpts...) if err := appendUnstructured(resources, dsWin); err != nil { fmt.Printf("error appending DaemonSet %s: %s\n", dsWin.GetName(), err.Error()) } } } return resources } ================================================ FILE: pkg/install/resources_test.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package install import ( "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) func TestResources(t *testing.T) { bsl := BackupStorageLocation(DefaultVeleroNamespace, "test", "test", "", make(map[string]string), []byte("test")) assert.Equal(t, "velero", bsl.ObjectMeta.Namespace) assert.Equal(t, "test", bsl.Spec.Provider) assert.Equal(t, "test", bsl.Spec.StorageType.ObjectStorage.Bucket) assert.Equal(t, make(map[string]string), bsl.Spec.Config) assert.Equal(t, []byte("test"), bsl.Spec.ObjectStorage.CACert) vsl := VolumeSnapshotLocation(DefaultVeleroNamespace, "test", make(map[string]string)) assert.Equal(t, "velero", vsl.ObjectMeta.Namespace) assert.Equal(t, "test", vsl.Spec.Provider) assert.Equal(t, make(map[string]string), vsl.Spec.Config) ns := Namespace("velero") assert.Equal(t, "velero", ns.Name) // For k8s version v1.25 and later, need to add the following labels to make // velero installation namespace has privileged version to work with // PSA(Pod Security Admission) and PSS(Pod Security Standards). assert.Equal(t, "privileged", ns.Labels["pod-security.kubernetes.io/enforce"]) assert.Equal(t, "latest", ns.Labels["pod-security.kubernetes.io/enforce-version"]) assert.Equal(t, "privileged", ns.Labels["pod-security.kubernetes.io/audit"]) assert.Equal(t, "latest", ns.Labels["pod-security.kubernetes.io/audit-version"]) assert.Equal(t, "privileged", ns.Labels["pod-security.kubernetes.io/warn"]) assert.Equal(t, "latest", ns.Labels["pod-security.kubernetes.io/warn-version"]) crb := ClusterRoleBinding(DefaultVeleroNamespace) // The CRB is a cluster-scoped resource assert.Empty(t, crb.ObjectMeta.Namespace) assert.Equal(t, "velero", crb.ObjectMeta.Name) assert.Equal(t, "velero", crb.Subjects[0].Namespace) customNamespaceCRB := ClusterRoleBinding("foo") // The CRB is a cluster-scoped resource assert.Empty(t, customNamespaceCRB.ObjectMeta.Namespace) assert.Equal(t, "velero-foo", customNamespaceCRB.ObjectMeta.Name) assert.Equal(t, "foo", customNamespaceCRB.Subjects[0].Namespace) sa := ServiceAccount(DefaultVeleroNamespace, map[string]string{"abcd": "cbd"}) assert.Equal(t, "velero", sa.ObjectMeta.Namespace) assert.Equal(t, "cbd", sa.ObjectMeta.Annotations["abcd"]) } func TestAllCRDs(t *testing.T) { list := AllCRDs() assert.Len(t, list.Items, 13) assert.Equal(t, Labels(), list.Items[0].GetLabels()) } func TestAllResources(t *testing.T) { option := &VeleroOptions{ Namespace: "velero", SecretData: []byte{'a'}, UseVolumeSnapshots: true, UseNodeAgent: true, UseNodeAgentWindows: true, } list := AllResources(option) objects := map[string][]unstructured.Unstructured{} for _, item := range list.Items { objects[item.GetKind()] = append(objects[item.GetKind()], item) } ns, exist := objects["Namespace"] require.True(t, exist) assert.Equal(t, "velero", ns[0].GetName()) _, exist = objects["ClusterRoleBinding"] assert.True(t, exist) _, exist = objects["ServiceAccount"] assert.True(t, exist) _, exist = objects["Secret"] assert.True(t, exist) _, exist = objects["BackupStorageLocation"] assert.True(t, exist) _, exist = objects["VolumeSnapshotLocation"] assert.True(t, exist) _, exist = objects["Deployment"] assert.True(t, exist) ds, exist := objects["DaemonSet"] assert.True(t, exist) assert.Len(t, ds, 2) } func TestAllResourcesWithPriorityClassName(t *testing.T) { testCases := []struct { name string serverPriorityClassName string nodeAgentPriorityClassName string useNodeAgent bool }{ { name: "with same priority class for server and node agent", serverPriorityClassName: "high-priority", nodeAgentPriorityClassName: "high-priority", useNodeAgent: true, }, { name: "with different priority classes for server and node agent", serverPriorityClassName: "high-priority", nodeAgentPriorityClassName: "medium-priority", useNodeAgent: true, }, { name: "with only server priority class", serverPriorityClassName: "high-priority", nodeAgentPriorityClassName: "", useNodeAgent: true, }, { name: "with only node agent priority class", serverPriorityClassName: "", nodeAgentPriorityClassName: "medium-priority", useNodeAgent: true, }, { name: "with priority class name without node agent", serverPriorityClassName: "high-priority", nodeAgentPriorityClassName: "medium-priority", useNodeAgent: false, }, { name: "without priority class name with node agent", serverPriorityClassName: "", nodeAgentPriorityClassName: "", useNodeAgent: true, }, { name: "without priority class name without node agent", serverPriorityClassName: "", nodeAgentPriorityClassName: "", useNodeAgent: false, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { // Create VeleroOptions with the priority class names options := &VeleroOptions{ Namespace: "velero", UseNodeAgent: tc.useNodeAgent, ServerPriorityClassName: tc.serverPriorityClassName, NodeAgentPriorityClassName: tc.nodeAgentPriorityClassName, } // Generate all resources resources := AllResources(options) // Find the deployment and verify priority class name deploymentFound := false daemonsetFound := false for i := range resources.Items { item := resources.Items[i] // Check deployment if item.GetKind() == "Deployment" && item.GetName() == "velero" { deploymentFound = true // Extract priority class name from the unstructured object priorityClassName, found, err := unstructured.NestedString( item.Object, "spec", "template", "spec", "priorityClassName", ) require.NoError(t, err) if tc.serverPriorityClassName != "" { assert.True(t, found, "Server priorityClassName should be set") assert.Equal(t, tc.serverPriorityClassName, priorityClassName, "Server deployment should have the correct priority class") } else { // If no priority class name was provided, it might not be set at all if found { assert.Empty(t, priorityClassName) } } } // Check daemonset if node agent is enabled if tc.useNodeAgent && item.GetKind() == "DaemonSet" && item.GetName() == "node-agent" { daemonsetFound = true // Extract priority class name from the unstructured object priorityClassName, found, err := unstructured.NestedString( item.Object, "spec", "template", "spec", "priorityClassName", ) require.NoError(t, err) if tc.nodeAgentPriorityClassName != "" { assert.True(t, found, "Node agent priorityClassName should be set") assert.Equal(t, tc.nodeAgentPriorityClassName, priorityClassName, "Node agent daemonset should have the correct priority class") } else { // If no priority class name was provided, it might not be set at all if found { assert.Empty(t, priorityClassName) } } } } // Verify we found the deployment assert.True(t, deploymentFound, "Deployment should be present in resources") // Verify we found the daemonset if node agent is enabled if tc.useNodeAgent { assert.True(t, daemonsetFound, "DaemonSet should be present when UseNodeAgent is true") } }) } } ================================================ FILE: pkg/itemblock/actions/pod_action.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actions import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/plugin/velero" "github.com/vmware-tanzu/velero/pkg/util/actionhelpers" ) // PodAction implements ItemBlockAction. type PodAction struct { log logrus.FieldLogger } // NewPodAction creates a new ItemBlockAction for pods. func NewPodAction(logger logrus.FieldLogger) *PodAction { return &PodAction{log: logger} } // AppliesTo returns a ResourceSelector that applies only to pods. func (a *PodAction) AppliesTo() (velero.ResourceSelector, error) { return velero.ResourceSelector{ IncludedResources: []string{"pods"}, }, nil } // GetRelatedItems scans the pod's spec.volumes for persistentVolumeClaim volumes and returns a // ResourceIdentifier list containing references to all of the persistentVolumeClaim volumes used by // the pod. This ensures that when a pod is backed up, all referenced PVCs are backed up along with the pod. func (a *PodAction) GetRelatedItems(item runtime.Unstructured, backup *v1.Backup) ([]velero.ResourceIdentifier, error) { a.log.Info("Executing pod ItemBlockAction") defer a.log.Info("Done executing pod ItemBlockAction") pod := new(corev1api.Pod) if err := runtime.DefaultUnstructuredConverter.FromUnstructured(item.UnstructuredContent(), pod); err != nil { return nil, errors.WithStack(err) } return actionhelpers.RelatedItemsForPod(pod, a.log), nil } func (a *PodAction) Name() string { return "PodItemBlockAction" } ================================================ FILE: pkg/itemblock/actions/pod_action_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actions import ( "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/runtime" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/plugin/velero" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func TestPodActionAppliesTo(t *testing.T) { a := NewPodAction(velerotest.NewLogger()) actual, err := a.AppliesTo() require.NoError(t, err) expected := velero.ResourceSelector{ IncludedResources: []string{"pods"}, } assert.Equal(t, expected, actual) } func TestPodActionGetRelatedItems(t *testing.T) { tests := []struct { name string pod runtime.Unstructured expected []velero.ResourceIdentifier }{ { name: "no spec.volumes", pod: velerotest.UnstructuredOrDie(` { "apiVersion": "v1", "kind": "Pod", "metadata": { "namespace": "foo", "name": "bar" } } `), }, { name: "persistentVolumeClaim without claimName", pod: velerotest.UnstructuredOrDie(` { "apiVersion": "v1", "kind": "Pod", "metadata": { "namespace": "foo", "name": "bar" }, "spec": { "volumes": [ { "persistentVolumeClaim": {} } ] } } `), }, { name: "full test, mix of volume types", pod: velerotest.UnstructuredOrDie(` { "apiVersion": "v1", "kind": "Pod", "metadata": { "namespace": "foo", "name": "bar" }, "spec": { "volumes": [ { "persistentVolumeClaim": {} }, { "emptyDir": {} }, { "persistentVolumeClaim": {"claimName": "claim1"} }, { "emptyDir": {} }, { "persistentVolumeClaim": {"claimName": "claim2"} } ] } } `), expected: []velero.ResourceIdentifier{ {GroupResource: kuberesource.PersistentVolumeClaims, Namespace: "foo", Name: "claim1"}, {GroupResource: kuberesource.PersistentVolumeClaims, Namespace: "foo", Name: "claim2"}, }, }, { name: "test priority class", pod: velerotest.UnstructuredOrDie(` { "apiVersion": "v1", "kind": "Pod", "metadata": { "namespace": "foo", "name": "bar" }, "spec": { "priorityClassName": "testPriorityClass" } } `), expected: []velero.ResourceIdentifier{ {GroupResource: kuberesource.PriorityClasses, Name: "testPriorityClass"}, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { a := NewPodAction(velerotest.NewLogger()) relatedItems, err := a.GetRelatedItems(test.pod, nil) require.NoError(t, err) assert.Equal(t, test.expected, relatedItems) }) } } ================================================ FILE: pkg/itemblock/actions/pvc_action.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actions import ( "context" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" crclient "sigs.k8s.io/controller-runtime/pkg/client" v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/kuberesource" plugincommon "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" "github.com/vmware-tanzu/velero/pkg/plugin/velero" "github.com/vmware-tanzu/velero/pkg/util/actionhelpers" "github.com/vmware-tanzu/velero/pkg/util/kube" ) // PVCAction inspects a PersistentVolumeClaim for the PersistentVolume // that it references and backs it up type PVCAction struct { log logrus.FieldLogger crClient crclient.Client // map[namespace]->[map[pvcVolumes]->[]podName] nsPVCs map[string]map[string][]string } func NewPVCAction(f client.Factory) plugincommon.HandlerInitializer { return func(logger logrus.FieldLogger) (any, error) { crClient, err := f.KubebuilderClient() if err != nil { return nil, errors.WithStack(err) } return &PVCAction{ log: logger, crClient: crClient, }, nil } } func (a *PVCAction) AppliesTo() (velero.ResourceSelector, error) { return velero.ResourceSelector{ IncludedResources: []string{"persistentvolumeclaims"}, }, nil } func (a *PVCAction) GetRelatedItems(item runtime.Unstructured, backup *v1.Backup) ([]velero.ResourceIdentifier, error) { a.log.Info("Executing PVC ItemBlockAction") defer a.log.Info("Done executing PVC ItemBlockAction") pvc := new(corev1api.PersistentVolumeClaim) if err := runtime.DefaultUnstructuredConverter.FromUnstructured(item.UnstructuredContent(), &pvc); err != nil { return nil, errors.Wrap(err, "unable to convert unstructured item to persistent volume claim") } if pvc.Status.Phase != corev1api.ClaimBound || pvc.Spec.VolumeName == "" { return nil, nil } // returns the PV for the PVC (shared with BIA additionalItems) relatedItems := actionhelpers.RelatedItemsForPVC(pvc, a.log) // Adds pods mounting this PVC to ensure that multiple pods mounting the same RWX // volume get backed up together. pvcs, err := a.getPVCList(pvc.Namespace) if err != nil { return nil, err } for _, pod := range pvcs[pvc.Name] { a.log.Infof("Adding related Pod %s to PVC %s", pod, pvc.Name) relatedItems = append(relatedItems, velero.ResourceIdentifier{ GroupResource: kuberesource.Pods, Namespace: pvc.Namespace, Name: pod, }) } // Gather groupedPVCs based on VGS label provided in the backup groupedPVCs, err := a.getGroupedPVCs(context.Background(), pvc, backup) if err != nil { return nil, err } // Add the groupedPVCs to relatedItems so that they processed in a single item block relatedItems = append(relatedItems, groupedPVCs...) return relatedItems, nil } func (a *PVCAction) getPVCList(ns string) (map[string][]string, error) { if a.nsPVCs == nil { a.nsPVCs = make(map[string]map[string][]string) } pvcList, ok := a.nsPVCs[ns] if ok { return pvcList, nil } pvcList = make(map[string][]string) pods := new(corev1api.PodList) err := a.crClient.List(context.Background(), pods, crclient.InNamespace(ns)) if err != nil { return nil, errors.Wrap(err, "failed to list pods") } for i := range pods.Items { if kube.IsPodRunning(&pods.Items[i]) != nil { a.log.Debugf("Pod %s is not running, not adding to Pod list for PVC IBA plugin", pods.Items[i].Name) continue } for _, volume := range pods.Items[i].Spec.Volumes { if volume.VolumeSource.PersistentVolumeClaim != nil { pvcList[volume.VolumeSource.PersistentVolumeClaim.ClaimName] = append(pvcList[volume.VolumeSource.PersistentVolumeClaim.ClaimName], pods.Items[i].Name) } } } a.nsPVCs[ns] = pvcList return pvcList, nil } func (a *PVCAction) Name() string { return "PVCItemBlockAction" } // getGroupedPVCs returns other PVCs in the same group based on the VGS label key in the Backup spec. func (a *PVCAction) getGroupedPVCs(ctx context.Context, pvc *corev1api.PersistentVolumeClaim, backup *v1.Backup) ([]velero.ResourceIdentifier, error) { var related []velero.ResourceIdentifier vgsLabelKey := backup.Spec.VolumeGroupSnapshotLabelKey if vgsLabelKey == "" { a.log.Debug("No VolumeGroupSnapshotLabelKey provided in backup spec; skipping PVC grouping") return nil, nil } groupID, ok := pvc.Labels[vgsLabelKey] if !ok || groupID == "" { // PVC does not belong to any VGS group or groupID has empty value a.log.Debug("PVC does not belong to any PVC group or group label value is empty; skipping PVC grouping") return nil, nil } pvcList := new(corev1api.PersistentVolumeClaimList) if err := a.crClient.List( ctx, pvcList, crclient.InNamespace(pvc.Namespace), crclient.MatchingLabels{vgsLabelKey: groupID}, ); err != nil { return nil, errors.Wrapf(err, "failed to list PVCs for VGS grouping with label %s=%s in namespace %s", vgsLabelKey, groupID, pvc.Namespace) } if len(pvcList.Items) <= 1 { // Only the current PVC exists in this group return nil, nil } for _, groupPVC := range pvcList.Items { if groupPVC.Name == pvc.Name { continue } a.log.Infof("Adding grouped PVC %s (group %s) to relatedItems for PVC %s", groupPVC.Name, groupID, pvc.Name) related = append(related, velero.ResourceIdentifier{ GroupResource: kuberesource.PersistentVolumeClaims, Namespace: groupPVC.Namespace, Name: groupPVC.Name, }) } return related, nil } ================================================ FILE: pkg/itemblock/actions/pvc_action_test.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actions import ( "fmt" "testing" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" factorymocks "github.com/vmware-tanzu/velero/pkg/client/mocks" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/plugin/velero" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func TestBackupPVAction(t *testing.T) { tests := []struct { name string pvc *corev1api.PersistentVolumeClaim pods []*corev1api.Pod expectedErr error expectedRelated []velero.ResourceIdentifier }{ { name: "Test no volumeName", pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").Phase(corev1api.ClaimBound).Result(), expectedErr: nil, expectedRelated: nil, }, { name: "Test empty volumeName", pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").VolumeName("").Phase(corev1api.ClaimBound).Result(), expectedErr: nil, expectedRelated: nil, }, { name: "Test no status phase", pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").VolumeName("testPV").Result(), expectedErr: nil, expectedRelated: nil, }, { name: "Test pending status phase", pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").VolumeName("testPV").Phase(corev1api.ClaimPending).Result(), expectedErr: nil, expectedRelated: nil, }, { name: "Test lost status phase", pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").VolumeName("testPV").Phase(corev1api.ClaimLost).Result(), expectedErr: nil, expectedRelated: nil, }, { name: "Test with volume", pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").VolumeName("testPV").Phase(corev1api.ClaimBound).Result(), expectedErr: nil, expectedRelated: []velero.ResourceIdentifier{ {GroupResource: kuberesource.PersistentVolumes, Name: "testPV"}, }, }, { name: "Test with volume and one running pod", pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").VolumeName("testPV").Phase(corev1api.ClaimBound).Result(), pods: []*corev1api.Pod{ builder.ForPod("velero", "testPod1").Volumes(builder.ForVolume("testPVC").PersistentVolumeClaimSource("testPVC").Result()).NodeName("velero").Phase(corev1api.PodRunning).Result(), }, expectedErr: nil, expectedRelated: []velero.ResourceIdentifier{ {GroupResource: kuberesource.PersistentVolumes, Name: "testPV"}, {GroupResource: kuberesource.Pods, Namespace: "velero", Name: "testPod1"}, }, }, { name: "Test with volume and multiple running pods", pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").VolumeName("testPV").Phase(corev1api.ClaimBound).Result(), pods: []*corev1api.Pod{ builder.ForPod("velero", "testPod1").Volumes(builder.ForVolume("testPVC").PersistentVolumeClaimSource("testPVC").Result()).NodeName("velero").Phase(corev1api.PodRunning).Result(), builder.ForPod("velero", "testPod2").Volumes(builder.ForVolume("testPVC").PersistentVolumeClaimSource("testPVC").Result()).NodeName("velero").Phase(corev1api.PodRunning).Result(), builder.ForPod("velero", "testPod3").Volumes(builder.ForVolume("testPVC").PersistentVolumeClaimSource("testPVC").Result()).NodeName("velero").Phase(corev1api.PodRunning).Result(), }, expectedErr: nil, expectedRelated: []velero.ResourceIdentifier{ {GroupResource: kuberesource.PersistentVolumes, Name: "testPV"}, {GroupResource: kuberesource.Pods, Namespace: "velero", Name: "testPod1"}, {GroupResource: kuberesource.Pods, Namespace: "velero", Name: "testPod2"}, {GroupResource: kuberesource.Pods, Namespace: "velero", Name: "testPod3"}, }, }, { name: "Test with volume and multiple running pods, some not running", pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").VolumeName("testPV").Phase(corev1api.ClaimBound).Result(), pods: []*corev1api.Pod{ builder.ForPod("velero", "testPod1").Volumes(builder.ForVolume("testPVC").PersistentVolumeClaimSource("testPVC").Result()).NodeName("velero").Phase(corev1api.PodSucceeded).Result(), builder.ForPod("velero", "testPod2").Volumes(builder.ForVolume("testPVC").PersistentVolumeClaimSource("testPVC").Result()).NodeName("velero").Phase(corev1api.PodRunning).Result(), builder.ForPod("velero", "testPod3").Volumes(builder.ForVolume("testPVC").PersistentVolumeClaimSource("testPVC").Result()).Phase(corev1api.PodRunning).Result(), }, expectedErr: nil, expectedRelated: []velero.ResourceIdentifier{ {GroupResource: kuberesource.PersistentVolumes, Name: "testPV"}, {GroupResource: kuberesource.Pods, Namespace: "velero", Name: "testPod2"}, }, }, { name: "Test with PVC grouping via VGS label", pvc: builder.ForPersistentVolumeClaim("velero", "testPVC-1").ObjectMeta(builder.WithLabels("velero.io/group", "db")).VolumeName("testPV-1").Phase(corev1api.ClaimBound).Result(), pods: []*corev1api.Pod{ builder.ForPod("velero", "testPod-1"). Volumes(builder.ForVolume("testPV-1").PersistentVolumeClaimSource("testPVC-1").Result()). NodeName("node"). Phase(corev1api.PodRunning).Result(), }, expectedErr: nil, expectedRelated: []velero.ResourceIdentifier{ {GroupResource: kuberesource.PersistentVolumes, Name: "testPV-1"}, {GroupResource: kuberesource.Pods, Namespace: "velero", Name: "testPod-1"}, {GroupResource: kuberesource.PersistentVolumeClaims, Namespace: "velero", Name: "groupedPVC"}, }, }, } backup := &v1.Backup{} logger := logrus.New() f := &factorymocks.Factory{} f.On("KubebuilderClient").Return(nil, fmt.Errorf("")) plugin := NewPVCAction(f) _, err := plugin(logger) require.Error(t, err) for _, tc := range tests { t.Run(tc.name, func(*testing.T) { crClient := velerotest.NewFakeControllerRuntimeClient(t) f := &factorymocks.Factory{} f.On("KubebuilderClient").Return(crClient, nil) plugin := NewPVCAction(f) i, err := plugin(logger) require.NoError(t, err) a := i.(*PVCAction) if tc.pvc != nil { require.NoError(t, crClient.Create(t.Context(), tc.pvc)) } for _, pod := range tc.pods { require.NoError(t, crClient.Create(t.Context(), pod)) } if tc.name == "Test with PVC grouping via VGS label" { groupedPVC := builder.ForPersistentVolumeClaim("velero", "groupedPVC").ObjectMeta(builder.WithLabels("velero.io/group", "db")).VolumeName("groupedPV").Phase(corev1api.ClaimBound).Result() require.NoError(t, crClient.Create(t.Context(), groupedPVC)) backup.Spec.VolumeGroupSnapshotLabelKey = "velero.io/group" } pvcMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&tc.pvc) require.NoError(t, err) relatedItems, err := a.GetRelatedItems(&unstructured.Unstructured{Object: pvcMap}, backup) if tc.expectedErr != nil { require.EqualError(t, err, tc.expectedErr.Error()) } else { require.NoError(t, err) } assert.Equal(t, tc.expectedRelated, relatedItems) }) } } // Test_getGroupedPVCs verifies the PVC grouping logic for VolumeGroupSnapshots. // This ensures only same-namespace PVCs with the same label key and value are included. func Test_getGroupedPVCs(t *testing.T) { tests := []struct { name string labelKey string groupValue string existingPVCs []*corev1api.PersistentVolumeClaim targetPVC *corev1api.PersistentVolumeClaim expectedRelated []velero.ResourceIdentifier expectError bool }{ { name: "No label key in spec", labelKey: "", targetPVC: builder.ForPersistentVolumeClaim("ns", "pvc-1").Result(), expectError: false, }, { name: "No group value", labelKey: "velero.io/group", groupValue: "", targetPVC: builder.ForPersistentVolumeClaim("ns", "pvc-1").Result(), expectError: false, }, { name: "Target PVC does not have the label", labelKey: "velero.io/group", targetPVC: builder.ForPersistentVolumeClaim("ns", "pvc-1").Result(), expectError: false, }, { name: "Target PVC has label, but no group matches", labelKey: "velero.io/group", groupValue: "group-1", targetPVC: builder.ForPersistentVolumeClaim("ns", "pvc-1").ObjectMeta(builder.WithLabels("velero.io/group", "group-1")).Result(), existingPVCs: []*corev1api.PersistentVolumeClaim{ builder.ForPersistentVolumeClaim("ns", "pvc-1").ObjectMeta(builder.WithLabels("velero.io/group", "group-1")).Result(), }, expectError: false, expectedRelated: nil, }, { name: "Multiple PVCs in the same group", labelKey: "velero.io/group", groupValue: "group-1", targetPVC: builder.ForPersistentVolumeClaim("ns", "pvc-1").ObjectMeta(builder.WithLabels("velero.io/group", "group-1")).Result(), existingPVCs: []*corev1api.PersistentVolumeClaim{ builder.ForPersistentVolumeClaim("ns", "pvc-1").ObjectMeta(builder.WithLabels("velero.io/group", "group-1")).Result(), builder.ForPersistentVolumeClaim("ns", "pvc-2").ObjectMeta(builder.WithLabels("velero.io/group", "group-1")).Result(), builder.ForPersistentVolumeClaim("ns", "pvc-3").ObjectMeta(builder.WithLabels("velero.io/group", "group-1")).Result(), }, expectError: false, expectedRelated: []velero.ResourceIdentifier{ {GroupResource: kuberesource.PersistentVolumeClaims, Namespace: "ns", Name: "pvc-2"}, {GroupResource: kuberesource.PersistentVolumeClaims, Namespace: "ns", Name: "pvc-3"}, }, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { crClient := velerotest.NewFakeControllerRuntimeClient(t) for _, pvc := range tc.existingPVCs { require.NoError(t, crClient.Create(t.Context(), pvc)) } logger := logrus.New() a := &PVCAction{ log: logger, crClient: crClient, } backup := builder.ForBackup("ns", "bkp").VolumeGroupSnapshotLabelKey(tc.labelKey).Result() related, err := a.getGroupedPVCs(t.Context(), tc.targetPVC, backup) if tc.expectError { require.Error(t, err) } else { require.NoError(t, err) } assert.ElementsMatch(t, tc.expectedRelated, related) }) } } ================================================ FILE: pkg/itemblock/actions/service_account_action.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actions import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerodiscovery "github.com/vmware-tanzu/velero/pkg/discovery" "github.com/vmware-tanzu/velero/pkg/plugin/velero" "github.com/vmware-tanzu/velero/pkg/util/actionhelpers" ) // ServiceAccountAction implements ItemBlockAction. type ServiceAccountAction struct { log logrus.FieldLogger clusterRoleBindings []actionhelpers.ClusterRoleBinding } // NewServiceAccountAction creates a new ItemBlockAction for service accounts. func NewServiceAccountAction(logger logrus.FieldLogger, clusterRoleBindingListers map[string]actionhelpers.ClusterRoleBindingLister, discoveryHelper velerodiscovery.Helper) (*ServiceAccountAction, error) { crbs, err := actionhelpers.ClusterRoleBindingsForAction(clusterRoleBindingListers, discoveryHelper) if err != nil { return nil, err } return &ServiceAccountAction{ log: logger, clusterRoleBindings: crbs, }, nil } // AppliesTo returns a ResourceSelector that applies only to service accounts. func (a *ServiceAccountAction) AppliesTo() (velero.ResourceSelector, error) { return velero.ResourceSelector{ IncludedResources: []string{"serviceaccounts"}, }, nil } // GetRelatedItems checks for any ClusterRoleBindings that have this service account as a subject, and // returns the ClusterRoleBinding and associated ClusterRole. func (a *ServiceAccountAction) GetRelatedItems(item runtime.Unstructured, backup *v1.Backup) ([]velero.ResourceIdentifier, error) { a.log.Info("Running ServiceAccount ItemBlockAction") defer a.log.Info("Done running ServiceAccount ItemBlockAction") objectMeta, err := meta.Accessor(item) if err != nil { return nil, errors.WithStack(err) } return actionhelpers.RelatedItemsForServiceAccount(objectMeta, a.clusterRoleBindings, a.log), nil } func (a *ServiceAccountAction) Name() string { return "ServiceAccountItemBlockAction" } ================================================ FILE: pkg/itemblock/actions/service_account_action_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actions import ( "fmt" "sort" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" rbacv1 "k8s.io/api/rbac/v1" rbacbeta "k8s.io/api/rbac/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/plugin/velero" velerotest "github.com/vmware-tanzu/velero/pkg/test" "github.com/vmware-tanzu/velero/pkg/util/actionhelpers" ) func newV1ClusterRoleBindingList(rbacCRBList []rbacv1.ClusterRoleBinding) []actionhelpers.ClusterRoleBinding { var crbs []actionhelpers.ClusterRoleBinding for _, c := range rbacCRBList { crbs = append(crbs, actionhelpers.V1ClusterRoleBinding{Crb: c}) } return crbs } func newV1beta1ClusterRoleBindingList(rbacCRBList []rbacbeta.ClusterRoleBinding) []actionhelpers.ClusterRoleBinding { var crbs []actionhelpers.ClusterRoleBinding for _, c := range rbacCRBList { crbs = append(crbs, actionhelpers.V1beta1ClusterRoleBinding{Crb: c}) } return crbs } type FakeV1ClusterRoleBindingLister struct { v1crbs []rbacv1.ClusterRoleBinding } func (f FakeV1ClusterRoleBindingLister) List() ([]actionhelpers.ClusterRoleBinding, error) { var crbs []actionhelpers.ClusterRoleBinding for _, c := range f.v1crbs { crbs = append(crbs, actionhelpers.V1ClusterRoleBinding{Crb: c}) } return crbs, nil } type FakeV1beta1ClusterRoleBindingLister struct { v1beta1crbs []rbacbeta.ClusterRoleBinding } func (f FakeV1beta1ClusterRoleBindingLister) List() ([]actionhelpers.ClusterRoleBinding, error) { var crbs []actionhelpers.ClusterRoleBinding for _, c := range f.v1beta1crbs { crbs = append(crbs, actionhelpers.V1beta1ClusterRoleBinding{Crb: c}) } return crbs, nil } func TestServiceAccountActionAppliesTo(t *testing.T) { // Instantiating the struct directly since using // NewServiceAccountAction requires a full Kubernetes clientset a := &ServiceAccountAction{} actual, err := a.AppliesTo() require.NoError(t, err) expected := velero.ResourceSelector{ IncludedResources: []string{"serviceaccounts"}, } assert.Equal(t, expected, actual) } func TestNewServiceAccountAction(t *testing.T) { tests := []struct { name string version string expectedCRBs []actionhelpers.ClusterRoleBinding }{ { name: "rbac v1 API instantiates an saAction", version: rbacv1.SchemeGroupVersion.Version, expectedCRBs: []actionhelpers.ClusterRoleBinding{ actionhelpers.V1ClusterRoleBinding{ Crb: rbacv1.ClusterRoleBinding{ ObjectMeta: metav1.ObjectMeta{ Name: "v1crb-1", }, }, }, actionhelpers.V1ClusterRoleBinding{ Crb: rbacv1.ClusterRoleBinding{ ObjectMeta: metav1.ObjectMeta{ Name: "v1crb-2", }, }, }, }, }, { name: "rbac v1beta1 API instantiates an saAction", version: rbacbeta.SchemeGroupVersion.Version, expectedCRBs: []actionhelpers.ClusterRoleBinding{ actionhelpers.V1beta1ClusterRoleBinding{ Crb: rbacbeta.ClusterRoleBinding{ ObjectMeta: metav1.ObjectMeta{ Name: "v1beta1crb-1", }, }, }, actionhelpers.V1beta1ClusterRoleBinding{ Crb: rbacbeta.ClusterRoleBinding{ ObjectMeta: metav1.ObjectMeta{ Name: "v1beta1crb-2", }, }, }, }, }, { name: "no RBAC API instantiates an saAction with empty slice", version: "", expectedCRBs: []actionhelpers.ClusterRoleBinding{}, }, } // Set up all of our fakes outside the test loop discoveryHelper := velerotest.FakeDiscoveryHelper{} logger := velerotest.NewLogger() v1crbs := []rbacv1.ClusterRoleBinding{ { ObjectMeta: metav1.ObjectMeta{ Name: "v1crb-1", }, }, { ObjectMeta: metav1.ObjectMeta{ Name: "v1crb-2", }, }, } v1beta1crbs := []rbacbeta.ClusterRoleBinding{ { ObjectMeta: metav1.ObjectMeta{ Name: "v1beta1crb-1", }, }, { ObjectMeta: metav1.ObjectMeta{ Name: "v1beta1crb-2", }, }, } clusterRoleBindingListers := map[string]actionhelpers.ClusterRoleBindingLister{ rbacv1.SchemeGroupVersion.Version: FakeV1ClusterRoleBindingLister{v1crbs: v1crbs}, rbacbeta.SchemeGroupVersion.Version: FakeV1beta1ClusterRoleBindingLister{v1beta1crbs: v1beta1crbs}, "": actionhelpers.NoopClusterRoleBindingLister{}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { // We only care about the preferred version, nothing else in the list discoveryHelper.APIGroupsList = []metav1.APIGroup{ { Name: rbacv1.GroupName, PreferredVersion: metav1.GroupVersionForDiscovery{ Version: test.version, }, }, } action, err := NewServiceAccountAction(logger, clusterRoleBindingListers, &discoveryHelper) require.NoError(t, err) assert.Equal(t, test.expectedCRBs, action.clusterRoleBindings) }) } } func TestServiceAccountActionExecute(t *testing.T) { tests := []struct { name string serviceAccount runtime.Unstructured crbs []rbacv1.ClusterRoleBinding expectedAdditionalItems []velero.ResourceIdentifier }{ { name: "no crbs", serviceAccount: velerotest.UnstructuredOrDie(` { "apiVersion": "v1", "kind": "ServiceAccount", "metadata": { "namespace": "velero", "name": "velero" } } `), crbs: nil, expectedAdditionalItems: nil, }, { name: "no matching crbs", serviceAccount: velerotest.UnstructuredOrDie(` { "apiVersion": "v1", "kind": "ServiceAccount", "metadata": { "namespace": "velero", "name": "velero" } } `), crbs: []rbacv1.ClusterRoleBinding{ { Subjects: []rbacv1.Subject{ { Kind: "non-matching-kind", Namespace: "non-matching-ns", Name: "non-matching-name", }, { Kind: "non-matching-kind", Namespace: "velero", Name: "velero", }, { Kind: rbacv1.ServiceAccountKind, Namespace: "non-matching-ns", Name: "velero", }, { Kind: rbacv1.ServiceAccountKind, Namespace: "velero", Name: "non-matching-name", }, }, RoleRef: rbacv1.RoleRef{ Name: "role", }, }, }, expectedAdditionalItems: nil, }, { name: "some matching crbs", serviceAccount: velerotest.UnstructuredOrDie(` { "apiVersion": "v1", "kind": "ServiceAccount", "metadata": { "namespace": "velero", "name": "velero" } } `), crbs: []rbacv1.ClusterRoleBinding{ { ObjectMeta: metav1.ObjectMeta{ Name: "crb-1", }, Subjects: []rbacv1.Subject{ { Kind: "non-matching-kind", Namespace: "non-matching-ns", Name: "non-matching-name", }, }, RoleRef: rbacv1.RoleRef{ Name: "role-1", }, }, { ObjectMeta: metav1.ObjectMeta{ Name: "crb-2", }, Subjects: []rbacv1.Subject{ { Kind: "non-matching-kind", Namespace: "non-matching-ns", Name: "non-matching-name", }, { Kind: rbacv1.ServiceAccountKind, Namespace: "velero", Name: "velero", }, }, RoleRef: rbacv1.RoleRef{ Name: "role-2", }, }, { ObjectMeta: metav1.ObjectMeta{ Name: "crb-3", }, Subjects: []rbacv1.Subject{ { Kind: rbacv1.ServiceAccountKind, Namespace: "velero", Name: "velero", }, }, RoleRef: rbacv1.RoleRef{ Name: "role-3", }, }, { ObjectMeta: metav1.ObjectMeta{ Name: "crb-4", }, Subjects: []rbacv1.Subject{ { Kind: rbacv1.ServiceAccountKind, Namespace: "velero", Name: "velero", }, { Kind: "non-matching-kind", Namespace: "non-matching-ns", Name: "non-matching-name", }, }, RoleRef: rbacv1.RoleRef{ Name: "role-4", }, }, }, expectedAdditionalItems: []velero.ResourceIdentifier{ { GroupResource: kuberesource.ClusterRoleBindings, Name: "crb-2", }, { GroupResource: kuberesource.ClusterRoleBindings, Name: "crb-3", }, { GroupResource: kuberesource.ClusterRoleBindings, Name: "crb-4", }, { GroupResource: kuberesource.ClusterRoles, Name: "role-2", }, { GroupResource: kuberesource.ClusterRoles, Name: "role-3", }, { GroupResource: kuberesource.ClusterRoles, Name: "role-4", }, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { // Create the action struct directly so we don't need to mock a clientset action := &ServiceAccountAction{ log: velerotest.NewLogger(), clusterRoleBindings: newV1ClusterRoleBindingList(test.crbs), } additional, err := action.GetRelatedItems(test.serviceAccount, nil) require.NoError(t, err) // ensure slices are ordered for valid comparison sort.Slice(test.expectedAdditionalItems, func(i, j int) bool { return fmt.Sprintf("%s.%s", test.expectedAdditionalItems[i].GroupResource.String(), test.expectedAdditionalItems[i].Name) < fmt.Sprintf("%s.%s", test.expectedAdditionalItems[j].GroupResource.String(), test.expectedAdditionalItems[j].Name) }) sort.Slice(additional, func(i, j int) bool { return fmt.Sprintf("%s.%s", additional[i].GroupResource.String(), additional[i].Name) < fmt.Sprintf("%s.%s", additional[j].GroupResource.String(), additional[j].Name) }) assert.Equal(t, test.expectedAdditionalItems, additional) }) } } func TestServiceAccountActionExecuteOnBeta1(t *testing.T) { tests := []struct { name string serviceAccount runtime.Unstructured crbs []rbacbeta.ClusterRoleBinding expectedAdditionalItems []velero.ResourceIdentifier }{ { name: "no crbs", serviceAccount: velerotest.UnstructuredOrDie(` { "apiVersion": "v1", "kind": "ServiceAccount", "metadata": { "namespace": "velero", "name": "velero" } } `), crbs: nil, expectedAdditionalItems: nil, }, { name: "no matching crbs", serviceAccount: velerotest.UnstructuredOrDie(` { "apiVersion": "v1", "kind": "ServiceAccount", "metadata": { "namespace": "velero", "name": "velero" } } `), crbs: []rbacbeta.ClusterRoleBinding{ { Subjects: []rbacbeta.Subject{ { Kind: "non-matching-kind", Namespace: "non-matching-ns", Name: "non-matching-name", }, { Kind: "non-matching-kind", Namespace: "velero", Name: "velero", }, { Kind: rbacbeta.ServiceAccountKind, Namespace: "non-matching-ns", Name: "velero", }, { Kind: rbacbeta.ServiceAccountKind, Namespace: "velero", Name: "non-matching-name", }, }, RoleRef: rbacbeta.RoleRef{ Name: "role", }, }, }, expectedAdditionalItems: nil, }, { name: "some matching crbs", serviceAccount: velerotest.UnstructuredOrDie(` { "apiVersion": "v1", "kind": "ServiceAccount", "metadata": { "namespace": "velero", "name": "velero" } } `), crbs: []rbacbeta.ClusterRoleBinding{ { ObjectMeta: metav1.ObjectMeta{ Name: "crb-1", }, Subjects: []rbacbeta.Subject{ { Kind: "non-matching-kind", Namespace: "non-matching-ns", Name: "non-matching-name", }, }, RoleRef: rbacbeta.RoleRef{ Name: "role-1", }, }, { ObjectMeta: metav1.ObjectMeta{ Name: "crb-2", }, Subjects: []rbacbeta.Subject{ { Kind: "non-matching-kind", Namespace: "non-matching-ns", Name: "non-matching-name", }, { Kind: rbacbeta.ServiceAccountKind, Namespace: "velero", Name: "velero", }, }, RoleRef: rbacbeta.RoleRef{ Name: "role-2", }, }, { ObjectMeta: metav1.ObjectMeta{ Name: "crb-3", }, Subjects: []rbacbeta.Subject{ { Kind: rbacbeta.ServiceAccountKind, Namespace: "velero", Name: "velero", }, }, RoleRef: rbacbeta.RoleRef{ Name: "role-3", }, }, { ObjectMeta: metav1.ObjectMeta{ Name: "crb-4", }, Subjects: []rbacbeta.Subject{ { Kind: rbacbeta.ServiceAccountKind, Namespace: "velero", Name: "velero", }, { Kind: "non-matching-kind", Namespace: "non-matching-ns", Name: "non-matching-name", }, }, RoleRef: rbacbeta.RoleRef{ Name: "role-4", }, }, }, expectedAdditionalItems: []velero.ResourceIdentifier{ { GroupResource: kuberesource.ClusterRoleBindings, Name: "crb-2", }, { GroupResource: kuberesource.ClusterRoleBindings, Name: "crb-3", }, { GroupResource: kuberesource.ClusterRoleBindings, Name: "crb-4", }, { GroupResource: kuberesource.ClusterRoles, Name: "role-2", }, { GroupResource: kuberesource.ClusterRoles, Name: "role-3", }, { GroupResource: kuberesource.ClusterRoles, Name: "role-4", }, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { // Create the action struct directly so we don't need to mock a clientset action := &ServiceAccountAction{ log: velerotest.NewLogger(), clusterRoleBindings: newV1beta1ClusterRoleBindingList(test.crbs), } additional, err := action.GetRelatedItems(test.serviceAccount, nil) require.NoError(t, err) // ensure slices are ordered for valid comparison sort.Slice(test.expectedAdditionalItems, func(i, j int) bool { return fmt.Sprintf("%s.%s", test.expectedAdditionalItems[i].GroupResource.String(), test.expectedAdditionalItems[i].Name) < fmt.Sprintf("%s.%s", test.expectedAdditionalItems[j].GroupResource.String(), test.expectedAdditionalItems[j].Name) }) sort.Slice(additional, func(i, j int) bool { return fmt.Sprintf("%s.%s", additional[i].GroupResource.String(), additional[i].Name) < fmt.Sprintf("%s.%s", additional[j].GroupResource.String(), additional[j].Name) }) assert.Equal(t, test.expectedAdditionalItems, additional) }) } } ================================================ FILE: pkg/itemblock/itemblock.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package itemblock import ( "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" ) type ItemBlock struct { Log logrus.FieldLogger Items []ItemBlockItem } type ItemBlockItem struct { Gr schema.GroupResource Item *unstructured.Unstructured PreferredGVR schema.GroupVersionResource } func (ib *ItemBlock) AddUnstructured(gr schema.GroupResource, item *unstructured.Unstructured, preferredGVR schema.GroupVersionResource) { ib.Items = append(ib.Items, ItemBlockItem{ Gr: gr, Item: item, PreferredGVR: preferredGVR, }) } // Could return multiple items if EnableAPIGroupVersions is set. The item matching the preferredGVR is returned first func (ib *ItemBlock) FindItem(gr schema.GroupResource, namespace, name string) []ItemBlockItem { var itemList []ItemBlockItem var returnList []ItemBlockItem for _, item := range ib.Items { if item.Gr == gr && item.Item != nil && item.Item.GetName() == name && item.Item.GetNamespace() == namespace { itemGV, err := schema.ParseGroupVersion(item.Item.GetAPIVersion()) if err == nil && item.PreferredGVR.GroupVersion() == itemGV { returnList = append(returnList, item) } else { itemList = append(itemList, item) } } } return append(returnList, itemList...) } ================================================ FILE: pkg/itemoperation/backup_operation.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package itemoperation import ( "github.com/vmware-tanzu/velero/pkg/plugin/velero" ) // BackupOperation stores information about an async item operation // started by a BackupItemAction plugin (v2 or later) type BackupOperation struct { Spec BackupOperationSpec `json:"spec"` Status OperationStatus `json:"status"` } func (in *BackupOperation) DeepCopy() *BackupOperation { if in == nil { return nil } out := new(BackupOperation) in.DeepCopyInto(out) return out } func (in *BackupOperation) DeepCopyInto(out *BackupOperation) { *out = *in in.Spec.DeepCopyInto(&out.Spec) in.Status.DeepCopyInto(&out.Status) } type BackupOperationSpec struct { // BackupName is the name of the Velero backup this item operation // is associated with. BackupName string `json:"backupName"` // BackupUID is the UID of the Velero backup this item operation // is associated with. BackupUID string `json:"backupUID"` // BackupItemAction is the name of the BackupItemAction plugin that started the operation BackupItemAction string `json:"backupItemAction"` // Kubernetes resource identifier for the item ResourceIdentifier velero.ResourceIdentifier `json:"resourceIdentifier"` // OperationID returned by the BIA plugin OperationID string `json:"operationID"` // Items needing to be added to the backup after all async operations have completed PostOperationItems []velero.ResourceIdentifier `json:"postOperationItems"` } func (in *BackupOperationSpec) DeepCopy() *BackupOperationSpec { if in == nil { return nil } out := new(BackupOperationSpec) in.DeepCopyInto(out) return out } func (in *BackupOperationSpec) DeepCopyInto(out *BackupOperationSpec) { *out = *in in.ResourceIdentifier.DeepCopyInto(&out.ResourceIdentifier) if in.PostOperationItems != nil { in, out := &in.PostOperationItems, &out.PostOperationItems *out = make([]velero.ResourceIdentifier, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } } ================================================ FILE: pkg/itemoperation/restore_operation.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package itemoperation import ( "github.com/vmware-tanzu/velero/pkg/plugin/velero" ) // RestoreOperation stores information about an async item operation // started by a RestoreItemAction plugin (v2 or later) type RestoreOperation struct { Spec RestoreOperationSpec `json:"spec"` Status OperationStatus `json:"status"` } func (in *RestoreOperation) DeepCopy() *RestoreOperation { if in == nil { return nil } out := new(RestoreOperation) in.DeepCopyInto(out) return out } func (in *RestoreOperation) DeepCopyInto(out *RestoreOperation) { *out = *in in.Spec.DeepCopyInto(&out.Spec) in.Status.DeepCopyInto(&out.Status) } type RestoreOperationSpec struct { // RestoreName is the name of the Velero restore this item operation // is associated with. RestoreName string `json:"restoreName"` // RestoreUID is the UID of the Velero restore this item operation // is associated with. RestoreUID string `json:"restoreUID"` // RestoreItemAction is the name of the RestoreItemAction plugin that started the operation RestoreItemAction string `json:"restoreItemAction"` // Kubernetes resource identifier for the item ResourceIdentifier velero.ResourceIdentifier `json:"resourceIdentifier"` // OperationID returned by the RIA plugin OperationID string `json:"operationID"` } func (in *RestoreOperationSpec) DeepCopy() *RestoreOperationSpec { if in == nil { return nil } out := new(RestoreOperationSpec) in.DeepCopyInto(out) return out } func (in *RestoreOperationSpec) DeepCopyInto(out *RestoreOperationSpec) { *out = *in in.ResourceIdentifier.DeepCopyInto(&out.ResourceIdentifier) } ================================================ FILE: pkg/itemoperation/shared.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package itemoperation import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // OperationPhase is the lifecycle phase of a Velero item operation type OperationPhase string type OperationStatus struct { // Phase is the current state of the item operation. Phase OperationPhase `json:"phase,omitempty"` // Error displays the reason for a failed operation Error string `json:"error,omitempty"` // Amount of operation completed (measured in OperationUnits) // i.e. number of bytes transferred for a volume NCompleted int64 `json:"nCompleted,omitempty"` // Total Amount of operation (measured in OperationUnits) // i.e. volume size in bytes NTotal int64 `json:"nTotal,omitempty"` // Units that NCompleted,NTotal are measured in // i.e. "bytes" OperationUnits string `json:"operationUnits,omitempty"` // Description of progress made // i.e. "processing", "Current phase: Running", etc. Description string `json:"description,omitempty"` // Created records the time the item operation was created Created *metav1.Time `json:"created,omitempty"` // Started records the time the item operation was started, if known // +optional // +nullable Started *metav1.Time `json:"started,omitempty"` // Updated records the time the item operation was updated, if known. // +optional // +nullable Updated *metav1.Time `json:"updated,omitempty"` } func (in *OperationStatus) DeepCopy() *OperationStatus { if in == nil { return nil } out := new(OperationStatus) in.DeepCopyInto(out) return out } func (in *OperationStatus) DeepCopyInto(out *OperationStatus) { *out = *in if in.Created != nil { in, out := &in.Created, &out.Created *out = (*in).DeepCopy() } if in.Started != nil { in, out := &in.Started, &out.Started *out = (*in).DeepCopy() } if in.Updated != nil { in, out := &in.Updated, &out.Updated *out = (*in).DeepCopy() } } const ( // OperationPhaseNew means the item operation has been created but not started // by the plugin OperationPhaseNew OperationPhase = "New" // OperationPhaseInProgress means the item operation has been created and started // by the plugin OperationPhaseInProgress OperationPhase = "InProgress" // OperationPhaseCompleted means the item operation was successfully completed // and can be used for restore. OperationPhaseCompleted OperationPhase = "Completed" // OperationPhaseFailed means the item operation ended with an error. OperationPhaseFailed OperationPhase = "Failed" ) ================================================ FILE: pkg/itemoperationmap/backup_operation_map.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package itemoperationmap import ( "bytes" "sync" "github.com/pkg/errors" "github.com/vmware-tanzu/velero/pkg/itemoperation" "github.com/vmware-tanzu/velero/pkg/persistence" "github.com/vmware-tanzu/velero/pkg/util/encode" ) type BackupItemOperationsMap struct { opsMap map[string]*OperationsForBackup opsLock sync.Mutex } // Returns a pointer to a new BackupItemOperationsMap func NewBackupItemOperationsMap() *BackupItemOperationsMap { return &BackupItemOperationsMap{opsMap: make(map[string]*OperationsForBackup)} } // returns a deep copy so we can minimize the time the map is locked func (m *BackupItemOperationsMap) GetOperationsForBackup( backupStore persistence.BackupStore, backupName string) (*OperationsForBackup, error) { var err error // lock operations map m.opsLock.Lock() defer m.opsLock.Unlock() operations, ok := m.opsMap[backupName] if !ok || len(operations.Operations) == 0 { operations = &OperationsForBackup{} operations.Operations, err = backupStore.GetBackupItemOperations(backupName) if err == nil { m.opsMap[backupName] = operations } } return operations.DeepCopy(), err } func (m *BackupItemOperationsMap) PutOperationsForBackup( operations *OperationsForBackup, backupName string) { // lock operations map m.opsLock.Lock() defer m.opsLock.Unlock() if operations != nil { m.opsMap[backupName] = operations } } func (m *BackupItemOperationsMap) DeleteOperationsForBackup(backupName string) { // lock operations map m.opsLock.Lock() defer m.opsLock.Unlock() delete(m.opsMap, backupName) } // UploadProgressAndPutOperationsForBackup will upload the item operations for this backup to // the object store and update the map for this backup with the modified operations func (m *BackupItemOperationsMap) UploadProgressAndPutOperationsForBackup( backupStore persistence.BackupStore, operations *OperationsForBackup, backupName string) error { m.opsLock.Lock() defer m.opsLock.Unlock() if operations == nil { return errors.New("nil operations passed in") } if err := operations.uploadProgress(backupStore, backupName); err != nil { return err } m.opsMap[backupName] = operations return nil } // UpdateForBackup will upload the item operations for this backup to // the object store, if it has changes not yet uploaded func (m *BackupItemOperationsMap) UpdateForBackup(backupStore persistence.BackupStore, backupName string) error { // lock operations map m.opsLock.Lock() defer m.opsLock.Unlock() operations, ok := m.opsMap[backupName] // if operations for this backup aren't found, or if there are no changes // or errors since last update, do nothing if !ok || (!operations.ChangesSinceUpdate && len(operations.ErrsSinceUpdate) == 0) { return nil } if err := operations.uploadProgress(backupStore, backupName); err != nil { return err } return nil } type OperationsForBackup struct { Operations []*itemoperation.BackupOperation ChangesSinceUpdate bool ErrsSinceUpdate []string } func (m *OperationsForBackup) DeepCopy() *OperationsForBackup { if m == nil { return nil } out := new(OperationsForBackup) m.DeepCopyInto(out) return out } func (m *OperationsForBackup) DeepCopyInto(out *OperationsForBackup) { *out = *m if m.Operations != nil { in, out := &m.Operations, &out.Operations *out = make([]*itemoperation.BackupOperation, len(*in)) for i := range *in { if (*in)[i] != nil { in, out := &(*in)[i], &(*out)[i] *out = new(itemoperation.BackupOperation) (*in).DeepCopyInto(*out) } } } if m.ErrsSinceUpdate != nil { in, out := &m.ErrsSinceUpdate, &out.ErrsSinceUpdate *out = make([]string, len(*in)) copy(*out, *in) } } func (m *OperationsForBackup) uploadProgress(backupStore persistence.BackupStore, backupName string) error { if len(m.Operations) > 0 { var backupItemOperations *bytes.Buffer backupItemOperations, errs := encode.ToJSONGzip(m.Operations, "backup item operations list") if errs != nil { return errors.Wrap(errs[0], "error encoding item operations json") } err := backupStore.PutBackupItemOperations(backupName, backupItemOperations) if err != nil { return errors.Wrap(err, "error uploading item operations json") } } m.ChangesSinceUpdate = false m.ErrsSinceUpdate = nil return nil } ================================================ FILE: pkg/itemoperationmap/restore_operation_map.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package itemoperationmap import ( "bytes" "sync" "github.com/pkg/errors" "github.com/vmware-tanzu/velero/pkg/itemoperation" "github.com/vmware-tanzu/velero/pkg/persistence" "github.com/vmware-tanzu/velero/pkg/util/encode" ) type RestoreItemOperationsMap struct { opsMap map[string]*OperationsForRestore opsLock sync.Mutex } // Returns a pointer to a new RestoreItemOperationsMap func NewRestoreItemOperationsMap() *RestoreItemOperationsMap { return &RestoreItemOperationsMap{opsMap: make(map[string]*OperationsForRestore)} } // returns a deep copy so we can minimize the time the map is locked func (m *RestoreItemOperationsMap) GetOperationsForRestore( backupStore persistence.BackupStore, restoreName string) (*OperationsForRestore, error) { var err error // lock operations map m.opsLock.Lock() defer m.opsLock.Unlock() operations, ok := m.opsMap[restoreName] if !ok || len(operations.Operations) == 0 { operations = &OperationsForRestore{} operations.Operations, err = backupStore.GetRestoreItemOperations(restoreName) if err == nil { m.opsMap[restoreName] = operations } } return operations.DeepCopy(), err } func (m *RestoreItemOperationsMap) PutOperationsForRestore( operations *OperationsForRestore, restoreName string) { // lock operations map m.opsLock.Lock() defer m.opsLock.Unlock() if operations != nil { m.opsMap[restoreName] = operations } } func (m *RestoreItemOperationsMap) DeleteOperationsForRestore(restoreName string) { // lock operations map m.opsLock.Lock() defer m.opsLock.Unlock() delete(m.opsMap, restoreName) } // UploadProgressAndPutOperationsForRestore will upload the item operations for this restore to // the object store and update the map for this restore with the modified operations func (m *RestoreItemOperationsMap) UploadProgressAndPutOperationsForRestore( backupStore persistence.BackupStore, operations *OperationsForRestore, restoreName string) error { m.opsLock.Lock() defer m.opsLock.Unlock() if operations == nil { return errors.New("nil operations passed in") } if err := operations.uploadProgress(backupStore, restoreName); err != nil { return err } m.opsMap[restoreName] = operations return nil } // UpdateForRestore will upload the item operations for this restore to // the object store, if it has changes not yet uploaded func (m *RestoreItemOperationsMap) UpdateForRestore(backupStore persistence.BackupStore, restoreName string) error { // lock operations map m.opsLock.Lock() defer m.opsLock.Unlock() operations, ok := m.opsMap[restoreName] // if operations for this restore aren't found, or if there are no changes // or errors since last update, do nothing if !ok || (!operations.ChangesSinceUpdate && len(operations.ErrsSinceUpdate) == 0) { return nil } if err := operations.uploadProgress(backupStore, restoreName); err != nil { return err } return nil } type OperationsForRestore struct { Operations []*itemoperation.RestoreOperation ChangesSinceUpdate bool ErrsSinceUpdate []string } func (m *OperationsForRestore) DeepCopy() *OperationsForRestore { if m == nil { return nil } out := new(OperationsForRestore) m.DeepCopyInto(out) return out } func (m *OperationsForRestore) DeepCopyInto(out *OperationsForRestore) { *out = *m if m.Operations != nil { in, out := &m.Operations, &out.Operations *out = make([]*itemoperation.RestoreOperation, len(*in)) for i := range *in { if (*in)[i] != nil { in, out := &(*in)[i], &(*out)[i] *out = new(itemoperation.RestoreOperation) (*in).DeepCopyInto(*out) } } } if m.ErrsSinceUpdate != nil { in, out := &m.ErrsSinceUpdate, &out.ErrsSinceUpdate *out = make([]string, len(*in)) copy(*out, *in) } } func (m *OperationsForRestore) uploadProgress(backupStore persistence.BackupStore, restoreName string) error { if len(m.Operations) > 0 { var restoreItemOperations *bytes.Buffer restoreItemOperations, errs := encode.ToJSONGzip(m.Operations, "restore item operations list") if errs != nil { return errors.Wrap(errs[0], "error encoding item operations json") } err := backupStore.PutRestoreItemOperations(restoreName, restoreItemOperations) if err != nil { return errors.Wrap(err, "error uploading item operations json") } } m.ChangesSinceUpdate = false m.ErrsSinceUpdate = nil return nil } ================================================ FILE: pkg/kopia/kopia_log.go ================================================ package kopia /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ import ( "context" "io" "github.com/kopia/kopia/repo/logging" "github.com/sirupsen/logrus" "go.uber.org/zap" "go.uber.org/zap/zapcore" ) type kopiaLog struct { module string logger logrus.FieldLogger } type repoLog struct { logger logrus.FieldLogger } // SetupKopiaLog sets the Kopia log handler to the specific context, Kopia modules // call the logger in the context to write logs func SetupKopiaLog(ctx context.Context, logger logrus.FieldLogger) context.Context { return logging.WithLogger(ctx, func(module string) logging.Logger { kpLog := &kopiaLog{module, logger} return zap.New(kpLog).Sugar() }) } func RepositoryLogger(logger logrus.FieldLogger) io.Writer { return &repoLog{logger: logger} } // Enabled decides whether a given logging level is enabled when logging a message func (kl *kopiaLog) Enabled(level zapcore.Level) bool { entry := kl.logger.WithField("null", "null") switch level { case zapcore.DebugLevel: return (entry.Logger.GetLevel() >= logrus.DebugLevel) case zapcore.InfoLevel: return (entry.Logger.GetLevel() >= logrus.InfoLevel) case zapcore.WarnLevel: return (entry.Logger.GetLevel() >= logrus.WarnLevel) case zapcore.ErrorLevel: return (entry.Logger.GetLevel() >= logrus.ErrorLevel) case zapcore.DPanicLevel: return (entry.Logger.GetLevel() >= logrus.PanicLevel) case zapcore.PanicLevel: return (entry.Logger.GetLevel() >= logrus.PanicLevel) case zapcore.FatalLevel: return (entry.Logger.GetLevel() >= logrus.FatalLevel) default: return false } } // With adds structured context to the Core. func (kl *kopiaLog) With(fields []zapcore.Field) zapcore.Core { copied := kl.logrusFields(fields) return &kopiaLog{ module: kl.module, logger: kl.logger.WithFields(copied), } } // Check determines whether the supplied Entry should be logged. If the entry // should be logged, the Core adds itself to the CheckedEntry and returns the result. func (kl *kopiaLog) Check(ent zapcore.Entry, ce *zapcore.CheckedEntry) *zapcore.CheckedEntry { if kl.Enabled(ent.Level) { return ce.AddCore(ent, kl) } return ce } // Write serializes the Entry and any Fields supplied at the log site and writes them to their destination. func (kl *kopiaLog) Write(ent zapcore.Entry, fields []zapcore.Field) error { copied := kl.logrusFieldsForWrite(ent, fields) logger := kl.logger.WithFields(copied) switch ent.Level { case zapcore.DebugLevel: logger.Debug(ent.Message) case zapcore.InfoLevel: logger.Info(ent.Message) case zapcore.WarnLevel: logger.Warn(ent.Message) case zapcore.ErrorLevel: // We see Kopia generates error logs for some normal cases or non-critical // cases. So Kopia's error logs are regarded as warning logs so that they don't // affect Velero's workflow. logger.Warn(ent.Message) case zapcore.DPanicLevel: logger.Panic(ent.Message) case zapcore.PanicLevel: logger.Panic(ent.Message) case zapcore.FatalLevel: logger.Fatal(ent.Message) } return nil } // Sync flushes buffered logs (if any). func (kl *kopiaLog) Sync() error { return nil } func (kl *kopiaLog) logrusFields(fields []zapcore.Field) logrus.Fields { if fields == nil { return logrus.Fields{} } m := zapcore.NewMapObjectEncoder() for _, field := range fields { field.AddTo(m) } return m.Fields } func (kl *kopiaLog) getLogModule() string { return "kopia/" + kl.module } func (kl *kopiaLog) logrusFieldsForWrite(ent zapcore.Entry, fields []zapcore.Field) logrus.Fields { copied := kl.logrusFields(fields) copied["logModule"] = kl.getLogModule() if ent.Caller.Function != "" { copied["function"] = ent.Caller.Function } path := ent.Caller.FullPath() if path != "undefined" { copied["path"] = path } if ent.LoggerName != "" { copied["logger name"] = ent.LoggerName } if ent.Stack != "" { copied["stack"] = ent.Stack } if ent.Level == zap.ErrorLevel { copied["sublevel"] = "error" } return copied } func (rl *repoLog) Write(p []byte) (int, error) { rl.logger.Debug(string(p)) return len(p), nil } ================================================ FILE: pkg/kopia/kopia_log_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package kopia import ( "testing" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap/zapcore" "github.com/vmware-tanzu/velero/pkg/test" ) func TestEnabled(t *testing.T) { testCases := []struct { name string level logrus.Level zapLevel zapcore.Level expected bool }{ { name: "check debug again debug", level: logrus.DebugLevel, zapLevel: zapcore.DebugLevel, expected: true, }, { name: "check debug again info", level: logrus.InfoLevel, zapLevel: zapcore.DebugLevel, expected: false, }, { name: "check info again debug", level: logrus.DebugLevel, zapLevel: zapcore.InfoLevel, expected: true, }, { name: "check info again info", level: logrus.InfoLevel, zapLevel: zapcore.InfoLevel, expected: true, }, { name: "check warn again warn", level: logrus.WarnLevel, zapLevel: zapcore.WarnLevel, expected: true, }, { name: "check info again error", level: logrus.ErrorLevel, zapLevel: zapcore.InfoLevel, expected: false, }, { name: "check error again error", level: logrus.ErrorLevel, zapLevel: zapcore.ErrorLevel, expected: true, }, { name: "check dppanic again panic", level: logrus.PanicLevel, zapLevel: zapcore.DPanicLevel, expected: true, }, { name: "check panic again error", level: logrus.ErrorLevel, zapLevel: zapcore.PanicLevel, expected: true, }, { name: "check fatal again fatal", level: logrus.FatalLevel, zapLevel: zapcore.FatalLevel, expected: true, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { log := kopiaLog{ logger: test.NewLoggerWithLevel(tc.level), } m := log.Enabled(tc.zapLevel) require.Equal(t, tc.expected, m) }) } } func TestLogrusFieldsForWrite(t *testing.T) { testCases := []struct { name string module string zapEntry zapcore.Entry zapFields []zapcore.Field expected logrus.Fields }{ { name: "debug with nil fields", module: "module-01", zapEntry: zapcore.Entry{ Level: zapcore.DebugLevel, }, zapFields: nil, expected: logrus.Fields{ "logModule": "kopia/module-01", }, }, { name: "error with nil fields", module: "module-02", zapEntry: zapcore.Entry{ Level: zapcore.ErrorLevel, }, zapFields: nil, expected: logrus.Fields{ "logModule": "kopia/module-02", "sublevel": "error", }, }, { name: "info with nil string filed", module: "module-03", zapEntry: zapcore.Entry{ Level: zapcore.InfoLevel, }, zapFields: []zapcore.Field{ { Key: "key-01", Type: zapcore.StringType, String: "value-01", }, }, expected: logrus.Fields{ "logModule": "kopia/module-03", "key-01": "value-01", }, }, { name: "info with logger name", module: "module-04", zapEntry: zapcore.Entry{ Level: zapcore.InfoLevel, LoggerName: "logger-name-01", }, zapFields: nil, expected: logrus.Fields{ "logModule": "kopia/module-04", "logger name": "logger-name-01", }, }, { name: "info with function name", module: "module-05", zapEntry: zapcore.Entry{ Level: zapcore.InfoLevel, Caller: zapcore.EntryCaller{ Function: "function-name-01", }, }, zapFields: nil, expected: logrus.Fields{ "logModule": "kopia/module-05", "function": "function-name-01", }, }, { name: "info with undefined path", module: "module-06", zapEntry: zapcore.Entry{ Level: zapcore.InfoLevel, Caller: zapcore.EntryCaller{ Defined: false, }, }, zapFields: nil, expected: logrus.Fields{ "logModule": "kopia/module-06", }, }, { name: "info with defined path", module: "module-06", zapEntry: zapcore.Entry{ Level: zapcore.InfoLevel, Caller: zapcore.EntryCaller{ Defined: true, File: "file-name-01", Line: 100, }, }, zapFields: nil, expected: logrus.Fields{ "logModule": "kopia/module-06", "path": "file-name-01:100", }, }, { name: "info with stack", module: "module-07", zapEntry: zapcore.Entry{ Level: zapcore.InfoLevel, Stack: "fake-stack", }, zapFields: nil, expected: logrus.Fields{ "logModule": "kopia/module-07", "stack": "fake-stack", }, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { log := kopiaLog{ module: tc.module, logger: test.NewLogger(), } m := log.logrusFieldsForWrite(tc.zapEntry, tc.zapFields) require.Equal(t, tc.expected, m) }) } } func TestWrite(t *testing.T) { testCases := []struct { name string ent zapcore.Entry logMessage string logLevel string shouldPanic bool }{ { name: "write debug", ent: zapcore.Entry{ Level: zapcore.DebugLevel, Message: "fake-message", }, logMessage: "fake-message", logLevel: "level=debug", }, { name: "write info", ent: zapcore.Entry{ Level: zapcore.InfoLevel, Message: "fake-message", }, logMessage: "fake-message", logLevel: "level=info", }, { name: "write warn", ent: zapcore.Entry{ Level: zapcore.WarnLevel, Message: "fake-message", }, logMessage: "fake-message", logLevel: "level=warn", }, { name: "write error", ent: zapcore.Entry{ Level: zapcore.ErrorLevel, Message: "fake-message", }, logMessage: "fake-message", logLevel: "level=warn", }, { name: "write DPanic", ent: zapcore.Entry{ Level: zapcore.DPanicLevel, Message: "fake-message", }, logMessage: "fake-message", logLevel: "level=panic", shouldPanic: true, }, { name: "write panic", ent: zapcore.Entry{ Level: zapcore.PanicLevel, Message: "fake-message", }, logMessage: "fake-message", logLevel: "level=panic", shouldPanic: true, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { logMessage := "" log := kopiaLog{ logger: test.NewSingleLogger(&logMessage), } if tc.shouldPanic { defer func() { r := recover() assert.NotNil(t, r) if len(tc.logMessage) > 0 { assert.Contains(t, logMessage, tc.logMessage) } if len(tc.logLevel) > 0 { assert.Contains(t, logMessage, tc.logLevel) } }() } err := log.Write(tc.ent, nil) require.NoError(t, err) if len(tc.logMessage) > 0 { assert.Contains(t, logMessage, tc.logMessage) } if len(tc.logLevel) > 0 { assert.Contains(t, logMessage, tc.logLevel) } }) } } ================================================ FILE: pkg/kuberesource/kuberesource.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package kuberesource import ( "k8s.io/apimachinery/pkg/runtime/schema" ) var ( ClusterRoleBindings = schema.GroupResource{Group: "rbac.authorization.k8s.io", Resource: "clusterrolebindings"} ClusterRoles = schema.GroupResource{Group: "rbac.authorization.k8s.io", Resource: "clusterroles"} CustomResourceDefinitions = schema.GroupResource{Group: "apiextensions.k8s.io", Resource: "customresourcedefinitions"} Jobs = schema.GroupResource{Group: "batch", Resource: "jobs"} Namespaces = schema.GroupResource{Group: "", Resource: "namespaces"} PersistentVolumeClaims = schema.GroupResource{Group: "", Resource: "persistentvolumeclaims"} PersistentVolumes = schema.GroupResource{Group: "", Resource: "persistentvolumes"} Pods = schema.GroupResource{Group: "", Resource: "pods"} ServiceAccounts = schema.GroupResource{Group: "", Resource: "serviceaccounts"} Secrets = schema.GroupResource{Group: "", Resource: "secrets"} VolumeSnapshotClasses = schema.GroupResource{Group: "snapshot.storage.k8s.io", Resource: "volumesnapshotclasses"} VolumeSnapshots = schema.GroupResource{Group: "snapshot.storage.k8s.io", Resource: "volumesnapshots"} VolumeGroupSnapshots = schema.GroupResource{Group: "snapshot.storage.k8s.io", Resource: "volumegroupsnapshots"} VolumeSnapshotContents = schema.GroupResource{Group: "snapshot.storage.k8s.io", Resource: "volumesnapshotcontents"} PriorityClasses = schema.GroupResource{Group: "scheduling.k8s.io", Resource: "priorityclasses"} DataUploads = schema.GroupResource{Group: "velero.io", Resource: "datauploads"} VGSKind = "VolumeGroupSnapshot" ) ================================================ FILE: pkg/label/label.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package label import ( "crypto/sha256" "crypto/sha3" "encoding/hex" "fmt" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/validation" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) // GetValidName converts an input string to valid Kubernetes label string in accordance to rfc1035 DNS Label spec // (https://github.com/kubernetes/community/blob/master/contributors/design-proposals/architecture/identifiers.md) // Length of the label is adjusted basis the DNS1035LabelMaxLength (defined at k8s.io/apimachinery/pkg/util/validation) // If length exceeds, we trim the label name to contain only max allowed characters // Additionally, the last 6 characters of the label name are replaced by the first 6 characters of the sha256 of original label func GetValidName(label string) string { if len(label) <= validation.DNS1035LabelMaxLength { return label } sha := sha256.Sum256([]byte(label)) strSha := hex.EncodeToString(sha[:]) charsFromLabel := validation.DNS1035LabelMaxLength - 6 if charsFromLabel < 0 { // Derive the label name from sha hash in case the DNS1035LabelMaxLength is less than 6 return string(strSha[validation.DNS1035LabelMaxLength]) } return label[:charsFromLabel] + strSha[:6] } // ReturnNameOrHash returns the original name if it is within the DNS1035LabelMaxLength limit, // otherwise it returns the sha3 Sum224 hash(length is 56) of the name. func ReturnNameOrHash(name string) string { if len(name) <= validation.DNS1035LabelMaxLength { return name } hash := sha3.Sum224([]byte(name)) return hex.EncodeToString(hash[:]) } // NewSelectorForBackup returns a Selector based on the backup name. // This is useful for interacting with Listers that need a Selector. func NewSelectorForBackup(name string) labels.Selector { return labels.SelectorFromSet(map[string]string{velerov1api.BackupNameLabel: GetValidName(name)}) } // NewListOptionsForBackup returns a ListOptions based on the backup name. // This is useful for interacting with client-go clients that needs a ListOptions. func NewListOptionsForBackup(name string) metav1.ListOptions { return metav1.ListOptions{ LabelSelector: fmt.Sprintf("%s=%s", velerov1api.BackupNameLabel, GetValidName(name)), } } // NewSelectorForRestore returns a Selector based on the restore name. // This is useful for interacting with Listers that need a Selector. func NewSelectorForRestore(name string) labels.Selector { return labels.SelectorFromSet(map[string]string{velerov1api.RestoreNameLabel: GetValidName(name)}) } ================================================ FILE: pkg/label/label_test.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package label import ( "testing" "github.com/stretchr/testify/assert" ) func TestGetValidLabelName(t *testing.T) { tests := []struct { name string label string expectedLabel string }{ { name: "valid label name should not be modified", label: "short label value", expectedLabel: "short label value", }, { name: "label with more than 63 characters should be modified", label: "this_is_a_very_long_label_value_that_will_be_rejected_by_Kubernetes", expectedLabel: "this_is_a_very_long_label_value_that_will_be_rejected_by_8d0722", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { labelVal := GetValidName(test.label) assert.Equal(t, test.expectedLabel, labelVal) }) } } func TestReturnNameOrHash(t *testing.T) { tests := []struct { name string label string expectedLabel string }{ { name: "valid label name should not be modified", label: "short label value", expectedLabel: "short label value", }, { name: "label with more than 63 characters should be modified", label: "this_is_a_very_long_label_value_that_will_be_rejected_by_Kubernetes", expectedLabel: "1a7399f2d00e268fc12daf431d6667319d1461e2609981070bb7e85c", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { labelVal := ReturnNameOrHash(test.label) assert.Equal(t, test.expectedLabel, labelVal) }) } } func TestNewSelectorForBackup(t *testing.T) { selector := NewSelectorForBackup("my-backup") assert.Equal(t, "velero.io/backup-name=my-backup", selector.String()) } func TestNewListOptionsForBackup(t *testing.T) { option := NewListOptionsForBackup("my-backup") assert.Equal(t, "velero.io/backup-name=my-backup", option.LabelSelector) } ================================================ FILE: pkg/metrics/metrics.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package metrics import ( "time" "github.com/prometheus/client_golang/prometheus" ) // ServerMetrics contains Prometheus metrics for the Velero server. type ServerMetrics struct { metrics map[string]prometheus.Collector } // Metrics returns the metrics map for testing purposes. func (m *ServerMetrics) Metrics() map[string]prometheus.Collector { return m.metrics } const ( metricNamespace = "velero" podVolumeMetricsNamespace = "podVolume" //Velero metrics backupTarballSizeBytesGauge = "backup_tarball_size_bytes" backupTotal = "backup_total" backupAttemptTotal = "backup_attempt_total" backupSuccessTotal = "backup_success_total" backupPartialFailureTotal = "backup_partial_failure_total" backupFailureTotal = "backup_failure_total" backupValidationFailureTotal = "backup_validation_failure_total" backupDurationSeconds = "backup_duration_seconds" backupDeletionAttemptTotal = "backup_deletion_attempt_total" backupDeletionSuccessTotal = "backup_deletion_success_total" backupDeletionFailureTotal = "backup_deletion_failure_total" backupLastSuccessfulTimestamp = "backup_last_successful_timestamp" backupItemsTotalGauge = "backup_items_total" backupItemsErrorsGauge = "backup_items_errors" backupWarningTotal = "backup_warning_total" backupLastStatus = "backup_last_status" backupLocationStatus = "backup_location_status_gauge" restoreTotal = "restore_total" restoreAttemptTotal = "restore_attempt_total" restoreValidationFailedTotal = "restore_validation_failed_total" restoreSuccessTotal = "restore_success_total" restorePartialFailureTotal = "restore_partial_failure_total" restoreFailedTotal = "restore_failed_total" volumeSnapshotAttemptTotal = "volume_snapshot_attempt_total" volumeSnapshotSuccessTotal = "volume_snapshot_success_total" volumeSnapshotFailureTotal = "volume_snapshot_failure_total" csiSnapshotAttemptTotal = "csi_snapshot_attempt_total" csiSnapshotSuccessTotal = "csi_snapshot_success_total" csiSnapshotFailureTotal = "csi_snapshot_failure_total" // pod volume metrics podVolumeBackupEnqueueTotal = "pod_volume_backup_enqueue_count" podVolumeBackupDequeueTotal = "pod_volume_backup_dequeue_count" podVolumeOperationLatencySeconds = "pod_volume_operation_latency_seconds" podVolumeOperationLatencyGaugeSeconds = "pod_volume_operation_latency_seconds_gauge" // data mover metrics DataUploadSuccessTotal = "data_upload_success_total" DataUploadFailureTotal = "data_upload_failure_total" DataUploadCancelTotal = "data_upload_cancel_total" DataDownloadSuccessTotal = "data_download_success_total" DataDownloadFailureTotal = "data_download_failure_total" DataDownloadCancelTotal = "data_download_cancel_total" // schedule metrics scheduleExpectedIntervalSeconds = "schedule_expected_interval_seconds" // repo maintenance metrics repoMaintenanceSuccessTotal = "repo_maintenance_success_total" repoMaintenanceFailureTotal = "repo_maintenance_failure_total" // repoMaintenanceDurationSeconds tracks the distribution of maintenance job durations. // Each completed job's duration is recorded in the appropriate bucket, allowing // analysis of individual job performance and trending over time. repoMaintenanceDurationSeconds = "repo_maintenance_duration_seconds" // Labels nodeMetricLabel = "node" podVolumeOperationLabel = "operation" bslNameLabel = "backup_location_name" pvbNameLabel = "pod_volume_backup" scheduleLabel = "schedule" backupNameLabel = "backupName" repositoryNameLabel = "repository_name" // metrics values BackupLastStatusSucc int64 = 1 BackupLastStatusFailure int64 = 0 ) // NewServerMetrics returns new ServerMetrics func NewServerMetrics() *ServerMetrics { return &ServerMetrics{ metrics: map[string]prometheus.Collector{ backupTarballSizeBytesGauge: prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: metricNamespace, Name: backupTarballSizeBytesGauge, Help: "Size, in bytes, of a backup", }, []string{scheduleLabel}, ), backupLastSuccessfulTimestamp: prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: metricNamespace, Name: backupLastSuccessfulTimestamp, Help: "Last time a backup ran successfully, Unix timestamp in seconds", }, []string{scheduleLabel}, ), backupTotal: prometheus.NewGauge( prometheus.GaugeOpts{ Namespace: metricNamespace, Name: backupTotal, Help: "Current number of existent backups", }, ), backupAttemptTotal: prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: metricNamespace, Name: backupAttemptTotal, Help: "Total number of attempted backups", }, []string{scheduleLabel}, ), backupSuccessTotal: prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: metricNamespace, Name: backupSuccessTotal, Help: "Total number of successful backups", }, []string{scheduleLabel}, ), backupPartialFailureTotal: prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: metricNamespace, Name: backupPartialFailureTotal, Help: "Total number of partially failed backups", }, []string{scheduleLabel}, ), backupFailureTotal: prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: metricNamespace, Name: backupFailureTotal, Help: "Total number of failed backups", }, []string{scheduleLabel}, ), backupValidationFailureTotal: prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: metricNamespace, Name: backupValidationFailureTotal, Help: "Total number of validation failed backups", }, []string{scheduleLabel}, ), backupDeletionAttemptTotal: prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: metricNamespace, Name: backupDeletionAttemptTotal, Help: "Total number of attempted backup deletions", }, []string{scheduleLabel}, ), backupDeletionSuccessTotal: prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: metricNamespace, Name: backupDeletionSuccessTotal, Help: "Total number of successful backup deletions", }, []string{scheduleLabel}, ), backupDeletionFailureTotal: prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: metricNamespace, Name: backupDeletionFailureTotal, Help: "Total number of failed backup deletions", }, []string{scheduleLabel}, ), backupDurationSeconds: prometheus.NewHistogramVec( prometheus.HistogramOpts{ Namespace: metricNamespace, Name: backupDurationSeconds, Help: "Time taken to complete backup, in seconds", Buckets: []float64{ toSeconds(1 * time.Minute), toSeconds(5 * time.Minute), toSeconds(10 * time.Minute), toSeconds(15 * time.Minute), toSeconds(30 * time.Minute), toSeconds(1 * time.Hour), toSeconds(2 * time.Hour), toSeconds(3 * time.Hour), toSeconds(4 * time.Hour), }, }, []string{scheduleLabel}, ), backupItemsTotalGauge: prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: metricNamespace, Name: backupItemsTotalGauge, Help: "Total number of items backed up", }, []string{scheduleLabel}, ), backupItemsErrorsGauge: prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: metricNamespace, Name: backupItemsErrorsGauge, Help: "Total number of errors encountered during backup", }, []string{scheduleLabel}, ), backupWarningTotal: prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: metricNamespace, Name: backupWarningTotal, Help: "Total number of warned backups", }, []string{scheduleLabel}, ), backupLastStatus: prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: metricNamespace, Name: backupLastStatus, Help: "Last status of the backup. A value of 1 is success, 0 is failure", }, []string{scheduleLabel}, ), backupLocationStatus: prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: metricNamespace, Name: backupLocationStatus, Help: "The status of backup location. A value of 1 is available, 0 is unavailable", }, []string{bslNameLabel}, ), restoreTotal: prometheus.NewGauge( prometheus.GaugeOpts{ Namespace: metricNamespace, Name: restoreTotal, Help: "Current number of existent restores", }, ), restoreAttemptTotal: prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: metricNamespace, Name: restoreAttemptTotal, Help: "Total number of attempted restores", }, []string{scheduleLabel}, ), restoreSuccessTotal: prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: metricNamespace, Name: restoreSuccessTotal, Help: "Total number of successful restores", }, []string{scheduleLabel}, ), restorePartialFailureTotal: prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: metricNamespace, Name: restorePartialFailureTotal, Help: "Total number of partially failed restores", }, []string{scheduleLabel}, ), restoreFailedTotal: prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: metricNamespace, Name: restoreFailedTotal, Help: "Total number of failed restores", }, []string{scheduleLabel}, ), restoreValidationFailedTotal: prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: metricNamespace, Name: restoreValidationFailedTotal, Help: "Total number of failed restores failing validations", }, []string{scheduleLabel}, ), volumeSnapshotAttemptTotal: prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: metricNamespace, Name: volumeSnapshotAttemptTotal, Help: "Total number of attempted volume snapshots", }, []string{scheduleLabel}, ), volumeSnapshotSuccessTotal: prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: metricNamespace, Name: volumeSnapshotSuccessTotal, Help: "Total number of successful volume snapshots", }, []string{scheduleLabel}, ), volumeSnapshotFailureTotal: prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: metricNamespace, Name: volumeSnapshotFailureTotal, Help: "Total number of failed volume snapshots", }, []string{scheduleLabel}, ), csiSnapshotAttemptTotal: prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: metricNamespace, Name: csiSnapshotAttemptTotal, Help: "Total number of CSI attempted volume snapshots", }, []string{scheduleLabel, backupNameLabel}, ), csiSnapshotSuccessTotal: prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: metricNamespace, Name: csiSnapshotSuccessTotal, Help: "Total number of CSI successful volume snapshots", }, []string{scheduleLabel, backupNameLabel}, ), csiSnapshotFailureTotal: prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: metricNamespace, Name: csiSnapshotFailureTotal, Help: "Total number of CSI failed volume snapshots", }, []string{scheduleLabel, backupNameLabel}, ), scheduleExpectedIntervalSeconds: prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: metricNamespace, Name: scheduleExpectedIntervalSeconds, Help: "Expected interval between consecutive scheduled backups, in seconds", }, []string{scheduleLabel}, ), repoMaintenanceSuccessTotal: prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: metricNamespace, Name: repoMaintenanceSuccessTotal, Help: "Total number of successful repo maintenance jobs", }, []string{repositoryNameLabel}, ), repoMaintenanceFailureTotal: prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: metricNamespace, Name: repoMaintenanceFailureTotal, Help: "Total number of failed repo maintenance jobs", }, []string{repositoryNameLabel}, ), repoMaintenanceDurationSeconds: prometheus.NewHistogramVec( prometheus.HistogramOpts{ Namespace: metricNamespace, Name: repoMaintenanceDurationSeconds, Help: "Time taken to complete repo maintenance jobs, in seconds", Buckets: []float64{ toSeconds(1 * time.Minute), toSeconds(5 * time.Minute), toSeconds(10 * time.Minute), toSeconds(15 * time.Minute), toSeconds(30 * time.Minute), toSeconds(1 * time.Hour), toSeconds(2 * time.Hour), toSeconds(3 * time.Hour), toSeconds(4 * time.Hour), }, }, []string{repositoryNameLabel}, ), }, } } func NewNodeMetrics() *ServerMetrics { return &ServerMetrics{ metrics: map[string]prometheus.Collector{ podVolumeBackupEnqueueTotal: prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: podVolumeMetricsNamespace, Name: podVolumeBackupEnqueueTotal, Help: "Total number of pod_volume_backup objects enqueued", }, []string{nodeMetricLabel}, ), podVolumeBackupDequeueTotal: prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: podVolumeMetricsNamespace, Name: podVolumeBackupDequeueTotal, Help: "Total number of pod_volume_backup objects dequeued", }, []string{nodeMetricLabel}, ), podVolumeOperationLatencyGaugeSeconds: prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: podVolumeMetricsNamespace, Name: podVolumeOperationLatencyGaugeSeconds, Help: "Gauge metric indicating time taken, in seconds, to perform pod volume operations", }, []string{nodeMetricLabel, podVolumeOperationLabel, backupNameLabel, pvbNameLabel}, ), podVolumeOperationLatencySeconds: prometheus.NewHistogramVec( prometheus.HistogramOpts{ Namespace: podVolumeMetricsNamespace, Name: podVolumeOperationLatencySeconds, Help: "Time taken to complete pod volume operations, in seconds", Buckets: []float64{ toSeconds(1 * time.Minute), toSeconds(5 * time.Minute), toSeconds(10 * time.Minute), toSeconds(15 * time.Minute), toSeconds(30 * time.Minute), toSeconds(1 * time.Hour), toSeconds(2 * time.Hour), toSeconds(3 * time.Hour), toSeconds(4 * time.Hour), }, }, []string{nodeMetricLabel, podVolumeOperationLabel, backupNameLabel, pvbNameLabel}, ), DataUploadSuccessTotal: prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: podVolumeMetricsNamespace, Name: DataUploadSuccessTotal, Help: "Total number of successful uploaded snapshots", }, []string{nodeMetricLabel}, ), DataUploadFailureTotal: prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: podVolumeMetricsNamespace, Name: DataUploadFailureTotal, Help: "Total number of failed uploaded snapshots", }, []string{nodeMetricLabel}, ), DataUploadCancelTotal: prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: podVolumeMetricsNamespace, Name: DataUploadCancelTotal, Help: "Total number of canceled uploaded snapshots", }, []string{nodeMetricLabel}, ), DataDownloadSuccessTotal: prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: podVolumeMetricsNamespace, Name: DataDownloadSuccessTotal, Help: "Total number of successful downloaded snapshots", }, []string{nodeMetricLabel}, ), DataDownloadFailureTotal: prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: podVolumeMetricsNamespace, Name: DataDownloadFailureTotal, Help: "Total number of failed downloaded snapshots", }, []string{nodeMetricLabel}, ), DataDownloadCancelTotal: prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: podVolumeMetricsNamespace, Name: DataDownloadCancelTotal, Help: "Total number of canceled downloaded snapshots", }, []string{nodeMetricLabel}, ), }, } } // RegisterAllMetrics registers all prometheus metrics. func (m *ServerMetrics) RegisterAllMetrics() { for _, pm := range m.metrics { prometheus.MustRegister(pm) } } // InitSchedule initializes counter metrics of a schedule. func (m *ServerMetrics) InitSchedule(scheduleName string) { if c, ok := m.metrics[backupAttemptTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(scheduleName).Add(0) } if c, ok := m.metrics[backupSuccessTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(scheduleName).Add(0) } if c, ok := m.metrics[backupPartialFailureTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(scheduleName).Add(0) } if c, ok := m.metrics[backupFailureTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(scheduleName).Add(0) } if c, ok := m.metrics[backupValidationFailureTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(scheduleName).Add(0) } if c, ok := m.metrics[backupDeletionAttemptTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(scheduleName).Add(0) } if c, ok := m.metrics[backupDeletionSuccessTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(scheduleName).Add(0) } if c, ok := m.metrics[backupDeletionFailureTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(scheduleName).Add(0) } if c, ok := m.metrics[backupItemsTotalGauge].(*prometheus.GaugeVec); ok { c.WithLabelValues(scheduleName).Add(0) } if c, ok := m.metrics[backupItemsErrorsGauge].(*prometheus.GaugeVec); ok { c.WithLabelValues(scheduleName).Add(0) } if c, ok := m.metrics[backupWarningTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(scheduleName).Add(0) } if c, ok := m.metrics[backupLastStatus].(*prometheus.GaugeVec); ok { c.WithLabelValues(scheduleName).Set(float64(1)) } if c, ok := m.metrics[restoreAttemptTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(scheduleName).Add(0) } if c, ok := m.metrics[restorePartialFailureTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(scheduleName).Add(0) } if c, ok := m.metrics[restoreFailedTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(scheduleName).Add(0) } if c, ok := m.metrics[restoreSuccessTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(scheduleName).Add(0) } if c, ok := m.metrics[restoreValidationFailedTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(scheduleName).Add(0) } if c, ok := m.metrics[volumeSnapshotSuccessTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(scheduleName).Add(0) } if c, ok := m.metrics[volumeSnapshotAttemptTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(scheduleName).Add(0) } if c, ok := m.metrics[volumeSnapshotFailureTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(scheduleName).Add(0) } if c, ok := m.metrics[csiSnapshotAttemptTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(scheduleName, "").Add(0) } if c, ok := m.metrics[csiSnapshotSuccessTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(scheduleName, "").Add(0) } if c, ok := m.metrics[csiSnapshotFailureTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(scheduleName, "").Add(0) } } // RemoveSchedule removes metrics associated with a specified schedule. func (m *ServerMetrics) RemoveSchedule(scheduleName string) { if g, ok := m.metrics[backupTarballSizeBytesGauge].(*prometheus.GaugeVec); ok { g.DeleteLabelValues(scheduleName) } if c, ok := m.metrics[backupAttemptTotal].(*prometheus.CounterVec); ok { c.DeleteLabelValues(scheduleName) } if c, ok := m.metrics[backupSuccessTotal].(*prometheus.CounterVec); ok { c.DeleteLabelValues(scheduleName) } if c, ok := m.metrics[backupPartialFailureTotal].(*prometheus.CounterVec); ok { c.DeleteLabelValues(scheduleName) } if c, ok := m.metrics[backupFailureTotal].(*prometheus.CounterVec); ok { c.DeleteLabelValues(scheduleName) } if c, ok := m.metrics[backupValidationFailureTotal].(*prometheus.CounterVec); ok { c.DeleteLabelValues(scheduleName) } if h, ok := m.metrics[backupDurationSeconds].(*prometheus.HistogramVec); ok { h.DeleteLabelValues(scheduleName) } if c, ok := m.metrics[backupDeletionAttemptTotal].(*prometheus.CounterVec); ok { c.DeleteLabelValues(scheduleName) } if c, ok := m.metrics[backupDeletionSuccessTotal].(*prometheus.CounterVec); ok { c.DeleteLabelValues(scheduleName) } if c, ok := m.metrics[backupDeletionFailureTotal].(*prometheus.CounterVec); ok { c.DeleteLabelValues(scheduleName) } if g, ok := m.metrics[backupLastSuccessfulTimestamp].(*prometheus.GaugeVec); ok { g.DeleteLabelValues(scheduleName) } if c, ok := m.metrics[backupItemsTotalGauge].(*prometheus.GaugeVec); ok { c.DeleteLabelValues(scheduleName) } if c, ok := m.metrics[backupItemsErrorsGauge].(*prometheus.GaugeVec); ok { c.DeleteLabelValues(scheduleName) } if c, ok := m.metrics[backupWarningTotal].(*prometheus.CounterVec); ok { c.DeleteLabelValues(scheduleName) } if c, ok := m.metrics[backupLastStatus].(*prometheus.GaugeVec); ok { c.DeleteLabelValues(scheduleName) } if c, ok := m.metrics[restoreAttemptTotal].(*prometheus.CounterVec); ok { c.DeleteLabelValues(scheduleName) } if c, ok := m.metrics[restorePartialFailureTotal].(*prometheus.CounterVec); ok { c.DeleteLabelValues(scheduleName) } if c, ok := m.metrics[restoreFailedTotal].(*prometheus.CounterVec); ok { c.DeleteLabelValues(scheduleName) } if c, ok := m.metrics[restoreSuccessTotal].(*prometheus.CounterVec); ok { c.DeleteLabelValues(scheduleName) } if c, ok := m.metrics[restoreValidationFailedTotal].(*prometheus.CounterVec); ok { c.DeleteLabelValues(scheduleName) } if c, ok := m.metrics[volumeSnapshotSuccessTotal].(*prometheus.CounterVec); ok { c.DeleteLabelValues(scheduleName) } if c, ok := m.metrics[volumeSnapshotAttemptTotal].(*prometheus.CounterVec); ok { c.DeleteLabelValues(scheduleName) } if c, ok := m.metrics[volumeSnapshotFailureTotal].(*prometheus.CounterVec); ok { c.DeleteLabelValues(scheduleName) } if c, ok := m.metrics[csiSnapshotAttemptTotal].(*prometheus.CounterVec); ok { c.DeleteLabelValues(scheduleName, "") } if c, ok := m.metrics[csiSnapshotSuccessTotal].(*prometheus.CounterVec); ok { c.DeleteLabelValues(scheduleName, "") } if c, ok := m.metrics[csiSnapshotFailureTotal].(*prometheus.CounterVec); ok { c.DeleteLabelValues(scheduleName, "") } if g, ok := m.metrics[scheduleExpectedIntervalSeconds].(*prometheus.GaugeVec); ok { g.DeleteLabelValues(scheduleName) } } // InitMetricsForNode initializes counter metrics for a node. func (m *ServerMetrics) InitMetricsForNode(node string) { if c, ok := m.metrics[podVolumeBackupEnqueueTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(node).Add(0) } if c, ok := m.metrics[podVolumeBackupDequeueTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(node).Add(0) } if c, ok := m.metrics[DataUploadSuccessTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(node).Add(0) } if c, ok := m.metrics[DataUploadFailureTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(node).Add(0) } if c, ok := m.metrics[DataUploadCancelTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(node).Add(0) } if c, ok := m.metrics[DataDownloadSuccessTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(node).Add(0) } if c, ok := m.metrics[DataDownloadFailureTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(node).Add(0) } if c, ok := m.metrics[DataDownloadCancelTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(node).Add(0) } } // RegisterPodVolumeBackupEnqueue records enqueuing of a PodVolumeBackup object. func (m *ServerMetrics) RegisterPodVolumeBackupEnqueue(node string) { if c, ok := m.metrics[podVolumeBackupEnqueueTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(node).Inc() } } // RegisterPodVolumeBackupDequeue records dequeuing of a PodVolumeBackup object. func (m *ServerMetrics) RegisterPodVolumeBackupDequeue(node string) { if c, ok := m.metrics[podVolumeBackupDequeueTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(node).Inc() } } // RegisterDataUploadSuccess records successful uploaded snapshots. func (m *ServerMetrics) RegisterDataUploadSuccess(node string) { if c, ok := m.metrics[DataUploadSuccessTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(node).Inc() } } // RegisterDataUploadFailure records failed uploaded snapshots. func (m *ServerMetrics) RegisterDataUploadFailure(node string) { if c, ok := m.metrics[DataUploadFailureTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(node).Inc() } } // RegisterDataUploadCancel records canceled uploaded snapshots. func (m *ServerMetrics) RegisterDataUploadCancel(node string) { if c, ok := m.metrics[DataUploadCancelTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(node).Inc() } } // RegisterDataDownloadSuccess records successful downloaded snapshots. func (m *ServerMetrics) RegisterDataDownloadSuccess(node string) { if c, ok := m.metrics[DataDownloadSuccessTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(node).Inc() } } // RegisterDataDownloadFailure records failed downloaded snapshots. func (m *ServerMetrics) RegisterDataDownloadFailure(node string) { if c, ok := m.metrics[DataDownloadFailureTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(node).Inc() } } // RegisterDataDownloadCancel records canceled downloaded snapshots. func (m *ServerMetrics) RegisterDataDownloadCancel(node string) { if c, ok := m.metrics[DataDownloadCancelTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(node).Inc() } } // ObservePodVolumeOpLatency records the number of seconds a pod volume operation took. func (m *ServerMetrics) ObservePodVolumeOpLatency(node, pvbName, opName, backupName string, seconds float64) { if h, ok := m.metrics[podVolumeOperationLatencySeconds].(*prometheus.HistogramVec); ok { h.WithLabelValues(node, opName, backupName, pvbName).Observe(seconds) } } // RegisterPodVolumeOpLatencyGauge registers the pod volume operation latency as a gauge metric. func (m *ServerMetrics) RegisterPodVolumeOpLatencyGauge(node, pvbName, opName, backupName string, seconds float64) { if g, ok := m.metrics[podVolumeOperationLatencyGaugeSeconds].(*prometheus.GaugeVec); ok { g.WithLabelValues(node, opName, backupName, pvbName).Set(seconds) } } // SetBackupTarballSizeBytesGauge records the size, in bytes, of a backup tarball. func (m *ServerMetrics) SetBackupTarballSizeBytesGauge(backupSchedule string, size int64) { if g, ok := m.metrics[backupTarballSizeBytesGauge].(*prometheus.GaugeVec); ok { g.WithLabelValues(backupSchedule).Set(float64(size)) } } // SetBackupLastSuccessfulTimestamp records the last time a backup ran successfully, Unix timestamp in seconds func (m *ServerMetrics) SetBackupLastSuccessfulTimestamp(backupSchedule string, time time.Time) { if g, ok := m.metrics[backupLastSuccessfulTimestamp].(*prometheus.GaugeVec); ok { g.WithLabelValues(backupSchedule).Set(float64(time.Unix())) } } // SetScheduleExpectedIntervalSeconds records the expected interval in seconds, // between consecutive backups for a schedule. func (m *ServerMetrics) SetScheduleExpectedIntervalSeconds(scheduleName string, seconds float64) { if g, ok := m.metrics[scheduleExpectedIntervalSeconds].(*prometheus.GaugeVec); ok { g.WithLabelValues(scheduleName).Set(seconds) } } // SetBackupTotal records the current number of existent backups. func (m *ServerMetrics) SetBackupTotal(numberOfBackups int64) { if g, ok := m.metrics[backupTotal].(prometheus.Gauge); ok { g.Set(float64(numberOfBackups)) } } // RegisterBackupAttempt records an backup attempt. func (m *ServerMetrics) RegisterBackupAttempt(backupSchedule string) { if c, ok := m.metrics[backupAttemptTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(backupSchedule).Inc() } } // RegisterBackupSuccess records a successful completion of a backup. func (m *ServerMetrics) RegisterBackupSuccess(backupSchedule string) { if c, ok := m.metrics[backupSuccessTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(backupSchedule).Inc() } m.SetBackupLastSuccessfulTimestamp(backupSchedule, time.Now()) } // RegisterBackupPartialFailure records a partially failed backup. func (m *ServerMetrics) RegisterBackupPartialFailure(backupSchedule string) { if c, ok := m.metrics[backupPartialFailureTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(backupSchedule).Inc() } } // RegisterBackupFailed records a failed backup. func (m *ServerMetrics) RegisterBackupFailed(backupSchedule string) { if c, ok := m.metrics[backupFailureTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(backupSchedule).Inc() } } // RegisterBackupValidationFailure records a validation failed backup. func (m *ServerMetrics) RegisterBackupValidationFailure(backupSchedule string) { if c, ok := m.metrics[backupValidationFailureTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(backupSchedule).Inc() } } // RegisterBackupDuration records the number of seconds a backup took. func (m *ServerMetrics) RegisterBackupDuration(backupSchedule string, seconds float64) { if c, ok := m.metrics[backupDurationSeconds].(*prometheus.HistogramVec); ok { c.WithLabelValues(backupSchedule).Observe(seconds) } } // RegisterBackupDeletionAttempt records the number of attempted backup deletions func (m *ServerMetrics) RegisterBackupDeletionAttempt(backupSchedule string) { if c, ok := m.metrics[backupDeletionAttemptTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(backupSchedule).Inc() } } // RegisterBackupDeletionFailed records the number of failed backup deletions func (m *ServerMetrics) RegisterBackupDeletionFailed(backupSchedule string) { if c, ok := m.metrics[backupDeletionFailureTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(backupSchedule).Inc() } } // RegisterBackupDeletionSuccess records the number of successful backup deletions func (m *ServerMetrics) RegisterBackupDeletionSuccess(backupSchedule string) { if c, ok := m.metrics[backupDeletionSuccessTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(backupSchedule).Inc() } } // RegisterBackupItemsTotalGauge records the number of items to be backed up. func (m *ServerMetrics) RegisterBackupItemsTotalGauge(backupSchedule string, items int) { if c, ok := m.metrics[backupItemsTotalGauge].(*prometheus.GaugeVec); ok { c.WithLabelValues(backupSchedule).Set(float64(items)) } } // RegisterBackupItemsErrorsGauge records the number of all error messages that were generated during // execution of the backup. func (m *ServerMetrics) RegisterBackupItemsErrorsGauge(backupSchedule string, items int) { if c, ok := m.metrics[backupItemsErrorsGauge].(*prometheus.GaugeVec); ok { c.WithLabelValues(backupSchedule).Set(float64(items)) } } // RegisterBackupWarning records a warned backup. func (m *ServerMetrics) RegisterBackupWarning(backupSchedule string) { if c, ok := m.metrics[backupWarningTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(backupSchedule).Inc() } } // RegisterBackupLastStatus records the last status of the backup. func (m *ServerMetrics) RegisterBackupLastStatus(backupSchedule string, lastStatus int64) { if g, ok := m.metrics[backupLastStatus].(*prometheus.GaugeVec); ok { g.WithLabelValues(backupSchedule).Set(float64(lastStatus)) } } // toSeconds translates a time.Duration value into a float64 // representing the number of seconds in that duration. func toSeconds(d time.Duration) float64 { return float64(d / time.Second) } // SetRestoreTotal records the current number of existent restores. func (m *ServerMetrics) SetRestoreTotal(numberOfRestores int64) { if g, ok := m.metrics[restoreTotal].(prometheus.Gauge); ok { g.Set(float64(numberOfRestores)) } } // RegisterRestoreAttempt records an attempt to restore a backup. func (m *ServerMetrics) RegisterRestoreAttempt(backupSchedule string) { if c, ok := m.metrics[restoreAttemptTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(backupSchedule).Inc() } } // RegisterRestoreSuccess records a successful (maybe partial) completion of a restore. func (m *ServerMetrics) RegisterRestoreSuccess(backupSchedule string) { if c, ok := m.metrics[restoreSuccessTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(backupSchedule).Inc() } } // RegisterRestorePartialFailure records a restore that partially failed. func (m *ServerMetrics) RegisterRestorePartialFailure(backupSchedule string) { if c, ok := m.metrics[restorePartialFailureTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(backupSchedule).Inc() } } // RegisterRestoreFailed records a restore that failed. func (m *ServerMetrics) RegisterRestoreFailed(backupSchedule string) { if c, ok := m.metrics[restoreFailedTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(backupSchedule).Inc() } } // RegisterRestoreValidationFailed records a restore that failed validation. func (m *ServerMetrics) RegisterRestoreValidationFailed(backupSchedule string) { if c, ok := m.metrics[restoreValidationFailedTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(backupSchedule).Inc() } } // RegisterVolumeSnapshotAttempts records an attempt to snapshot a volume. func (m *ServerMetrics) RegisterVolumeSnapshotAttempts(backupSchedule string, volumeSnapshotsAttempted int) { if c, ok := m.metrics[volumeSnapshotAttemptTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(backupSchedule).Add(float64(volumeSnapshotsAttempted)) } } // RegisterVolumeSnapshotSuccesses records a completed volume snapshot. func (m *ServerMetrics) RegisterVolumeSnapshotSuccesses(backupSchedule string, volumeSnapshotsCompleted int) { if c, ok := m.metrics[volumeSnapshotSuccessTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(backupSchedule).Add(float64(volumeSnapshotsCompleted)) } } // RegisterVolumeSnapshotFailures records a failed volume snapshot. func (m *ServerMetrics) RegisterVolumeSnapshotFailures(backupSchedule string, volumeSnapshotsFailed int) { if c, ok := m.metrics[volumeSnapshotFailureTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(backupSchedule).Add(float64(volumeSnapshotsFailed)) } } // RegisterCSISnapshotAttempts records an attempt to snapshot a volume by CSI plugin. func (m *ServerMetrics) RegisterCSISnapshotAttempts(backupSchedule, backupName string, csiSnapshotsAttempted int) { if c, ok := m.metrics[csiSnapshotAttemptTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(backupSchedule, backupName).Add(float64(csiSnapshotsAttempted)) } } // RegisterCSISnapshotSuccesses records a completed volume snapshot by CSI plugin. func (m *ServerMetrics) RegisterCSISnapshotSuccesses(backupSchedule, backupName string, csiSnapshotCompleted int) { if c, ok := m.metrics[csiSnapshotSuccessTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(backupSchedule, backupName).Add(float64(csiSnapshotCompleted)) } } // RegisterCSISnapshotFailures records a failed volume snapshot by CSI plugin. func (m *ServerMetrics) RegisterCSISnapshotFailures(backupSchedule, backupName string, csiSnapshotsFailed int) { if c, ok := m.metrics[csiSnapshotFailureTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(backupSchedule, backupName).Add(float64(csiSnapshotsFailed)) } } // RegisterBackupLocationAvailable records the availability of a backup location. func (m *ServerMetrics) RegisterBackupLocationAvailable(backupLocationName string) { if g, ok := m.metrics[backupLocationStatus].(*prometheus.GaugeVec); ok { g.WithLabelValues(backupLocationName).Set(float64(1)) } } // RegisterBackupLocationUnavailable records the availability of a backup location. func (m *ServerMetrics) RegisterBackupLocationUnavailable(backupLocationName string) { if g, ok := m.metrics[backupLocationStatus].(*prometheus.GaugeVec); ok { g.WithLabelValues(backupLocationName).Set(float64(0)) } } // RegisterRepoMaintenanceSuccess records a successful repo maintenance job. func (m *ServerMetrics) RegisterRepoMaintenanceSuccess(repositoryName string) { if c, ok := m.metrics[repoMaintenanceSuccessTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(repositoryName).Inc() } } // RegisterRepoMaintenanceFailure records a failed repo maintenance job. func (m *ServerMetrics) RegisterRepoMaintenanceFailure(repositoryName string) { if c, ok := m.metrics[repoMaintenanceFailureTotal].(*prometheus.CounterVec); ok { c.WithLabelValues(repositoryName).Inc() } } // ObserveRepoMaintenanceDuration records the number of seconds a repo maintenance job took. func (m *ServerMetrics) ObserveRepoMaintenanceDuration(repositoryName string, seconds float64) { if h, ok := m.metrics[repoMaintenanceDurationSeconds].(*prometheus.HistogramVec); ok { h.WithLabelValues(repositoryName).Observe(seconds) } } ================================================ FILE: pkg/metrics/metrics_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package metrics import ( "testing" "time" "github.com/prometheus/client_golang/prometheus" dto "github.com/prometheus/client_model/go" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) // TestBackupMetricsWithAdhocBackups verifies that metrics are properly recorded // for both scheduled and adhoc (non-scheduled) backups. func TestBackupMetricsWithAdhocBackups(t *testing.T) { tests := []struct { name string scheduleName string expectedLabel string description string }{ { name: "scheduled backup metrics", scheduleName: "daily-backup", expectedLabel: "daily-backup", description: "Metrics should be recorded with the schedule name label", }, { name: "adhoc backup metrics with empty schedule", scheduleName: "", expectedLabel: "", description: "Metrics should be recorded with empty schedule label for adhoc backups", }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { // Create a new metrics instance m := NewServerMetrics() // Test backup attempt metric t.Run("RegisterBackupAttempt", func(t *testing.T) { m.RegisterBackupAttempt(tc.scheduleName) metric := getMetricValue(t, m.metrics[backupAttemptTotal].(*prometheus.CounterVec), tc.expectedLabel) assert.Equal(t, float64(1), metric, tc.description) }) // Test backup success metric t.Run("RegisterBackupSuccess", func(t *testing.T) { m.RegisterBackupSuccess(tc.scheduleName) metric := getMetricValue(t, m.metrics[backupSuccessTotal].(*prometheus.CounterVec), tc.expectedLabel) assert.Equal(t, float64(1), metric, tc.description) }) // Test backup failure metric t.Run("RegisterBackupFailed", func(t *testing.T) { m.RegisterBackupFailed(tc.scheduleName) metric := getMetricValue(t, m.metrics[backupFailureTotal].(*prometheus.CounterVec), tc.expectedLabel) assert.Equal(t, float64(1), metric, tc.description) }) // Test backup partial failure metric t.Run("RegisterBackupPartialFailure", func(t *testing.T) { m.RegisterBackupPartialFailure(tc.scheduleName) metric := getMetricValue(t, m.metrics[backupPartialFailureTotal].(*prometheus.CounterVec), tc.expectedLabel) assert.Equal(t, float64(1), metric, tc.description) }) // Test backup validation failure metric t.Run("RegisterBackupValidationFailure", func(t *testing.T) { m.RegisterBackupValidationFailure(tc.scheduleName) metric := getMetricValue(t, m.metrics[backupValidationFailureTotal].(*prometheus.CounterVec), tc.expectedLabel) assert.Equal(t, float64(1), metric, tc.description) }) // Test backup warning metric t.Run("RegisterBackupWarning", func(t *testing.T) { m.RegisterBackupWarning(tc.scheduleName) metric := getMetricValue(t, m.metrics[backupWarningTotal].(*prometheus.CounterVec), tc.expectedLabel) assert.Equal(t, float64(1), metric, tc.description) }) // Test backup items total gauge t.Run("RegisterBackupItemsTotalGauge", func(t *testing.T) { m.RegisterBackupItemsTotalGauge(tc.scheduleName, 100) metric := getMetricValue(t, m.metrics[backupItemsTotalGauge].(*prometheus.GaugeVec), tc.expectedLabel) assert.Equal(t, float64(100), metric, tc.description) }) // Test backup items errors gauge t.Run("RegisterBackupItemsErrorsGauge", func(t *testing.T) { m.RegisterBackupItemsErrorsGauge(tc.scheduleName, 5) metric := getMetricValue(t, m.metrics[backupItemsErrorsGauge].(*prometheus.GaugeVec), tc.expectedLabel) assert.Equal(t, float64(5), metric, tc.description) }) // Test backup duration metric t.Run("RegisterBackupDuration", func(t *testing.T) { m.RegisterBackupDuration(tc.scheduleName, 120.5) // For histogram, we check the count metric := getHistogramCount(t, m.metrics[backupDurationSeconds].(*prometheus.HistogramVec), tc.expectedLabel) assert.Equal(t, uint64(1), metric, tc.description) }) // Test backup last status metric t.Run("RegisterBackupLastStatus", func(t *testing.T) { m.RegisterBackupLastStatus(tc.scheduleName, BackupLastStatusSucc) metric := getMetricValue(t, m.metrics[backupLastStatus].(*prometheus.GaugeVec), tc.expectedLabel) assert.Equal(t, float64(BackupLastStatusSucc), metric, tc.description) }) // Test backup tarball size metric t.Run("SetBackupTarballSizeBytesGauge", func(t *testing.T) { m.SetBackupTarballSizeBytesGauge(tc.scheduleName, 1024*1024) metric := getMetricValue(t, m.metrics[backupTarballSizeBytesGauge].(*prometheus.GaugeVec), tc.expectedLabel) assert.Equal(t, float64(1024*1024), metric, tc.description) }) // Test backup last successful timestamp t.Run("SetBackupLastSuccessfulTimestamp", func(t *testing.T) { testTime := time.Now() m.SetBackupLastSuccessfulTimestamp(tc.scheduleName, testTime) metric := getMetricValue(t, m.metrics[backupLastSuccessfulTimestamp].(*prometheus.GaugeVec), tc.expectedLabel) assert.Equal(t, float64(testTime.Unix()), metric, tc.description) }) }) } } // TestRestoreMetricsWithAdhocBackups verifies that restore metrics are properly recorded // for restores from both scheduled and adhoc backups. func TestRestoreMetricsWithAdhocBackups(t *testing.T) { tests := []struct { name string scheduleName string expectedLabel string description string }{ { name: "restore from scheduled backup", scheduleName: "daily-backup", expectedLabel: "daily-backup", description: "Restore metrics should use the backup's schedule name", }, { name: "restore from adhoc backup", scheduleName: "", expectedLabel: "", description: "Restore metrics should have empty schedule label for adhoc backup restores", }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { m := NewServerMetrics() // Test restore attempt metric t.Run("RegisterRestoreAttempt", func(t *testing.T) { m.RegisterRestoreAttempt(tc.scheduleName) metric := getMetricValue(t, m.metrics[restoreAttemptTotal].(*prometheus.CounterVec), tc.expectedLabel) assert.Equal(t, float64(1), metric, tc.description) }) // Test restore success metric t.Run("RegisterRestoreSuccess", func(t *testing.T) { m.RegisterRestoreSuccess(tc.scheduleName) metric := getMetricValue(t, m.metrics[restoreSuccessTotal].(*prometheus.CounterVec), tc.expectedLabel) assert.Equal(t, float64(1), metric, tc.description) }) // Test restore failed metric t.Run("RegisterRestoreFailed", func(t *testing.T) { m.RegisterRestoreFailed(tc.scheduleName) metric := getMetricValue(t, m.metrics[restoreFailedTotal].(*prometheus.CounterVec), tc.expectedLabel) assert.Equal(t, float64(1), metric, tc.description) }) // Test restore partial failure metric t.Run("RegisterRestorePartialFailure", func(t *testing.T) { m.RegisterRestorePartialFailure(tc.scheduleName) metric := getMetricValue(t, m.metrics[restorePartialFailureTotal].(*prometheus.CounterVec), tc.expectedLabel) assert.Equal(t, float64(1), metric, tc.description) }) // Test restore validation failed metric t.Run("RegisterRestoreValidationFailed", func(t *testing.T) { m.RegisterRestoreValidationFailed(tc.scheduleName) metric := getMetricValue(t, m.metrics[restoreValidationFailedTotal].(*prometheus.CounterVec), tc.expectedLabel) assert.Equal(t, float64(1), metric, tc.description) }) }) } } // TestMultipleAdhocBackupsShareMetrics verifies that multiple adhoc backups // accumulate metrics under the same empty schedule label. func TestMultipleAdhocBackupsShareMetrics(t *testing.T) { m := NewServerMetrics() // Simulate multiple adhoc backup attempts for i := 0; i < 5; i++ { m.RegisterBackupAttempt("") } // Simulate some successes and failures m.RegisterBackupSuccess("") m.RegisterBackupSuccess("") m.RegisterBackupFailed("") m.RegisterBackupPartialFailure("") m.RegisterBackupValidationFailure("") // Verify accumulated metrics attemptMetric := getMetricValue(t, m.metrics[backupAttemptTotal].(*prometheus.CounterVec), "") assert.Equal(t, float64(5), attemptMetric, "All adhoc backup attempts should be counted together") successMetric := getMetricValue(t, m.metrics[backupSuccessTotal].(*prometheus.CounterVec), "") assert.Equal(t, float64(2), successMetric, "All adhoc backup successes should be counted together") failureMetric := getMetricValue(t, m.metrics[backupFailureTotal].(*prometheus.CounterVec), "") assert.Equal(t, float64(1), failureMetric, "All adhoc backup failures should be counted together") partialFailureMetric := getMetricValue(t, m.metrics[backupPartialFailureTotal].(*prometheus.CounterVec), "") assert.Equal(t, float64(1), partialFailureMetric, "All adhoc partial failures should be counted together") validationFailureMetric := getMetricValue(t, m.metrics[backupValidationFailureTotal].(*prometheus.CounterVec), "") assert.Equal(t, float64(1), validationFailureMetric, "All adhoc validation failures should be counted together") } // TestSetScheduleExpectedIntervalSeconds verifies that the expected interval metric // is properly recorded for schedules. func TestSetScheduleExpectedIntervalSeconds(t *testing.T) { tests := []struct { name string scheduleName string intervalSeconds float64 description string }{ { name: "every 5 minutes schedule", scheduleName: "frequent-backup", intervalSeconds: 300, description: "Expected interval should be 5m in seconds", }, { name: "daily schedule", scheduleName: "daily-backup", intervalSeconds: 86400, description: "Expected interval should be 24h in seconds", }, { name: "monthly schedule", scheduleName: "monthly-backup", intervalSeconds: 2678400, // 31 days in seconds description: "Expected interval should be 31 days in seconds", }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { m := NewServerMetrics() m.SetScheduleExpectedIntervalSeconds(tc.scheduleName, tc.intervalSeconds) metric := getMetricValue(t, m.metrics[scheduleExpectedIntervalSeconds].(*prometheus.GaugeVec), tc.scheduleName) assert.Equal(t, tc.intervalSeconds, metric, tc.description) }) } } // TestScheduleExpectedIntervalNotInitializedByDefault verifies that the expected // interval metric is not initialized by InitSchedule, so it only appears for // schedules with a valid cron expression. func TestScheduleExpectedIntervalNotInitializedByDefault(t *testing.T) { m := NewServerMetrics() m.InitSchedule("test-schedule") // The metric should not have any values after InitSchedule ch := make(chan prometheus.Metric, 1) m.metrics[scheduleExpectedIntervalSeconds].(*prometheus.GaugeVec).Collect(ch) close(ch) count := 0 for range ch { count++ } assert.Equal(t, 0, count, "scheduleExpectedIntervalSeconds should not be initialized by InitSchedule") } // TestRemoveScheduleCleansUpExpectedInterval verifies that RemoveSchedule // cleans up the expected interval metric. func TestRemoveScheduleCleansUpExpectedInterval(t *testing.T) { m := NewServerMetrics() m.InitSchedule("test-schedule") m.SetScheduleExpectedIntervalSeconds("test-schedule", 3600) // Verify metric exists metric := getMetricValue(t, m.metrics[scheduleExpectedIntervalSeconds].(*prometheus.GaugeVec), "test-schedule") assert.Equal(t, float64(3600), metric) // Remove schedule and verify metric is cleaned up m.RemoveSchedule("test-schedule") ch := make(chan prometheus.Metric, 1) m.metrics[scheduleExpectedIntervalSeconds].(*prometheus.GaugeVec).Collect(ch) close(ch) count := 0 for range ch { count++ } assert.Equal(t, 0, count, "scheduleExpectedIntervalSeconds should be removed after RemoveSchedule") } // TestInitScheduleWithEmptyName verifies that InitSchedule works correctly // with an empty schedule name (for adhoc backups). func TestInitScheduleWithEmptyName(t *testing.T) { m := NewServerMetrics() // Initialize metrics for empty schedule (adhoc backups) m.InitSchedule("") // Verify all metrics are initialized with 0 metrics := []string{ backupAttemptTotal, backupSuccessTotal, backupPartialFailureTotal, backupFailureTotal, backupValidationFailureTotal, backupDeletionAttemptTotal, backupDeletionSuccessTotal, backupDeletionFailureTotal, backupItemsTotalGauge, backupItemsErrorsGauge, backupWarningTotal, restoreAttemptTotal, restorePartialFailureTotal, restoreFailedTotal, restoreSuccessTotal, restoreValidationFailedTotal, volumeSnapshotSuccessTotal, volumeSnapshotAttemptTotal, volumeSnapshotFailureTotal, } for _, metricName := range metrics { t.Run(metricName, func(t *testing.T) { var value float64 switch vec := m.metrics[metricName].(type) { case *prometheus.CounterVec: value = getMetricValue(t, vec, "") case *prometheus.GaugeVec: value = getMetricValue(t, vec, "") } assert.Equal(t, float64(0), value, "Metric %s should be initialized to 0 for empty schedule", metricName) }) } // Special case: backupLastStatus should be initialized to 1 (success) lastStatusValue := getMetricValue(t, m.metrics[backupLastStatus].(*prometheus.GaugeVec), "") assert.Equal(t, float64(1), lastStatusValue, "backupLastStatus should be initialized to 1 for empty schedule") } // Helper function to get metric value from a CounterVec or GaugeVec func getMetricValue(t *testing.T, vec prometheus.Collector, scheduleLabel string) float64 { t.Helper() ch := make(chan prometheus.Metric, 1) vec.Collect(ch) close(ch) for metric := range ch { dto := &dto.Metric{} err := metric.Write(dto) require.NoError(t, err) // Check if this metric has the expected schedule label hasCorrectLabel := false for _, label := range dto.Label { if *label.Name == "schedule" && *label.Value == scheduleLabel { hasCorrectLabel = true break } } if hasCorrectLabel { if dto.Counter != nil { return *dto.Counter.Value } if dto.Gauge != nil { return *dto.Gauge.Value } } } t.Fatalf("Metric with schedule label '%s' not found", scheduleLabel) return 0 } // Helper function to get histogram count func getHistogramCount(t *testing.T, vec *prometheus.HistogramVec, scheduleLabel string) uint64 { t.Helper() ch := make(chan prometheus.Metric, 1) vec.Collect(ch) close(ch) for metric := range ch { dto := &dto.Metric{} err := metric.Write(dto) require.NoError(t, err) // Check if this metric has the expected schedule label hasCorrectLabel := false for _, label := range dto.Label { if *label.Name == "schedule" && *label.Value == scheduleLabel { hasCorrectLabel = true break } } if hasCorrectLabel && dto.Histogram != nil { return *dto.Histogram.SampleCount } } t.Fatalf("Histogram with schedule label '%s' not found", scheduleLabel) return 0 } // TestRepoMaintenanceMetrics verifies that repo maintenance metrics are properly recorded. func TestRepoMaintenanceMetrics(t *testing.T) { tests := []struct { name string repositoryName string description string }{ { name: "maintenance job metrics for repository", repositoryName: "default-restic-abcd", description: "Metrics should be recorded with the repository name label", }, { name: "maintenance job metrics for different repository", repositoryName: "velero-backup-repo-xyz", description: "Metrics should be recorded with different repository name", }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { m := NewServerMetrics() // Test repo maintenance success metric t.Run("RegisterRepoMaintenanceSuccess", func(t *testing.T) { m.RegisterRepoMaintenanceSuccess(tc.repositoryName) metric := getMaintenanceMetricValue(t, m.metrics[repoMaintenanceSuccessTotal].(*prometheus.CounterVec), tc.repositoryName) assert.Equal(t, float64(1), metric, tc.description) }) // Test repo maintenance failure metric t.Run("RegisterRepoMaintenanceFailure", func(t *testing.T) { m.RegisterRepoMaintenanceFailure(tc.repositoryName) metric := getMaintenanceMetricValue(t, m.metrics[repoMaintenanceFailureTotal].(*prometheus.CounterVec), tc.repositoryName) assert.Equal(t, float64(1), metric, tc.description) }) // Test repo maintenance duration metric t.Run("ObserveRepoMaintenanceDuration", func(t *testing.T) { m.ObserveRepoMaintenanceDuration(tc.repositoryName, 300.5) // For histogram, we check the count metric := getMaintenanceHistogramCount(t, m.metrics[repoMaintenanceDurationSeconds].(*prometheus.HistogramVec), tc.repositoryName) assert.Equal(t, uint64(1), metric, tc.description) }) }) } } // TestMultipleRepoMaintenanceJobsAccumulate verifies that multiple repo maintenance jobs // accumulate metrics under the same repository label. func TestMultipleRepoMaintenanceJobsAccumulate(t *testing.T) { m := NewServerMetrics() repoName := "default-restic-test" // Simulate multiple repo maintenance job executions m.RegisterRepoMaintenanceSuccess(repoName) m.RegisterRepoMaintenanceSuccess(repoName) m.RegisterRepoMaintenanceSuccess(repoName) m.RegisterRepoMaintenanceFailure(repoName) m.RegisterRepoMaintenanceFailure(repoName) // Record multiple durations m.ObserveRepoMaintenanceDuration(repoName, 120.5) m.ObserveRepoMaintenanceDuration(repoName, 180.3) m.ObserveRepoMaintenanceDuration(repoName, 90.7) // Verify accumulated metrics successMetric := getMaintenanceMetricValue(t, m.metrics[repoMaintenanceSuccessTotal].(*prometheus.CounterVec), repoName) assert.Equal(t, float64(3), successMetric, "All repo maintenance successes should be counted") failureMetric := getMaintenanceMetricValue(t, m.metrics[repoMaintenanceFailureTotal].(*prometheus.CounterVec), repoName) assert.Equal(t, float64(2), failureMetric, "All repo maintenance failures should be counted") durationCount := getMaintenanceHistogramCount(t, m.metrics[repoMaintenanceDurationSeconds].(*prometheus.HistogramVec), repoName) assert.Equal(t, uint64(3), durationCount, "All repo maintenance durations should be observed") } // Helper function to get metric value from a CounterVec with repository_name label func getMaintenanceMetricValue(t *testing.T, vec prometheus.Collector, repositoryName string) float64 { t.Helper() ch := make(chan prometheus.Metric, 1) vec.Collect(ch) close(ch) for metric := range ch { dto := &dto.Metric{} err := metric.Write(dto) require.NoError(t, err) // Check if this metric has the expected repository_name label hasCorrectLabel := false for _, label := range dto.Label { if *label.Name == "repository_name" && *label.Value == repositoryName { hasCorrectLabel = true break } } if hasCorrectLabel { if dto.Counter != nil { return *dto.Counter.Value } if dto.Gauge != nil { return *dto.Gauge.Value } } } t.Fatalf("Metric with repository_name label '%s' not found", repositoryName) return 0 } // Helper function to get histogram count with repository_name label func getMaintenanceHistogramCount(t *testing.T, vec *prometheus.HistogramVec, repositoryName string) uint64 { t.Helper() ch := make(chan prometheus.Metric, 1) vec.Collect(ch) close(ch) for metric := range ch { dto := &dto.Metric{} err := metric.Write(dto) require.NoError(t, err) // Check if this metric has the expected repository_name label hasCorrectLabel := false for _, label := range dto.Label { if *label.Name == "repository_name" && *label.Value == repositoryName { hasCorrectLabel = true break } } if hasCorrectLabel && dto.Histogram != nil { return *dto.Histogram.SampleCount } } t.Fatalf("Histogram with repository_name label '%s' not found", repositoryName) return 0 } ================================================ FILE: pkg/nodeagent/node_agent.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package nodeagent import ( "context" "encoding/json" "fmt" "github.com/pkg/errors" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/kubernetes" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" velerotypes "github.com/vmware-tanzu/velero/pkg/types" "github.com/vmware-tanzu/velero/pkg/util/kube" ) const ( // daemonSet is the name of the Velero node agent daemonset on linux nodes. daemonSet = "node-agent" // daemonsetWindows is the name of the Velero node agent daemonset on Windows nodes. daemonsetWindows = "node-agent-windows" // nodeAgentRole marks pods with node-agent role on all nodes. nodeAgentRole = "node-agent" // HostPodVolumeMount is the name of the volume in node-agent for host-pod mount HostPodVolumeMount = "host-pods" // HostPodVolumeMountPoint is the mount point of the volume in node-agent for host-pod mount HostPodVolumeMountPoint = "host_pods" ) var ( ErrDaemonSetNotFound = errors.New("daemonset not found") ErrNodeAgentLabelNotFound = errors.New("node-agent label not found") ErrNodeAgentAnnotationNotFound = errors.New("node-agent annotation not found") ErrNodeAgentTolerationNotFound = errors.New("node-agent toleration not found") ) func IsRunningOnLinux(ctx context.Context, kubeClient kubernetes.Interface, namespace string) error { return isRunning(ctx, kubeClient, namespace, daemonSet) } func IsRunningOnWindows(ctx context.Context, kubeClient kubernetes.Interface, namespace string) error { return isRunning(ctx, kubeClient, namespace, daemonsetWindows) } func isRunning(ctx context.Context, kubeClient kubernetes.Interface, namespace string, daemonset string) error { if _, err := kubeClient.AppsV1().DaemonSets(namespace).Get(ctx, daemonset, metav1.GetOptions{}); apierrors.IsNotFound(err) { return ErrDaemonSetNotFound } else if err != nil { return err } else { return nil } } // KbClientIsRunningInNode checks if the node agent pod is running properly in a specified node through kube client. If not, return the error found func KbClientIsRunningInNode(ctx context.Context, namespace string, nodeName string, kubeClient kubernetes.Interface) error { return isRunningInNode(ctx, namespace, nodeName, nil, kubeClient) } // IsRunningInNode checks if the node agent pod is running properly in a specified node through controller client. If not, return the error found func IsRunningInNode(ctx context.Context, namespace string, nodeName string, crClient ctrlclient.Client) error { return isRunningInNode(ctx, namespace, nodeName, crClient, nil) } func isRunningInNode(ctx context.Context, namespace string, nodeName string, crClient ctrlclient.Client, kubeClient kubernetes.Interface) error { if nodeName == "" { return errors.New("node name is empty") } pods := new(corev1api.PodList) parsedSelector, err := labels.Parse(fmt.Sprintf("role=%s", nodeAgentRole)) if err != nil { return errors.Wrap(err, "fail to parse selector") } if crClient != nil { err = crClient.List(ctx, pods, &ctrlclient.ListOptions{LabelSelector: parsedSelector}) } else { pods, err = kubeClient.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{LabelSelector: parsedSelector.String()}) } if err != nil { return errors.Wrap(err, "failed to list node-agent pods") } for i := range pods.Items { if kube.IsPodRunning(&pods.Items[i]) != nil { continue } if pods.Items[i].Spec.NodeName == nodeName { return nil } } return errors.Errorf("daemonset pod not found in running state in node %s", nodeName) } func GetPodSpec(ctx context.Context, kubeClient kubernetes.Interface, namespace string, osType string) (*corev1api.PodSpec, error) { dsName := daemonSet if osType == kube.NodeOSWindows { dsName = daemonsetWindows } ds, err := kubeClient.AppsV1().DaemonSets(namespace).Get(ctx, dsName, metav1.GetOptions{}) if err != nil { return nil, errors.Wrapf(err, "error to get %s daemonset", dsName) } return &ds.Spec.Template.Spec, nil } func GetConfigs(ctx context.Context, namespace string, kubeClient kubernetes.Interface, configName string) (*velerotypes.NodeAgentConfigs, error) { cm, err := kubeClient.CoreV1().ConfigMaps(namespace).Get(ctx, configName, metav1.GetOptions{}) if err != nil { return nil, errors.Wrapf(err, "error to get node agent configs %s", configName) } if cm.Data == nil { return nil, errors.Errorf("data is not available in config map %s", configName) } if len(cm.Data) > 1 { return nil, errors.Errorf("more than one keys are found in ConfigMap %s's data. only expect one", configName) } jsonString := "" for _, v := range cm.Data { jsonString = v } configs := &velerotypes.NodeAgentConfigs{} err = json.Unmarshal([]byte(jsonString), configs) if err != nil { return nil, errors.Wrapf(err, "error to unmarshall configs from %s", configName) } return configs, nil } func GetLabelValue(ctx context.Context, kubeClient kubernetes.Interface, namespace string, key string, osType string) (string, error) { dsName := daemonSet if osType == kube.NodeOSWindows { dsName = daemonsetWindows } ds, err := kubeClient.AppsV1().DaemonSets(namespace).Get(ctx, dsName, metav1.GetOptions{}) if err != nil { return "", errors.Wrapf(err, "error getting %s daemonset", dsName) } if ds.Spec.Template.Labels == nil { return "", ErrNodeAgentLabelNotFound } val, found := ds.Spec.Template.Labels[key] if !found { return "", ErrNodeAgentLabelNotFound } return val, nil } func GetAnnotationValue(ctx context.Context, kubeClient kubernetes.Interface, namespace string, key string, osType string) (string, error) { dsName := daemonSet if osType == kube.NodeOSWindows { dsName = daemonsetWindows } ds, err := kubeClient.AppsV1().DaemonSets(namespace).Get(ctx, dsName, metav1.GetOptions{}) if err != nil { return "", errors.Wrapf(err, "error getting %s daemonset", dsName) } if ds.Spec.Template.Annotations == nil { return "", ErrNodeAgentAnnotationNotFound } val, found := ds.Spec.Template.Annotations[key] if !found { return "", ErrNodeAgentAnnotationNotFound } return val, nil } func GetToleration(ctx context.Context, kubeClient kubernetes.Interface, namespace string, key string, osType string) (*corev1api.Toleration, error) { dsName := daemonSet if osType == kube.NodeOSWindows { dsName = daemonsetWindows } ds, err := kubeClient.AppsV1().DaemonSets(namespace).Get(ctx, dsName, metav1.GetOptions{}) if err != nil { return nil, errors.Wrapf(err, "error getting %s daemonset", dsName) } for i, t := range ds.Spec.Template.Spec.Tolerations { if t.Key == key { return &ds.Spec.Template.Spec.Tolerations[i], nil } } return nil, ErrNodeAgentTolerationNotFound } func GetHostPodPath(ctx context.Context, kubeClient kubernetes.Interface, namespace string, osType string) (string, error) { dsName := daemonSet if osType == kube.NodeOSWindows { dsName = daemonsetWindows } ds, err := kubeClient.AppsV1().DaemonSets(namespace).Get(ctx, dsName, metav1.GetOptions{}) if err != nil { return "", errors.Wrapf(err, "error getting daemonset %s", dsName) } var volume *corev1api.Volume for _, v := range ds.Spec.Template.Spec.Volumes { if v.Name == HostPodVolumeMount { volume = &v break } } if volume == nil { return "", errors.New("host pod volume is not found") } if volume.HostPath == nil { return "", errors.New("host pod volume is not a host path volume") } if volume.HostPath.Path == "" { return "", errors.New("host pod volume path is empty") } return volume.HostPath.Path, nil } func HostPodVolumeMountPath() string { return "/" + HostPodVolumeMountPoint } func HostPodVolumeMountPathWin() string { return "\\" + HostPodVolumeMountPoint } ================================================ FILE: pkg/nodeagent/node_agent_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package nodeagent import ( "testing" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" appsv1api "k8s.io/api/apps/v1" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes/fake" clientTesting "k8s.io/client-go/testing" clientFake "sigs.k8s.io/controller-runtime/pkg/client/fake" "github.com/vmware-tanzu/velero/pkg/builder" velerotypes "github.com/vmware-tanzu/velero/pkg/types" "github.com/vmware-tanzu/velero/pkg/util/kube" ) type reactor struct { verb string resource string reactorFunc clientTesting.ReactionFunc } func TestIsRunning(t *testing.T) { ds := &appsv1api.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-ns", Name: "node-agent", }, TypeMeta: metav1.TypeMeta{ Kind: "DaemonSet", }, } tests := []struct { name string kubeClientObj []runtime.Object namespace string kubeReactors []reactor expectErr string }{ { name: "ds is not found", namespace: "fake-ns", expectErr: "daemonset not found", }, { name: "ds get error", namespace: "fake-ns", kubeReactors: []reactor{ { verb: "get", resource: "daemonsets", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, errors.New("fake-get-error") }, }, }, expectErr: "fake-get-error", }, { name: "succeed", namespace: "fake-ns", kubeClientObj: []runtime.Object{ ds, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...) for _, reactor := range test.kubeReactors { fakeKubeClient.Fake.PrependReactor(reactor.verb, reactor.resource, reactor.reactorFunc) } err := isRunning(t.Context(), fakeKubeClient, test.namespace, daemonSet) if test.expectErr == "" { assert.NoError(t, err) } else { assert.EqualError(t, err, test.expectErr) } }) } } func TestIsRunningInNode(t *testing.T) { scheme := runtime.NewScheme() corev1api.AddToScheme(scheme) nonNodeAgentPod := builder.ForPod("fake-ns", "fake-pod").Result() nodeAgentPodNotRunning := builder.ForPod("fake-ns", "fake-pod").Labels(map[string]string{"role": "node-agent"}).Result() nodeAgentPodRunning1 := builder.ForPod("fake-ns", "fake-pod-1").Labels(map[string]string{"role": "node-agent"}).Phase(corev1api.PodRunning).Result() nodeAgentPodRunning2 := builder.ForPod("fake-ns", "fake-pod-2").Labels(map[string]string{"role": "node-agent"}).Phase(corev1api.PodRunning).Result() nodeAgentPodRunning3 := builder.ForPod("fake-ns", "fake-pod-3"). Labels(map[string]string{"role": "node-agent"}). Phase(corev1api.PodRunning). NodeName("fake-node"). Result() tests := []struct { name string kubeClientObj []runtime.Object nodeName string expectErr string }{ { name: "node name is empty", expectErr: "node name is empty", }, { name: "ds pod not found", nodeName: "fake-node", kubeClientObj: []runtime.Object{ nonNodeAgentPod, }, expectErr: "daemonset pod not found in running state in node fake-node", }, { name: "ds po are not all running", nodeName: "fake-node", kubeClientObj: []runtime.Object{ nodeAgentPodNotRunning, nodeAgentPodRunning1, }, expectErr: "daemonset pod not found in running state in node fake-node", }, { name: "ds pods wrong node name", nodeName: "fake-node", kubeClientObj: []runtime.Object{ nodeAgentPodNotRunning, nodeAgentPodRunning1, nodeAgentPodRunning2, }, expectErr: "daemonset pod not found in running state in node fake-node", }, { name: "succeed", nodeName: "fake-node", kubeClientObj: []runtime.Object{ nodeAgentPodNotRunning, nodeAgentPodRunning1, nodeAgentPodRunning2, nodeAgentPodRunning3, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeClientBuilder := clientFake.NewClientBuilder() fakeClientBuilder = fakeClientBuilder.WithScheme(scheme) fakeClient := fakeClientBuilder.WithRuntimeObjects(test.kubeClientObj...).Build() err := IsRunningInNode(t.Context(), "", test.nodeName, fakeClient) if test.expectErr == "" { assert.NoError(t, err) } else { assert.EqualError(t, err, test.expectErr) } }) } } func TestGetPodSpec(t *testing.T) { podSpec := corev1api.PodSpec{ NodeName: "fake-node", } daemonSet := &appsv1api.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-ns", Name: "node-agent", }, TypeMeta: metav1.TypeMeta{ Kind: "DaemonSet", }, Spec: appsv1api.DaemonSetSpec{ Template: corev1api.PodTemplateSpec{ Spec: podSpec, }, }, } tests := []struct { name string kubeClientObj []runtime.Object namespace string expectErr string expectSpec corev1api.PodSpec }{ { name: "ds is not found", namespace: "fake-ns", expectErr: "error to get node-agent daemonset: daemonsets.apps \"node-agent\" not found", }, { name: "succeed", namespace: "fake-ns", kubeClientObj: []runtime.Object{ daemonSet, }, expectSpec: podSpec, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...) spec, err := GetPodSpec(t.Context(), fakeKubeClient, test.namespace, kube.NodeOSLinux) if test.expectErr == "" { require.NoError(t, err) assert.Equal(t, *spec, test.expectSpec) } else { assert.EqualError(t, err, test.expectErr) } }) } } func TestGetConfigs(t *testing.T) { cm := builder.ForConfigMap("fake-ns", "node-agent-config").Result() cmWithInvalidDataFormat := builder.ForConfigMap("fake-ns", "node-agent-config").Data("fake-key", "wrong").Result() cmWithoutCocurrentData := builder.ForConfigMap("fake-ns", "node-agent-config").Data("fake-key", "{\"someothers\":{\"someother\": 10}}").Result() cmWithValidData := builder.ForConfigMap("fake-ns", "node-agent-config").Data("fake-key", "{\"loadConcurrency\":{\"globalConfig\": 5}}").Result() cmWithPriorityClass := builder.ForConfigMap("fake-ns", "node-agent-config").Data("fake-key", "{\"priorityClassName\": \"high-priority\"}").Result() cmWithPriorityClassAndOther := builder.ForConfigMap("fake-ns", "node-agent-config").Data("fake-key", "{\"priorityClassName\": \"low-priority\", \"loadConcurrency\":{\"globalConfig\": 3}}").Result() cmWithMultipleKeysInData := builder.ForConfigMap("fake-ns", "node-agent-config").Data("fake-key-1", "{}", "fake-key-2", "{}").Result() tests := []struct { name string kubeClientObj []runtime.Object namespace string kubeReactors []reactor expectResult *velerotypes.NodeAgentConfigs expectErr string }{ { name: "cm get error", namespace: "fake-ns", kubeReactors: []reactor{ { verb: "get", resource: "configmaps", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, errors.New("fake-get-error") }, }, }, expectErr: "error to get node agent configs node-agent-config: fake-get-error", }, { name: "cm's data is nil", namespace: "fake-ns", kubeClientObj: []runtime.Object{ cm, }, expectErr: "data is not available in config map node-agent-config", }, { name: "cm's data is with invalid format", namespace: "fake-ns", kubeClientObj: []runtime.Object{ cmWithInvalidDataFormat, }, expectErr: "error to unmarshall configs from node-agent-config: invalid character 'w' looking for beginning of value", }, { name: "concurrency configs are not found", namespace: "fake-ns", kubeClientObj: []runtime.Object{ cmWithoutCocurrentData, }, expectResult: &velerotypes.NodeAgentConfigs{}, }, { name: "success", namespace: "fake-ns", kubeClientObj: []runtime.Object{ cmWithValidData, }, expectResult: &velerotypes.NodeAgentConfigs{ LoadConcurrency: &velerotypes.LoadConcurrency{ GlobalConfig: 5, }, }, }, { name: "configmap with priority class name", namespace: "fake-ns", kubeClientObj: []runtime.Object{ cmWithPriorityClass, }, expectResult: &velerotypes.NodeAgentConfigs{ PriorityClassName: "high-priority", }, }, { name: "configmap with priority class and other configs", namespace: "fake-ns", kubeClientObj: []runtime.Object{ cmWithPriorityClassAndOther, }, expectResult: &velerotypes.NodeAgentConfigs{ PriorityClassName: "low-priority", LoadConcurrency: &velerotypes.LoadConcurrency{ GlobalConfig: 3, }, }, }, { name: "ConfigMap's Data has more than one key", namespace: "fake-ns", kubeClientObj: []runtime.Object{ cmWithMultipleKeysInData, }, expectErr: "more than one keys are found in ConfigMap node-agent-config's data. only expect one", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...) for _, reactor := range test.kubeReactors { fakeKubeClient.Fake.PrependReactor(reactor.verb, reactor.resource, reactor.reactorFunc) } result, err := GetConfigs(t.Context(), test.namespace, fakeKubeClient, "node-agent-config") if test.expectErr == "" { require.NoError(t, err) if test.expectResult == nil { assert.Nil(t, result) } else { // Check PriorityClassName assert.Equal(t, test.expectResult.PriorityClassName, result.PriorityClassName) // Check LoadConcurrency if test.expectResult.LoadConcurrency == nil { assert.Nil(t, result.LoadConcurrency) } else { assert.Equal(t, *test.expectResult.LoadConcurrency, *result.LoadConcurrency) } } } else { assert.EqualError(t, err, test.expectErr) } }) } } func TestGetLabelValue(t *testing.T) { daemonSet := &appsv1api.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-ns", Name: "node-agent", }, TypeMeta: metav1.TypeMeta{ Kind: "DaemonSet", }, } daemonSetWithOtherLabel := &appsv1api.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-ns", Name: "node-agent", }, TypeMeta: metav1.TypeMeta{ Kind: "DaemonSet", }, Spec: appsv1api.DaemonSetSpec{ Template: corev1api.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ "fake-other-label": "fake-value-1", }, }, }, }, } daemonSetWithLabel := &appsv1api.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-ns", Name: "node-agent", }, TypeMeta: metav1.TypeMeta{ Kind: "DaemonSet", }, Spec: appsv1api.DaemonSetSpec{ Template: corev1api.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ "fake-label": "fake-value-2", }, }, }, }, } daemonSetWithEmptyLabel := &appsv1api.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-ns", Name: "node-agent", }, TypeMeta: metav1.TypeMeta{ Kind: "DaemonSet", }, Spec: appsv1api.DaemonSetSpec{ Template: corev1api.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ "fake-label": "", }, }, }, }, } tests := []struct { name string kubeClientObj []runtime.Object namespace string expectedValue string expectErr string }{ { name: "ds get error", namespace: "fake-ns", expectErr: "error getting node-agent daemonset: daemonsets.apps \"node-agent\" not found", }, { name: "no label", namespace: "fake-ns", kubeClientObj: []runtime.Object{ daemonSet, }, expectErr: ErrNodeAgentLabelNotFound.Error(), }, { name: "no expecting label", namespace: "fake-ns", kubeClientObj: []runtime.Object{ daemonSetWithOtherLabel, }, expectErr: ErrNodeAgentLabelNotFound.Error(), }, { name: "expecting label", namespace: "fake-ns", kubeClientObj: []runtime.Object{ daemonSetWithLabel, }, expectedValue: "fake-value-2", }, { name: "expecting empty label", namespace: "fake-ns", kubeClientObj: []runtime.Object{ daemonSetWithEmptyLabel, }, expectedValue: "", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...) value, err := GetLabelValue(t.Context(), fakeKubeClient, test.namespace, "fake-label", kube.NodeOSLinux) if test.expectErr == "" { require.NoError(t, err) assert.Equal(t, test.expectedValue, value) } else { assert.EqualError(t, err, test.expectErr) } }) } } func TestGetAnnotationValue(t *testing.T) { daemonSet := &appsv1api.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-ns", Name: "node-agent", }, TypeMeta: metav1.TypeMeta{ Kind: "DaemonSet", }, } daemonSetWithOtherAnnotation := &appsv1api.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-ns", Name: "node-agent", }, TypeMeta: metav1.TypeMeta{ Kind: "DaemonSet", }, Spec: appsv1api.DaemonSetSpec{ Template: corev1api.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ "fake-other-annotation": "fake-value-1", }, }, }, }, } daemonSetWithAnnotation := &appsv1api.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-ns", Name: "node-agent", }, TypeMeta: metav1.TypeMeta{ Kind: "DaemonSet", }, Spec: appsv1api.DaemonSetSpec{ Template: corev1api.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ "fake-annotation": "fake-value-2", }, }, }, }, } daemonSetWithEmptyAnnotation := &appsv1api.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-ns", Name: "node-agent", }, TypeMeta: metav1.TypeMeta{ Kind: "DaemonSet", }, Spec: appsv1api.DaemonSetSpec{ Template: corev1api.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ "fake-annotation": "", }, }, }, }, } tests := []struct { name string kubeClientObj []runtime.Object namespace string expectedValue string expectErr string }{ { name: "ds get error", namespace: "fake-ns", expectErr: "error getting node-agent daemonset: daemonsets.apps \"node-agent\" not found", }, { name: "no annotation", namespace: "fake-ns", kubeClientObj: []runtime.Object{ daemonSet, }, expectErr: ErrNodeAgentAnnotationNotFound.Error(), }, { name: "no expecting annotation", namespace: "fake-ns", kubeClientObj: []runtime.Object{ daemonSetWithOtherAnnotation, }, expectErr: ErrNodeAgentAnnotationNotFound.Error(), }, { name: "expecting annotation", namespace: "fake-ns", kubeClientObj: []runtime.Object{ daemonSetWithAnnotation, }, expectedValue: "fake-value-2", }, { name: "expecting empty annotation", namespace: "fake-ns", kubeClientObj: []runtime.Object{ daemonSetWithEmptyAnnotation, }, expectedValue: "", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...) value, err := GetAnnotationValue(t.Context(), fakeKubeClient, test.namespace, "fake-annotation", kube.NodeOSLinux) if test.expectErr == "" { require.NoError(t, err) assert.Equal(t, test.expectedValue, value) } else { assert.EqualError(t, err, test.expectErr) } }) } } func TestGetToleration(t *testing.T) { daemonSet := &appsv1api.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-ns", Name: "node-agent", }, TypeMeta: metav1.TypeMeta{ Kind: "DaemonSet", }, } daemonSetWithOtherToleration := &appsv1api.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-ns", Name: "node-agent", }, TypeMeta: metav1.TypeMeta{ Kind: "DaemonSet", }, Spec: appsv1api.DaemonSetSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ Tolerations: []corev1api.Toleration{ { Key: "other-toleration-key", }, }, }, }, }, } daemonSetWithToleration := &appsv1api.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-ns", Name: "node-agent", }, TypeMeta: metav1.TypeMeta{ Kind: "DaemonSet", }, Spec: appsv1api.DaemonSetSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ Tolerations: []corev1api.Toleration{ { Key: "fake-toleration", Value: "true", }, }, }, }, }, } tests := []struct { name string kubeClientObj []runtime.Object namespace string expectedValue corev1api.Toleration expectErr string }{ // { // name: "ds get error", // namespace: "fake-ns", // expectErr: "error getting node-agent daemonset: daemonsets.apps \"node-agent\" not found", // }, { name: "no toleration", namespace: "fake-ns", kubeClientObj: []runtime.Object{ daemonSet, }, expectErr: ErrNodeAgentTolerationNotFound.Error(), }, { name: "no expecting toleration", namespace: "fake-ns", kubeClientObj: []runtime.Object{ daemonSetWithOtherToleration, }, expectErr: ErrNodeAgentTolerationNotFound.Error(), }, { name: "expecting toleration", namespace: "fake-ns", kubeClientObj: []runtime.Object{ daemonSetWithToleration, }, expectedValue: corev1api.Toleration{ Key: "fake-toleration", Value: "true", }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...) value, err := GetToleration(t.Context(), fakeKubeClient, test.namespace, "fake-toleration", kube.NodeOSLinux) if test.expectErr == "" { require.NoError(t, err) assert.Equal(t, test.expectedValue, *value) } else { assert.EqualError(t, err, test.expectErr) } }) } } func TestGetHostPodPath(t *testing.T) { daemonSet := &appsv1api.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-ns", Name: "node-agent", }, TypeMeta: metav1.TypeMeta{ Kind: "DaemonSet", }, } daemonSetWithHostPodVolume := &appsv1api.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-ns", Name: "node-agent", }, TypeMeta: metav1.TypeMeta{ Kind: "DaemonSet", }, Spec: appsv1api.DaemonSetSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ Volumes: []corev1api.Volume{ { Name: HostPodVolumeMount, }, }, }, }, }, } daemonSetWithHostPodVolumeAndEmptyPath := &appsv1api.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-ns", Name: "node-agent", }, TypeMeta: metav1.TypeMeta{ Kind: "DaemonSet", }, Spec: appsv1api.DaemonSetSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ Volumes: []corev1api.Volume{ { Name: HostPodVolumeMount, VolumeSource: corev1api.VolumeSource{ HostPath: &corev1api.HostPathVolumeSource{}, }, }, }, }, }, }, } daemonSetWithHostPodVolumeAndValidPath := &appsv1api.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-ns", Name: "node-agent", }, TypeMeta: metav1.TypeMeta{ Kind: "DaemonSet", }, Spec: appsv1api.DaemonSetSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ Volumes: []corev1api.Volume{ { Name: HostPodVolumeMount, VolumeSource: corev1api.VolumeSource{ HostPath: &corev1api.HostPathVolumeSource{ Path: "/var/lib/kubelet/pods", }, }, }, }, }, }, }, } tests := []struct { name string kubeClientObj []runtime.Object namespace string osType string expectedValue string expectErr string }{ { name: "ds get error", namespace: "fake-ns", osType: kube.NodeOSWindows, kubeClientObj: []runtime.Object{ daemonSet, }, expectErr: "error getting daemonset node-agent-windows: daemonsets.apps \"node-agent-windows\" not found", }, { name: "no host pod volume", namespace: "fake-ns", osType: kube.NodeOSLinux, kubeClientObj: []runtime.Object{ daemonSet, }, expectErr: "host pod volume is not found", }, { name: "no host pod volume path", namespace: "fake-ns", osType: kube.NodeOSLinux, kubeClientObj: []runtime.Object{ daemonSetWithHostPodVolume, }, expectErr: "host pod volume is not a host path volume", }, { name: "empty host pod volume path", namespace: "fake-ns", osType: kube.NodeOSLinux, kubeClientObj: []runtime.Object{ daemonSetWithHostPodVolumeAndEmptyPath, }, expectErr: "host pod volume path is empty", }, { name: "succeed", namespace: "fake-ns", osType: kube.NodeOSLinux, kubeClientObj: []runtime.Object{ daemonSetWithHostPodVolumeAndValidPath, }, expectedValue: "/var/lib/kubelet/pods", }, { name: "succeed on empty os type", namespace: "fake-ns", kubeClientObj: []runtime.Object{ daemonSetWithHostPodVolumeAndValidPath, }, expectedValue: "/var/lib/kubelet/pods", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...) path, err := GetHostPodPath(t.Context(), fakeKubeClient, test.namespace, test.osType) if test.expectErr == "" { require.NoError(t, err) assert.Equal(t, test.expectedValue, path) } else { assert.EqualError(t, err, test.expectErr) } }) } } ================================================ FILE: pkg/persistence/in_memory_object_store.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package persistence import ( "bytes" "errors" "io" "strings" "time" ) type BucketData map[string][]byte // inMemoryObjectStore is a simple implementation of the ObjectStore interface // that stores its data in-memory/in-proc. This is mainly intended to be used // as a test fake. type inMemoryObjectStore struct { Data map[string]BucketData Config map[string]string } func newInMemoryObjectStore(buckets ...string) *inMemoryObjectStore { o := &inMemoryObjectStore{ Data: make(map[string]BucketData), } for _, bucket := range buckets { o.Data[bucket] = make(map[string][]byte) } return o } // // Interface Implementation // func (o *inMemoryObjectStore) Init(config map[string]string) error { o.Config = config return nil } func (o *inMemoryObjectStore) PutObject(bucket, key string, body io.Reader) error { bucketData, ok := o.Data[bucket] if !ok { return errors.New("bucket not found") } obj, err := io.ReadAll(body) if err != nil { return err } bucketData[key] = obj return nil } func (o *inMemoryObjectStore) ObjectExists(bucket, key string) (bool, error) { bucketData, ok := o.Data[bucket] if !ok { return false, errors.New("bucket not found") } _, ok = bucketData[key] return ok, nil } func (o *inMemoryObjectStore) GetObject(bucket, key string) (io.ReadCloser, error) { bucketData, ok := o.Data[bucket] if !ok { return nil, errors.New("bucket not found") } obj, ok := bucketData[key] if !ok { return nil, errors.New("key not found") } return io.NopCloser(bytes.NewReader(obj)), nil } func (o *inMemoryObjectStore) ListCommonPrefixes(bucket, prefix, delimiter string) ([]string, error) { keys, err := o.ListObjects(bucket, prefix) if err != nil { return nil, err } // For each key, check if it has an instance of the delimiter *after* the prefix. // If not, skip it; if so, return the prefix of the key up to/including the delimiter. var prefixes []string for _, key := range keys { // everything after 'prefix' afterPrefix := key[len(prefix):] // index of the *start* of 'delimiter' in 'afterPrefix' delimiterStart := strings.Index(afterPrefix, delimiter) if delimiterStart == -1 { continue } // return the prefix, plus everything after the prefix and before // the delimiter, plus the delimiter fullPrefix := prefix + afterPrefix[0:delimiterStart] + delimiter prefixes = append(prefixes, fullPrefix) } return prefixes, nil } func (o *inMemoryObjectStore) ListObjects(bucket, prefix string) ([]string, error) { bucketData, ok := o.Data[bucket] if !ok { return nil, errors.New("bucket not found") } var objs []string for key := range bucketData { if strings.HasPrefix(key, prefix) { objs = append(objs, key) } } return objs, nil } func (o *inMemoryObjectStore) DeleteObject(bucket, key string) error { bucketData, ok := o.Data[bucket] if !ok { return errors.New("bucket not found") } delete(bucketData, key) return nil } func (o *inMemoryObjectStore) CreateSignedURL(bucket, key string, ttl time.Duration) (string, error) { bucketData, ok := o.Data[bucket] if !ok { return "", errors.New("bucket not found") } _, ok = bucketData[key] if !ok { return "", errors.New("key not found") } return "a-url", nil } // // Test Helper Methods // func (o *inMemoryObjectStore) ClearBucket(bucket string) { if _, ok := o.Data[bucket]; !ok { return } o.Data[bucket] = make(map[string][]byte) } ================================================ FILE: pkg/persistence/mocks/backup_store.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ // Code generated by mockery v2.42.2. DO NOT EDIT. package mocks import ( io "io" mock "github.com/stretchr/testify/mock" itemoperation "github.com/vmware-tanzu/velero/pkg/itemoperation" persistence "github.com/vmware-tanzu/velero/pkg/persistence" results "github.com/vmware-tanzu/velero/pkg/util/results" v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" volume "github.com/vmware-tanzu/velero/internal/volume" volumesnapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" ) // BackupStore is an autogenerated mock type for the BackupStore type type BackupStore struct { mock.Mock } // BackupExists provides a mock function with given fields: bucket, backupName func (_m *BackupStore) BackupExists(bucket string, backupName string) (bool, error) { ret := _m.Called(bucket, backupName) if len(ret) == 0 { panic("no return value specified for BackupExists") } var r0 bool var r1 error if rf, ok := ret.Get(0).(func(string, string) (bool, error)); ok { return rf(bucket, backupName) } if rf, ok := ret.Get(0).(func(string, string) bool); ok { r0 = rf(bucket, backupName) } else { r0 = ret.Get(0).(bool) } if rf, ok := ret.Get(1).(func(string, string) error); ok { r1 = rf(bucket, backupName) } else { r1 = ret.Error(1) } return r0, r1 } // DeleteBackup provides a mock function with given fields: name func (_m *BackupStore) DeleteBackup(name string) error { ret := _m.Called(name) if len(ret) == 0 { panic("no return value specified for DeleteBackup") } var r0 error if rf, ok := ret.Get(0).(func(string) error); ok { r0 = rf(name) } else { r0 = ret.Error(0) } return r0 } // DeleteRestore provides a mock function with given fields: name func (_m *BackupStore) DeleteRestore(name string) error { ret := _m.Called(name) if len(ret) == 0 { panic("no return value specified for DeleteRestore") } var r0 error if rf, ok := ret.Get(0).(func(string) error); ok { r0 = rf(name) } else { r0 = ret.Error(0) } return r0 } // GetBackupContents provides a mock function with given fields: name func (_m *BackupStore) GetBackupContents(name string) (io.ReadCloser, error) { ret := _m.Called(name) if len(ret) == 0 { panic("no return value specified for GetBackupContents") } var r0 io.ReadCloser var r1 error if rf, ok := ret.Get(0).(func(string) (io.ReadCloser, error)); ok { return rf(name) } if rf, ok := ret.Get(0).(func(string) io.ReadCloser); ok { r0 = rf(name) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(io.ReadCloser) } } if rf, ok := ret.Get(1).(func(string) error); ok { r1 = rf(name) } else { r1 = ret.Error(1) } return r0, r1 } // GetBackupItemOperations provides a mock function with given fields: name func (_m *BackupStore) GetBackupItemOperations(name string) ([]*itemoperation.BackupOperation, error) { ret := _m.Called(name) if len(ret) == 0 { panic("no return value specified for GetBackupItemOperations") } var r0 []*itemoperation.BackupOperation var r1 error if rf, ok := ret.Get(0).(func(string) ([]*itemoperation.BackupOperation, error)); ok { return rf(name) } if rf, ok := ret.Get(0).(func(string) []*itemoperation.BackupOperation); ok { r0 = rf(name) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]*itemoperation.BackupOperation) } } if rf, ok := ret.Get(1).(func(string) error); ok { r1 = rf(name) } else { r1 = ret.Error(1) } return r0, r1 } // GetBackupMetadata provides a mock function with given fields: name func (_m *BackupStore) GetBackupMetadata(name string) (*v1.Backup, error) { ret := _m.Called(name) if len(ret) == 0 { panic("no return value specified for GetBackupMetadata") } var r0 *v1.Backup var r1 error if rf, ok := ret.Get(0).(func(string) (*v1.Backup, error)); ok { return rf(name) } if rf, ok := ret.Get(0).(func(string) *v1.Backup); ok { r0 = rf(name) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(*v1.Backup) } } if rf, ok := ret.Get(1).(func(string) error); ok { r1 = rf(name) } else { r1 = ret.Error(1) } return r0, r1 } // GetBackupVolumeInfos provides a mock function with given fields: name func (_m *BackupStore) GetBackupVolumeInfos(name string) ([]*volume.BackupVolumeInfo, error) { ret := _m.Called(name) if len(ret) == 0 { panic("no return value specified for GetBackupVolumeInfos") } var r0 []*volume.BackupVolumeInfo var r1 error if rf, ok := ret.Get(0).(func(string) ([]*volume.BackupVolumeInfo, error)); ok { return rf(name) } if rf, ok := ret.Get(0).(func(string) []*volume.BackupVolumeInfo); ok { r0 = rf(name) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]*volume.BackupVolumeInfo) } } if rf, ok := ret.Get(1).(func(string) error); ok { r1 = rf(name) } else { r1 = ret.Error(1) } return r0, r1 } // GetBackupVolumeSnapshots provides a mock function with given fields: name func (_m *BackupStore) GetBackupVolumeSnapshots(name string) ([]*volume.Snapshot, error) { ret := _m.Called(name) if len(ret) == 0 { panic("no return value specified for GetBackupVolumeSnapshots") } var r0 []*volume.Snapshot var r1 error if rf, ok := ret.Get(0).(func(string) ([]*volume.Snapshot, error)); ok { return rf(name) } if rf, ok := ret.Get(0).(func(string) []*volume.Snapshot); ok { r0 = rf(name) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]*volume.Snapshot) } } if rf, ok := ret.Get(1).(func(string) error); ok { r1 = rf(name) } else { r1 = ret.Error(1) } return r0, r1 } // GetCSIVolumeSnapshotClasses provides a mock function with given fields: name func (_m *BackupStore) GetCSIVolumeSnapshotClasses(name string) ([]*volumesnapshotv1.VolumeSnapshotClass, error) { ret := _m.Called(name) if len(ret) == 0 { panic("no return value specified for GetCSIVolumeSnapshotClasses") } var r0 []*volumesnapshotv1.VolumeSnapshotClass var r1 error if rf, ok := ret.Get(0).(func(string) ([]*volumesnapshotv1.VolumeSnapshotClass, error)); ok { return rf(name) } if rf, ok := ret.Get(0).(func(string) []*volumesnapshotv1.VolumeSnapshotClass); ok { r0 = rf(name) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]*volumesnapshotv1.VolumeSnapshotClass) } } if rf, ok := ret.Get(1).(func(string) error); ok { r1 = rf(name) } else { r1 = ret.Error(1) } return r0, r1 } // GetCSIVolumeSnapshotContents provides a mock function with given fields: name func (_m *BackupStore) GetCSIVolumeSnapshotContents(name string) ([]*volumesnapshotv1.VolumeSnapshotContent, error) { ret := _m.Called(name) if len(ret) == 0 { panic("no return value specified for GetCSIVolumeSnapshotContents") } var r0 []*volumesnapshotv1.VolumeSnapshotContent var r1 error if rf, ok := ret.Get(0).(func(string) ([]*volumesnapshotv1.VolumeSnapshotContent, error)); ok { return rf(name) } if rf, ok := ret.Get(0).(func(string) []*volumesnapshotv1.VolumeSnapshotContent); ok { r0 = rf(name) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]*volumesnapshotv1.VolumeSnapshotContent) } } if rf, ok := ret.Get(1).(func(string) error); ok { r1 = rf(name) } else { r1 = ret.Error(1) } return r0, r1 } // GetCSIVolumeSnapshots provides a mock function with given fields: name func (_m *BackupStore) GetCSIVolumeSnapshots(name string) ([]*volumesnapshotv1.VolumeSnapshot, error) { ret := _m.Called(name) if len(ret) == 0 { panic("no return value specified for GetCSIVolumeSnapshots") } var r0 []*volumesnapshotv1.VolumeSnapshot var r1 error if rf, ok := ret.Get(0).(func(string) ([]*volumesnapshotv1.VolumeSnapshot, error)); ok { return rf(name) } if rf, ok := ret.Get(0).(func(string) []*volumesnapshotv1.VolumeSnapshot); ok { r0 = rf(name) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]*volumesnapshotv1.VolumeSnapshot) } } if rf, ok := ret.Get(1).(func(string) error); ok { r1 = rf(name) } else { r1 = ret.Error(1) } return r0, r1 } // GetDownloadURL provides a mock function with given fields: target func (_m *BackupStore) GetDownloadURL(target v1.DownloadTarget) (string, error) { ret := _m.Called(target) if len(ret) == 0 { panic("no return value specified for GetDownloadURL") } var r0 string var r1 error if rf, ok := ret.Get(0).(func(v1.DownloadTarget) (string, error)); ok { return rf(target) } if rf, ok := ret.Get(0).(func(v1.DownloadTarget) string); ok { r0 = rf(target) } else { r0 = ret.Get(0).(string) } if rf, ok := ret.Get(1).(func(v1.DownloadTarget) error); ok { r1 = rf(target) } else { r1 = ret.Error(1) } return r0, r1 } // GetPodVolumeBackups provides a mock function with given fields: name func (_m *BackupStore) GetPodVolumeBackups(name string) ([]*v1.PodVolumeBackup, error) { ret := _m.Called(name) if len(ret) == 0 { panic("no return value specified for GetPodVolumeBackups") } var r0 []*v1.PodVolumeBackup var r1 error if rf, ok := ret.Get(0).(func(string) ([]*v1.PodVolumeBackup, error)); ok { return rf(name) } if rf, ok := ret.Get(0).(func(string) []*v1.PodVolumeBackup); ok { r0 = rf(name) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]*v1.PodVolumeBackup) } } if rf, ok := ret.Get(1).(func(string) error); ok { r1 = rf(name) } else { r1 = ret.Error(1) } return r0, r1 } // GetRestoreItemOperations provides a mock function with given fields: name func (_m *BackupStore) GetRestoreItemOperations(name string) ([]*itemoperation.RestoreOperation, error) { ret := _m.Called(name) if len(ret) == 0 { panic("no return value specified for GetRestoreItemOperations") } var r0 []*itemoperation.RestoreOperation var r1 error if rf, ok := ret.Get(0).(func(string) ([]*itemoperation.RestoreOperation, error)); ok { return rf(name) } if rf, ok := ret.Get(0).(func(string) []*itemoperation.RestoreOperation); ok { r0 = rf(name) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]*itemoperation.RestoreOperation) } } if rf, ok := ret.Get(1).(func(string) error); ok { r1 = rf(name) } else { r1 = ret.Error(1) } return r0, r1 } // GetRestoreResults provides a mock function with given fields: name func (_m *BackupStore) GetRestoreResults(name string) (map[string]results.Result, error) { ret := _m.Called(name) if len(ret) == 0 { panic("no return value specified for GetRestoreResults") } var r0 map[string]results.Result var r1 error if rf, ok := ret.Get(0).(func(string) (map[string]results.Result, error)); ok { return rf(name) } if rf, ok := ret.Get(0).(func(string) map[string]results.Result); ok { r0 = rf(name) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(map[string]results.Result) } } if rf, ok := ret.Get(1).(func(string) error); ok { r1 = rf(name) } else { r1 = ret.Error(1) } return r0, r1 } // GetRestoredResourceList provides a mock function with given fields: name func (_m *BackupStore) GetRestoredResourceList(name string) (map[string][]string, error) { ret := _m.Called(name) if len(ret) == 0 { panic("no return value specified for GetRestoredResourceList") } var r0 map[string][]string var r1 error if rf, ok := ret.Get(0).(func(string) (map[string][]string, error)); ok { return rf(name) } if rf, ok := ret.Get(0).(func(string) map[string][]string); ok { r0 = rf(name) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(map[string][]string) } } if rf, ok := ret.Get(1).(func(string) error); ok { r1 = rf(name) } else { r1 = ret.Error(1) } return r0, r1 } // IsValid provides a mock function with given fields: func (_m *BackupStore) IsValid() error { ret := _m.Called() if len(ret) == 0 { panic("no return value specified for IsValid") } var r0 error if rf, ok := ret.Get(0).(func() error); ok { r0 = rf() } else { r0 = ret.Error(0) } return r0 } // ListBackups provides a mock function with given fields: func (_m *BackupStore) ListBackups() ([]string, error) { ret := _m.Called() if len(ret) == 0 { panic("no return value specified for ListBackups") } var r0 []string var r1 error if rf, ok := ret.Get(0).(func() ([]string, error)); ok { return rf() } if rf, ok := ret.Get(0).(func() []string); ok { r0 = rf() } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]string) } } if rf, ok := ret.Get(1).(func() error); ok { r1 = rf() } else { r1 = ret.Error(1) } return r0, r1 } // PutBackup provides a mock function with given fields: info func (_m *BackupStore) PutBackup(info persistence.BackupInfo) error { ret := _m.Called(info) if len(ret) == 0 { panic("no return value specified for PutBackup") } var r0 error if rf, ok := ret.Get(0).(func(persistence.BackupInfo) error); ok { r0 = rf(info) } else { r0 = ret.Error(0) } return r0 } // PutBackupContents provides a mock function with given fields: backup, backupContents func (_m *BackupStore) PutBackupContents(backup string, backupContents io.Reader) error { ret := _m.Called(backup, backupContents) if len(ret) == 0 { panic("no return value specified for PutBackupContents") } var r0 error if rf, ok := ret.Get(0).(func(string, io.Reader) error); ok { r0 = rf(backup, backupContents) } else { r0 = ret.Error(0) } return r0 } // PutBackupItemOperations provides a mock function with given fields: backup, backupItemOperations func (_m *BackupStore) PutBackupItemOperations(backup string, backupItemOperations io.Reader) error { ret := _m.Called(backup, backupItemOperations) if len(ret) == 0 { panic("no return value specified for PutBackupItemOperations") } var r0 error if rf, ok := ret.Get(0).(func(string, io.Reader) error); ok { r0 = rf(backup, backupItemOperations) } else { r0 = ret.Error(0) } return r0 } // PutBackupMetadata provides a mock function with given fields: backup, backupMetadata func (_m *BackupStore) PutBackupMetadata(backup string, backupMetadata io.Reader) error { ret := _m.Called(backup, backupMetadata) if len(ret) == 0 { panic("no return value specified for PutBackupMetadata") } var r0 error if rf, ok := ret.Get(0).(func(string, io.Reader) error); ok { r0 = rf(backup, backupMetadata) } else { r0 = ret.Error(0) } return r0 } // PutBackupVolumeInfos provides a mock function with given fields: name, volumeInfo func (_m *BackupStore) PutBackupVolumeInfos(name string, volumeInfo io.Reader) error { ret := _m.Called(name, volumeInfo) if len(ret) == 0 { panic("no return value specified for PutBackupVolumeInfos") } var r0 error if rf, ok := ret.Get(0).(func(string, io.Reader) error); ok { r0 = rf(name, volumeInfo) } else { r0 = ret.Error(0) } return r0 } // PutRestoreItemOperations provides a mock function with given fields: restore, restoreItemOperations func (_m *BackupStore) PutRestoreItemOperations(restore string, restoreItemOperations io.Reader) error { ret := _m.Called(restore, restoreItemOperations) if len(ret) == 0 { panic("no return value specified for PutRestoreItemOperations") } var r0 error if rf, ok := ret.Get(0).(func(string, io.Reader) error); ok { r0 = rf(restore, restoreItemOperations) } else { r0 = ret.Error(0) } return r0 } // PutRestoreLog provides a mock function with given fields: backup, restore, log func (_m *BackupStore) PutRestoreLog(backup string, restore string, log io.Reader) error { ret := _m.Called(backup, restore, log) if len(ret) == 0 { panic("no return value specified for PutRestoreLog") } var r0 error if rf, ok := ret.Get(0).(func(string, string, io.Reader) error); ok { r0 = rf(backup, restore, log) } else { r0 = ret.Error(0) } return r0 } // PutRestoreResults provides a mock function with given fields: backup, restore, _a2 func (_m *BackupStore) PutRestoreResults(backup string, restore string, _a2 io.Reader) error { ret := _m.Called(backup, restore, _a2) if len(ret) == 0 { panic("no return value specified for PutRestoreResults") } var r0 error if rf, ok := ret.Get(0).(func(string, string, io.Reader) error); ok { r0 = rf(backup, restore, _a2) } else { r0 = ret.Error(0) } return r0 } // PutRestoreVolumeInfo provides a mock function with given fields: restore, volumeInfo func (_m *BackupStore) PutRestoreVolumeInfo(restore string, volumeInfo io.Reader) error { ret := _m.Called(restore, volumeInfo) if len(ret) == 0 { panic("no return value specified for PutRestoreVolumeInfo") } var r0 error if rf, ok := ret.Get(0).(func(string, io.Reader) error); ok { r0 = rf(restore, volumeInfo) } else { r0 = ret.Error(0) } return r0 } // PutRestoredResourceList provides a mock function with given fields: restore, _a1 func (_m *BackupStore) PutRestoredResourceList(restore string, _a1 io.Reader) error { ret := _m.Called(restore, _a1) if len(ret) == 0 { panic("no return value specified for PutRestoredResourceList") } var r0 error if rf, ok := ret.Get(0).(func(string, io.Reader) error); ok { r0 = rf(restore, _a1) } else { r0 = ret.Error(0) } return r0 } // NewBackupStore creates a new instance of BackupStore. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. func NewBackupStore(t interface { mock.TestingT Cleanup(func()) }) *BackupStore { mock := &BackupStore{} mock.Mock.Test(t) t.Cleanup(func() { mock.AssertExpectations(t) }) return mock } ================================================ FILE: pkg/persistence/mocks/object_store.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ // Code generated by mockery v1.0.0. DO NOT EDIT. package mocks import io "io" import mock "github.com/stretchr/testify/mock" import time "time" // ObjectStore is an autogenerated mock type for the ObjectStore type type ObjectStore struct { mock.Mock } // CreateSignedURL provides a mock function with given fields: bucket, key, ttl func (_m *ObjectStore) CreateSignedURL(bucket string, key string, ttl time.Duration) (string, error) { ret := _m.Called(bucket, key, ttl) var r0 string if rf, ok := ret.Get(0).(func(string, string, time.Duration) string); ok { r0 = rf(bucket, key, ttl) } else { r0 = ret.Get(0).(string) } var r1 error if rf, ok := ret.Get(1).(func(string, string, time.Duration) error); ok { r1 = rf(bucket, key, ttl) } else { r1 = ret.Error(1) } return r0, r1 } // DeleteObject provides a mock function with given fields: bucket, key func (_m *ObjectStore) DeleteObject(bucket string, key string) error { ret := _m.Called(bucket, key) var r0 error if rf, ok := ret.Get(0).(func(string, string) error); ok { r0 = rf(bucket, key) } else { r0 = ret.Error(0) } return r0 } // GetObject provides a mock function with given fields: bucket, key func (_m *ObjectStore) GetObject(bucket string, key string) (io.ReadCloser, error) { ret := _m.Called(bucket, key) var r0 io.ReadCloser if rf, ok := ret.Get(0).(func(string, string) io.ReadCloser); ok { r0 = rf(bucket, key) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(io.ReadCloser) } } var r1 error if rf, ok := ret.Get(1).(func(string, string) error); ok { r1 = rf(bucket, key) } else { r1 = ret.Error(1) } return r0, r1 } // Init provides a mock function with given fields: config func (_m *ObjectStore) Init(config map[string]string) error { ret := _m.Called(config) var r0 error if rf, ok := ret.Get(0).(func(map[string]string) error); ok { r0 = rf(config) } else { r0 = ret.Error(0) } return r0 } // ListCommonPrefixes provides a mock function with given fields: bucket, prefix, delimiter func (_m *ObjectStore) ListCommonPrefixes(bucket string, prefix string, delimiter string) ([]string, error) { ret := _m.Called(bucket, prefix, delimiter) var r0 []string if rf, ok := ret.Get(0).(func(string, string, string) []string); ok { r0 = rf(bucket, prefix, delimiter) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]string) } } var r1 error if rf, ok := ret.Get(1).(func(string, string, string) error); ok { r1 = rf(bucket, prefix, delimiter) } else { r1 = ret.Error(1) } return r0, r1 } // ListObjects provides a mock function with given fields: bucket, prefix func (_m *ObjectStore) ListObjects(bucket string, prefix string) ([]string, error) { ret := _m.Called(bucket, prefix) var r0 []string if rf, ok := ret.Get(0).(func(string, string) []string); ok { r0 = rf(bucket, prefix) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]string) } } var r1 error if rf, ok := ret.Get(1).(func(string, string) error); ok { r1 = rf(bucket, prefix) } else { r1 = ret.Error(1) } return r0, r1 } // ObjectExists provides a mock function with given fields: bucket, key func (_m *ObjectStore) ObjectExists(bucket string, key string) (bool, error) { ret := _m.Called(bucket, key) var r0 bool if rf, ok := ret.Get(0).(func(string, string) bool); ok { r0 = rf(bucket, key) } else { r0 = ret.Get(0).(bool) } var r1 error if rf, ok := ret.Get(1).(func(string, string) error); ok { r1 = rf(bucket, key) } else { r1 = ret.Error(1) } return r0, r1 } // PutObject provides a mock function with given fields: bucket, key, body func (_m *ObjectStore) PutObject(bucket string, key string, body io.Reader) error { ret := _m.Called(bucket, key, body) var r0 error if rf, ok := ret.Get(0).(func(string, string, io.Reader) error); ok { r0 = rf(bucket, key, body) } else { r0 = ret.Error(0) } return r0 } ================================================ FILE: pkg/persistence/object_store.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package persistence import ( "compress/gzip" "encoding/json" "io" "strings" "time" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/runtime/serializer" kerrors "k8s.io/apimachinery/pkg/util/errors" "github.com/vmware-tanzu/velero/internal/credentials" "github.com/vmware-tanzu/velero/internal/volume" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/itemoperation" "github.com/vmware-tanzu/velero/pkg/plugin/velero" "github.com/vmware-tanzu/velero/pkg/util" "github.com/vmware-tanzu/velero/pkg/util/results" ) type BackupInfo struct { Name string Metadata, Contents, Log, BackupResults, PodVolumeBackups, VolumeSnapshots, BackupItemOperations, BackupResourceList, CSIVolumeSnapshotClasses, BackupVolumeInfo io.Reader } // BackupStore defines operations for creating, retrieving, and deleting // Velero backup and restore data in/from a persistent backup store. type BackupStore interface { IsValid() error ListBackups() ([]string, error) PutBackup(info BackupInfo) error PutBackupMetadata(backup string, backupMetadata io.Reader) error PutBackupItemOperations(backup string, backupItemOperations io.Reader) error PutBackupContents(backup string, backupContents io.Reader) error GetBackupMetadata(name string) (*velerov1api.Backup, error) GetBackupItemOperations(name string) ([]*itemoperation.BackupOperation, error) GetBackupVolumeSnapshots(name string) ([]*volume.Snapshot, error) GetPodVolumeBackups(name string) ([]*velerov1api.PodVolumeBackup, error) GetBackupContents(name string) (io.ReadCloser, error) GetCSIVolumeSnapshots(name string) ([]*snapshotv1api.VolumeSnapshot, error) GetCSIVolumeSnapshotClasses(name string) ([]*snapshotv1api.VolumeSnapshotClass, error) PutBackupVolumeInfos(name string, volumeInfo io.Reader) error GetBackupVolumeInfos(name string) ([]*volume.BackupVolumeInfo, error) GetRestoreResults(name string) (map[string]results.Result, error) // BackupExists checks if the backup metadata file exists in object storage. BackupExists(bucket, backupName string) (bool, error) DeleteBackup(name string) error PutRestoreLog(backup, restore string, log io.Reader) error PutRestoreResults(backup, restore string, results io.Reader) error PutRestoredResourceList(restore string, results io.Reader) error PutRestoreItemOperations(restore string, restoreItemOperations io.Reader) error GetRestoreItemOperations(name string) ([]*itemoperation.RestoreOperation, error) PutRestoreVolumeInfo(restore string, volumeInfo io.Reader) error DeleteRestore(name string) error GetRestoredResourceList(name string) (map[string][]string, error) GetDownloadURL(target velerov1api.DownloadTarget) (string, error) } // DownloadURLTTL is how long a download URL is valid for. const DownloadURLTTL = 10 * time.Minute type objectBackupStore struct { objectStore velero.ObjectStore bucket string layout *ObjectStoreLayout logger logrus.FieldLogger } // ObjectStoreGetter is a type that can get a velero.ObjectStore // from a provider name. type ObjectStoreGetter interface { GetObjectStore(provider string) (velero.ObjectStore, error) } // ObjectBackupStoreGetter is a type that can get a velero.BackupStore for a // given BackupStorageLocation and ObjectStore. type ObjectBackupStoreGetter interface { Get(location *velerov1api.BackupStorageLocation, objectStoreGetter ObjectStoreGetter, logger logrus.FieldLogger) (BackupStore, error) } type objectBackupStoreGetter struct { credentialStore credentials.FileStore secretStore credentials.SecretStore } // NewObjectBackupStoreGetter returns a ObjectBackupStoreGetter that can get a velero.BackupStore. func NewObjectBackupStoreGetter(credentialStore credentials.FileStore) ObjectBackupStoreGetter { return &objectBackupStoreGetter{credentialStore: credentialStore} } // NewObjectBackupStoreGetterWithSecretStore returns an ObjectBackupStoreGetter with SecretStore // support for resolving caCertRef from Kubernetes Secrets. func NewObjectBackupStoreGetterWithSecretStore(credentialStore credentials.FileStore, secretStore credentials.SecretStore) ObjectBackupStoreGetter { return &objectBackupStoreGetter{ credentialStore: credentialStore, secretStore: secretStore, } } func (b *objectBackupStoreGetter) Get(location *velerov1api.BackupStorageLocation, objectStoreGetter ObjectStoreGetter, logger logrus.FieldLogger) (BackupStore, error) { if location.Spec.ObjectStorage == nil { return nil, errors.New("backup storage location does not use object storage") } if location.Spec.Provider == "" { return nil, errors.New("object storage provider name must not be empty") } // trim off any leading/trailing slashes bucket := strings.Trim(location.Spec.ObjectStorage.Bucket, "/") prefix := strings.Trim(location.Spec.ObjectStorage.Prefix, "/") // if there are any slashes in the middle of 'bucket', the user // probably put / in the bucket field, which we // don't support. // Exception: MRAP ARNs (arn:aws:s3::...) legitimately contain slashes. if strings.Contains(bucket, "/") && !strings.HasPrefix(bucket, "arn:aws:s3:") { return nil, errors.Errorf("backup storage location's bucket name %q must not contain a '/' (if using a prefix, put it in the 'Prefix' field instead)", location.Spec.ObjectStorage.Bucket) } // Pass a new map into the object store rather than modifying the passed-in // location. This prevents Velero controllers from accidentally modifying // the in-cluster BSL with data which doesn't belong in Spec.Config objectStoreConfig := make(map[string]string) if location.Spec.Config != nil { for key, val := range location.Spec.Config { objectStoreConfig[key] = val } } // add the bucket name and prefix to the config map so that object stores // can use them when initializing. The AWS object store uses the bucket // name to determine the bucket's region when setting up its client. objectStoreConfig["bucket"] = bucket objectStoreConfig["prefix"] = prefix // Only include a CACert if it's specified in order to maintain compatibility with plugins that don't expect it. // Prefer caCertRef (from Secret) over inline caCert (deprecated). if location.Spec.ObjectStorage.CACertRef != nil { if b.secretStore != nil { caCertString, err := b.secretStore.Get(location.Spec.ObjectStorage.CACertRef) if err != nil { return nil, errors.Wrap(err, "error getting CA certificate from secret") } objectStoreConfig["caCert"] = caCertString } } else if location.Spec.ObjectStorage.CACert != nil { objectStoreConfig["caCert"] = string(location.Spec.ObjectStorage.CACert) } // If the BSL specifies a credential, fetch its path on disk and pass to // plugin via the config. if location.Spec.Credential != nil { credsFile, err := b.credentialStore.Path(location.Spec.Credential) if err != nil { return nil, errors.Wrap(err, "unable to get credentials") } objectStoreConfig["credentialsFile"] = credsFile } objectStore, err := objectStoreGetter.GetObjectStore(location.Spec.Provider) if err != nil { return nil, err } if err := objectStore.Init(objectStoreConfig); err != nil { return nil, err } log := logger.WithFields(logrus.Fields(map[string]any{ "bucket": bucket, "prefix": prefix, })) return &objectBackupStore{ objectStore: objectStore, bucket: bucket, layout: NewObjectStoreLayout(prefix), logger: log, }, nil } func (s *objectBackupStore) IsValid() error { dirs, err := s.objectStore.ListCommonPrefixes(s.bucket, s.layout.rootPrefix, "/") if err != nil { return errors.WithStack(err) } var invalid []string for _, dir := range dirs { subdir := strings.TrimSuffix(strings.TrimPrefix(dir, s.layout.rootPrefix), "/") if !s.layout.isValidSubdir(subdir) { invalid = append(invalid, subdir) } } if len(invalid) > 0 { // don't include more than 3 invalid dirs in the error message if len(invalid) > 3 { return errors.Errorf("Backup store contains %d invalid top-level directories: %v", len(invalid), append(invalid[:3], "...")) } return errors.Errorf("Backup store contains invalid top-level directories: %v", invalid) } return nil } func (s *objectBackupStore) ListBackups() ([]string, error) { prefixes, err := s.objectStore.ListCommonPrefixes(s.bucket, s.layout.subdirs["backups"], "/") if err != nil { return nil, err } if len(prefixes) == 0 { return []string{}, nil } output := make([]string, 0, len(prefixes)) for _, prefix := range prefixes { // values returned from a call to ObjectStore's // ListCommonPrefixes method return the *full* prefix, inclusive // of s.backupsPrefix, and include the delimiter ("/") as a suffix. Trim // each of those off to get the backup name. backupName := strings.TrimSuffix(strings.TrimPrefix(prefix, s.layout.subdirs["backups"]), "/") output = append(output, backupName) } return output, nil } func (s *objectBackupStore) PutBackup(info BackupInfo) error { if err := seekAndPutObject(s.objectStore, s.bucket, s.layout.getBackupLogKey(info.Name), info.Log); err != nil { // Uploading the log file is best-effort; if it fails, we log the error but it doesn't impact the // backup's status. s.logger.WithError(err).WithField("backup", info.Name).Error("Error uploading log file") } if err := seekAndPutObject(s.objectStore, s.bucket, s.layout.getBackupMetadataKey(info.Name), info.Metadata); err != nil { // failure to upload metadata file is a hard-stop return err } if err := seekAndPutObject(s.objectStore, s.bucket, s.layout.getBackupContentsKey(info.Name), info.Contents); err != nil { deleteErr := s.objectStore.DeleteObject(s.bucket, s.layout.getBackupMetadataKey(info.Name)) return kerrors.NewAggregate([]error{err, deleteErr}) } // Since the logic for all of these files is the exact same except for the name and the contents, // use a map literal to iterate through them and write them to the bucket. var backupObjs = map[string]io.Reader{ s.layout.getPodVolumeBackupsKey(info.Name): info.PodVolumeBackups, s.layout.getBackupVolumeSnapshotsKey(info.Name): info.VolumeSnapshots, s.layout.getBackupItemOperationsKey(info.Name): info.BackupItemOperations, s.layout.getBackupResourceListKey(info.Name): info.BackupResourceList, s.layout.getBackupResultsKey(info.Name): info.BackupResults, s.layout.getBackupVolumeInfoKey(info.Name): info.BackupVolumeInfo, } for key, reader := range backupObjs { if err := seekAndPutObject(s.objectStore, s.bucket, key, reader); err != nil { errs := []error{err} // attempt to clean up the backup contents and metadata if we fail to upload and of the extra files. deleteErr := s.objectStore.DeleteObject(s.bucket, s.layout.getBackupContentsKey(info.Name)) errs = append(errs, deleteErr) deleteErr = s.objectStore.DeleteObject(s.bucket, s.layout.getBackupMetadataKey(info.Name)) errs = append(errs, deleteErr) return kerrors.NewAggregate(errs) } } return nil } func (s *objectBackupStore) GetBackupMetadata(name string) (*velerov1api.Backup, error) { metadataKey := s.layout.getBackupMetadataKey(name) res, err := s.objectStore.GetObject(s.bucket, metadataKey) if err != nil { return nil, err } defer res.Close() data, err := io.ReadAll(res) if err != nil { return nil, errors.WithStack(err) } codecFactory := serializer.NewCodecFactory(util.VeleroScheme) decoder := codecFactory.UniversalDecoder(velerov1api.SchemeGroupVersion) obj, _, err := decoder.Decode(data, nil, nil) if err != nil { return nil, errors.WithStack(err) } backupObj, ok := obj.(*velerov1api.Backup) if !ok { return nil, errors.Errorf("unexpected type for %s/%s: %T", s.bucket, metadataKey, obj) } return backupObj, nil } func (s *objectBackupStore) PutBackupMetadata(backup string, backupMetadata io.Reader) error { return seekAndPutObject(s.objectStore, s.bucket, s.layout.getBackupMetadataKey(backup), backupMetadata) } func (s *objectBackupStore) GetBackupVolumeSnapshots(name string) ([]*volume.Snapshot, error) { // if the volumesnapshots file doesn't exist, we don't want to return an error, since // a legacy backup or a backup with no snapshots would not have this file, so check for // its existence before attempting to get its contents. res, err := tryGet(s.objectStore, s.bucket, s.layout.getBackupVolumeSnapshotsKey(name)) if err != nil { return nil, err } if res == nil { return nil, nil } defer res.Close() var volumeSnapshots []*volume.Snapshot if err := decode(res, &volumeSnapshots); err != nil { return nil, err } return volumeSnapshots, nil } func (s *objectBackupStore) GetBackupItemOperations(name string) ([]*itemoperation.BackupOperation, error) { // if the itemoperations file doesn't exist, we don't want to return an error, since // a legacy backup or a backup with no async operations would not have this file, so check for // its existence before attempting to get its contents. res, err := tryGet(s.objectStore, s.bucket, s.layout.getBackupItemOperationsKey(name)) if err != nil { return nil, err } if res == nil { return nil, nil } defer res.Close() var backupItemOperations []*itemoperation.BackupOperation if err := decode(res, &backupItemOperations); err != nil { return nil, err } return backupItemOperations, nil } func (s *objectBackupStore) GetRestoreItemOperations(name string) ([]*itemoperation.RestoreOperation, error) { // if the itemoperations file doesn't exist, we don't want to return an error, since // a legacy restore or a restore with no async operations would not have this file, so check for // its existence before attempting to get its contents. res, err := tryGet(s.objectStore, s.bucket, s.layout.getRestoreItemOperationsKey(name)) if err != nil { return nil, err } if res == nil { return nil, nil } defer res.Close() var restoreItemOperations []*itemoperation.RestoreOperation if err := decode(res, &restoreItemOperations); err != nil { return nil, err } return restoreItemOperations, nil } // tryGet returns the object with the given key if it exists, nil if it does not exist, // or an error if it was unable to check existence or get the object. func tryGet(objectStore velero.ObjectStore, bucket, key string) (io.ReadCloser, error) { exists, err := objectStore.ObjectExists(bucket, key) if err != nil { return nil, errors.WithStack(err) } if !exists { return nil, nil } return objectStore.GetObject(bucket, key) } // decode extracts a .json.gz file reader into the object pointed to // by 'into'. func decode(jsongzReader io.Reader, into any) error { gzr, err := gzip.NewReader(jsongzReader) if err != nil { return errors.WithStack(err) } defer gzr.Close() if err := json.NewDecoder(gzr).Decode(into); err != nil { return errors.Wrap(err, "error decoding object data") } return nil } func (s *objectBackupStore) GetCSIVolumeSnapshotClasses(name string) ([]*snapshotv1api.VolumeSnapshotClass, error) { res, err := tryGet(s.objectStore, s.bucket, s.layout.getCSIVolumeSnapshotClassesKey(name)) if err != nil { return nil, err } if res == nil { // this indicates that the no CSI volumesnapshots were prensent in the backup return nil, nil } defer res.Close() var csiVSClasses []*snapshotv1api.VolumeSnapshotClass if err := decode(res, &csiVSClasses); err != nil { return nil, err } return csiVSClasses, nil } func (s *objectBackupStore) GetCSIVolumeSnapshots(name string) ([]*snapshotv1api.VolumeSnapshot, error) { res, err := tryGet(s.objectStore, s.bucket, s.layout.getCSIVolumeSnapshotKey(name)) if err != nil { return nil, err } if res == nil { // this indicates that the no CSI volumesnapshots were prensent in the backup return nil, nil } defer res.Close() var csiSnaps []*snapshotv1api.VolumeSnapshot if err := decode(res, &csiSnaps); err != nil { return nil, err } return csiSnaps, nil } func (s *objectBackupStore) GetCSIVolumeSnapshotContents(name string) ([]*snapshotv1api.VolumeSnapshotContent, error) { res, err := tryGet(s.objectStore, s.bucket, s.layout.getCSIVolumeSnapshotContentsKey(name)) if err != nil { return nil, err } if res == nil { // this indicates that the no CSI volumesnapshotcontents were prensent in the backup return nil, nil } defer res.Close() var snapConts []*snapshotv1api.VolumeSnapshotContent if err := decode(res, &snapConts); err != nil { return nil, err } return snapConts, nil } func (s *objectBackupStore) GetPodVolumeBackups(name string) ([]*velerov1api.PodVolumeBackup, error) { // if the podvolumebackups file doesn't exist, we don't want to return an error, since // a legacy backup or a backup with no pod volume backups would not have this file, so // check for its existence before attempting to get its contents. res, err := tryGet(s.objectStore, s.bucket, s.layout.getPodVolumeBackupsKey(name)) if err != nil { return nil, err } if res == nil { return nil, nil } defer res.Close() var podVolumeBackups []*velerov1api.PodVolumeBackup if err := decode(res, &podVolumeBackups); err != nil { return nil, err } return podVolumeBackups, nil } func (s *objectBackupStore) GetBackupVolumeInfos(name string) ([]*volume.BackupVolumeInfo, error) { volumeInfos := make([]*volume.BackupVolumeInfo, 0) res, err := tryGet(s.objectStore, s.bucket, s.layout.getBackupVolumeInfoKey(name)) if err != nil { return volumeInfos, err } if res == nil { return volumeInfos, nil } defer res.Close() if err := decode(res, &volumeInfos); err != nil { return volumeInfos, err } return volumeInfos, nil } func (s *objectBackupStore) PutBackupVolumeInfos(name string, volumeInfo io.Reader) error { return s.objectStore.PutObject(s.bucket, s.layout.getBackupVolumeInfoKey(name), volumeInfo) } func (s *objectBackupStore) GetRestoreResults(name string) (map[string]results.Result, error) { results := make(map[string]results.Result) res, err := tryGet(s.objectStore, s.bucket, s.layout.getRestoreResultsKey(name)) if err != nil { return results, err } if res == nil { return results, nil } defer res.Close() if err := decode(res, &results); err != nil { return results, err } return results, nil } func (s *objectBackupStore) GetBackupContents(name string) (io.ReadCloser, error) { return s.objectStore.GetObject(s.bucket, s.layout.getBackupContentsKey(name)) } func (s *objectBackupStore) BackupExists(bucket, backupName string) (bool, error) { return s.objectStore.ObjectExists(bucket, s.layout.getBackupMetadataKey(backupName)) } func (s *objectBackupStore) DeleteBackup(name string) error { objects, err := s.objectStore.ListObjects(s.bucket, s.layout.getBackupDir(name)) if err != nil { return err } var errs []error for _, key := range objects { s.logger.WithFields(logrus.Fields{ "key": key, }).Debug("Trying to delete object") if err := s.objectStore.DeleteObject(s.bucket, key); err != nil { errs = append(errs, err) } } return errors.WithStack(kerrors.NewAggregate(errs)) } func (s *objectBackupStore) DeleteRestore(name string) error { objects, err := s.objectStore.ListObjects(s.bucket, s.layout.getRestoreDir(name)) if err != nil { return err } var errs []error for _, key := range objects { s.logger.WithFields(logrus.Fields{ "key": key, }).Debug("Trying to delete object") if err := s.objectStore.DeleteObject(s.bucket, key); err != nil { errs = append(errs, err) } } return errors.WithStack(kerrors.NewAggregate(errs)) } func (s *objectBackupStore) PutRestoreLog(backup string, restore string, log io.Reader) error { return s.objectStore.PutObject(s.bucket, s.layout.getRestoreLogKey(restore), log) } func (s *objectBackupStore) PutRestoreResults(backup string, restore string, results io.Reader) error { return s.objectStore.PutObject(s.bucket, s.layout.getRestoreResultsKey(restore), results) } func (s *objectBackupStore) PutRestoredResourceList(restore string, list io.Reader) error { return s.objectStore.PutObject(s.bucket, s.layout.getRestoreResourceListKey(restore), list) } func (s *objectBackupStore) PutRestoreItemOperations(restore string, restoreItemOperations io.Reader) error { return seekAndPutObject(s.objectStore, s.bucket, s.layout.getRestoreItemOperationsKey(restore), restoreItemOperations) } func (s *objectBackupStore) PutRestoreVolumeInfo(restore string, volumeInfo io.Reader) error { return seekAndPutObject(s.objectStore, s.bucket, s.layout.getRestoreVolumeInfoKey(restore), volumeInfo) } func (s *objectBackupStore) PutBackupItemOperations(backup string, backupItemOperations io.Reader) error { return seekAndPutObject(s.objectStore, s.bucket, s.layout.getBackupItemOperationsKey(backup), backupItemOperations) } func (s *objectBackupStore) PutBackupContents(backup string, backupContents io.Reader) error { return seekAndPutObject(s.objectStore, s.bucket, s.layout.getBackupContentsKey(backup), backupContents) } func (s *objectBackupStore) GetDownloadURL(target velerov1api.DownloadTarget) (string, error) { switch target.Kind { case velerov1api.DownloadTargetKindBackupContents: return s.objectStore.CreateSignedURL(s.bucket, s.layout.getBackupContentsKey(target.Name), DownloadURLTTL) case velerov1api.DownloadTargetKindBackupLog: return s.objectStore.CreateSignedURL(s.bucket, s.layout.getBackupLogKey(target.Name), DownloadURLTTL) case velerov1api.DownloadTargetKindBackupVolumeSnapshots: return s.objectStore.CreateSignedURL(s.bucket, s.layout.getBackupVolumeSnapshotsKey(target.Name), DownloadURLTTL) case velerov1api.DownloadTargetKindBackupItemOperations: return s.objectStore.CreateSignedURL(s.bucket, s.layout.getBackupItemOperationsKey(target.Name), DownloadURLTTL) case velerov1api.DownloadTargetKindRestoreItemOperations: return s.objectStore.CreateSignedURL(s.bucket, s.layout.getRestoreItemOperationsKey(target.Name), DownloadURLTTL) case velerov1api.DownloadTargetKindBackupResourceList: return s.objectStore.CreateSignedURL(s.bucket, s.layout.getBackupResourceListKey(target.Name), DownloadURLTTL) case velerov1api.DownloadTargetKindRestoreLog: return s.objectStore.CreateSignedURL(s.bucket, s.layout.getRestoreLogKey(target.Name), DownloadURLTTL) case velerov1api.DownloadTargetKindRestoreResults: return s.objectStore.CreateSignedURL(s.bucket, s.layout.getRestoreResultsKey(target.Name), DownloadURLTTL) case velerov1api.DownloadTargetKindRestoreResourceList: return s.objectStore.CreateSignedURL(s.bucket, s.layout.getRestoreResourceListKey(target.Name), DownloadURLTTL) case velerov1api.DownloadTargetKindCSIBackupVolumeSnapshots: return s.objectStore.CreateSignedURL(s.bucket, s.layout.getCSIVolumeSnapshotKey(target.Name), DownloadURLTTL) case velerov1api.DownloadTargetKindCSIBackupVolumeSnapshotContents: return s.objectStore.CreateSignedURL(s.bucket, s.layout.getCSIVolumeSnapshotContentsKey(target.Name), DownloadURLTTL) case velerov1api.DownloadTargetKindBackupResults: return s.objectStore.CreateSignedURL(s.bucket, s.layout.getBackupResultsKey(target.Name), DownloadURLTTL) case velerov1api.DownloadTargetKindBackupVolumeInfos: return s.objectStore.CreateSignedURL(s.bucket, s.layout.getBackupVolumeInfoKey(target.Name), DownloadURLTTL) case velerov1api.DownloadTargetKindRestoreVolumeInfo: return s.objectStore.CreateSignedURL(s.bucket, s.layout.getRestoreVolumeInfoKey(target.Name), DownloadURLTTL) default: return "", errors.Errorf("unsupported download target kind %q", target.Kind) } } func (s *objectBackupStore) GetRestoredResourceList(name string) (map[string][]string, error) { list := make(map[string][]string) res, err := tryGet(s.objectStore, s.bucket, s.layout.getRestoreResourceListKey(name)) if err != nil { return list, err } if res == nil { return list, nil } defer res.Close() if err := decode(res, &list); err != nil { return list, err } return list, nil } func seekToBeginning(r io.Reader) error { seeker, ok := r.(io.Seeker) if !ok { return nil } _, err := seeker.Seek(0, 0) return err } func seekAndPutObject(objectStore velero.ObjectStore, bucket, key string, file io.Reader) error { if file == nil { return nil } if err := seekToBeginning(file); err != nil { return errors.WithStack(err) } return objectStore.PutObject(bucket, key, file) } ================================================ FILE: pkg/persistence/object_store_layout.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package persistence import ( "fmt" "path" "strings" ) // ObjectStoreLayout defines how Velero's persisted files map to // keys in an object storage bucket. type ObjectStoreLayout struct { rootPrefix string subdirs map[string]string } func NewObjectStoreLayout(prefix string) *ObjectStoreLayout { if prefix != "" && !strings.HasSuffix(prefix, "/") { prefix = prefix + "/" } subdirs := map[string]string{ "backups": path.Join(prefix, "backups") + "/", "restores": path.Join(prefix, "restores") + "/", "restic": path.Join(prefix, "restic") + "/", "metadata": path.Join(prefix, "metadata") + "/", "plugins": path.Join(prefix, "plugins") + "/", "kopia": path.Join(prefix, "kopia") + "/", } return &ObjectStoreLayout{ rootPrefix: prefix, subdirs: subdirs, } } // GetResticDir returns the full prefix representing the restic // directory within an object storage bucket containing a backup // store. func (l *ObjectStoreLayout) GetResticDir() string { return l.subdirs["restic"] } func (l *ObjectStoreLayout) isValidSubdir(name string) bool { _, ok := l.subdirs[name] return ok } func (l *ObjectStoreLayout) getBackupDir(backup string) string { return path.Join(l.subdirs["backups"], backup) + "/" } func (l *ObjectStoreLayout) getRestoreDir(restore string) string { return path.Join(l.subdirs["restores"], restore) + "/" } func (l *ObjectStoreLayout) getBackupMetadataKey(backup string) string { return path.Join(l.subdirs["backups"], backup, "velero-backup.json") } func (l *ObjectStoreLayout) getBackupContentsKey(backup string) string { return path.Join(l.subdirs["backups"], backup, fmt.Sprintf("%s.tar.gz", backup)) } func (l *ObjectStoreLayout) getBackupLogKey(backup string) string { return path.Join(l.subdirs["backups"], backup, fmt.Sprintf("%s-logs.gz", backup)) } func (l *ObjectStoreLayout) getPodVolumeBackupsKey(backup string) string { return path.Join(l.subdirs["backups"], backup, fmt.Sprintf("%s-podvolumebackups.json.gz", backup)) } func (l *ObjectStoreLayout) getBackupVolumeSnapshotsKey(backup string) string { return path.Join(l.subdirs["backups"], backup, fmt.Sprintf("%s-volumesnapshots.json.gz", backup)) } func (l *ObjectStoreLayout) getBackupItemOperationsKey(backup string) string { return path.Join(l.subdirs["backups"], backup, fmt.Sprintf("%s-itemoperations.json.gz", backup)) } func (l *ObjectStoreLayout) getBackupResourceListKey(backup string) string { return path.Join(l.subdirs["backups"], backup, fmt.Sprintf("%s-resource-list.json.gz", backup)) } func (l *ObjectStoreLayout) getRestoreLogKey(restore string) string { return path.Join(l.subdirs["restores"], restore, fmt.Sprintf("restore-%s-logs.gz", restore)) } func (l *ObjectStoreLayout) getRestoreResultsKey(restore string) string { return path.Join(l.subdirs["restores"], restore, fmt.Sprintf("restore-%s-results.gz", restore)) } func (l *ObjectStoreLayout) getRestoreResourceListKey(restore string) string { return path.Join(l.subdirs["restores"], restore, fmt.Sprintf("restore-%s-resource-list.json.gz", restore)) } func (l *ObjectStoreLayout) getRestoreItemOperationsKey(restore string) string { return path.Join(l.subdirs["restores"], restore, fmt.Sprintf("restore-%s-itemoperations.json.gz", restore)) } func (l *ObjectStoreLayout) getCSIVolumeSnapshotKey(backup string) string { return path.Join(l.subdirs["backups"], backup, fmt.Sprintf("%s-csi-volumesnapshots.json.gz", backup)) } func (l *ObjectStoreLayout) getCSIVolumeSnapshotContentsKey(backup string) string { return path.Join(l.subdirs["backups"], backup, fmt.Sprintf("%s-csi-volumesnapshotcontents.json.gz", backup)) } func (l *ObjectStoreLayout) getCSIVolumeSnapshotClassesKey(backup string) string { return path.Join(l.subdirs["backups"], backup, fmt.Sprintf("%s-csi-volumesnapshotclasses.json.gz", backup)) } func (l *ObjectStoreLayout) getBackupResultsKey(backup string) string { return path.Join(l.subdirs["backups"], backup, fmt.Sprintf("%s-results.gz", backup)) } func (l *ObjectStoreLayout) getBackupVolumeInfoKey(backup string) string { return path.Join(l.subdirs["backups"], backup, fmt.Sprintf("%s-volumeinfo.json.gz", backup)) } func (l *ObjectStoreLayout) getRestoreVolumeInfoKey(restore string) string { return path.Join(l.subdirs["restores"], restore, fmt.Sprintf("%s-volumeinfo.json.gz", restore)) } ================================================ FILE: pkg/persistence/object_store_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package persistence import ( "bytes" "compress/gzip" "encoding/json" "errors" "fmt" "io" "sort" "strings" "testing" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "github.com/vmware-tanzu/velero/internal/credentials" "github.com/vmware-tanzu/velero/internal/volume" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/itemoperation" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/plugin/velero" providermocks "github.com/vmware-tanzu/velero/pkg/plugin/velero/mocks" velerotest "github.com/vmware-tanzu/velero/pkg/test" "github.com/vmware-tanzu/velero/pkg/util/encode" "github.com/vmware-tanzu/velero/pkg/util/results" ) type objectBackupStoreTestHarness struct { // embedded to reduce verbosity when calling methods *objectBackupStore objectStore *inMemoryObjectStore bucket, prefix string } func newObjectBackupStoreTestHarness(bucket, prefix string) *objectBackupStoreTestHarness { objectStore := newInMemoryObjectStore(bucket) return &objectBackupStoreTestHarness{ objectBackupStore: &objectBackupStore{ objectStore: objectStore, bucket: bucket, layout: NewObjectStoreLayout(prefix), logger: velerotest.NewLogger(), }, objectStore: objectStore, bucket: bucket, prefix: prefix, } } func TestIsValid(t *testing.T) { tests := []struct { name string prefix string storageData BucketData expectErr bool }{ { name: "empty backup store with no prefix is valid", expectErr: false, }, { name: "empty backup store with a prefix is valid", prefix: "bar", expectErr: false, }, { name: "backup store with no prefix and only unsupported directories is invalid", storageData: map[string][]byte{ "backup-1/velero-backup.json": {}, "backup-2/velero-backup.json": {}, }, expectErr: true, }, { name: "backup store with a prefix and only unsupported directories is invalid", prefix: "backups", storageData: map[string][]byte{ "backups/backup-1/velero-backup.json": {}, "backups/backup-2/velero-backup.json": {}, }, expectErr: true, }, { name: "backup store with no prefix and both supported and unsupported directories is invalid", storageData: map[string][]byte{ "backups/backup-1/velero-backup.json": {}, "backups/backup-2/velero-backup.json": {}, "restores/restore-1/foo": {}, "unsupported-dir/foo": {}, }, expectErr: true, }, { name: "backup store with a prefix and both supported and unsupported directories is invalid", prefix: "cluster-1", storageData: map[string][]byte{ "cluster-1/backups/backup-1/velero-backup.json": {}, "cluster-1/backups/backup-2/velero-backup.json": {}, "cluster-1/restores/restore-1/foo": {}, "cluster-1/unsupported-dir/foo": {}, }, expectErr: true, }, { name: "backup store with no prefix and only supported directories is valid", storageData: map[string][]byte{ "backups/backup-1/velero-backup.json": {}, "backups/backup-2/velero-backup.json": {}, "restores/restore-1/foo": {}, }, expectErr: false, }, { name: "backup store with a prefix and only supported directories is valid", prefix: "cluster-1", storageData: map[string][]byte{ "cluster-1/backups/backup-1/velero-backup.json": {}, "cluster-1/backups/backup-2/velero-backup.json": {}, "cluster-1/restores/restore-1/foo": {}, }, expectErr: false, }, { name: "backup store with plugins directory is valid", storageData: map[string][]byte{ "plugins/vsphere/foo": {}, }, expectErr: false, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { harness := newObjectBackupStoreTestHarness("foo", tc.prefix) for key, obj := range tc.storageData { require.NoError(t, harness.objectStore.PutObject(harness.bucket, key, bytes.NewReader(obj))) } err := harness.IsValid() if tc.expectErr { assert.Error(t, err) } else { assert.NoError(t, err) } }) } } func TestListBackups(t *testing.T) { tests := []struct { name string prefix string storageData BucketData expectedRes []string expectedErr string }{ { name: "normal case", storageData: map[string][]byte{ "backups/backup-1/velero-backup.json": encodeToBytes(builder.ForBackup("", "backup-1").Result()), "backups/backup-2/velero-backup.json": encodeToBytes(builder.ForBackup("", "backup-2").Result()), }, expectedRes: []string{"backup-1", "backup-2"}, }, { name: "normal case with backup store prefix", prefix: "velero-backups/", storageData: map[string][]byte{ "velero-backups/backups/backup-1/velero-backup.json": encodeToBytes(builder.ForBackup("", "backup-1").Result()), "velero-backups/backups/backup-2/velero-backup.json": encodeToBytes(builder.ForBackup("", "backup-2").Result()), }, expectedRes: []string{"backup-1", "backup-2"}, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { harness := newObjectBackupStoreTestHarness("foo", tc.prefix) for key, obj := range tc.storageData { require.NoError(t, harness.objectStore.PutObject(harness.bucket, key, bytes.NewReader(obj))) } res, err := harness.ListBackups() velerotest.AssertErrorMatches(t, tc.expectedErr, err) sort.Strings(tc.expectedRes) sort.Strings(res) assert.Equal(t, tc.expectedRes, res) }) } } func TestPutBackup(t *testing.T) { tests := []struct { name string prefix string metadata io.Reader contents io.Reader log io.Reader podVolumeBackup io.Reader snapshots io.Reader backupItemOperations io.Reader resourceList io.Reader backupVolumeInfo io.Reader expectedErr string expectedKeys []string }{ { name: "normal case", metadata: newStringReadSeeker("metadata"), contents: newStringReadSeeker("contents"), log: newStringReadSeeker("log"), podVolumeBackup: newStringReadSeeker("podVolumeBackup"), snapshots: newStringReadSeeker("snapshots"), backupItemOperations: newStringReadSeeker("backupItemOperations"), resourceList: newStringReadSeeker("resourceList"), backupVolumeInfo: newStringReadSeeker("backupVolumeInfo"), expectedErr: "", expectedKeys: []string{ "backups/backup-1/velero-backup.json", "backups/backup-1/backup-1.tar.gz", "backups/backup-1/backup-1-logs.gz", "backups/backup-1/backup-1-podvolumebackups.json.gz", "backups/backup-1/backup-1-volumesnapshots.json.gz", "backups/backup-1/backup-1-itemoperations.json.gz", "backups/backup-1/backup-1-resource-list.json.gz", "backups/backup-1/backup-1-volumeinfo.json.gz", }, }, { name: "normal case with backup store prefix", prefix: "prefix-1/", metadata: newStringReadSeeker("metadata"), contents: newStringReadSeeker("contents"), log: newStringReadSeeker("log"), podVolumeBackup: newStringReadSeeker("podVolumeBackup"), snapshots: newStringReadSeeker("snapshots"), backupItemOperations: newStringReadSeeker("backupItemOperations"), resourceList: newStringReadSeeker("resourceList"), backupVolumeInfo: newStringReadSeeker("backupVolumeInfo"), expectedErr: "", expectedKeys: []string{ "prefix-1/backups/backup-1/velero-backup.json", "prefix-1/backups/backup-1/backup-1.tar.gz", "prefix-1/backups/backup-1/backup-1-logs.gz", "prefix-1/backups/backup-1/backup-1-podvolumebackups.json.gz", "prefix-1/backups/backup-1/backup-1-volumesnapshots.json.gz", "prefix-1/backups/backup-1/backup-1-itemoperations.json.gz", "prefix-1/backups/backup-1/backup-1-resource-list.json.gz", "prefix-1/backups/backup-1/backup-1-volumeinfo.json.gz", }, }, { name: "error on metadata upload does not upload data", metadata: new(errorReader), contents: newStringReadSeeker("contents"), log: newStringReadSeeker("log"), podVolumeBackup: newStringReadSeeker("podVolumeBackup"), snapshots: newStringReadSeeker("snapshots"), backupItemOperations: newStringReadSeeker("backupItemOperations"), resourceList: newStringReadSeeker("resourceList"), backupVolumeInfo: newStringReadSeeker("backupVolumeInfo"), expectedErr: "error readers return errors", expectedKeys: []string{"backups/backup-1/backup-1-logs.gz"}, }, { name: "error on data upload deletes metadata", metadata: newStringReadSeeker("metadata"), contents: new(errorReader), log: newStringReadSeeker("log"), snapshots: newStringReadSeeker("snapshots"), backupItemOperations: newStringReadSeeker("backupItemOperations"), resourceList: newStringReadSeeker("resourceList"), backupVolumeInfo: newStringReadSeeker("backupVolumeInfo"), expectedErr: "error readers return errors", expectedKeys: []string{"backups/backup-1/backup-1-logs.gz"}, }, { name: "error on log upload is ok", metadata: newStringReadSeeker("foo"), contents: newStringReadSeeker("bar"), log: new(errorReader), podVolumeBackup: newStringReadSeeker("podVolumeBackup"), snapshots: newStringReadSeeker("snapshots"), backupItemOperations: newStringReadSeeker("backupItemOperations"), resourceList: newStringReadSeeker("resourceList"), backupVolumeInfo: newStringReadSeeker("backupVolumeInfo"), expectedErr: "", expectedKeys: []string{ "backups/backup-1/velero-backup.json", "backups/backup-1/backup-1.tar.gz", "backups/backup-1/backup-1-podvolumebackups.json.gz", "backups/backup-1/backup-1-volumesnapshots.json.gz", "backups/backup-1/backup-1-itemoperations.json.gz", "backups/backup-1/backup-1-resource-list.json.gz", "backups/backup-1/backup-1-volumeinfo.json.gz", }, }, { name: "data should be uploaded even when metadata is nil", metadata: nil, contents: newStringReadSeeker("contents"), log: newStringReadSeeker("log"), podVolumeBackup: newStringReadSeeker("podVolumeBackup"), snapshots: newStringReadSeeker("snapshots"), resourceList: newStringReadSeeker("resourceList"), backupVolumeInfo: newStringReadSeeker("backupVolumeInfo"), expectedErr: "", expectedKeys: []string{ "backups/backup-1/backup-1.tar.gz", "backups/backup-1/backup-1-logs.gz", "backups/backup-1/backup-1-podvolumebackups.json.gz", "backups/backup-1/backup-1-volumesnapshots.json.gz", "backups/backup-1/backup-1-resource-list.json.gz", "backups/backup-1/backup-1-volumeinfo.json.gz", }, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { harness := newObjectBackupStoreTestHarness("foo", tc.prefix) backupInfo := BackupInfo{ Name: "backup-1", Metadata: tc.metadata, Contents: tc.contents, Log: tc.log, PodVolumeBackups: tc.podVolumeBackup, VolumeSnapshots: tc.snapshots, BackupItemOperations: tc.backupItemOperations, BackupResourceList: tc.resourceList, BackupVolumeInfo: tc.backupVolumeInfo, } err := harness.PutBackup(backupInfo) velerotest.AssertErrorMatches(t, tc.expectedErr, err) assert.Len(t, harness.objectStore.Data[harness.bucket], len(tc.expectedKeys)) for _, key := range tc.expectedKeys { assert.Contains(t, harness.objectStore.Data[harness.bucket], key) } }) } } func TestGetBackupMetadata(t *testing.T) { tests := []struct { name string backupName string key string obj metav1.Object wantErr error }{ { name: "metadata file returns correctly", backupName: "foo", key: "backups/foo/velero-backup.json", obj: builder.ForBackup(velerov1api.DefaultNamespace, "foo").Result(), }, { name: "no metadata file returns an error", backupName: "foo", wantErr: errors.New("key not found"), }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { harness := newObjectBackupStoreTestHarness("test-bucket", "") if tc.obj != nil { jsonBytes, err := json.Marshal(tc.obj) require.NoError(t, err) require.NoError(t, harness.objectStore.PutObject(harness.bucket, tc.key, bytes.NewReader(jsonBytes))) } res, err := harness.GetBackupMetadata(tc.backupName) if tc.wantErr != nil { assert.Equal(t, tc.wantErr, err) } else { require.NoError(t, err) assert.Equal(t, tc.obj.GetNamespace(), res.Namespace) assert.Equal(t, tc.obj.GetName(), res.Name) } }) } } func TestGetBackupVolumeSnapshots(t *testing.T) { harness := newObjectBackupStoreTestHarness("test-bucket", "") // volumesnapshots file not found should not error harness.objectStore.PutObject(harness.bucket, "backups/test-backup/velero-backup.json", newStringReadSeeker("foo")) res, err := harness.GetBackupVolumeSnapshots("test-backup") require.NoError(t, err) assert.Nil(t, res) // volumesnapshots file containing invalid data should error harness.objectStore.PutObject(harness.bucket, "backups/test-backup/test-backup-volumesnapshots.json.gz", newStringReadSeeker("foo")) _, err = harness.GetBackupVolumeSnapshots("test-backup") require.Error(t, err) // volumesnapshots file containing gzipped json data should return correctly snapshots := []*volume.Snapshot{ { Spec: volume.SnapshotSpec{ BackupName: "test-backup", PersistentVolumeName: "pv-1", }, }, { Spec: volume.SnapshotSpec{ BackupName: "test-backup", PersistentVolumeName: "pv-2", }, }, } obj := new(bytes.Buffer) gzw := gzip.NewWriter(obj) require.NoError(t, json.NewEncoder(gzw).Encode(snapshots)) require.NoError(t, gzw.Close()) require.NoError(t, harness.objectStore.PutObject(harness.bucket, "backups/test-backup/test-backup-volumesnapshots.json.gz", obj)) res, err = harness.GetBackupVolumeSnapshots("test-backup") require.NoError(t, err) assert.Equal(t, snapshots, res) } func TestGetBackupItemOperations(t *testing.T) { harness := newObjectBackupStoreTestHarness("test-bucket", "") // itemoperations file not found should not error harness.objectStore.PutObject(harness.bucket, "backups/test-backup/velero-backup.json", newStringReadSeeker("foo")) res, err := harness.GetBackupItemOperations("test-backup") require.NoError(t, err) assert.Nil(t, res) // itemoperations file containing invalid data should error harness.objectStore.PutObject(harness.bucket, "backups/test-backup/test-backup-itemoperations.json.gz", newStringReadSeeker("foo")) _, err = harness.GetBackupItemOperations("test-backup") require.Error(t, err) // itemoperations file containing gzipped json data should return correctly operations := []*itemoperation.BackupOperation{ { Spec: itemoperation.BackupOperationSpec{ BackupName: "test-backup", ResourceIdentifier: velero.ResourceIdentifier{ GroupResource: kuberesource.Pods, Namespace: "ns", Name: "item-1", }, }, }, { Spec: itemoperation.BackupOperationSpec{ BackupName: "test-backup", ResourceIdentifier: velero.ResourceIdentifier{ GroupResource: kuberesource.Pods, Namespace: "ns", Name: "item-2", }, }, }, } obj := new(bytes.Buffer) gzw := gzip.NewWriter(obj) require.NoError(t, json.NewEncoder(gzw).Encode(operations)) require.NoError(t, gzw.Close()) require.NoError(t, harness.objectStore.PutObject(harness.bucket, "backups/test-backup/test-backup-itemoperations.json.gz", obj)) res, err = harness.GetBackupItemOperations("test-backup") require.NoError(t, err) assert.Equal(t, operations, res) } func TestGetRestoreItemOperations(t *testing.T) { harness := newObjectBackupStoreTestHarness("test-bucket", "") // itemoperations file not found should not error res, err := harness.GetRestoreItemOperations("test-restore") require.NoError(t, err) assert.Nil(t, res) // itemoperations file containing invalid data should error harness.objectStore.PutObject(harness.bucket, "restores/test-restore/restore-test-restore-itemoperations.json.gz", newStringReadSeeker("foo")) _, err = harness.GetRestoreItemOperations("test-restore") require.Error(t, err) // itemoperations file containing gzipped json data should return correctly operations := []*itemoperation.RestoreOperation{ { Spec: itemoperation.RestoreOperationSpec{ RestoreName: "test-restore", ResourceIdentifier: velero.ResourceIdentifier{ GroupResource: kuberesource.Pods, Namespace: "ns", Name: "item-1", }, }, }, { Spec: itemoperation.RestoreOperationSpec{ RestoreName: "test-restore", ResourceIdentifier: velero.ResourceIdentifier{ GroupResource: kuberesource.Pods, Namespace: "ns", Name: "item-2", }, }, }, } obj := new(bytes.Buffer) gzw := gzip.NewWriter(obj) require.NoError(t, json.NewEncoder(gzw).Encode(operations)) require.NoError(t, gzw.Close()) require.NoError(t, harness.objectStore.PutObject(harness.bucket, "restores/test-restore/restore-test-restore-itemoperations.json.gz", obj)) res, err = harness.GetRestoreItemOperations("test-restore") require.NoError(t, err) assert.Equal(t, operations, res) } func TestGetBackupContents(t *testing.T) { harness := newObjectBackupStoreTestHarness("test-bucket", "") harness.objectStore.PutObject(harness.bucket, "backups/test-backup/test-backup.tar.gz", newStringReadSeeker("foo")) rc, err := harness.GetBackupContents("test-backup") require.NoError(t, err) require.NotNil(t, rc) data, err := io.ReadAll(rc) require.NoError(t, err) assert.Equal(t, "foo", string(data)) } func TestDeleteBackup(t *testing.T) { tests := []struct { name string prefix string listObjectsError error deleteErrors []error expectedErr string }{ { name: "normal case", }, { name: "normal case with backup store prefix", prefix: "velero-backups/", }, { name: "some delete errors, do as much as we can", deleteErrors: []error{errors.New("a"), nil, errors.New("c")}, expectedErr: "[a, c]", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { objectStore := new(providermocks.ObjectStore) backupStore := &objectBackupStore{ objectStore: objectStore, bucket: "test-bucket", layout: NewObjectStoreLayout(test.prefix), logger: velerotest.NewLogger(), } defer objectStore.AssertExpectations(t) objects := []string{test.prefix + "backups/bak/velero-backup.json", test.prefix + "backups/bak/bak.tar.gz", test.prefix + "backups/bak/bak.log.gz"} objectStore.On("ListObjects", backupStore.bucket, test.prefix+"backups/bak/").Return(objects, test.listObjectsError) for i, obj := range objects { var err error if i < len(test.deleteErrors) { err = test.deleteErrors[i] } objectStore.On("DeleteObject", backupStore.bucket, obj).Return(err) } err := backupStore.DeleteBackup("bak") velerotest.AssertErrorMatches(t, test.expectedErr, err) }) } } func TestDeleteRestore(t *testing.T) { tests := []struct { name string prefix string listObjectsError error deleteErrors []error expectedErr string }{ { name: "normal case", }, { name: "normal case with backup store prefix", prefix: "velero-backups/", }, { name: "some delete errors, do as much as we can", deleteErrors: []error{errors.New("a"), nil, errors.New("c")}, expectedErr: "[a, c]", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { objectStore := new(providermocks.ObjectStore) backupStore := &objectBackupStore{ objectStore: objectStore, bucket: "test-bucket", layout: NewObjectStoreLayout(test.prefix), logger: velerotest.NewLogger(), } defer objectStore.AssertExpectations(t) objects := []string{test.prefix + "restores/bak/velero-restore.json", test.prefix + "restores/bak/bak.tar.gz", test.prefix + "restores/bak/bak.log.gz"} objectStore.On("ListObjects", backupStore.bucket, test.prefix+"restores/bak/").Return(objects, test.listObjectsError) for i, obj := range objects { var err error if i < len(test.deleteErrors) { err = test.deleteErrors[i] } objectStore.On("DeleteObject", backupStore.bucket, obj).Return(err) } err := backupStore.DeleteRestore("bak") velerotest.AssertErrorMatches(t, test.expectedErr, err) }) } } func TestGetDownloadURL(t *testing.T) { tests := []struct { name string targetName string expectedKeyByKind map[velerov1api.DownloadTargetKind]string prefix string }{ { name: "backup", targetName: "my-backup", expectedKeyByKind: map[velerov1api.DownloadTargetKind]string{ velerov1api.DownloadTargetKindBackupContents: "backups/my-backup/my-backup.tar.gz", velerov1api.DownloadTargetKindBackupLog: "backups/my-backup/my-backup-logs.gz", velerov1api.DownloadTargetKindBackupVolumeSnapshots: "backups/my-backup/my-backup-volumesnapshots.json.gz", velerov1api.DownloadTargetKindBackupItemOperations: "backups/my-backup/my-backup-itemoperations.json.gz", velerov1api.DownloadTargetKindBackupResourceList: "backups/my-backup/my-backup-resource-list.json.gz", }, }, { name: "backup with prefix", targetName: "my-backup", prefix: "velero-backups/", expectedKeyByKind: map[velerov1api.DownloadTargetKind]string{ velerov1api.DownloadTargetKindBackupContents: "velero-backups/backups/my-backup/my-backup.tar.gz", velerov1api.DownloadTargetKindBackupLog: "velero-backups/backups/my-backup/my-backup-logs.gz", velerov1api.DownloadTargetKindBackupVolumeSnapshots: "velero-backups/backups/my-backup/my-backup-volumesnapshots.json.gz", velerov1api.DownloadTargetKindBackupItemOperations: "velero-backups/backups/my-backup/my-backup-itemoperations.json.gz", velerov1api.DownloadTargetKindBackupResourceList: "velero-backups/backups/my-backup/my-backup-resource-list.json.gz", }, }, { name: "backup with multiple dashes", targetName: "b-cool-20170913154901-20170913154902", expectedKeyByKind: map[velerov1api.DownloadTargetKind]string{ velerov1api.DownloadTargetKindBackupContents: "backups/b-cool-20170913154901-20170913154902/b-cool-20170913154901-20170913154902.tar.gz", velerov1api.DownloadTargetKindBackupLog: "backups/b-cool-20170913154901-20170913154902/b-cool-20170913154901-20170913154902-logs.gz", velerov1api.DownloadTargetKindBackupVolumeSnapshots: "backups/b-cool-20170913154901-20170913154902/b-cool-20170913154901-20170913154902-volumesnapshots.json.gz", velerov1api.DownloadTargetKindBackupItemOperations: "backups/b-cool-20170913154901-20170913154902/b-cool-20170913154901-20170913154902-itemoperations.json.gz", velerov1api.DownloadTargetKindBackupResourceList: "backups/b-cool-20170913154901-20170913154902/b-cool-20170913154901-20170913154902-resource-list.json.gz", }, }, { name: "scheduled backup", targetName: "my-backup-20170913154901", expectedKeyByKind: map[velerov1api.DownloadTargetKind]string{ velerov1api.DownloadTargetKindBackupContents: "backups/my-backup-20170913154901/my-backup-20170913154901.tar.gz", velerov1api.DownloadTargetKindBackupLog: "backups/my-backup-20170913154901/my-backup-20170913154901-logs.gz", velerov1api.DownloadTargetKindBackupVolumeSnapshots: "backups/my-backup-20170913154901/my-backup-20170913154901-volumesnapshots.json.gz", velerov1api.DownloadTargetKindBackupItemOperations: "backups/my-backup-20170913154901/my-backup-20170913154901-itemoperations.json.gz", velerov1api.DownloadTargetKindBackupResourceList: "backups/my-backup-20170913154901/my-backup-20170913154901-resource-list.json.gz", }, }, { name: "scheduled backup with prefix", targetName: "my-backup-20170913154901", prefix: "velero-backups/", expectedKeyByKind: map[velerov1api.DownloadTargetKind]string{ velerov1api.DownloadTargetKindBackupContents: "velero-backups/backups/my-backup-20170913154901/my-backup-20170913154901.tar.gz", velerov1api.DownloadTargetKindBackupLog: "velero-backups/backups/my-backup-20170913154901/my-backup-20170913154901-logs.gz", velerov1api.DownloadTargetKindBackupVolumeSnapshots: "velero-backups/backups/my-backup-20170913154901/my-backup-20170913154901-volumesnapshots.json.gz", velerov1api.DownloadTargetKindBackupItemOperations: "velero-backups/backups/my-backup-20170913154901/my-backup-20170913154901-itemoperations.json.gz", velerov1api.DownloadTargetKindBackupResourceList: "velero-backups/backups/my-backup-20170913154901/my-backup-20170913154901-resource-list.json.gz", }, }, { name: "restore", targetName: "my-backup", expectedKeyByKind: map[velerov1api.DownloadTargetKind]string{ velerov1api.DownloadTargetKindRestoreLog: "restores/my-backup/restore-my-backup-logs.gz", velerov1api.DownloadTargetKindRestoreResults: "restores/my-backup/restore-my-backup-results.gz", velerov1api.DownloadTargetKindRestoreItemOperations: "restores/my-backup/restore-my-backup-itemoperations.json.gz", velerov1api.DownloadTargetKindRestoreResourceList: "restores/my-backup/restore-my-backup-resource-list.json.gz", }, }, { name: "restore with prefix", targetName: "my-backup", prefix: "velero-backups/", expectedKeyByKind: map[velerov1api.DownloadTargetKind]string{ velerov1api.DownloadTargetKindRestoreLog: "velero-backups/restores/my-backup/restore-my-backup-logs.gz", velerov1api.DownloadTargetKindRestoreResults: "velero-backups/restores/my-backup/restore-my-backup-results.gz", velerov1api.DownloadTargetKindRestoreItemOperations: "velero-backups/restores/my-backup/restore-my-backup-itemoperations.json.gz", velerov1api.DownloadTargetKindRestoreResourceList: "velero-backups/restores/my-backup/restore-my-backup-resource-list.json.gz", }, }, { name: "restore with multiple dashes", targetName: "b-cool-20170913154901-20170913154902", expectedKeyByKind: map[velerov1api.DownloadTargetKind]string{ velerov1api.DownloadTargetKindRestoreLog: "restores/b-cool-20170913154901-20170913154902/restore-b-cool-20170913154901-20170913154902-logs.gz", velerov1api.DownloadTargetKindRestoreResults: "restores/b-cool-20170913154901-20170913154902/restore-b-cool-20170913154901-20170913154902-results.gz", velerov1api.DownloadTargetKindRestoreItemOperations: "restores/b-cool-20170913154901-20170913154902/restore-b-cool-20170913154901-20170913154902-itemoperations.json.gz", velerov1api.DownloadTargetKindRestoreResourceList: "restores/b-cool-20170913154901-20170913154902/restore-b-cool-20170913154901-20170913154902-resource-list.json.gz", }, }, { name: "", targetName: "my-backup", expectedKeyByKind: map[velerov1api.DownloadTargetKind]string{ velerov1api.DownloadTargetKindBackupVolumeInfos: "backups/my-backup/my-backup-volumeinfo.json.gz", }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { harness := newObjectBackupStoreTestHarness("test-bucket", test.prefix) for kind, expectedKey := range test.expectedKeyByKind { t.Run(string(kind), func(t *testing.T) { require.NoError(t, harness.objectStore.PutObject("test-bucket", expectedKey, newStringReadSeeker("foo"))) url, err := harness.GetDownloadURL(velerov1api.DownloadTarget{Kind: kind, Name: test.targetName}) require.NoError(t, err) assert.Equal(t, "a-url", url) }) } }) } } func TestGetCSIVolumeSnapshotClasses(t *testing.T) { harness := newObjectBackupStoreTestHarness("test-bucket", "") // file not found should not error res, err := harness.GetCSIVolumeSnapshotClasses("test-backup") require.NoError(t, err) assert.Nil(t, res) // file containing invalid data should error harness.objectStore.PutObject(harness.bucket, "backups/test-backup/test-backup-csi-volumesnapshotclasses.json.gz", newStringReadSeeker("foo")) _, err = harness.GetCSIVolumeSnapshotClasses("test-backup") require.Error(t, err) // file containing gzipped json data should return correctly classes := []*snapshotv1api.VolumeSnapshotClass{ { Driver: "driver", }, } obj := new(bytes.Buffer) gzw := gzip.NewWriter(obj) require.NoError(t, json.NewEncoder(gzw).Encode(classes)) require.NoError(t, gzw.Close()) require.NoError(t, harness.objectStore.PutObject(harness.bucket, "backups/test-backup/test-backup-csi-volumesnapshotclasses.json.gz", obj)) res, err = harness.GetCSIVolumeSnapshotClasses("test-backup") require.NoError(t, err) assert.Equal(t, classes, res) } func TestGetCSIVolumeSnapshots(t *testing.T) { harness := newObjectBackupStoreTestHarness("test-bucket", "") // file not found should not error res, err := harness.GetCSIVolumeSnapshots("test-backup") require.NoError(t, err) assert.Nil(t, res) // file containing invalid data should error harness.objectStore.PutObject(harness.bucket, "backups/test-backup/test-backup-csi-volumesnapshots.json.gz", newStringReadSeeker("foo")) _, err = harness.GetCSIVolumeSnapshots("test-backup") require.Error(t, err) // file containing gzipped json data should return correctly snapshots := []*snapshotv1api.VolumeSnapshot{ { Spec: snapshotv1api.VolumeSnapshotSpec{ Source: snapshotv1api.VolumeSnapshotSource{ VolumeSnapshotContentName: nil, }, }, }, } obj := new(bytes.Buffer) gzw := gzip.NewWriter(obj) require.NoError(t, json.NewEncoder(gzw).Encode(snapshots)) require.NoError(t, gzw.Close()) require.NoError(t, harness.objectStore.PutObject(harness.bucket, "backups/test-backup/test-backup-csi-volumesnapshots.json.gz", obj)) res, err = harness.GetCSIVolumeSnapshots("test-backup") require.NoError(t, err) assert.Equal(t, snapshots, res) } type objectStoreGetter map[string]velero.ObjectStore func (osg objectStoreGetter) GetObjectStore(provider string) (velero.ObjectStore, error) { res, ok := osg[provider] if !ok { return nil, errors.New("object store not found") } return res, nil } // TestNewObjectBackupStore runs the NewObjectBackupStoreGetter constructor and ensures // that it provides a BackupStore with a correctly constructed ObjectBackupStore or // that an appropriate error is returned. func TestNewObjectBackupStoreGetter(t *testing.T) { tests := []struct { name string location *velerov1api.BackupStorageLocation objectStoreGetter objectStoreGetter credFileStore credentials.FileStore fileStoreErr error wantBucket string wantPrefix string wantErr string }{ { name: "when location does not use object storage, a backup store can't be retrieved", location: new(velerov1api.BackupStorageLocation), credFileStore: velerotest.NewFakeCredentialsFileStore("", nil), wantErr: "backup storage location does not use object storage", }, { name: "when object storage does not specify a provider, a backup store can't be retrieved", location: builder.ForBackupStorageLocation("", "").Bucket("").Result(), credFileStore: velerotest.NewFakeCredentialsFileStore("", nil), wantErr: "object storage provider name must not be empty", }, { name: "when the Bucket field has a '/' in the middle, a backup store can't be retrieved", location: builder.ForBackupStorageLocation("", "").Provider("provider-1").Bucket("invalid/bucket").Result(), credFileStore: velerotest.NewFakeCredentialsFileStore("", nil), wantErr: "backup storage location's bucket name \"invalid/bucket\" must not contain a '/' (if using a prefix, put it in the 'Prefix' field instead)", }, { name: "when the credential selector is invalid, a backup store can't be retrieved", location: builder.ForBackupStorageLocation("", "").Provider("provider-1").Bucket("bucket").Credential( builder.ForSecretKeySelector("does-not-exist", "does-not-exist").Result(), ).Result(), credFileStore: velerotest.NewFakeCredentialsFileStore("", fmt.Errorf("secret does not exist")), wantErr: "unable to get credentials: secret does not exist", }, { name: "when Bucket has a leading and trailing slash, they are both stripped", location: builder.ForBackupStorageLocation("", "").Provider("provider-1").Bucket("/bucket/").Result(), objectStoreGetter: objectStoreGetter{ "provider-1": newInMemoryObjectStore("bucket"), }, credFileStore: velerotest.NewFakeCredentialsFileStore("", nil), wantBucket: "bucket", }, { name: "when Prefix has a leading and trailing slash, the leading slash is stripped and the trailing slash is left", location: builder.ForBackupStorageLocation("", "").Provider("provider-1").Bucket("bucket").Prefix("/prefix/").Result(), objectStoreGetter: objectStoreGetter{ "provider-1": newInMemoryObjectStore("bucket"), }, credFileStore: velerotest.NewFakeCredentialsFileStore("", nil), wantBucket: "bucket", wantPrefix: "prefix/", }, { name: "when Prefix has no leading or trailing slash, a trailing slash is added", location: builder.ForBackupStorageLocation("", "").Provider("provider-1").Bucket("bucket").Prefix("prefix").Result(), objectStoreGetter: objectStoreGetter{ "provider-1": newInMemoryObjectStore("bucket"), }, credFileStore: velerotest.NewFakeCredentialsFileStore("", nil), wantBucket: "bucket", wantPrefix: "prefix/", }, { name: "when the Bucket field is an MRAP ARN, it should be valid", location: builder.ForBackupStorageLocation("", "").Provider("provider-1").Bucket("arn:aws:s3::123456789012:accesspoint/abcdef0123456.mrap").Result(), objectStoreGetter: objectStoreGetter{ "provider-1": newInMemoryObjectStore("arn:aws:s3::123456789012:accesspoint/abcdef0123456.mrap"), }, credFileStore: velerotest.NewFakeCredentialsFileStore("", nil), wantBucket: "arn:aws:s3::123456789012:accesspoint/abcdef0123456.mrap", }, { name: "when the Bucket field is an MRAP ARN with trailing slash, it should be valid and trimmed", location: builder.ForBackupStorageLocation("", "").Provider("provider-1").Bucket("arn:aws:s3::123456789012:accesspoint/abcdef0123456.mrap/").Result(), objectStoreGetter: objectStoreGetter{ "provider-1": newInMemoryObjectStore("arn:aws:s3::123456789012:accesspoint/abcdef0123456.mrap"), }, credFileStore: velerotest.NewFakeCredentialsFileStore("", nil), wantBucket: "arn:aws:s3::123456789012:accesspoint/abcdef0123456.mrap", }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { getter := NewObjectBackupStoreGetter(tc.credFileStore) res, err := getter.Get(tc.location, tc.objectStoreGetter, velerotest.NewLogger()) if tc.wantErr != "" { require.EqualError(t, err, tc.wantErr) } else { require.NoError(t, err) store, ok := res.(*objectBackupStore) require.True(t, ok) assert.Equal(t, tc.wantBucket, store.bucket) assert.Equal(t, tc.wantPrefix, store.layout.rootPrefix) } }) } } // TestNewObjectBackupStoreGetterConfig runs the NewObjectBackupStoreGetter constructor and ensures // that it initializes the ObjectBackupStore with the correct config. func TestNewObjectBackupStoreGetterConfig(t *testing.T) { provider := "provider" bucket := "bucket" tests := []struct { name string location *velerov1api.BackupStorageLocation getter ObjectBackupStoreGetter credentialPath string wantConfig map[string]string }{ { name: "location with bucket but no prefix has config initialized with bucket and empty prefix", location: builder.ForBackupStorageLocation("", "").Provider(provider).Bucket(bucket).Result(), getter: NewObjectBackupStoreGetter(velerotest.NewFakeCredentialsFileStore("", nil)), wantConfig: map[string]string{ "bucket": "bucket", "prefix": "", }, }, { name: "location with bucket and prefix has config initialized with bucket and prefix", location: builder.ForBackupStorageLocation("", "").Provider(provider).Bucket(bucket).Prefix("prefix").Result(), getter: NewObjectBackupStoreGetter(velerotest.NewFakeCredentialsFileStore("", nil)), wantConfig: map[string]string{ "bucket": "bucket", "prefix": "prefix", }, }, { name: "location with CACert is initialized with caCert", location: builder.ForBackupStorageLocation("", "").Provider(provider).Bucket(bucket).CACert([]byte("cacert-data")).Result(), getter: NewObjectBackupStoreGetter(velerotest.NewFakeCredentialsFileStore("", nil)), wantConfig: map[string]string{ "bucket": "bucket", "prefix": "", "caCert": "cacert-data", }, }, { name: "location with Credential is initialized with path of serialized secret", location: builder.ForBackupStorageLocation("", "").Provider(provider).Bucket(bucket).Credential( builder.ForSecretKeySelector("does-not-exist", "does-not-exist").Result(), ).Result(), getter: NewObjectBackupStoreGetter(velerotest.NewFakeCredentialsFileStore("/tmp/credentials/secret-file", nil)), wantConfig: map[string]string{ "bucket": "bucket", "prefix": "", "credentialsFile": "/tmp/credentials/secret-file", }, }, { name: "location with CACertRef is initialized with caCert from secret", location: builder.ForBackupStorageLocation("", "").Provider(provider).Bucket(bucket).CACertRef( builder.ForSecretKeySelector("cacert-secret", "ca.crt").Result(), ).Result(), getter: NewObjectBackupStoreGetterWithSecretStore( velerotest.NewFakeCredentialsFileStore("", nil), velerotest.NewFakeCredentialsSecretStore("cacert-from-secret", nil), ), wantConfig: map[string]string{ "bucket": "bucket", "prefix": "", "caCert": "cacert-from-secret", }, }, { name: "location with CACertRef and no SecretStore uses no caCert", location: builder.ForBackupStorageLocation("", "").Provider(provider).Bucket(bucket).CACertRef( builder.ForSecretKeySelector("cacert-secret", "ca.crt").Result(), ).Result(), getter: NewObjectBackupStoreGetter(velerotest.NewFakeCredentialsFileStore("", nil)), wantConfig: map[string]string{ "bucket": "bucket", "prefix": "", }, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { objStore := newInMemoryObjectStore(bucket) objStoreGetter := &objectStoreGetter{provider: objStore} _, err := tc.getter.Get(tc.location, objStoreGetter, velerotest.NewLogger()) require.NoError(t, err) require.Equal(t, tc.wantConfig, objStore.Config) }) } } func TestGetBackupVolumeInfos(t *testing.T) { tests := []struct { name string volumeInfo []*volume.BackupVolumeInfo volumeInfoStr string expectedErr string expectedResult []*volume.BackupVolumeInfo }{ { name: "No VolumeInfos, expect no error.", }, { name: "Valid BackupVolumeInfo, should pass.", volumeInfo: []*volume.BackupVolumeInfo{ { PVCName: "pvcName", PVName: "pvName", Skipped: true, SnapshotDataMoved: false, }, }, expectedResult: []*volume.BackupVolumeInfo{ { PVCName: "pvcName", PVName: "pvName", Skipped: true, SnapshotDataMoved: false, }, }, }, { name: "Invalid BackupVolumeInfo string, should also pass.", volumeInfoStr: `[{"abc": "123", "def": "456", "pvcName": "pvcName"}]`, expectedResult: []*volume.BackupVolumeInfo{ { PVCName: "pvcName", }, }, }, } harness := newObjectBackupStoreTestHarness("test-bucket", "") for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { if tc.volumeInfo != nil { obj := new(bytes.Buffer) gzw := gzip.NewWriter(obj) require.NoError(t, json.NewEncoder(gzw).Encode(tc.volumeInfo)) require.NoError(t, gzw.Close()) harness.objectStore.PutObject(harness.bucket, "backups/test-backup/test-backup-volumeinfo.json.gz", obj) } if tc.volumeInfoStr != "" { obj := new(bytes.Buffer) gzw := gzip.NewWriter(obj) _, err := gzw.Write([]byte(tc.volumeInfoStr)) require.NoError(t, err) require.NoError(t, gzw.Close()) harness.objectStore.PutObject(harness.bucket, "backups/test-backup/test-backup-volumeinfo.json.gz", obj) } result, err := harness.GetBackupVolumeInfos("test-backup") if tc.expectedErr != "" { require.Equal(t, tc.expectedErr, err.Error()) } else { if err != nil { fmt.Println(err.Error()) } require.NoError(t, err) } if len(tc.expectedResult) > 0 { require.Equal(t, tc.expectedResult, result) } }) } } func TestGetRestoreResults(t *testing.T) { harness := newObjectBackupStoreTestHarness("test-bucket", "") // file not found should not error _, err := harness.GetRestoreResults("test-restore") require.NoError(t, err) // file containing invalid data should error harness.objectStore.PutObject(harness.bucket, "restores/test-restore/restore-test-restore-results.gz", newStringReadSeeker("foo")) _, err = harness.GetRestoreResults("test-restore") require.Error(t, err) // file containing gzipped json data should return correctly contents := map[string]results.Result{ "warnings": {Cluster: []string{"cluster warning"}}, "errors": {Namespaces: map[string][]string{"test-ns": {"namespace error"}}}, } obj := new(bytes.Buffer) gzw := gzip.NewWriter(obj) require.NoError(t, json.NewEncoder(gzw).Encode(contents)) require.NoError(t, gzw.Close()) require.NoError(t, harness.objectStore.PutObject(harness.bucket, "restores/test-restore/restore-test-restore-results.gz", obj)) res, err := harness.GetRestoreResults("test-restore") require.NoError(t, err) assert.Equal(t, contents["warnings"], res["warnings"]) assert.Equal(t, contents["errors"], res["errors"]) } func TestGetRestoredResourceList(t *testing.T) { harness := newObjectBackupStoreTestHarness("test-bucket", "") // file not found should not error _, err := harness.GetRestoredResourceList("test-restore") require.NoError(t, err) // file containing invalid data should error harness.objectStore.PutObject(harness.bucket, "restores/test-restore/restore-test-restore-resource-list.json.gz", newStringReadSeeker("foo")) _, err = harness.GetRestoredResourceList("test-restore") require.Error(t, err) // file containing gzipped json data should return correctly list := map[string][]string{ "pod": {"test-ns/pod1(created)", "test-ns/pod2(skipped)"}, } obj := new(bytes.Buffer) gzw := gzip.NewWriter(obj) require.NoError(t, json.NewEncoder(gzw).Encode(list)) require.NoError(t, gzw.Close()) require.NoError(t, harness.objectStore.PutObject(harness.bucket, "restores/test-restore/restore-test-restore-resource-list.json.gz", obj)) res, err := harness.GetRestoredResourceList("test-restore") require.NoError(t, err) assert.Equal(t, list["pod"], res["pod"]) } func TestPutBackupVolumeInfos(t *testing.T) { tests := []struct { name string prefix string expectedErr string expectedKeys []string }{ { name: "normal case", expectedErr: "", expectedKeys: []string{ "backups/backup-1/backup-1-volumeinfo.json.gz", }, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { harness := newObjectBackupStoreTestHarness("foo", tc.prefix) volumeInfos := []*volume.BackupVolumeInfo{ { PVCName: "test", }, } buf := new(bytes.Buffer) gzw := gzip.NewWriter(buf) defer gzw.Close() require.NoError(t, json.NewEncoder(gzw).Encode(volumeInfos)) bufferContent := buf.Bytes() err := harness.PutBackupVolumeInfos("backup-1", buf) velerotest.AssertErrorMatches(t, tc.expectedErr, err) assert.Len(t, harness.objectStore.Data[harness.bucket], len(tc.expectedKeys)) for _, key := range tc.expectedKeys { assert.Contains(t, harness.objectStore.Data[harness.bucket], key) assert.Equal(t, harness.objectStore.Data[harness.bucket][key], bufferContent) } }) } } func encodeToBytes(obj runtime.Object) []byte { res, err := encode.Encode(obj, "json") if err != nil { panic(err) } return res } type stringReadSeeker struct { *strings.Reader } func newStringReadSeeker(s string) *stringReadSeeker { return &stringReadSeeker{ Reader: strings.NewReader(s), } } func (srs *stringReadSeeker) Seek(offset int64, whence int) (int64, error) { return 0, nil } type errorReader struct{} func (r *errorReader) Read([]byte) (int, error) { return 0, errors.New("error readers return errors") } ================================================ FILE: pkg/plugin/clientmgmt/backupitemaction/v1/restartable_backup_item_action.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v1 import ( "github.com/pkg/errors" "k8s.io/apimachinery/pkg/runtime" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt/process" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" "github.com/vmware-tanzu/velero/pkg/plugin/velero" biav1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/backupitemaction/v1" ) // AdaptedBackupItemAction is a backup item action adapted to the v1 BackupItemAction API type AdaptedBackupItemAction struct { Kind common.PluginKind // Get returns a restartable BackupItemAction for the given name and process, wrapping if necessary GetRestartable func(name string, restartableProcess process.RestartableProcess) biav1.BackupItemAction } func AdaptedBackupItemActions() []AdaptedBackupItemAction { return []AdaptedBackupItemAction{ { Kind: common.PluginKindBackupItemAction, GetRestartable: func(name string, restartableProcess process.RestartableProcess) biav1.BackupItemAction { return NewRestartableBackupItemAction(name, restartableProcess) }, }, } } // RestartableBackupItemAction is a backup item action for a given implementation (such as "pod"). It is associated with // a restartableProcess, which may be shared and used to run multiple plugins. At the beginning of each method // call, the restartableBackupItemAction asks its restartableProcess to restart itself if needed (e.g. if the // process terminated for any reason), then it proceeds with the actual call. type RestartableBackupItemAction struct { Key process.KindAndName SharedPluginProcess process.RestartableProcess } // NewRestartableBackupItemAction returns a new RestartableBackupItemAction. func NewRestartableBackupItemAction(name string, sharedPluginProcess process.RestartableProcess) *RestartableBackupItemAction { r := &RestartableBackupItemAction{ Key: process.KindAndName{Kind: common.PluginKindBackupItemAction, Name: name}, SharedPluginProcess: sharedPluginProcess, } return r } // getBackupItemAction returns the backup item action for this restartableBackupItemAction. It does *not* restart the // plugin process. func (r *RestartableBackupItemAction) getBackupItemAction() (biav1.BackupItemAction, error) { plugin, err := r.SharedPluginProcess.GetByKindAndName(r.Key) if err != nil { return nil, err } backupItemAction, ok := plugin.(biav1.BackupItemAction) if !ok { return nil, errors.Errorf("plugin %T is not a BackupItemAction", plugin) } return backupItemAction, nil } // getDelegate restarts the plugin process (if needed) and returns the backup item action for this restartableBackupItemAction. func (r *RestartableBackupItemAction) getDelegate() (biav1.BackupItemAction, error) { if err := r.SharedPluginProcess.ResetIfNeeded(); err != nil { return nil, err } return r.getBackupItemAction() } // AppliesTo restarts the plugin's process if needed, then delegates the call. func (r *RestartableBackupItemAction) AppliesTo() (velero.ResourceSelector, error) { delegate, err := r.getDelegate() if err != nil { return velero.ResourceSelector{}, err } return delegate.AppliesTo() } // Execute restarts the plugin's process if needed, then delegates the call. func (r *RestartableBackupItemAction) Execute(item runtime.Unstructured, backup *api.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, error) { delegate, err := r.getDelegate() if err != nil { return nil, nil, err } return delegate.Execute(item, backup) } ================================================ FILE: pkg/plugin/clientmgmt/backupitemaction/v1/restartable_backup_item_action_test.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v1 import ( "testing" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "github.com/vmware-tanzu/velero/internal/restartabletest" v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt/process" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" "github.com/vmware-tanzu/velero/pkg/plugin/velero" mocks "github.com/vmware-tanzu/velero/pkg/plugin/velero/mocks/backupitemaction/v1" ) func TestRestartableGetBackupItemAction(t *testing.T) { tests := []struct { name string plugin any getError error expectedError string }{ { name: "error getting by kind and name", getError: errors.Errorf("get error"), expectedError: "get error", }, { name: "wrong type", plugin: 3, expectedError: "plugin int is not a BackupItemAction", }, { name: "happy path", plugin: new(mocks.BackupItemAction), }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { p := new(restartabletest.MockRestartableProcess) defer p.AssertExpectations(t) name := "pod" key := process.KindAndName{Kind: common.PluginKindBackupItemAction, Name: name} p.On("GetByKindAndName", key).Return(tc.plugin, tc.getError) r := NewRestartableBackupItemAction(name, p) a, err := r.getBackupItemAction() if tc.expectedError != "" { assert.EqualError(t, err, tc.expectedError) return } require.NoError(t, err) assert.Equal(t, tc.plugin, a) }) } } func TestRestartableBackupItemActionGetDelegate(t *testing.T) { p := new(restartabletest.MockRestartableProcess) defer p.AssertExpectations(t) // Reset error p.On("ResetIfNeeded").Return(errors.Errorf("reset error")).Once() name := "pod" r := NewRestartableBackupItemAction(name, p) a, err := r.getDelegate() assert.Nil(t, a) require.EqualError(t, err, "reset error") // Happy path p.On("ResetIfNeeded").Return(nil) expected := new(mocks.BackupItemAction) key := process.KindAndName{Kind: common.PluginKindBackupItemAction, Name: name} p.On("GetByKindAndName", key).Return(expected, nil) a, err = r.getDelegate() require.NoError(t, err) assert.Equal(t, expected, a) } func TestRestartableBackupItemActionDelegatedFunctions(t *testing.T) { b := new(v1.Backup) pv := &unstructured.Unstructured{ Object: map[string]any{ "color": "blue", }, } pvToReturn := &unstructured.Unstructured{ Object: map[string]any{ "color": "green", }, } additionalItems := []velero.ResourceIdentifier{ { GroupResource: schema.GroupResource{Group: "velero.io", Resource: "backups"}, }, } restartabletest.RunRestartableDelegateTests( t, common.PluginKindBackupItemAction, func(key process.KindAndName, p process.RestartableProcess) any { return &RestartableBackupItemAction{ Key: key, SharedPluginProcess: p, } }, func() restartabletest.Mockable { return new(mocks.BackupItemAction) }, restartabletest.RestartableDelegateTest{ Function: "AppliesTo", Inputs: []any{}, ExpectedErrorOutputs: []any{velero.ResourceSelector{}, errors.Errorf("reset error")}, ExpectedDelegateOutputs: []any{velero.ResourceSelector{IncludedNamespaces: []string{"a"}}, errors.Errorf("delegate error")}, }, restartabletest.RestartableDelegateTest{ Function: "Execute", Inputs: []any{pv, b}, ExpectedErrorOutputs: []any{nil, ([]velero.ResourceIdentifier)(nil), errors.Errorf("reset error")}, ExpectedDelegateOutputs: []any{pvToReturn, additionalItems, errors.Errorf("delegate error")}, }, ) } ================================================ FILE: pkg/plugin/clientmgmt/backupitemaction/v2/restartable_backup_item_action.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v2 import ( "github.com/pkg/errors" "k8s.io/apimachinery/pkg/runtime" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" biav1cli "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt/backupitemaction/v1" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt/process" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" "github.com/vmware-tanzu/velero/pkg/plugin/velero" biav2 "github.com/vmware-tanzu/velero/pkg/plugin/velero/backupitemaction/v2" ) // AdaptedBackupItemAction is a v1 BackupItemAction adapted to implement the v2 API type AdaptedBackupItemAction struct { Kind common.PluginKind // Get returns a restartable BackupItemAction for the given name and process, wrapping if necessary GetRestartable func(name string, restartableProcess process.RestartableProcess) biav2.BackupItemAction } func AdaptedBackupItemActions() []AdaptedBackupItemAction { return []AdaptedBackupItemAction{ { Kind: common.PluginKindBackupItemActionV2, GetRestartable: func(name string, restartableProcess process.RestartableProcess) biav2.BackupItemAction { return NewRestartableBackupItemAction(name, restartableProcess) }, }, { Kind: common.PluginKindBackupItemAction, GetRestartable: func(name string, restartableProcess process.RestartableProcess) biav2.BackupItemAction { return NewAdaptedV1RestartableBackupItemAction(biav1cli.NewRestartableBackupItemAction(name, restartableProcess)) }, }, } } // restartableBackupItemAction is a backup item action for a given implementation (such as "pod"). It is associated with // a restartableProcess, which may be shared and used to run multiple plugins. At the beginning of each method // call, the restartableBackupItemAction asks its restartableProcess to restart itself if needed (e.g. if the // process terminated for any reason), then it proceeds with the actual call. type RestartableBackupItemAction struct { Key process.KindAndName SharedPluginProcess process.RestartableProcess } // NewRestartableBackupItemAction returns a new RestartableBackupItemAction. func NewRestartableBackupItemAction(name string, sharedPluginProcess process.RestartableProcess) *RestartableBackupItemAction { r := &RestartableBackupItemAction{ Key: process.KindAndName{Kind: common.PluginKindBackupItemActionV2, Name: name}, SharedPluginProcess: sharedPluginProcess, } return r } // getBackupItemAction returns the backup item action for this restartableBackupItemAction. It does *not* restart the // plugin process. func (r *RestartableBackupItemAction) getBackupItemAction() (biav2.BackupItemAction, error) { plugin, err := r.SharedPluginProcess.GetByKindAndName(r.Key) if err != nil { return nil, err } backupItemAction, ok := plugin.(biav2.BackupItemAction) if !ok { return nil, errors.Errorf("plugin %T (returned for %v) is not a BackupItemActionV2", plugin, r.Key) } return backupItemAction, nil } // getDelegate restarts the plugin process (if needed) and returns the backup item action for this restartableBackupItemAction. func (r *RestartableBackupItemAction) getDelegate() (biav2.BackupItemAction, error) { if err := r.SharedPluginProcess.ResetIfNeeded(); err != nil { return nil, err } return r.getBackupItemAction() } // Name returns the plugin's name. func (r *RestartableBackupItemAction) Name() string { return r.Key.Name } // AppliesTo restarts the plugin's process if needed, then delegates the call. func (r *RestartableBackupItemAction) AppliesTo() (velero.ResourceSelector, error) { delegate, err := r.getDelegate() if err != nil { return velero.ResourceSelector{}, err } return delegate.AppliesTo() } // Execute restarts the plugin's process if needed, then delegates the call. func (r *RestartableBackupItemAction) Execute(item runtime.Unstructured, backup *api.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { delegate, err := r.getDelegate() if err != nil { return nil, nil, "", nil, err } return delegate.Execute(item, backup) } // Progress restarts the plugin's process if needed, then delegates the call. func (r *RestartableBackupItemAction) Progress(operationID string, backup *api.Backup) (velero.OperationProgress, error) { delegate, err := r.getDelegate() if err != nil { return velero.OperationProgress{}, err } return delegate.Progress(operationID, backup) } // Cancel restarts the plugin's process if needed, then delegates the call. func (r *RestartableBackupItemAction) Cancel(operationID string, backup *api.Backup) error { delegate, err := r.getDelegate() if err != nil { return err } return delegate.Cancel(operationID, backup) } type AdaptedV1RestartableBackupItemAction struct { V1Restartable *biav1cli.RestartableBackupItemAction } // NewAdaptedV1RestartableBackupItemAction returns a new v1 RestartableBackupItemAction adapted to v2 func NewAdaptedV1RestartableBackupItemAction(v1Restartable *biav1cli.RestartableBackupItemAction) *AdaptedV1RestartableBackupItemAction { r := &AdaptedV1RestartableBackupItemAction{ V1Restartable: v1Restartable, } return r } // Name restarts the plugin's name. func (r *AdaptedV1RestartableBackupItemAction) Name() string { return r.V1Restartable.Key.Name } // AppliesTo delegates to the v1 AppliesTo call. func (r *AdaptedV1RestartableBackupItemAction) AppliesTo() (velero.ResourceSelector, error) { return r.V1Restartable.AppliesTo() } // Execute delegates to the v1 Execute call, returning an empty operationID. func (r *AdaptedV1RestartableBackupItemAction) Execute(item runtime.Unstructured, backup *api.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { updatedItem, additionalItems, err := r.V1Restartable.Execute(item, backup) return updatedItem, additionalItems, "", nil, err } // Progress returns with an error since v1 plugins will never return an operationID, which means that // any operationID passed in here will be invalid. func (r *AdaptedV1RestartableBackupItemAction) Progress(operationID string, backup *api.Backup) (velero.OperationProgress, error) { return velero.OperationProgress{}, biav2.AsyncOperationsNotSupportedError() } // Cancel just returns without error since v1 plugins don't implement it. func (r *AdaptedV1RestartableBackupItemAction) Cancel(operationID string, backup *api.Backup) error { return nil } ================================================ FILE: pkg/plugin/clientmgmt/backupitemaction/v2/restartable_backup_item_action_test.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v2 import ( "testing" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "github.com/vmware-tanzu/velero/internal/restartabletest" v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt/process" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" "github.com/vmware-tanzu/velero/pkg/plugin/velero" mocksv2 "github.com/vmware-tanzu/velero/pkg/plugin/velero/mocks/backupitemaction/v2" ) func TestRestartableGetBackupItemAction(t *testing.T) { tests := []struct { name string plugin any getError error expectedError string }{ { name: "error getting by kind and name", getError: errors.Errorf("get error"), expectedError: "get error", }, { name: "wrong type", plugin: 3, expectedError: "plugin int (returned for {BackupItemActionV2 pod}) is not a BackupItemActionV2", }, { name: "happy path", plugin: new(mocksv2.BackupItemAction), }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { p := new(restartabletest.MockRestartableProcess) defer p.AssertExpectations(t) name := "pod" key := process.KindAndName{Kind: common.PluginKindBackupItemActionV2, Name: name} p.On("GetByKindAndName", key).Return(tc.plugin, tc.getError) r := NewRestartableBackupItemAction(name, p) a, err := r.getBackupItemAction() if tc.expectedError != "" { assert.EqualError(t, err, tc.expectedError) return } require.NoError(t, err) assert.Equal(t, tc.plugin, a) }) } } func TestRestartableBackupItemActionGetDelegate(t *testing.T) { p := new(restartabletest.MockRestartableProcess) defer p.AssertExpectations(t) // Reset error p.On("ResetIfNeeded").Return(errors.Errorf("reset error")).Once() name := "pod" r := NewRestartableBackupItemAction(name, p) a, err := r.getDelegate() assert.Nil(t, a) require.EqualError(t, err, "reset error") // Happy path p.On("ResetIfNeeded").Return(nil) expected := new(mocksv2.BackupItemAction) key := process.KindAndName{Kind: common.PluginKindBackupItemActionV2, Name: name} p.On("GetByKindAndName", key).Return(expected, nil) a, err = r.getDelegate() require.NoError(t, err) assert.Equal(t, expected, a) } func TestRestartableBackupItemActionDelegatedFunctions(t *testing.T) { b := new(v1.Backup) pv := &unstructured.Unstructured{ Object: map[string]any{ "color": "blue", }, } oid := "operation1" pvToReturn := &unstructured.Unstructured{ Object: map[string]any{ "color": "green", }, } additionalItems := []velero.ResourceIdentifier{ { GroupResource: schema.GroupResource{Group: "velero.io", Resource: "backups"}, }, } restartabletest.RunRestartableDelegateTests( t, common.PluginKindBackupItemAction, func(key process.KindAndName, p process.RestartableProcess) any { return &RestartableBackupItemAction{ Key: key, SharedPluginProcess: p, } }, func() restartabletest.Mockable { return new(mocksv2.BackupItemAction) }, restartabletest.RestartableDelegateTest{ Function: "AppliesTo", Inputs: []any{}, ExpectedErrorOutputs: []any{velero.ResourceSelector{}, errors.Errorf("reset error")}, ExpectedDelegateOutputs: []any{velero.ResourceSelector{IncludedNamespaces: []string{"a"}}, errors.Errorf("delegate error")}, }, restartabletest.RestartableDelegateTest{ Function: "Execute", Inputs: []any{pv, b}, ExpectedErrorOutputs: []any{nil, ([]velero.ResourceIdentifier)(nil), "", ([]velero.ResourceIdentifier)(nil), errors.Errorf("reset error")}, ExpectedDelegateOutputs: []any{pvToReturn, additionalItems, "", ([]velero.ResourceIdentifier)(nil), errors.Errorf("delegate error")}, }, restartabletest.RestartableDelegateTest{ Function: "Progress", Inputs: []any{oid, b}, ExpectedErrorOutputs: []any{velero.OperationProgress{}, errors.Errorf("reset error")}, ExpectedDelegateOutputs: []any{velero.OperationProgress{}, errors.Errorf("delegate error")}, }, restartabletest.RestartableDelegateTest{ Function: "Cancel", Inputs: []any{oid, b}, ExpectedErrorOutputs: []any{errors.Errorf("reset error")}, ExpectedDelegateOutputs: []any{errors.Errorf("delegate error")}, }, ) } ================================================ FILE: pkg/plugin/clientmgmt/itemblockaction/v1/restartable_item_block_action.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v1 import ( "github.com/pkg/errors" "k8s.io/apimachinery/pkg/runtime" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt/process" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" "github.com/vmware-tanzu/velero/pkg/plugin/velero" ibav1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/itemblockaction/v1" ) // AdaptedItemBlockAction is an ItemBlock action adapted to the v1 ItemBlockAction API type AdaptedItemBlockAction struct { Kind common.PluginKind // Get returns a restartable ItemBlockAction for the given name and process, wrapping if necessary GetRestartable func(name string, restartableProcess process.RestartableProcess) ibav1.ItemBlockAction } func AdaptedItemBlockActions() []AdaptedItemBlockAction { return []AdaptedItemBlockAction{ { Kind: common.PluginKindItemBlockAction, GetRestartable: func(name string, restartableProcess process.RestartableProcess) ibav1.ItemBlockAction { return NewRestartableItemBlockAction(name, restartableProcess) }, }, } } // RestartableItemBlockAction is an ItemBlock action for a given implementation (such as "pod"). It is associated with // a restartableProcess, which may be shared and used to run multiple plugins. At the beginning of each method // call, the restartableItemBlockAction asks its restartableProcess to restart itself if needed (e.g. if the // process terminated for any reason), then it proceeds with the actual call. type RestartableItemBlockAction struct { Key process.KindAndName SharedPluginProcess process.RestartableProcess } // NewRestartableItemBlockAction returns a new RestartableItemBlockAction. func NewRestartableItemBlockAction(name string, sharedPluginProcess process.RestartableProcess) *RestartableItemBlockAction { r := &RestartableItemBlockAction{ Key: process.KindAndName{Kind: common.PluginKindItemBlockAction, Name: name}, SharedPluginProcess: sharedPluginProcess, } return r } // getItemBlockAction returns the ItemBlock action for this restartableItemBlockAction. It does *not* restart the // plugin process. func (r *RestartableItemBlockAction) getItemBlockAction() (ibav1.ItemBlockAction, error) { plugin, err := r.SharedPluginProcess.GetByKindAndName(r.Key) if err != nil { return nil, err } itemBlockAction, ok := plugin.(ibav1.ItemBlockAction) if !ok { return nil, errors.Errorf("plugin %T is not an ItemBlockAction", plugin) } return itemBlockAction, nil } // getDelegate restarts the plugin process (if needed) and returns the ItemBlock action for this restartableItemBlockAction. func (r *RestartableItemBlockAction) getDelegate() (ibav1.ItemBlockAction, error) { if err := r.SharedPluginProcess.ResetIfNeeded(); err != nil { return nil, err } return r.getItemBlockAction() } // Name returns the plugin's name. func (r *RestartableItemBlockAction) Name() string { return r.Key.Name } // AppliesTo restarts the plugin's process if needed, then delegates the call. func (r *RestartableItemBlockAction) AppliesTo() (velero.ResourceSelector, error) { delegate, err := r.getDelegate() if err != nil { return velero.ResourceSelector{}, err } return delegate.AppliesTo() } // GetRelatedItems restarts the plugin's process if needed, then delegates the call. func (r *RestartableItemBlockAction) GetRelatedItems(item runtime.Unstructured, backup *api.Backup) ([]velero.ResourceIdentifier, error) { delegate, err := r.getDelegate() if err != nil { return nil, err } return delegate.GetRelatedItems(item, backup) } ================================================ FILE: pkg/plugin/clientmgmt/itemblockaction/v1/restartable_item_block_action_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v1 import ( "testing" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "github.com/vmware-tanzu/velero/internal/restartabletest" v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt/process" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" "github.com/vmware-tanzu/velero/pkg/plugin/velero" mocks "github.com/vmware-tanzu/velero/pkg/plugin/velero/mocks/itemblockaction/v1" ) func TestRestartableGetItemBlockAction(t *testing.T) { tests := []struct { name string plugin any getError error expectedError string }{ { name: "error getting by kind and name", getError: errors.Errorf("get error"), expectedError: "get error", }, { name: "wrong type", plugin: 3, expectedError: "plugin int is not an ItemBlockAction", }, { name: "happy path", plugin: new(mocks.ItemBlockAction), }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { p := new(restartabletest.MockRestartableProcess) defer p.AssertExpectations(t) name := "pod" key := process.KindAndName{Kind: common.PluginKindItemBlockAction, Name: name} p.On("GetByKindAndName", key).Return(tc.plugin, tc.getError) r := NewRestartableItemBlockAction(name, p) a, err := r.getItemBlockAction() if tc.expectedError != "" { assert.EqualError(t, err, tc.expectedError) return } require.NoError(t, err) assert.Equal(t, tc.plugin, a) }) } } func TestRestartableItemBlockActionGetDelegate(t *testing.T) { p := new(restartabletest.MockRestartableProcess) defer p.AssertExpectations(t) // Reset error p.On("ResetIfNeeded").Return(errors.Errorf("reset error")).Once() name := "pod" r := NewRestartableItemBlockAction(name, p) a, err := r.getDelegate() assert.Nil(t, a) require.EqualError(t, err, "reset error") // Happy path p.On("ResetIfNeeded").Return(nil) expected := new(mocks.ItemBlockAction) key := process.KindAndName{Kind: common.PluginKindItemBlockAction, Name: name} p.On("GetByKindAndName", key).Return(expected, nil) a, err = r.getDelegate() require.NoError(t, err) assert.Equal(t, expected, a) } func TestRestartableItemBlockActionDelegatedFunctions(t *testing.T) { b := new(v1.Backup) pv := &unstructured.Unstructured{ Object: map[string]any{ "color": "blue", }, } relatedItems := []velero.ResourceIdentifier{ { GroupResource: schema.GroupResource{Group: "velero.io", Resource: "backups"}, }, } restartabletest.RunRestartableDelegateTests( t, common.PluginKindItemBlockAction, func(key process.KindAndName, p process.RestartableProcess) any { return &RestartableItemBlockAction{ Key: key, SharedPluginProcess: p, } }, func() restartabletest.Mockable { return new(mocks.ItemBlockAction) }, restartabletest.RestartableDelegateTest{ Function: "AppliesTo", Inputs: []any{}, ExpectedErrorOutputs: []any{velero.ResourceSelector{}, errors.Errorf("reset error")}, ExpectedDelegateOutputs: []any{velero.ResourceSelector{IncludedNamespaces: []string{"a"}}, errors.Errorf("delegate error")}, }, restartabletest.RestartableDelegateTest{ Function: "GetRelatedItems", Inputs: []any{pv, b}, ExpectedErrorOutputs: []any{([]velero.ResourceIdentifier)(nil), errors.Errorf("reset error")}, ExpectedDelegateOutputs: []any{relatedItems, errors.Errorf("delegate error")}, }, ) } ================================================ FILE: pkg/plugin/clientmgmt/manager.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package clientmgmt import ( "errors" "fmt" "strings" "sync" "github.com/sirupsen/logrus" biav1cli "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt/backupitemaction/v1" biav2cli "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt/backupitemaction/v2" ibav1cli "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt/itemblockaction/v1" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt/process" riav1cli "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt/restoreitemaction/v1" riav2cli "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt/restoreitemaction/v2" vsv1cli "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt/volumesnapshotter/v1" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" "github.com/vmware-tanzu/velero/pkg/plugin/velero" biav1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/backupitemaction/v1" biav2 "github.com/vmware-tanzu/velero/pkg/plugin/velero/backupitemaction/v2" ibav1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/itemblockaction/v1" riav1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/restoreitemaction/v1" riav2 "github.com/vmware-tanzu/velero/pkg/plugin/velero/restoreitemaction/v2" vsv1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/volumesnapshotter/v1" ) // Manager manages the lifecycles of plugins. type Manager interface { // GetObjectStore returns the ObjectStore plugin for name. GetObjectStore(name string) (velero.ObjectStore, error) // GetVolumeSnapshotter returns the VolumeSnapshotter plugin for name. GetVolumeSnapshotter(name string) (vsv1.VolumeSnapshotter, error) // GetBackupItemActions returns all v1 backup item action plugins. GetBackupItemActions() ([]biav1.BackupItemAction, error) // GetBackupItemAction returns the backup item action plugin for name. GetBackupItemAction(name string) (biav1.BackupItemAction, error) // GetBackupItemActionsV2 returns all v2 backup item action plugins (including those adapted from v1). GetBackupItemActionsV2() ([]biav2.BackupItemAction, error) // GetBackupItemActionV2 returns the backup item action plugin for name. GetBackupItemActionV2(name string) (biav2.BackupItemAction, error) // GetRestoreItemActions returns all restore item action plugins. GetRestoreItemActions() ([]riav1.RestoreItemAction, error) // GetRestoreItemAction returns the restore item action plugin for name. GetRestoreItemAction(name string) (riav1.RestoreItemAction, error) // GetRestoreItemActionsV2 returns all v2 restore item action plugins. GetRestoreItemActionsV2() ([]riav2.RestoreItemAction, error) // GetRestoreItemActionV2 returns the restore item action plugin for name. GetRestoreItemActionV2(name string) (riav2.RestoreItemAction, error) // GetDeleteItemActions returns all delete item action plugins. GetDeleteItemActions() ([]velero.DeleteItemAction, error) // GetDeleteItemAction returns the delete item action plugin for name. GetDeleteItemAction(name string) (velero.DeleteItemAction, error) // GetItemBlockActions returns all v1 ItemBlock action plugins. GetItemBlockActions() ([]ibav1.ItemBlockAction, error) // GetItemBlockAction returns the ItemBlock action plugin for name. GetItemBlockAction(name string) (ibav1.ItemBlockAction, error) // CleanupClients terminates all of the Manager's running plugin processes. CleanupClients() } // Used checking for adapted plugin versions var pluginNotFoundErrType = &process.PluginNotFoundError{} // manager implements Manager. type manager struct { logger logrus.FieldLogger logLevel logrus.Level registry process.Registry restartableProcessFactory process.RestartableProcessFactory // lock guards restartableProcesses lock sync.Mutex restartableProcesses map[string]process.RestartableProcess } // NewManager constructs a manager for getting plugins. func NewManager(logger logrus.FieldLogger, level logrus.Level, registry process.Registry) Manager { return &manager{ logger: logger, logLevel: level, registry: registry, restartableProcessFactory: process.NewRestartableProcessFactory(), restartableProcesses: make(map[string]process.RestartableProcess), } } func (m *manager) CleanupClients() { m.lock.Lock() for _, restartableProcess := range m.restartableProcesses { restartableProcess.Stop() } m.lock.Unlock() } // getRestartableProcess returns a restartableProcess for a plugin identified by kind and name, creating a // restartableProcess if it is the first time it has been requested. func (m *manager) getRestartableProcess(kind common.PluginKind, name string) (process.RestartableProcess, error) { m.lock.Lock() defer m.lock.Unlock() logger := m.logger.WithFields(logrus.Fields{ "kind": kind.String(), "name": name, }) logger.Debug("looking for plugin in registry") info, err := m.registry.Get(kind, name) if err != nil { return nil, err } logger = logger.WithField("command", info.Command) restartableProcess, found := m.restartableProcesses[info.Command] if found { logger.Debug("found preexisting restartable plugin process") return restartableProcess, nil } logger.Debug("creating new restartable plugin process") restartableProcess, err = m.restartableProcessFactory.NewRestartableProcess(info.Command, m.logger, m.logLevel) if err != nil { return nil, err } m.restartableProcesses[info.Command] = restartableProcess return restartableProcess, nil } // GetObjectStore returns a restartableObjectStore for name. func (m *manager) GetObjectStore(name string) (velero.ObjectStore, error) { name = sanitizeName(name) restartableProcess, err := m.getRestartableProcess(common.PluginKindObjectStore, name) if err != nil { return nil, err } r := NewRestartableObjectStore(name, restartableProcess) return r, nil } // GetVolumeSnapshotter returns a restartableVolumeSnapshotter for name. func (m *manager) GetVolumeSnapshotter(name string) (vsv1.VolumeSnapshotter, error) { name = sanitizeName(name) for _, adaptedVolumeSnapshotter := range vsv1cli.AdaptedVolumeSnapshotters() { restartableProcess, err := m.getRestartableProcess(adaptedVolumeSnapshotter.Kind, name) // Check if plugin was not found if errors.As(err, &pluginNotFoundErrType) { continue } if err != nil { return nil, err } return adaptedVolumeSnapshotter.GetRestartable(name, restartableProcess), nil } return nil, fmt.Errorf("unable to get valid VolumeSnapshotter for %q", name) } // GetBackupItemActions returns all backup item actions as restartableBackupItemActions. func (m *manager) GetBackupItemActions() ([]biav1.BackupItemAction, error) { list := m.registry.List(common.PluginKindBackupItemAction) actions := make([]biav1.BackupItemAction, 0, len(list)) for i := range list { id := list[i] r, err := m.GetBackupItemAction(id.Name) if err != nil { return nil, err } actions = append(actions, r) } return actions, nil } // GetBackupItemAction returns a restartableBackupItemAction for name. func (m *manager) GetBackupItemAction(name string) (biav1.BackupItemAction, error) { name = sanitizeName(name) for _, adaptedBackupItemAction := range biav1cli.AdaptedBackupItemActions() { restartableProcess, err := m.getRestartableProcess(adaptedBackupItemAction.Kind, name) // Check if plugin was not found if errors.As(err, &pluginNotFoundErrType) { continue } if err != nil { return nil, err } return adaptedBackupItemAction.GetRestartable(name, restartableProcess), nil } return nil, fmt.Errorf("unable to get valid BackupItemAction for %q", name) } // GetBackupItemActionsV2 returns all v2 backup item actions as RestartableBackupItemActions. func (m *manager) GetBackupItemActionsV2() ([]biav2.BackupItemAction, error) { list := m.registry.List(common.PluginKindBackupItemActionV2) actions := make([]biav2.BackupItemAction, 0, len(list)) for i := range list { id := list[i] r, err := m.GetBackupItemActionV2(id.Name) if err != nil { return nil, err } actions = append(actions, r) } return actions, nil } // GetBackupItemActionV2 returns a v2 restartableBackupItemAction for name. func (m *manager) GetBackupItemActionV2(name string) (biav2.BackupItemAction, error) { name = sanitizeName(name) for _, adaptedBackupItemAction := range biav2cli.AdaptedBackupItemActions() { restartableProcess, err := m.getRestartableProcess(adaptedBackupItemAction.Kind, name) // Check if plugin was not found if errors.As(err, &pluginNotFoundErrType) { continue } if err != nil { return nil, err } return adaptedBackupItemAction.GetRestartable(name, restartableProcess), nil } return nil, fmt.Errorf("unable to get valid BackupItemActionV2 for %q", name) } // GetRestoreItemActions returns all restore item actions as restartableRestoreItemActions. func (m *manager) GetRestoreItemActions() ([]riav1.RestoreItemAction, error) { list := m.registry.List(common.PluginKindRestoreItemAction) actions := make([]riav1.RestoreItemAction, 0, len(list)) for i := range list { id := list[i] r, err := m.GetRestoreItemAction(id.Name) if err != nil { return nil, err } actions = append(actions, r) } return actions, nil } // GetRestoreItemAction returns a restartableRestoreItemAction for name. func (m *manager) GetRestoreItemAction(name string) (riav1.RestoreItemAction, error) { name = sanitizeName(name) for _, adaptedRestoreItemAction := range riav1cli.AdaptedRestoreItemActions() { restartableProcess, err := m.getRestartableProcess(adaptedRestoreItemAction.Kind, name) // Check if plugin was not found if errors.As(err, &pluginNotFoundErrType) { continue } if err != nil { return nil, err } return adaptedRestoreItemAction.GetRestartable(name, restartableProcess), nil } return nil, fmt.Errorf("unable to get valid RestoreItemAction for %q", name) } // GetRestoreItemActionsV2 returns all v2 restore item actions as restartableRestoreItemActions. func (m *manager) GetRestoreItemActionsV2() ([]riav2.RestoreItemAction, error) { list := m.registry.List(common.PluginKindRestoreItemActionV2) actions := make([]riav2.RestoreItemAction, 0, len(list)) for i := range list { id := list[i] r, err := m.GetRestoreItemActionV2(id.Name) if err != nil { return nil, err } actions = append(actions, r) } return actions, nil } // GetRestoreItemActionV2 returns a v2 restartableRestoreItemAction for name. func (m *manager) GetRestoreItemActionV2(name string) (riav2.RestoreItemAction, error) { name = sanitizeName(name) for _, adaptedRestoreItemAction := range riav2cli.AdaptedRestoreItemActions() { restartableProcess, err := m.getRestartableProcess(adaptedRestoreItemAction.Kind, name) // Check if plugin was not found if errors.As(err, &pluginNotFoundErrType) { continue } if err != nil { return nil, err } return adaptedRestoreItemAction.GetRestartable(name, restartableProcess), nil } return nil, fmt.Errorf("unable to get valid RestoreItemActionV2 for %q", name) } // GetDeleteItemActions returns all delete item actions as restartableDeleteItemActions. func (m *manager) GetDeleteItemActions() ([]velero.DeleteItemAction, error) { list := m.registry.List(common.PluginKindDeleteItemAction) actions := make([]velero.DeleteItemAction, 0, len(list)) for i := range list { id := list[i] r, err := m.GetDeleteItemAction(id.Name) if err != nil { return nil, err } actions = append(actions, r) } return actions, nil } // GetDeleteItemAction returns a restartableDeleteItemAction for name. func (m *manager) GetDeleteItemAction(name string) (velero.DeleteItemAction, error) { name = sanitizeName(name) restartableProcess, err := m.getRestartableProcess(common.PluginKindDeleteItemAction, name) if err != nil { return nil, err } r := NewRestartableDeleteItemAction(name, restartableProcess) return r, nil } // GetItemBlockActions returns all ItemBlock actions as restartableItemBlockActions. func (m *manager) GetItemBlockActions() ([]ibav1.ItemBlockAction, error) { list := m.registry.List(common.PluginKindItemBlockAction) actions := make([]ibav1.ItemBlockAction, 0, len(list)) for i := range list { id := list[i] r, err := m.GetItemBlockAction(id.Name) if err != nil { return nil, err } actions = append(actions, r) } return actions, nil } // GetItemBlockAction returns a restartableItemBlockAction for name. func (m *manager) GetItemBlockAction(name string) (ibav1.ItemBlockAction, error) { name = sanitizeName(name) for _, adaptedItemBlockAction := range ibav1cli.AdaptedItemBlockActions() { restartableProcess, err := m.getRestartableProcess(adaptedItemBlockAction.Kind, name) // Check if plugin was not found if errors.As(err, &pluginNotFoundErrType) { continue } if err != nil { return nil, err } return adaptedItemBlockAction.GetRestartable(name, restartableProcess), nil } return nil, fmt.Errorf("unable to get valid ItemBlockAction for %q", name) } // sanitizeName adds "velero.io" to legacy plugins that weren't namespaced. func sanitizeName(name string) string { // Backwards compatibility with non-namespaced Velero plugins, following principle of least surprise // since DeleteItemActions were not bundled with Velero when plugins were non-namespaced. if !strings.Contains(name, "/") { name = "velero.io/" + name } return name } ================================================ FILE: pkg/plugin/clientmgmt/manager_test.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package clientmgmt import ( "fmt" "testing" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "github.com/vmware-tanzu/velero/internal/restartabletest" biav1cli "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt/backupitemaction/v1" biav2cli "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt/backupitemaction/v2" ibav1cli "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt/itemblockaction/v1" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt/process" riav1cli "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt/restoreitemaction/v1" riav2cli "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt/restoreitemaction/v2" vsv1cli "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt/volumesnapshotter/v1" "github.com/vmware-tanzu/velero/pkg/plugin/framework" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" "github.com/vmware-tanzu/velero/pkg/test" ) type mockRegistry struct { mock.Mock } func (r *mockRegistry) DiscoverPlugins() error { args := r.Called() return args.Error(0) } func (r *mockRegistry) List(kind common.PluginKind) []framework.PluginIdentifier { args := r.Called(kind) return args.Get(0).([]framework.PluginIdentifier) } func (r *mockRegistry) Get(kind common.PluginKind, name string) (framework.PluginIdentifier, error) { args := r.Called(kind, name) var id framework.PluginIdentifier if args.Get(0) != nil { id = args.Get(0).(framework.PluginIdentifier) } return id, args.Error(1) } func TestNewManager(t *testing.T) { logger := test.NewLogger() logLevel := logrus.InfoLevel registry := &mockRegistry{} defer registry.AssertExpectations(t) m := NewManager(logger, logLevel, registry).(*manager) assert.Equal(t, logger, m.logger) assert.Equal(t, logLevel, m.logLevel) assert.Equal(t, registry, m.registry) assert.NotNil(t, m.restartableProcesses) assert.Empty(t, m.restartableProcesses) } type mockRestartableProcessFactory struct { mock.Mock } func (f *mockRestartableProcessFactory) NewRestartableProcess(command string, logger logrus.FieldLogger, logLevel logrus.Level) (process.RestartableProcess, error) { args := f.Called(command, logger, logLevel) var rp process.RestartableProcess if args.Get(0) != nil { rp = args.Get(0).(process.RestartableProcess) } return rp, args.Error(1) } func TestGetRestartableProcess(t *testing.T) { logger := test.NewLogger() logLevel := logrus.InfoLevel registry := &mockRegistry{} defer registry.AssertExpectations(t) m := NewManager(logger, logLevel, registry).(*manager) factory := &mockRestartableProcessFactory{} defer factory.AssertExpectations(t) m.restartableProcessFactory = factory // Test 1: registry error pluginKind := common.PluginKindBackupItemAction pluginName := "pod" registry.On("Get", pluginKind, pluginName).Return(nil, errors.Errorf("registry")).Once() rp, err := m.getRestartableProcess(pluginKind, pluginName) assert.Nil(t, rp) require.EqualError(t, err, "registry") // Test 2: registry ok, factory error podID := framework.PluginIdentifier{ Command: "/command", Kind: pluginKind, Name: pluginName, } registry.On("Get", pluginKind, pluginName).Return(podID, nil) factory.On("NewRestartableProcess", podID.Command, logger, logLevel).Return(nil, errors.Errorf("factory")).Once() rp, err = m.getRestartableProcess(pluginKind, pluginName) assert.Nil(t, rp) require.EqualError(t, err, "factory") // Test 3: registry ok, factory ok restartableProcess := &restartabletest.MockRestartableProcess{} defer restartableProcess.AssertExpectations(t) factory.On("NewRestartableProcess", podID.Command, logger, logLevel).Return(restartableProcess, nil).Once() rp, err = m.getRestartableProcess(pluginKind, pluginName) require.NoError(t, err) assert.Equal(t, restartableProcess, rp) // Test 4: retrieve from cache rp, err = m.getRestartableProcess(pluginKind, pluginName) require.NoError(t, err) assert.Equal(t, restartableProcess, rp) } func TestCleanupClients(t *testing.T) { logger := test.NewLogger() logLevel := logrus.InfoLevel registry := &mockRegistry{} defer registry.AssertExpectations(t) m := NewManager(logger, logLevel, registry).(*manager) for i := 0; i < 5; i++ { rp := &restartabletest.MockRestartableProcess{} defer rp.AssertExpectations(t) rp.On("Stop") m.restartableProcesses[fmt.Sprintf("rp%d", i)] = rp } m.CleanupClients() } func TestGetObjectStore(t *testing.T) { getPluginTest(t, common.PluginKindObjectStore, "velero.io/aws", func(m Manager, name string) (any, error) { return m.GetObjectStore(name) }, func(name string, sharedPluginProcess process.RestartableProcess) any { return &restartableObjectStore{ key: process.KindAndName{Kind: common.PluginKindObjectStore, Name: name}, sharedPluginProcess: sharedPluginProcess, } }, true, ) } func TestGetVolumeSnapshotter(t *testing.T) { getPluginTest(t, common.PluginKindVolumeSnapshotter, "velero.io/aws", func(m Manager, name string) (any, error) { return m.GetVolumeSnapshotter(name) }, func(name string, sharedPluginProcess process.RestartableProcess) any { return &vsv1cli.RestartableVolumeSnapshotter{ Key: process.KindAndName{Kind: common.PluginKindVolumeSnapshotter, Name: name}, SharedPluginProcess: sharedPluginProcess, } }, true, ) } func TestGetBackupItemAction(t *testing.T) { getPluginTest(t, common.PluginKindBackupItemAction, "velero.io/pod", func(m Manager, name string) (any, error) { return m.GetBackupItemAction(name) }, func(name string, sharedPluginProcess process.RestartableProcess) any { return &biav1cli.RestartableBackupItemAction{ Key: process.KindAndName{Kind: common.PluginKindBackupItemAction, Name: name}, SharedPluginProcess: sharedPluginProcess, } }, false, ) } func TestGetBackupItemActionV2(t *testing.T) { getPluginTest(t, common.PluginKindBackupItemActionV2, "velero.io/pod", func(m Manager, name string) (any, error) { return m.GetBackupItemActionV2(name) }, func(name string, sharedPluginProcess process.RestartableProcess) any { return &biav2cli.RestartableBackupItemAction{ Key: process.KindAndName{Kind: common.PluginKindBackupItemActionV2, Name: name}, SharedPluginProcess: sharedPluginProcess, } }, false, ) } func TestGetRestoreItemAction(t *testing.T) { getPluginTest(t, common.PluginKindRestoreItemAction, "velero.io/pod", func(m Manager, name string) (any, error) { return m.GetRestoreItemAction(name) }, func(name string, sharedPluginProcess process.RestartableProcess) any { return &riav1cli.RestartableRestoreItemAction{ Key: process.KindAndName{Kind: common.PluginKindRestoreItemAction, Name: name}, SharedPluginProcess: sharedPluginProcess, } }, false, ) } func TestGetRestoreItemActionV2(t *testing.T) { getPluginTest(t, common.PluginKindRestoreItemActionV2, "velero.io/pod", func(m Manager, name string) (any, error) { return m.GetRestoreItemActionV2(name) }, func(name string, sharedPluginProcess process.RestartableProcess) any { return &riav2cli.RestartableRestoreItemAction{ Key: process.KindAndName{Kind: common.PluginKindRestoreItemActionV2, Name: name}, SharedPluginProcess: sharedPluginProcess, } }, false, ) } func TestGetItemBlockAction(t *testing.T) { getPluginTest(t, common.PluginKindItemBlockAction, "velero.io/pod", func(m Manager, name string) (any, error) { return m.GetItemBlockAction(name) }, func(name string, sharedPluginProcess process.RestartableProcess) any { return &ibav1cli.RestartableItemBlockAction{ Key: process.KindAndName{Kind: common.PluginKindItemBlockAction, Name: name}, SharedPluginProcess: sharedPluginProcess, } }, false, ) } func getPluginTest( t *testing.T, kind common.PluginKind, name string, getPluginFunc func(m Manager, name string) (any, error), expectedResultFunc func(name string, sharedPluginProcess process.RestartableProcess) any, reinitializable bool, ) { t.Helper() logger := test.NewLogger() logLevel := logrus.InfoLevel registry := &mockRegistry{} defer registry.AssertExpectations(t) m := NewManager(logger, logLevel, registry).(*manager) factory := &mockRestartableProcessFactory{} defer factory.AssertExpectations(t) m.restartableProcessFactory = factory pluginKind := kind pluginName := name pluginID := framework.PluginIdentifier{ Command: "/command", Kind: pluginKind, Name: pluginName, } registry.On("Get", pluginKind, pluginName).Return(pluginID, nil) restartableProcess := &restartabletest.MockRestartableProcess{} defer restartableProcess.AssertExpectations(t) // Test 1: error getting restartable process factory.On("NewRestartableProcess", pluginID.Command, logger, logLevel).Return(nil, errors.Errorf("NewRestartableProcess")).Once() actual, err := getPluginFunc(m, pluginName) assert.Nil(t, actual) require.EqualError(t, err, "NewRestartableProcess") // Test 2: happy path factory.On("NewRestartableProcess", pluginID.Command, logger, logLevel).Return(restartableProcess, nil).Once() expected := expectedResultFunc(name, restartableProcess) if reinitializable { key := process.KindAndName{Kind: pluginID.Kind, Name: pluginID.Name} restartableProcess.On("AddReinitializer", key, expected) } actual, err = getPluginFunc(m, pluginName) require.NoError(t, err) assert.Equal(t, expected, actual) } func TestGetBackupItemActions(t *testing.T) { tests := []struct { name string names []string newRestartableProcessError error expectedError string }{ { name: "No items", names: []string{}, }, { name: "Error getting restartable process", names: []string{"velero.io/a", "velero.io/b", "velero.io/c"}, newRestartableProcessError: errors.Errorf("NewRestartableProcess"), expectedError: "NewRestartableProcess", }, { name: "Happy path", names: []string{"velero.io/a", "velero.io/b", "velero.io/c"}, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { logger := test.NewLogger() logLevel := logrus.InfoLevel registry := &mockRegistry{} defer registry.AssertExpectations(t) m := NewManager(logger, logLevel, registry).(*manager) factory := &mockRestartableProcessFactory{} defer factory.AssertExpectations(t) m.restartableProcessFactory = factory pluginKind := common.PluginKindBackupItemAction var pluginIDs []framework.PluginIdentifier for i := range tc.names { pluginID := framework.PluginIdentifier{ Command: "/command", Kind: pluginKind, Name: tc.names[i], } pluginIDs = append(pluginIDs, pluginID) } registry.On("List", pluginKind).Return(pluginIDs) var expectedActions []any for i := range pluginIDs { pluginID := pluginIDs[i] pluginName := pluginID.Name registry.On("Get", pluginKind, pluginName).Return(pluginID, nil) restartableProcess := &restartabletest.MockRestartableProcess{} defer restartableProcess.AssertExpectations(t) expected := &biav1cli.RestartableBackupItemAction{ Key: process.KindAndName{Kind: pluginKind, Name: pluginName}, SharedPluginProcess: restartableProcess, } if tc.newRestartableProcessError != nil { // Test 1: error getting restartable process factory.On("NewRestartableProcess", pluginID.Command, logger, logLevel).Return(nil, errors.Errorf("NewRestartableProcess")).Once() break } // Test 2: happy path if i == 0 { factory.On("NewRestartableProcess", pluginID.Command, logger, logLevel).Return(restartableProcess, nil).Once() } expectedActions = append(expectedActions, expected) } backupItemActions, err := m.GetBackupItemActions() if tc.newRestartableProcessError != nil { assert.Nil(t, backupItemActions) assert.EqualError(t, err, "NewRestartableProcess") } else { require.NoError(t, err) var actual []any for i := range backupItemActions { actual = append(actual, backupItemActions[i]) } assert.Equal(t, expectedActions, actual) } }) } } func TestGetBackupItemActionsV2(t *testing.T) { tests := []struct { name string names []string newRestartableProcessError error expectedError string }{ { name: "No items", names: []string{}, }, { name: "Error getting restartable process", names: []string{"velero.io/a", "velero.io/b", "velero.io/c"}, newRestartableProcessError: errors.Errorf("NewRestartableProcess"), expectedError: "NewRestartableProcess", }, { name: "Happy path", names: []string{"velero.io/a", "velero.io/b", "velero.io/c"}, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { logger := test.NewLogger() logLevel := logrus.InfoLevel registry := &mockRegistry{} defer registry.AssertExpectations(t) m := NewManager(logger, logLevel, registry).(*manager) factory := &mockRestartableProcessFactory{} defer factory.AssertExpectations(t) m.restartableProcessFactory = factory pluginKind := common.PluginKindBackupItemActionV2 var pluginIDs []framework.PluginIdentifier for i := range tc.names { pluginID := framework.PluginIdentifier{ Command: "/command", Kind: pluginKind, Name: tc.names[i], } pluginIDs = append(pluginIDs, pluginID) } registry.On("List", pluginKind).Return(pluginIDs) var expectedActions []any for i := range pluginIDs { pluginID := pluginIDs[i] pluginName := pluginID.Name registry.On("Get", pluginKind, pluginName).Return(pluginID, nil) restartableProcess := &restartabletest.MockRestartableProcess{} defer restartableProcess.AssertExpectations(t) expected := &biav2cli.RestartableBackupItemAction{ Key: process.KindAndName{Kind: pluginKind, Name: pluginName}, SharedPluginProcess: restartableProcess, } if tc.newRestartableProcessError != nil { // Test 1: error getting restartable process factory.On("NewRestartableProcess", pluginID.Command, logger, logLevel).Return(nil, errors.Errorf("NewRestartableProcess")).Once() break } // Test 2: happy path if i == 0 { factory.On("NewRestartableProcess", pluginID.Command, logger, logLevel).Return(restartableProcess, nil).Once() } expectedActions = append(expectedActions, expected) } backupItemActions, err := m.GetBackupItemActionsV2() if tc.newRestartableProcessError != nil { assert.Nil(t, backupItemActions) assert.EqualError(t, err, "NewRestartableProcess") } else { require.NoError(t, err) var actual []any for i := range backupItemActions { actual = append(actual, backupItemActions[i]) } assert.Equal(t, expectedActions, actual) } }) } } func TestGetRestoreItemActions(t *testing.T) { tests := []struct { name string names []string newRestartableProcessError error expectedError string }{ { name: "No items", names: []string{}, }, { name: "Error getting restartable process", names: []string{"velero.io/a", "velero.io/b", "velero.io/c"}, newRestartableProcessError: errors.Errorf("NewRestartableProcess"), expectedError: "NewRestartableProcess", }, { name: "Happy path", names: []string{"velero.io/a", "velero.io/b", "velero.io/c"}, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { logger := test.NewLogger() logLevel := logrus.InfoLevel registry := &mockRegistry{} defer registry.AssertExpectations(t) m := NewManager(logger, logLevel, registry).(*manager) factory := &mockRestartableProcessFactory{} defer factory.AssertExpectations(t) m.restartableProcessFactory = factory pluginKind := common.PluginKindRestoreItemAction var pluginIDs []framework.PluginIdentifier for i := range tc.names { pluginID := framework.PluginIdentifier{ Command: "/command", Kind: pluginKind, Name: tc.names[i], } pluginIDs = append(pluginIDs, pluginID) } registry.On("List", pluginKind).Return(pluginIDs) var expectedActions []any for i := range pluginIDs { pluginID := pluginIDs[i] pluginName := pluginID.Name registry.On("Get", pluginKind, pluginName).Return(pluginID, nil) restartableProcess := &restartabletest.MockRestartableProcess{} defer restartableProcess.AssertExpectations(t) expected := &riav1cli.RestartableRestoreItemAction{ Key: process.KindAndName{Kind: pluginKind, Name: pluginName}, SharedPluginProcess: restartableProcess, } if tc.newRestartableProcessError != nil { // Test 1: error getting restartable process factory.On("NewRestartableProcess", pluginID.Command, logger, logLevel).Return(nil, errors.Errorf("NewRestartableProcess")).Once() break } // Test 2: happy path if i == 0 { factory.On("NewRestartableProcess", pluginID.Command, logger, logLevel).Return(restartableProcess, nil).Once() } expectedActions = append(expectedActions, expected) } restoreItemActions, err := m.GetRestoreItemActions() if tc.newRestartableProcessError != nil { assert.Nil(t, restoreItemActions) assert.EqualError(t, err, "NewRestartableProcess") } else { require.NoError(t, err) var actual []any for i := range restoreItemActions { actual = append(actual, restoreItemActions[i]) } assert.Equal(t, expectedActions, actual) } }) } } func TestGetRestoreItemActionsV2(t *testing.T) { tests := []struct { name string names []string newRestartableProcessError error expectedError string }{ { name: "No items", names: []string{}, }, { name: "Error getting restartable process", names: []string{"velero.io/a", "velero.io/b", "velero.io/c"}, newRestartableProcessError: errors.Errorf("NewRestartableProcess"), expectedError: "NewRestartableProcess", }, { name: "Happy path", names: []string{"velero.io/a", "velero.io/b", "velero.io/c"}, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { logger := test.NewLogger() logLevel := logrus.InfoLevel registry := &mockRegistry{} defer registry.AssertExpectations(t) m := NewManager(logger, logLevel, registry).(*manager) factory := &mockRestartableProcessFactory{} defer factory.AssertExpectations(t) m.restartableProcessFactory = factory pluginKind := common.PluginKindRestoreItemActionV2 var pluginIDs []framework.PluginIdentifier for i := range tc.names { pluginID := framework.PluginIdentifier{ Command: "/command", Kind: pluginKind, Name: tc.names[i], } pluginIDs = append(pluginIDs, pluginID) } registry.On("List", pluginKind).Return(pluginIDs) var expectedActions []any for i := range pluginIDs { pluginID := pluginIDs[i] pluginName := pluginID.Name registry.On("Get", pluginKind, pluginName).Return(pluginID, nil) restartableProcess := &restartabletest.MockRestartableProcess{} defer restartableProcess.AssertExpectations(t) expected := &riav2cli.RestartableRestoreItemAction{ Key: process.KindAndName{Kind: pluginKind, Name: pluginName}, SharedPluginProcess: restartableProcess, } if tc.newRestartableProcessError != nil { // Test 1: error getting restartable process factory.On("NewRestartableProcess", pluginID.Command, logger, logLevel).Return(nil, errors.Errorf("NewRestartableProcess")).Once() break } // Test 2: happy path if i == 0 { factory.On("NewRestartableProcess", pluginID.Command, logger, logLevel).Return(restartableProcess, nil).Once() } expectedActions = append(expectedActions, expected) } restoreItemActions, err := m.GetRestoreItemActionsV2() if tc.newRestartableProcessError != nil { assert.Nil(t, restoreItemActions) assert.EqualError(t, err, "NewRestartableProcess") } else { require.NoError(t, err) var actual []any for i := range restoreItemActions { actual = append(actual, restoreItemActions[i]) } assert.Equal(t, expectedActions, actual) } }) } } func TestGetDeleteItemAction(t *testing.T) { getPluginTest(t, common.PluginKindDeleteItemAction, "velero.io/deleter", func(m Manager, name string) (any, error) { return m.GetDeleteItemAction(name) }, func(name string, sharedPluginProcess process.RestartableProcess) any { return &restartableDeleteItemAction{ key: process.KindAndName{Kind: common.PluginKindDeleteItemAction, Name: name}, sharedPluginProcess: sharedPluginProcess, } }, false, ) } func TestGetDeleteItemActions(t *testing.T) { tests := []struct { name string names []string newRestartableProcessError error expectedError string }{ { name: "No items", names: []string{}, }, { name: "Error getting restartable process", names: []string{"velero.io/a", "velero.io/b", "velero.io/c"}, newRestartableProcessError: errors.Errorf("NewRestartableProcess"), expectedError: "NewRestartableProcess", }, { name: "Happy path", names: []string{"velero.io/a", "velero.io/b", "velero.io/c"}, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { logger := test.NewLogger() logLevel := logrus.InfoLevel registry := &mockRegistry{} defer registry.AssertExpectations(t) m := NewManager(logger, logLevel, registry).(*manager) factory := &mockRestartableProcessFactory{} defer factory.AssertExpectations(t) m.restartableProcessFactory = factory pluginKind := common.PluginKindDeleteItemAction var pluginIDs []framework.PluginIdentifier for i := range tc.names { pluginID := framework.PluginIdentifier{ Command: "/command", Kind: pluginKind, Name: tc.names[i], } pluginIDs = append(pluginIDs, pluginID) } registry.On("List", pluginKind).Return(pluginIDs) var expectedActions []any for i := range pluginIDs { pluginID := pluginIDs[i] pluginName := pluginID.Name registry.On("Get", pluginKind, pluginName).Return(pluginID, nil) restartableProcess := &restartabletest.MockRestartableProcess{} defer restartableProcess.AssertExpectations(t) expected := &restartableDeleteItemAction{ key: process.KindAndName{Kind: pluginKind, Name: pluginName}, sharedPluginProcess: restartableProcess, } if tc.newRestartableProcessError != nil { // Test 1: error getting restartable process factory.On("NewRestartableProcess", pluginID.Command, logger, logLevel).Return(nil, errors.Errorf("NewRestartableProcess")).Once() break } // Test 2: happy path if i == 0 { factory.On("NewRestartableProcess", pluginID.Command, logger, logLevel).Return(restartableProcess, nil).Once() } expectedActions = append(expectedActions, expected) } deleteItemActions, err := m.GetDeleteItemActions() if tc.newRestartableProcessError != nil { assert.Nil(t, deleteItemActions) assert.EqualError(t, err, "NewRestartableProcess") } else { require.NoError(t, err) var actual []any for i := range deleteItemActions { actual = append(actual, deleteItemActions[i]) } assert.Equal(t, expectedActions, actual) } }) } } func TestGetItemBlockActions(t *testing.T) { tests := []struct { name string names []string newRestartableProcessError error expectedError string }{ { name: "No items", names: []string{}, }, { name: "Error getting restartable process", names: []string{"velero.io/a", "velero.io/b", "velero.io/c"}, newRestartableProcessError: errors.Errorf("NewRestartableProcess"), expectedError: "NewRestartableProcess", }, { name: "Happy path", names: []string{"velero.io/a", "velero.io/b", "velero.io/c"}, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { logger := test.NewLogger() logLevel := logrus.InfoLevel registry := &mockRegistry{} defer registry.AssertExpectations(t) m := NewManager(logger, logLevel, registry).(*manager) factory := &mockRestartableProcessFactory{} defer factory.AssertExpectations(t) m.restartableProcessFactory = factory pluginKind := common.PluginKindItemBlockAction var pluginIDs []framework.PluginIdentifier for i := range tc.names { pluginID := framework.PluginIdentifier{ Command: "/command", Kind: pluginKind, Name: tc.names[i], } pluginIDs = append(pluginIDs, pluginID) } registry.On("List", pluginKind).Return(pluginIDs) var expectedActions []any for i := range pluginIDs { pluginID := pluginIDs[i] pluginName := pluginID.Name registry.On("Get", pluginKind, pluginName).Return(pluginID, nil) restartableProcess := &restartabletest.MockRestartableProcess{} defer restartableProcess.AssertExpectations(t) expected := &ibav1cli.RestartableItemBlockAction{ Key: process.KindAndName{Kind: pluginKind, Name: pluginName}, SharedPluginProcess: restartableProcess, } if tc.newRestartableProcessError != nil { // Test 1: error getting restartable process factory.On("NewRestartableProcess", pluginID.Command, logger, logLevel).Return(nil, errors.Errorf("NewRestartableProcess")).Once() break } // Test 2: happy path if i == 0 { factory.On("NewRestartableProcess", pluginID.Command, logger, logLevel).Return(restartableProcess, nil).Once() } expectedActions = append(expectedActions, expected) } itemBlockActions, err := m.GetItemBlockActions() if tc.newRestartableProcessError != nil { assert.Nil(t, itemBlockActions) assert.EqualError(t, err, "NewRestartableProcess") } else { require.NoError(t, err) var actual []any for i := range itemBlockActions { actual = append(actual, itemBlockActions[i]) } assert.Equal(t, expectedActions, actual) } }) } } func TestSanitizeName(t *testing.T) { tests := []struct { name, pluginName, expectedName string }{ { name: "Legacy, non-namespaced plugin", pluginName: "aws", expectedName: "velero.io/aws", }, { name: "A Velero plugin", pluginName: "velero.io/aws", expectedName: "velero.io/aws", }, { name: "A non-Velero plugin with a Velero namespace", pluginName: "velero.io/plugin-for-velero", expectedName: "velero.io/plugin-for-velero", }, { name: "A non-Velero plugin with a non-Velero namespace", pluginName: "digitalocean.com/velero", expectedName: "digitalocean.com/velero", }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { sanitizedName := sanitizeName(tc.pluginName) assert.Equal(t, tc.expectedName, sanitizedName) }) } } ================================================ FILE: pkg/plugin/clientmgmt/process/client_builder.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ // Package clientmgmt contains the plugin client for Velero. package process import ( "context" "os" "os/exec" hclog "github.com/hashicorp/go-hclog" hcplugin "github.com/hashicorp/go-plugin" "github.com/sirupsen/logrus" "github.com/vmware-tanzu/velero/pkg/plugin/framework" biav2 "github.com/vmware-tanzu/velero/pkg/plugin/framework/backupitemaction/v2" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" ibav1 "github.com/vmware-tanzu/velero/pkg/plugin/framework/itemblockaction/v1" riav2 "github.com/vmware-tanzu/velero/pkg/plugin/framework/restoreitemaction/v2" ) // clientBuilder builds go-plugin Clients. type clientBuilder struct { commandName string commandArgs []string clientLogger logrus.FieldLogger pluginLogger hclog.Logger } // newClientBuilder returns a new clientBuilder with commandName to name. If the command matches the currently running // process (i.e. velero), this also sets commandArgs to the internal Velero command to run plugins. func newClientBuilder(command string, logger logrus.FieldLogger, logLevel logrus.Level) *clientBuilder { b := &clientBuilder{ commandName: command, clientLogger: logger, pluginLogger: newLogrusAdapter(logger, logLevel), } if command == os.Args[0] { // For plugins compiled into the velero executable, we need to run "velero run-plugins" b.commandArgs = []string{"run-plugins"} } // exclude "velero" and "server" from "velero server --flags ..." b.commandArgs = append(b.commandArgs, os.Args[2:]...) return b } func newLogrusAdapter(pluginLogger logrus.FieldLogger, logLevel logrus.Level) *logrusAdapter { return &logrusAdapter{impl: pluginLogger, level: logLevel} } func (b *clientBuilder) clientConfig() *hcplugin.ClientConfig { return &hcplugin.ClientConfig{ HandshakeConfig: framework.Handshake(), AllowedProtocols: []hcplugin.Protocol{hcplugin.ProtocolGRPC}, Plugins: map[string]hcplugin.Plugin{ string(common.PluginKindBackupItemAction): framework.NewBackupItemActionPlugin(common.ClientLogger(b.clientLogger)), string(common.PluginKindBackupItemActionV2): biav2.NewBackupItemActionPlugin(common.ClientLogger(b.clientLogger)), string(common.PluginKindVolumeSnapshotter): framework.NewVolumeSnapshotterPlugin(common.ClientLogger(b.clientLogger)), string(common.PluginKindObjectStore): framework.NewObjectStorePlugin(common.ClientLogger(b.clientLogger)), string(common.PluginKindPluginLister): &framework.PluginListerPlugin{}, string(common.PluginKindRestoreItemAction): framework.NewRestoreItemActionPlugin(common.ClientLogger(b.clientLogger)), string(common.PluginKindRestoreItemActionV2): riav2.NewRestoreItemActionPlugin(common.ClientLogger(b.clientLogger)), string(common.PluginKindDeleteItemAction): framework.NewDeleteItemActionPlugin(common.ClientLogger(b.clientLogger)), string(common.PluginKindItemBlockAction): ibav1.NewItemBlockActionPlugin(common.ClientLogger(b.clientLogger)), }, Logger: b.pluginLogger, Cmd: exec.CommandContext(context.Background(), b.commandName, b.commandArgs...), //nolint:gosec // Internal call. No need to check the command line. } } // client creates a new go-plugin Client with support for all of Velero's plugin kinds (BackupItemAction, VolumeSnapshotter, // ObjectStore, PluginLister, RestoreItemAction). func (b *clientBuilder) client() *hcplugin.Client { return hcplugin.NewClient(b.clientConfig()) } ================================================ FILE: pkg/plugin/clientmgmt/process/client_builder_test.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package process import ( "os" "os/exec" "testing" hcplugin "github.com/hashicorp/go-plugin" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/vmware-tanzu/velero/pkg/plugin/framework" biav2 "github.com/vmware-tanzu/velero/pkg/plugin/framework/backupitemaction/v2" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" ibav1 "github.com/vmware-tanzu/velero/pkg/plugin/framework/itemblockaction/v1" riav2 "github.com/vmware-tanzu/velero/pkg/plugin/framework/restoreitemaction/v2" "github.com/vmware-tanzu/velero/pkg/test" ) func TestNewClientBuilder(t *testing.T) { logger := test.NewLogger() logLevel := logrus.InfoLevel cb := newClientBuilder("velero", logger, logLevel) assert.Equal(t, "velero", cb.commandName) assert.Equal(t, newLogrusAdapter(logger, logLevel), cb.pluginLogger) cb = newClientBuilder(os.Args[0], logger, logLevel) assert.Equal(t, cb.commandName, os.Args[0]) assert.Equal(t, newLogrusAdapter(logger, logLevel), cb.pluginLogger) } func TestClientConfig(t *testing.T) { logger := test.NewLogger() logLevel := logrus.InfoLevel cb := newClientBuilder("velero", logger, logLevel) expected := &hcplugin.ClientConfig{ HandshakeConfig: framework.Handshake(), AllowedProtocols: []hcplugin.Protocol{hcplugin.ProtocolGRPC}, Plugins: map[string]hcplugin.Plugin{ string(common.PluginKindBackupItemAction): framework.NewBackupItemActionPlugin(common.ClientLogger(logger)), string(common.PluginKindBackupItemActionV2): biav2.NewBackupItemActionPlugin(common.ClientLogger(logger)), string(common.PluginKindVolumeSnapshotter): framework.NewVolumeSnapshotterPlugin(common.ClientLogger(logger)), string(common.PluginKindObjectStore): framework.NewObjectStorePlugin(common.ClientLogger(logger)), string(common.PluginKindPluginLister): &framework.PluginListerPlugin{}, string(common.PluginKindRestoreItemAction): framework.NewRestoreItemActionPlugin(common.ClientLogger(logger)), string(common.PluginKindRestoreItemActionV2): riav2.NewRestoreItemActionPlugin(common.ClientLogger(logger)), string(common.PluginKindDeleteItemAction): framework.NewDeleteItemActionPlugin(common.ClientLogger(logger)), string(common.PluginKindItemBlockAction): ibav1.NewItemBlockActionPlugin(common.ClientLogger(logger)), }, Logger: cb.pluginLogger, Cmd: exec.CommandContext(t.Context(), cb.commandName, cb.commandArgs...), } cc := cb.clientConfig() assert.Equal(t, expected.HandshakeConfig, cc.HandshakeConfig) assert.Equal(t, expected.AllowedProtocols, cc.AllowedProtocols) assert.Equal(t, expected.Plugins, cc.Plugins) } ================================================ FILE: pkg/plugin/clientmgmt/process/logrus_adapter.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package process import ( "fmt" "io" "log" hclog "github.com/hashicorp/go-hclog" "github.com/sirupsen/logrus" ) const pluginNameField = "pluginName" // logrusAdapter implements the hclog.Logger interface and // delegates all calls to a logrus logger. type logrusAdapter struct { impl logrus.FieldLogger level logrus.Level name string } // args are alternating key, value pairs, where the keys // are expected to be strings, and values can be any type. func argsToFields(args ...any) logrus.Fields { fields := make(map[string]any) for i := 0; i < len(args); i += 2 { switch args[i] { case "time", "timestamp", "level": // remove `time` & `timestamp` because this info will be added // by the Velero logger and we don't want to have duplicated // fields. // // remove `level` because it'll be added by the Velero logger based // on the call we make (and go-plugin is determining which level // to log at based on the hclog-compatible `@level` field which // we're adding via HcLogLevelHook). default: var val any if i+1 < len(args) { val = args[i+1] } fields[fmt.Sprintf("%v", args[i])] = val } } return logrus.Fields(fields) } // Trace emits a message and key/value pairs at the DEBUG level // (logrus doesn't have a TRACE level) func (l *logrusAdapter) Trace(msg string, args ...any) { l.Debug(msg, args...) } // Debug emits a message and key/value pairs at the DEBUG level func (l *logrusAdapter) Debug(msg string, args ...any) { l.impl.WithFields(argsToFields(args...)).Debug(msg) } // Info emits a message and key/value pairs at the INFO level func (l *logrusAdapter) Info(msg string, args ...any) { l.impl.WithFields(argsToFields(args...)).Info(msg) } // Warn emits a message and key/value pairs at the WARN level func (l *logrusAdapter) Warn(msg string, args ...any) { l.impl.WithFields(argsToFields(args...)).Warn(msg) } // Error emits a message and key/value pairs at the ERROR level func (l *logrusAdapter) Error(msg string, args ...any) { l.impl.WithFields(argsToFields(args...)).Error(msg) } // IsTrace indicates if TRACE logs would be emitted. This and the other Is* guards // are used to elide expensive logging code based on the current level. func (l *logrusAdapter) IsTrace() bool { return l.IsDebug() } // IsDebug indicates if DEBUG logs would be emitted. This and the other Is* guards // are used to elide expensive logging code based on the current level. func (l *logrusAdapter) IsDebug() bool { return l.level <= logrus.DebugLevel } // IsInfo indicates if INFO logs would be emitted. This and the other Is* guards // are used to elide expensive logging code based on the current level. func (l *logrusAdapter) IsInfo() bool { return l.level <= logrus.InfoLevel } // IsWarn indicates if WARN logs would be emitted. This and the other Is* guards // are used to elide expensive logging code based on the current level. func (l *logrusAdapter) IsWarn() bool { return l.level <= logrus.WarnLevel } // IsError indicates if ERROR logs would be emitted. This and the other Is* guards // are used to elide expensive logging code based on the current level. func (l *logrusAdapter) IsError() bool { return l.level <= logrus.ErrorLevel } // With creates a sublogger that will always have the given key/value pairs func (l *logrusAdapter) With(args ...any) hclog.Logger { return &logrusAdapter{ impl: l.impl.WithFields(argsToFields(args...)), level: l.level, } } // Named creates a logger that will add a `pluginName` field with the name string // as the value. If the logger already has a name, the new value will be appended // to the current name. func (l *logrusAdapter) Named(name string) hclog.Logger { var newName string if l.name == "" { newName = name } else { newName = l.name + "." + name } return l.ResetNamed(newName) } // ResetNamed creates a logger that will add a `pluginName` field with the name string // as the value. This sets the name of the logger to the value directly, unlike `Named` // which appends the given value to the current name. func (l *logrusAdapter) ResetNamed(name string) hclog.Logger { return &logrusAdapter{ impl: l.impl.WithField(pluginNameField, name), level: l.level, name: name, } } // StandardLogger returns a value that conforms to the stdlib log.Logger interface func (l *logrusAdapter) StandardLogger(opts *hclog.StandardLoggerOptions) *log.Logger { panic("not implemented") } // Updates the level. This should affect all sub-loggers as well. If an // implementation cannot update the level on the fly, it should no-op. func (l *logrusAdapter) SetLevel(_ hclog.Level) { } // ImpliedArgs returns With key/value pairs func (l *logrusAdapter) ImpliedArgs() []any { panic("not implemented") } // Args are alternating key, val pairs // keys must be strings // vals can be any type, but display is implementation specific // Emit a message and key/value pairs at a provided log level func (l *logrusAdapter) Log(level hclog.Level, msg string, args ...any) { switch level { case hclog.Trace: l.Trace(msg, args...) case hclog.Debug: l.Debug(msg, args...) case hclog.Info: l.Info(msg, args...) case hclog.Warn: l.Warn(msg, args...) case hclog.Error: l.Error(msg, args...) } } // Returns the Name of the logger func (l *logrusAdapter) Name() string { return l.name } // Return a value that conforms to io.Writer, which can be passed into log.SetOutput() func (l *logrusAdapter) StandardWriter(opts *hclog.StandardLoggerOptions) io.Writer { panic("not implemented") } ================================================ FILE: pkg/plugin/clientmgmt/process/logrus_adapter_test.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package process import ( "testing" "time" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" ) func TestArgsToFields(t *testing.T) { tests := []struct { name string args []any expectedFields logrus.Fields }{ { name: "empty args results in empty map of fields", args: []any{}, expectedFields: logrus.Fields(map[string]any{}), }, { name: "matching string keys/values are correctly set as fields", args: []any{"key-1", "value-1", "key-2", "value-2"}, expectedFields: logrus.Fields(map[string]any{ "key-1": "value-1", "key-2": "value-2", }), }, { name: "time/timestamp/level entries are removed", args: []any{"time", time.Now(), "key-1", "value-1", "timestamp", time.Now(), "key-2", "value-2", "level", "WARN"}, expectedFields: logrus.Fields(map[string]any{ "key-1": "value-1", "key-2": "value-2", }), }, { name: "odd number of args adds the last arg as a field with a nil value", args: []any{"key-1", "value-1", "key-2", "value-2", "key-3"}, expectedFields: logrus.Fields(map[string]any{ "key-1": "value-1", "key-2": "value-2", "key-3": nil, }), }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { assert.Equal(t, test.expectedFields, argsToFields(test.args...)) }) } } ================================================ FILE: pkg/plugin/clientmgmt/process/process.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package process import ( plugin "github.com/hashicorp/go-plugin" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" ) type Factory interface { newProcess(command string, logger logrus.FieldLogger, logLevel logrus.Level) (Process, error) } type processFactory struct { } func newProcessFactory() Factory { return &processFactory{} } func (pf *processFactory) newProcess(command string, logger logrus.FieldLogger, logLevel logrus.Level) (Process, error) { return newProcess(command, logger, logLevel) } type Process interface { dispense(key KindAndName) (any, error) exited() bool kill() } type process struct { client *plugin.Client protocolClient plugin.ClientProtocol } func newProcess(command string, logger logrus.FieldLogger, logLevel logrus.Level) (Process, error) { builder := newClientBuilder(command, logger.WithField("cmd", command), logLevel) // This creates a new go-plugin Client that has its own unique exec.Cmd for launching the plugin process. client := builder.client() // This launches the plugin process. protocolClient, err := client.Client() if err != nil { return nil, err } p := &process{ client: client, protocolClient: protocolClient, } return p, nil } // removeFeaturesFlag looks for and removes the '--features' arg // as well as the arg immediately following it (the flag value). func removeFeaturesFlag(args []string) []string { var commandArgs []string var featureFlag bool for _, arg := range args { // if this arg is the flag name, skip it if arg == "--features" { featureFlag = true continue } // if the last arg we saw was the flag name, then // this arg is the value for the flag, so skip it if featureFlag { featureFlag = false continue } // otherwise, keep the arg commandArgs = append(commandArgs, arg) } return commandArgs } func (r *process) dispense(key KindAndName) (any, error) { // This calls GRPCClient(clientConn) on the plugin instance registered for key.name. dispensed, err := r.protocolClient.Dispense(key.Kind.String()) if err != nil { return nil, errors.WithStack(err) } // Currently all plugins except for PluginLister dispense clientDispenser instances. if clientDispenser, ok := dispensed.(common.ClientDispenser); ok { if key.Name == "" { return nil, errors.Errorf("%s plugin requested but name is missing", key.Kind.String()) } // Get the instance that implements our plugin interface (e.g. ObjectStore) that is a gRPC-based // client dispensed = clientDispenser.ClientFor(key.Name) } return dispensed, nil } func (r *process) exited() bool { return r.client.Exited() } func (r *process) kill() { r.client.Kill() } ================================================ FILE: pkg/plugin/clientmgmt/process/process_test.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package process import ( "testing" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "github.com/vmware-tanzu/velero/pkg/plugin/framework" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" ) type mockClientProtocol struct { mock.Mock } func (cp *mockClientProtocol) Close() error { args := cp.Called() return args.Error(0) } func (cp *mockClientProtocol) Dispense(name string) (any, error) { args := cp.Called(name) return args.Get(0), args.Error(1) } func (cp *mockClientProtocol) Ping() error { args := cp.Called() return args.Error(0) } type mockClientDispenser struct { mock.Mock } func (cd *mockClientDispenser) ClientFor(name string) any { args := cd.Called(name) return args.Get(0) } func TestDispense(t *testing.T) { tests := []struct { name string missingKeyName bool dispenseError error clientDispenser bool expectedError string }{ { name: "protocol client dispense error", dispenseError: errors.Errorf("protocol client dispense"), expectedError: "protocol client dispense", }, { name: "plugin lister, no error", }, { name: "client dispenser, missing key name", clientDispenser: true, missingKeyName: true, expectedError: "ObjectStore plugin requested but name is missing", }, { name: "client dispenser, have key name", clientDispenser: true, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { p := new(process) protocolClient := new(mockClientProtocol) defer protocolClient.AssertExpectations(t) p.protocolClient = protocolClient clientDispenser := new(mockClientDispenser) defer clientDispenser.AssertExpectations(t) var client any key := KindAndName{} if tc.clientDispenser { key.Kind = common.PluginKindObjectStore protocolClient.On("Dispense", key.Kind.String()).Return(clientDispenser, tc.dispenseError) if !tc.missingKeyName { key.Name = "aws" client = &framework.BackupItemActionGRPCClient{} clientDispenser.On("ClientFor", key.Name).Return(client) } } else { key.Kind = common.PluginKindPluginLister client = &framework.PluginListerGRPCClient{} protocolClient.On("Dispense", key.Kind.String()).Return(client, tc.dispenseError) } dispensed, err := p.dispense(key) if tc.expectedError != "" { assert.EqualError(t, err, tc.expectedError) return } require.NoError(t, err) assert.Equal(t, client, dispensed) }) } } func Test_removeFeaturesFlag(t *testing.T) { tests := []struct { name string commandArgs []string want []string }{ { name: "when commandArgs is nil, a nil slice is returned", commandArgs: nil, want: nil, }, { name: "when commandArgs is empty, a nil slice is returned", commandArgs: []string{}, want: nil, }, { name: "when commandArgs does not contain --features, it is returned as-is", commandArgs: []string{"--log-level", "debug", "--another-flag", "foo"}, want: []string{"--log-level", "debug", "--another-flag", "foo"}, }, { name: "when --features is the only flag, a nil slice is returned", commandArgs: []string{"--features", "EnableCSI"}, want: nil, }, { name: "when --features is the first flag, it's properly removed", commandArgs: []string{"--features", "EnableCSI", "--log-level", "debug", "--another-flag", "foo"}, want: []string{"--log-level", "debug", "--another-flag", "foo"}, }, { name: "when --features is the last flag, it's properly removed", commandArgs: []string{"--log-level", "debug", "--another-flag", "foo", "--features", "EnableCSI"}, want: []string{"--log-level", "debug", "--another-flag", "foo"}, }, { name: "when --features is neither the first nor last flag, it's properly removed", commandArgs: []string{"--log-level", "debug", "--features", "EnableCSI", "--another-flag", "foo"}, want: []string{"--log-level", "debug", "--another-flag", "foo"}, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { assert.Equal(t, tc.want, removeFeaturesFlag(tc.commandArgs)) }) } } ================================================ FILE: pkg/plugin/clientmgmt/process/registry.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package process import ( "fmt" "os" "path/filepath" "strings" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/vmware-tanzu/velero/pkg/plugin/framework" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" "github.com/vmware-tanzu/velero/pkg/util/filesystem" ) // Registry manages information about available plugins. type Registry interface { // DiscoverPlugins discovers all available plugins. DiscoverPlugins() error // List returns all PluginIdentifiers for kind. List(kind common.PluginKind) []framework.PluginIdentifier // Get returns the PluginIdentifier for kind and name. Get(kind common.PluginKind, name string) (framework.PluginIdentifier, error) } // KindAndName is a convenience struct that combines a PluginKind and a name. type KindAndName struct { Kind common.PluginKind Name string } // registry implements Registry. type registry struct { // dir is the directory to search for plugins. dir string logger logrus.FieldLogger logLevel logrus.Level processFactory Factory fs filesystem.Interface pluginsByID map[KindAndName]framework.PluginIdentifier pluginsByKind map[common.PluginKind][]framework.PluginIdentifier } // NewRegistry returns a new registry. func NewRegistry(dir string, logger logrus.FieldLogger, logLevel logrus.Level) Registry { return ®istry{ dir: dir, logger: logger, logLevel: logLevel, processFactory: newProcessFactory(), fs: filesystem.NewFileSystem(), pluginsByID: make(map[KindAndName]framework.PluginIdentifier), pluginsByKind: make(map[common.PluginKind][]framework.PluginIdentifier), } } func (r *registry) DiscoverPlugins() error { plugins, err := r.readPluginsDir(r.dir) if err != nil { return err } // Start by adding velero's internal plugins commands := []string{os.Args[0]} // Then add the discovered plugin executables commands = append(commands, plugins...) return r.discoverPlugins(commands) } func (r *registry) discoverPlugins(commands []string) error { for _, command := range commands { plugins, err := r.listPlugins(command) if err != nil { return err } for _, plugin := range plugins { r.logger.WithFields(logrus.Fields{ "kind": plugin.Kind, "name": plugin.Name, "command": command, }).Info("registering plugin") if err := r.register(plugin); err != nil { return err } } } return nil } // List returns info about all plugin binaries that implement the given // PluginKind. func (r *registry) List(kind common.PluginKind) []framework.PluginIdentifier { return r.pluginsByKind[kind] } // Get returns info about a plugin with the given name and kind, or an // error if one cannot be found. func (r *registry) Get(kind common.PluginKind, name string) (framework.PluginIdentifier, error) { p, found := r.pluginsByID[KindAndName{Kind: kind, Name: name}] if !found { return framework.PluginIdentifier{}, newPluginNotFoundError(kind, name) } return p, nil } // readPluginsDir recursively reads dir looking for plugins. func (r *registry) readPluginsDir(dir string) ([]string, error) { if _, err := r.fs.Stat(dir); err != nil { if os.IsNotExist(err) { return []string{}, nil } return nil, errors.WithStack(err) } files, err := r.fs.ReadDir(dir) if err != nil { return nil, errors.WithStack(err) } fullPaths := make([]string, 0, len(files)) for _, file := range files { fullPath := filepath.Join(dir, file.Name()) if file.IsDir() { subDirPaths, err := r.readPluginsDir(fullPath) if err != nil { return nil, err } fullPaths = append(fullPaths, subDirPaths...) continue } if !executable(file) { r.logger.Warnf("Searching plugin skip file %s, not executable, mode %v, ext %s", file.Name(), file.Mode(), strings.ToLower(filepath.Ext(file.Name()))) continue } fullPaths = append(fullPaths, fullPath) } return fullPaths, nil } // executable determines if a file is executable. func executable(info os.FileInfo) bool { return executableLinux(info) || executableWindows(info) } func executableWindows(info os.FileInfo) bool { ext := strings.ToLower(filepath.Ext(info.Name())) return (ext == ".exe") } func executableLinux(info os.FileInfo) bool { /* When we AND the mode with 0111: - 0100 (user executable) - 0010 (group executable) - 0001 (other executable) the result will be 0 if and only if none of the executable bits is set. */ return (info.Mode() & 0111) != 0 } // listPlugins executes command, queries it for registered plugins, and returns the list of PluginIdentifiers. func (r *registry) listPlugins(command string) ([]framework.PluginIdentifier, error) { process, err := r.processFactory.newProcess(command, r.logger, r.logLevel) if err != nil { return nil, err } defer process.kill() plugin, err := process.dispense(KindAndName{Kind: common.PluginKindPluginLister}) if err != nil { return nil, err } lister, ok := plugin.(framework.PluginLister) if !ok { return nil, errors.Errorf("%T is not a PluginLister", plugin) } return lister.ListPlugins() } // register registers a PluginIdentifier with the registry. func (r *registry) register(id framework.PluginIdentifier) error { key := KindAndName{Kind: id.Kind, Name: id.Name} if existing, found := r.pluginsByID[key]; found { return newDuplicatePluginRegistrationError(existing, id) } // no need to pass list of existing plugins since the check if this exists was done above if err := common.ValidatePluginName(id.Name, nil); err != nil { return errors.Errorf("invalid plugin name %q: %s", id.Name, err) } r.pluginsByID[key] = id r.pluginsByKind[id.Kind] = append(r.pluginsByKind[id.Kind], id) // if id.Kind is adaptable to newer plugin versions, list it under the other versions as well // If BackupItemAction is adaptable to BackupItemActionV2, then it would be listed under both // kinds if kinds, ok := common.PluginKindsAdaptableTo[id.Kind]; ok { for _, kind := range kinds { r.pluginsByKind[kind] = append(r.pluginsByKind[kind], id) } } return nil } // pluginNotFoundError indicates a plugin could not be located for kind and name. type PluginNotFoundError struct { kind common.PluginKind name string } // newPluginNotFoundError returns a new pluginNotFoundError for kind and name. func newPluginNotFoundError(kind common.PluginKind, name string) *PluginNotFoundError { return &PluginNotFoundError{ kind: kind, name: name, } } func (e *PluginNotFoundError) Error() string { return fmt.Sprintf("unable to locate %v plugin named %s", e.kind, e.name) } type duplicatePluginRegistrationError struct { existing framework.PluginIdentifier duplicate framework.PluginIdentifier } func newDuplicatePluginRegistrationError(existing, duplicate framework.PluginIdentifier) *duplicatePluginRegistrationError { return &duplicatePluginRegistrationError{ existing: existing, duplicate: duplicate, } } func (e *duplicatePluginRegistrationError) Error() string { return fmt.Sprintf( "unable to register plugin (kind=%s, name=%s, command=%s) because another plugin is already registered for this kind and name (command=%s)", string(e.duplicate.Kind), e.duplicate.Name, e.duplicate.Command, e.existing.Command, ) } ================================================ FILE: pkg/plugin/clientmgmt/process/registry_test.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package process import ( "io/fs" "os" "sort" "testing" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/vmware-tanzu/velero/pkg/test" ) func TestNewRegistry(t *testing.T) { logger := test.NewLogger() logLevel := logrus.InfoLevel dir := "/plugins" r := NewRegistry(dir, logger, logLevel).(*registry) assert.Equal(t, dir, r.dir) assert.Equal(t, logger, r.logger) assert.Equal(t, logLevel, r.logLevel) assert.NotNil(t, r.pluginsByID) assert.Empty(t, r.pluginsByID) assert.NotNil(t, r.pluginsByKind) assert.Empty(t, r.pluginsByKind) } type fakeFileInfo struct { fs.FileInfo name string mode os.FileMode } func (f *fakeFileInfo) Mode() os.FileMode { return f.mode } func (f *fakeFileInfo) Name() string { return f.name } func TestExecutable(t *testing.T) { tests := []struct { name string fileName string mode uint32 expectExecutable bool }{ { name: "no perms", mode: 0000, }, { name: "r--r--r--", mode: 0444, }, { name: "rw-rw-rw-", mode: 0666, }, { name: "--x------", mode: 0100, expectExecutable: true, }, { name: "-----x---", mode: 0010, expectExecutable: true, }, { name: "--------x", mode: 0001, expectExecutable: true, }, { name: "rwxrwxrwx", mode: 0777, expectExecutable: true, }, { name: "windows lower case", fileName: "test.exe", mode: 0, expectExecutable: true, }, { name: "windows upper case", fileName: "test.EXE", mode: 0, expectExecutable: true, }, { name: "windows wrong ext", fileName: "test.EXE1", mode: 0, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { info := &fakeFileInfo{ name: test.fileName, mode: os.FileMode(test.mode), } assert.Equal(t, test.expectExecutable, executable(info)) }) } } func TestReadPluginsDir(t *testing.T) { logger := test.NewLogger() logLevel := logrus.InfoLevel dir := "/plugins" r := NewRegistry(dir, logger, logLevel).(*registry) r.fs = test.NewFakeFileSystem(). WithFileAndMode("/plugins/executable1", []byte("plugin1"), 0755). WithFileAndMode("/plugins/nonexecutable2", []byte("plugin2"), 0644). WithFileAndMode("/plugins/executable3", []byte("plugin3"), 0755). WithFileAndMode("/plugins/nested/executable4", []byte("plugin4"), 0755). WithFileAndMode("/plugins/nested/nonexecutable5", []byte("plugin4"), 0644). WithFileAndMode("/plugins/nested/win-exe1.exe", []byte("plugin4"), 0600). WithFileAndMode("/plugins/nested/WIN-EXE2.EXE", []byte("plugin4"), 0600) plugins, err := r.readPluginsDir(dir) require.NoError(t, err) expected := []string{ "/plugins/executable1", "/plugins/executable3", "/plugins/nested/executable4", "/plugins/nested/win-exe1.exe", "/plugins/nested/WIN-EXE2.EXE", } sort.Strings(plugins) sort.Strings(expected) assert.Equal(t, expected, plugins) } ================================================ FILE: pkg/plugin/clientmgmt/process/restartable_process.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package process import ( "sync" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) type RestartableProcessFactory interface { NewRestartableProcess(command string, logger logrus.FieldLogger, logLevel logrus.Level) (RestartableProcess, error) } type restartableProcessFactory struct { } func NewRestartableProcessFactory() RestartableProcessFactory { return &restartableProcessFactory{} } func (rpf *restartableProcessFactory) NewRestartableProcess(command string, logger logrus.FieldLogger, logLevel logrus.Level) (RestartableProcess, error) { return newRestartableProcess(command, logger, logLevel) } type RestartableProcess interface { AddReinitializer(key KindAndName, r Reinitializer) Reset() error ResetIfNeeded() error GetByKindAndName(key KindAndName) (any, error) Stop() } // restartableProcess encapsulates the lifecycle for all plugins contained in a single executable file. It is able // to restart a plugin process if it is terminated for any reason. If this happens, all plugins are reinitialized using // the original configuration data. type restartableProcess struct { command string logger logrus.FieldLogger logLevel logrus.Level // lock guards all of the fields below lock sync.RWMutex process Process plugins map[KindAndName]any reinitializers map[KindAndName]Reinitializer resetFailures int } // reinitializer is capable of reinitializing a restartable plugin instance using the newly dispensed plugin. type Reinitializer interface { // reinitialize reinitializes a restartable plugin instance using the newly dispensed plugin. Reinitialize(dispensed any) error } // newRestartableProcess creates a new restartableProcess for the given command and options. func newRestartableProcess(command string, logger logrus.FieldLogger, logLevel logrus.Level) (RestartableProcess, error) { p := &restartableProcess{ command: command, logger: logger, logLevel: logLevel, plugins: make(map[KindAndName]any), reinitializers: make(map[KindAndName]Reinitializer), } // This launches the process err := p.Reset() return p, err } // AddReinitializer registers the reinitializer r for key. func (p *restartableProcess) AddReinitializer(key KindAndName, r Reinitializer) { p.lock.Lock() defer p.lock.Unlock() p.reinitializers[key] = r } // Reset acquires the lock and calls resetLH. func (p *restartableProcess) Reset() error { p.lock.Lock() defer p.lock.Unlock() return p.resetLH() } // resetLH (re)launches the plugin process. It redispenses all previously dispensed plugins and reinitializes all the // registered reinitializers using the newly dispensed plugins. // // Callers of resetLH *must* acquire the lock before calling it. func (p *restartableProcess) resetLH() error { if p.resetFailures > 10 { return errors.Errorf("unable to restart plugin process: exceeded maximum number of reset failures") } process, err := newProcess(p.command, p.logger, p.logLevel) if err != nil { p.resetFailures++ return err } p.process = process // Redispense any previously dispensed plugins, reinitializing if necessary. // Start by creating a new map to hold the newly dispensed plugins. newPlugins := make(map[KindAndName]any) for key := range p.plugins { // Re-dispense dispensed, err := p.process.dispense(key) if err != nil { p.resetFailures++ return err } // Store in the new map newPlugins[key] = dispensed // Reinitialize if r, found := p.reinitializers[key]; found { if err := r.Reinitialize(dispensed); err != nil { p.resetFailures++ return err } } } // Make sure we update p's plugins! p.plugins = newPlugins p.resetFailures = 0 return nil } // ResetIfNeeded checks if the plugin process has exited and resets p if it has. func (p *restartableProcess) ResetIfNeeded() error { p.lock.Lock() defer p.lock.Unlock() if p.process.exited() { p.logger.Info("Plugin process exited - restarting.") return p.resetLH() } return nil } // GetByKindAndName acquires the lock and calls getByKindAndNameLH. func (p *restartableProcess) GetByKindAndName(key KindAndName) (any, error) { p.lock.Lock() defer p.lock.Unlock() return p.getByKindAndNameLH(key) } // getByKindAndNameLH returns the dispensed plugin for key. If the plugin hasn't been dispensed before, it dispenses a // new one. func (p *restartableProcess) getByKindAndNameLH(key KindAndName) (any, error) { dispensed, found := p.plugins[key] if found { return dispensed, nil } dispensed, err := p.process.dispense(key) if err != nil { return nil, err } p.plugins[key] = dispensed return p.plugins[key], nil } // stop terminates the plugin process. func (p *restartableProcess) Stop() { p.lock.Lock() p.process.kill() p.lock.Unlock() } ================================================ FILE: pkg/plugin/clientmgmt/restartable_delete_item_action.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package clientmgmt import ( "github.com/pkg/errors" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt/process" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" "github.com/vmware-tanzu/velero/pkg/plugin/velero" ) // restartableDeleteItemAction is a delete item action for a given implementation (such as "pod"). It is associated with // a restartableProcess, which may be shared and used to run multiple plugins. At the beginning of each method // call, the restartableDeleteItemAction asks its restartableProcess to restart itself if needed (e.g. if the // process terminated for any reason), then it proceeds with the actual call. type restartableDeleteItemAction struct { key process.KindAndName sharedPluginProcess process.RestartableProcess } // NewRestartableDeleteItemAction returns a new restartableDeleteItemAction. func NewRestartableDeleteItemAction(name string, sharedPluginProcess process.RestartableProcess) *restartableDeleteItemAction { r := &restartableDeleteItemAction{ key: process.KindAndName{Kind: common.PluginKindDeleteItemAction, Name: name}, sharedPluginProcess: sharedPluginProcess, } return r } // getDeleteItemAction returns the delete item action for this restartableDeleteItemAction. It does *not* restart the // plugin process. func (r *restartableDeleteItemAction) getDeleteItemAction() (velero.DeleteItemAction, error) { plugin, err := r.sharedPluginProcess.GetByKindAndName(r.key) if err != nil { return nil, err } deleteItemAction, ok := plugin.(velero.DeleteItemAction) if !ok { return nil, errors.Errorf("plugin %T is not a DeleteItemAction", plugin) } return deleteItemAction, nil } // getDelegate restarts the plugin process (if needed) and returns the delete item action for this restartableDeleteItemAction. func (r *restartableDeleteItemAction) getDelegate() (velero.DeleteItemAction, error) { if err := r.sharedPluginProcess.ResetIfNeeded(); err != nil { return nil, err } return r.getDeleteItemAction() } // AppliesTo restarts the plugin's process if needed, then delegates the call. func (r *restartableDeleteItemAction) AppliesTo() (velero.ResourceSelector, error) { delegate, err := r.getDelegate() if err != nil { return velero.ResourceSelector{}, err } return delegate.AppliesTo() } // Execute restarts the plugin's process if needed, then delegates the call. func (r *restartableDeleteItemAction) Execute(input *velero.DeleteItemActionExecuteInput) error { delegate, err := r.getDelegate() if err != nil { return err } return delegate.Execute(input) } ================================================ FILE: pkg/plugin/clientmgmt/restartable_delete_item_action_test.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package clientmgmt import ( "testing" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "github.com/vmware-tanzu/velero/internal/restartabletest" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt/process" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" "github.com/vmware-tanzu/velero/pkg/plugin/velero" "github.com/vmware-tanzu/velero/pkg/plugin/velero/mocks" ) func TestRestartableGetDeleteItemAction(t *testing.T) { tests := []struct { name string plugin any getError error expectedError string }{ { name: "error getting by kind and name", getError: errors.Errorf("get error"), expectedError: "get error", }, { name: "wrong type", plugin: 3, expectedError: "plugin int is not a DeleteItemAction", }, { name: "happy path", plugin: new(mocks.DeleteItemAction), }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { p := new(restartabletest.MockRestartableProcess) defer p.AssertExpectations(t) name := "pod" key := process.KindAndName{Kind: common.PluginKindDeleteItemAction, Name: name} p.On("GetByKindAndName", key).Return(tc.plugin, tc.getError) r := NewRestartableDeleteItemAction(name, p) a, err := r.getDeleteItemAction() if tc.expectedError != "" { assert.EqualError(t, err, tc.expectedError) return } require.NoError(t, err) assert.Equal(t, tc.plugin, a) }) } } func TestRestartableDeleteItemActionGetDelegate(t *testing.T) { p := new(restartabletest.MockRestartableProcess) defer p.AssertExpectations(t) // Reset error p.On("ResetIfNeeded").Return(errors.Errorf("reset error")).Once() name := "pod" r := NewRestartableDeleteItemAction(name, p) a, err := r.getDelegate() assert.Nil(t, a) require.EqualError(t, err, "reset error") // Happy path // Currently broken since this mocks out the restore item action interface p.On("ResetIfNeeded").Return(nil) expected := new(mocks.DeleteItemAction) key := process.KindAndName{Kind: common.PluginKindDeleteItemAction, Name: name} p.On("GetByKindAndName", key).Return(expected, nil) a, err = r.getDelegate() require.NoError(t, err) assert.Equal(t, expected, a) } func TestRestartableDeleteItemActionDelegatedFunctions(t *testing.T) { pv := &unstructured.Unstructured{ Object: map[string]any{ "color": "blue", }, } backup := &api.Backup{} input := &velero.DeleteItemActionExecuteInput{ Item: pv, Backup: backup, } restartabletest.RunRestartableDelegateTests( t, common.PluginKindDeleteItemAction, func(key process.KindAndName, p process.RestartableProcess) any { return &restartableDeleteItemAction{ key: key, sharedPluginProcess: p, } }, func() restartabletest.Mockable { // Currently broken because this mocks the restore item action interface return new(mocks.DeleteItemAction) }, restartabletest.RestartableDelegateTest{ Function: "AppliesTo", Inputs: []any{}, ExpectedErrorOutputs: []any{velero.ResourceSelector{}, errors.Errorf("reset error")}, ExpectedDelegateOutputs: []any{velero.ResourceSelector{IncludedNamespaces: []string{"a"}}, errors.Errorf("delegate error")}, }, restartabletest.RestartableDelegateTest{ Function: "Execute", Inputs: []any{input}, ExpectedErrorOutputs: []any{errors.Errorf("reset error")}, ExpectedDelegateOutputs: []any{errors.Errorf("delegate error")}, }, ) } ================================================ FILE: pkg/plugin/clientmgmt/restartable_object_store.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package clientmgmt import ( "io" "time" "github.com/pkg/errors" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt/process" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" "github.com/vmware-tanzu/velero/pkg/plugin/velero" ) // restartableObjectStore is an object store for a given implementation (such as "aws"). It is associated with // a restartableProcess, which may be shared and used to run multiple plugins. At the beginning of each method // call, the restartableObjectStore asks its restartableProcess to restart itself if needed (e.g. if the // process terminated for any reason), then it proceeds with the actual call. type restartableObjectStore struct { key process.KindAndName sharedPluginProcess process.RestartableProcess // config contains the data used to initialize the plugin. It is used to reinitialize the plugin in the event its // sharedPluginProcess gets restarted. config map[string]string } // NewRestartableObjectStore returns a new restartableObjectStore. func NewRestartableObjectStore(name string, sharedPluginProcess process.RestartableProcess) *restartableObjectStore { key := process.KindAndName{Kind: common.PluginKindObjectStore, Name: name} r := &restartableObjectStore{ key: key, sharedPluginProcess: sharedPluginProcess, } // Register our reinitializer so we can reinitialize after a restart with r.config. sharedPluginProcess.AddReinitializer(key, r) return r } // reinitialize reinitializes a re-dispensed plugin using the initial data passed to Init(). func (r *restartableObjectStore) Reinitialize(dispensed any) error { objectStore, ok := dispensed.(velero.ObjectStore) if !ok { return errors.Errorf("plugin %T is not a ObjectStore", dispensed) } return r.init(objectStore, r.config) } // getObjectStore returns the object store for this restartableObjectStore. It does *not* restart the // plugin process. func (r *restartableObjectStore) getObjectStore() (velero.ObjectStore, error) { plugin, err := r.sharedPluginProcess.GetByKindAndName(r.key) if err != nil { return nil, err } objectStore, ok := plugin.(velero.ObjectStore) if !ok { return nil, errors.Errorf("plugin %T is not a ObjectStore", plugin) } return objectStore, nil } // getDelegate restarts the plugin process (if needed) and returns the object store for this restartableObjectStore. func (r *restartableObjectStore) getDelegate() (velero.ObjectStore, error) { if err := r.sharedPluginProcess.ResetIfNeeded(); err != nil { return nil, err } return r.getObjectStore() } // Init initializes the object store instance using config. If this is the first invocation, r stores config for future // reinitialization needs. Init does NOT restart the shared plugin process. Init may only be called once. func (r *restartableObjectStore) Init(config map[string]string) error { if r.config != nil { return errors.Errorf("already initialized") } // Not using getDelegate() to avoid possible infinite recursion delegate, err := r.getObjectStore() if err != nil { return err } r.config = config return r.init(delegate, config) } // init calls Init on objectStore with config. This is split out from Init() so that both Init() and reinitialize() may // call it using a specific ObjectStore. func (r *restartableObjectStore) init(objectStore velero.ObjectStore, config map[string]string) error { return objectStore.Init(config) } // PutObject restarts the plugin's process if needed, then delegates the call. func (r *restartableObjectStore) PutObject(bucket string, key string, body io.Reader) error { delegate, err := r.getDelegate() if err != nil { return err } return delegate.PutObject(bucket, key, body) } // ObjectExists restarts the plugin's process if needed, then delegates the call. func (r *restartableObjectStore) ObjectExists(bucket, key string) (bool, error) { delegate, err := r.getDelegate() if err != nil { return false, err } return delegate.ObjectExists(bucket, key) } // GetObject restarts the plugin's process if needed, then delegates the call. func (r *restartableObjectStore) GetObject(bucket string, key string) (io.ReadCloser, error) { delegate, err := r.getDelegate() if err != nil { return nil, err } return delegate.GetObject(bucket, key) } // ListCommonPrefixes restarts the plugin's process if needed, then delegates the call. func (r *restartableObjectStore) ListCommonPrefixes(bucket string, prefix string, delimiter string) ([]string, error) { delegate, err := r.getDelegate() if err != nil { return nil, err } return delegate.ListCommonPrefixes(bucket, prefix, delimiter) } // ListObjects restarts the plugin's process if needed, then delegates the call. func (r *restartableObjectStore) ListObjects(bucket string, prefix string) ([]string, error) { delegate, err := r.getDelegate() if err != nil { return nil, err } return delegate.ListObjects(bucket, prefix) } // DeleteObject restarts the plugin's process if needed, then delegates the call. func (r *restartableObjectStore) DeleteObject(bucket string, key string) error { delegate, err := r.getDelegate() if err != nil { return err } return delegate.DeleteObject(bucket, key) } // CreateSignedURL restarts the plugin's process if needed, then delegates the call. func (r *restartableObjectStore) CreateSignedURL(bucket string, key string, ttl time.Duration) (string, error) { delegate, err := r.getDelegate() if err != nil { return "", err } return delegate.CreateSignedURL(bucket, key, ttl) } ================================================ FILE: pkg/plugin/clientmgmt/restartable_object_store_test.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package clientmgmt import ( "io" "strings" "testing" "time" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/vmware-tanzu/velero/internal/restartabletest" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt/process" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" providermocks "github.com/vmware-tanzu/velero/pkg/plugin/velero/mocks" ) func TestRestartableGetObjectStore(t *testing.T) { tests := []struct { name string plugin any getError error expectedError string }{ { name: "error getting by kind and name", getError: errors.Errorf("get error"), expectedError: "get error", }, { name: "wrong type", plugin: 3, expectedError: "plugin int is not a ObjectStore", }, { name: "happy path", plugin: new(providermocks.ObjectStore), }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { p := new(restartabletest.MockRestartableProcess) p.Test(t) defer p.AssertExpectations(t) name := "aws" key := process.KindAndName{Kind: common.PluginKindObjectStore, Name: name} p.On("GetByKindAndName", key).Return(tc.plugin, tc.getError) r := &restartableObjectStore{ key: key, sharedPluginProcess: p, } a, err := r.getObjectStore() if tc.expectedError != "" { assert.EqualError(t, err, tc.expectedError) return } require.NoError(t, err) assert.Equal(t, tc.plugin, a) }) } } func TestRestartableObjectStoreReinitialize(t *testing.T) { p := new(restartabletest.MockRestartableProcess) p.Test(t) defer p.AssertExpectations(t) name := "aws" key := process.KindAndName{Kind: common.PluginKindObjectStore, Name: name} r := &restartableObjectStore{ key: key, sharedPluginProcess: p, config: map[string]string{ "color": "blue", }, } err := r.Reinitialize(3) require.EqualError(t, err, "plugin int is not a ObjectStore") objectStore := new(providermocks.ObjectStore) objectStore.Test(t) defer objectStore.AssertExpectations(t) objectStore.On("Init", r.config).Return(errors.Errorf("init error")).Once() err = r.Reinitialize(objectStore) require.EqualError(t, err, "init error") objectStore.On("Init", r.config).Return(nil) err = r.Reinitialize(objectStore) assert.NoError(t, err) } func TestRestartableObjectStoreGetDelegate(t *testing.T) { p := new(restartabletest.MockRestartableProcess) p.Test(t) defer p.AssertExpectations(t) // Reset error p.On("ResetIfNeeded").Return(errors.Errorf("reset error")).Once() name := "aws" key := process.KindAndName{Kind: common.PluginKindObjectStore, Name: name} r := &restartableObjectStore{ key: key, sharedPluginProcess: p, } a, err := r.getDelegate() assert.Nil(t, a) require.EqualError(t, err, "reset error") // Happy path p.On("ResetIfNeeded").Return(nil) objectStore := new(providermocks.ObjectStore) objectStore.Test(t) defer objectStore.AssertExpectations(t) p.On("GetByKindAndName", key).Return(objectStore, nil) a, err = r.getDelegate() require.NoError(t, err) assert.Equal(t, objectStore, a) } func TestRestartableObjectStoreInit(t *testing.T) { p := new(restartabletest.MockRestartableProcess) p.Test(t) defer p.AssertExpectations(t) // getObjectStore error name := "aws" key := process.KindAndName{Kind: common.PluginKindObjectStore, Name: name} r := &restartableObjectStore{ key: key, sharedPluginProcess: p, } p.On("GetByKindAndName", key).Return(nil, errors.Errorf("GetByKindAndName error")).Once() config := map[string]string{ "color": "blue", } err := r.Init(config) require.EqualError(t, err, "GetByKindAndName error") // Delegate returns error objectStore := new(providermocks.ObjectStore) objectStore.Test(t) defer objectStore.AssertExpectations(t) p.On("GetByKindAndName", key).Return(objectStore, nil) objectStore.On("Init", config).Return(errors.Errorf("Init error")).Once() err = r.Init(config) require.EqualError(t, err, "Init error") // wipe this out because the previous failed Init call set it r.config = nil // Happy path objectStore.On("Init", config).Return(nil) err = r.Init(config) require.NoError(t, err) assert.Equal(t, config, r.config) // Calling Init twice is forbidden err = r.Init(config) assert.EqualError(t, err, "already initialized") } func TestRestartableObjectStoreDelegatedFunctions(t *testing.T) { restartabletest.RunRestartableDelegateTests( t, common.PluginKindObjectStore, func(key process.KindAndName, p process.RestartableProcess) any { return &restartableObjectStore{ key: key, sharedPluginProcess: p, } }, func() restartabletest.Mockable { return new(providermocks.ObjectStore) }, restartabletest.RestartableDelegateTest{ Function: "PutObject", Inputs: []any{"bucket", "key", strings.NewReader("body")}, ExpectedErrorOutputs: []any{errors.Errorf("reset error")}, ExpectedDelegateOutputs: []any{errors.Errorf("delegate error")}, }, restartabletest.RestartableDelegateTest{ Function: "GetObject", Inputs: []any{"bucket", "key"}, ExpectedErrorOutputs: []any{nil, errors.Errorf("reset error")}, ExpectedDelegateOutputs: []any{io.NopCloser(strings.NewReader("object")), errors.Errorf("delegate error")}, }, restartabletest.RestartableDelegateTest{ Function: "ListCommonPrefixes", Inputs: []any{"bucket", "prefix", "delimiter"}, ExpectedErrorOutputs: []any{([]string)(nil), errors.Errorf("reset error")}, ExpectedDelegateOutputs: []any{[]string{"a", "b"}, errors.Errorf("delegate error")}, }, restartabletest.RestartableDelegateTest{ Function: "ListObjects", Inputs: []any{"bucket", "prefix"}, ExpectedErrorOutputs: []any{([]string)(nil), errors.Errorf("reset error")}, ExpectedDelegateOutputs: []any{[]string{"a", "b"}, errors.Errorf("delegate error")}, }, restartabletest.RestartableDelegateTest{ Function: "DeleteObject", Inputs: []any{"bucket", "key"}, ExpectedErrorOutputs: []any{errors.Errorf("reset error")}, ExpectedDelegateOutputs: []any{errors.Errorf("delegate error")}, }, restartabletest.RestartableDelegateTest{ Function: "CreateSignedURL", Inputs: []any{"bucket", "key", 30 * time.Minute}, ExpectedErrorOutputs: []any{"", errors.Errorf("reset error")}, ExpectedDelegateOutputs: []any{"signedURL", errors.Errorf("delegate error")}, }, ) } ================================================ FILE: pkg/plugin/clientmgmt/restoreitemaction/v1/restartable_restore_item_action.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v1 import ( "github.com/pkg/errors" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt/process" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" "github.com/vmware-tanzu/velero/pkg/plugin/velero" riav1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/restoreitemaction/v1" ) // AdaptedRestoreItemAction is a restore item action adapted to the v1 RestoreItemAction API type AdaptedRestoreItemAction struct { Kind common.PluginKind // Get returns a restartable RestoreItemAction for the given name and process, wrapping if necessary GetRestartable func(name string, restartableProcess process.RestartableProcess) riav1.RestoreItemAction } func AdaptedRestoreItemActions() []AdaptedRestoreItemAction { return []AdaptedRestoreItemAction{ { Kind: common.PluginKindRestoreItemAction, GetRestartable: func(name string, restartableProcess process.RestartableProcess) riav1.RestoreItemAction { return NewRestartableRestoreItemAction(name, restartableProcess) }, }, } } // RestartableRestoreItemAction is a restore item action for a given implementation (such as "pod"). It is associated with // a restartableProcess, which may be shared and used to run multiple plugins. At the beginning of each method // call, the RestartableRestoreItemAction asks its restartableProcess to restart itself if needed (e.g. if the // process terminated for any reason), then it proceeds with the actual call. type RestartableRestoreItemAction struct { Key process.KindAndName SharedPluginProcess process.RestartableProcess } // NewRestartableRestoreItemAction returns a new RestartableRestoreItemAction. func NewRestartableRestoreItemAction(name string, sharedPluginProcess process.RestartableProcess) *RestartableRestoreItemAction { r := &RestartableRestoreItemAction{ Key: process.KindAndName{Kind: common.PluginKindRestoreItemAction, Name: name}, SharedPluginProcess: sharedPluginProcess, } return r } // getRestoreItemAction returns the restore item action for this RestartableRestoreItemAction. It does *not* restart the // plugin process. func (r *RestartableRestoreItemAction) getRestoreItemAction() (riav1.RestoreItemAction, error) { plugin, err := r.SharedPluginProcess.GetByKindAndName(r.Key) if err != nil { return nil, err } restoreItemAction, ok := plugin.(riav1.RestoreItemAction) if !ok { return nil, errors.Errorf("plugin %T is not a RestoreItemAction", plugin) } return restoreItemAction, nil } // getDelegate restarts the plugin process (if needed) and returns the restore item action for this RestartableRestoreItemAction. func (r *RestartableRestoreItemAction) getDelegate() (riav1.RestoreItemAction, error) { if err := r.SharedPluginProcess.ResetIfNeeded(); err != nil { return nil, err } return r.getRestoreItemAction() } // AppliesTo restarts the plugin's process if needed, then delegates the call. func (r RestartableRestoreItemAction) AppliesTo() (velero.ResourceSelector, error) { delegate, err := r.getDelegate() if err != nil { return velero.ResourceSelector{}, err } return delegate.AppliesTo() } // Execute restarts the plugin's process if needed, then delegates the call. func (r *RestartableRestoreItemAction) Execute(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { delegate, err := r.getDelegate() if err != nil { return nil, err } return delegate.Execute(input) } ================================================ FILE: pkg/plugin/clientmgmt/restoreitemaction/v1/restartable_restore_item_action_test.go ================================================ /* Copyright 2018, 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v1 import ( "testing" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "github.com/vmware-tanzu/velero/internal/restartabletest" v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt/process" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" "github.com/vmware-tanzu/velero/pkg/plugin/velero" mocks "github.com/vmware-tanzu/velero/pkg/plugin/velero/mocks/restoreitemaction/v1" ) func TestRestartableGetRestoreItemAction(t *testing.T) { tests := []struct { name string plugin any getError error expectedError string }{ { name: "error getting by kind and name", getError: errors.Errorf("get error"), expectedError: "get error", }, { name: "wrong type", plugin: 3, expectedError: "plugin int is not a RestoreItemAction", }, { name: "happy path", plugin: new(mocks.RestoreItemAction), }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { p := new(restartabletest.MockRestartableProcess) defer p.AssertExpectations(t) name := "pod" key := process.KindAndName{Kind: common.PluginKindRestoreItemAction, Name: name} p.On("GetByKindAndName", key).Return(tc.plugin, tc.getError) r := NewRestartableRestoreItemAction(name, p) a, err := r.getRestoreItemAction() if tc.expectedError != "" { assert.EqualError(t, err, tc.expectedError) return } require.NoError(t, err) assert.Equal(t, tc.plugin, a) }) } } func TestRestartableRestoreItemActionGetDelegate(t *testing.T) { p := new(restartabletest.MockRestartableProcess) defer p.AssertExpectations(t) // Reset error p.On("ResetIfNeeded").Return(errors.Errorf("reset error")).Once() name := "pod" r := NewRestartableRestoreItemAction(name, p) a, err := r.getDelegate() assert.Nil(t, a) require.EqualError(t, err, "reset error") // Happy path p.On("ResetIfNeeded").Return(nil) expected := new(mocks.RestoreItemAction) key := process.KindAndName{Kind: common.PluginKindRestoreItemAction, Name: name} p.On("GetByKindAndName", key).Return(expected, nil) a, err = r.getDelegate() require.NoError(t, err) assert.Equal(t, expected, a) } func TestRestartableRestoreItemActionDelegatedFunctions(t *testing.T) { pv := &unstructured.Unstructured{ Object: map[string]any{ "color": "blue", }, } input := &velero.RestoreItemActionExecuteInput{ Item: pv, ItemFromBackup: pv, Restore: new(v1.Restore), } output := &velero.RestoreItemActionExecuteOutput{ UpdatedItem: &unstructured.Unstructured{ Object: map[string]any{ "color": "green", }, }, } restartabletest.RunRestartableDelegateTests( t, common.PluginKindRestoreItemAction, func(key process.KindAndName, p process.RestartableProcess) any { return &RestartableRestoreItemAction{ Key: key, SharedPluginProcess: p, } }, func() restartabletest.Mockable { return new(mocks.RestoreItemAction) }, restartabletest.RestartableDelegateTest{ Function: "AppliesTo", Inputs: []any{}, ExpectedErrorOutputs: []any{velero.ResourceSelector{}, errors.Errorf("reset error")}, ExpectedDelegateOutputs: []any{velero.ResourceSelector{IncludedNamespaces: []string{"a"}}, errors.Errorf("delegate error")}, }, restartabletest.RestartableDelegateTest{ Function: "Execute", Inputs: []any{input}, ExpectedErrorOutputs: []any{nil, errors.Errorf("reset error")}, ExpectedDelegateOutputs: []any{output, errors.Errorf("delegate error")}, }, ) } ================================================ FILE: pkg/plugin/clientmgmt/restoreitemaction/v2/restartable_restore_item_action.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v2 import ( "github.com/pkg/errors" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt/process" riav1cli "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt/restoreitemaction/v1" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" "github.com/vmware-tanzu/velero/pkg/plugin/velero" riav2 "github.com/vmware-tanzu/velero/pkg/plugin/velero/restoreitemaction/v2" ) // AdaptedRestoreItemAction is a v1 RestoreItemAction adapted to implement the v2 API type AdaptedRestoreItemAction struct { Kind common.PluginKind // Get returns a restartable RestoreItemAction for the given name and process, wrapping if necessary GetRestartable func(name string, restartableProcess process.RestartableProcess) riav2.RestoreItemAction } func AdaptedRestoreItemActions() []AdaptedRestoreItemAction { return []AdaptedRestoreItemAction{ { Kind: common.PluginKindRestoreItemActionV2, GetRestartable: func(name string, restartableProcess process.RestartableProcess) riav2.RestoreItemAction { return NewRestartableRestoreItemAction(name, restartableProcess) }, }, { Kind: common.PluginKindRestoreItemAction, GetRestartable: func(name string, restartableProcess process.RestartableProcess) riav2.RestoreItemAction { return NewAdaptedV1RestartableRestoreItemAction(riav1cli.NewRestartableRestoreItemAction(name, restartableProcess)) }, }, } } // RestartableRestoreItemAction is a restore item action for a given implementation (such as "pod"). It is associated with // a restartableProcess, which may be shared and used to run multiple plugins. At the beginning of each method // call, the RestartableRestoreItemAction asks its restartableProcess to restart itself if needed (e.g. if the // process terminated for any reason), then it proceeds with the actual call. type RestartableRestoreItemAction struct { Key process.KindAndName SharedPluginProcess process.RestartableProcess } // NewRestartableRestoreItemAction returns a new RestartableRestoreItemAction. func NewRestartableRestoreItemAction(name string, sharedPluginProcess process.RestartableProcess) *RestartableRestoreItemAction { r := &RestartableRestoreItemAction{ Key: process.KindAndName{Kind: common.PluginKindRestoreItemActionV2, Name: name}, SharedPluginProcess: sharedPluginProcess, } return r } // getRestoreItemAction returns the restore item action for this RestartableRestoreItemAction. It does *not* restart the // plugin process. func (r *RestartableRestoreItemAction) getRestoreItemAction() (riav2.RestoreItemAction, error) { plugin, err := r.SharedPluginProcess.GetByKindAndName(r.Key) if err != nil { return nil, err } restoreItemAction, ok := plugin.(riav2.RestoreItemAction) if !ok { return nil, errors.Errorf("plugin %T is not a RestoreItemActionV2", plugin) } return restoreItemAction, nil } // getDelegate restarts the plugin process (if needed) and returns the restore item action for this RestartableRestoreItemAction. func (r *RestartableRestoreItemAction) getDelegate() (riav2.RestoreItemAction, error) { if err := r.SharedPluginProcess.ResetIfNeeded(); err != nil { return nil, err } return r.getRestoreItemAction() } // Name returns the plugin's name. func (r *RestartableRestoreItemAction) Name() string { return r.Key.Name } // AppliesTo restarts the plugin's process if needed, then delegates the call. func (r RestartableRestoreItemAction) AppliesTo() (velero.ResourceSelector, error) { delegate, err := r.getDelegate() if err != nil { return velero.ResourceSelector{}, err } return delegate.AppliesTo() } // Execute restarts the plugin's process if needed, then delegates the call. func (r *RestartableRestoreItemAction) Execute(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { delegate, err := r.getDelegate() if err != nil { return nil, err } return delegate.Execute(input) } // Progress restarts the plugin's process if needed, then delegates the call. func (r *RestartableRestoreItemAction) Progress(operationID string, restore *api.Restore) (velero.OperationProgress, error) { delegate, err := r.getDelegate() if err != nil { return velero.OperationProgress{}, err } return delegate.Progress(operationID, restore) } // Cancel restarts the plugin's process if needed, then delegates the call. func (r *RestartableRestoreItemAction) Cancel(operationID string, restore *api.Restore) error { delegate, err := r.getDelegate() if err != nil { return err } return delegate.Cancel(operationID, restore) } // AreAdditionalItemsReady restarts the plugin's process if needed, then delegates the call. func (r *RestartableRestoreItemAction) AreAdditionalItemsReady(additionalItems []velero.ResourceIdentifier, restore *api.Restore) (bool, error) { delegate, err := r.getDelegate() if err != nil { return false, err } return delegate.AreAdditionalItemsReady(additionalItems, restore) } type AdaptedV1RestartableRestoreItemAction struct { V1Restartable *riav1cli.RestartableRestoreItemAction } // NewAdaptedV1RestartableRestoreItemAction returns a new v1 RestartableRestoreItemAction adapted to v2 func NewAdaptedV1RestartableRestoreItemAction(v1Restartable *riav1cli.RestartableRestoreItemAction) *AdaptedV1RestartableRestoreItemAction { r := &AdaptedV1RestartableRestoreItemAction{ V1Restartable: v1Restartable, } return r } // Name restarts the plugin's name. func (r *AdaptedV1RestartableRestoreItemAction) Name() string { return r.V1Restartable.Key.Name } // AppliesTo delegates to the v1 AppliesTo call. func (r *AdaptedV1RestartableRestoreItemAction) AppliesTo() (velero.ResourceSelector, error) { return r.V1Restartable.AppliesTo() } // Execute delegates to the v1 Execute call, returning an empty operationID. func (r *AdaptedV1RestartableRestoreItemAction) Execute(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { return r.V1Restartable.Execute(input) } // Progress returns with an error since v1 plugins will never return an operationID, which means that // any operationID passed in here will be invalid. func (r *AdaptedV1RestartableRestoreItemAction) Progress(operationID string, restore *api.Restore) (velero.OperationProgress, error) { return velero.OperationProgress{}, riav2.AsyncOperationsNotSupportedError() } // Cancel just returns without error since v1 plugins don't implement it. func (r *AdaptedV1RestartableRestoreItemAction) Cancel(operationID string, restore *api.Restore) error { return nil } // AreAdditionalItemsReady just returns true since v1 plugins don't wait for items. func (r *AdaptedV1RestartableRestoreItemAction) AreAdditionalItemsReady(additionalItems []velero.ResourceIdentifier, restore *api.Restore) (bool, error) { return true, nil } ================================================ FILE: pkg/plugin/clientmgmt/restoreitemaction/v2/restartable_restore_item_action_test.go ================================================ /* Copyright 2018, 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v2 import ( "testing" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "github.com/vmware-tanzu/velero/internal/restartabletest" v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt/process" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" "github.com/vmware-tanzu/velero/pkg/plugin/velero" mocks "github.com/vmware-tanzu/velero/pkg/plugin/velero/mocks/restoreitemaction/v2" ) func TestRestartableGetRestoreItemAction(t *testing.T) { tests := []struct { name string plugin any getError error expectedError string }{ { name: "error getting by kind and name", getError: errors.Errorf("get error"), expectedError: "get error", }, { name: "wrong type", plugin: 3, expectedError: "plugin int is not a RestoreItemActionV2", }, { name: "happy path", plugin: new(mocks.RestoreItemAction), }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { p := new(restartabletest.MockRestartableProcess) defer p.AssertExpectations(t) name := "pod" key := process.KindAndName{Kind: common.PluginKindRestoreItemActionV2, Name: name} p.On("GetByKindAndName", key).Return(tc.plugin, tc.getError) r := NewRestartableRestoreItemAction(name, p) a, err := r.getRestoreItemAction() if tc.expectedError != "" { assert.EqualError(t, err, tc.expectedError) return } require.NoError(t, err) assert.Equal(t, tc.plugin, a) }) } } func TestRestartableRestoreItemActionGetDelegate(t *testing.T) { p := new(restartabletest.MockRestartableProcess) defer p.AssertExpectations(t) // Reset error p.On("ResetIfNeeded").Return(errors.Errorf("reset error")).Once() name := "pod" r := NewRestartableRestoreItemAction(name, p) a, err := r.getDelegate() assert.Nil(t, a) require.EqualError(t, err, "reset error") // Happy path p.On("ResetIfNeeded").Return(nil) expected := new(mocks.RestoreItemAction) key := process.KindAndName{Kind: common.PluginKindRestoreItemActionV2, Name: name} p.On("GetByKindAndName", key).Return(expected, nil) a, err = r.getDelegate() require.NoError(t, err) assert.Equal(t, expected, a) } func TestRestartableRestoreItemActionDelegatedFunctions(t *testing.T) { pv := &unstructured.Unstructured{ Object: map[string]any{ "color": "blue", }, } input := &velero.RestoreItemActionExecuteInput{ Item: pv, ItemFromBackup: pv, Restore: new(v1.Restore), } output := &velero.RestoreItemActionExecuteOutput{ UpdatedItem: &unstructured.Unstructured{ Object: map[string]any{ "color": "green", }, }, } r := new(v1.Restore) oid := "operation1" additionalItems := []velero.ResourceIdentifier{} restartabletest.RunRestartableDelegateTests( t, common.PluginKindRestoreItemActionV2, func(key process.KindAndName, p process.RestartableProcess) any { return &RestartableRestoreItemAction{ Key: key, SharedPluginProcess: p, } }, func() restartabletest.Mockable { return new(mocks.RestoreItemAction) }, restartabletest.RestartableDelegateTest{ Function: "AppliesTo", Inputs: []any{}, ExpectedErrorOutputs: []any{velero.ResourceSelector{}, errors.Errorf("reset error")}, ExpectedDelegateOutputs: []any{velero.ResourceSelector{IncludedNamespaces: []string{"a"}}, errors.Errorf("delegate error")}, }, restartabletest.RestartableDelegateTest{ Function: "Execute", Inputs: []any{input}, ExpectedErrorOutputs: []any{nil, errors.Errorf("reset error")}, ExpectedDelegateOutputs: []any{output, errors.Errorf("delegate error")}, }, restartabletest.RestartableDelegateTest{ Function: "Progress", Inputs: []any{oid, r}, ExpectedErrorOutputs: []any{velero.OperationProgress{}, errors.Errorf("reset error")}, ExpectedDelegateOutputs: []any{velero.OperationProgress{}, errors.Errorf("delegate error")}, }, restartabletest.RestartableDelegateTest{ Function: "Cancel", Inputs: []any{oid, r}, ExpectedErrorOutputs: []any{errors.Errorf("reset error")}, ExpectedDelegateOutputs: []any{errors.Errorf("delegate error")}, }, restartabletest.RestartableDelegateTest{ Function: "AreAdditionalItemsReady", Inputs: []any{additionalItems, r}, ExpectedErrorOutputs: []any{false, errors.Errorf("reset error")}, ExpectedDelegateOutputs: []any{true, errors.Errorf("delegate error")}, }, ) } ================================================ FILE: pkg/plugin/clientmgmt/volumesnapshotter/v1/restartable_volume_snapshotter.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v1 import ( "github.com/pkg/errors" "k8s.io/apimachinery/pkg/runtime" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt/process" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" vsv1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/volumesnapshotter/v1" ) // AdaptedVolumeSnapshotter is a volume snapshotter adapted to the v1 VolumeSnapshotter API type AdaptedVolumeSnapshotter struct { Kind common.PluginKind // Get returns a restartable VolumeSnapshotter for the given name and process, wrapping if necessary GetRestartable func(name string, restartableProcess process.RestartableProcess) vsv1.VolumeSnapshotter } func AdaptedVolumeSnapshotters() []AdaptedVolumeSnapshotter { return []AdaptedVolumeSnapshotter{ { Kind: common.PluginKindVolumeSnapshotter, GetRestartable: func(name string, restartableProcess process.RestartableProcess) vsv1.VolumeSnapshotter { return NewRestartableVolumeSnapshotter(name, restartableProcess) }, }, } } // RestartableVolumeSnapshotter is a volume snapshotter for a given implementation (such as "aws"). It is associated with // a restartableProcess, which may be shared and used to run multiple plugins. At the beginning of each method // call, the restartableVolumeSnapshotter asks its restartableProcess to restart itself if needed (e.g. if the // process terminated for any reason), then it proceeds with the actual call. type RestartableVolumeSnapshotter struct { Key process.KindAndName SharedPluginProcess process.RestartableProcess config map[string]string } // NewRestartableVolumeSnapshotter returns a new restartableVolumeSnapshotter. func NewRestartableVolumeSnapshotter(name string, sharedPluginProcess process.RestartableProcess) *RestartableVolumeSnapshotter { key := process.KindAndName{Kind: common.PluginKindVolumeSnapshotter, Name: name} r := &RestartableVolumeSnapshotter{ Key: key, SharedPluginProcess: sharedPluginProcess, } // Register our reinitializer so we can reinitialize after a restart with r.config. sharedPluginProcess.AddReinitializer(key, r) return r } // reinitialize reinitializes a re-dispensed plugin using the initial data passed to Init(). func (r *RestartableVolumeSnapshotter) Reinitialize(dispensed any) error { volumeSnapshotter, ok := dispensed.(vsv1.VolumeSnapshotter) if !ok { return errors.Errorf("plugin %T is not a VolumeSnapshotter", dispensed) } return r.init(volumeSnapshotter, r.config) } // getVolumeSnapshotter returns the volume snapshotter for this restartableVolumeSnapshotter. It does *not* restart the // plugin process. func (r *RestartableVolumeSnapshotter) getVolumeSnapshotter() (vsv1.VolumeSnapshotter, error) { plugin, err := r.SharedPluginProcess.GetByKindAndName(r.Key) if err != nil { return nil, err } volumeSnapshotter, ok := plugin.(vsv1.VolumeSnapshotter) if !ok { return nil, errors.Errorf("plugin %T is not a VolumeSnapshotter", plugin) } return volumeSnapshotter, nil } // getDelegate restarts the plugin process (if needed) and returns the volume snapshotter for this RestartableVolumeSnapshotter. func (r *RestartableVolumeSnapshotter) getDelegate() (vsv1.VolumeSnapshotter, error) { if err := r.SharedPluginProcess.ResetIfNeeded(); err != nil { return nil, err } return r.getVolumeSnapshotter() } // Init initializes the volume snapshotter instance using config. If this is the first invocation, r stores config for future // reinitialization needs. Init does NOT restart the shared plugin process. Init may only be called once. func (r *RestartableVolumeSnapshotter) Init(config map[string]string) error { if r.config != nil { return errors.Errorf("already initialized") } // Not using getDelegate() to avoid possible infinite recursion delegate, err := r.getVolumeSnapshotter() if err != nil { return err } r.config = config return r.init(delegate, config) } // init calls Init on volumeSnapshotter with config. This is split out from Init() so that both Init() and reinitialize() may // call it using a specific VolumeSnapshotter. func (r *RestartableVolumeSnapshotter) init(volumeSnapshotter vsv1.VolumeSnapshotter, config map[string]string) error { return volumeSnapshotter.Init(config) } // CreateVolumeFromSnapshot restarts the plugin's process if needed, then delegates the call. func (r *RestartableVolumeSnapshotter) CreateVolumeFromSnapshot(snapshotID string, volumeType string, volumeAZ string, iops *int64) (volumeID string, err error) { delegate, err := r.getDelegate() if err != nil { return "", err } return delegate.CreateVolumeFromSnapshot(snapshotID, volumeType, volumeAZ, iops) } // GetVolumeID restarts the plugin's process if needed, then delegates the call. func (r *RestartableVolumeSnapshotter) GetVolumeID(pv runtime.Unstructured) (string, error) { delegate, err := r.getDelegate() if err != nil { return "", err } return delegate.GetVolumeID(pv) } // SetVolumeID restarts the plugin's process if needed, then delegates the call. func (r *RestartableVolumeSnapshotter) SetVolumeID(pv runtime.Unstructured, volumeID string) (runtime.Unstructured, error) { delegate, err := r.getDelegate() if err != nil { return nil, err } return delegate.SetVolumeID(pv, volumeID) } // GetVolumeInfo restarts the plugin's process if needed, then delegates the call. func (r *RestartableVolumeSnapshotter) GetVolumeInfo(volumeID string, volumeAZ string) (string, *int64, error) { delegate, err := r.getDelegate() if err != nil { return "", nil, err } return delegate.GetVolumeInfo(volumeID, volumeAZ) } // CreateSnapshot restarts the plugin's process if needed, then delegates the call. func (r *RestartableVolumeSnapshotter) CreateSnapshot(volumeID string, volumeAZ string, tags map[string]string) (snapshotID string, err error) { delegate, err := r.getDelegate() if err != nil { return "", err } return delegate.CreateSnapshot(volumeID, volumeAZ, tags) } // DeleteSnapshot restarts the plugin's process if needed, then delegates the call. func (r *RestartableVolumeSnapshotter) DeleteSnapshot(snapshotID string) error { delegate, err := r.getDelegate() if err != nil { return err } return delegate.DeleteSnapshot(snapshotID) } ================================================ FILE: pkg/plugin/clientmgmt/volumesnapshotter/v1/restartable_volume_snapshotter_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v1 import ( "testing" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "github.com/vmware-tanzu/velero/internal/restartabletest" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt/process" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" providermocks "github.com/vmware-tanzu/velero/pkg/plugin/velero/mocks/volumesnapshotter/v1" ) func TestRestartableGetVolumeSnapshotter(t *testing.T) { tests := []struct { name string plugin any getError error expectedError string }{ { name: "error getting by kind and name", getError: errors.Errorf("get error"), expectedError: "get error", }, { name: "wrong type", plugin: 3, expectedError: "plugin int is not a VolumeSnapshotter", }, { name: "happy path", plugin: new(providermocks.VolumeSnapshotter), }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { p := new(restartabletest.MockRestartableProcess) p.Test(t) defer p.AssertExpectations(t) name := "aws" key := process.KindAndName{Kind: common.PluginKindVolumeSnapshotter, Name: name} p.On("GetByKindAndName", key).Return(tc.plugin, tc.getError) r := &RestartableVolumeSnapshotter{ Key: key, SharedPluginProcess: p, } a, err := r.getVolumeSnapshotter() if tc.expectedError != "" { assert.EqualError(t, err, tc.expectedError) return } require.NoError(t, err) assert.Equal(t, tc.plugin, a) }) } } func TestRestartableVolumeSnapshotterReinitialize(t *testing.T) { p := new(restartabletest.MockRestartableProcess) p.Test(t) defer p.AssertExpectations(t) name := "aws" key := process.KindAndName{Kind: common.PluginKindVolumeSnapshotter, Name: name} r := &RestartableVolumeSnapshotter{ Key: key, SharedPluginProcess: p, config: map[string]string{ "color": "blue", }, } err := r.Reinitialize(3) require.EqualError(t, err, "plugin int is not a VolumeSnapshotter") volumeSnapshotter := new(providermocks.VolumeSnapshotter) volumeSnapshotter.Test(t) defer volumeSnapshotter.AssertExpectations(t) volumeSnapshotter.On("Init", r.config).Return(errors.Errorf("init error")).Once() err = r.Reinitialize(volumeSnapshotter) require.EqualError(t, err, "init error") volumeSnapshotter.On("Init", r.config).Return(nil) err = r.Reinitialize(volumeSnapshotter) assert.NoError(t, err) } func TestRestartableVolumeSnapshotterGetDelegate(t *testing.T) { p := new(restartabletest.MockRestartableProcess) p.Test(t) defer p.AssertExpectations(t) // Reset error p.On("ResetIfNeeded").Return(errors.Errorf("reset error")).Once() name := "aws" key := process.KindAndName{Kind: common.PluginKindVolumeSnapshotter, Name: name} r := &RestartableVolumeSnapshotter{ Key: key, SharedPluginProcess: p, } a, err := r.getDelegate() assert.Nil(t, a) require.EqualError(t, err, "reset error") // Happy path p.On("ResetIfNeeded").Return(nil) volumeSnapshotter := new(providermocks.VolumeSnapshotter) volumeSnapshotter.Test(t) defer volumeSnapshotter.AssertExpectations(t) p.On("GetByKindAndName", key).Return(volumeSnapshotter, nil) a, err = r.getDelegate() require.NoError(t, err) assert.Equal(t, volumeSnapshotter, a) } func TestRestartableVolumeSnapshotterInit(t *testing.T) { p := new(restartabletest.MockRestartableProcess) p.Test(t) defer p.AssertExpectations(t) // getVolumeSnapshottererror name := "aws" key := process.KindAndName{Kind: common.PluginKindVolumeSnapshotter, Name: name} r := &RestartableVolumeSnapshotter{ Key: key, SharedPluginProcess: p, } p.On("GetByKindAndName", key).Return(nil, errors.Errorf("GetByKindAndName error")).Once() config := map[string]string{ "color": "blue", } err := r.Init(config) require.EqualError(t, err, "GetByKindAndName error") // Delegate returns error volumeSnapshotter := new(providermocks.VolumeSnapshotter) volumeSnapshotter.Test(t) defer volumeSnapshotter.AssertExpectations(t) p.On("GetByKindAndName", key).Return(volumeSnapshotter, nil) volumeSnapshotter.On("Init", config).Return(errors.Errorf("Init error")).Once() err = r.Init(config) require.EqualError(t, err, "Init error") // wipe this out because the previous failed Init call set it r.config = nil // Happy path volumeSnapshotter.On("Init", config).Return(nil) err = r.Init(config) require.NoError(t, err) assert.Equal(t, config, r.config) // Calling Init twice is forbidden err = r.Init(config) assert.EqualError(t, err, "already initialized") } func TestRestartableVolumeSnapshotterDelegatedFunctions(t *testing.T) { pv := &unstructured.Unstructured{ Object: map[string]any{ "color": "blue", }, } pvToReturn := &unstructured.Unstructured{ Object: map[string]any{ "color": "green", }, } restartabletest.RunRestartableDelegateTests( t, common.PluginKindVolumeSnapshotter, func(key process.KindAndName, p process.RestartableProcess) any { return &RestartableVolumeSnapshotter{ Key: key, SharedPluginProcess: p, } }, func() restartabletest.Mockable { return new(providermocks.VolumeSnapshotter) }, restartabletest.RestartableDelegateTest{ Function: "CreateVolumeFromSnapshot", Inputs: []any{"snapshotID", "volumeID", "volumeAZ", to.Ptr(int64(10000))}, ExpectedErrorOutputs: []any{"", errors.Errorf("reset error")}, ExpectedDelegateOutputs: []any{"volumeID", errors.Errorf("delegate error")}, }, restartabletest.RestartableDelegateTest{ Function: "GetVolumeID", Inputs: []any{pv}, ExpectedErrorOutputs: []any{"", errors.Errorf("reset error")}, ExpectedDelegateOutputs: []any{"volumeID", errors.Errorf("delegate error")}, }, restartabletest.RestartableDelegateTest{ Function: "SetVolumeID", Inputs: []any{pv, "volumeID"}, ExpectedErrorOutputs: []any{nil, errors.Errorf("reset error")}, ExpectedDelegateOutputs: []any{pvToReturn, errors.Errorf("delegate error")}, }, restartabletest.RestartableDelegateTest{ Function: "GetVolumeInfo", Inputs: []any{"volumeID", "volumeAZ"}, ExpectedErrorOutputs: []any{"", (*int64)(nil), errors.Errorf("reset error")}, ExpectedDelegateOutputs: []any{"volumeType", to.Ptr(int64(10000)), errors.Errorf("delegate error")}, }, restartabletest.RestartableDelegateTest{ Function: "CreateSnapshot", Inputs: []any{"volumeID", "volumeAZ", map[string]string{"a": "b"}}, ExpectedErrorOutputs: []any{"", errors.Errorf("reset error")}, ExpectedDelegateOutputs: []any{"snapshotID", errors.Errorf("delegate error")}, }, restartabletest.RestartableDelegateTest{ Function: "DeleteSnapshot", Inputs: []any{"snapshotID"}, ExpectedErrorOutputs: []any{errors.Errorf("reset error")}, ExpectedDelegateOutputs: []any{errors.Errorf("delegate error")}, }, ) } ================================================ FILE: pkg/plugin/framework/action_resolver.go ================================================ /* Copyright the Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package framework import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime/schema" "github.com/vmware-tanzu/velero/pkg/discovery" "github.com/vmware-tanzu/velero/pkg/plugin/velero" biav1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/backupitemaction/v1" biav2 "github.com/vmware-tanzu/velero/pkg/plugin/velero/backupitemaction/v2" ibav1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/itemblockaction/v1" riav1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/restoreitemaction/v1" riav2 "github.com/vmware-tanzu/velero/pkg/plugin/velero/restoreitemaction/v2" "github.com/vmware-tanzu/velero/pkg/util/collections" ) /* Velero has a variety of Actions that can be executed on Kubernetes resources. The Actions (BackupItemAction, RestoreItemAction and others) implement the Applicable interface which returns a ResourceSelector for the Action. The ResourceSelector can specify namespaces, resource names and labels to include or exclude. The ResourceSelector is resolved into lists of namespaces and resources present in the backup to be matched against. These lists and the label selector are then used to decide whether or not the ResolvedAction should be used for a particular resource. */ // ResolvedAction is an action that has had the namespaces, resources names and labels to include or exclude resolved type ResolvedAction interface { // ShouldUse returns true if the resolved namespaces, resource names and labels match those passed in the parameters. // metadata is optional and may be nil ShouldUse(groupResource schema.GroupResource, namespace string, metadata metav1.Object, log logrus.FieldLogger) bool } // resolvedAction is a core struct that holds the resolved namespaces, resource names and labels type resolvedAction struct { ResourceIncludesExcludes *collections.IncludesExcludes NamespaceIncludesExcludes *collections.IncludesExcludes Selector labels.Selector } func (recv resolvedAction) ShouldUse(groupResource schema.GroupResource, namespace string, metadata metav1.Object, log logrus.FieldLogger) bool { if !recv.ResourceIncludesExcludes.ShouldInclude(groupResource.String()) { log.Debug("Skipping action because it does not apply to this resource") return false } if namespace != "" && !recv.NamespaceIncludesExcludes.ShouldInclude(namespace) { log.Debug("Skipping action because it does not apply to this namespace") return false } if namespace == "" && !recv.NamespaceIncludesExcludes.IncludeEverything() { log.Debug("Skipping action because resource is cluster-scoped and action only applies to specific namespaces") return false } if metadata != nil && !recv.Selector.Matches(labels.Set(metadata.GetLabels())) { log.Debug("Skipping action because label selector does not match") return false } return true } // resolveAction resolves the resources, namespaces and selector into fully-qualified versions func resolveAction(helper discovery.Helper, action velero.Applicable) (resources *collections.IncludesExcludes, namespaces *collections.IncludesExcludes, selector labels.Selector, err error) { resourceSelector, err := action.AppliesTo() if err != nil { return nil, nil, nil, err } resources = collections.GetResourceIncludesExcludes(helper, resourceSelector.IncludedResources, resourceSelector.ExcludedResources) namespaces = collections.NewIncludesExcludes().Includes(resourceSelector.IncludedNamespaces...).Excludes(resourceSelector.ExcludedNamespaces...) selector = labels.Everything() if resourceSelector.LabelSelector != "" { if selector, err = labels.Parse(resourceSelector.LabelSelector); err != nil { return nil, nil, nil, err } } return } type BackupItemResolvedAction struct { biav1.BackupItemAction resolvedAction } func NewBackupItemActionResolver(actions []biav1.BackupItemAction) BackupItemActionResolver { return BackupItemActionResolver{ actions: actions, } } type BackupItemResolvedActionV2 struct { biav2.BackupItemAction resolvedAction } func NewBackupItemActionResolverV2(actions []biav2.BackupItemAction) BackupItemActionResolverV2 { return BackupItemActionResolverV2{ actions: actions, } } func NewRestoreItemActionResolver(actions []riav1.RestoreItemAction) RestoreItemActionResolver { return RestoreItemActionResolver{ actions: actions, } } func NewRestoreItemActionResolverV2(actions []riav2.RestoreItemAction) RestoreItemActionResolverV2 { return RestoreItemActionResolverV2{ actions: actions, } } func NewDeleteItemActionResolver(actions []velero.DeleteItemAction) DeleteItemActionResolver { return DeleteItemActionResolver{ actions: actions, } } type ActionResolver interface { ResolveAction(helper discovery.Helper, action velero.Applicable, log logrus.FieldLogger) (ResolvedAction, error) } type BackupItemActionResolver struct { actions []biav1.BackupItemAction } func (recv BackupItemActionResolver) ResolveActions(helper discovery.Helper, log logrus.FieldLogger) ([]BackupItemResolvedAction, error) { var resolved []BackupItemResolvedAction for _, action := range recv.actions { resources, namespaces, selector, err := resolveAction(helper, action) if err != nil { return nil, err } res := BackupItemResolvedAction{ BackupItemAction: action, resolvedAction: resolvedAction{ ResourceIncludesExcludes: resources, NamespaceIncludesExcludes: namespaces, Selector: selector, }, } resolved = append(resolved, res) } return resolved, nil } type BackupItemActionResolverV2 struct { actions []biav2.BackupItemAction } func (recv BackupItemActionResolverV2) ResolveActions(helper discovery.Helper, log logrus.FieldLogger) ([]BackupItemResolvedActionV2, error) { var resolved []BackupItemResolvedActionV2 for _, action := range recv.actions { log.Debugf("resolving BackupItemAction for: %v", action) resources, namespaces, selector, err := resolveAction(helper, action) if err != nil { log.WithError(errors.WithStack(err)).Debugf("resolveAction error, action: %v", action) return nil, err } res := BackupItemResolvedActionV2{ BackupItemAction: action, resolvedAction: resolvedAction{ ResourceIncludesExcludes: resources, NamespaceIncludesExcludes: namespaces, Selector: selector, }, } resolved = append(resolved, res) } return resolved, nil } type RestoreItemResolvedAction struct { riav1.RestoreItemAction resolvedAction } type RestoreItemResolvedActionV2 struct { riav2.RestoreItemAction resolvedAction } type RestoreItemActionResolver struct { actions []riav1.RestoreItemAction } func (recv RestoreItemActionResolver) ResolveActions(helper discovery.Helper, log logrus.FieldLogger) ([]RestoreItemResolvedAction, error) { var resolved []RestoreItemResolvedAction for _, action := range recv.actions { resources, namespaces, selector, err := resolveAction(helper, action) if err != nil { return nil, err } res := RestoreItemResolvedAction{ RestoreItemAction: action, resolvedAction: resolvedAction{ ResourceIncludesExcludes: resources, NamespaceIncludesExcludes: namespaces, Selector: selector, }, } resolved = append(resolved, res) } return resolved, nil } type RestoreItemActionResolverV2 struct { actions []riav2.RestoreItemAction } func (recv RestoreItemActionResolverV2) ResolveActions(helper discovery.Helper, log logrus.FieldLogger) ([]RestoreItemResolvedActionV2, error) { var resolved []RestoreItemResolvedActionV2 for _, action := range recv.actions { resources, namespaces, selector, err := resolveAction(helper, action) if err != nil { return nil, err } res := RestoreItemResolvedActionV2{ RestoreItemAction: action, resolvedAction: resolvedAction{ ResourceIncludesExcludes: resources, NamespaceIncludesExcludes: namespaces, Selector: selector, }, } resolved = append(resolved, res) } return resolved, nil } type DeleteItemResolvedAction struct { velero.DeleteItemAction resolvedAction } type DeleteItemActionResolver struct { actions []velero.DeleteItemAction } func (recv DeleteItemActionResolver) ResolveActions(helper discovery.Helper, log logrus.FieldLogger) ([]DeleteItemResolvedAction, error) { var resolved []DeleteItemResolvedAction for _, action := range recv.actions { resources, namespaces, selector, err := resolveAction(helper, action) if err != nil { return nil, err } res := DeleteItemResolvedAction{ DeleteItemAction: action, resolvedAction: resolvedAction{ ResourceIncludesExcludes: resources, NamespaceIncludesExcludes: namespaces, Selector: selector, }, } resolved = append(resolved, res) } return resolved, nil } type ItemBlockResolvedAction struct { ibav1.ItemBlockAction resolvedAction } type ItemBlockActionResolver struct { actions []ibav1.ItemBlockAction } func NewItemBlockActionResolver(actions []ibav1.ItemBlockAction) ItemBlockActionResolver { return ItemBlockActionResolver{ actions: actions, } } func (recv ItemBlockActionResolver) ResolveActions(helper discovery.Helper, log logrus.FieldLogger) ([]ItemBlockResolvedAction, error) { var resolved []ItemBlockResolvedAction for _, action := range recv.actions { resources, namespaces, selector, err := resolveAction(helper, action) if err != nil { return nil, err } res := ItemBlockResolvedAction{ ItemBlockAction: action, resolvedAction: resolvedAction{ ResourceIncludesExcludes: resources, NamespaceIncludesExcludes: namespaces, Selector: selector, }, } resolved = append(resolved, res) } return resolved, nil } ================================================ FILE: pkg/plugin/framework/action_resolver_test.go ================================================ /* Copyright the Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package framework import ( "testing" "k8s.io/apimachinery/pkg/labels" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/runtime/schema" "github.com/vmware-tanzu/velero/pkg/plugin/velero" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) type mockApplicable struct { selector velero.ResourceSelector } func (recv mockApplicable) AppliesTo() (velero.ResourceSelector, error) { return recv.selector, nil } func TestActionResolverNamespace(t *testing.T) { discoveryHelper := velerotest.NewFakeDiscoveryHelper(false, map[schema.GroupVersionResource]schema.GroupVersionResource{}) namespaceMatchApplicable := mockApplicable{ selector: velero.ResourceSelector{ IncludedNamespaces: []string{"default"}, }, } resources, namespaces, selector, err := resolveAction(discoveryHelper, namespaceMatchApplicable) require.NoError(t, err) require.Equal(t, []string{"default"}, namespaces.GetIncludes()) require.Empty(t, namespaces.GetExcludes()) require.Empty(t, resources.GetIncludes()) require.Empty(t, resources.GetExcludes()) require.True(t, selector.Empty()) } func TestActionResolverResource(t *testing.T) { pvGVR := schema.GroupVersionResource{ Group: "", Version: "v1", Resource: "persistentvolumes", } discoveryHelper := velerotest.NewFakeDiscoveryHelper(false, map[schema.GroupVersionResource]schema.GroupVersionResource{pvGVR: pvGVR}) namespaceMatchApplicable := mockApplicable{ selector: velero.ResourceSelector{ IncludedResources: []string{"persistentvolumes"}, }, } resources, namespaces, selector, err := resolveAction(discoveryHelper, namespaceMatchApplicable) require.NoError(t, err) require.Empty(t, namespaces.GetIncludes()) require.Empty(t, namespaces.GetExcludes()) require.True(t, resources.ShouldInclude("persistentvolumes")) require.Empty(t, resources.GetExcludes()) require.True(t, selector.Empty()) } func TestActionResolverLabel(t *testing.T) { discoveryHelper := velerotest.NewFakeDiscoveryHelper(false, map[schema.GroupVersionResource]schema.GroupVersionResource{}) namespaceMatchApplicable := mockApplicable{ selector: velero.ResourceSelector{ LabelSelector: "myLabel=true", }, } checkLabel, err := labels.ConvertSelectorToLabelsMap("myLabel=true") require.NoError(t, err) resources, namespaces, selector, err := resolveAction(discoveryHelper, namespaceMatchApplicable) require.NoError(t, err) require.Empty(t, namespaces.GetIncludes()) require.Empty(t, namespaces.GetExcludes()) require.Empty(t, resources.GetIncludes()) require.Empty(t, resources.GetExcludes()) require.True(t, selector.Matches(checkLabel)) } ================================================ FILE: pkg/plugin/framework/backup_item_action.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package framework import ( "context" plugin "github.com/hashicorp/go-plugin" "google.golang.org/grpc" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" protobiav1 "github.com/vmware-tanzu/velero/pkg/plugin/generated" ) // BackupItemActionPlugin is an implementation of go-plugin's Plugin // interface with support for gRPC for the backup/ItemAction // interface. type BackupItemActionPlugin struct { plugin.NetRPCUnsupportedPlugin *common.PluginBase } // GRPCClient returns a clientDispenser for BackupItemAction gRPC clients. func (p *BackupItemActionPlugin) GRPCClient(_ context.Context, _ *plugin.GRPCBroker, clientConn *grpc.ClientConn) (any, error) { return common.NewClientDispenser(p.ClientLogger, clientConn, newBackupItemActionGRPCClient), nil } // GRPCServer registers a BackupItemAction gRPC server. func (p *BackupItemActionPlugin) GRPCServer(_ *plugin.GRPCBroker, server *grpc.Server) error { protobiav1.RegisterBackupItemActionServer(server, &BackupItemActionGRPCServer{mux: p.ServerMux}) return nil } ================================================ FILE: pkg/plugin/framework/backup_item_action_client.go ================================================ /* Copyright 2017, 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package framework import ( "encoding/json" "context" "github.com/pkg/errors" "google.golang.org/grpc" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" protobiav1 "github.com/vmware-tanzu/velero/pkg/plugin/generated" "github.com/vmware-tanzu/velero/pkg/plugin/velero" ) // NewBackupItemActionPlugin constructs a BackupItemActionPlugin. func NewBackupItemActionPlugin(options ...common.PluginOption) *BackupItemActionPlugin { return &BackupItemActionPlugin{ PluginBase: common.NewPluginBase(options...), } } // BackupItemActionGRPCClient implements the backup/ItemAction interface and uses a // gRPC client to make calls to the plugin server. type BackupItemActionGRPCClient struct { *common.ClientBase grpcClient protobiav1.BackupItemActionClient } func newBackupItemActionGRPCClient(base *common.ClientBase, clientConn *grpc.ClientConn) any { return &BackupItemActionGRPCClient{ ClientBase: base, grpcClient: protobiav1.NewBackupItemActionClient(clientConn), } } func (c *BackupItemActionGRPCClient) AppliesTo() (velero.ResourceSelector, error) { req := &protobiav1.BackupItemActionAppliesToRequest{ Plugin: c.Plugin, } res, err := c.grpcClient.AppliesTo(context.Background(), req) if err != nil { return velero.ResourceSelector{}, common.FromGRPCError(err) } if res.ResourceSelector == nil { return velero.ResourceSelector{}, nil } return velero.ResourceSelector{ IncludedNamespaces: res.ResourceSelector.IncludedNamespaces, ExcludedNamespaces: res.ResourceSelector.ExcludedNamespaces, IncludedResources: res.ResourceSelector.IncludedResources, ExcludedResources: res.ResourceSelector.ExcludedResources, LabelSelector: res.ResourceSelector.Selector, }, nil } func (c *BackupItemActionGRPCClient) Execute(item runtime.Unstructured, backup *api.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, error) { itemJSON, err := json.Marshal(item.UnstructuredContent()) if err != nil { return nil, nil, errors.WithStack(err) } backupJSON, err := json.Marshal(backup) if err != nil { return nil, nil, errors.WithStack(err) } req := &protobiav1.ExecuteRequest{ Plugin: c.Plugin, Item: itemJSON, Backup: backupJSON, } res, err := c.grpcClient.Execute(context.Background(), req) if err != nil { return nil, nil, common.FromGRPCError(err) } var updatedItem unstructured.Unstructured if err := json.Unmarshal(res.Item, &updatedItem); err != nil { return nil, nil, errors.WithStack(err) } var additionalItems []velero.ResourceIdentifier for _, itm := range res.AdditionalItems { newItem := velero.ResourceIdentifier{ GroupResource: schema.GroupResource{ Group: itm.Group, Resource: itm.Resource, }, Namespace: itm.Namespace, Name: itm.Name, } additionalItems = append(additionalItems, newItem) } return &updatedItem, additionalItems, nil } ================================================ FILE: pkg/plugin/framework/backup_item_action_server.go ================================================ /* Copyright 2017, 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package framework import ( "encoding/json" "context" "github.com/pkg/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" proto "github.com/vmware-tanzu/velero/pkg/plugin/generated" "github.com/vmware-tanzu/velero/pkg/plugin/velero" biav1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/backupitemaction/v1" ) // BackupItemActionGRPCServer implements the proto-generated BackupItemAction interface, and accepts // gRPC calls and forwards them to an implementation of the pluggable interface. type BackupItemActionGRPCServer struct { mux *common.ServerMux } func (s *BackupItemActionGRPCServer) getImpl(name string) (biav1.BackupItemAction, error) { impl, err := s.mux.GetHandler(name) if err != nil { return nil, err } itemAction, ok := impl.(biav1.BackupItemAction) if !ok { return nil, errors.Errorf("%T is not a backup item action", impl) } return itemAction, nil } func (s *BackupItemActionGRPCServer) AppliesTo( ctx context.Context, req *proto.BackupItemActionAppliesToRequest) ( response *proto.BackupItemActionAppliesToResponse, err error) { defer func() { if recoveredErr := common.HandlePanic(recover()); recoveredErr != nil { err = recoveredErr } }() impl, err := s.getImpl(req.Plugin) if err != nil { return nil, common.NewGRPCError(err) } resourceSelector, err := impl.AppliesTo() if err != nil { return nil, common.NewGRPCError(err) } return &proto.BackupItemActionAppliesToResponse{ ResourceSelector: &proto.ResourceSelector{ IncludedNamespaces: resourceSelector.IncludedNamespaces, ExcludedNamespaces: resourceSelector.ExcludedNamespaces, IncludedResources: resourceSelector.IncludedResources, ExcludedResources: resourceSelector.ExcludedResources, Selector: resourceSelector.LabelSelector, }, }, nil } func (s *BackupItemActionGRPCServer) Execute( ctx context.Context, req *proto.ExecuteRequest) (response *proto.ExecuteResponse, err error) { defer func() { if recoveredErr := common.HandlePanic(recover()); recoveredErr != nil { err = recoveredErr } }() impl, err := s.getImpl(req.Plugin) if err != nil { return nil, common.NewGRPCError(err) } var item unstructured.Unstructured var backup api.Backup if err := json.Unmarshal(req.Item, &item); err != nil { return nil, common.NewGRPCError(errors.WithStack(err)) } if err := json.Unmarshal(req.Backup, &backup); err != nil { return nil, common.NewGRPCError(errors.WithStack(err)) } updatedItem, additionalItems, err := impl.Execute(&item, &backup) if err != nil { return nil, common.NewGRPCError(err) } // If the plugin implementation returned a nil updatedItem (meaning no modifications), reset updatedItem to the // original item. var updatedItemJSON []byte if updatedItem == nil { updatedItemJSON = req.Item } else { updatedItemJSON, err = json.Marshal(updatedItem.UnstructuredContent()) if err != nil { return nil, common.NewGRPCError(errors.WithStack(err)) } } res := &proto.ExecuteResponse{ Item: updatedItemJSON, } for _, item := range additionalItems { res.AdditionalItems = append(res.AdditionalItems, backupResourceIdentifierToProto(item)) } return res, nil } func backupResourceIdentifierToProto(id velero.ResourceIdentifier) *proto.ResourceIdentifier { return &proto.ResourceIdentifier{ Group: id.Group, Resource: id.Resource, Namespace: id.Namespace, Name: id.Name, } } ================================================ FILE: pkg/plugin/framework/backup_item_action_test.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package framework import ( "encoding/json" "testing" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" proto "github.com/vmware-tanzu/velero/pkg/plugin/generated" "github.com/vmware-tanzu/velero/pkg/plugin/velero" mocks "github.com/vmware-tanzu/velero/pkg/plugin/velero/mocks/backupitemaction/v1" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func TestBackupItemActionGRPCServerExecute(t *testing.T) { invalidItem := []byte("this is gibberish json") validItem := []byte(` { "apiVersion": "v1", "kind": "ConfigMap", "metadata": { "namespace": "myns", "name": "myconfigmap" }, "data": { "key": "value" } }`) var validItemObject unstructured.Unstructured err := json.Unmarshal(validItem, &validItemObject) require.NoError(t, err) updatedItem := []byte(` { "apiVersion": "v1", "kind": "ConfigMap", "metadata": { "namespace": "myns", "name": "myconfigmap" }, "data": { "key": "changed!" } }`) var updatedItemObject unstructured.Unstructured err = json.Unmarshal(updatedItem, &updatedItemObject) require.NoError(t, err) invalidBackup := []byte("this is gibberish json") validBackup := []byte(` { "apiVersion": "velero.io/v1", "kind": "Backup", "metadata": { "namespace": "myns", "name": "mybackup" }, "spec": { "includedNamespaces": ["*"], "includedResources": ["*"], "ttl": "60m" } }`) var validBackupObject v1.Backup err = json.Unmarshal(validBackup, &validBackupObject) require.NoError(t, err) tests := []struct { name string backup []byte item []byte implUpdatedItem runtime.Unstructured implAdditionalItems []velero.ResourceIdentifier implError error expectError bool skipMock bool }{ { name: "error unmarshaling item", item: invalidItem, backup: validBackup, expectError: true, skipMock: true, }, { name: "error unmarshaling backup", item: validItem, backup: invalidBackup, expectError: true, skipMock: true, }, { name: "error running impl", item: validItem, backup: validBackup, implError: errors.New("impl error"), expectError: true, }, { name: "nil updatedItem / no additionalItems", item: validItem, backup: validBackup, }, { name: "same updatedItem / some additionalItems", item: validItem, backup: validBackup, implUpdatedItem: &validItemObject, implAdditionalItems: []velero.ResourceIdentifier{ { GroupResource: schema.GroupResource{Group: "v1", Resource: "pods"}, Namespace: "myns", Name: "mypod", }, }, }, { name: "different updatedItem", item: validItem, backup: validBackup, implUpdatedItem: &updatedItemObject, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { itemAction := &mocks.BackupItemAction{} defer itemAction.AssertExpectations(t) if !test.skipMock { itemAction.On("Execute", &validItemObject, &validBackupObject).Return(test.implUpdatedItem, test.implAdditionalItems, test.implError) } s := &BackupItemActionGRPCServer{mux: &common.ServerMux{ ServerLog: velerotest.NewLogger(), Handlers: map[string]any{ "xyz": itemAction, }, }} req := &proto.ExecuteRequest{ Plugin: "xyz", Item: test.item, Backup: test.backup, } resp, err := s.Execute(t.Context(), req) // Verify error assert.Equal(t, test.expectError, err != nil) if err != nil { return } require.NotNil(t, resp) // Verify updated item updatedItem := test.implUpdatedItem if updatedItem == nil { // If the impl returned nil for its updatedItem, we should expect the plugin to return the original item updatedItem = &validItemObject } var respItem unstructured.Unstructured err = json.Unmarshal(resp.Item, &respItem) require.NoError(t, err) assert.Equal(t, updatedItem, &respItem) // Verify additional items var expectedAdditionalItems []*proto.ResourceIdentifier for _, item := range test.implAdditionalItems { expectedAdditionalItems = append(expectedAdditionalItems, backupResourceIdentifierToProto(item)) } assert.Equal(t, expectedAdditionalItems, resp.AdditionalItems) }) } } ================================================ FILE: pkg/plugin/framework/backupitemaction/v2/backup_item_action.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v2 import ( "context" plugin "github.com/hashicorp/go-plugin" "google.golang.org/grpc" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" protobiav2 "github.com/vmware-tanzu/velero/pkg/plugin/generated/backupitemaction/v2" ) // BackupItemActionPlugin is an implementation of go-plugin's Plugin // interface with support for gRPC for the backup/ItemAction // interface. type BackupItemActionPlugin struct { plugin.NetRPCUnsupportedPlugin *common.PluginBase } // GRPCClient returns a clientDispenser for BackupItemAction gRPC clients. func (p *BackupItemActionPlugin) GRPCClient(_ context.Context, _ *plugin.GRPCBroker, clientConn *grpc.ClientConn) (any, error) { return common.NewClientDispenser(p.ClientLogger, clientConn, newBackupItemActionGRPCClient), nil } // GRPCServer registers a BackupItemAction gRPC server. func (p *BackupItemActionPlugin) GRPCServer(_ *plugin.GRPCBroker, server *grpc.Server) error { protobiav2.RegisterBackupItemActionServer(server, &BackupItemActionGRPCServer{mux: p.ServerMux}) return nil } ================================================ FILE: pkg/plugin/framework/backupitemaction/v2/backup_item_action_client.go ================================================ /* Copyright 2017, 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v2 import ( "encoding/json" "context" "github.com/pkg/errors" "google.golang.org/grpc" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" protobiav2 "github.com/vmware-tanzu/velero/pkg/plugin/generated/backupitemaction/v2" "github.com/vmware-tanzu/velero/pkg/plugin/velero" ) // NewBackupItemActionPlugin constructs a BackupItemActionPlugin. func NewBackupItemActionPlugin(options ...common.PluginOption) *BackupItemActionPlugin { return &BackupItemActionPlugin{ PluginBase: common.NewPluginBase(options...), } } // BackupItemActionGRPCClient implements the backup/ItemAction interface and uses a // gRPC client to make calls to the plugin server. type BackupItemActionGRPCClient struct { *common.ClientBase grpcClient protobiav2.BackupItemActionClient } func newBackupItemActionGRPCClient(base *common.ClientBase, clientConn *grpc.ClientConn) any { return &BackupItemActionGRPCClient{ ClientBase: base, grpcClient: protobiav2.NewBackupItemActionClient(clientConn), } } func (c *BackupItemActionGRPCClient) AppliesTo() (velero.ResourceSelector, error) { req := &protobiav2.BackupItemActionAppliesToRequest{ Plugin: c.Plugin, } res, err := c.grpcClient.AppliesTo(context.Background(), req) if err != nil { return velero.ResourceSelector{}, common.FromGRPCError(err) } if res.ResourceSelector == nil { return velero.ResourceSelector{}, nil } return velero.ResourceSelector{ IncludedNamespaces: res.ResourceSelector.IncludedNamespaces, ExcludedNamespaces: res.ResourceSelector.ExcludedNamespaces, IncludedResources: res.ResourceSelector.IncludedResources, ExcludedResources: res.ResourceSelector.ExcludedResources, LabelSelector: res.ResourceSelector.Selector, }, nil } func (c *BackupItemActionGRPCClient) Execute(item runtime.Unstructured, backup *api.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { itemJSON, err := json.Marshal(item.UnstructuredContent()) if err != nil { return nil, nil, "", nil, errors.WithStack(err) } backupJSON, err := json.Marshal(backup) if err != nil { return nil, nil, "", nil, errors.WithStack(err) } req := &protobiav2.ExecuteRequest{ Plugin: c.Plugin, Item: itemJSON, Backup: backupJSON, } res, err := c.grpcClient.Execute(context.Background(), req) if err != nil { return nil, nil, "", nil, common.FromGRPCError(err) } var updatedItem unstructured.Unstructured if err := json.Unmarshal(res.Item, &updatedItem); err != nil { return nil, nil, "", nil, errors.WithStack(err) } var additionalItems []velero.ResourceIdentifier for _, itm := range res.AdditionalItems { newItem := velero.ResourceIdentifier{ GroupResource: schema.GroupResource{ Group: itm.Group, Resource: itm.Resource, }, Namespace: itm.Namespace, Name: itm.Name, } additionalItems = append(additionalItems, newItem) } var postOperationItems []velero.ResourceIdentifier for _, itm := range res.PostOperationItems { newItem := velero.ResourceIdentifier{ GroupResource: schema.GroupResource{ Group: itm.Group, Resource: itm.Resource, }, Namespace: itm.Namespace, Name: itm.Name, } postOperationItems = append(postOperationItems, newItem) } return &updatedItem, additionalItems, res.OperationID, postOperationItems, nil } func (c *BackupItemActionGRPCClient) Progress(operationID string, backup *api.Backup) (velero.OperationProgress, error) { backupJSON, err := json.Marshal(backup) if err != nil { return velero.OperationProgress{}, errors.WithStack(err) } req := &protobiav2.BackupItemActionProgressRequest{ Plugin: c.Plugin, OperationID: operationID, Backup: backupJSON, } res, err := c.grpcClient.Progress(context.Background(), req) if err != nil { return velero.OperationProgress{}, common.FromGRPCError(err) } return velero.OperationProgress{ Completed: res.Progress.Completed, Err: res.Progress.Err, NCompleted: res.Progress.NCompleted, NTotal: res.Progress.NTotal, OperationUnits: res.Progress.OperationUnits, Description: res.Progress.Description, Started: res.Progress.Started.AsTime(), Updated: res.Progress.Updated.AsTime(), }, nil } func (c *BackupItemActionGRPCClient) Cancel(operationID string, backup *api.Backup) error { backupJSON, err := json.Marshal(backup) if err != nil { return errors.WithStack(err) } req := &protobiav2.BackupItemActionCancelRequest{ Plugin: c.Plugin, OperationID: operationID, Backup: backupJSON, } _, err = c.grpcClient.Cancel(context.Background(), req) if err != nil { return common.FromGRPCError(err) } return nil } // This shouldn't be called on the GRPC client since the RestartableBackupItemAction won't delegate // this method func (c *BackupItemActionGRPCClient) Name() string { return "" } ================================================ FILE: pkg/plugin/framework/backupitemaction/v2/backup_item_action_server.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v2 import ( "encoding/json" "context" "github.com/pkg/errors" "google.golang.org/protobuf/types/known/emptypb" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" timestamppb "google.golang.org/protobuf/types/known/timestamppb" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" proto "github.com/vmware-tanzu/velero/pkg/plugin/generated" protobiav2 "github.com/vmware-tanzu/velero/pkg/plugin/generated/backupitemaction/v2" "github.com/vmware-tanzu/velero/pkg/plugin/velero" biav2 "github.com/vmware-tanzu/velero/pkg/plugin/velero/backupitemaction/v2" ) // BackupItemActionGRPCServer implements the proto-generated BackupItemAction interface, and accepts // gRPC calls and forwards them to an implementation of the pluggable interface. type BackupItemActionGRPCServer struct { mux *common.ServerMux } func (s *BackupItemActionGRPCServer) getImpl(name string) (biav2.BackupItemAction, error) { impl, err := s.mux.GetHandler(name) if err != nil { return nil, err } itemAction, ok := impl.(biav2.BackupItemAction) if !ok { return nil, errors.Errorf("%T is not a backup item action", impl) } return itemAction, nil } func (s *BackupItemActionGRPCServer) AppliesTo( ctx context.Context, req *protobiav2.BackupItemActionAppliesToRequest) ( response *protobiav2.BackupItemActionAppliesToResponse, err error) { defer func() { if recoveredErr := common.HandlePanic(recover()); recoveredErr != nil { err = recoveredErr } }() impl, err := s.getImpl(req.Plugin) if err != nil { return nil, common.NewGRPCError(err) } resourceSelector, err := impl.AppliesTo() if err != nil { return nil, common.NewGRPCError(err) } return &protobiav2.BackupItemActionAppliesToResponse{ ResourceSelector: &proto.ResourceSelector{ IncludedNamespaces: resourceSelector.IncludedNamespaces, ExcludedNamespaces: resourceSelector.ExcludedNamespaces, IncludedResources: resourceSelector.IncludedResources, ExcludedResources: resourceSelector.ExcludedResources, Selector: resourceSelector.LabelSelector, }, }, nil } func (s *BackupItemActionGRPCServer) Execute( ctx context.Context, req *protobiav2.ExecuteRequest) (response *protobiav2.ExecuteResponse, err error) { defer func() { if recoveredErr := common.HandlePanic(recover()); recoveredErr != nil { err = recoveredErr } }() impl, err := s.getImpl(req.Plugin) if err != nil { return nil, common.NewGRPCError(err) } var item unstructured.Unstructured var backup api.Backup if err := json.Unmarshal(req.Item, &item); err != nil { return nil, common.NewGRPCError(errors.WithStack(err)) } if err := json.Unmarshal(req.Backup, &backup); err != nil { return nil, common.NewGRPCError(errors.WithStack(err)) } updatedItem, additionalItems, operationID, postOperationItems, err := impl.Execute(&item, &backup) if err != nil { return nil, common.NewGRPCError(err) } // If the plugin implementation returned a nil updatedItem (meaning no modifications), reset updatedItem to the // original item. var updatedItemJSON []byte if updatedItem == nil { updatedItemJSON = req.Item } else { updatedItemJSON, err = json.Marshal(updatedItem.UnstructuredContent()) if err != nil { return nil, common.NewGRPCError(errors.WithStack(err)) } } res := &protobiav2.ExecuteResponse{ Item: updatedItemJSON, OperationID: operationID, } for _, item := range additionalItems { res.AdditionalItems = append(res.AdditionalItems, backupResourceIdentifierToProto(item)) } for _, item := range postOperationItems { res.PostOperationItems = append(res.PostOperationItems, backupResourceIdentifierToProto(item)) } return res, nil } func (s *BackupItemActionGRPCServer) Progress( ctx context.Context, req *protobiav2.BackupItemActionProgressRequest) ( response *protobiav2.BackupItemActionProgressResponse, err error) { defer func() { if recoveredErr := common.HandlePanic(recover()); recoveredErr != nil { err = recoveredErr } }() impl, err := s.getImpl(req.Plugin) if err != nil { return nil, common.NewGRPCError(err) } var backup api.Backup if err := json.Unmarshal(req.Backup, &backup); err != nil { return nil, common.NewGRPCError(errors.WithStack(err)) } progress, err := impl.Progress(req.OperationID, &backup) if err != nil { return nil, common.NewGRPCError(err) } res := &protobiav2.BackupItemActionProgressResponse{ Progress: &proto.OperationProgress{ Completed: progress.Completed, Err: progress.Err, NCompleted: progress.NCompleted, NTotal: progress.NTotal, OperationUnits: progress.OperationUnits, Description: progress.Description, Started: timestamppb.New(progress.Started), Updated: timestamppb.New(progress.Updated), }, } return res, nil } func (s *BackupItemActionGRPCServer) Cancel( ctx context.Context, req *protobiav2.BackupItemActionCancelRequest) ( response *emptypb.Empty, err error) { defer func() { if recoveredErr := common.HandlePanic(recover()); recoveredErr != nil { err = recoveredErr } }() impl, err := s.getImpl(req.Plugin) if err != nil { return nil, common.NewGRPCError(err) } var backup api.Backup if err := json.Unmarshal(req.Backup, &backup); err != nil { return nil, common.NewGRPCError(errors.WithStack(err)) } err = impl.Cancel(req.OperationID, &backup) if err != nil { return nil, common.NewGRPCError(err) } return &emptypb.Empty{}, nil } func backupResourceIdentifierToProto(id velero.ResourceIdentifier) *proto.ResourceIdentifier { return &proto.ResourceIdentifier{ Group: id.Group, Resource: id.Resource, Namespace: id.Namespace, Name: id.Name, } } // This shouldn't be called on the GRPC server since the server won't ever receive this request, as // the RestartableBackupItemAction in Velero won't delegate this to the server func (s *BackupItemActionGRPCServer) Name() string { return "" } ================================================ FILE: pkg/plugin/framework/backupitemaction/v2/backup_item_action_test.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v2 import ( "encoding/json" "testing" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" proto "github.com/vmware-tanzu/velero/pkg/plugin/generated" protobiav2 "github.com/vmware-tanzu/velero/pkg/plugin/generated/backupitemaction/v2" "github.com/vmware-tanzu/velero/pkg/plugin/velero" mocks "github.com/vmware-tanzu/velero/pkg/plugin/velero/mocks/backupitemaction/v2" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func TestBackupItemActionGRPCServerExecute(t *testing.T) { invalidItem := []byte("this is gibberish json") validItem := []byte(` { "apiVersion": "v1", "kind": "ConfigMap", "metadata": { "namespace": "myns", "name": "myconfigmap" }, "data": { "key": "value" } }`) var validItemObject unstructured.Unstructured err := json.Unmarshal(validItem, &validItemObject) require.NoError(t, err) updatedItem := []byte(` { "apiVersion": "v1", "kind": "ConfigMap", "metadata": { "namespace": "myns", "name": "myconfigmap" }, "data": { "key": "changed!" } }`) var updatedItemObject unstructured.Unstructured err = json.Unmarshal(updatedItem, &updatedItemObject) require.NoError(t, err) invalidBackup := []byte("this is gibberish json") validBackup := []byte(` { "apiVersion": "velero.io/v1", "kind": "Backup", "metadata": { "namespace": "myns", "name": "mybackup" }, "spec": { "includedNamespaces": ["*"], "includedResources": ["*"], "ttl": "60m" } }`) var validBackupObject v1.Backup err = json.Unmarshal(validBackup, &validBackupObject) require.NoError(t, err) tests := []struct { name string backup []byte item []byte implUpdatedItem runtime.Unstructured implAdditionalItems []velero.ResourceIdentifier implOperationID string implPostOperationItems []velero.ResourceIdentifier implError error expectError bool skipMock bool }{ { name: "error unmarshaling item", item: invalidItem, backup: validBackup, expectError: true, skipMock: true, }, { name: "error unmarshaling backup", item: validItem, backup: invalidBackup, expectError: true, skipMock: true, }, { name: "error running impl", item: validItem, backup: validBackup, implError: errors.New("impl error"), expectError: true, }, { name: "nil updatedItem / no additionalItems", item: validItem, backup: validBackup, }, { name: "same updatedItem / some additionalItems", item: validItem, backup: validBackup, implUpdatedItem: &validItemObject, implAdditionalItems: []velero.ResourceIdentifier{ { GroupResource: schema.GroupResource{Group: "v1", Resource: "pods"}, Namespace: "myns", Name: "mypod", }, }, }, { name: "different updatedItem", item: validItem, backup: validBackup, implUpdatedItem: &updatedItemObject, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { itemAction := &mocks.BackupItemAction{} defer itemAction.AssertExpectations(t) if !test.skipMock { itemAction.On("Execute", &validItemObject, &validBackupObject).Return(test.implUpdatedItem, test.implAdditionalItems, test.implOperationID, test.implPostOperationItems, test.implError) } s := &BackupItemActionGRPCServer{mux: &common.ServerMux{ ServerLog: velerotest.NewLogger(), Handlers: map[string]any{ "xyz": itemAction, }, }} req := &protobiav2.ExecuteRequest{ Plugin: "xyz", Item: test.item, Backup: test.backup, } resp, err := s.Execute(t.Context(), req) // Verify error assert.Equal(t, test.expectError, err != nil) if err != nil { return } require.NotNil(t, resp) // Verify updated item updatedItem := test.implUpdatedItem if updatedItem == nil { // If the impl returned nil for its updatedItem, we should expect the plugin to return the original item updatedItem = &validItemObject } var respItem unstructured.Unstructured err = json.Unmarshal(resp.Item, &respItem) require.NoError(t, err) assert.Equal(t, updatedItem, &respItem) // Verify additional items var expectedAdditionalItems []*proto.ResourceIdentifier for _, item := range test.implAdditionalItems { expectedAdditionalItems = append(expectedAdditionalItems, backupResourceIdentifierToProto(item)) } assert.Equal(t, expectedAdditionalItems, resp.AdditionalItems) }) } } ================================================ FILE: pkg/plugin/framework/common/client_dispenser.go ================================================ /* Copyright 2018, 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package common import ( "github.com/sirupsen/logrus" "google.golang.org/grpc" ) // ClientBase implements client and contains shared fields common to all clients. type ClientBase struct { Plugin string Logger logrus.FieldLogger } type ClientDispenser interface { ClientFor(name string) any } // clientDispenser supports the initialization and retrieval of multiple implementations for a single plugin kind, such as // "aws" and "azure" implementations of the object store plugin. type clientDispenser struct { // logger is the log the plugin should use. logger logrus.FieldLogger // clienConn is shared among all implementations for this client. clientConn *grpc.ClientConn // initFunc returns a client that implements a plugin interface, such as ObjectStore. initFunc clientInitFunc // clients keeps track of all the initialized implementations. clients map[string]any } type clientInitFunc func(base *ClientBase, clientConn *grpc.ClientConn) any // newClientDispenser creates a new clientDispenser. func NewClientDispenser(logger logrus.FieldLogger, clientConn *grpc.ClientConn, initFunc clientInitFunc) *clientDispenser { return &clientDispenser{ clientConn: clientConn, logger: logger, initFunc: initFunc, clients: make(map[string]any), } } // ClientFor returns a gRPC client stub for the implementation of a plugin named name. If the client stub does not // currently exist, clientFor creates it. func (cd *clientDispenser) ClientFor(name string) any { if client, found := cd.clients[name]; found { return client } base := &ClientBase{ Plugin: name, Logger: cd.logger, } // Initialize the plugin (e.g. newBackupItemActionGRPCClient()) client := cd.initFunc(base, cd.clientConn) cd.clients[name] = client return client } ================================================ FILE: pkg/plugin/framework/common/client_dispenser_test.go ================================================ /* Copyright 2018, 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package common import ( "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/grpc" "github.com/vmware-tanzu/velero/pkg/test" ) type fakeClient struct { base *ClientBase clientConn *grpc.ClientConn } func TestNewClientDispenser(t *testing.T) { logger := test.NewLogger() clientConn := new(grpc.ClientConn) c := 3 initFunc := func(base *ClientBase, clientConn *grpc.ClientConn) any { return c } cd := NewClientDispenser(logger, clientConn, initFunc) assert.Equal(t, clientConn, cd.clientConn) assert.NotNil(t, cd.clients) assert.Empty(t, cd.clients) } func TestClientFor(t *testing.T) { logger := test.NewLogger() clientConn := new(grpc.ClientConn) c := new(fakeClient) count := 0 initFunc := func(base *ClientBase, clientConn *grpc.ClientConn) any { c.base = base c.clientConn = clientConn count++ return c } cd := NewClientDispenser(logger, clientConn, initFunc) actual := cd.ClientFor("pod") require.IsType(t, &fakeClient{}, actual) typed := actual.(*fakeClient) assert.Equal(t, 1, count) assert.Equal(t, &typed, &c) expectedBase := &ClientBase{ Plugin: "pod", Logger: logger, } assert.Equal(t, expectedBase, typed.base) assert.Equal(t, clientConn, typed.clientConn) // Make sure we reuse a previous client actual = cd.ClientFor("pod") require.IsType(t, &fakeClient{}, actual) typed = actual.(*fakeClient) assert.Equal(t, 1, count) } ================================================ FILE: pkg/plugin/framework/common/client_errors.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package common import ( "google.golang.org/grpc/status" proto "github.com/vmware-tanzu/velero/pkg/plugin/generated" ) // FromGRPCError takes a gRPC status error, extracts a stack trace // from the details if it exists, and returns an error that can // provide information about where it was created. // // This function should be used in the internal plugin client code to convert // all errors returned from the plugin server before they're passed back to // the rest of the Velero codebase. This will enable them to display location // information when they're logged. func FromGRPCError(err error) error { statusErr, ok := status.FromError(err) if !ok { return statusErr.Err() } for _, detail := range statusErr.Details() { if t, ok := detail.(*proto.Stack); ok { return &ProtoStackError{ error: err, stack: t, } } } return err } type ProtoStackError struct { error stack *proto.Stack } func (e *ProtoStackError) File() string { if e.stack == nil || len(e.stack.Frames) < 1 { return "" } return e.stack.Frames[0].File } func (e *ProtoStackError) Line() int32 { if e.stack == nil || len(e.stack.Frames) < 1 { return 0 } return e.stack.Frames[0].Line } func (e *ProtoStackError) Function() string { if e.stack == nil || len(e.stack.Frames) < 1 { return "" } return e.stack.Frames[0].Function } ================================================ FILE: pkg/plugin/framework/common/handle_panic.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package common import ( "runtime/debug" "github.com/pkg/errors" "google.golang.org/grpc/codes" ) // HandlePanic is a panic handler for the server half of velero plugins. func HandlePanic(p any) error { if p == nil { return nil } // If p is an error with a stack trace, we want to retain // it to preserve the stack trace. Otherwise, create a new // error here. var err error if panicErr, ok := p.(error); !ok { err = errors.Errorf("plugin panicked: %v", p) } else { if _, ok := panicErr.(StackTracer); ok { err = panicErr } else { errWithStacktrace := errors.Errorf("%v, stack trace: %s", panicErr, debug.Stack()) err = errors.Wrap(errWithStacktrace, "plugin panicked") } } return NewGRPCErrorWithCode(err, codes.Aborted) } ================================================ FILE: pkg/plugin/framework/common/plugin_base.go ================================================ /* Copyright 2018, 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package common import ( "github.com/sirupsen/logrus" ) type PluginBase struct { ClientLogger logrus.FieldLogger *ServerMux } func NewPluginBase(options ...PluginOption) *PluginBase { base := new(PluginBase) for _, option := range options { option(base) } return base } type PluginOption func(base *PluginBase) func ClientLogger(logger logrus.FieldLogger) PluginOption { return func(base *PluginBase) { base.ClientLogger = logger } } func ServerLogger(logger logrus.FieldLogger) PluginOption { return func(base *PluginBase) { base.ServerMux = NewServerMux(logger) } } ================================================ FILE: pkg/plugin/framework/common/plugin_base_test.go ================================================ /* Copyright 2018, 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package common import ( "testing" "github.com/stretchr/testify/assert" "github.com/vmware-tanzu/velero/pkg/test" ) func TestClientLogger(t *testing.T) { base := &PluginBase{} logger := test.NewLogger() f := ClientLogger(logger) f(base) assert.Equal(t, logger, base.ClientLogger) } func TestServerLogger(t *testing.T) { base := &PluginBase{} logger := test.NewLogger() f := ServerLogger(logger) f(base) assert.Equal(t, NewServerMux(logger), base.ServerMux) } ================================================ FILE: pkg/plugin/framework/common/plugin_config.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package common import ( "context" "fmt" "github.com/pkg/errors" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" ) func PluginConfigLabelSelector(kind PluginKind, name string) string { return fmt.Sprintf("velero.io/plugin-config,%s=%s", name, kind) } func GetPluginConfig(kind PluginKind, name string, client corev1client.ConfigMapInterface) (*corev1api.ConfigMap, error) { opts := metav1.ListOptions{ // velero.io/plugin-config: true // velero.io/pod-volume-restore: RestoreItemAction LabelSelector: PluginConfigLabelSelector(kind, name), } list, err := client.List(context.Background(), opts) if err != nil { return nil, errors.WithStack(err) } if len(list.Items) == 0 { return nil, nil } if len(list.Items) > 1 { var items []string for _, item := range list.Items { items = append(items, item.Name) } return nil, errors.Errorf("found more than one ConfigMap matching label selector %q: %v", opts.LabelSelector, items) } return &list.Items[0], nil } ================================================ FILE: pkg/plugin/framework/common/plugin_config_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package common import ( "reflect" "testing" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes/fake" velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) func TestGetPluginConfig(t *testing.T) { type args struct { kind PluginKind name string objects []runtime.Object } pluginLabelsMap := map[string]string{"velero.io/plugin-config": "", "foo": "RestoreItemAction"} testConfigMap := &corev1api.ConfigMap{ TypeMeta: metav1.TypeMeta{ Kind: "ConfigMap", }, ObjectMeta: metav1.ObjectMeta{ Name: "foo-config", Namespace: velerov1.DefaultNamespace, Labels: pluginLabelsMap, }, } tests := []struct { name string args args want *corev1api.ConfigMap wantErr bool }{ { name: "should return nil if no config map found", args: args{ kind: PluginKindRestoreItemAction, name: "foo", objects: []runtime.Object{}, }, want: nil, wantErr: false, }, { name: "should return error if more than one config map found", args: args{ kind: PluginKindRestoreItemAction, name: "foo", objects: []runtime.Object{ &corev1api.ConfigMap{ TypeMeta: metav1.TypeMeta{ Kind: "ConfigMap", }, ObjectMeta: metav1.ObjectMeta{ Name: "foo-config", Namespace: velerov1.DefaultNamespace, Labels: pluginLabelsMap, }, }, &corev1api.ConfigMap{ TypeMeta: metav1.TypeMeta{ Kind: "ConfigMap", }, ObjectMeta: metav1.ObjectMeta{ Name: "foo-config-duplicate", Namespace: velerov1.DefaultNamespace, Labels: pluginLabelsMap, }, }, }, }, want: nil, wantErr: true, }, { name: "should return pointer to configmap if only one config map with label found", args: args{ kind: PluginKindRestoreItemAction, name: "foo", objects: []runtime.Object{ testConfigMap, }, }, want: testConfigMap, wantErr: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { fakeClient := fake.NewSimpleClientset(tt.args.objects...) got, err := GetPluginConfig(tt.args.kind, tt.args.name, fakeClient.CoreV1().ConfigMaps(velerov1.DefaultNamespace)) if (err != nil) != tt.wantErr { t.Errorf("GetPluginConfig() error = %v, wantErr %v", err, tt.wantErr) return } if !reflect.DeepEqual(got, tt.want) { t.Errorf("GetPluginConfig() = %v, want %v", got, tt.want) } }) } } ================================================ FILE: pkg/plugin/framework/common/plugin_kinds.go ================================================ /* Copyright 2018, 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package common // PluginKind is a type alias for a string that describes // the kind of a Velero-supported plugin. type PluginKind string // String returns the string for k. func (k PluginKind) String() string { return string(k) } const ( // PluginKindObjectStore represents an object store plugin. PluginKindObjectStore PluginKind = "ObjectStore" // PluginKindVolumeSnapshotter represents a volume snapshotter plugin. PluginKindVolumeSnapshotter PluginKind = "VolumeSnapshotter" // PluginKindBackupItemAction represents a backup item action plugin. PluginKindBackupItemAction PluginKind = "BackupItemAction" // PluginKindBackupItemActionV2 represents a v2 backup item action plugin. PluginKindBackupItemActionV2 PluginKind = "BackupItemActionV2" // PluginKindRestoreItemAction represents a restore item action plugin. PluginKindRestoreItemAction PluginKind = "RestoreItemAction" // PluginKindRestoreItemActionV2 represents a v2 restore item action plugin. PluginKindRestoreItemActionV2 PluginKind = "RestoreItemActionV2" // PluginKindDeleteItemAction represents a delete item action plugin. PluginKindDeleteItemAction PluginKind = "DeleteItemAction" // PluginKindItemBlockAction represents a v1 ItemBlock action plugin. PluginKindItemBlockAction PluginKind = "ItemBlockAction" // PluginKindPluginLister represents a plugin lister plugin. PluginKindPluginLister PluginKind = "PluginLister" ) // PluginKindsAdaptableTo if there are plugin kinds that are adaptable to newer API versions, list them here. // The older (adaptable) version is the key, and the value is the full list of newer // plugin kinds that are capable of adapting it. var PluginKindsAdaptableTo = map[PluginKind][]PluginKind{ PluginKindBackupItemAction: {PluginKindBackupItemActionV2}, PluginKindRestoreItemAction: {PluginKindRestoreItemActionV2}, } // AllPluginKinds contains all the valid plugin kinds that Velero supports, excluding PluginLister because that is not a // kind that a developer would ever need to implement (it's handled by Velero and the Velero plugin library code). func AllPluginKinds() map[string]PluginKind { allPluginKinds := make(map[string]PluginKind) allPluginKinds[PluginKindObjectStore.String()] = PluginKindObjectStore allPluginKinds[PluginKindVolumeSnapshotter.String()] = PluginKindVolumeSnapshotter allPluginKinds[PluginKindBackupItemAction.String()] = PluginKindBackupItemAction allPluginKinds[PluginKindBackupItemActionV2.String()] = PluginKindBackupItemActionV2 allPluginKinds[PluginKindRestoreItemAction.String()] = PluginKindRestoreItemAction allPluginKinds[PluginKindRestoreItemActionV2.String()] = PluginKindRestoreItemActionV2 allPluginKinds[PluginKindDeleteItemAction.String()] = PluginKindDeleteItemAction allPluginKinds[PluginKindItemBlockAction.String()] = PluginKindItemBlockAction return allPluginKinds } ================================================ FILE: pkg/plugin/framework/common/server_errors.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package common import ( "github.com/pkg/errors" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/protoadapt" proto "github.com/vmware-tanzu/velero/pkg/plugin/generated" "github.com/vmware-tanzu/velero/pkg/util/logging" ) // NewGRPCErrorWithCode wraps err in a gRPC status error with the error's stack trace // included in the details if it exists. This provides an easy way to send // stack traces from plugin servers across the wire to the plugin client. // // This function should be used in the internal plugin server code to wrap // all errors before they're returned. func NewGRPCErrorWithCode(err error, code codes.Code, details ...protoadapt.MessageV1) error { // if it's already a gRPC status error, use it; otherwise, create a new one statusErr, ok := status.FromError(err) if !ok { statusErr = status.New(code, err.Error()) } // get a Stack for the error and add it to details if stack := ErrorStack(err); stack != nil { details = append(details, stack) } statusErr, err = statusErr.WithDetails(details...) if err != nil { return status.Errorf(codes.Unknown, "error adding details to the gRPC error: %v", err) } return statusErr.Err() } // NewGRPCError is a convenience function for creating a new gRPC error // with code = codes.Unknown func NewGRPCError(err error, details ...protoadapt.MessageV1) error { return NewGRPCErrorWithCode(err, codes.Unknown, details...) } // ErrorStack gets a stack trace, if it exists, from the provided error, and // returns it as a *proto.Stack. func ErrorStack(err error) *proto.Stack { stackTracer, ok := err.(StackTracer) if !ok { return nil } stackTrace := new(proto.Stack) for _, frame := range stackTracer.StackTrace() { location := logging.GetFrameLocationInfo(frame) stackTrace.Frames = append(stackTrace.Frames, &proto.StackFrame{ File: location.File, Line: int32(location.Line), Function: location.Function, }) } return stackTrace } type StackTracer interface { StackTrace() errors.StackTrace } ================================================ FILE: pkg/plugin/framework/common/server_mux.go ================================================ /* Copyright 2018, 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package common import ( "strings" "github.com/pkg/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation" ) // HandlerInitializer is a function that initializes and returns a new instance of one of Velero's plugin interfaces // (ObjectStore, VolumeSnapshotter, BackupItemAction, RestoreItemAction). type HandlerInitializer func(logger logrus.FieldLogger) (any, error) // ServerMux manages multiple implementations of a single plugin kind, such as pod and pvc BackupItemActions. type ServerMux struct { kind PluginKind initializers map[string]HandlerInitializer Handlers map[string]any ServerLog logrus.FieldLogger } // NewServerMux returns a new ServerMux. func NewServerMux(logger logrus.FieldLogger) *ServerMux { return &ServerMux{ initializers: make(map[string]HandlerInitializer), Handlers: make(map[string]any), ServerLog: logger, } } // register validates the plugin name and registers the // initializer for the given name. func (m *ServerMux) Register(name string, f HandlerInitializer) { if err := ValidatePluginName(name, m.Names()); err != nil { m.ServerLog.Errorf("invalid plugin name %q: %s", name, err) return } m.initializers[name] = f } // names returns a list of all registered implementations. func (m *ServerMux) Names() []string { return sets.StringKeySet(m.initializers).List() } // GetHandler returns the instance for a plugin with the given name. If an instance has already been initialized, // that is returned. Otherwise, the instance is initialized by calling its initialization function. func (m *ServerMux) GetHandler(name string) (any, error) { if instance, found := m.Handlers[name]; found { return instance, nil } initializer, found := m.initializers[name] if !found { return nil, errors.Errorf("%v plugin: %s was not found or has an invalid name format", m.kind, name) } instance, err := initializer(m.ServerLog) if err != nil { return nil, err } m.Handlers[name] = instance return m.Handlers[name], nil } // ValidatePluginName checks if the given name: // - the plugin name has two parts separated by '/' // - non of the above parts is empty // - the prefix is a valid DNS subdomain name // - a plugin with the same name does not already exist (if list of existing names is passed in) func ValidatePluginName(name string, existingNames []string) error { // validate there is one "/" and two parts parts := strings.Split(name, "/") if len(parts) != 2 { return errors.Errorf("plugin name must have exactly two parts separated by a `/`. Accepted format: /. %s is invalid", name) } // validate both prefix and name are non-empty if parts[0] == "" || parts[1] == "" { return errors.Errorf("both parts of the plugin name must be non-empty. Accepted format: /. %s is invalid", name) } // validate that the prefix is a DNS subdomain if errs := validation.IsDNS1123Subdomain(parts[0]); len(errs) != 0 { return errors.Errorf("first part of the plugin name must be a valid DNS subdomain. Accepted format: /. first part %q is invalid: %s", parts[0], strings.Join(errs, "; ")) } for _, existingName := range existingNames { if strings.Compare(name, existingName) == 0 { return errors.New("plugin name " + existingName + " already exists") } } return nil } ================================================ FILE: pkg/plugin/framework/common/server_mux_test.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package common import ( "strings" "testing" ) func TestValidPluginName(t *testing.T) { successCases := []struct { pluginName string existingNames []string }{ {"example.io/azure", []string{"velero.io/aws"}}, {"with-dashes/name", []string{"velero.io/aws"}}, {"prefix/Uppercase_Is_OK_123", []string{"velero.io/aws"}}, {"example-with-dash.io/azure", []string{"velero.io/aws"}}, {"1.2.3.4/5678", []string{"velero.io/aws"}}, {"example.io/azure", []string{"velero.io/aws"}}, {"example.io/azure", []string{""}}, {"example.io/azure", nil}, {strings.Repeat("a", 253) + "/name", []string{"velero.io/aws"}}, } for i, tt := range successCases { t.Run(tt.pluginName, func(t *testing.T) { if err := ValidatePluginName(tt.pluginName, tt.existingNames); err != nil { t.Errorf("case[%d]: %q: expected success: %v", i, successCases[i], err) } }) } errorCases := []struct { pluginName string existingNames []string }{ {"", []string{"velero.io/aws"}}, {"single", []string{"velero.io/aws"}}, {"/", []string{"velero.io/aws"}}, {"//", []string{"velero.io/aws"}}, {"///", []string{"velero.io/aws"}}, {"a/", []string{"velero.io/aws"}}, {"/a", []string{"velero.io/aws"}}, {"velero.io/aws", []string{"velero.io/aws"}}, {"Uppercase_Is_OK_123/name", []string{"velero.io/aws"}}, {strings.Repeat("a", 254) + "/name", []string{"velero.io/aws"}}, {"ospecialchars%^=@", []string{"velero.io/aws"}}, } for i, tt := range errorCases { t.Run(tt.pluginName, func(t *testing.T) { if err := ValidatePluginName(tt.pluginName, tt.existingNames); err == nil { t.Errorf("case[%d]: %q: expected failure.", i, errorCases[i]) } }) } } ================================================ FILE: pkg/plugin/framework/delete_item_action.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package framework import ( "context" plugin "github.com/hashicorp/go-plugin" "google.golang.org/grpc" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" proto "github.com/vmware-tanzu/velero/pkg/plugin/generated" ) // DeleteItemActionPlugin is an implementation of go-plugin's Plugin // interface with support for gRPC for the restore/ItemAction // interface. type DeleteItemActionPlugin struct { plugin.NetRPCUnsupportedPlugin *common.PluginBase } // GRPCClient returns a DeleteItemAction gRPC client. func (p *DeleteItemActionPlugin) GRPCClient(_ context.Context, _ *plugin.GRPCBroker, clientConn *grpc.ClientConn) (any, error) { return common.NewClientDispenser(p.ClientLogger, clientConn, newDeleteItemActionGRPCClient), nil } // GRPCServer registers a DeleteItemAction gRPC server. func (p *DeleteItemActionPlugin) GRPCServer(_ *plugin.GRPCBroker, server *grpc.Server) error { proto.RegisterDeleteItemActionServer(server, &DeleteItemActionGRPCServer{mux: p.ServerMux}) return nil } ================================================ FILE: pkg/plugin/framework/delete_item_action_client.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package framework import ( "encoding/json" "context" "github.com/pkg/errors" "google.golang.org/grpc" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" proto "github.com/vmware-tanzu/velero/pkg/plugin/generated" "github.com/vmware-tanzu/velero/pkg/plugin/velero" ) var _ velero.DeleteItemAction = &DeleteItemActionGRPCClient{} // NewDeleteItemActionPlugin constructs a DeleteItemActionPlugin. func NewDeleteItemActionPlugin(options ...common.PluginOption) *DeleteItemActionPlugin { return &DeleteItemActionPlugin{ PluginBase: common.NewPluginBase(options...), } } // DeleteItemActionGRPCClient implements the DeleteItemAction interface and uses a // gRPC client to make calls to the plugin server. type DeleteItemActionGRPCClient struct { *common.ClientBase grpcClient proto.DeleteItemActionClient } func newDeleteItemActionGRPCClient(base *common.ClientBase, clientConn *grpc.ClientConn) any { return &DeleteItemActionGRPCClient{ ClientBase: base, grpcClient: proto.NewDeleteItemActionClient(clientConn), } } func (c *DeleteItemActionGRPCClient) AppliesTo() (velero.ResourceSelector, error) { res, err := c.grpcClient.AppliesTo(context.Background(), &proto.DeleteItemActionAppliesToRequest{Plugin: c.Plugin}) if err != nil { return velero.ResourceSelector{}, common.FromGRPCError(err) } if res.ResourceSelector == nil { return velero.ResourceSelector{}, nil } return velero.ResourceSelector{ IncludedNamespaces: res.ResourceSelector.IncludedNamespaces, ExcludedNamespaces: res.ResourceSelector.ExcludedNamespaces, IncludedResources: res.ResourceSelector.IncludedResources, ExcludedResources: res.ResourceSelector.ExcludedResources, LabelSelector: res.ResourceSelector.Selector, }, nil } func (c *DeleteItemActionGRPCClient) Execute(input *velero.DeleteItemActionExecuteInput) error { itemJSON, err := json.Marshal(input.Item.UnstructuredContent()) if err != nil { return errors.WithStack(err) } backupJSON, err := json.Marshal(input.Backup) if err != nil { return errors.WithStack(err) } req := &proto.DeleteItemActionExecuteRequest{ Plugin: c.Plugin, Item: itemJSON, Backup: backupJSON, } // First return item is just an empty struct no matter what. if _, err = c.grpcClient.Execute(context.Background(), req); err != nil { return common.FromGRPCError(err) } return nil } ================================================ FILE: pkg/plugin/framework/delete_item_action_server.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package framework import ( "encoding/json" "context" "github.com/pkg/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" proto "github.com/vmware-tanzu/velero/pkg/plugin/generated" "github.com/vmware-tanzu/velero/pkg/plugin/velero" ) // DeleteItemActionGRPCServer implements the proto-generated DeleteItemActionServer interface, and accepts // gRPC calls and forwards them to an implementation of the pluggable interface. type DeleteItemActionGRPCServer struct { mux *common.ServerMux } func (s *DeleteItemActionGRPCServer) getImpl(name string) (velero.DeleteItemAction, error) { impl, err := s.mux.GetHandler(name) if err != nil { return nil, err } itemAction, ok := impl.(velero.DeleteItemAction) if !ok { return nil, errors.Errorf("%T is not a delete item action", impl) } return itemAction, nil } func (s *DeleteItemActionGRPCServer) AppliesTo(ctx context.Context, req *proto.DeleteItemActionAppliesToRequest) (response *proto.DeleteItemActionAppliesToResponse, err error) { defer func() { if recoveredErr := common.HandlePanic(recover()); recoveredErr != nil { err = recoveredErr } }() impl, err := s.getImpl(req.Plugin) if err != nil { return nil, common.NewGRPCError(err) } resourceSelector, err := impl.AppliesTo() if err != nil { return nil, common.NewGRPCError(err) } return &proto.DeleteItemActionAppliesToResponse{ ResourceSelector: &proto.ResourceSelector{ IncludedNamespaces: resourceSelector.IncludedNamespaces, ExcludedNamespaces: resourceSelector.ExcludedNamespaces, IncludedResources: resourceSelector.IncludedResources, ExcludedResources: resourceSelector.ExcludedResources, Selector: resourceSelector.LabelSelector, }, }, nil } func (s *DeleteItemActionGRPCServer) Execute(ctx context.Context, req *proto.DeleteItemActionExecuteRequest) (_ *proto.Empty, err error) { defer func() { if recoveredErr := common.HandlePanic(recover()); recoveredErr != nil { err = recoveredErr } }() impl, err := s.getImpl(req.Plugin) if err != nil { return nil, common.NewGRPCError(err) } var ( item unstructured.Unstructured backup api.Backup ) if err := json.Unmarshal(req.Item, &item); err != nil { return nil, common.NewGRPCError(errors.WithStack(err)) } if err = json.Unmarshal(req.Backup, &backup); err != nil { return nil, common.NewGRPCError(errors.WithStack(err)) } if err := impl.Execute(&velero.DeleteItemActionExecuteInput{ Item: &item, Backup: &backup, }); err != nil { return nil, common.NewGRPCError(err) } return &proto.Empty{}, nil } ================================================ FILE: pkg/plugin/framework/doc.go ================================================ // Package framework is the common package that any plugin client // will need to import, for example, both plugin authors and Velero core. package framework ================================================ FILE: pkg/plugin/framework/examples_test.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package framework import ( "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/runtime" "github.com/vmware-tanzu/velero/pkg/plugin/velero" ) func ExampleNewServer_volumeSnapshotter() { NewServer(). // call the server RegisterVolumeSnapshotter("example.io/volumesnapshotter", newVolumeSnapshotter). // register the plugin with a valid name RegisterDeleteItemAction("example.io/delete-item-action", newDeleteItemAction). Serve() // serve the plugin } func newVolumeSnapshotter(logger logrus.FieldLogger) (any, error) { return &VolumeSnapshotter{FieldLogger: logger}, nil } type VolumeSnapshotter struct { FieldLogger logrus.FieldLogger } // Implement all methods for the VolumeSnapshotter interface... func (b *VolumeSnapshotter) Init(config map[string]string) error { b.FieldLogger.Infof("VolumeSnapshotter.Init called") // ... return nil } func (b *VolumeSnapshotter) CreateVolumeFromSnapshot(snapshotID, volumeType, volumeAZ string, iops *int64) (volumeID string, err error) { b.FieldLogger.Infof("CreateVolumeFromSnapshot called") // ... return "volumeID", nil } func (b *VolumeSnapshotter) GetVolumeID(pv runtime.Unstructured) (string, error) { b.FieldLogger.Infof("GetVolumeID called") // ... return "volumeID", nil } func (b *VolumeSnapshotter) SetVolumeID(pv runtime.Unstructured, volumeID string) (runtime.Unstructured, error) { b.FieldLogger.Infof("SetVolumeID called") // ... return nil, nil } func (b *VolumeSnapshotter) GetVolumeInfo(volumeID, volumeAZ string) (string, *int64, error) { b.FieldLogger.Infof("GetVolumeInfo called") // ... return "volumeFilesystemType", nil, nil } func (b *VolumeSnapshotter) CreateSnapshot(volumeID, volumeAZ string, tags map[string]string) (snapshotID string, err error) { b.FieldLogger.Infof("CreateSnapshot called") // ... return "snapshotID", nil } func (b *VolumeSnapshotter) DeleteSnapshot(snapshotID string) error { b.FieldLogger.Infof("DeleteSnapshot called") // ... return nil } // Implement all methods for the DeleteItemAction interface func newDeleteItemAction(logger logrus.FieldLogger) (any, error) { return DeleteItemAction{FieldLogger: logger}, nil } type DeleteItemAction struct { FieldLogger logrus.FieldLogger } func (d *DeleteItemAction) AppliesTo() (velero.ResourceSelector, error) { d.FieldLogger.Infof("AppliesTo called") // ... return velero.ResourceSelector{}, nil } func (d *DeleteItemAction) Execute(input *velero.DeleteItemActionExecuteInput) error { d.FieldLogger.Infof("Execute called") // ... return nil } ================================================ FILE: pkg/plugin/framework/handshake.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package framework import plugin "github.com/hashicorp/go-plugin" // Handshake returns the configuration information that allows go-plugin clients and servers to perform a handshake. func Handshake() plugin.HandshakeConfig { return plugin.HandshakeConfig{ // The ProtocolVersion is the version that must match between Velero framework // and Velero client plugins. This should be bumped whenever a change happens in // one or the other that makes it so that they can't safely communicate. ProtocolVersion: 2, MagicCookieKey: "VELERO_PLUGIN", MagicCookieValue: "hello", } } ================================================ FILE: pkg/plugin/framework/import_test.go ================================================ package framework import ( "os/exec" "path/filepath" "regexp" "runtime" "strings" "testing" "github.com/stretchr/testify/require" ) // test that this package do not import cloud provider // Prevent https://github.com/vmware-tanzu/velero/issues/8207 and https://github.com/vmware-tanzu/velero/issues/8157 func TestPkgImportNoCloudProvider(t *testing.T) { _, filename, _, ok := runtime.Caller(0) if !ok { t.Fatalf("No caller information") } t.Logf("Current test file path: %s", filename) t.Logf("Current test directory: %s", filepath.Dir(filename)) // should be this package name // go list -f {{.Deps}} ./ cmd := exec.CommandContext( t.Context(), "go", "list", "-f", "{{.Deps}}", ".", ) // set cmd.Dir to this package even if executed from different dir cmd.Dir = filepath.Dir(filename) output, err := cmd.Output() require.NoError(t, err) // split dep by line, replace space with newline deps := strings.ReplaceAll(string(output), " ", "\n") require.NotEmpty(t, deps) // ignore k8s.io k8sio, err := regexp.Compile("^k8s.io") require.NoError(t, err) cloudProvider, err := regexp.Compile("aws|cloud.google.com|azure") require.NoError(t, err) cloudProviderDeps := []string{} for _, dep := range strings.Split(deps, "\n") { if !k8sio.MatchString(dep) { if cloudProvider.MatchString(dep) { cloudProviderDeps = append(cloudProviderDeps, dep) } } } require.Empty(t, cloudProviderDeps) } ================================================ FILE: pkg/plugin/framework/interface.go ================================================ /* Copyright 2017, 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package framework import plugin "github.com/hashicorp/go-plugin" // Interface represents a Velero plugin. type Interface interface { plugin.Plugin // names returns a list of all the registered implementations for this plugin (such as "pod" and "pvc" for // BackupItemAction). Names() []string } ================================================ FILE: pkg/plugin/framework/itemblockaction/v1/item_block_action.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v1 import ( "context" plugin "github.com/hashicorp/go-plugin" "google.golang.org/grpc" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" protoibav1 "github.com/vmware-tanzu/velero/pkg/plugin/generated/itemblockaction/v1" ) // ItemBlockActionPlugin is an implementation of go-plugin's Plugin // interface with support for gRPC for the backup/ItemAction // interface. type ItemBlockActionPlugin struct { plugin.NetRPCUnsupportedPlugin *common.PluginBase } // GRPCClient returns a clientDispenser for ItemBlockAction gRPC clients. func (p *ItemBlockActionPlugin) GRPCClient(_ context.Context, _ *plugin.GRPCBroker, clientConn *grpc.ClientConn) (any, error) { return common.NewClientDispenser(p.ClientLogger, clientConn, newItemBlockActionGRPCClient), nil } // GRPCServer registers a ItemBlockAction gRPC server. func (p *ItemBlockActionPlugin) GRPCServer(_ *plugin.GRPCBroker, server *grpc.Server) error { protoibav1.RegisterItemBlockActionServer(server, &ItemBlockActionGRPCServer{mux: p.ServerMux}) return nil } ================================================ FILE: pkg/plugin/framework/itemblockaction/v1/item_block_action_client.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v1 import ( "encoding/json" "context" "github.com/pkg/errors" "google.golang.org/grpc" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" protoibav1 "github.com/vmware-tanzu/velero/pkg/plugin/generated/itemblockaction/v1" "github.com/vmware-tanzu/velero/pkg/plugin/velero" ) // NewItemBlockActionPlugin constructs a ItemBlockActionPlugin. func NewItemBlockActionPlugin(options ...common.PluginOption) *ItemBlockActionPlugin { return &ItemBlockActionPlugin{ PluginBase: common.NewPluginBase(options...), } } // ItemBlockActionGRPCClient implements the backup/ItemAction interface and uses a // gRPC client to make calls to the plugin server. type ItemBlockActionGRPCClient struct { *common.ClientBase grpcClient protoibav1.ItemBlockActionClient } func newItemBlockActionGRPCClient(base *common.ClientBase, clientConn *grpc.ClientConn) any { return &ItemBlockActionGRPCClient{ ClientBase: base, grpcClient: protoibav1.NewItemBlockActionClient(clientConn), } } func (c *ItemBlockActionGRPCClient) AppliesTo() (velero.ResourceSelector, error) { req := &protoibav1.ItemBlockActionAppliesToRequest{ Plugin: c.Plugin, } res, err := c.grpcClient.AppliesTo(context.Background(), req) if err != nil { return velero.ResourceSelector{}, common.FromGRPCError(err) } if res.ResourceSelector == nil { return velero.ResourceSelector{}, nil } return velero.ResourceSelector{ IncludedNamespaces: res.ResourceSelector.IncludedNamespaces, ExcludedNamespaces: res.ResourceSelector.ExcludedNamespaces, IncludedResources: res.ResourceSelector.IncludedResources, ExcludedResources: res.ResourceSelector.ExcludedResources, LabelSelector: res.ResourceSelector.Selector, }, nil } func (c *ItemBlockActionGRPCClient) GetRelatedItems(item runtime.Unstructured, backup *api.Backup) ([]velero.ResourceIdentifier, error) { itemJSON, err := json.Marshal(item.UnstructuredContent()) if err != nil { return nil, errors.WithStack(err) } backupJSON, err := json.Marshal(backup) if err != nil { return nil, errors.WithStack(err) } req := &protoibav1.ItemBlockActionGetRelatedItemsRequest{ Plugin: c.Plugin, Item: itemJSON, Backup: backupJSON, } res, err := c.grpcClient.GetRelatedItems(context.Background(), req) if err != nil { return nil, common.FromGRPCError(err) } var relatedItems []velero.ResourceIdentifier for _, itm := range res.RelatedItems { newItem := velero.ResourceIdentifier{ GroupResource: schema.GroupResource{ Group: itm.Group, Resource: itm.Resource, }, Namespace: itm.Namespace, Name: itm.Name, } relatedItems = append(relatedItems, newItem) } return relatedItems, nil } // This shouldn't be called on the GRPC client since the RestartableItemBlockAction won't delegate // this method func (c *ItemBlockActionGRPCClient) Name() string { return "" } ================================================ FILE: pkg/plugin/framework/itemblockaction/v1/item_block_action_server.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v1 import ( "encoding/json" "context" "github.com/pkg/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" proto "github.com/vmware-tanzu/velero/pkg/plugin/generated" protoibav1 "github.com/vmware-tanzu/velero/pkg/plugin/generated/itemblockaction/v1" "github.com/vmware-tanzu/velero/pkg/plugin/velero" ibav1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/itemblockaction/v1" ) // ItemBlockActionGRPCServer implements the proto-generated ItemBlockAction interface, and accepts // gRPC calls and forwards them to an implementation of the pluggable interface. type ItemBlockActionGRPCServer struct { mux *common.ServerMux } func (s *ItemBlockActionGRPCServer) getImpl(name string) (ibav1.ItemBlockAction, error) { impl, err := s.mux.GetHandler(name) if err != nil { return nil, err } itemAction, ok := impl.(ibav1.ItemBlockAction) if !ok { return nil, errors.Errorf("%T is not an ItemBlock action", impl) } return itemAction, nil } func (s *ItemBlockActionGRPCServer) AppliesTo( ctx context.Context, req *protoibav1.ItemBlockActionAppliesToRequest) ( response *protoibav1.ItemBlockActionAppliesToResponse, err error) { defer func() { if recoveredErr := common.HandlePanic(recover()); recoveredErr != nil { err = recoveredErr } }() impl, err := s.getImpl(req.Plugin) if err != nil { return nil, common.NewGRPCError(err) } resourceSelector, err := impl.AppliesTo() if err != nil { return nil, common.NewGRPCError(err) } return &protoibav1.ItemBlockActionAppliesToResponse{ ResourceSelector: &proto.ResourceSelector{ IncludedNamespaces: resourceSelector.IncludedNamespaces, ExcludedNamespaces: resourceSelector.ExcludedNamespaces, IncludedResources: resourceSelector.IncludedResources, ExcludedResources: resourceSelector.ExcludedResources, Selector: resourceSelector.LabelSelector, }, }, nil } func (s *ItemBlockActionGRPCServer) GetRelatedItems( ctx context.Context, req *protoibav1.ItemBlockActionGetRelatedItemsRequest) (response *protoibav1.ItemBlockActionGetRelatedItemsResponse, err error) { defer func() { if recoveredErr := common.HandlePanic(recover()); recoveredErr != nil { err = recoveredErr } }() impl, err := s.getImpl(req.Plugin) if err != nil { return nil, common.NewGRPCError(err) } var item unstructured.Unstructured var backup api.Backup if err := json.Unmarshal(req.Item, &item); err != nil { return nil, common.NewGRPCError(errors.WithStack(err)) } if err := json.Unmarshal(req.Backup, &backup); err != nil { return nil, common.NewGRPCError(errors.WithStack(err)) } relatedItems, err := impl.GetRelatedItems(&item, &backup) if err != nil { return nil, common.NewGRPCError(err) } res := &protoibav1.ItemBlockActionGetRelatedItemsResponse{} for _, item := range relatedItems { res.RelatedItems = append(res.RelatedItems, backupResourceIdentifierToProto(item)) } return res, nil } func backupResourceIdentifierToProto(id velero.ResourceIdentifier) *proto.ResourceIdentifier { return &proto.ResourceIdentifier{ Group: id.Group, Resource: id.Resource, Namespace: id.Namespace, Name: id.Name, } } // This shouldn't be called on the GRPC server since the server won't ever receive this request, as // the RestartableItemBlockAction in Velero won't delegate this to the server func (s *ItemBlockActionGRPCServer) Name() string { return "" } ================================================ FILE: pkg/plugin/framework/itemblockaction/v1/item_block_action_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v1 import ( "encoding/json" "testing" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" proto "github.com/vmware-tanzu/velero/pkg/plugin/generated" protoibav1 "github.com/vmware-tanzu/velero/pkg/plugin/generated/itemblockaction/v1" "github.com/vmware-tanzu/velero/pkg/plugin/velero" mocks "github.com/vmware-tanzu/velero/pkg/plugin/velero/mocks/itemblockaction/v1" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func TestItemBlockActionGRPCServerGetRelatedItems(t *testing.T) { invalidItem := []byte("this is gibberish json") validItem := []byte(` { "apiVersion": "v1", "kind": "ConfigMap", "metadata": { "namespace": "myns", "name": "myconfigmap" }, "data": { "key": "value" } }`) var validItemObject unstructured.Unstructured err := json.Unmarshal(validItem, &validItemObject) require.NoError(t, err) invalidBackup := []byte("this is gibberish json") validBackup := []byte(` { "apiVersion": "velero.io/v1", "kind": "Backup", "metadata": { "namespace": "myns", "name": "mybackup" }, "spec": { "includedNamespaces": ["*"], "includedResources": ["*"], "ttl": "60m" } }`) var validBackupObject v1.Backup err = json.Unmarshal(validBackup, &validBackupObject) require.NoError(t, err) tests := []struct { name string backup []byte item []byte implRelatedItems []velero.ResourceIdentifier implError error expectError bool skipMock bool }{ { name: "error unmarshaling item", item: invalidItem, backup: validBackup, expectError: true, skipMock: true, }, { name: "error unmarshaling backup", item: validItem, backup: invalidBackup, expectError: true, skipMock: true, }, { name: "error running impl", item: validItem, backup: validBackup, implError: errors.New("impl error"), expectError: true, }, { name: "no relatedItems", item: validItem, backup: validBackup, }, { name: "some relatedItems", item: validItem, backup: validBackup, implRelatedItems: []velero.ResourceIdentifier{ { GroupResource: schema.GroupResource{Group: "v1", Resource: "pods"}, Namespace: "myns", Name: "mypod", }, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { itemAction := &mocks.ItemBlockAction{} defer itemAction.AssertExpectations(t) if !test.skipMock { itemAction.On("GetRelatedItems", &validItemObject, &validBackupObject).Return(test.implRelatedItems, test.implError) } s := &ItemBlockActionGRPCServer{mux: &common.ServerMux{ ServerLog: velerotest.NewLogger(), Handlers: map[string]any{ "xyz": itemAction, }, }} req := &protoibav1.ItemBlockActionGetRelatedItemsRequest{ Plugin: "xyz", Item: test.item, Backup: test.backup, } resp, err := s.GetRelatedItems(t.Context(), req) // Verify error assert.Equal(t, test.expectError, err != nil) if err != nil { return } require.NotNil(t, resp) // Verify related items var expectedRelatedItems []*proto.ResourceIdentifier for _, item := range test.implRelatedItems { expectedRelatedItems = append(expectedRelatedItems, backupResourceIdentifierToProto(item)) } assert.Equal(t, expectedRelatedItems, resp.RelatedItems) }) } } ================================================ FILE: pkg/plugin/framework/logger.go ================================================ /* Copyright 2017, 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package framework import ( "github.com/sirupsen/logrus" "github.com/vmware-tanzu/velero/pkg/util/logging" ) // newLogger returns a logger that is suitable for use within an // Velero plugin. func newLogger() *logrus.Logger { logger := logrus.New() /* !!!DO NOT SET THE OUTPUT TO STDOUT!!! go-plugin uses stdout for a communications protocol between client and server. stderr is used for log messages from server to client. The velero server makes sure they are logged to stdout. */ // we use the JSON formatter because go-plugin will parse incoming // JSON on stderr and use it to create structured log entries. logger.Formatter = &logrus.JSONFormatter{ FieldMap: logrus.FieldMap{ // this is the hclog-compatible message field logrus.FieldKeyMsg: "@message", }, // Velero server already adds timestamps when emitting logs, so // don't do it within the plugin. DisableTimestamp: true, } // set a logger name for the location hook which will signal to the Velero // server logger that the location has been set within a hook. logger.Hooks.Add((&logging.LogLocationHook{}).WithLoggerName("plugin")) // make sure we attempt to record the error location logger.Hooks.Add(&logging.ErrorLocationHook{}) // this hook adjusts the string representation of WarnLevel to "warn" // rather than "warning" to make it parseable by go-plugin within the // Velero server code logger.Hooks.Add(&logging.HcLogLevelHook{}) return logger } ================================================ FILE: pkg/plugin/framework/logger_test.go ================================================ /* Copyright 2018, 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package framework import ( "testing" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/vmware-tanzu/velero/pkg/util/logging" ) func TestNewLogger(t *testing.T) { l := newLogger() expectedFormatter := &logrus.JSONFormatter{ FieldMap: logrus.FieldMap{ logrus.FieldKeyMsg: "@message", }, DisableTimestamp: true, } assert.Equal(t, expectedFormatter, l.Formatter) expectedHooks := []logrus.Hook{ (&logging.LogLocationHook{}).WithLoggerName("plugin"), &logging.ErrorLocationHook{}, &logging.HcLogLevelHook{}, } for _, level := range logrus.AllLevels { assert.Equal(t, expectedHooks, l.Hooks[level]) } } ================================================ FILE: pkg/plugin/framework/object_store.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package framework import ( "context" plugin "github.com/hashicorp/go-plugin" "google.golang.org/grpc" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" proto "github.com/vmware-tanzu/velero/pkg/plugin/generated" ) // ObjectStorePlugin is an implementation of go-plugin's Plugin // interface with support for gRPC for the cloudprovider/ObjectStore // interface. type ObjectStorePlugin struct { plugin.NetRPCUnsupportedPlugin *common.PluginBase } // GRPCClient returns an ObjectStore gRPC client. func (p *ObjectStorePlugin) GRPCClient(_ context.Context, _ *plugin.GRPCBroker, clientConn *grpc.ClientConn) (any, error) { return common.NewClientDispenser(p.ClientLogger, clientConn, newObjectStoreGRPCClient), nil } // GRPCServer registers an ObjectStore gRPC server. func (p *ObjectStorePlugin) GRPCServer(_ *plugin.GRPCBroker, server *grpc.Server) error { proto.RegisterObjectStoreServer(server, &ObjectStoreGRPCServer{mux: p.ServerMux}) return nil } ================================================ FILE: pkg/plugin/framework/object_store_client.go ================================================ /* Copyright 2017, 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package framework import ( "io" "time" "context" "github.com/pkg/errors" "google.golang.org/grpc" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" proto "github.com/vmware-tanzu/velero/pkg/plugin/generated" ) const byteChunkSize = 16384 // NewObjectStorePlugin construct an ObjectStorePlugin. func NewObjectStorePlugin(options ...common.PluginOption) *ObjectStorePlugin { return &ObjectStorePlugin{ PluginBase: common.NewPluginBase(options...), } } // ObjectStoreGRPCClient implements the ObjectStore interface and uses a // gRPC client to make calls to the plugin server. type ObjectStoreGRPCClient struct { *common.ClientBase grpcClient proto.ObjectStoreClient } func newObjectStoreGRPCClient(base *common.ClientBase, clientConn *grpc.ClientConn) any { return &ObjectStoreGRPCClient{ ClientBase: base, grpcClient: proto.NewObjectStoreClient(clientConn), } } // Init prepares the ObjectStore for usage using the provided map of // configuration key-value pairs. It returns an error if the ObjectStore // cannot be initialized from the provided config. func (c *ObjectStoreGRPCClient) Init(config map[string]string) error { req := &proto.ObjectStoreInitRequest{ Plugin: c.Plugin, Config: config, } if _, err := c.grpcClient.Init(context.Background(), req); err != nil { return common.FromGRPCError(err) } return nil } // PutObject creates a new object using the data in body within the specified // object storage bucket with the given key. func (c *ObjectStoreGRPCClient) PutObject(bucket, key string, body io.Reader) error { stream, err := c.grpcClient.PutObject(context.Background()) if err != nil { return common.FromGRPCError(err) } // read from the provider io.Reader into chunks, and send each one over // the gRPC stream chunk := make([]byte, byteChunkSize) for { n, err := body.Read(chunk) if err == io.EOF { if _, resErr := stream.CloseAndRecv(); resErr != nil { return common.FromGRPCError(resErr) } return nil } if err != nil { if err := stream.CloseSend(); err != nil { return common.FromGRPCError(err) } return errors.WithStack(err) } if err := stream.Send(&proto.PutObjectRequest{Plugin: c.Plugin, Bucket: bucket, Key: key, Body: chunk[0:n]}); err != nil { return common.FromGRPCError(err) } } } // ObjectExists checks if there is an object with the given key in the object storage bucket. func (c *ObjectStoreGRPCClient) ObjectExists(bucket, key string) (bool, error) { req := &proto.ObjectExistsRequest{ Plugin: c.Plugin, Bucket: bucket, Key: key, } res, err := c.grpcClient.ObjectExists(context.Background(), req) if err != nil { return false, err } return res.Exists, nil } // GetObject retrieves the object with the given key from the specified // bucket in object storage. func (c *ObjectStoreGRPCClient) GetObject(bucket, key string) (io.ReadCloser, error) { req := &proto.GetObjectRequest{ Plugin: c.Plugin, Bucket: bucket, Key: key, } stream, err := c.grpcClient.GetObject(context.Background(), req) if err != nil { return nil, common.FromGRPCError(err) } receive := func() ([]byte, error) { data, err := stream.Recv() if err == io.EOF { // we need to return io.EOF errors unwrapped so that // calling code sees them as io.EOF and knows to stop // reading. return nil, err } if err != nil { return nil, common.FromGRPCError(err) } return data.Data, nil } close := func() error { if err := stream.CloseSend(); err != nil { return common.FromGRPCError(err) } return nil } return &StreamReadCloser{receive: receive, close: close}, nil } // ListCommonPrefixes gets a list of all object key prefixes that come // after the provided prefix and before the provided delimiter (this is // often used to simulate a directory hierarchy in object storage). func (c *ObjectStoreGRPCClient) ListCommonPrefixes(bucket, prefix, delimiter string) ([]string, error) { req := &proto.ListCommonPrefixesRequest{ Plugin: c.Plugin, Bucket: bucket, Prefix: prefix, Delimiter: delimiter, } res, err := c.grpcClient.ListCommonPrefixes(context.Background(), req) if err != nil { return nil, common.FromGRPCError(err) } return res.Prefixes, nil } // ListObjects gets a list of all objects in bucket that have the same prefix. func (c *ObjectStoreGRPCClient) ListObjects(bucket, prefix string) ([]string, error) { req := &proto.ListObjectsRequest{ Plugin: c.Plugin, Bucket: bucket, Prefix: prefix, } res, err := c.grpcClient.ListObjects(context.Background(), req) if err != nil { return nil, common.FromGRPCError(err) } return res.Keys, nil } // DeleteObject removes object with the specified key from the given // bucket. func (c *ObjectStoreGRPCClient) DeleteObject(bucket, key string) error { req := &proto.DeleteObjectRequest{ Plugin: c.Plugin, Bucket: bucket, Key: key, } if _, err := c.grpcClient.DeleteObject(context.Background(), req); err != nil { return common.FromGRPCError(err) } return nil } // CreateSignedURL creates a pre-signed URL for the given bucket and key that expires after ttl. func (c *ObjectStoreGRPCClient) CreateSignedURL(bucket, key string, ttl time.Duration) (string, error) { req := &proto.CreateSignedURLRequest{ Plugin: c.Plugin, Bucket: bucket, Key: key, Ttl: int64(ttl), } res, err := c.grpcClient.CreateSignedURL(context.Background(), req) if err != nil { return "", common.FromGRPCError(err) } return res.Url, nil } ================================================ FILE: pkg/plugin/framework/object_store_server.go ================================================ /* Copyright 2017, 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package framework import ( "io" "time" "context" "github.com/pkg/errors" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" proto "github.com/vmware-tanzu/velero/pkg/plugin/generated" "github.com/vmware-tanzu/velero/pkg/plugin/velero" ) // ObjectStoreGRPCServer implements the proto-generated ObjectStoreServer interface, and accepts // gRPC calls and forwards them to an implementation of the pluggable interface. type ObjectStoreGRPCServer struct { mux *common.ServerMux } func (s *ObjectStoreGRPCServer) getImpl(name string) (velero.ObjectStore, error) { impl, err := s.mux.GetHandler(name) if err != nil { return nil, err } itemAction, ok := impl.(velero.ObjectStore) if !ok { return nil, errors.Errorf("%T is not an object store", impl) } return itemAction, nil } // Init prepares the ObjectStore for usage using the provided map of // configuration key-value pairs. It returns an error if the ObjectStore // cannot be initialized from the provided config. func (s *ObjectStoreGRPCServer) Init(ctx context.Context, req *proto.ObjectStoreInitRequest) (response *proto.Empty, err error) { defer func() { if recoveredErr := common.HandlePanic(recover()); recoveredErr != nil { err = recoveredErr } }() impl, err := s.getImpl(req.Plugin) if err != nil { return nil, common.NewGRPCError(err) } if err := impl.Init(req.Config); err != nil { return nil, common.NewGRPCError(err) } return &proto.Empty{}, nil } // PutObject creates a new object using the data in body within the specified // object storage bucket with the given key. func (s *ObjectStoreGRPCServer) PutObject(stream proto.ObjectStore_PutObjectServer) (err error) { defer func() { if recoveredErr := common.HandlePanic(recover()); recoveredErr != nil { err = recoveredErr } }() // we need to read the first chunk ahead of time to get the bucket and key; // in our receive method, we'll use `first` on the first call firstChunk, err := stream.Recv() if err != nil { return common.NewGRPCError(errors.WithStack(err)) } impl, err := s.getImpl(firstChunk.Plugin) if err != nil { return common.NewGRPCError(err) } bucket := firstChunk.Bucket key := firstChunk.Key receive := func() ([]byte, error) { if firstChunk != nil { res := firstChunk.Body firstChunk = nil return res, nil } data, err := stream.Recv() if err == io.EOF { // we need to return io.EOF errors unwrapped so that // calling code sees them as io.EOF and knows to stop // reading. return nil, err } if err != nil { return nil, errors.WithStack(err) } return data.Body, nil } close := func() error { return nil } if err := impl.PutObject(bucket, key, &StreamReadCloser{receive: receive, close: close}); err != nil { return common.NewGRPCError(err) } if err := stream.SendAndClose(&proto.Empty{}); err != nil { return common.NewGRPCError(errors.WithStack(err)) } return nil } // ObjectExists checks if there is an object with the given key in the object storage bucket. func (s *ObjectStoreGRPCServer) ObjectExists(ctx context.Context, req *proto.ObjectExistsRequest) (response *proto.ObjectExistsResponse, err error) { defer func() { if recoveredErr := common.HandlePanic(recover()); recoveredErr != nil { err = recoveredErr } }() impl, err := s.getImpl(req.Plugin) if err != nil { return nil, common.NewGRPCError(err) } exists, err := impl.ObjectExists(req.Bucket, req.Key) if err != nil { return nil, common.NewGRPCError(err) } return &proto.ObjectExistsResponse{Exists: exists}, nil } // GetObject retrieves the object with the given key from the specified // bucket in object storage. func (s *ObjectStoreGRPCServer) GetObject(req *proto.GetObjectRequest, stream proto.ObjectStore_GetObjectServer) (err error) { defer func() { if recoveredErr := common.HandlePanic(recover()); recoveredErr != nil { err = recoveredErr } }() impl, err := s.getImpl(req.Plugin) if err != nil { return common.NewGRPCError(err) } rdr, err := impl.GetObject(req.Bucket, req.Key) if err != nil { return common.NewGRPCError(err) } defer rdr.Close() chunk := make([]byte, byteChunkSize) for { n, err := rdr.Read(chunk) if err != nil && err != io.EOF { return common.NewGRPCError(errors.WithStack(err)) } if n == 0 { return nil } if err := stream.Send(&proto.Bytes{Data: chunk[0:n]}); err != nil { return common.NewGRPCError(errors.WithStack(err)) } } } // ListCommonPrefixes gets a list of all object key prefixes that start with // the specified prefix and stop at the next instance of the provided delimiter // (this is often used to simulate a directory hierarchy in object storage). func (s *ObjectStoreGRPCServer) ListCommonPrefixes(ctx context.Context, req *proto.ListCommonPrefixesRequest) (response *proto.ListCommonPrefixesResponse, err error) { defer func() { if recoveredErr := common.HandlePanic(recover()); recoveredErr != nil { err = recoveredErr } }() impl, err := s.getImpl(req.Plugin) if err != nil { return nil, common.NewGRPCError(err) } prefixes, err := impl.ListCommonPrefixes(req.Bucket, req.Prefix, req.Delimiter) if err != nil { return nil, common.NewGRPCError(err) } return &proto.ListCommonPrefixesResponse{Prefixes: prefixes}, nil } // ListObjects gets a list of all objects in bucket that have the same prefix. func (s *ObjectStoreGRPCServer) ListObjects(ctx context.Context, req *proto.ListObjectsRequest) (response *proto.ListObjectsResponse, err error) { defer func() { if recoveredErr := common.HandlePanic(recover()); recoveredErr != nil { err = recoveredErr } }() impl, err := s.getImpl(req.Plugin) if err != nil { return nil, common.NewGRPCError(err) } keys, err := impl.ListObjects(req.Bucket, req.Prefix) if err != nil { return nil, common.NewGRPCError(err) } return &proto.ListObjectsResponse{Keys: keys}, nil } // DeleteObject removes object with the specified key from the given // bucket. func (s *ObjectStoreGRPCServer) DeleteObject(ctx context.Context, req *proto.DeleteObjectRequest) (response *proto.Empty, err error) { defer func() { if recoveredErr := common.HandlePanic(recover()); recoveredErr != nil { err = recoveredErr } }() impl, err := s.getImpl(req.Plugin) if err != nil { return nil, common.NewGRPCError(err) } if err := impl.DeleteObject(req.Bucket, req.Key); err != nil { return nil, common.NewGRPCError(err) } return &proto.Empty{}, nil } // CreateSignedURL creates a pre-signed URL for the given bucket and key that expires after ttl. func (s *ObjectStoreGRPCServer) CreateSignedURL(ctx context.Context, req *proto.CreateSignedURLRequest) (response *proto.CreateSignedURLResponse, err error) { defer func() { if recoveredErr := common.HandlePanic(recover()); recoveredErr != nil { err = recoveredErr } }() impl, err := s.getImpl(req.Plugin) if err != nil { return nil, common.NewGRPCError(err) } url, err := impl.CreateSignedURL(req.Bucket, req.Key, time.Duration(req.Ttl)) if err != nil { return nil, common.NewGRPCError(err) } return &proto.CreateSignedURLResponse{Url: url}, nil } ================================================ FILE: pkg/plugin/framework/plugin_lister.go ================================================ /* Copyright 2018, 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package framework import ( "context" plugin "github.com/hashicorp/go-plugin" "github.com/pkg/errors" "google.golang.org/grpc" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" proto "github.com/vmware-tanzu/velero/pkg/plugin/generated" ) // PluginIdentifier uniquely identifies a plugin by command, kind, and name. type PluginIdentifier struct { Command string Kind common.PluginKind Name string } // PluginLister lists plugins. type PluginLister interface { ListPlugins() ([]PluginIdentifier, error) } // pluginLister implements PluginLister. type pluginLister struct { plugins []PluginIdentifier } // NewPluginLister returns a new PluginLister for plugins. func NewPluginLister(plugins ...PluginIdentifier) PluginLister { return &pluginLister{plugins: plugins} } // ListPlugins returns the pluginLister's plugins. func (pl *pluginLister) ListPlugins() ([]PluginIdentifier, error) { return pl.plugins, nil } // PluginListerPlugin is a go-plugin Plugin for a PluginLister. type PluginListerPlugin struct { plugin.NetRPCUnsupportedPlugin impl PluginLister } // NewPluginListerPlugin creates a new PluginListerPlugin with impl as the server-side implementation. func NewPluginListerPlugin(impl PluginLister) *PluginListerPlugin { return &PluginListerPlugin{impl: impl} } ////////////////////////////////////////////////////////////////////////////// // client code ////////////////////////////////////////////////////////////////////////////// // GRPCClient returns a PluginLister gRPC client. func (p *PluginListerPlugin) GRPCClient(_ context.Context, _ *plugin.GRPCBroker, clientConn *grpc.ClientConn) (any, error) { return &PluginListerGRPCClient{grpcClient: proto.NewPluginListerClient(clientConn)}, nil } // PluginListerGRPCClient implements PluginLister and uses a gRPC client to make calls to the plugin server. type PluginListerGRPCClient struct { grpcClient proto.PluginListerClient } // ListPlugins uses the gRPC client to request the list of plugins from the server. It translates the protobuf response // to []PluginIdentifier. func (c *PluginListerGRPCClient) ListPlugins() ([]PluginIdentifier, error) { resp, err := c.grpcClient.ListPlugins(context.Background(), &proto.Empty{}) if err != nil { return nil, err } ret := make([]PluginIdentifier, len(resp.Plugins)) for i, id := range resp.Plugins { if _, ok := common.AllPluginKinds()[id.Kind]; !ok { return nil, errors.Errorf("invalid plugin kind: %s", id.Kind) } ret[i] = PluginIdentifier{ Command: id.Command, Kind: common.PluginKind(id.Kind), Name: id.Name, } } return ret, nil } ////////////////////////////////////////////////////////////////////////////// // server code ////////////////////////////////////////////////////////////////////////////// // GRPCServer registers a PluginLister gRPC server. func (p *PluginListerPlugin) GRPCServer(_ *plugin.GRPCBroker, server *grpc.Server) error { proto.RegisterPluginListerServer(server, &PluginListerGRPCServer{impl: p.impl}) return nil } // PluginListerGRPCServer implements the proto-generated PluginLister gRPC service interface. It accepts gRPC calls, // forwards them to impl, and translates the responses to protobuf. type PluginListerGRPCServer struct { impl PluginLister } // ListPlugins returns a list of registered plugins, delegating to s.impl to perform the listing. func (s *PluginListerGRPCServer) ListPlugins(ctx context.Context, req *proto.Empty) (*proto.ListPluginsResponse, error) { list, err := s.impl.ListPlugins() if err != nil { return nil, err } plugins := make([]*proto.PluginIdentifier, len(list)) for i, id := range list { if _, ok := common.AllPluginKinds()[id.Kind.String()]; !ok { return nil, errors.Errorf("invalid plugin kind: %s", id.Kind) } plugins[i] = &proto.PluginIdentifier{ Command: id.Command, Kind: id.Kind.String(), Name: id.Name, } } ret := &proto.ListPluginsResponse{ Plugins: plugins, } return ret, nil } ================================================ FILE: pkg/plugin/framework/plugin_types_test.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package framework import ( "testing" plugin "github.com/hashicorp/go-plugin" "github.com/stretchr/testify/assert" ) func TestPluginImplementationsAreGRPCPlugins(t *testing.T) { pluginImpls := []any{ new(VolumeSnapshotterPlugin), new(BackupItemActionPlugin), new(ObjectStorePlugin), new(PluginListerPlugin), new(RestoreItemActionPlugin), } for _, impl := range pluginImpls { _, ok := impl.(plugin.GRPCPlugin) assert.True(t, ok, "plugin implementation %T does not implement the go-plugin.GRPCPlugin interface", impl) } } ================================================ FILE: pkg/plugin/framework/restore_item_action.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package framework import ( "context" plugin "github.com/hashicorp/go-plugin" "google.golang.org/grpc" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" proto "github.com/vmware-tanzu/velero/pkg/plugin/generated" ) // RestoreItemActionPlugin is an implementation of go-plugin's Plugin // interface with support for gRPC for the restore/ItemAction // interface. type RestoreItemActionPlugin struct { plugin.NetRPCUnsupportedPlugin *common.PluginBase } // GRPCClient returns a RestoreItemAction gRPC client. func (p *RestoreItemActionPlugin) GRPCClient(_ context.Context, _ *plugin.GRPCBroker, clientConn *grpc.ClientConn) (any, error) { return common.NewClientDispenser(p.ClientLogger, clientConn, newRestoreItemActionGRPCClient), nil } // GRPCServer registers a RestoreItemAction gRPC server. func (p *RestoreItemActionPlugin) GRPCServer(_ *plugin.GRPCBroker, server *grpc.Server) error { proto.RegisterRestoreItemActionServer(server, &RestoreItemActionGRPCServer{mux: p.ServerMux}) return nil } ================================================ FILE: pkg/plugin/framework/restore_item_action_client.go ================================================ /* Copyright 2017, 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package framework import ( "encoding/json" "context" "github.com/pkg/errors" "google.golang.org/grpc" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" proto "github.com/vmware-tanzu/velero/pkg/plugin/generated" "github.com/vmware-tanzu/velero/pkg/plugin/velero" riav1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/restoreitemaction/v1" ) var _ riav1.RestoreItemAction = &RestoreItemActionGRPCClient{} // NewRestoreItemActionPlugin constructs a RestoreItemActionPlugin. func NewRestoreItemActionPlugin(options ...common.PluginOption) *RestoreItemActionPlugin { return &RestoreItemActionPlugin{ PluginBase: common.NewPluginBase(options...), } } // RestoreItemActionGRPCClient implements the backup/ItemAction interface and uses a // gRPC client to make calls to the plugin server. type RestoreItemActionGRPCClient struct { *common.ClientBase grpcClient proto.RestoreItemActionClient } func newRestoreItemActionGRPCClient(base *common.ClientBase, clientConn *grpc.ClientConn) any { return &RestoreItemActionGRPCClient{ ClientBase: base, grpcClient: proto.NewRestoreItemActionClient(clientConn), } } func (c *RestoreItemActionGRPCClient) AppliesTo() (velero.ResourceSelector, error) { res, err := c.grpcClient.AppliesTo(context.Background(), &proto.RestoreItemActionAppliesToRequest{Plugin: c.Plugin}) if err != nil { return velero.ResourceSelector{}, common.FromGRPCError(err) } if res.ResourceSelector == nil { return velero.ResourceSelector{}, nil } return velero.ResourceSelector{ IncludedNamespaces: res.ResourceSelector.IncludedNamespaces, ExcludedNamespaces: res.ResourceSelector.ExcludedNamespaces, IncludedResources: res.ResourceSelector.IncludedResources, ExcludedResources: res.ResourceSelector.ExcludedResources, LabelSelector: res.ResourceSelector.Selector, }, nil } func (c *RestoreItemActionGRPCClient) Execute(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { itemJSON, err := json.Marshal(input.Item.UnstructuredContent()) if err != nil { return nil, errors.WithStack(err) } itemFromBackupJSON, err := json.Marshal(input.ItemFromBackup.UnstructuredContent()) if err != nil { return nil, errors.WithStack(err) } restoreJSON, err := json.Marshal(input.Restore) if err != nil { return nil, errors.WithStack(err) } req := &proto.RestoreItemActionExecuteRequest{ Plugin: c.Plugin, Item: itemJSON, ItemFromBackup: itemFromBackupJSON, Restore: restoreJSON, } res, err := c.grpcClient.Execute(context.Background(), req) if err != nil { return nil, common.FromGRPCError(err) } var updatedItem unstructured.Unstructured if err := json.Unmarshal(res.Item, &updatedItem); err != nil { return nil, errors.WithStack(err) } var additionalItems []velero.ResourceIdentifier for _, itm := range res.AdditionalItems { newItem := velero.ResourceIdentifier{ GroupResource: schema.GroupResource{ Group: itm.Group, Resource: itm.Resource, }, Namespace: itm.Namespace, Name: itm.Name, } additionalItems = append(additionalItems, newItem) } return &velero.RestoreItemActionExecuteOutput{ UpdatedItem: &updatedItem, AdditionalItems: additionalItems, SkipRestore: res.SkipRestore, }, nil } ================================================ FILE: pkg/plugin/framework/restore_item_action_server.go ================================================ /* Copyright 2017, 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package framework import ( "encoding/json" "context" "github.com/pkg/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" proto "github.com/vmware-tanzu/velero/pkg/plugin/generated" "github.com/vmware-tanzu/velero/pkg/plugin/velero" riav1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/restoreitemaction/v1" ) // RestoreItemActionGRPCServer implements the proto-generated RestoreItemActionServer interface, and accepts // gRPC calls and forwards them to an implementation of the pluggable interface. type RestoreItemActionGRPCServer struct { mux *common.ServerMux } func (s *RestoreItemActionGRPCServer) getImpl(name string) (riav1.RestoreItemAction, error) { impl, err := s.mux.GetHandler(name) if err != nil { return nil, err } itemAction, ok := impl.(riav1.RestoreItemAction) if !ok { return nil, errors.Errorf("%T is not a restore item action", impl) } return itemAction, nil } func (s *RestoreItemActionGRPCServer) AppliesTo(ctx context.Context, req *proto.RestoreItemActionAppliesToRequest) (response *proto.RestoreItemActionAppliesToResponse, err error) { defer func() { if recoveredErr := common.HandlePanic(recover()); recoveredErr != nil { err = recoveredErr } }() impl, err := s.getImpl(req.Plugin) if err != nil { return nil, common.NewGRPCError(err) } resourceSelector, err := impl.AppliesTo() if err != nil { return nil, common.NewGRPCError(err) } return &proto.RestoreItemActionAppliesToResponse{ ResourceSelector: &proto.ResourceSelector{ IncludedNamespaces: resourceSelector.IncludedNamespaces, ExcludedNamespaces: resourceSelector.ExcludedNamespaces, IncludedResources: resourceSelector.IncludedResources, ExcludedResources: resourceSelector.ExcludedResources, Selector: resourceSelector.LabelSelector, }, }, nil } func (s *RestoreItemActionGRPCServer) Execute(ctx context.Context, req *proto.RestoreItemActionExecuteRequest) (response *proto.RestoreItemActionExecuteResponse, err error) { defer func() { if recoveredErr := common.HandlePanic(recover()); recoveredErr != nil { err = recoveredErr } }() impl, err := s.getImpl(req.Plugin) if err != nil { return nil, common.NewGRPCError(err) } var ( item unstructured.Unstructured itemFromBackup unstructured.Unstructured restoreObj api.Restore ) if err := json.Unmarshal(req.Item, &item); err != nil { return nil, common.NewGRPCError(errors.WithStack(err)) } if err := json.Unmarshal(req.ItemFromBackup, &itemFromBackup); err != nil { return nil, common.NewGRPCError(errors.WithStack(err)) } if err := json.Unmarshal(req.Restore, &restoreObj); err != nil { return nil, common.NewGRPCError(errors.WithStack(err)) } executeOutput, err := impl.Execute(&velero.RestoreItemActionExecuteInput{ Item: &item, ItemFromBackup: &itemFromBackup, Restore: &restoreObj, }) if err != nil { return nil, common.NewGRPCError(err) } // If the plugin implementation returned a nil updateItem (meaning no modifications), reset updatedItem to the // original item. var updatedItemJSON []byte if executeOutput.UpdatedItem == nil { updatedItemJSON = req.Item } else { updatedItemJSON, err = json.Marshal(executeOutput.UpdatedItem.UnstructuredContent()) if err != nil { return nil, common.NewGRPCError(errors.WithStack(err)) } } res := &proto.RestoreItemActionExecuteResponse{ Item: updatedItemJSON, SkipRestore: executeOutput.SkipRestore, } for _, item := range executeOutput.AdditionalItems { res.AdditionalItems = append(res.AdditionalItems, restoreResourceIdentifierToProto(item)) } return res, nil } func restoreResourceIdentifierToProto(id velero.ResourceIdentifier) *proto.ResourceIdentifier { return &proto.ResourceIdentifier{ Group: id.Group, Resource: id.Resource, Namespace: id.Namespace, Name: id.Name, } } ================================================ FILE: pkg/plugin/framework/restoreitemaction/v2/restore_item_action.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v2 import ( "context" plugin "github.com/hashicorp/go-plugin" "google.golang.org/grpc" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" protoriav2 "github.com/vmware-tanzu/velero/pkg/plugin/generated/restoreitemaction/v2" ) // RestoreItemActionPlugin is an implementation of go-plugin's Plugin // interface with support for gRPC for the restore/ItemAction // interface. type RestoreItemActionPlugin struct { plugin.NetRPCUnsupportedPlugin *common.PluginBase } // GRPCClient returns a RestoreItemAction gRPC client. func (p *RestoreItemActionPlugin) GRPCClient(_ context.Context, _ *plugin.GRPCBroker, clientConn *grpc.ClientConn) (any, error) { return common.NewClientDispenser(p.ClientLogger, clientConn, newRestoreItemActionGRPCClient), nil } // GRPCServer registers a RestoreItemAction gRPC server. func (p *RestoreItemActionPlugin) GRPCServer(_ *plugin.GRPCBroker, server *grpc.Server) error { protoriav2.RegisterRestoreItemActionServer(server, &RestoreItemActionGRPCServer{mux: p.ServerMux}) return nil } ================================================ FILE: pkg/plugin/framework/restoreitemaction/v2/restore_item_action_client.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v2 import ( "encoding/json" "context" "github.com/pkg/errors" "google.golang.org/grpc" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" protoriav2 "github.com/vmware-tanzu/velero/pkg/plugin/generated/restoreitemaction/v2" "github.com/vmware-tanzu/velero/pkg/plugin/velero" riav2 "github.com/vmware-tanzu/velero/pkg/plugin/velero/restoreitemaction/v2" ) var _ riav2.RestoreItemAction = &RestoreItemActionGRPCClient{} // NewRestoreItemActionPlugin constructs a RestoreItemActionPlugin. func NewRestoreItemActionPlugin(options ...common.PluginOption) *RestoreItemActionPlugin { return &RestoreItemActionPlugin{ PluginBase: common.NewPluginBase(options...), } } // RestoreItemActionGRPCClient implements the backup/ItemAction interface and uses a // gRPC client to make calls to the plugin server. type RestoreItemActionGRPCClient struct { *common.ClientBase grpcClient protoriav2.RestoreItemActionClient } func newRestoreItemActionGRPCClient(base *common.ClientBase, clientConn *grpc.ClientConn) any { return &RestoreItemActionGRPCClient{ ClientBase: base, grpcClient: protoriav2.NewRestoreItemActionClient(clientConn), } } func (c *RestoreItemActionGRPCClient) AppliesTo() (velero.ResourceSelector, error) { res, err := c.grpcClient.AppliesTo(context.Background(), &protoriav2.RestoreItemActionAppliesToRequest{Plugin: c.Plugin}) if err != nil { return velero.ResourceSelector{}, common.FromGRPCError(err) } if res.ResourceSelector == nil { return velero.ResourceSelector{}, nil } return velero.ResourceSelector{ IncludedNamespaces: res.ResourceSelector.IncludedNamespaces, ExcludedNamespaces: res.ResourceSelector.ExcludedNamespaces, IncludedResources: res.ResourceSelector.IncludedResources, ExcludedResources: res.ResourceSelector.ExcludedResources, LabelSelector: res.ResourceSelector.Selector, }, nil } func (c *RestoreItemActionGRPCClient) Execute(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { itemJSON, err := json.Marshal(input.Item.UnstructuredContent()) if err != nil { return nil, errors.WithStack(err) } itemFromBackupJSON, err := json.Marshal(input.ItemFromBackup.UnstructuredContent()) if err != nil { return nil, errors.WithStack(err) } restoreJSON, err := json.Marshal(input.Restore) if err != nil { return nil, errors.WithStack(err) } req := &protoriav2.RestoreItemActionExecuteRequest{ Plugin: c.Plugin, Item: itemJSON, ItemFromBackup: itemFromBackupJSON, Restore: restoreJSON, } res, err := c.grpcClient.Execute(context.Background(), req) if err != nil { return nil, common.FromGRPCError(err) } var updatedItem unstructured.Unstructured if err := json.Unmarshal(res.Item, &updatedItem); err != nil { return nil, errors.WithStack(err) } var additionalItems []velero.ResourceIdentifier for _, itm := range res.AdditionalItems { newItem := velero.ResourceIdentifier{ GroupResource: schema.GroupResource{ Group: itm.Group, Resource: itm.Resource, }, Namespace: itm.Namespace, Name: itm.Name, } additionalItems = append(additionalItems, newItem) } return &velero.RestoreItemActionExecuteOutput{ UpdatedItem: &updatedItem, AdditionalItems: additionalItems, SkipRestore: res.SkipRestore, OperationID: res.OperationID, WaitForAdditionalItems: res.WaitForAdditionalItems, AdditionalItemsReadyTimeout: res.AdditionalItemsReadyTimeout.AsDuration(), }, nil } func (c *RestoreItemActionGRPCClient) Progress(operationID string, restore *api.Restore) (velero.OperationProgress, error) { restoreJSON, err := json.Marshal(restore) if err != nil { return velero.OperationProgress{}, errors.WithStack(err) } req := &protoriav2.RestoreItemActionProgressRequest{ Plugin: c.Plugin, OperationID: operationID, Restore: restoreJSON, } res, err := c.grpcClient.Progress(context.Background(), req) if err != nil { return velero.OperationProgress{}, common.FromGRPCError(err) } return velero.OperationProgress{ Completed: res.Progress.Completed, Err: res.Progress.Err, NCompleted: res.Progress.NCompleted, NTotal: res.Progress.NTotal, OperationUnits: res.Progress.OperationUnits, Description: res.Progress.Description, Started: res.Progress.Started.AsTime(), Updated: res.Progress.Updated.AsTime(), }, nil } func (c *RestoreItemActionGRPCClient) Cancel(operationID string, restore *api.Restore) error { restoreJSON, err := json.Marshal(restore) if err != nil { return errors.WithStack(err) } req := &protoriav2.RestoreItemActionCancelRequest{ Plugin: c.Plugin, OperationID: operationID, Restore: restoreJSON, } _, err = c.grpcClient.Cancel(context.Background(), req) if err != nil { return common.FromGRPCError(err) } return nil } func (c *RestoreItemActionGRPCClient) AreAdditionalItemsReady(additionalItems []velero.ResourceIdentifier, restore *api.Restore) (bool, error) { restoreJSON, err := json.Marshal(restore) if err != nil { return false, errors.WithStack(err) } req := &protoriav2.RestoreItemActionItemsReadyRequest{ Plugin: c.Plugin, Restore: restoreJSON, } for _, item := range additionalItems { req.AdditionalItems = append(req.AdditionalItems, restoreResourceIdentifierToProto(item)) } res, err := c.grpcClient.AreAdditionalItemsReady(context.Background(), req) if err != nil { return false, common.FromGRPCError(err) } return res.Ready, nil } // This shouldn't be called on the GRPC client since the RestartableRestoreItemAction won't delegate // this method func (c *RestoreItemActionGRPCClient) Name() string { return "" } ================================================ FILE: pkg/plugin/framework/restoreitemaction/v2/restore_item_action_server.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v2 import ( "encoding/json" "context" "github.com/pkg/errors" "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/emptypb" "google.golang.org/protobuf/types/known/timestamppb" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" proto "github.com/vmware-tanzu/velero/pkg/plugin/generated" protoriav2 "github.com/vmware-tanzu/velero/pkg/plugin/generated/restoreitemaction/v2" "github.com/vmware-tanzu/velero/pkg/plugin/velero" riav2 "github.com/vmware-tanzu/velero/pkg/plugin/velero/restoreitemaction/v2" ) // RestoreItemActionGRPCServer implements the proto-generated RestoreItemActionServer interface, and accepts // gRPC calls and forwards them to an implementation of the pluggable interface. type RestoreItemActionGRPCServer struct { mux *common.ServerMux } func (s *RestoreItemActionGRPCServer) getImpl(name string) (riav2.RestoreItemAction, error) { impl, err := s.mux.GetHandler(name) if err != nil { return nil, err } itemAction, ok := impl.(riav2.RestoreItemAction) if !ok { return nil, errors.Errorf("%T is not a restore item action (v2)", impl) } return itemAction, nil } func (s *RestoreItemActionGRPCServer) AppliesTo(ctx context.Context, req *protoriav2.RestoreItemActionAppliesToRequest) (response *protoriav2.RestoreItemActionAppliesToResponse, err error) { defer func() { if recoveredErr := common.HandlePanic(recover()); recoveredErr != nil { err = recoveredErr } }() impl, err := s.getImpl(req.Plugin) if err != nil { return nil, common.NewGRPCError(err) } resourceSelector, err := impl.AppliesTo() if err != nil { return nil, common.NewGRPCError(err) } return &protoriav2.RestoreItemActionAppliesToResponse{ ResourceSelector: &proto.ResourceSelector{ IncludedNamespaces: resourceSelector.IncludedNamespaces, ExcludedNamespaces: resourceSelector.ExcludedNamespaces, IncludedResources: resourceSelector.IncludedResources, ExcludedResources: resourceSelector.ExcludedResources, Selector: resourceSelector.LabelSelector, }, }, nil } func (s *RestoreItemActionGRPCServer) Execute(ctx context.Context, req *protoriav2.RestoreItemActionExecuteRequest) (response *protoriav2.RestoreItemActionExecuteResponse, err error) { defer func() { if recoveredErr := common.HandlePanic(recover()); recoveredErr != nil { err = recoveredErr } }() impl, err := s.getImpl(req.Plugin) if err != nil { return nil, common.NewGRPCError(err) } var ( item unstructured.Unstructured itemFromBackup unstructured.Unstructured restoreObj api.Restore ) if err := json.Unmarshal(req.Item, &item); err != nil { return nil, common.NewGRPCError(errors.WithStack(err)) } if err := json.Unmarshal(req.ItemFromBackup, &itemFromBackup); err != nil { return nil, common.NewGRPCError(errors.WithStack(err)) } if err := json.Unmarshal(req.Restore, &restoreObj); err != nil { return nil, common.NewGRPCError(errors.WithStack(err)) } executeOutput, err := impl.Execute(&velero.RestoreItemActionExecuteInput{ Item: &item, ItemFromBackup: &itemFromBackup, Restore: &restoreObj, }) if err != nil { return nil, common.NewGRPCError(err) } // If the plugin implementation returned a nil updateItem (meaning no modifications), reset updatedItem to the // original item. var updatedItemJSON []byte if executeOutput.UpdatedItem == nil { updatedItemJSON = req.Item } else { updatedItemJSON, err = json.Marshal(executeOutput.UpdatedItem.UnstructuredContent()) if err != nil { return nil, common.NewGRPCError(errors.WithStack(err)) } } res := &protoriav2.RestoreItemActionExecuteResponse{ Item: updatedItemJSON, SkipRestore: executeOutput.SkipRestore, OperationID: executeOutput.OperationID, WaitForAdditionalItems: executeOutput.WaitForAdditionalItems, AdditionalItemsReadyTimeout: durationpb.New(executeOutput.AdditionalItemsReadyTimeout), } for _, item := range executeOutput.AdditionalItems { res.AdditionalItems = append(res.AdditionalItems, restoreResourceIdentifierToProto(item)) } return res, nil } func (s *RestoreItemActionGRPCServer) Progress(ctx context.Context, req *protoriav2.RestoreItemActionProgressRequest) ( response *protoriav2.RestoreItemActionProgressResponse, err error) { defer func() { if recoveredErr := common.HandlePanic(recover()); recoveredErr != nil { err = recoveredErr } }() impl, err := s.getImpl(req.Plugin) if err != nil { return nil, common.NewGRPCError(err) } var restore api.Restore if err := json.Unmarshal(req.Restore, &restore); err != nil { return nil, common.NewGRPCError(errors.WithStack(err)) } progress, err := impl.Progress(req.OperationID, &restore) if err != nil { return nil, common.NewGRPCError(err) } res := &protoriav2.RestoreItemActionProgressResponse{ Progress: &proto.OperationProgress{ Completed: progress.Completed, Err: progress.Err, NCompleted: progress.NCompleted, NTotal: progress.NTotal, OperationUnits: progress.OperationUnits, Description: progress.Description, Started: timestamppb.New(progress.Started), Updated: timestamppb.New(progress.Updated), }, } return res, nil } func (s *RestoreItemActionGRPCServer) Cancel( ctx context.Context, req *protoriav2.RestoreItemActionCancelRequest) ( response *emptypb.Empty, err error) { defer func() { if recoveredErr := common.HandlePanic(recover()); recoveredErr != nil { err = recoveredErr } }() impl, err := s.getImpl(req.Plugin) if err != nil { return nil, common.NewGRPCError(err) } var restore api.Restore if err := json.Unmarshal(req.Restore, &restore); err != nil { return nil, common.NewGRPCError(errors.WithStack(err)) } err = impl.Cancel(req.OperationID, &restore) if err != nil { return nil, common.NewGRPCError(err) } return &emptypb.Empty{}, nil } func (s *RestoreItemActionGRPCServer) AreAdditionalItemsReady(ctx context.Context, req *protoriav2.RestoreItemActionItemsReadyRequest) ( response *protoriav2.RestoreItemActionItemsReadyResponse, err error) { defer func() { if recoveredErr := common.HandlePanic(recover()); recoveredErr != nil { err = recoveredErr } }() impl, err := s.getImpl(req.Plugin) if err != nil { return nil, common.NewGRPCError(err) } var restore api.Restore if err := json.Unmarshal(req.Restore, &restore); err != nil { return nil, common.NewGRPCError(errors.WithStack(err)) } var additionalItems []velero.ResourceIdentifier for _, itm := range req.AdditionalItems { newItem := velero.ResourceIdentifier{ GroupResource: schema.GroupResource{ Group: itm.Group, Resource: itm.Resource, }, Namespace: itm.Namespace, Name: itm.Name, } additionalItems = append(additionalItems, newItem) } ready, err := impl.AreAdditionalItemsReady(additionalItems, &restore) if err != nil { return nil, common.NewGRPCError(err) } res := &protoriav2.RestoreItemActionItemsReadyResponse{ Ready: ready, } return res, nil } func restoreResourceIdentifierToProto(id velero.ResourceIdentifier) *proto.ResourceIdentifier { return &proto.ResourceIdentifier{ Group: id.Group, Resource: id.Resource, Namespace: id.Namespace, Name: id.Name, } } // This shouldn't be called on the GRPC server since the server won't ever receive this request, as // the RestartableRestoreItemAction in Velero won't delegate this to the server func (s *RestoreItemActionGRPCServer) Name() string { return "" } ================================================ FILE: pkg/plugin/framework/server.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package framework import ( "os" "strings" plugin "github.com/hashicorp/go-plugin" "github.com/sirupsen/logrus" "github.com/spf13/pflag" "github.com/vmware-tanzu/velero/pkg/cmd/server/config" biav2 "github.com/vmware-tanzu/velero/pkg/plugin/framework/backupitemaction/v2" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" ibav1 "github.com/vmware-tanzu/velero/pkg/plugin/framework/itemblockaction/v1" riav2 "github.com/vmware-tanzu/velero/pkg/plugin/framework/restoreitemaction/v2" ) // Server serves registered plugin implementations. type Server interface { // BindFlags defines the plugin server's command-line flags // on the provided FlagSet. If you're not sure what flag set // to use, pflag.CommandLine is the default set of command-line // flags. // // This method must be called prior to calling .Serve(). BindFlags(flags *pflag.FlagSet) Server // GetConfig return the config parsed from the flags GetConfig() *config.Config // RegisterBackupItemAction registers a backup item action. Accepted format // for the plugin name is /. RegisterBackupItemAction(pluginName string, initializer common.HandlerInitializer) Server // RegisterBackupItemActions registers multiple backup item actions. RegisterBackupItemActions(map[string]common.HandlerInitializer) Server // RegisterBackupItemActionV2 registers a v2 backup item action. Accepted format // for the plugin name is /. RegisterBackupItemActionV2(pluginName string, initializer common.HandlerInitializer) Server // RegisterBackupItemActionsV2 registers multiple v2 backup item actions. RegisterBackupItemActionsV2(map[string]common.HandlerInitializer) Server // RegisterVolumeSnapshotter registers a volume snapshotter. Accepted format // for the plugin name is /. RegisterVolumeSnapshotter(pluginName string, initializer common.HandlerInitializer) Server // RegisterVolumeSnapshotters registers multiple volume snapshotters. RegisterVolumeSnapshotters(map[string]common.HandlerInitializer) Server // RegisterObjectStore registers an object store. Accepted format // for the plugin name is /. RegisterObjectStore(pluginName string, initializer common.HandlerInitializer) Server // RegisterObjectStores registers multiple object stores. RegisterObjectStores(map[string]common.HandlerInitializer) Server // RegisterRestoreItemAction registers a restore item action. Accepted format // for the plugin name is /. RegisterRestoreItemAction(pluginName string, initializer common.HandlerInitializer) Server // RegisterRestoreItemActions registers multiple restore item actions. RegisterRestoreItemActions(map[string]common.HandlerInitializer) Server // RegisterRestoreItemActionV2 registers a v2 restore item action. Accepted format // for the plugin name is /. RegisterRestoreItemActionV2(pluginName string, initializer common.HandlerInitializer) Server // RegisterRestoreItemActionsV2 registers multiple v2 restore item actions. RegisterRestoreItemActionsV2(map[string]common.HandlerInitializer) Server // RegisterDeleteItemAction registers a delete item action. Accepted format // for the plugin name is /. RegisterDeleteItemAction(pluginName string, initializer common.HandlerInitializer) Server // RegisterDeleteItemActions registers multiple Delete item actions. RegisterDeleteItemActions(map[string]common.HandlerInitializer) Server // RegisterItemBlockAction registers a ItemBlock action. Accepted format // for the plugin name is /. RegisterItemBlockAction(pluginName string, initializer common.HandlerInitializer) Server // RegisterItemBlockActions registers multiple ItemBlock actions. RegisterItemBlockActions(map[string]common.HandlerInitializer) Server // Server runs the plugin server. Serve() } // server implements Server. type server struct { config *config.Config log *logrus.Logger flagSet *pflag.FlagSet backupItemAction *BackupItemActionPlugin backupItemActionV2 *biav2.BackupItemActionPlugin volumeSnapshotter *VolumeSnapshotterPlugin objectStore *ObjectStorePlugin restoreItemAction *RestoreItemActionPlugin restoreItemActionV2 *riav2.RestoreItemActionPlugin deleteItemAction *DeleteItemActionPlugin itemBlockAction *ibav1.ItemBlockActionPlugin } // NewServer returns a new Server func NewServer() Server { log := newLogger() return &server{ config: config.GetDefaultConfig(), log: log, backupItemAction: NewBackupItemActionPlugin(common.ServerLogger(log)), backupItemActionV2: biav2.NewBackupItemActionPlugin(common.ServerLogger(log)), volumeSnapshotter: NewVolumeSnapshotterPlugin(common.ServerLogger(log)), objectStore: NewObjectStorePlugin(common.ServerLogger(log)), restoreItemAction: NewRestoreItemActionPlugin(common.ServerLogger(log)), restoreItemActionV2: riav2.NewRestoreItemActionPlugin(common.ServerLogger(log)), deleteItemAction: NewDeleteItemActionPlugin(common.ServerLogger(log)), itemBlockAction: ibav1.NewItemBlockActionPlugin(common.ServerLogger(log)), } } func (s *server) BindFlags(flags *pflag.FlagSet) Server { s.flagSet = flags s.config.BindFlags(flags) s.flagSet.ParseErrorsWhitelist.UnknownFlags = true // Velero.io word list : ignore return s } func (s *server) GetConfig() *config.Config { return s.config } func (s *server) RegisterBackupItemAction(name string, initializer common.HandlerInitializer) Server { s.backupItemAction.Register(name, initializer) return s } func (s *server) RegisterBackupItemActions(m map[string]common.HandlerInitializer) Server { for name := range m { s.RegisterBackupItemAction(name, m[name]) } return s } func (s *server) RegisterBackupItemActionV2(name string, initializer common.HandlerInitializer) Server { s.backupItemActionV2.Register(name, initializer) return s } func (s *server) RegisterBackupItemActionsV2(m map[string]common.HandlerInitializer) Server { for name := range m { s.RegisterBackupItemActionV2(name, m[name]) } return s } func (s *server) RegisterVolumeSnapshotter(name string, initializer common.HandlerInitializer) Server { s.volumeSnapshotter.Register(name, initializer) return s } func (s *server) RegisterVolumeSnapshotters(m map[string]common.HandlerInitializer) Server { for name := range m { s.RegisterVolumeSnapshotter(name, m[name]) } return s } func (s *server) RegisterObjectStore(name string, initializer common.HandlerInitializer) Server { s.objectStore.Register(name, initializer) return s } func (s *server) RegisterObjectStores(m map[string]common.HandlerInitializer) Server { for name := range m { s.RegisterObjectStore(name, m[name]) } return s } func (s *server) RegisterRestoreItemAction(name string, initializer common.HandlerInitializer) Server { s.restoreItemAction.Register(name, initializer) return s } func (s *server) RegisterRestoreItemActions(m map[string]common.HandlerInitializer) Server { for name := range m { s.RegisterRestoreItemAction(name, m[name]) } return s } func (s *server) RegisterRestoreItemActionV2(name string, initializer common.HandlerInitializer) Server { s.restoreItemActionV2.Register(name, initializer) return s } func (s *server) RegisterRestoreItemActionsV2(m map[string]common.HandlerInitializer) Server { for name := range m { s.RegisterRestoreItemActionV2(name, m[name]) } return s } func (s *server) RegisterDeleteItemAction(name string, initializer common.HandlerInitializer) Server { s.deleteItemAction.Register(name, initializer) return s } func (s *server) RegisterDeleteItemActions(m map[string]common.HandlerInitializer) Server { for name := range m { s.RegisterDeleteItemAction(name, m[name]) } return s } func (s *server) RegisterItemBlockAction(name string, initializer common.HandlerInitializer) Server { s.itemBlockAction.Register(name, initializer) return s } func (s *server) RegisterItemBlockActions(m map[string]common.HandlerInitializer) Server { for name := range m { s.RegisterItemBlockAction(name, m[name]) } return s } // getNames returns a list of PluginIdentifiers registered with plugin. func getNames(command string, kind common.PluginKind, plugin Interface) []PluginIdentifier { var pluginIdentifiers []PluginIdentifier for _, name := range plugin.Names() { id := PluginIdentifier{Command: command, Kind: kind, Name: name} pluginIdentifiers = append(pluginIdentifiers, id) } return pluginIdentifiers } func (s *server) Serve() { if s.flagSet != nil && !s.flagSet.Parsed() { s.log.Debugf("Parsing flags") if err := s.flagSet.Parse(os.Args[1:]); err != nil { s.log.Errorf("fail to parse the flags: %s", err.Error()) return } } s.log.Level = s.config.LogLevel.Parse() s.log.Debugf("Setting log level to %s", strings.ToUpper(s.log.Level.String())) command := os.Args[0] var pluginIdentifiers []PluginIdentifier pluginIdentifiers = append(pluginIdentifiers, getNames(command, common.PluginKindBackupItemAction, s.backupItemAction)...) pluginIdentifiers = append(pluginIdentifiers, getNames(command, common.PluginKindBackupItemActionV2, s.backupItemActionV2)...) pluginIdentifiers = append(pluginIdentifiers, getNames(command, common.PluginKindVolumeSnapshotter, s.volumeSnapshotter)...) pluginIdentifiers = append(pluginIdentifiers, getNames(command, common.PluginKindObjectStore, s.objectStore)...) pluginIdentifiers = append(pluginIdentifiers, getNames(command, common.PluginKindRestoreItemAction, s.restoreItemAction)...) pluginIdentifiers = append(pluginIdentifiers, getNames(command, common.PluginKindRestoreItemActionV2, s.restoreItemActionV2)...) pluginIdentifiers = append(pluginIdentifiers, getNames(command, common.PluginKindDeleteItemAction, s.deleteItemAction)...) pluginIdentifiers = append(pluginIdentifiers, getNames(command, common.PluginKindItemBlockAction, s.itemBlockAction)...) pluginLister := NewPluginLister(pluginIdentifiers...) plugin.Serve(&plugin.ServeConfig{ HandshakeConfig: Handshake(), Plugins: map[string]plugin.Plugin{ string(common.PluginKindBackupItemAction): s.backupItemAction, string(common.PluginKindBackupItemActionV2): s.backupItemActionV2, string(common.PluginKindVolumeSnapshotter): s.volumeSnapshotter, string(common.PluginKindObjectStore): s.objectStore, string(common.PluginKindPluginLister): NewPluginListerPlugin(pluginLister), string(common.PluginKindRestoreItemAction): s.restoreItemAction, string(common.PluginKindRestoreItemActionV2): s.restoreItemActionV2, string(common.PluginKindDeleteItemAction): s.deleteItemAction, string(common.PluginKindItemBlockAction): s.itemBlockAction, }, GRPCServer: plugin.DefaultGRPCServer, }) } ================================================ FILE: pkg/plugin/framework/stream_reader.go ================================================ /* Copyright 2017, 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package framework import ( "bytes" "io" ) // ReceiveFunc is a function that either returns a slice // of an arbitrary number of bytes OR an error. Returning // an io.EOF means there is no more data to be read; any // other error is considered an actual error. type ReceiveFunc func() ([]byte, error) // CloseFunc is used to signal to the source of data that // the StreamReadCloser has been closed. type CloseFunc func() error // StreamReadCloser wraps a ReceiveFunc and a CloseSendFunc // to implement io.ReadCloser. type StreamReadCloser struct { buf *bytes.Buffer receive ReceiveFunc close CloseFunc } func (s *StreamReadCloser) Read(p []byte) (n int, err error) { for { // if buf exists and holds at least as much as we're trying to read, // read from the buffer if s.buf != nil && s.buf.Len() >= len(p) { return s.buf.Read(p) } // if buf is nil, create it if s.buf == nil { s.buf = new(bytes.Buffer) } // buf exists but doesn't hold enough data to fill p, so // receive again. If we get an EOF, return what's in the // buffer; else, write the new data to the buffer and // try another read. data, err := s.receive() if err == io.EOF { return s.buf.Read(p) } if err != nil { return 0, err } if _, err := s.buf.Write(data); err != nil { return 0, err } } } func (s *StreamReadCloser) Close() error { return s.close() } ================================================ FILE: pkg/plugin/framework/stream_reader_test.go ================================================ /* Copyright 2017, 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package framework import ( "bytes" "io" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) type stringByteReceiver struct { buf *bytes.Buffer chunkSize int } func (r *stringByteReceiver) Receive() ([]byte, error) { chunk := make([]byte, r.chunkSize) n, err := r.buf.Read(chunk) if err != nil { return nil, err } return chunk[0:n], nil } func (r *stringByteReceiver) CloseSend() error { r.buf = nil return nil } func TestStreamReader(t *testing.T) { s := "hello world, it's me, streamreader!!!!!" rdr := &stringByteReceiver{ buf: bytes.NewBufferString(s), chunkSize: 3, } sr := &StreamReadCloser{ receive: rdr.Receive, close: rdr.CloseSend, } res, err := io.ReadAll(sr) require.NoError(t, err) assert.Equal(t, s, string(res)) } ================================================ FILE: pkg/plugin/framework/validation.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package framework import ( "github.com/pkg/errors" "k8s.io/apimachinery/pkg/util/sets" ) // ValidateObjectStoreConfigKeys ensures that an object store's config // is valid by making sure each `config` key is in the `validKeys` list. // The special keys "bucket" and "prefix" are always considered valid. func ValidateObjectStoreConfigKeys(config map[string]string, validKeys ...string) error { // `bucket` and `prefix` are automatically added to all object // store config by velero, so add them as valid keys. return validateConfigKeys(config, append(validKeys, "bucket", "prefix", "caCert")...) } // ValidateVolumeSnapshotterConfigKeys ensures that a volume snapshotter's // config is valid by making sure each `config` key is in the `validKeys` list. func ValidateVolumeSnapshotterConfigKeys(config map[string]string, validKeys ...string) error { return validateConfigKeys(config, validKeys...) } func validateConfigKeys(config map[string]string, validKeys ...string) error { validKeysSet := sets.NewString(validKeys...) var invalidKeys []string for k := range config { if !validKeysSet.Has(k) { invalidKeys = append(invalidKeys, k) } } if len(invalidKeys) > 0 { return errors.Errorf("config has invalid keys %v; valid keys are %v", invalidKeys, validKeys) } return nil } ================================================ FILE: pkg/plugin/framework/validation_test.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package framework import ( "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestValidateConfigKeys(t *testing.T) { require.NoError(t, validateConfigKeys(nil)) require.NoError(t, validateConfigKeys(map[string]string{})) require.NoError(t, validateConfigKeys(map[string]string{"foo": "bar"}, "foo")) require.NoError(t, validateConfigKeys(map[string]string{"foo": "bar", "bar": "baz"}, "foo", "bar")) require.Error(t, validateConfigKeys(map[string]string{"foo": "bar"})) require.Error(t, validateConfigKeys(map[string]string{"foo": "bar"}, "Foo")) require.Error(t, validateConfigKeys(map[string]string{"foo": "bar", "boo": ""}, "foo")) require.NoError(t, ValidateObjectStoreConfigKeys(map[string]string{"bucket": "foo"})) assert.Error(t, ValidateVolumeSnapshotterConfigKeys(map[string]string{"bucket": "foo"})) } ================================================ FILE: pkg/plugin/framework/volume_snapshotter.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package framework import ( "context" plugin "github.com/hashicorp/go-plugin" "google.golang.org/grpc" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" proto "github.com/vmware-tanzu/velero/pkg/plugin/generated" ) // VolumeSnapshotterPlugin is an implementation of go-plugin's Plugin // interface with support for gRPC for the cloudprovider/VolumeSnapshotter // interface. type VolumeSnapshotterPlugin struct { plugin.NetRPCUnsupportedPlugin *common.PluginBase } // GRPCClient returns a VolumeSnapshotter gRPC client. func (p *VolumeSnapshotterPlugin) GRPCClient(_ context.Context, _ *plugin.GRPCBroker, clientConn *grpc.ClientConn) (any, error) { return common.NewClientDispenser(p.ClientLogger, clientConn, newVolumeSnapshotterGRPCClient), nil } // GRPCServer registers a VolumeSnapshotter gRPC server. func (p *VolumeSnapshotterPlugin) GRPCServer(_ *plugin.GRPCBroker, server *grpc.Server) error { proto.RegisterVolumeSnapshotterServer(server, &VolumeSnapshotterGRPCServer{mux: p.ServerMux}) return nil } ================================================ FILE: pkg/plugin/framework/volume_snapshotter_client.go ================================================ /* Copyright 2017, 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package framework import ( "encoding/json" "context" "github.com/pkg/errors" "google.golang.org/grpc" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" proto "github.com/vmware-tanzu/velero/pkg/plugin/generated" ) // NewVolumeSnapshotterPlugin constructs a VolumeSnapshotterPlugin. func NewVolumeSnapshotterPlugin(options ...common.PluginOption) *VolumeSnapshotterPlugin { return &VolumeSnapshotterPlugin{ PluginBase: common.NewPluginBase(options...), } } // VolumeSnapshotterGRPCClient implements the cloudprovider.VolumeSnapshotter interface and uses a // gRPC client to make calls to the plugin server. type VolumeSnapshotterGRPCClient struct { *common.ClientBase grpcClient proto.VolumeSnapshotterClient } func newVolumeSnapshotterGRPCClient(base *common.ClientBase, clientConn *grpc.ClientConn) any { return &VolumeSnapshotterGRPCClient{ ClientBase: base, grpcClient: proto.NewVolumeSnapshotterClient(clientConn), } } // Init prepares the VolumeSnapshotter for usage using the provided map of // configuration key-value pairs. It returns an error if the VolumeSnapshotter // cannot be initialized from the provided config. func (c *VolumeSnapshotterGRPCClient) Init(config map[string]string) error { req := &proto.VolumeSnapshotterInitRequest{ Plugin: c.Plugin, Config: config, } if _, err := c.grpcClient.Init(context.Background(), req); err != nil { return common.FromGRPCError(err) } return nil } // CreateVolumeFromSnapshot creates a new block volume, initialized from the provided snapshot, // and with the specified type and IOPS (if using provisioned IOPS). func (c *VolumeSnapshotterGRPCClient) CreateVolumeFromSnapshot(snapshotID, volumeType, volumeAZ string, iops *int64) (string, error) { req := &proto.CreateVolumeRequest{ Plugin: c.Plugin, SnapshotID: snapshotID, VolumeType: volumeType, VolumeAZ: volumeAZ, } if iops == nil { req.Iops = 0 } else { req.Iops = *iops } res, err := c.grpcClient.CreateVolumeFromSnapshot(context.Background(), req) if err != nil { return "", common.FromGRPCError(err) } return res.VolumeID, nil } // GetVolumeInfo returns the type and IOPS (if using provisioned IOPS) for a specified block // volume. func (c *VolumeSnapshotterGRPCClient) GetVolumeInfo(volumeID, volumeAZ string) (string, *int64, error) { req := &proto.GetVolumeInfoRequest{ Plugin: c.Plugin, VolumeID: volumeID, VolumeAZ: volumeAZ, } res, err := c.grpcClient.GetVolumeInfo(context.Background(), req) if err != nil { return "", nil, common.FromGRPCError(err) } var iops *int64 if res.Iops != 0 { iops = &res.Iops } return res.VolumeType, iops, nil } // CreateSnapshot creates a snapshot of the specified block volume, and applies the provided // set of tags to the snapshot. func (c *VolumeSnapshotterGRPCClient) CreateSnapshot(volumeID, volumeAZ string, tags map[string]string) (string, error) { req := &proto.CreateSnapshotRequest{ Plugin: c.Plugin, VolumeID: volumeID, VolumeAZ: volumeAZ, Tags: tags, } res, err := c.grpcClient.CreateSnapshot(context.Background(), req) if err != nil { return "", common.FromGRPCError(err) } return res.SnapshotID, nil } // DeleteSnapshot deletes the specified volume snapshot. func (c *VolumeSnapshotterGRPCClient) DeleteSnapshot(snapshotID string) error { req := &proto.DeleteSnapshotRequest{ Plugin: c.Plugin, SnapshotID: snapshotID, } if _, err := c.grpcClient.DeleteSnapshot(context.Background(), req); err != nil { return common.FromGRPCError(err) } return nil } func (c *VolumeSnapshotterGRPCClient) GetVolumeID(pv runtime.Unstructured) (string, error) { encodedPV, err := json.Marshal(pv.UnstructuredContent()) if err != nil { return "", errors.WithStack(err) } req := &proto.GetVolumeIDRequest{ Plugin: c.Plugin, PersistentVolume: encodedPV, } resp, err := c.grpcClient.GetVolumeID(context.Background(), req) if err != nil { return "", common.FromGRPCError(err) } return resp.VolumeID, nil } func (c *VolumeSnapshotterGRPCClient) SetVolumeID(pv runtime.Unstructured, volumeID string) (runtime.Unstructured, error) { encodedPV, err := json.Marshal(pv.UnstructuredContent()) if err != nil { return nil, errors.WithStack(err) } req := &proto.SetVolumeIDRequest{ Plugin: c.Plugin, PersistentVolume: encodedPV, VolumeID: volumeID, } resp, err := c.grpcClient.SetVolumeID(context.Background(), req) if err != nil { return nil, common.FromGRPCError(err) } var updatedPV unstructured.Unstructured if err := json.Unmarshal(resp.PersistentVolume, &updatedPV); err != nil { return nil, errors.WithStack(err) } return &updatedPV, nil } ================================================ FILE: pkg/plugin/framework/volume_snapshotter_server.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package framework import ( "encoding/json" "context" "github.com/pkg/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" proto "github.com/vmware-tanzu/velero/pkg/plugin/generated" vsv1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/volumesnapshotter/v1" ) // VolumeSnapshotterGRPCServer implements the proto-generated VolumeSnapshotterServer interface, and accepts // gRPC calls and forwards them to an implementation of the pluggable interface. type VolumeSnapshotterGRPCServer struct { mux *common.ServerMux } func (s *VolumeSnapshotterGRPCServer) getImpl(name string) (vsv1.VolumeSnapshotter, error) { impl, err := s.mux.GetHandler(name) if err != nil { return nil, err } volumeSnapshotter, ok := impl.(vsv1.VolumeSnapshotter) if !ok { return nil, errors.Errorf("%T is not a volume snapshotter", impl) } return volumeSnapshotter, nil } // Init prepares the VolumeSnapshotter for usage using the provided map of // configuration key-value pairs. It returns an error if the VolumeSnapshotter // cannot be initialized from the provided config. func (s *VolumeSnapshotterGRPCServer) Init(ctx context.Context, req *proto.VolumeSnapshotterInitRequest) (response *proto.Empty, err error) { defer func() { if recoveredErr := common.HandlePanic(recover()); recoveredErr != nil { err = recoveredErr } }() impl, err := s.getImpl(req.Plugin) if err != nil { return nil, common.NewGRPCError(err) } if err := impl.Init(req.Config); err != nil { return nil, common.NewGRPCError(err) } return &proto.Empty{}, nil } // CreateVolumeFromSnapshot creates a new block volume, initialized from the provided snapshot, // and with the specified type and IOPS (if using provisioned IOPS). func (s *VolumeSnapshotterGRPCServer) CreateVolumeFromSnapshot(ctx context.Context, req *proto.CreateVolumeRequest) (response *proto.CreateVolumeResponse, err error) { defer func() { if recoveredErr := common.HandlePanic(recover()); recoveredErr != nil { err = recoveredErr } }() impl, err := s.getImpl(req.Plugin) if err != nil { return nil, common.NewGRPCError(err) } snapshotID := req.SnapshotID volumeType := req.VolumeType volumeAZ := req.VolumeAZ var iops *int64 if req.Iops != 0 { iops = &req.Iops } volumeID, err := impl.CreateVolumeFromSnapshot(snapshotID, volumeType, volumeAZ, iops) if err != nil { return nil, common.NewGRPCError(err) } return &proto.CreateVolumeResponse{VolumeID: volumeID}, nil } // GetVolumeInfo returns the type and IOPS (if using provisioned IOPS) for a specified block // volume. func (s *VolumeSnapshotterGRPCServer) GetVolumeInfo(ctx context.Context, req *proto.GetVolumeInfoRequest) (response *proto.GetVolumeInfoResponse, err error) { defer func() { if recoveredErr := common.HandlePanic(recover()); recoveredErr != nil { err = recoveredErr } }() impl, err := s.getImpl(req.Plugin) if err != nil { return nil, common.NewGRPCError(err) } volumeType, iops, err := impl.GetVolumeInfo(req.VolumeID, req.VolumeAZ) if err != nil { return nil, common.NewGRPCError(err) } res := &proto.GetVolumeInfoResponse{ VolumeType: volumeType, } if iops != nil { res.Iops = *iops } return res, nil } // CreateSnapshot creates a snapshot of the specified block volume, and applies the provided // set of tags to the snapshot. func (s *VolumeSnapshotterGRPCServer) CreateSnapshot(ctx context.Context, req *proto.CreateSnapshotRequest) (response *proto.CreateSnapshotResponse, err error) { defer func() { if recoveredErr := common.HandlePanic(recover()); recoveredErr != nil { err = recoveredErr } }() impl, err := s.getImpl(req.Plugin) if err != nil { return nil, common.NewGRPCError(err) } snapshotID, err := impl.CreateSnapshot(req.VolumeID, req.VolumeAZ, req.Tags) if err != nil { return nil, common.NewGRPCError(err) } return &proto.CreateSnapshotResponse{SnapshotID: snapshotID}, nil } // DeleteSnapshot deletes the specified volume snapshot. func (s *VolumeSnapshotterGRPCServer) DeleteSnapshot(ctx context.Context, req *proto.DeleteSnapshotRequest) (response *proto.Empty, err error) { defer func() { if recoveredErr := common.HandlePanic(recover()); recoveredErr != nil { err = recoveredErr } }() impl, err := s.getImpl(req.Plugin) if err != nil { return nil, common.NewGRPCError(err) } if err := impl.DeleteSnapshot(req.SnapshotID); err != nil { return nil, common.NewGRPCError(err) } return &proto.Empty{}, nil } func (s *VolumeSnapshotterGRPCServer) GetVolumeID(ctx context.Context, req *proto.GetVolumeIDRequest) (response *proto.GetVolumeIDResponse, err error) { defer func() { if recoveredErr := common.HandlePanic(recover()); recoveredErr != nil { err = recoveredErr } }() impl, err := s.getImpl(req.Plugin) if err != nil { return nil, common.NewGRPCError(err) } var pv unstructured.Unstructured if err := json.Unmarshal(req.PersistentVolume, &pv); err != nil { return nil, common.NewGRPCError(errors.WithStack(err)) } volumeID, err := impl.GetVolumeID(&pv) if err != nil { return nil, common.NewGRPCError(err) } return &proto.GetVolumeIDResponse{VolumeID: volumeID}, nil } func (s *VolumeSnapshotterGRPCServer) SetVolumeID(ctx context.Context, req *proto.SetVolumeIDRequest) (response *proto.SetVolumeIDResponse, err error) { defer func() { if recoveredErr := common.HandlePanic(recover()); recoveredErr != nil { err = recoveredErr } }() impl, err := s.getImpl(req.Plugin) if err != nil { return nil, common.NewGRPCError(err) } var pv unstructured.Unstructured if err := json.Unmarshal(req.PersistentVolume, &pv); err != nil { return nil, common.NewGRPCError(errors.WithStack(err)) } updatedPV, err := impl.SetVolumeID(&pv, req.VolumeID) if err != nil { return nil, common.NewGRPCError(err) } updatedPVBytes, err := json.Marshal(updatedPV.UnstructuredContent()) if err != nil { return nil, common.NewGRPCError(err) } return &proto.SetVolumeIDResponse{PersistentVolume: updatedPVBytes}, nil } ================================================ FILE: pkg/plugin/generated/BackupItemAction.pb.go ================================================ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.33.0 // protoc v4.25.2 // source: BackupItemAction.proto package generated import ( protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" sync "sync" ) const ( // Verify that this generated code is sufficiently up-to-date. _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) // Verify that runtime/protoimpl is sufficiently up-to-date. _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) type ExecuteRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` Item []byte `protobuf:"bytes,2,opt,name=item,proto3" json:"item,omitempty"` Backup []byte `protobuf:"bytes,3,opt,name=backup,proto3" json:"backup,omitempty"` } func (x *ExecuteRequest) Reset() { *x = ExecuteRequest{} if protoimpl.UnsafeEnabled { mi := &file_BackupItemAction_proto_msgTypes[0] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *ExecuteRequest) String() string { return protoimpl.X.MessageStringOf(x) } func (*ExecuteRequest) ProtoMessage() {} func (x *ExecuteRequest) ProtoReflect() protoreflect.Message { mi := &file_BackupItemAction_proto_msgTypes[0] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use ExecuteRequest.ProtoReflect.Descriptor instead. func (*ExecuteRequest) Descriptor() ([]byte, []int) { return file_BackupItemAction_proto_rawDescGZIP(), []int{0} } func (x *ExecuteRequest) GetPlugin() string { if x != nil { return x.Plugin } return "" } func (x *ExecuteRequest) GetItem() []byte { if x != nil { return x.Item } return nil } func (x *ExecuteRequest) GetBackup() []byte { if x != nil { return x.Backup } return nil } type ExecuteResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Item []byte `protobuf:"bytes,1,opt,name=item,proto3" json:"item,omitempty"` AdditionalItems []*ResourceIdentifier `protobuf:"bytes,2,rep,name=additionalItems,proto3" json:"additionalItems,omitempty"` } func (x *ExecuteResponse) Reset() { *x = ExecuteResponse{} if protoimpl.UnsafeEnabled { mi := &file_BackupItemAction_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *ExecuteResponse) String() string { return protoimpl.X.MessageStringOf(x) } func (*ExecuteResponse) ProtoMessage() {} func (x *ExecuteResponse) ProtoReflect() protoreflect.Message { mi := &file_BackupItemAction_proto_msgTypes[1] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use ExecuteResponse.ProtoReflect.Descriptor instead. func (*ExecuteResponse) Descriptor() ([]byte, []int) { return file_BackupItemAction_proto_rawDescGZIP(), []int{1} } func (x *ExecuteResponse) GetItem() []byte { if x != nil { return x.Item } return nil } func (x *ExecuteResponse) GetAdditionalItems() []*ResourceIdentifier { if x != nil { return x.AdditionalItems } return nil } type BackupItemActionAppliesToRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` } func (x *BackupItemActionAppliesToRequest) Reset() { *x = BackupItemActionAppliesToRequest{} if protoimpl.UnsafeEnabled { mi := &file_BackupItemAction_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *BackupItemActionAppliesToRequest) String() string { return protoimpl.X.MessageStringOf(x) } func (*BackupItemActionAppliesToRequest) ProtoMessage() {} func (x *BackupItemActionAppliesToRequest) ProtoReflect() protoreflect.Message { mi := &file_BackupItemAction_proto_msgTypes[2] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use BackupItemActionAppliesToRequest.ProtoReflect.Descriptor instead. func (*BackupItemActionAppliesToRequest) Descriptor() ([]byte, []int) { return file_BackupItemAction_proto_rawDescGZIP(), []int{2} } func (x *BackupItemActionAppliesToRequest) GetPlugin() string { if x != nil { return x.Plugin } return "" } type BackupItemActionAppliesToResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields ResourceSelector *ResourceSelector `protobuf:"bytes,1,opt,name=ResourceSelector,proto3" json:"ResourceSelector,omitempty"` } func (x *BackupItemActionAppliesToResponse) Reset() { *x = BackupItemActionAppliesToResponse{} if protoimpl.UnsafeEnabled { mi := &file_BackupItemAction_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *BackupItemActionAppliesToResponse) String() string { return protoimpl.X.MessageStringOf(x) } func (*BackupItemActionAppliesToResponse) ProtoMessage() {} func (x *BackupItemActionAppliesToResponse) ProtoReflect() protoreflect.Message { mi := &file_BackupItemAction_proto_msgTypes[3] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use BackupItemActionAppliesToResponse.ProtoReflect.Descriptor instead. func (*BackupItemActionAppliesToResponse) Descriptor() ([]byte, []int) { return file_BackupItemAction_proto_rawDescGZIP(), []int{3} } func (x *BackupItemActionAppliesToResponse) GetResourceSelector() *ResourceSelector { if x != nil { return x.ResourceSelector } return nil } var File_BackupItemAction_proto protoreflect.FileDescriptor var file_BackupItemAction_proto_rawDesc = []byte{ 0x0a, 0x16, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x09, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x1a, 0x0c, 0x53, 0x68, 0x61, 0x72, 0x65, 0x64, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x54, 0x0a, 0x0e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x22, 0x6e, 0x0a, 0x0f, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x12, 0x47, 0x0a, 0x0f, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x52, 0x0f, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x22, 0x3a, 0x0a, 0x20, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x22, 0x6c, 0x0a, 0x21, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x47, 0x0a, 0x10, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x10, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x32, 0xbc, 0x01, 0x0a, 0x10, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x66, 0x0a, 0x09, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x12, 0x2b, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x40, 0x0a, 0x07, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x12, 0x19, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x35, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x76, 0x6d, 0x77, 0x61, 0x72, 0x65, 0x2d, 0x74, 0x61, 0x6e, 0x7a, 0x75, 0x2f, 0x76, 0x65, 0x6c, 0x65, 0x72, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x2f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( file_BackupItemAction_proto_rawDescOnce sync.Once file_BackupItemAction_proto_rawDescData = file_BackupItemAction_proto_rawDesc ) func file_BackupItemAction_proto_rawDescGZIP() []byte { file_BackupItemAction_proto_rawDescOnce.Do(func() { file_BackupItemAction_proto_rawDescData = protoimpl.X.CompressGZIP(file_BackupItemAction_proto_rawDescData) }) return file_BackupItemAction_proto_rawDescData } var file_BackupItemAction_proto_msgTypes = make([]protoimpl.MessageInfo, 4) var file_BackupItemAction_proto_goTypes = []interface{}{ (*ExecuteRequest)(nil), // 0: generated.ExecuteRequest (*ExecuteResponse)(nil), // 1: generated.ExecuteResponse (*BackupItemActionAppliesToRequest)(nil), // 2: generated.BackupItemActionAppliesToRequest (*BackupItemActionAppliesToResponse)(nil), // 3: generated.BackupItemActionAppliesToResponse (*ResourceIdentifier)(nil), // 4: generated.ResourceIdentifier (*ResourceSelector)(nil), // 5: generated.ResourceSelector } var file_BackupItemAction_proto_depIdxs = []int32{ 4, // 0: generated.ExecuteResponse.additionalItems:type_name -> generated.ResourceIdentifier 5, // 1: generated.BackupItemActionAppliesToResponse.ResourceSelector:type_name -> generated.ResourceSelector 2, // 2: generated.BackupItemAction.AppliesTo:input_type -> generated.BackupItemActionAppliesToRequest 0, // 3: generated.BackupItemAction.Execute:input_type -> generated.ExecuteRequest 3, // 4: generated.BackupItemAction.AppliesTo:output_type -> generated.BackupItemActionAppliesToResponse 1, // 5: generated.BackupItemAction.Execute:output_type -> generated.ExecuteResponse 4, // [4:6] is the sub-list for method output_type 2, // [2:4] is the sub-list for method input_type 2, // [2:2] is the sub-list for extension type_name 2, // [2:2] is the sub-list for extension extendee 0, // [0:2] is the sub-list for field type_name } func init() { file_BackupItemAction_proto_init() } func file_BackupItemAction_proto_init() { if File_BackupItemAction_proto != nil { return } file_Shared_proto_init() if !protoimpl.UnsafeEnabled { file_BackupItemAction_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ExecuteRequest); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_BackupItemAction_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ExecuteResponse); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_BackupItemAction_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*BackupItemActionAppliesToRequest); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_BackupItemAction_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*BackupItemActionAppliesToResponse); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_BackupItemAction_proto_rawDesc, NumEnums: 0, NumMessages: 4, NumExtensions: 0, NumServices: 1, }, GoTypes: file_BackupItemAction_proto_goTypes, DependencyIndexes: file_BackupItemAction_proto_depIdxs, MessageInfos: file_BackupItemAction_proto_msgTypes, }.Build() File_BackupItemAction_proto = out.File file_BackupItemAction_proto_rawDesc = nil file_BackupItemAction_proto_goTypes = nil file_BackupItemAction_proto_depIdxs = nil } ================================================ FILE: pkg/plugin/generated/BackupItemAction_grpc.pb.go ================================================ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.3.0 // - protoc v4.25.2 // source: BackupItemAction.proto package generated import ( context "context" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" ) // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. // Requires gRPC-Go v1.32.0 or later. const _ = grpc.SupportPackageIsVersion7 const ( BackupItemAction_AppliesTo_FullMethodName = "/generated.BackupItemAction/AppliesTo" BackupItemAction_Execute_FullMethodName = "/generated.BackupItemAction/Execute" ) // BackupItemActionClient is the client API for BackupItemAction service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. type BackupItemActionClient interface { AppliesTo(ctx context.Context, in *BackupItemActionAppliesToRequest, opts ...grpc.CallOption) (*BackupItemActionAppliesToResponse, error) Execute(ctx context.Context, in *ExecuteRequest, opts ...grpc.CallOption) (*ExecuteResponse, error) } type backupItemActionClient struct { cc grpc.ClientConnInterface } func NewBackupItemActionClient(cc grpc.ClientConnInterface) BackupItemActionClient { return &backupItemActionClient{cc} } func (c *backupItemActionClient) AppliesTo(ctx context.Context, in *BackupItemActionAppliesToRequest, opts ...grpc.CallOption) (*BackupItemActionAppliesToResponse, error) { out := new(BackupItemActionAppliesToResponse) err := c.cc.Invoke(ctx, BackupItemAction_AppliesTo_FullMethodName, in, out, opts...) if err != nil { return nil, err } return out, nil } func (c *backupItemActionClient) Execute(ctx context.Context, in *ExecuteRequest, opts ...grpc.CallOption) (*ExecuteResponse, error) { out := new(ExecuteResponse) err := c.cc.Invoke(ctx, BackupItemAction_Execute_FullMethodName, in, out, opts...) if err != nil { return nil, err } return out, nil } // BackupItemActionServer is the server API for BackupItemAction service. // All implementations should embed UnimplementedBackupItemActionServer // for forward compatibility type BackupItemActionServer interface { AppliesTo(context.Context, *BackupItemActionAppliesToRequest) (*BackupItemActionAppliesToResponse, error) Execute(context.Context, *ExecuteRequest) (*ExecuteResponse, error) } // UnimplementedBackupItemActionServer should be embedded to have forward compatible implementations. type UnimplementedBackupItemActionServer struct { } func (UnimplementedBackupItemActionServer) AppliesTo(context.Context, *BackupItemActionAppliesToRequest) (*BackupItemActionAppliesToResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method AppliesTo not implemented") } func (UnimplementedBackupItemActionServer) Execute(context.Context, *ExecuteRequest) (*ExecuteResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method Execute not implemented") } // UnsafeBackupItemActionServer may be embedded to opt out of forward compatibility for this service. // Use of this interface is not recommended, as added methods to BackupItemActionServer will // result in compilation errors. type UnsafeBackupItemActionServer interface { mustEmbedUnimplementedBackupItemActionServer() } func RegisterBackupItemActionServer(s grpc.ServiceRegistrar, srv BackupItemActionServer) { s.RegisterService(&BackupItemAction_ServiceDesc, srv) } func _BackupItemAction_AppliesTo_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(BackupItemActionAppliesToRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(BackupItemActionServer).AppliesTo(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: BackupItemAction_AppliesTo_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(BackupItemActionServer).AppliesTo(ctx, req.(*BackupItemActionAppliesToRequest)) } return interceptor(ctx, in, info, handler) } func _BackupItemAction_Execute_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(ExecuteRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(BackupItemActionServer).Execute(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: BackupItemAction_Execute_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(BackupItemActionServer).Execute(ctx, req.(*ExecuteRequest)) } return interceptor(ctx, in, info, handler) } // BackupItemAction_ServiceDesc is the grpc.ServiceDesc for BackupItemAction service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) var BackupItemAction_ServiceDesc = grpc.ServiceDesc{ ServiceName: "generated.BackupItemAction", HandlerType: (*BackupItemActionServer)(nil), Methods: []grpc.MethodDesc{ { MethodName: "AppliesTo", Handler: _BackupItemAction_AppliesTo_Handler, }, { MethodName: "Execute", Handler: _BackupItemAction_Execute_Handler, }, }, Streams: []grpc.StreamDesc{}, Metadata: "BackupItemAction.proto", } ================================================ FILE: pkg/plugin/generated/DeleteItemAction.pb.go ================================================ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.33.0 // protoc v4.25.2 // source: DeleteItemAction.proto package generated import ( protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" sync "sync" ) const ( // Verify that this generated code is sufficiently up-to-date. _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) // Verify that runtime/protoimpl is sufficiently up-to-date. _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) type DeleteItemActionExecuteRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` Item []byte `protobuf:"bytes,2,opt,name=item,proto3" json:"item,omitempty"` Backup []byte `protobuf:"bytes,3,opt,name=backup,proto3" json:"backup,omitempty"` } func (x *DeleteItemActionExecuteRequest) Reset() { *x = DeleteItemActionExecuteRequest{} if protoimpl.UnsafeEnabled { mi := &file_DeleteItemAction_proto_msgTypes[0] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *DeleteItemActionExecuteRequest) String() string { return protoimpl.X.MessageStringOf(x) } func (*DeleteItemActionExecuteRequest) ProtoMessage() {} func (x *DeleteItemActionExecuteRequest) ProtoReflect() protoreflect.Message { mi := &file_DeleteItemAction_proto_msgTypes[0] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use DeleteItemActionExecuteRequest.ProtoReflect.Descriptor instead. func (*DeleteItemActionExecuteRequest) Descriptor() ([]byte, []int) { return file_DeleteItemAction_proto_rawDescGZIP(), []int{0} } func (x *DeleteItemActionExecuteRequest) GetPlugin() string { if x != nil { return x.Plugin } return "" } func (x *DeleteItemActionExecuteRequest) GetItem() []byte { if x != nil { return x.Item } return nil } func (x *DeleteItemActionExecuteRequest) GetBackup() []byte { if x != nil { return x.Backup } return nil } type DeleteItemActionAppliesToRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` } func (x *DeleteItemActionAppliesToRequest) Reset() { *x = DeleteItemActionAppliesToRequest{} if protoimpl.UnsafeEnabled { mi := &file_DeleteItemAction_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *DeleteItemActionAppliesToRequest) String() string { return protoimpl.X.MessageStringOf(x) } func (*DeleteItemActionAppliesToRequest) ProtoMessage() {} func (x *DeleteItemActionAppliesToRequest) ProtoReflect() protoreflect.Message { mi := &file_DeleteItemAction_proto_msgTypes[1] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use DeleteItemActionAppliesToRequest.ProtoReflect.Descriptor instead. func (*DeleteItemActionAppliesToRequest) Descriptor() ([]byte, []int) { return file_DeleteItemAction_proto_rawDescGZIP(), []int{1} } func (x *DeleteItemActionAppliesToRequest) GetPlugin() string { if x != nil { return x.Plugin } return "" } type DeleteItemActionAppliesToResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields ResourceSelector *ResourceSelector `protobuf:"bytes,1,opt,name=ResourceSelector,proto3" json:"ResourceSelector,omitempty"` } func (x *DeleteItemActionAppliesToResponse) Reset() { *x = DeleteItemActionAppliesToResponse{} if protoimpl.UnsafeEnabled { mi := &file_DeleteItemAction_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *DeleteItemActionAppliesToResponse) String() string { return protoimpl.X.MessageStringOf(x) } func (*DeleteItemActionAppliesToResponse) ProtoMessage() {} func (x *DeleteItemActionAppliesToResponse) ProtoReflect() protoreflect.Message { mi := &file_DeleteItemAction_proto_msgTypes[2] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use DeleteItemActionAppliesToResponse.ProtoReflect.Descriptor instead. func (*DeleteItemActionAppliesToResponse) Descriptor() ([]byte, []int) { return file_DeleteItemAction_proto_rawDescGZIP(), []int{2} } func (x *DeleteItemActionAppliesToResponse) GetResourceSelector() *ResourceSelector { if x != nil { return x.ResourceSelector } return nil } var File_DeleteItemAction_proto protoreflect.FileDescriptor var file_DeleteItemAction_proto_rawDesc = []byte{ 0x0a, 0x16, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x09, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x1a, 0x0c, 0x53, 0x68, 0x61, 0x72, 0x65, 0x64, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x64, 0x0a, 0x1e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x22, 0x3a, 0x0a, 0x20, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x22, 0x6c, 0x0a, 0x21, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x47, 0x0a, 0x10, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x10, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x32, 0xc2, 0x01, 0x0a, 0x10, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x66, 0x0a, 0x09, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x12, 0x2b, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x46, 0x0a, 0x07, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x12, 0x29, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x42, 0x35, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x76, 0x6d, 0x77, 0x61, 0x72, 0x65, 0x2d, 0x74, 0x61, 0x6e, 0x7a, 0x75, 0x2f, 0x76, 0x65, 0x6c, 0x65, 0x72, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x2f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( file_DeleteItemAction_proto_rawDescOnce sync.Once file_DeleteItemAction_proto_rawDescData = file_DeleteItemAction_proto_rawDesc ) func file_DeleteItemAction_proto_rawDescGZIP() []byte { file_DeleteItemAction_proto_rawDescOnce.Do(func() { file_DeleteItemAction_proto_rawDescData = protoimpl.X.CompressGZIP(file_DeleteItemAction_proto_rawDescData) }) return file_DeleteItemAction_proto_rawDescData } var file_DeleteItemAction_proto_msgTypes = make([]protoimpl.MessageInfo, 3) var file_DeleteItemAction_proto_goTypes = []interface{}{ (*DeleteItemActionExecuteRequest)(nil), // 0: generated.DeleteItemActionExecuteRequest (*DeleteItemActionAppliesToRequest)(nil), // 1: generated.DeleteItemActionAppliesToRequest (*DeleteItemActionAppliesToResponse)(nil), // 2: generated.DeleteItemActionAppliesToResponse (*ResourceSelector)(nil), // 3: generated.ResourceSelector (*Empty)(nil), // 4: generated.Empty } var file_DeleteItemAction_proto_depIdxs = []int32{ 3, // 0: generated.DeleteItemActionAppliesToResponse.ResourceSelector:type_name -> generated.ResourceSelector 1, // 1: generated.DeleteItemAction.AppliesTo:input_type -> generated.DeleteItemActionAppliesToRequest 0, // 2: generated.DeleteItemAction.Execute:input_type -> generated.DeleteItemActionExecuteRequest 2, // 3: generated.DeleteItemAction.AppliesTo:output_type -> generated.DeleteItemActionAppliesToResponse 4, // 4: generated.DeleteItemAction.Execute:output_type -> generated.Empty 3, // [3:5] is the sub-list for method output_type 1, // [1:3] is the sub-list for method input_type 1, // [1:1] is the sub-list for extension type_name 1, // [1:1] is the sub-list for extension extendee 0, // [0:1] is the sub-list for field type_name } func init() { file_DeleteItemAction_proto_init() } func file_DeleteItemAction_proto_init() { if File_DeleteItemAction_proto != nil { return } file_Shared_proto_init() if !protoimpl.UnsafeEnabled { file_DeleteItemAction_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*DeleteItemActionExecuteRequest); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_DeleteItemAction_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*DeleteItemActionAppliesToRequest); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_DeleteItemAction_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*DeleteItemActionAppliesToResponse); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_DeleteItemAction_proto_rawDesc, NumEnums: 0, NumMessages: 3, NumExtensions: 0, NumServices: 1, }, GoTypes: file_DeleteItemAction_proto_goTypes, DependencyIndexes: file_DeleteItemAction_proto_depIdxs, MessageInfos: file_DeleteItemAction_proto_msgTypes, }.Build() File_DeleteItemAction_proto = out.File file_DeleteItemAction_proto_rawDesc = nil file_DeleteItemAction_proto_goTypes = nil file_DeleteItemAction_proto_depIdxs = nil } ================================================ FILE: pkg/plugin/generated/DeleteItemAction_grpc.pb.go ================================================ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.3.0 // - protoc v4.25.2 // source: DeleteItemAction.proto package generated import ( context "context" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" ) // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. // Requires gRPC-Go v1.32.0 or later. const _ = grpc.SupportPackageIsVersion7 const ( DeleteItemAction_AppliesTo_FullMethodName = "/generated.DeleteItemAction/AppliesTo" DeleteItemAction_Execute_FullMethodName = "/generated.DeleteItemAction/Execute" ) // DeleteItemActionClient is the client API for DeleteItemAction service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. type DeleteItemActionClient interface { AppliesTo(ctx context.Context, in *DeleteItemActionAppliesToRequest, opts ...grpc.CallOption) (*DeleteItemActionAppliesToResponse, error) Execute(ctx context.Context, in *DeleteItemActionExecuteRequest, opts ...grpc.CallOption) (*Empty, error) } type deleteItemActionClient struct { cc grpc.ClientConnInterface } func NewDeleteItemActionClient(cc grpc.ClientConnInterface) DeleteItemActionClient { return &deleteItemActionClient{cc} } func (c *deleteItemActionClient) AppliesTo(ctx context.Context, in *DeleteItemActionAppliesToRequest, opts ...grpc.CallOption) (*DeleteItemActionAppliesToResponse, error) { out := new(DeleteItemActionAppliesToResponse) err := c.cc.Invoke(ctx, DeleteItemAction_AppliesTo_FullMethodName, in, out, opts...) if err != nil { return nil, err } return out, nil } func (c *deleteItemActionClient) Execute(ctx context.Context, in *DeleteItemActionExecuteRequest, opts ...grpc.CallOption) (*Empty, error) { out := new(Empty) err := c.cc.Invoke(ctx, DeleteItemAction_Execute_FullMethodName, in, out, opts...) if err != nil { return nil, err } return out, nil } // DeleteItemActionServer is the server API for DeleteItemAction service. // All implementations should embed UnimplementedDeleteItemActionServer // for forward compatibility type DeleteItemActionServer interface { AppliesTo(context.Context, *DeleteItemActionAppliesToRequest) (*DeleteItemActionAppliesToResponse, error) Execute(context.Context, *DeleteItemActionExecuteRequest) (*Empty, error) } // UnimplementedDeleteItemActionServer should be embedded to have forward compatible implementations. type UnimplementedDeleteItemActionServer struct { } func (UnimplementedDeleteItemActionServer) AppliesTo(context.Context, *DeleteItemActionAppliesToRequest) (*DeleteItemActionAppliesToResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method AppliesTo not implemented") } func (UnimplementedDeleteItemActionServer) Execute(context.Context, *DeleteItemActionExecuteRequest) (*Empty, error) { return nil, status.Errorf(codes.Unimplemented, "method Execute not implemented") } // UnsafeDeleteItemActionServer may be embedded to opt out of forward compatibility for this service. // Use of this interface is not recommended, as added methods to DeleteItemActionServer will // result in compilation errors. type UnsafeDeleteItemActionServer interface { mustEmbedUnimplementedDeleteItemActionServer() } func RegisterDeleteItemActionServer(s grpc.ServiceRegistrar, srv DeleteItemActionServer) { s.RegisterService(&DeleteItemAction_ServiceDesc, srv) } func _DeleteItemAction_AppliesTo_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(DeleteItemActionAppliesToRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(DeleteItemActionServer).AppliesTo(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: DeleteItemAction_AppliesTo_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(DeleteItemActionServer).AppliesTo(ctx, req.(*DeleteItemActionAppliesToRequest)) } return interceptor(ctx, in, info, handler) } func _DeleteItemAction_Execute_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(DeleteItemActionExecuteRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(DeleteItemActionServer).Execute(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: DeleteItemAction_Execute_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(DeleteItemActionServer).Execute(ctx, req.(*DeleteItemActionExecuteRequest)) } return interceptor(ctx, in, info, handler) } // DeleteItemAction_ServiceDesc is the grpc.ServiceDesc for DeleteItemAction service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) var DeleteItemAction_ServiceDesc = grpc.ServiceDesc{ ServiceName: "generated.DeleteItemAction", HandlerType: (*DeleteItemActionServer)(nil), Methods: []grpc.MethodDesc{ { MethodName: "AppliesTo", Handler: _DeleteItemAction_AppliesTo_Handler, }, { MethodName: "Execute", Handler: _DeleteItemAction_Execute_Handler, }, }, Streams: []grpc.StreamDesc{}, Metadata: "DeleteItemAction.proto", } ================================================ FILE: pkg/plugin/generated/ObjectStore.pb.go ================================================ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.33.0 // protoc v4.25.2 // source: ObjectStore.proto package generated import ( protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" sync "sync" ) const ( // Verify that this generated code is sufficiently up-to-date. _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) // Verify that runtime/protoimpl is sufficiently up-to-date. _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) type PutObjectRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` Bucket string `protobuf:"bytes,2,opt,name=bucket,proto3" json:"bucket,omitempty"` Key string `protobuf:"bytes,3,opt,name=key,proto3" json:"key,omitempty"` Body []byte `protobuf:"bytes,4,opt,name=body,proto3" json:"body,omitempty"` } func (x *PutObjectRequest) Reset() { *x = PutObjectRequest{} if protoimpl.UnsafeEnabled { mi := &file_ObjectStore_proto_msgTypes[0] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *PutObjectRequest) String() string { return protoimpl.X.MessageStringOf(x) } func (*PutObjectRequest) ProtoMessage() {} func (x *PutObjectRequest) ProtoReflect() protoreflect.Message { mi := &file_ObjectStore_proto_msgTypes[0] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use PutObjectRequest.ProtoReflect.Descriptor instead. func (*PutObjectRequest) Descriptor() ([]byte, []int) { return file_ObjectStore_proto_rawDescGZIP(), []int{0} } func (x *PutObjectRequest) GetPlugin() string { if x != nil { return x.Plugin } return "" } func (x *PutObjectRequest) GetBucket() string { if x != nil { return x.Bucket } return "" } func (x *PutObjectRequest) GetKey() string { if x != nil { return x.Key } return "" } func (x *PutObjectRequest) GetBody() []byte { if x != nil { return x.Body } return nil } type ObjectExistsRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` Bucket string `protobuf:"bytes,2,opt,name=bucket,proto3" json:"bucket,omitempty"` Key string `protobuf:"bytes,3,opt,name=key,proto3" json:"key,omitempty"` } func (x *ObjectExistsRequest) Reset() { *x = ObjectExistsRequest{} if protoimpl.UnsafeEnabled { mi := &file_ObjectStore_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *ObjectExistsRequest) String() string { return protoimpl.X.MessageStringOf(x) } func (*ObjectExistsRequest) ProtoMessage() {} func (x *ObjectExistsRequest) ProtoReflect() protoreflect.Message { mi := &file_ObjectStore_proto_msgTypes[1] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use ObjectExistsRequest.ProtoReflect.Descriptor instead. func (*ObjectExistsRequest) Descriptor() ([]byte, []int) { return file_ObjectStore_proto_rawDescGZIP(), []int{1} } func (x *ObjectExistsRequest) GetPlugin() string { if x != nil { return x.Plugin } return "" } func (x *ObjectExistsRequest) GetBucket() string { if x != nil { return x.Bucket } return "" } func (x *ObjectExistsRequest) GetKey() string { if x != nil { return x.Key } return "" } type ObjectExistsResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Exists bool `protobuf:"varint,1,opt,name=exists,proto3" json:"exists,omitempty"` } func (x *ObjectExistsResponse) Reset() { *x = ObjectExistsResponse{} if protoimpl.UnsafeEnabled { mi := &file_ObjectStore_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *ObjectExistsResponse) String() string { return protoimpl.X.MessageStringOf(x) } func (*ObjectExistsResponse) ProtoMessage() {} func (x *ObjectExistsResponse) ProtoReflect() protoreflect.Message { mi := &file_ObjectStore_proto_msgTypes[2] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use ObjectExistsResponse.ProtoReflect.Descriptor instead. func (*ObjectExistsResponse) Descriptor() ([]byte, []int) { return file_ObjectStore_proto_rawDescGZIP(), []int{2} } func (x *ObjectExistsResponse) GetExists() bool { if x != nil { return x.Exists } return false } type GetObjectRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` Bucket string `protobuf:"bytes,2,opt,name=bucket,proto3" json:"bucket,omitempty"` Key string `protobuf:"bytes,3,opt,name=key,proto3" json:"key,omitempty"` } func (x *GetObjectRequest) Reset() { *x = GetObjectRequest{} if protoimpl.UnsafeEnabled { mi := &file_ObjectStore_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *GetObjectRequest) String() string { return protoimpl.X.MessageStringOf(x) } func (*GetObjectRequest) ProtoMessage() {} func (x *GetObjectRequest) ProtoReflect() protoreflect.Message { mi := &file_ObjectStore_proto_msgTypes[3] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use GetObjectRequest.ProtoReflect.Descriptor instead. func (*GetObjectRequest) Descriptor() ([]byte, []int) { return file_ObjectStore_proto_rawDescGZIP(), []int{3} } func (x *GetObjectRequest) GetPlugin() string { if x != nil { return x.Plugin } return "" } func (x *GetObjectRequest) GetBucket() string { if x != nil { return x.Bucket } return "" } func (x *GetObjectRequest) GetKey() string { if x != nil { return x.Key } return "" } type Bytes struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Data []byte `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` } func (x *Bytes) Reset() { *x = Bytes{} if protoimpl.UnsafeEnabled { mi := &file_ObjectStore_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *Bytes) String() string { return protoimpl.X.MessageStringOf(x) } func (*Bytes) ProtoMessage() {} func (x *Bytes) ProtoReflect() protoreflect.Message { mi := &file_ObjectStore_proto_msgTypes[4] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use Bytes.ProtoReflect.Descriptor instead. func (*Bytes) Descriptor() ([]byte, []int) { return file_ObjectStore_proto_rawDescGZIP(), []int{4} } func (x *Bytes) GetData() []byte { if x != nil { return x.Data } return nil } type ListCommonPrefixesRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` Bucket string `protobuf:"bytes,2,opt,name=bucket,proto3" json:"bucket,omitempty"` Delimiter string `protobuf:"bytes,3,opt,name=delimiter,proto3" json:"delimiter,omitempty"` Prefix string `protobuf:"bytes,4,opt,name=prefix,proto3" json:"prefix,omitempty"` } func (x *ListCommonPrefixesRequest) Reset() { *x = ListCommonPrefixesRequest{} if protoimpl.UnsafeEnabled { mi := &file_ObjectStore_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *ListCommonPrefixesRequest) String() string { return protoimpl.X.MessageStringOf(x) } func (*ListCommonPrefixesRequest) ProtoMessage() {} func (x *ListCommonPrefixesRequest) ProtoReflect() protoreflect.Message { mi := &file_ObjectStore_proto_msgTypes[5] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use ListCommonPrefixesRequest.ProtoReflect.Descriptor instead. func (*ListCommonPrefixesRequest) Descriptor() ([]byte, []int) { return file_ObjectStore_proto_rawDescGZIP(), []int{5} } func (x *ListCommonPrefixesRequest) GetPlugin() string { if x != nil { return x.Plugin } return "" } func (x *ListCommonPrefixesRequest) GetBucket() string { if x != nil { return x.Bucket } return "" } func (x *ListCommonPrefixesRequest) GetDelimiter() string { if x != nil { return x.Delimiter } return "" } func (x *ListCommonPrefixesRequest) GetPrefix() string { if x != nil { return x.Prefix } return "" } type ListCommonPrefixesResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Prefixes []string `protobuf:"bytes,1,rep,name=prefixes,proto3" json:"prefixes,omitempty"` } func (x *ListCommonPrefixesResponse) Reset() { *x = ListCommonPrefixesResponse{} if protoimpl.UnsafeEnabled { mi := &file_ObjectStore_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *ListCommonPrefixesResponse) String() string { return protoimpl.X.MessageStringOf(x) } func (*ListCommonPrefixesResponse) ProtoMessage() {} func (x *ListCommonPrefixesResponse) ProtoReflect() protoreflect.Message { mi := &file_ObjectStore_proto_msgTypes[6] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use ListCommonPrefixesResponse.ProtoReflect.Descriptor instead. func (*ListCommonPrefixesResponse) Descriptor() ([]byte, []int) { return file_ObjectStore_proto_rawDescGZIP(), []int{6} } func (x *ListCommonPrefixesResponse) GetPrefixes() []string { if x != nil { return x.Prefixes } return nil } type ListObjectsRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` Bucket string `protobuf:"bytes,2,opt,name=bucket,proto3" json:"bucket,omitempty"` Prefix string `protobuf:"bytes,3,opt,name=prefix,proto3" json:"prefix,omitempty"` } func (x *ListObjectsRequest) Reset() { *x = ListObjectsRequest{} if protoimpl.UnsafeEnabled { mi := &file_ObjectStore_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *ListObjectsRequest) String() string { return protoimpl.X.MessageStringOf(x) } func (*ListObjectsRequest) ProtoMessage() {} func (x *ListObjectsRequest) ProtoReflect() protoreflect.Message { mi := &file_ObjectStore_proto_msgTypes[7] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use ListObjectsRequest.ProtoReflect.Descriptor instead. func (*ListObjectsRequest) Descriptor() ([]byte, []int) { return file_ObjectStore_proto_rawDescGZIP(), []int{7} } func (x *ListObjectsRequest) GetPlugin() string { if x != nil { return x.Plugin } return "" } func (x *ListObjectsRequest) GetBucket() string { if x != nil { return x.Bucket } return "" } func (x *ListObjectsRequest) GetPrefix() string { if x != nil { return x.Prefix } return "" } type ListObjectsResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Keys []string `protobuf:"bytes,1,rep,name=keys,proto3" json:"keys,omitempty"` } func (x *ListObjectsResponse) Reset() { *x = ListObjectsResponse{} if protoimpl.UnsafeEnabled { mi := &file_ObjectStore_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *ListObjectsResponse) String() string { return protoimpl.X.MessageStringOf(x) } func (*ListObjectsResponse) ProtoMessage() {} func (x *ListObjectsResponse) ProtoReflect() protoreflect.Message { mi := &file_ObjectStore_proto_msgTypes[8] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use ListObjectsResponse.ProtoReflect.Descriptor instead. func (*ListObjectsResponse) Descriptor() ([]byte, []int) { return file_ObjectStore_proto_rawDescGZIP(), []int{8} } func (x *ListObjectsResponse) GetKeys() []string { if x != nil { return x.Keys } return nil } type DeleteObjectRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` Bucket string `protobuf:"bytes,2,opt,name=bucket,proto3" json:"bucket,omitempty"` Key string `protobuf:"bytes,3,opt,name=key,proto3" json:"key,omitempty"` } func (x *DeleteObjectRequest) Reset() { *x = DeleteObjectRequest{} if protoimpl.UnsafeEnabled { mi := &file_ObjectStore_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *DeleteObjectRequest) String() string { return protoimpl.X.MessageStringOf(x) } func (*DeleteObjectRequest) ProtoMessage() {} func (x *DeleteObjectRequest) ProtoReflect() protoreflect.Message { mi := &file_ObjectStore_proto_msgTypes[9] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use DeleteObjectRequest.ProtoReflect.Descriptor instead. func (*DeleteObjectRequest) Descriptor() ([]byte, []int) { return file_ObjectStore_proto_rawDescGZIP(), []int{9} } func (x *DeleteObjectRequest) GetPlugin() string { if x != nil { return x.Plugin } return "" } func (x *DeleteObjectRequest) GetBucket() string { if x != nil { return x.Bucket } return "" } func (x *DeleteObjectRequest) GetKey() string { if x != nil { return x.Key } return "" } type CreateSignedURLRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` Bucket string `protobuf:"bytes,2,opt,name=bucket,proto3" json:"bucket,omitempty"` Key string `protobuf:"bytes,3,opt,name=key,proto3" json:"key,omitempty"` Ttl int64 `protobuf:"varint,4,opt,name=ttl,proto3" json:"ttl,omitempty"` } func (x *CreateSignedURLRequest) Reset() { *x = CreateSignedURLRequest{} if protoimpl.UnsafeEnabled { mi := &file_ObjectStore_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *CreateSignedURLRequest) String() string { return protoimpl.X.MessageStringOf(x) } func (*CreateSignedURLRequest) ProtoMessage() {} func (x *CreateSignedURLRequest) ProtoReflect() protoreflect.Message { mi := &file_ObjectStore_proto_msgTypes[10] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use CreateSignedURLRequest.ProtoReflect.Descriptor instead. func (*CreateSignedURLRequest) Descriptor() ([]byte, []int) { return file_ObjectStore_proto_rawDescGZIP(), []int{10} } func (x *CreateSignedURLRequest) GetPlugin() string { if x != nil { return x.Plugin } return "" } func (x *CreateSignedURLRequest) GetBucket() string { if x != nil { return x.Bucket } return "" } func (x *CreateSignedURLRequest) GetKey() string { if x != nil { return x.Key } return "" } func (x *CreateSignedURLRequest) GetTtl() int64 { if x != nil { return x.Ttl } return 0 } type CreateSignedURLResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Url string `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"` } func (x *CreateSignedURLResponse) Reset() { *x = CreateSignedURLResponse{} if protoimpl.UnsafeEnabled { mi := &file_ObjectStore_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *CreateSignedURLResponse) String() string { return protoimpl.X.MessageStringOf(x) } func (*CreateSignedURLResponse) ProtoMessage() {} func (x *CreateSignedURLResponse) ProtoReflect() protoreflect.Message { mi := &file_ObjectStore_proto_msgTypes[11] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use CreateSignedURLResponse.ProtoReflect.Descriptor instead. func (*CreateSignedURLResponse) Descriptor() ([]byte, []int) { return file_ObjectStore_proto_rawDescGZIP(), []int{11} } func (x *CreateSignedURLResponse) GetUrl() string { if x != nil { return x.Url } return "" } type ObjectStoreInitRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` Config map[string]string `protobuf:"bytes,2,rep,name=config,proto3" json:"config,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` } func (x *ObjectStoreInitRequest) Reset() { *x = ObjectStoreInitRequest{} if protoimpl.UnsafeEnabled { mi := &file_ObjectStore_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *ObjectStoreInitRequest) String() string { return protoimpl.X.MessageStringOf(x) } func (*ObjectStoreInitRequest) ProtoMessage() {} func (x *ObjectStoreInitRequest) ProtoReflect() protoreflect.Message { mi := &file_ObjectStore_proto_msgTypes[12] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use ObjectStoreInitRequest.ProtoReflect.Descriptor instead. func (*ObjectStoreInitRequest) Descriptor() ([]byte, []int) { return file_ObjectStore_proto_rawDescGZIP(), []int{12} } func (x *ObjectStoreInitRequest) GetPlugin() string { if x != nil { return x.Plugin } return "" } func (x *ObjectStoreInitRequest) GetConfig() map[string]string { if x != nil { return x.Config } return nil } var File_ObjectStore_proto protoreflect.FileDescriptor var file_ObjectStore_proto_rawDesc = []byte{ 0x0a, 0x11, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x09, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x1a, 0x0c, 0x53, 0x68, 0x61, 0x72, 0x65, 0x64, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x68, 0x0a, 0x10, 0x50, 0x75, 0x74, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x22, 0x57, 0x0a, 0x13, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x45, 0x78, 0x69, 0x73, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x22, 0x2e, 0x0a, 0x14, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x45, 0x78, 0x69, 0x73, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x65, 0x78, 0x69, 0x73, 0x74, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x65, 0x78, 0x69, 0x73, 0x74, 0x73, 0x22, 0x54, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x22, 0x1b, 0x0a, 0x05, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0x81, 0x01, 0x0a, 0x19, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x64, 0x65, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x64, 0x65, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x22, 0x38, 0x0a, 0x1a, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, 0x73, 0x22, 0x5c, 0x0a, 0x12, 0x4c, 0x69, 0x73, 0x74, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x22, 0x29, 0x0a, 0x13, 0x4c, 0x69, 0x73, 0x74, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x22, 0x57, 0x0a, 0x13, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x22, 0x6c, 0x0a, 0x16, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, 0x69, 0x67, 0x6e, 0x65, 0x64, 0x55, 0x52, 0x4c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x74, 0x74, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x03, 0x74, 0x74, 0x6c, 0x22, 0x2b, 0x0a, 0x17, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, 0x69, 0x67, 0x6e, 0x65, 0x64, 0x55, 0x52, 0x4c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x22, 0xb2, 0x01, 0x0a, 0x16, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x45, 0x0a, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x1a, 0x39, 0x0a, 0x0b, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x32, 0xe4, 0x04, 0x0a, 0x0b, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x3b, 0x0a, 0x04, 0x49, 0x6e, 0x69, 0x74, 0x12, 0x21, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x3c, 0x0a, 0x09, 0x50, 0x75, 0x74, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x1b, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x50, 0x75, 0x74, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x28, 0x01, 0x12, 0x4f, 0x0a, 0x0c, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x45, 0x78, 0x69, 0x73, 0x74, 0x73, 0x12, 0x1e, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x45, 0x78, 0x69, 0x73, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x45, 0x78, 0x69, 0x73, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3c, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x1b, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x42, 0x79, 0x74, 0x65, 0x73, 0x30, 0x01, 0x12, 0x61, 0x0a, 0x12, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, 0x73, 0x12, 0x24, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4c, 0x0a, 0x0b, 0x4c, 0x69, 0x73, 0x74, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x1d, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x40, 0x0a, 0x0c, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x1e, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x58, 0x0a, 0x0f, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, 0x69, 0x67, 0x6e, 0x65, 0x64, 0x55, 0x52, 0x4c, 0x12, 0x21, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, 0x69, 0x67, 0x6e, 0x65, 0x64, 0x55, 0x52, 0x4c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, 0x69, 0x67, 0x6e, 0x65, 0x64, 0x55, 0x52, 0x4c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x35, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x76, 0x6d, 0x77, 0x61, 0x72, 0x65, 0x2d, 0x74, 0x61, 0x6e, 0x7a, 0x75, 0x2f, 0x76, 0x65, 0x6c, 0x65, 0x72, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x2f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( file_ObjectStore_proto_rawDescOnce sync.Once file_ObjectStore_proto_rawDescData = file_ObjectStore_proto_rawDesc ) func file_ObjectStore_proto_rawDescGZIP() []byte { file_ObjectStore_proto_rawDescOnce.Do(func() { file_ObjectStore_proto_rawDescData = protoimpl.X.CompressGZIP(file_ObjectStore_proto_rawDescData) }) return file_ObjectStore_proto_rawDescData } var file_ObjectStore_proto_msgTypes = make([]protoimpl.MessageInfo, 14) var file_ObjectStore_proto_goTypes = []interface{}{ (*PutObjectRequest)(nil), // 0: generated.PutObjectRequest (*ObjectExistsRequest)(nil), // 1: generated.ObjectExistsRequest (*ObjectExistsResponse)(nil), // 2: generated.ObjectExistsResponse (*GetObjectRequest)(nil), // 3: generated.GetObjectRequest (*Bytes)(nil), // 4: generated.Bytes (*ListCommonPrefixesRequest)(nil), // 5: generated.ListCommonPrefixesRequest (*ListCommonPrefixesResponse)(nil), // 6: generated.ListCommonPrefixesResponse (*ListObjectsRequest)(nil), // 7: generated.ListObjectsRequest (*ListObjectsResponse)(nil), // 8: generated.ListObjectsResponse (*DeleteObjectRequest)(nil), // 9: generated.DeleteObjectRequest (*CreateSignedURLRequest)(nil), // 10: generated.CreateSignedURLRequest (*CreateSignedURLResponse)(nil), // 11: generated.CreateSignedURLResponse (*ObjectStoreInitRequest)(nil), // 12: generated.ObjectStoreInitRequest nil, // 13: generated.ObjectStoreInitRequest.ConfigEntry (*Empty)(nil), // 14: generated.Empty } var file_ObjectStore_proto_depIdxs = []int32{ 13, // 0: generated.ObjectStoreInitRequest.config:type_name -> generated.ObjectStoreInitRequest.ConfigEntry 12, // 1: generated.ObjectStore.Init:input_type -> generated.ObjectStoreInitRequest 0, // 2: generated.ObjectStore.PutObject:input_type -> generated.PutObjectRequest 1, // 3: generated.ObjectStore.ObjectExists:input_type -> generated.ObjectExistsRequest 3, // 4: generated.ObjectStore.GetObject:input_type -> generated.GetObjectRequest 5, // 5: generated.ObjectStore.ListCommonPrefixes:input_type -> generated.ListCommonPrefixesRequest 7, // 6: generated.ObjectStore.ListObjects:input_type -> generated.ListObjectsRequest 9, // 7: generated.ObjectStore.DeleteObject:input_type -> generated.DeleteObjectRequest 10, // 8: generated.ObjectStore.CreateSignedURL:input_type -> generated.CreateSignedURLRequest 14, // 9: generated.ObjectStore.Init:output_type -> generated.Empty 14, // 10: generated.ObjectStore.PutObject:output_type -> generated.Empty 2, // 11: generated.ObjectStore.ObjectExists:output_type -> generated.ObjectExistsResponse 4, // 12: generated.ObjectStore.GetObject:output_type -> generated.Bytes 6, // 13: generated.ObjectStore.ListCommonPrefixes:output_type -> generated.ListCommonPrefixesResponse 8, // 14: generated.ObjectStore.ListObjects:output_type -> generated.ListObjectsResponse 14, // 15: generated.ObjectStore.DeleteObject:output_type -> generated.Empty 11, // 16: generated.ObjectStore.CreateSignedURL:output_type -> generated.CreateSignedURLResponse 9, // [9:17] is the sub-list for method output_type 1, // [1:9] is the sub-list for method input_type 1, // [1:1] is the sub-list for extension type_name 1, // [1:1] is the sub-list for extension extendee 0, // [0:1] is the sub-list for field type_name } func init() { file_ObjectStore_proto_init() } func file_ObjectStore_proto_init() { if File_ObjectStore_proto != nil { return } file_Shared_proto_init() if !protoimpl.UnsafeEnabled { file_ObjectStore_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*PutObjectRequest); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_ObjectStore_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ObjectExistsRequest); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_ObjectStore_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ObjectExistsResponse); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_ObjectStore_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*GetObjectRequest); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_ObjectStore_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Bytes); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_ObjectStore_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ListCommonPrefixesRequest); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_ObjectStore_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ListCommonPrefixesResponse); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_ObjectStore_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ListObjectsRequest); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_ObjectStore_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ListObjectsResponse); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_ObjectStore_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*DeleteObjectRequest); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_ObjectStore_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*CreateSignedURLRequest); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_ObjectStore_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*CreateSignedURLResponse); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_ObjectStore_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ObjectStoreInitRequest); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_ObjectStore_proto_rawDesc, NumEnums: 0, NumMessages: 14, NumExtensions: 0, NumServices: 1, }, GoTypes: file_ObjectStore_proto_goTypes, DependencyIndexes: file_ObjectStore_proto_depIdxs, MessageInfos: file_ObjectStore_proto_msgTypes, }.Build() File_ObjectStore_proto = out.File file_ObjectStore_proto_rawDesc = nil file_ObjectStore_proto_goTypes = nil file_ObjectStore_proto_depIdxs = nil } ================================================ FILE: pkg/plugin/generated/ObjectStore_grpc.pb.go ================================================ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.3.0 // - protoc v4.25.2 // source: ObjectStore.proto package generated import ( context "context" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" ) // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. // Requires gRPC-Go v1.32.0 or later. const _ = grpc.SupportPackageIsVersion7 const ( ObjectStore_Init_FullMethodName = "/generated.ObjectStore/Init" ObjectStore_PutObject_FullMethodName = "/generated.ObjectStore/PutObject" ObjectStore_ObjectExists_FullMethodName = "/generated.ObjectStore/ObjectExists" ObjectStore_GetObject_FullMethodName = "/generated.ObjectStore/GetObject" ObjectStore_ListCommonPrefixes_FullMethodName = "/generated.ObjectStore/ListCommonPrefixes" ObjectStore_ListObjects_FullMethodName = "/generated.ObjectStore/ListObjects" ObjectStore_DeleteObject_FullMethodName = "/generated.ObjectStore/DeleteObject" ObjectStore_CreateSignedURL_FullMethodName = "/generated.ObjectStore/CreateSignedURL" ) // ObjectStoreClient is the client API for ObjectStore service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. type ObjectStoreClient interface { Init(ctx context.Context, in *ObjectStoreInitRequest, opts ...grpc.CallOption) (*Empty, error) PutObject(ctx context.Context, opts ...grpc.CallOption) (ObjectStore_PutObjectClient, error) ObjectExists(ctx context.Context, in *ObjectExistsRequest, opts ...grpc.CallOption) (*ObjectExistsResponse, error) GetObject(ctx context.Context, in *GetObjectRequest, opts ...grpc.CallOption) (ObjectStore_GetObjectClient, error) ListCommonPrefixes(ctx context.Context, in *ListCommonPrefixesRequest, opts ...grpc.CallOption) (*ListCommonPrefixesResponse, error) ListObjects(ctx context.Context, in *ListObjectsRequest, opts ...grpc.CallOption) (*ListObjectsResponse, error) DeleteObject(ctx context.Context, in *DeleteObjectRequest, opts ...grpc.CallOption) (*Empty, error) CreateSignedURL(ctx context.Context, in *CreateSignedURLRequest, opts ...grpc.CallOption) (*CreateSignedURLResponse, error) } type objectStoreClient struct { cc grpc.ClientConnInterface } func NewObjectStoreClient(cc grpc.ClientConnInterface) ObjectStoreClient { return &objectStoreClient{cc} } func (c *objectStoreClient) Init(ctx context.Context, in *ObjectStoreInitRequest, opts ...grpc.CallOption) (*Empty, error) { out := new(Empty) err := c.cc.Invoke(ctx, ObjectStore_Init_FullMethodName, in, out, opts...) if err != nil { return nil, err } return out, nil } func (c *objectStoreClient) PutObject(ctx context.Context, opts ...grpc.CallOption) (ObjectStore_PutObjectClient, error) { stream, err := c.cc.NewStream(ctx, &ObjectStore_ServiceDesc.Streams[0], ObjectStore_PutObject_FullMethodName, opts...) if err != nil { return nil, err } x := &objectStorePutObjectClient{stream} return x, nil } type ObjectStore_PutObjectClient interface { Send(*PutObjectRequest) error CloseAndRecv() (*Empty, error) grpc.ClientStream } type objectStorePutObjectClient struct { grpc.ClientStream } func (x *objectStorePutObjectClient) Send(m *PutObjectRequest) error { return x.ClientStream.SendMsg(m) } func (x *objectStorePutObjectClient) CloseAndRecv() (*Empty, error) { if err := x.ClientStream.CloseSend(); err != nil { return nil, err } m := new(Empty) if err := x.ClientStream.RecvMsg(m); err != nil { return nil, err } return m, nil } func (c *objectStoreClient) ObjectExists(ctx context.Context, in *ObjectExistsRequest, opts ...grpc.CallOption) (*ObjectExistsResponse, error) { out := new(ObjectExistsResponse) err := c.cc.Invoke(ctx, ObjectStore_ObjectExists_FullMethodName, in, out, opts...) if err != nil { return nil, err } return out, nil } func (c *objectStoreClient) GetObject(ctx context.Context, in *GetObjectRequest, opts ...grpc.CallOption) (ObjectStore_GetObjectClient, error) { stream, err := c.cc.NewStream(ctx, &ObjectStore_ServiceDesc.Streams[1], ObjectStore_GetObject_FullMethodName, opts...) if err != nil { return nil, err } x := &objectStoreGetObjectClient{stream} if err := x.ClientStream.SendMsg(in); err != nil { return nil, err } if err := x.ClientStream.CloseSend(); err != nil { return nil, err } return x, nil } type ObjectStore_GetObjectClient interface { Recv() (*Bytes, error) grpc.ClientStream } type objectStoreGetObjectClient struct { grpc.ClientStream } func (x *objectStoreGetObjectClient) Recv() (*Bytes, error) { m := new(Bytes) if err := x.ClientStream.RecvMsg(m); err != nil { return nil, err } return m, nil } func (c *objectStoreClient) ListCommonPrefixes(ctx context.Context, in *ListCommonPrefixesRequest, opts ...grpc.CallOption) (*ListCommonPrefixesResponse, error) { out := new(ListCommonPrefixesResponse) err := c.cc.Invoke(ctx, ObjectStore_ListCommonPrefixes_FullMethodName, in, out, opts...) if err != nil { return nil, err } return out, nil } func (c *objectStoreClient) ListObjects(ctx context.Context, in *ListObjectsRequest, opts ...grpc.CallOption) (*ListObjectsResponse, error) { out := new(ListObjectsResponse) err := c.cc.Invoke(ctx, ObjectStore_ListObjects_FullMethodName, in, out, opts...) if err != nil { return nil, err } return out, nil } func (c *objectStoreClient) DeleteObject(ctx context.Context, in *DeleteObjectRequest, opts ...grpc.CallOption) (*Empty, error) { out := new(Empty) err := c.cc.Invoke(ctx, ObjectStore_DeleteObject_FullMethodName, in, out, opts...) if err != nil { return nil, err } return out, nil } func (c *objectStoreClient) CreateSignedURL(ctx context.Context, in *CreateSignedURLRequest, opts ...grpc.CallOption) (*CreateSignedURLResponse, error) { out := new(CreateSignedURLResponse) err := c.cc.Invoke(ctx, ObjectStore_CreateSignedURL_FullMethodName, in, out, opts...) if err != nil { return nil, err } return out, nil } // ObjectStoreServer is the server API for ObjectStore service. // All implementations should embed UnimplementedObjectStoreServer // for forward compatibility type ObjectStoreServer interface { Init(context.Context, *ObjectStoreInitRequest) (*Empty, error) PutObject(ObjectStore_PutObjectServer) error ObjectExists(context.Context, *ObjectExistsRequest) (*ObjectExistsResponse, error) GetObject(*GetObjectRequest, ObjectStore_GetObjectServer) error ListCommonPrefixes(context.Context, *ListCommonPrefixesRequest) (*ListCommonPrefixesResponse, error) ListObjects(context.Context, *ListObjectsRequest) (*ListObjectsResponse, error) DeleteObject(context.Context, *DeleteObjectRequest) (*Empty, error) CreateSignedURL(context.Context, *CreateSignedURLRequest) (*CreateSignedURLResponse, error) } // UnimplementedObjectStoreServer should be embedded to have forward compatible implementations. type UnimplementedObjectStoreServer struct { } func (UnimplementedObjectStoreServer) Init(context.Context, *ObjectStoreInitRequest) (*Empty, error) { return nil, status.Errorf(codes.Unimplemented, "method Init not implemented") } func (UnimplementedObjectStoreServer) PutObject(ObjectStore_PutObjectServer) error { return status.Errorf(codes.Unimplemented, "method PutObject not implemented") } func (UnimplementedObjectStoreServer) ObjectExists(context.Context, *ObjectExistsRequest) (*ObjectExistsResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method ObjectExists not implemented") } func (UnimplementedObjectStoreServer) GetObject(*GetObjectRequest, ObjectStore_GetObjectServer) error { return status.Errorf(codes.Unimplemented, "method GetObject not implemented") } func (UnimplementedObjectStoreServer) ListCommonPrefixes(context.Context, *ListCommonPrefixesRequest) (*ListCommonPrefixesResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method ListCommonPrefixes not implemented") } func (UnimplementedObjectStoreServer) ListObjects(context.Context, *ListObjectsRequest) (*ListObjectsResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method ListObjects not implemented") } func (UnimplementedObjectStoreServer) DeleteObject(context.Context, *DeleteObjectRequest) (*Empty, error) { return nil, status.Errorf(codes.Unimplemented, "method DeleteObject not implemented") } func (UnimplementedObjectStoreServer) CreateSignedURL(context.Context, *CreateSignedURLRequest) (*CreateSignedURLResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method CreateSignedURL not implemented") } // UnsafeObjectStoreServer may be embedded to opt out of forward compatibility for this service. // Use of this interface is not recommended, as added methods to ObjectStoreServer will // result in compilation errors. type UnsafeObjectStoreServer interface { mustEmbedUnimplementedObjectStoreServer() } func RegisterObjectStoreServer(s grpc.ServiceRegistrar, srv ObjectStoreServer) { s.RegisterService(&ObjectStore_ServiceDesc, srv) } func _ObjectStore_Init_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(ObjectStoreInitRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(ObjectStoreServer).Init(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: ObjectStore_Init_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ObjectStoreServer).Init(ctx, req.(*ObjectStoreInitRequest)) } return interceptor(ctx, in, info, handler) } func _ObjectStore_PutObject_Handler(srv interface{}, stream grpc.ServerStream) error { return srv.(ObjectStoreServer).PutObject(&objectStorePutObjectServer{stream}) } type ObjectStore_PutObjectServer interface { SendAndClose(*Empty) error Recv() (*PutObjectRequest, error) grpc.ServerStream } type objectStorePutObjectServer struct { grpc.ServerStream } func (x *objectStorePutObjectServer) SendAndClose(m *Empty) error { return x.ServerStream.SendMsg(m) } func (x *objectStorePutObjectServer) Recv() (*PutObjectRequest, error) { m := new(PutObjectRequest) if err := x.ServerStream.RecvMsg(m); err != nil { return nil, err } return m, nil } func _ObjectStore_ObjectExists_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(ObjectExistsRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(ObjectStoreServer).ObjectExists(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: ObjectStore_ObjectExists_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ObjectStoreServer).ObjectExists(ctx, req.(*ObjectExistsRequest)) } return interceptor(ctx, in, info, handler) } func _ObjectStore_GetObject_Handler(srv interface{}, stream grpc.ServerStream) error { m := new(GetObjectRequest) if err := stream.RecvMsg(m); err != nil { return err } return srv.(ObjectStoreServer).GetObject(m, &objectStoreGetObjectServer{stream}) } type ObjectStore_GetObjectServer interface { Send(*Bytes) error grpc.ServerStream } type objectStoreGetObjectServer struct { grpc.ServerStream } func (x *objectStoreGetObjectServer) Send(m *Bytes) error { return x.ServerStream.SendMsg(m) } func _ObjectStore_ListCommonPrefixes_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(ListCommonPrefixesRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(ObjectStoreServer).ListCommonPrefixes(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: ObjectStore_ListCommonPrefixes_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ObjectStoreServer).ListCommonPrefixes(ctx, req.(*ListCommonPrefixesRequest)) } return interceptor(ctx, in, info, handler) } func _ObjectStore_ListObjects_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(ListObjectsRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(ObjectStoreServer).ListObjects(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: ObjectStore_ListObjects_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ObjectStoreServer).ListObjects(ctx, req.(*ListObjectsRequest)) } return interceptor(ctx, in, info, handler) } func _ObjectStore_DeleteObject_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(DeleteObjectRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(ObjectStoreServer).DeleteObject(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: ObjectStore_DeleteObject_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ObjectStoreServer).DeleteObject(ctx, req.(*DeleteObjectRequest)) } return interceptor(ctx, in, info, handler) } func _ObjectStore_CreateSignedURL_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(CreateSignedURLRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(ObjectStoreServer).CreateSignedURL(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: ObjectStore_CreateSignedURL_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ObjectStoreServer).CreateSignedURL(ctx, req.(*CreateSignedURLRequest)) } return interceptor(ctx, in, info, handler) } // ObjectStore_ServiceDesc is the grpc.ServiceDesc for ObjectStore service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) var ObjectStore_ServiceDesc = grpc.ServiceDesc{ ServiceName: "generated.ObjectStore", HandlerType: (*ObjectStoreServer)(nil), Methods: []grpc.MethodDesc{ { MethodName: "Init", Handler: _ObjectStore_Init_Handler, }, { MethodName: "ObjectExists", Handler: _ObjectStore_ObjectExists_Handler, }, { MethodName: "ListCommonPrefixes", Handler: _ObjectStore_ListCommonPrefixes_Handler, }, { MethodName: "ListObjects", Handler: _ObjectStore_ListObjects_Handler, }, { MethodName: "DeleteObject", Handler: _ObjectStore_DeleteObject_Handler, }, { MethodName: "CreateSignedURL", Handler: _ObjectStore_CreateSignedURL_Handler, }, }, Streams: []grpc.StreamDesc{ { StreamName: "PutObject", Handler: _ObjectStore_PutObject_Handler, ClientStreams: true, }, { StreamName: "GetObject", Handler: _ObjectStore_GetObject_Handler, ServerStreams: true, }, }, Metadata: "ObjectStore.proto", } ================================================ FILE: pkg/plugin/generated/PluginLister.pb.go ================================================ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.33.0 // protoc v4.25.2 // source: PluginLister.proto package generated import ( protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" sync "sync" ) const ( // Verify that this generated code is sufficiently up-to-date. _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) // Verify that runtime/protoimpl is sufficiently up-to-date. _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) type PluginIdentifier struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Command string `protobuf:"bytes,1,opt,name=command,proto3" json:"command,omitempty"` Kind string `protobuf:"bytes,2,opt,name=kind,proto3" json:"kind,omitempty"` Name string `protobuf:"bytes,3,opt,name=name,proto3" json:"name,omitempty"` } func (x *PluginIdentifier) Reset() { *x = PluginIdentifier{} if protoimpl.UnsafeEnabled { mi := &file_PluginLister_proto_msgTypes[0] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *PluginIdentifier) String() string { return protoimpl.X.MessageStringOf(x) } func (*PluginIdentifier) ProtoMessage() {} func (x *PluginIdentifier) ProtoReflect() protoreflect.Message { mi := &file_PluginLister_proto_msgTypes[0] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use PluginIdentifier.ProtoReflect.Descriptor instead. func (*PluginIdentifier) Descriptor() ([]byte, []int) { return file_PluginLister_proto_rawDescGZIP(), []int{0} } func (x *PluginIdentifier) GetCommand() string { if x != nil { return x.Command } return "" } func (x *PluginIdentifier) GetKind() string { if x != nil { return x.Kind } return "" } func (x *PluginIdentifier) GetName() string { if x != nil { return x.Name } return "" } type ListPluginsResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Plugins []*PluginIdentifier `protobuf:"bytes,1,rep,name=plugins,proto3" json:"plugins,omitempty"` } func (x *ListPluginsResponse) Reset() { *x = ListPluginsResponse{} if protoimpl.UnsafeEnabled { mi := &file_PluginLister_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *ListPluginsResponse) String() string { return protoimpl.X.MessageStringOf(x) } func (*ListPluginsResponse) ProtoMessage() {} func (x *ListPluginsResponse) ProtoReflect() protoreflect.Message { mi := &file_PluginLister_proto_msgTypes[1] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use ListPluginsResponse.ProtoReflect.Descriptor instead. func (*ListPluginsResponse) Descriptor() ([]byte, []int) { return file_PluginLister_proto_rawDescGZIP(), []int{1} } func (x *ListPluginsResponse) GetPlugins() []*PluginIdentifier { if x != nil { return x.Plugins } return nil } var File_PluginLister_proto protoreflect.FileDescriptor var file_PluginLister_proto_rawDesc = []byte{ 0x0a, 0x12, 0x50, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x09, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x1a, 0x0c, 0x53, 0x68, 0x61, 0x72, 0x65, 0x64, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x54, 0x0a, 0x10, 0x50, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x4c, 0x0a, 0x13, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x35, 0x0a, 0x07, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x50, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x52, 0x07, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x73, 0x32, 0x4f, 0x0a, 0x0c, 0x50, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x72, 0x12, 0x3f, 0x0a, 0x0b, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x73, 0x12, 0x10, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1e, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x35, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x76, 0x6d, 0x77, 0x61, 0x72, 0x65, 0x2d, 0x74, 0x61, 0x6e, 0x7a, 0x75, 0x2f, 0x76, 0x65, 0x6c, 0x65, 0x72, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x2f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( file_PluginLister_proto_rawDescOnce sync.Once file_PluginLister_proto_rawDescData = file_PluginLister_proto_rawDesc ) func file_PluginLister_proto_rawDescGZIP() []byte { file_PluginLister_proto_rawDescOnce.Do(func() { file_PluginLister_proto_rawDescData = protoimpl.X.CompressGZIP(file_PluginLister_proto_rawDescData) }) return file_PluginLister_proto_rawDescData } var file_PluginLister_proto_msgTypes = make([]protoimpl.MessageInfo, 2) var file_PluginLister_proto_goTypes = []interface{}{ (*PluginIdentifier)(nil), // 0: generated.PluginIdentifier (*ListPluginsResponse)(nil), // 1: generated.ListPluginsResponse (*Empty)(nil), // 2: generated.Empty } var file_PluginLister_proto_depIdxs = []int32{ 0, // 0: generated.ListPluginsResponse.plugins:type_name -> generated.PluginIdentifier 2, // 1: generated.PluginLister.ListPlugins:input_type -> generated.Empty 1, // 2: generated.PluginLister.ListPlugins:output_type -> generated.ListPluginsResponse 2, // [2:3] is the sub-list for method output_type 1, // [1:2] is the sub-list for method input_type 1, // [1:1] is the sub-list for extension type_name 1, // [1:1] is the sub-list for extension extendee 0, // [0:1] is the sub-list for field type_name } func init() { file_PluginLister_proto_init() } func file_PluginLister_proto_init() { if File_PluginLister_proto != nil { return } file_Shared_proto_init() if !protoimpl.UnsafeEnabled { file_PluginLister_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*PluginIdentifier); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_PluginLister_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ListPluginsResponse); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_PluginLister_proto_rawDesc, NumEnums: 0, NumMessages: 2, NumExtensions: 0, NumServices: 1, }, GoTypes: file_PluginLister_proto_goTypes, DependencyIndexes: file_PluginLister_proto_depIdxs, MessageInfos: file_PluginLister_proto_msgTypes, }.Build() File_PluginLister_proto = out.File file_PluginLister_proto_rawDesc = nil file_PluginLister_proto_goTypes = nil file_PluginLister_proto_depIdxs = nil } ================================================ FILE: pkg/plugin/generated/PluginLister_grpc.pb.go ================================================ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.3.0 // - protoc v4.25.2 // source: PluginLister.proto package generated import ( context "context" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" ) // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. // Requires gRPC-Go v1.32.0 or later. const _ = grpc.SupportPackageIsVersion7 const ( PluginLister_ListPlugins_FullMethodName = "/generated.PluginLister/ListPlugins" ) // PluginListerClient is the client API for PluginLister service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. type PluginListerClient interface { ListPlugins(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*ListPluginsResponse, error) } type pluginListerClient struct { cc grpc.ClientConnInterface } func NewPluginListerClient(cc grpc.ClientConnInterface) PluginListerClient { return &pluginListerClient{cc} } func (c *pluginListerClient) ListPlugins(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*ListPluginsResponse, error) { out := new(ListPluginsResponse) err := c.cc.Invoke(ctx, PluginLister_ListPlugins_FullMethodName, in, out, opts...) if err != nil { return nil, err } return out, nil } // PluginListerServer is the server API for PluginLister service. // All implementations should embed UnimplementedPluginListerServer // for forward compatibility type PluginListerServer interface { ListPlugins(context.Context, *Empty) (*ListPluginsResponse, error) } // UnimplementedPluginListerServer should be embedded to have forward compatible implementations. type UnimplementedPluginListerServer struct { } func (UnimplementedPluginListerServer) ListPlugins(context.Context, *Empty) (*ListPluginsResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method ListPlugins not implemented") } // UnsafePluginListerServer may be embedded to opt out of forward compatibility for this service. // Use of this interface is not recommended, as added methods to PluginListerServer will // result in compilation errors. type UnsafePluginListerServer interface { mustEmbedUnimplementedPluginListerServer() } func RegisterPluginListerServer(s grpc.ServiceRegistrar, srv PluginListerServer) { s.RegisterService(&PluginLister_ServiceDesc, srv) } func _PluginLister_ListPlugins_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(Empty) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(PluginListerServer).ListPlugins(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: PluginLister_ListPlugins_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(PluginListerServer).ListPlugins(ctx, req.(*Empty)) } return interceptor(ctx, in, info, handler) } // PluginLister_ServiceDesc is the grpc.ServiceDesc for PluginLister service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) var PluginLister_ServiceDesc = grpc.ServiceDesc{ ServiceName: "generated.PluginLister", HandlerType: (*PluginListerServer)(nil), Methods: []grpc.MethodDesc{ { MethodName: "ListPlugins", Handler: _PluginLister_ListPlugins_Handler, }, }, Streams: []grpc.StreamDesc{}, Metadata: "PluginLister.proto", } ================================================ FILE: pkg/plugin/generated/RestoreItemAction.pb.go ================================================ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.33.0 // protoc v4.25.2 // source: RestoreItemAction.proto package generated import ( protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" sync "sync" ) const ( // Verify that this generated code is sufficiently up-to-date. _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) // Verify that runtime/protoimpl is sufficiently up-to-date. _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) type RestoreItemActionExecuteRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` Item []byte `protobuf:"bytes,2,opt,name=item,proto3" json:"item,omitempty"` Restore []byte `protobuf:"bytes,3,opt,name=restore,proto3" json:"restore,omitempty"` ItemFromBackup []byte `protobuf:"bytes,4,opt,name=itemFromBackup,proto3" json:"itemFromBackup,omitempty"` } func (x *RestoreItemActionExecuteRequest) Reset() { *x = RestoreItemActionExecuteRequest{} if protoimpl.UnsafeEnabled { mi := &file_RestoreItemAction_proto_msgTypes[0] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *RestoreItemActionExecuteRequest) String() string { return protoimpl.X.MessageStringOf(x) } func (*RestoreItemActionExecuteRequest) ProtoMessage() {} func (x *RestoreItemActionExecuteRequest) ProtoReflect() protoreflect.Message { mi := &file_RestoreItemAction_proto_msgTypes[0] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use RestoreItemActionExecuteRequest.ProtoReflect.Descriptor instead. func (*RestoreItemActionExecuteRequest) Descriptor() ([]byte, []int) { return file_RestoreItemAction_proto_rawDescGZIP(), []int{0} } func (x *RestoreItemActionExecuteRequest) GetPlugin() string { if x != nil { return x.Plugin } return "" } func (x *RestoreItemActionExecuteRequest) GetItem() []byte { if x != nil { return x.Item } return nil } func (x *RestoreItemActionExecuteRequest) GetRestore() []byte { if x != nil { return x.Restore } return nil } func (x *RestoreItemActionExecuteRequest) GetItemFromBackup() []byte { if x != nil { return x.ItemFromBackup } return nil } type RestoreItemActionExecuteResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Item []byte `protobuf:"bytes,1,opt,name=item,proto3" json:"item,omitempty"` AdditionalItems []*ResourceIdentifier `protobuf:"bytes,2,rep,name=additionalItems,proto3" json:"additionalItems,omitempty"` SkipRestore bool `protobuf:"varint,3,opt,name=skipRestore,proto3" json:"skipRestore,omitempty"` } func (x *RestoreItemActionExecuteResponse) Reset() { *x = RestoreItemActionExecuteResponse{} if protoimpl.UnsafeEnabled { mi := &file_RestoreItemAction_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *RestoreItemActionExecuteResponse) String() string { return protoimpl.X.MessageStringOf(x) } func (*RestoreItemActionExecuteResponse) ProtoMessage() {} func (x *RestoreItemActionExecuteResponse) ProtoReflect() protoreflect.Message { mi := &file_RestoreItemAction_proto_msgTypes[1] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use RestoreItemActionExecuteResponse.ProtoReflect.Descriptor instead. func (*RestoreItemActionExecuteResponse) Descriptor() ([]byte, []int) { return file_RestoreItemAction_proto_rawDescGZIP(), []int{1} } func (x *RestoreItemActionExecuteResponse) GetItem() []byte { if x != nil { return x.Item } return nil } func (x *RestoreItemActionExecuteResponse) GetAdditionalItems() []*ResourceIdentifier { if x != nil { return x.AdditionalItems } return nil } func (x *RestoreItemActionExecuteResponse) GetSkipRestore() bool { if x != nil { return x.SkipRestore } return false } type RestoreItemActionAppliesToRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` } func (x *RestoreItemActionAppliesToRequest) Reset() { *x = RestoreItemActionAppliesToRequest{} if protoimpl.UnsafeEnabled { mi := &file_RestoreItemAction_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *RestoreItemActionAppliesToRequest) String() string { return protoimpl.X.MessageStringOf(x) } func (*RestoreItemActionAppliesToRequest) ProtoMessage() {} func (x *RestoreItemActionAppliesToRequest) ProtoReflect() protoreflect.Message { mi := &file_RestoreItemAction_proto_msgTypes[2] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use RestoreItemActionAppliesToRequest.ProtoReflect.Descriptor instead. func (*RestoreItemActionAppliesToRequest) Descriptor() ([]byte, []int) { return file_RestoreItemAction_proto_rawDescGZIP(), []int{2} } func (x *RestoreItemActionAppliesToRequest) GetPlugin() string { if x != nil { return x.Plugin } return "" } type RestoreItemActionAppliesToResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields ResourceSelector *ResourceSelector `protobuf:"bytes,1,opt,name=ResourceSelector,proto3" json:"ResourceSelector,omitempty"` } func (x *RestoreItemActionAppliesToResponse) Reset() { *x = RestoreItemActionAppliesToResponse{} if protoimpl.UnsafeEnabled { mi := &file_RestoreItemAction_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *RestoreItemActionAppliesToResponse) String() string { return protoimpl.X.MessageStringOf(x) } func (*RestoreItemActionAppliesToResponse) ProtoMessage() {} func (x *RestoreItemActionAppliesToResponse) ProtoReflect() protoreflect.Message { mi := &file_RestoreItemAction_proto_msgTypes[3] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use RestoreItemActionAppliesToResponse.ProtoReflect.Descriptor instead. func (*RestoreItemActionAppliesToResponse) Descriptor() ([]byte, []int) { return file_RestoreItemAction_proto_rawDescGZIP(), []int{3} } func (x *RestoreItemActionAppliesToResponse) GetResourceSelector() *ResourceSelector { if x != nil { return x.ResourceSelector } return nil } var File_RestoreItemAction_proto protoreflect.FileDescriptor var file_RestoreItemAction_proto_rawDesc = []byte{ 0x0a, 0x17, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x09, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x1a, 0x0c, 0x53, 0x68, 0x61, 0x72, 0x65, 0x64, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x8f, 0x01, 0x0a, 0x1f, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x72, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x26, 0x0a, 0x0e, 0x69, 0x74, 0x65, 0x6d, 0x46, 0x72, 0x6f, 0x6d, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0e, 0x69, 0x74, 0x65, 0x6d, 0x46, 0x72, 0x6f, 0x6d, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x22, 0xa1, 0x01, 0x0a, 0x20, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x12, 0x47, 0x0a, 0x0f, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x52, 0x0f, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x12, 0x20, 0x0a, 0x0b, 0x73, 0x6b, 0x69, 0x70, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0b, 0x73, 0x6b, 0x69, 0x70, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x22, 0x3b, 0x0a, 0x21, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x22, 0x6d, 0x0a, 0x22, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x47, 0x0a, 0x10, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x10, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x32, 0xe1, 0x01, 0x0a, 0x11, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x68, 0x0a, 0x09, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x12, 0x2c, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2d, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x62, 0x0a, 0x07, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x12, 0x2a, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2b, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x35, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x76, 0x6d, 0x77, 0x61, 0x72, 0x65, 0x2d, 0x74, 0x61, 0x6e, 0x7a, 0x75, 0x2f, 0x76, 0x65, 0x6c, 0x65, 0x72, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x2f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( file_RestoreItemAction_proto_rawDescOnce sync.Once file_RestoreItemAction_proto_rawDescData = file_RestoreItemAction_proto_rawDesc ) func file_RestoreItemAction_proto_rawDescGZIP() []byte { file_RestoreItemAction_proto_rawDescOnce.Do(func() { file_RestoreItemAction_proto_rawDescData = protoimpl.X.CompressGZIP(file_RestoreItemAction_proto_rawDescData) }) return file_RestoreItemAction_proto_rawDescData } var file_RestoreItemAction_proto_msgTypes = make([]protoimpl.MessageInfo, 4) var file_RestoreItemAction_proto_goTypes = []interface{}{ (*RestoreItemActionExecuteRequest)(nil), // 0: generated.RestoreItemActionExecuteRequest (*RestoreItemActionExecuteResponse)(nil), // 1: generated.RestoreItemActionExecuteResponse (*RestoreItemActionAppliesToRequest)(nil), // 2: generated.RestoreItemActionAppliesToRequest (*RestoreItemActionAppliesToResponse)(nil), // 3: generated.RestoreItemActionAppliesToResponse (*ResourceIdentifier)(nil), // 4: generated.ResourceIdentifier (*ResourceSelector)(nil), // 5: generated.ResourceSelector } var file_RestoreItemAction_proto_depIdxs = []int32{ 4, // 0: generated.RestoreItemActionExecuteResponse.additionalItems:type_name -> generated.ResourceIdentifier 5, // 1: generated.RestoreItemActionAppliesToResponse.ResourceSelector:type_name -> generated.ResourceSelector 2, // 2: generated.RestoreItemAction.AppliesTo:input_type -> generated.RestoreItemActionAppliesToRequest 0, // 3: generated.RestoreItemAction.Execute:input_type -> generated.RestoreItemActionExecuteRequest 3, // 4: generated.RestoreItemAction.AppliesTo:output_type -> generated.RestoreItemActionAppliesToResponse 1, // 5: generated.RestoreItemAction.Execute:output_type -> generated.RestoreItemActionExecuteResponse 4, // [4:6] is the sub-list for method output_type 2, // [2:4] is the sub-list for method input_type 2, // [2:2] is the sub-list for extension type_name 2, // [2:2] is the sub-list for extension extendee 0, // [0:2] is the sub-list for field type_name } func init() { file_RestoreItemAction_proto_init() } func file_RestoreItemAction_proto_init() { if File_RestoreItemAction_proto != nil { return } file_Shared_proto_init() if !protoimpl.UnsafeEnabled { file_RestoreItemAction_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*RestoreItemActionExecuteRequest); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_RestoreItemAction_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*RestoreItemActionExecuteResponse); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_RestoreItemAction_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*RestoreItemActionAppliesToRequest); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_RestoreItemAction_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*RestoreItemActionAppliesToResponse); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_RestoreItemAction_proto_rawDesc, NumEnums: 0, NumMessages: 4, NumExtensions: 0, NumServices: 1, }, GoTypes: file_RestoreItemAction_proto_goTypes, DependencyIndexes: file_RestoreItemAction_proto_depIdxs, MessageInfos: file_RestoreItemAction_proto_msgTypes, }.Build() File_RestoreItemAction_proto = out.File file_RestoreItemAction_proto_rawDesc = nil file_RestoreItemAction_proto_goTypes = nil file_RestoreItemAction_proto_depIdxs = nil } ================================================ FILE: pkg/plugin/generated/RestoreItemAction_grpc.pb.go ================================================ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.3.0 // - protoc v4.25.2 // source: RestoreItemAction.proto package generated import ( context "context" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" ) // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. // Requires gRPC-Go v1.32.0 or later. const _ = grpc.SupportPackageIsVersion7 const ( RestoreItemAction_AppliesTo_FullMethodName = "/generated.RestoreItemAction/AppliesTo" RestoreItemAction_Execute_FullMethodName = "/generated.RestoreItemAction/Execute" ) // RestoreItemActionClient is the client API for RestoreItemAction service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. type RestoreItemActionClient interface { AppliesTo(ctx context.Context, in *RestoreItemActionAppliesToRequest, opts ...grpc.CallOption) (*RestoreItemActionAppliesToResponse, error) Execute(ctx context.Context, in *RestoreItemActionExecuteRequest, opts ...grpc.CallOption) (*RestoreItemActionExecuteResponse, error) } type restoreItemActionClient struct { cc grpc.ClientConnInterface } func NewRestoreItemActionClient(cc grpc.ClientConnInterface) RestoreItemActionClient { return &restoreItemActionClient{cc} } func (c *restoreItemActionClient) AppliesTo(ctx context.Context, in *RestoreItemActionAppliesToRequest, opts ...grpc.CallOption) (*RestoreItemActionAppliesToResponse, error) { out := new(RestoreItemActionAppliesToResponse) err := c.cc.Invoke(ctx, RestoreItemAction_AppliesTo_FullMethodName, in, out, opts...) if err != nil { return nil, err } return out, nil } func (c *restoreItemActionClient) Execute(ctx context.Context, in *RestoreItemActionExecuteRequest, opts ...grpc.CallOption) (*RestoreItemActionExecuteResponse, error) { out := new(RestoreItemActionExecuteResponse) err := c.cc.Invoke(ctx, RestoreItemAction_Execute_FullMethodName, in, out, opts...) if err != nil { return nil, err } return out, nil } // RestoreItemActionServer is the server API for RestoreItemAction service. // All implementations should embed UnimplementedRestoreItemActionServer // for forward compatibility type RestoreItemActionServer interface { AppliesTo(context.Context, *RestoreItemActionAppliesToRequest) (*RestoreItemActionAppliesToResponse, error) Execute(context.Context, *RestoreItemActionExecuteRequest) (*RestoreItemActionExecuteResponse, error) } // UnimplementedRestoreItemActionServer should be embedded to have forward compatible implementations. type UnimplementedRestoreItemActionServer struct { } func (UnimplementedRestoreItemActionServer) AppliesTo(context.Context, *RestoreItemActionAppliesToRequest) (*RestoreItemActionAppliesToResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method AppliesTo not implemented") } func (UnimplementedRestoreItemActionServer) Execute(context.Context, *RestoreItemActionExecuteRequest) (*RestoreItemActionExecuteResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method Execute not implemented") } // UnsafeRestoreItemActionServer may be embedded to opt out of forward compatibility for this service. // Use of this interface is not recommended, as added methods to RestoreItemActionServer will // result in compilation errors. type UnsafeRestoreItemActionServer interface { mustEmbedUnimplementedRestoreItemActionServer() } func RegisterRestoreItemActionServer(s grpc.ServiceRegistrar, srv RestoreItemActionServer) { s.RegisterService(&RestoreItemAction_ServiceDesc, srv) } func _RestoreItemAction_AppliesTo_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(RestoreItemActionAppliesToRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(RestoreItemActionServer).AppliesTo(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: RestoreItemAction_AppliesTo_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(RestoreItemActionServer).AppliesTo(ctx, req.(*RestoreItemActionAppliesToRequest)) } return interceptor(ctx, in, info, handler) } func _RestoreItemAction_Execute_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(RestoreItemActionExecuteRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(RestoreItemActionServer).Execute(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: RestoreItemAction_Execute_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(RestoreItemActionServer).Execute(ctx, req.(*RestoreItemActionExecuteRequest)) } return interceptor(ctx, in, info, handler) } // RestoreItemAction_ServiceDesc is the grpc.ServiceDesc for RestoreItemAction service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) var RestoreItemAction_ServiceDesc = grpc.ServiceDesc{ ServiceName: "generated.RestoreItemAction", HandlerType: (*RestoreItemActionServer)(nil), Methods: []grpc.MethodDesc{ { MethodName: "AppliesTo", Handler: _RestoreItemAction_AppliesTo_Handler, }, { MethodName: "Execute", Handler: _RestoreItemAction_Execute_Handler, }, }, Streams: []grpc.StreamDesc{}, Metadata: "RestoreItemAction.proto", } ================================================ FILE: pkg/plugin/generated/Shared.pb.go ================================================ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.33.0 // protoc v4.25.2 // source: Shared.proto package generated import ( protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" timestamppb "google.golang.org/protobuf/types/known/timestamppb" reflect "reflect" sync "sync" ) const ( // Verify that this generated code is sufficiently up-to-date. _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) // Verify that runtime/protoimpl is sufficiently up-to-date. _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) type Empty struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields } func (x *Empty) Reset() { *x = Empty{} if protoimpl.UnsafeEnabled { mi := &file_Shared_proto_msgTypes[0] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *Empty) String() string { return protoimpl.X.MessageStringOf(x) } func (*Empty) ProtoMessage() {} func (x *Empty) ProtoReflect() protoreflect.Message { mi := &file_Shared_proto_msgTypes[0] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use Empty.ProtoReflect.Descriptor instead. func (*Empty) Descriptor() ([]byte, []int) { return file_Shared_proto_rawDescGZIP(), []int{0} } type Stack struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Frames []*StackFrame `protobuf:"bytes,1,rep,name=frames,proto3" json:"frames,omitempty"` } func (x *Stack) Reset() { *x = Stack{} if protoimpl.UnsafeEnabled { mi := &file_Shared_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *Stack) String() string { return protoimpl.X.MessageStringOf(x) } func (*Stack) ProtoMessage() {} func (x *Stack) ProtoReflect() protoreflect.Message { mi := &file_Shared_proto_msgTypes[1] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use Stack.ProtoReflect.Descriptor instead. func (*Stack) Descriptor() ([]byte, []int) { return file_Shared_proto_rawDescGZIP(), []int{1} } func (x *Stack) GetFrames() []*StackFrame { if x != nil { return x.Frames } return nil } type StackFrame struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields File string `protobuf:"bytes,1,opt,name=file,proto3" json:"file,omitempty"` Line int32 `protobuf:"varint,2,opt,name=line,proto3" json:"line,omitempty"` Function string `protobuf:"bytes,3,opt,name=function,proto3" json:"function,omitempty"` } func (x *StackFrame) Reset() { *x = StackFrame{} if protoimpl.UnsafeEnabled { mi := &file_Shared_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *StackFrame) String() string { return protoimpl.X.MessageStringOf(x) } func (*StackFrame) ProtoMessage() {} func (x *StackFrame) ProtoReflect() protoreflect.Message { mi := &file_Shared_proto_msgTypes[2] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use StackFrame.ProtoReflect.Descriptor instead. func (*StackFrame) Descriptor() ([]byte, []int) { return file_Shared_proto_rawDescGZIP(), []int{2} } func (x *StackFrame) GetFile() string { if x != nil { return x.File } return "" } func (x *StackFrame) GetLine() int32 { if x != nil { return x.Line } return 0 } func (x *StackFrame) GetFunction() string { if x != nil { return x.Function } return "" } type ResourceIdentifier struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Group string `protobuf:"bytes,1,opt,name=group,proto3" json:"group,omitempty"` Resource string `protobuf:"bytes,2,opt,name=resource,proto3" json:"resource,omitempty"` Namespace string `protobuf:"bytes,3,opt,name=namespace,proto3" json:"namespace,omitempty"` Name string `protobuf:"bytes,4,opt,name=name,proto3" json:"name,omitempty"` } func (x *ResourceIdentifier) Reset() { *x = ResourceIdentifier{} if protoimpl.UnsafeEnabled { mi := &file_Shared_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *ResourceIdentifier) String() string { return protoimpl.X.MessageStringOf(x) } func (*ResourceIdentifier) ProtoMessage() {} func (x *ResourceIdentifier) ProtoReflect() protoreflect.Message { mi := &file_Shared_proto_msgTypes[3] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use ResourceIdentifier.ProtoReflect.Descriptor instead. func (*ResourceIdentifier) Descriptor() ([]byte, []int) { return file_Shared_proto_rawDescGZIP(), []int{3} } func (x *ResourceIdentifier) GetGroup() string { if x != nil { return x.Group } return "" } func (x *ResourceIdentifier) GetResource() string { if x != nil { return x.Resource } return "" } func (x *ResourceIdentifier) GetNamespace() string { if x != nil { return x.Namespace } return "" } func (x *ResourceIdentifier) GetName() string { if x != nil { return x.Name } return "" } type ResourceSelector struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields IncludedNamespaces []string `protobuf:"bytes,1,rep,name=includedNamespaces,proto3" json:"includedNamespaces,omitempty"` ExcludedNamespaces []string `protobuf:"bytes,2,rep,name=excludedNamespaces,proto3" json:"excludedNamespaces,omitempty"` IncludedResources []string `protobuf:"bytes,3,rep,name=includedResources,proto3" json:"includedResources,omitempty"` ExcludedResources []string `protobuf:"bytes,4,rep,name=excludedResources,proto3" json:"excludedResources,omitempty"` Selector string `protobuf:"bytes,5,opt,name=selector,proto3" json:"selector,omitempty"` } func (x *ResourceSelector) Reset() { *x = ResourceSelector{} if protoimpl.UnsafeEnabled { mi := &file_Shared_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *ResourceSelector) String() string { return protoimpl.X.MessageStringOf(x) } func (*ResourceSelector) ProtoMessage() {} func (x *ResourceSelector) ProtoReflect() protoreflect.Message { mi := &file_Shared_proto_msgTypes[4] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use ResourceSelector.ProtoReflect.Descriptor instead. func (*ResourceSelector) Descriptor() ([]byte, []int) { return file_Shared_proto_rawDescGZIP(), []int{4} } func (x *ResourceSelector) GetIncludedNamespaces() []string { if x != nil { return x.IncludedNamespaces } return nil } func (x *ResourceSelector) GetExcludedNamespaces() []string { if x != nil { return x.ExcludedNamespaces } return nil } func (x *ResourceSelector) GetIncludedResources() []string { if x != nil { return x.IncludedResources } return nil } func (x *ResourceSelector) GetExcludedResources() []string { if x != nil { return x.ExcludedResources } return nil } func (x *ResourceSelector) GetSelector() string { if x != nil { return x.Selector } return "" } type OperationProgress struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Completed bool `protobuf:"varint,1,opt,name=completed,proto3" json:"completed,omitempty"` Err string `protobuf:"bytes,2,opt,name=err,proto3" json:"err,omitempty"` NCompleted int64 `protobuf:"varint,3,opt,name=nCompleted,proto3" json:"nCompleted,omitempty"` NTotal int64 `protobuf:"varint,4,opt,name=nTotal,proto3" json:"nTotal,omitempty"` OperationUnits string `protobuf:"bytes,5,opt,name=operationUnits,proto3" json:"operationUnits,omitempty"` Description string `protobuf:"bytes,6,opt,name=description,proto3" json:"description,omitempty"` Started *timestamppb.Timestamp `protobuf:"bytes,7,opt,name=started,proto3" json:"started,omitempty"` Updated *timestamppb.Timestamp `protobuf:"bytes,8,opt,name=updated,proto3" json:"updated,omitempty"` } func (x *OperationProgress) Reset() { *x = OperationProgress{} if protoimpl.UnsafeEnabled { mi := &file_Shared_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *OperationProgress) String() string { return protoimpl.X.MessageStringOf(x) } func (*OperationProgress) ProtoMessage() {} func (x *OperationProgress) ProtoReflect() protoreflect.Message { mi := &file_Shared_proto_msgTypes[5] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use OperationProgress.ProtoReflect.Descriptor instead. func (*OperationProgress) Descriptor() ([]byte, []int) { return file_Shared_proto_rawDescGZIP(), []int{5} } func (x *OperationProgress) GetCompleted() bool { if x != nil { return x.Completed } return false } func (x *OperationProgress) GetErr() string { if x != nil { return x.Err } return "" } func (x *OperationProgress) GetNCompleted() int64 { if x != nil { return x.NCompleted } return 0 } func (x *OperationProgress) GetNTotal() int64 { if x != nil { return x.NTotal } return 0 } func (x *OperationProgress) GetOperationUnits() string { if x != nil { return x.OperationUnits } return "" } func (x *OperationProgress) GetDescription() string { if x != nil { return x.Description } return "" } func (x *OperationProgress) GetStarted() *timestamppb.Timestamp { if x != nil { return x.Started } return nil } func (x *OperationProgress) GetUpdated() *timestamppb.Timestamp { if x != nil { return x.Updated } return nil } var File_Shared_proto protoreflect.FileDescriptor var file_Shared_proto_rawDesc = []byte{ 0x0a, 0x0c, 0x53, 0x68, 0x61, 0x72, 0x65, 0x64, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x09, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x07, 0x0a, 0x05, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x36, 0x0a, 0x05, 0x53, 0x74, 0x61, 0x63, 0x6b, 0x12, 0x2d, 0x0a, 0x06, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x53, 0x74, 0x61, 0x63, 0x6b, 0x46, 0x72, 0x61, 0x6d, 0x65, 0x52, 0x06, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x73, 0x22, 0x50, 0x0a, 0x0a, 0x53, 0x74, 0x61, 0x63, 0x6b, 0x46, 0x72, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x78, 0x0a, 0x12, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x12, 0x14, 0x0a, 0x05, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x12, 0x1a, 0x0a, 0x08, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0xea, 0x01, 0x0a, 0x10, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x2e, 0x0a, 0x12, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x12, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x12, 0x2e, 0x0a, 0x12, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x12, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x12, 0x2c, 0x0a, 0x11, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x64, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x11, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x64, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x12, 0x2c, 0x0a, 0x11, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x64, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x11, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x64, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x22, 0xb1, 0x02, 0x0a, 0x11, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x65, 0x72, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x65, 0x72, 0x72, 0x12, 0x1e, 0x0a, 0x0a, 0x6e, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x6e, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x6e, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x6e, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x12, 0x26, 0x0a, 0x0e, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x55, 0x6e, 0x69, 0x74, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x55, 0x6e, 0x69, 0x74, 0x73, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x34, 0x0a, 0x07, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x07, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x12, 0x34, 0x0a, 0x07, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x07, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x42, 0x35, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x76, 0x6d, 0x77, 0x61, 0x72, 0x65, 0x2d, 0x74, 0x61, 0x6e, 0x7a, 0x75, 0x2f, 0x76, 0x65, 0x6c, 0x65, 0x72, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x2f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( file_Shared_proto_rawDescOnce sync.Once file_Shared_proto_rawDescData = file_Shared_proto_rawDesc ) func file_Shared_proto_rawDescGZIP() []byte { file_Shared_proto_rawDescOnce.Do(func() { file_Shared_proto_rawDescData = protoimpl.X.CompressGZIP(file_Shared_proto_rawDescData) }) return file_Shared_proto_rawDescData } var file_Shared_proto_msgTypes = make([]protoimpl.MessageInfo, 6) var file_Shared_proto_goTypes = []interface{}{ (*Empty)(nil), // 0: generated.Empty (*Stack)(nil), // 1: generated.Stack (*StackFrame)(nil), // 2: generated.StackFrame (*ResourceIdentifier)(nil), // 3: generated.ResourceIdentifier (*ResourceSelector)(nil), // 4: generated.ResourceSelector (*OperationProgress)(nil), // 5: generated.OperationProgress (*timestamppb.Timestamp)(nil), // 6: google.protobuf.Timestamp } var file_Shared_proto_depIdxs = []int32{ 2, // 0: generated.Stack.frames:type_name -> generated.StackFrame 6, // 1: generated.OperationProgress.started:type_name -> google.protobuf.Timestamp 6, // 2: generated.OperationProgress.updated:type_name -> google.protobuf.Timestamp 3, // [3:3] is the sub-list for method output_type 3, // [3:3] is the sub-list for method input_type 3, // [3:3] is the sub-list for extension type_name 3, // [3:3] is the sub-list for extension extendee 0, // [0:3] is the sub-list for field type_name } func init() { file_Shared_proto_init() } func file_Shared_proto_init() { if File_Shared_proto != nil { return } if !protoimpl.UnsafeEnabled { file_Shared_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Empty); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_Shared_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Stack); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_Shared_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*StackFrame); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_Shared_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ResourceIdentifier); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_Shared_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ResourceSelector); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_Shared_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*OperationProgress); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_Shared_proto_rawDesc, NumEnums: 0, NumMessages: 6, NumExtensions: 0, NumServices: 0, }, GoTypes: file_Shared_proto_goTypes, DependencyIndexes: file_Shared_proto_depIdxs, MessageInfos: file_Shared_proto_msgTypes, }.Build() File_Shared_proto = out.File file_Shared_proto_rawDesc = nil file_Shared_proto_goTypes = nil file_Shared_proto_depIdxs = nil } ================================================ FILE: pkg/plugin/generated/VolumeSnapshotter.pb.go ================================================ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.33.0 // protoc v4.25.2 // source: VolumeSnapshotter.proto package generated import ( protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" sync "sync" ) const ( // Verify that this generated code is sufficiently up-to-date. _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) // Verify that runtime/protoimpl is sufficiently up-to-date. _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) type CreateVolumeRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` SnapshotID string `protobuf:"bytes,2,opt,name=snapshotID,proto3" json:"snapshotID,omitempty"` VolumeType string `protobuf:"bytes,3,opt,name=volumeType,proto3" json:"volumeType,omitempty"` VolumeAZ string `protobuf:"bytes,4,opt,name=volumeAZ,proto3" json:"volumeAZ,omitempty"` Iops int64 `protobuf:"varint,5,opt,name=iops,proto3" json:"iops,omitempty"` } func (x *CreateVolumeRequest) Reset() { *x = CreateVolumeRequest{} if protoimpl.UnsafeEnabled { mi := &file_VolumeSnapshotter_proto_msgTypes[0] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *CreateVolumeRequest) String() string { return protoimpl.X.MessageStringOf(x) } func (*CreateVolumeRequest) ProtoMessage() {} func (x *CreateVolumeRequest) ProtoReflect() protoreflect.Message { mi := &file_VolumeSnapshotter_proto_msgTypes[0] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use CreateVolumeRequest.ProtoReflect.Descriptor instead. func (*CreateVolumeRequest) Descriptor() ([]byte, []int) { return file_VolumeSnapshotter_proto_rawDescGZIP(), []int{0} } func (x *CreateVolumeRequest) GetPlugin() string { if x != nil { return x.Plugin } return "" } func (x *CreateVolumeRequest) GetSnapshotID() string { if x != nil { return x.SnapshotID } return "" } func (x *CreateVolumeRequest) GetVolumeType() string { if x != nil { return x.VolumeType } return "" } func (x *CreateVolumeRequest) GetVolumeAZ() string { if x != nil { return x.VolumeAZ } return "" } func (x *CreateVolumeRequest) GetIops() int64 { if x != nil { return x.Iops } return 0 } type CreateVolumeResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields VolumeID string `protobuf:"bytes,1,opt,name=volumeID,proto3" json:"volumeID,omitempty"` } func (x *CreateVolumeResponse) Reset() { *x = CreateVolumeResponse{} if protoimpl.UnsafeEnabled { mi := &file_VolumeSnapshotter_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *CreateVolumeResponse) String() string { return protoimpl.X.MessageStringOf(x) } func (*CreateVolumeResponse) ProtoMessage() {} func (x *CreateVolumeResponse) ProtoReflect() protoreflect.Message { mi := &file_VolumeSnapshotter_proto_msgTypes[1] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use CreateVolumeResponse.ProtoReflect.Descriptor instead. func (*CreateVolumeResponse) Descriptor() ([]byte, []int) { return file_VolumeSnapshotter_proto_rawDescGZIP(), []int{1} } func (x *CreateVolumeResponse) GetVolumeID() string { if x != nil { return x.VolumeID } return "" } type GetVolumeInfoRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` VolumeID string `protobuf:"bytes,2,opt,name=volumeID,proto3" json:"volumeID,omitempty"` VolumeAZ string `protobuf:"bytes,3,opt,name=volumeAZ,proto3" json:"volumeAZ,omitempty"` } func (x *GetVolumeInfoRequest) Reset() { *x = GetVolumeInfoRequest{} if protoimpl.UnsafeEnabled { mi := &file_VolumeSnapshotter_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *GetVolumeInfoRequest) String() string { return protoimpl.X.MessageStringOf(x) } func (*GetVolumeInfoRequest) ProtoMessage() {} func (x *GetVolumeInfoRequest) ProtoReflect() protoreflect.Message { mi := &file_VolumeSnapshotter_proto_msgTypes[2] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use GetVolumeInfoRequest.ProtoReflect.Descriptor instead. func (*GetVolumeInfoRequest) Descriptor() ([]byte, []int) { return file_VolumeSnapshotter_proto_rawDescGZIP(), []int{2} } func (x *GetVolumeInfoRequest) GetPlugin() string { if x != nil { return x.Plugin } return "" } func (x *GetVolumeInfoRequest) GetVolumeID() string { if x != nil { return x.VolumeID } return "" } func (x *GetVolumeInfoRequest) GetVolumeAZ() string { if x != nil { return x.VolumeAZ } return "" } type GetVolumeInfoResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields VolumeType string `protobuf:"bytes,1,opt,name=volumeType,proto3" json:"volumeType,omitempty"` Iops int64 `protobuf:"varint,2,opt,name=iops,proto3" json:"iops,omitempty"` } func (x *GetVolumeInfoResponse) Reset() { *x = GetVolumeInfoResponse{} if protoimpl.UnsafeEnabled { mi := &file_VolumeSnapshotter_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *GetVolumeInfoResponse) String() string { return protoimpl.X.MessageStringOf(x) } func (*GetVolumeInfoResponse) ProtoMessage() {} func (x *GetVolumeInfoResponse) ProtoReflect() protoreflect.Message { mi := &file_VolumeSnapshotter_proto_msgTypes[3] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use GetVolumeInfoResponse.ProtoReflect.Descriptor instead. func (*GetVolumeInfoResponse) Descriptor() ([]byte, []int) { return file_VolumeSnapshotter_proto_rawDescGZIP(), []int{3} } func (x *GetVolumeInfoResponse) GetVolumeType() string { if x != nil { return x.VolumeType } return "" } func (x *GetVolumeInfoResponse) GetIops() int64 { if x != nil { return x.Iops } return 0 } type CreateSnapshotRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` VolumeID string `protobuf:"bytes,2,opt,name=volumeID,proto3" json:"volumeID,omitempty"` VolumeAZ string `protobuf:"bytes,3,opt,name=volumeAZ,proto3" json:"volumeAZ,omitempty"` Tags map[string]string `protobuf:"bytes,4,rep,name=tags,proto3" json:"tags,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` } func (x *CreateSnapshotRequest) Reset() { *x = CreateSnapshotRequest{} if protoimpl.UnsafeEnabled { mi := &file_VolumeSnapshotter_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *CreateSnapshotRequest) String() string { return protoimpl.X.MessageStringOf(x) } func (*CreateSnapshotRequest) ProtoMessage() {} func (x *CreateSnapshotRequest) ProtoReflect() protoreflect.Message { mi := &file_VolumeSnapshotter_proto_msgTypes[4] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use CreateSnapshotRequest.ProtoReflect.Descriptor instead. func (*CreateSnapshotRequest) Descriptor() ([]byte, []int) { return file_VolumeSnapshotter_proto_rawDescGZIP(), []int{4} } func (x *CreateSnapshotRequest) GetPlugin() string { if x != nil { return x.Plugin } return "" } func (x *CreateSnapshotRequest) GetVolumeID() string { if x != nil { return x.VolumeID } return "" } func (x *CreateSnapshotRequest) GetVolumeAZ() string { if x != nil { return x.VolumeAZ } return "" } func (x *CreateSnapshotRequest) GetTags() map[string]string { if x != nil { return x.Tags } return nil } type CreateSnapshotResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields SnapshotID string `protobuf:"bytes,1,opt,name=snapshotID,proto3" json:"snapshotID,omitempty"` } func (x *CreateSnapshotResponse) Reset() { *x = CreateSnapshotResponse{} if protoimpl.UnsafeEnabled { mi := &file_VolumeSnapshotter_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *CreateSnapshotResponse) String() string { return protoimpl.X.MessageStringOf(x) } func (*CreateSnapshotResponse) ProtoMessage() {} func (x *CreateSnapshotResponse) ProtoReflect() protoreflect.Message { mi := &file_VolumeSnapshotter_proto_msgTypes[5] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use CreateSnapshotResponse.ProtoReflect.Descriptor instead. func (*CreateSnapshotResponse) Descriptor() ([]byte, []int) { return file_VolumeSnapshotter_proto_rawDescGZIP(), []int{5} } func (x *CreateSnapshotResponse) GetSnapshotID() string { if x != nil { return x.SnapshotID } return "" } type DeleteSnapshotRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` SnapshotID string `protobuf:"bytes,2,opt,name=snapshotID,proto3" json:"snapshotID,omitempty"` } func (x *DeleteSnapshotRequest) Reset() { *x = DeleteSnapshotRequest{} if protoimpl.UnsafeEnabled { mi := &file_VolumeSnapshotter_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *DeleteSnapshotRequest) String() string { return protoimpl.X.MessageStringOf(x) } func (*DeleteSnapshotRequest) ProtoMessage() {} func (x *DeleteSnapshotRequest) ProtoReflect() protoreflect.Message { mi := &file_VolumeSnapshotter_proto_msgTypes[6] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use DeleteSnapshotRequest.ProtoReflect.Descriptor instead. func (*DeleteSnapshotRequest) Descriptor() ([]byte, []int) { return file_VolumeSnapshotter_proto_rawDescGZIP(), []int{6} } func (x *DeleteSnapshotRequest) GetPlugin() string { if x != nil { return x.Plugin } return "" } func (x *DeleteSnapshotRequest) GetSnapshotID() string { if x != nil { return x.SnapshotID } return "" } type GetVolumeIDRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` PersistentVolume []byte `protobuf:"bytes,2,opt,name=persistentVolume,proto3" json:"persistentVolume,omitempty"` } func (x *GetVolumeIDRequest) Reset() { *x = GetVolumeIDRequest{} if protoimpl.UnsafeEnabled { mi := &file_VolumeSnapshotter_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *GetVolumeIDRequest) String() string { return protoimpl.X.MessageStringOf(x) } func (*GetVolumeIDRequest) ProtoMessage() {} func (x *GetVolumeIDRequest) ProtoReflect() protoreflect.Message { mi := &file_VolumeSnapshotter_proto_msgTypes[7] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use GetVolumeIDRequest.ProtoReflect.Descriptor instead. func (*GetVolumeIDRequest) Descriptor() ([]byte, []int) { return file_VolumeSnapshotter_proto_rawDescGZIP(), []int{7} } func (x *GetVolumeIDRequest) GetPlugin() string { if x != nil { return x.Plugin } return "" } func (x *GetVolumeIDRequest) GetPersistentVolume() []byte { if x != nil { return x.PersistentVolume } return nil } type GetVolumeIDResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields VolumeID string `protobuf:"bytes,1,opt,name=volumeID,proto3" json:"volumeID,omitempty"` } func (x *GetVolumeIDResponse) Reset() { *x = GetVolumeIDResponse{} if protoimpl.UnsafeEnabled { mi := &file_VolumeSnapshotter_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *GetVolumeIDResponse) String() string { return protoimpl.X.MessageStringOf(x) } func (*GetVolumeIDResponse) ProtoMessage() {} func (x *GetVolumeIDResponse) ProtoReflect() protoreflect.Message { mi := &file_VolumeSnapshotter_proto_msgTypes[8] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use GetVolumeIDResponse.ProtoReflect.Descriptor instead. func (*GetVolumeIDResponse) Descriptor() ([]byte, []int) { return file_VolumeSnapshotter_proto_rawDescGZIP(), []int{8} } func (x *GetVolumeIDResponse) GetVolumeID() string { if x != nil { return x.VolumeID } return "" } type SetVolumeIDRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` PersistentVolume []byte `protobuf:"bytes,2,opt,name=persistentVolume,proto3" json:"persistentVolume,omitempty"` VolumeID string `protobuf:"bytes,3,opt,name=volumeID,proto3" json:"volumeID,omitempty"` } func (x *SetVolumeIDRequest) Reset() { *x = SetVolumeIDRequest{} if protoimpl.UnsafeEnabled { mi := &file_VolumeSnapshotter_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *SetVolumeIDRequest) String() string { return protoimpl.X.MessageStringOf(x) } func (*SetVolumeIDRequest) ProtoMessage() {} func (x *SetVolumeIDRequest) ProtoReflect() protoreflect.Message { mi := &file_VolumeSnapshotter_proto_msgTypes[9] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use SetVolumeIDRequest.ProtoReflect.Descriptor instead. func (*SetVolumeIDRequest) Descriptor() ([]byte, []int) { return file_VolumeSnapshotter_proto_rawDescGZIP(), []int{9} } func (x *SetVolumeIDRequest) GetPlugin() string { if x != nil { return x.Plugin } return "" } func (x *SetVolumeIDRequest) GetPersistentVolume() []byte { if x != nil { return x.PersistentVolume } return nil } func (x *SetVolumeIDRequest) GetVolumeID() string { if x != nil { return x.VolumeID } return "" } type SetVolumeIDResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields PersistentVolume []byte `protobuf:"bytes,1,opt,name=persistentVolume,proto3" json:"persistentVolume,omitempty"` } func (x *SetVolumeIDResponse) Reset() { *x = SetVolumeIDResponse{} if protoimpl.UnsafeEnabled { mi := &file_VolumeSnapshotter_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *SetVolumeIDResponse) String() string { return protoimpl.X.MessageStringOf(x) } func (*SetVolumeIDResponse) ProtoMessage() {} func (x *SetVolumeIDResponse) ProtoReflect() protoreflect.Message { mi := &file_VolumeSnapshotter_proto_msgTypes[10] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use SetVolumeIDResponse.ProtoReflect.Descriptor instead. func (*SetVolumeIDResponse) Descriptor() ([]byte, []int) { return file_VolumeSnapshotter_proto_rawDescGZIP(), []int{10} } func (x *SetVolumeIDResponse) GetPersistentVolume() []byte { if x != nil { return x.PersistentVolume } return nil } type VolumeSnapshotterInitRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` Config map[string]string `protobuf:"bytes,2,rep,name=config,proto3" json:"config,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` } func (x *VolumeSnapshotterInitRequest) Reset() { *x = VolumeSnapshotterInitRequest{} if protoimpl.UnsafeEnabled { mi := &file_VolumeSnapshotter_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *VolumeSnapshotterInitRequest) String() string { return protoimpl.X.MessageStringOf(x) } func (*VolumeSnapshotterInitRequest) ProtoMessage() {} func (x *VolumeSnapshotterInitRequest) ProtoReflect() protoreflect.Message { mi := &file_VolumeSnapshotter_proto_msgTypes[11] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use VolumeSnapshotterInitRequest.ProtoReflect.Descriptor instead. func (*VolumeSnapshotterInitRequest) Descriptor() ([]byte, []int) { return file_VolumeSnapshotter_proto_rawDescGZIP(), []int{11} } func (x *VolumeSnapshotterInitRequest) GetPlugin() string { if x != nil { return x.Plugin } return "" } func (x *VolumeSnapshotterInitRequest) GetConfig() map[string]string { if x != nil { return x.Config } return nil } var File_VolumeSnapshotter_proto protoreflect.FileDescriptor var file_VolumeSnapshotter_proto_rawDesc = []byte{ 0x0a, 0x17, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x74, 0x65, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x09, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x1a, 0x0c, 0x53, 0x68, 0x61, 0x72, 0x65, 0x64, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x9d, 0x01, 0x0a, 0x13, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x49, 0x44, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x49, 0x44, 0x12, 0x1e, 0x0a, 0x0a, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x54, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x41, 0x5a, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x41, 0x5a, 0x12, 0x12, 0x0a, 0x04, 0x69, 0x6f, 0x70, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x69, 0x6f, 0x70, 0x73, 0x22, 0x32, 0x0a, 0x14, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x44, 0x22, 0x66, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x1a, 0x0a, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x44, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x44, 0x12, 0x1a, 0x0a, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x41, 0x5a, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x41, 0x5a, 0x22, 0x4b, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x54, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x69, 0x6f, 0x70, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x69, 0x6f, 0x70, 0x73, 0x22, 0xe0, 0x01, 0x0a, 0x15, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x1a, 0x0a, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x44, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x44, 0x12, 0x1a, 0x0a, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x41, 0x5a, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x41, 0x5a, 0x12, 0x3e, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x54, 0x61, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x74, 0x61, 0x67, 0x73, 0x1a, 0x37, 0x0a, 0x09, 0x54, 0x61, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x38, 0x0a, 0x16, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x49, 0x44, 0x22, 0x4f, 0x0a, 0x15, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x49, 0x44, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x49, 0x44, 0x22, 0x58, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x44, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x2a, 0x0a, 0x10, 0x70, 0x65, 0x72, 0x73, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x74, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x10, 0x70, 0x65, 0x72, 0x73, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x74, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x22, 0x31, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x44, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x44, 0x22, 0x74, 0x0a, 0x12, 0x53, 0x65, 0x74, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x44, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x2a, 0x0a, 0x10, 0x70, 0x65, 0x72, 0x73, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x74, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x10, 0x70, 0x65, 0x72, 0x73, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x74, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x44, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x44, 0x22, 0x41, 0x0a, 0x13, 0x53, 0x65, 0x74, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x44, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2a, 0x0a, 0x10, 0x70, 0x65, 0x72, 0x73, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x74, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x10, 0x70, 0x65, 0x72, 0x73, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x74, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x22, 0xbe, 0x01, 0x0a, 0x1c, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x74, 0x65, 0x72, 0x49, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x4b, 0x0a, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x74, 0x65, 0x72, 0x49, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x1a, 0x39, 0x0a, 0x0b, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x32, 0xc0, 0x04, 0x0a, 0x11, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x74, 0x65, 0x72, 0x12, 0x41, 0x0a, 0x04, 0x49, 0x6e, 0x69, 0x74, 0x12, 0x27, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x74, 0x65, 0x72, 0x49, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x5b, 0x0a, 0x18, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x46, 0x72, 0x6f, 0x6d, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x12, 0x1e, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x52, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x1f, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x47, 0x65, 0x74, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x47, 0x65, 0x74, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x55, 0x0a, 0x0e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x12, 0x20, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x44, 0x0a, 0x0e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x12, 0x20, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x4c, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x44, 0x12, 0x1d, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x47, 0x65, 0x74, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x44, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x47, 0x65, 0x74, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x44, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4c, 0x0a, 0x0b, 0x53, 0x65, 0x74, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x44, 0x12, 0x1d, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x53, 0x65, 0x74, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x44, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x53, 0x65, 0x74, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x44, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x35, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x76, 0x6d, 0x77, 0x61, 0x72, 0x65, 0x2d, 0x74, 0x61, 0x6e, 0x7a, 0x75, 0x2f, 0x76, 0x65, 0x6c, 0x65, 0x72, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x2f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( file_VolumeSnapshotter_proto_rawDescOnce sync.Once file_VolumeSnapshotter_proto_rawDescData = file_VolumeSnapshotter_proto_rawDesc ) func file_VolumeSnapshotter_proto_rawDescGZIP() []byte { file_VolumeSnapshotter_proto_rawDescOnce.Do(func() { file_VolumeSnapshotter_proto_rawDescData = protoimpl.X.CompressGZIP(file_VolumeSnapshotter_proto_rawDescData) }) return file_VolumeSnapshotter_proto_rawDescData } var file_VolumeSnapshotter_proto_msgTypes = make([]protoimpl.MessageInfo, 14) var file_VolumeSnapshotter_proto_goTypes = []interface{}{ (*CreateVolumeRequest)(nil), // 0: generated.CreateVolumeRequest (*CreateVolumeResponse)(nil), // 1: generated.CreateVolumeResponse (*GetVolumeInfoRequest)(nil), // 2: generated.GetVolumeInfoRequest (*GetVolumeInfoResponse)(nil), // 3: generated.GetVolumeInfoResponse (*CreateSnapshotRequest)(nil), // 4: generated.CreateSnapshotRequest (*CreateSnapshotResponse)(nil), // 5: generated.CreateSnapshotResponse (*DeleteSnapshotRequest)(nil), // 6: generated.DeleteSnapshotRequest (*GetVolumeIDRequest)(nil), // 7: generated.GetVolumeIDRequest (*GetVolumeIDResponse)(nil), // 8: generated.GetVolumeIDResponse (*SetVolumeIDRequest)(nil), // 9: generated.SetVolumeIDRequest (*SetVolumeIDResponse)(nil), // 10: generated.SetVolumeIDResponse (*VolumeSnapshotterInitRequest)(nil), // 11: generated.VolumeSnapshotterInitRequest nil, // 12: generated.CreateSnapshotRequest.TagsEntry nil, // 13: generated.VolumeSnapshotterInitRequest.ConfigEntry (*Empty)(nil), // 14: generated.Empty } var file_VolumeSnapshotter_proto_depIdxs = []int32{ 12, // 0: generated.CreateSnapshotRequest.tags:type_name -> generated.CreateSnapshotRequest.TagsEntry 13, // 1: generated.VolumeSnapshotterInitRequest.config:type_name -> generated.VolumeSnapshotterInitRequest.ConfigEntry 11, // 2: generated.VolumeSnapshotter.Init:input_type -> generated.VolumeSnapshotterInitRequest 0, // 3: generated.VolumeSnapshotter.CreateVolumeFromSnapshot:input_type -> generated.CreateVolumeRequest 2, // 4: generated.VolumeSnapshotter.GetVolumeInfo:input_type -> generated.GetVolumeInfoRequest 4, // 5: generated.VolumeSnapshotter.CreateSnapshot:input_type -> generated.CreateSnapshotRequest 6, // 6: generated.VolumeSnapshotter.DeleteSnapshot:input_type -> generated.DeleteSnapshotRequest 7, // 7: generated.VolumeSnapshotter.GetVolumeID:input_type -> generated.GetVolumeIDRequest 9, // 8: generated.VolumeSnapshotter.SetVolumeID:input_type -> generated.SetVolumeIDRequest 14, // 9: generated.VolumeSnapshotter.Init:output_type -> generated.Empty 1, // 10: generated.VolumeSnapshotter.CreateVolumeFromSnapshot:output_type -> generated.CreateVolumeResponse 3, // 11: generated.VolumeSnapshotter.GetVolumeInfo:output_type -> generated.GetVolumeInfoResponse 5, // 12: generated.VolumeSnapshotter.CreateSnapshot:output_type -> generated.CreateSnapshotResponse 14, // 13: generated.VolumeSnapshotter.DeleteSnapshot:output_type -> generated.Empty 8, // 14: generated.VolumeSnapshotter.GetVolumeID:output_type -> generated.GetVolumeIDResponse 10, // 15: generated.VolumeSnapshotter.SetVolumeID:output_type -> generated.SetVolumeIDResponse 9, // [9:16] is the sub-list for method output_type 2, // [2:9] is the sub-list for method input_type 2, // [2:2] is the sub-list for extension type_name 2, // [2:2] is the sub-list for extension extendee 0, // [0:2] is the sub-list for field type_name } func init() { file_VolumeSnapshotter_proto_init() } func file_VolumeSnapshotter_proto_init() { if File_VolumeSnapshotter_proto != nil { return } file_Shared_proto_init() if !protoimpl.UnsafeEnabled { file_VolumeSnapshotter_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*CreateVolumeRequest); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_VolumeSnapshotter_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*CreateVolumeResponse); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_VolumeSnapshotter_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*GetVolumeInfoRequest); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_VolumeSnapshotter_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*GetVolumeInfoResponse); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_VolumeSnapshotter_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*CreateSnapshotRequest); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_VolumeSnapshotter_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*CreateSnapshotResponse); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_VolumeSnapshotter_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*DeleteSnapshotRequest); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_VolumeSnapshotter_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*GetVolumeIDRequest); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_VolumeSnapshotter_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*GetVolumeIDResponse); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_VolumeSnapshotter_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*SetVolumeIDRequest); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_VolumeSnapshotter_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*SetVolumeIDResponse); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_VolumeSnapshotter_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*VolumeSnapshotterInitRequest); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_VolumeSnapshotter_proto_rawDesc, NumEnums: 0, NumMessages: 14, NumExtensions: 0, NumServices: 1, }, GoTypes: file_VolumeSnapshotter_proto_goTypes, DependencyIndexes: file_VolumeSnapshotter_proto_depIdxs, MessageInfos: file_VolumeSnapshotter_proto_msgTypes, }.Build() File_VolumeSnapshotter_proto = out.File file_VolumeSnapshotter_proto_rawDesc = nil file_VolumeSnapshotter_proto_goTypes = nil file_VolumeSnapshotter_proto_depIdxs = nil } ================================================ FILE: pkg/plugin/generated/VolumeSnapshotter_grpc.pb.go ================================================ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.3.0 // - protoc v4.25.2 // source: VolumeSnapshotter.proto package generated import ( context "context" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" ) // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. // Requires gRPC-Go v1.32.0 or later. const _ = grpc.SupportPackageIsVersion7 const ( VolumeSnapshotter_Init_FullMethodName = "/generated.VolumeSnapshotter/Init" VolumeSnapshotter_CreateVolumeFromSnapshot_FullMethodName = "/generated.VolumeSnapshotter/CreateVolumeFromSnapshot" VolumeSnapshotter_GetVolumeInfo_FullMethodName = "/generated.VolumeSnapshotter/GetVolumeInfo" VolumeSnapshotter_CreateSnapshot_FullMethodName = "/generated.VolumeSnapshotter/CreateSnapshot" VolumeSnapshotter_DeleteSnapshot_FullMethodName = "/generated.VolumeSnapshotter/DeleteSnapshot" VolumeSnapshotter_GetVolumeID_FullMethodName = "/generated.VolumeSnapshotter/GetVolumeID" VolumeSnapshotter_SetVolumeID_FullMethodName = "/generated.VolumeSnapshotter/SetVolumeID" ) // VolumeSnapshotterClient is the client API for VolumeSnapshotter service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. type VolumeSnapshotterClient interface { Init(ctx context.Context, in *VolumeSnapshotterInitRequest, opts ...grpc.CallOption) (*Empty, error) CreateVolumeFromSnapshot(ctx context.Context, in *CreateVolumeRequest, opts ...grpc.CallOption) (*CreateVolumeResponse, error) GetVolumeInfo(ctx context.Context, in *GetVolumeInfoRequest, opts ...grpc.CallOption) (*GetVolumeInfoResponse, error) CreateSnapshot(ctx context.Context, in *CreateSnapshotRequest, opts ...grpc.CallOption) (*CreateSnapshotResponse, error) DeleteSnapshot(ctx context.Context, in *DeleteSnapshotRequest, opts ...grpc.CallOption) (*Empty, error) GetVolumeID(ctx context.Context, in *GetVolumeIDRequest, opts ...grpc.CallOption) (*GetVolumeIDResponse, error) SetVolumeID(ctx context.Context, in *SetVolumeIDRequest, opts ...grpc.CallOption) (*SetVolumeIDResponse, error) } type volumeSnapshotterClient struct { cc grpc.ClientConnInterface } func NewVolumeSnapshotterClient(cc grpc.ClientConnInterface) VolumeSnapshotterClient { return &volumeSnapshotterClient{cc} } func (c *volumeSnapshotterClient) Init(ctx context.Context, in *VolumeSnapshotterInitRequest, opts ...grpc.CallOption) (*Empty, error) { out := new(Empty) err := c.cc.Invoke(ctx, VolumeSnapshotter_Init_FullMethodName, in, out, opts...) if err != nil { return nil, err } return out, nil } func (c *volumeSnapshotterClient) CreateVolumeFromSnapshot(ctx context.Context, in *CreateVolumeRequest, opts ...grpc.CallOption) (*CreateVolumeResponse, error) { out := new(CreateVolumeResponse) err := c.cc.Invoke(ctx, VolumeSnapshotter_CreateVolumeFromSnapshot_FullMethodName, in, out, opts...) if err != nil { return nil, err } return out, nil } func (c *volumeSnapshotterClient) GetVolumeInfo(ctx context.Context, in *GetVolumeInfoRequest, opts ...grpc.CallOption) (*GetVolumeInfoResponse, error) { out := new(GetVolumeInfoResponse) err := c.cc.Invoke(ctx, VolumeSnapshotter_GetVolumeInfo_FullMethodName, in, out, opts...) if err != nil { return nil, err } return out, nil } func (c *volumeSnapshotterClient) CreateSnapshot(ctx context.Context, in *CreateSnapshotRequest, opts ...grpc.CallOption) (*CreateSnapshotResponse, error) { out := new(CreateSnapshotResponse) err := c.cc.Invoke(ctx, VolumeSnapshotter_CreateSnapshot_FullMethodName, in, out, opts...) if err != nil { return nil, err } return out, nil } func (c *volumeSnapshotterClient) DeleteSnapshot(ctx context.Context, in *DeleteSnapshotRequest, opts ...grpc.CallOption) (*Empty, error) { out := new(Empty) err := c.cc.Invoke(ctx, VolumeSnapshotter_DeleteSnapshot_FullMethodName, in, out, opts...) if err != nil { return nil, err } return out, nil } func (c *volumeSnapshotterClient) GetVolumeID(ctx context.Context, in *GetVolumeIDRequest, opts ...grpc.CallOption) (*GetVolumeIDResponse, error) { out := new(GetVolumeIDResponse) err := c.cc.Invoke(ctx, VolumeSnapshotter_GetVolumeID_FullMethodName, in, out, opts...) if err != nil { return nil, err } return out, nil } func (c *volumeSnapshotterClient) SetVolumeID(ctx context.Context, in *SetVolumeIDRequest, opts ...grpc.CallOption) (*SetVolumeIDResponse, error) { out := new(SetVolumeIDResponse) err := c.cc.Invoke(ctx, VolumeSnapshotter_SetVolumeID_FullMethodName, in, out, opts...) if err != nil { return nil, err } return out, nil } // VolumeSnapshotterServer is the server API for VolumeSnapshotter service. // All implementations should embed UnimplementedVolumeSnapshotterServer // for forward compatibility type VolumeSnapshotterServer interface { Init(context.Context, *VolumeSnapshotterInitRequest) (*Empty, error) CreateVolumeFromSnapshot(context.Context, *CreateVolumeRequest) (*CreateVolumeResponse, error) GetVolumeInfo(context.Context, *GetVolumeInfoRequest) (*GetVolumeInfoResponse, error) CreateSnapshot(context.Context, *CreateSnapshotRequest) (*CreateSnapshotResponse, error) DeleteSnapshot(context.Context, *DeleteSnapshotRequest) (*Empty, error) GetVolumeID(context.Context, *GetVolumeIDRequest) (*GetVolumeIDResponse, error) SetVolumeID(context.Context, *SetVolumeIDRequest) (*SetVolumeIDResponse, error) } // UnimplementedVolumeSnapshotterServer should be embedded to have forward compatible implementations. type UnimplementedVolumeSnapshotterServer struct { } func (UnimplementedVolumeSnapshotterServer) Init(context.Context, *VolumeSnapshotterInitRequest) (*Empty, error) { return nil, status.Errorf(codes.Unimplemented, "method Init not implemented") } func (UnimplementedVolumeSnapshotterServer) CreateVolumeFromSnapshot(context.Context, *CreateVolumeRequest) (*CreateVolumeResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method CreateVolumeFromSnapshot not implemented") } func (UnimplementedVolumeSnapshotterServer) GetVolumeInfo(context.Context, *GetVolumeInfoRequest) (*GetVolumeInfoResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method GetVolumeInfo not implemented") } func (UnimplementedVolumeSnapshotterServer) CreateSnapshot(context.Context, *CreateSnapshotRequest) (*CreateSnapshotResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method CreateSnapshot not implemented") } func (UnimplementedVolumeSnapshotterServer) DeleteSnapshot(context.Context, *DeleteSnapshotRequest) (*Empty, error) { return nil, status.Errorf(codes.Unimplemented, "method DeleteSnapshot not implemented") } func (UnimplementedVolumeSnapshotterServer) GetVolumeID(context.Context, *GetVolumeIDRequest) (*GetVolumeIDResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method GetVolumeID not implemented") } func (UnimplementedVolumeSnapshotterServer) SetVolumeID(context.Context, *SetVolumeIDRequest) (*SetVolumeIDResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method SetVolumeID not implemented") } // UnsafeVolumeSnapshotterServer may be embedded to opt out of forward compatibility for this service. // Use of this interface is not recommended, as added methods to VolumeSnapshotterServer will // result in compilation errors. type UnsafeVolumeSnapshotterServer interface { mustEmbedUnimplementedVolumeSnapshotterServer() } func RegisterVolumeSnapshotterServer(s grpc.ServiceRegistrar, srv VolumeSnapshotterServer) { s.RegisterService(&VolumeSnapshotter_ServiceDesc, srv) } func _VolumeSnapshotter_Init_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(VolumeSnapshotterInitRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(VolumeSnapshotterServer).Init(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: VolumeSnapshotter_Init_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(VolumeSnapshotterServer).Init(ctx, req.(*VolumeSnapshotterInitRequest)) } return interceptor(ctx, in, info, handler) } func _VolumeSnapshotter_CreateVolumeFromSnapshot_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(CreateVolumeRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(VolumeSnapshotterServer).CreateVolumeFromSnapshot(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: VolumeSnapshotter_CreateVolumeFromSnapshot_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(VolumeSnapshotterServer).CreateVolumeFromSnapshot(ctx, req.(*CreateVolumeRequest)) } return interceptor(ctx, in, info, handler) } func _VolumeSnapshotter_GetVolumeInfo_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(GetVolumeInfoRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(VolumeSnapshotterServer).GetVolumeInfo(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: VolumeSnapshotter_GetVolumeInfo_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(VolumeSnapshotterServer).GetVolumeInfo(ctx, req.(*GetVolumeInfoRequest)) } return interceptor(ctx, in, info, handler) } func _VolumeSnapshotter_CreateSnapshot_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(CreateSnapshotRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(VolumeSnapshotterServer).CreateSnapshot(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: VolumeSnapshotter_CreateSnapshot_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(VolumeSnapshotterServer).CreateSnapshot(ctx, req.(*CreateSnapshotRequest)) } return interceptor(ctx, in, info, handler) } func _VolumeSnapshotter_DeleteSnapshot_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(DeleteSnapshotRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(VolumeSnapshotterServer).DeleteSnapshot(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: VolumeSnapshotter_DeleteSnapshot_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(VolumeSnapshotterServer).DeleteSnapshot(ctx, req.(*DeleteSnapshotRequest)) } return interceptor(ctx, in, info, handler) } func _VolumeSnapshotter_GetVolumeID_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(GetVolumeIDRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(VolumeSnapshotterServer).GetVolumeID(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: VolumeSnapshotter_GetVolumeID_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(VolumeSnapshotterServer).GetVolumeID(ctx, req.(*GetVolumeIDRequest)) } return interceptor(ctx, in, info, handler) } func _VolumeSnapshotter_SetVolumeID_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(SetVolumeIDRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(VolumeSnapshotterServer).SetVolumeID(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: VolumeSnapshotter_SetVolumeID_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(VolumeSnapshotterServer).SetVolumeID(ctx, req.(*SetVolumeIDRequest)) } return interceptor(ctx, in, info, handler) } // VolumeSnapshotter_ServiceDesc is the grpc.ServiceDesc for VolumeSnapshotter service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) var VolumeSnapshotter_ServiceDesc = grpc.ServiceDesc{ ServiceName: "generated.VolumeSnapshotter", HandlerType: (*VolumeSnapshotterServer)(nil), Methods: []grpc.MethodDesc{ { MethodName: "Init", Handler: _VolumeSnapshotter_Init_Handler, }, { MethodName: "CreateVolumeFromSnapshot", Handler: _VolumeSnapshotter_CreateVolumeFromSnapshot_Handler, }, { MethodName: "GetVolumeInfo", Handler: _VolumeSnapshotter_GetVolumeInfo_Handler, }, { MethodName: "CreateSnapshot", Handler: _VolumeSnapshotter_CreateSnapshot_Handler, }, { MethodName: "DeleteSnapshot", Handler: _VolumeSnapshotter_DeleteSnapshot_Handler, }, { MethodName: "GetVolumeID", Handler: _VolumeSnapshotter_GetVolumeID_Handler, }, { MethodName: "SetVolumeID", Handler: _VolumeSnapshotter_SetVolumeID_Handler, }, }, Streams: []grpc.StreamDesc{}, Metadata: "VolumeSnapshotter.proto", } ================================================ FILE: pkg/plugin/generated/backupitemaction/v2/BackupItemAction.pb.go ================================================ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.33.0 // protoc v4.25.2 // source: backupitemaction/v2/BackupItemAction.proto package v2 import ( generated "github.com/vmware-tanzu/velero/pkg/plugin/generated" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" emptypb "google.golang.org/protobuf/types/known/emptypb" reflect "reflect" sync "sync" ) const ( // Verify that this generated code is sufficiently up-to-date. _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) // Verify that runtime/protoimpl is sufficiently up-to-date. _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) type ExecuteRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` Item []byte `protobuf:"bytes,2,opt,name=item,proto3" json:"item,omitempty"` Backup []byte `protobuf:"bytes,3,opt,name=backup,proto3" json:"backup,omitempty"` } func (x *ExecuteRequest) Reset() { *x = ExecuteRequest{} if protoimpl.UnsafeEnabled { mi := &file_backupitemaction_v2_BackupItemAction_proto_msgTypes[0] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *ExecuteRequest) String() string { return protoimpl.X.MessageStringOf(x) } func (*ExecuteRequest) ProtoMessage() {} func (x *ExecuteRequest) ProtoReflect() protoreflect.Message { mi := &file_backupitemaction_v2_BackupItemAction_proto_msgTypes[0] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use ExecuteRequest.ProtoReflect.Descriptor instead. func (*ExecuteRequest) Descriptor() ([]byte, []int) { return file_backupitemaction_v2_BackupItemAction_proto_rawDescGZIP(), []int{0} } func (x *ExecuteRequest) GetPlugin() string { if x != nil { return x.Plugin } return "" } func (x *ExecuteRequest) GetItem() []byte { if x != nil { return x.Item } return nil } func (x *ExecuteRequest) GetBackup() []byte { if x != nil { return x.Backup } return nil } type ExecuteResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Item []byte `protobuf:"bytes,1,opt,name=item,proto3" json:"item,omitempty"` AdditionalItems []*generated.ResourceIdentifier `protobuf:"bytes,2,rep,name=additionalItems,proto3" json:"additionalItems,omitempty"` OperationID string `protobuf:"bytes,3,opt,name=operationID,proto3" json:"operationID,omitempty"` PostOperationItems []*generated.ResourceIdentifier `protobuf:"bytes,4,rep,name=postOperationItems,proto3" json:"postOperationItems,omitempty"` } func (x *ExecuteResponse) Reset() { *x = ExecuteResponse{} if protoimpl.UnsafeEnabled { mi := &file_backupitemaction_v2_BackupItemAction_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *ExecuteResponse) String() string { return protoimpl.X.MessageStringOf(x) } func (*ExecuteResponse) ProtoMessage() {} func (x *ExecuteResponse) ProtoReflect() protoreflect.Message { mi := &file_backupitemaction_v2_BackupItemAction_proto_msgTypes[1] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use ExecuteResponse.ProtoReflect.Descriptor instead. func (*ExecuteResponse) Descriptor() ([]byte, []int) { return file_backupitemaction_v2_BackupItemAction_proto_rawDescGZIP(), []int{1} } func (x *ExecuteResponse) GetItem() []byte { if x != nil { return x.Item } return nil } func (x *ExecuteResponse) GetAdditionalItems() []*generated.ResourceIdentifier { if x != nil { return x.AdditionalItems } return nil } func (x *ExecuteResponse) GetOperationID() string { if x != nil { return x.OperationID } return "" } func (x *ExecuteResponse) GetPostOperationItems() []*generated.ResourceIdentifier { if x != nil { return x.PostOperationItems } return nil } type BackupItemActionAppliesToRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` } func (x *BackupItemActionAppliesToRequest) Reset() { *x = BackupItemActionAppliesToRequest{} if protoimpl.UnsafeEnabled { mi := &file_backupitemaction_v2_BackupItemAction_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *BackupItemActionAppliesToRequest) String() string { return protoimpl.X.MessageStringOf(x) } func (*BackupItemActionAppliesToRequest) ProtoMessage() {} func (x *BackupItemActionAppliesToRequest) ProtoReflect() protoreflect.Message { mi := &file_backupitemaction_v2_BackupItemAction_proto_msgTypes[2] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use BackupItemActionAppliesToRequest.ProtoReflect.Descriptor instead. func (*BackupItemActionAppliesToRequest) Descriptor() ([]byte, []int) { return file_backupitemaction_v2_BackupItemAction_proto_rawDescGZIP(), []int{2} } func (x *BackupItemActionAppliesToRequest) GetPlugin() string { if x != nil { return x.Plugin } return "" } type BackupItemActionAppliesToResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields ResourceSelector *generated.ResourceSelector `protobuf:"bytes,1,opt,name=ResourceSelector,proto3" json:"ResourceSelector,omitempty"` } func (x *BackupItemActionAppliesToResponse) Reset() { *x = BackupItemActionAppliesToResponse{} if protoimpl.UnsafeEnabled { mi := &file_backupitemaction_v2_BackupItemAction_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *BackupItemActionAppliesToResponse) String() string { return protoimpl.X.MessageStringOf(x) } func (*BackupItemActionAppliesToResponse) ProtoMessage() {} func (x *BackupItemActionAppliesToResponse) ProtoReflect() protoreflect.Message { mi := &file_backupitemaction_v2_BackupItemAction_proto_msgTypes[3] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use BackupItemActionAppliesToResponse.ProtoReflect.Descriptor instead. func (*BackupItemActionAppliesToResponse) Descriptor() ([]byte, []int) { return file_backupitemaction_v2_BackupItemAction_proto_rawDescGZIP(), []int{3} } func (x *BackupItemActionAppliesToResponse) GetResourceSelector() *generated.ResourceSelector { if x != nil { return x.ResourceSelector } return nil } type BackupItemActionProgressRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` OperationID string `protobuf:"bytes,2,opt,name=operationID,proto3" json:"operationID,omitempty"` Backup []byte `protobuf:"bytes,3,opt,name=backup,proto3" json:"backup,omitempty"` } func (x *BackupItemActionProgressRequest) Reset() { *x = BackupItemActionProgressRequest{} if protoimpl.UnsafeEnabled { mi := &file_backupitemaction_v2_BackupItemAction_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *BackupItemActionProgressRequest) String() string { return protoimpl.X.MessageStringOf(x) } func (*BackupItemActionProgressRequest) ProtoMessage() {} func (x *BackupItemActionProgressRequest) ProtoReflect() protoreflect.Message { mi := &file_backupitemaction_v2_BackupItemAction_proto_msgTypes[4] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use BackupItemActionProgressRequest.ProtoReflect.Descriptor instead. func (*BackupItemActionProgressRequest) Descriptor() ([]byte, []int) { return file_backupitemaction_v2_BackupItemAction_proto_rawDescGZIP(), []int{4} } func (x *BackupItemActionProgressRequest) GetPlugin() string { if x != nil { return x.Plugin } return "" } func (x *BackupItemActionProgressRequest) GetOperationID() string { if x != nil { return x.OperationID } return "" } func (x *BackupItemActionProgressRequest) GetBackup() []byte { if x != nil { return x.Backup } return nil } type BackupItemActionProgressResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Progress *generated.OperationProgress `protobuf:"bytes,1,opt,name=progress,proto3" json:"progress,omitempty"` } func (x *BackupItemActionProgressResponse) Reset() { *x = BackupItemActionProgressResponse{} if protoimpl.UnsafeEnabled { mi := &file_backupitemaction_v2_BackupItemAction_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *BackupItemActionProgressResponse) String() string { return protoimpl.X.MessageStringOf(x) } func (*BackupItemActionProgressResponse) ProtoMessage() {} func (x *BackupItemActionProgressResponse) ProtoReflect() protoreflect.Message { mi := &file_backupitemaction_v2_BackupItemAction_proto_msgTypes[5] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use BackupItemActionProgressResponse.ProtoReflect.Descriptor instead. func (*BackupItemActionProgressResponse) Descriptor() ([]byte, []int) { return file_backupitemaction_v2_BackupItemAction_proto_rawDescGZIP(), []int{5} } func (x *BackupItemActionProgressResponse) GetProgress() *generated.OperationProgress { if x != nil { return x.Progress } return nil } type BackupItemActionCancelRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` OperationID string `protobuf:"bytes,2,opt,name=operationID,proto3" json:"operationID,omitempty"` Backup []byte `protobuf:"bytes,3,opt,name=backup,proto3" json:"backup,omitempty"` } func (x *BackupItemActionCancelRequest) Reset() { *x = BackupItemActionCancelRequest{} if protoimpl.UnsafeEnabled { mi := &file_backupitemaction_v2_BackupItemAction_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *BackupItemActionCancelRequest) String() string { return protoimpl.X.MessageStringOf(x) } func (*BackupItemActionCancelRequest) ProtoMessage() {} func (x *BackupItemActionCancelRequest) ProtoReflect() protoreflect.Message { mi := &file_backupitemaction_v2_BackupItemAction_proto_msgTypes[6] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use BackupItemActionCancelRequest.ProtoReflect.Descriptor instead. func (*BackupItemActionCancelRequest) Descriptor() ([]byte, []int) { return file_backupitemaction_v2_BackupItemAction_proto_rawDescGZIP(), []int{6} } func (x *BackupItemActionCancelRequest) GetPlugin() string { if x != nil { return x.Plugin } return "" } func (x *BackupItemActionCancelRequest) GetOperationID() string { if x != nil { return x.OperationID } return "" } func (x *BackupItemActionCancelRequest) GetBackup() []byte { if x != nil { return x.Backup } return nil } var File_backupitemaction_v2_BackupItemAction_proto protoreflect.FileDescriptor var file_backupitemaction_v2_BackupItemAction_proto_rawDesc = []byte{ 0x0a, 0x2a, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x69, 0x74, 0x65, 0x6d, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x76, 0x32, 0x2f, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x02, 0x76, 0x32, 0x1a, 0x0c, 0x53, 0x68, 0x61, 0x72, 0x65, 0x64, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x54, 0x0a, 0x0e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x22, 0xdf, 0x01, 0x0a, 0x0f, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x12, 0x47, 0x0a, 0x0f, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x52, 0x0f, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x12, 0x20, 0x0a, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x12, 0x4d, 0x0a, 0x12, 0x70, 0x6f, 0x73, 0x74, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x52, 0x12, 0x70, 0x6f, 0x73, 0x74, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x22, 0x3a, 0x0a, 0x20, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x22, 0x6c, 0x0a, 0x21, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x47, 0x0a, 0x10, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x10, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x22, 0x73, 0x0a, 0x1f, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x20, 0x0a, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x22, 0x5c, 0x0a, 0x20, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x38, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x22, 0x71, 0x0a, 0x1d, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x20, 0x0a, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x32, 0xbc, 0x02, 0x0a, 0x10, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x58, 0x0a, 0x09, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x12, 0x24, 0x2e, 0x76, 0x32, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x76, 0x32, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x32, 0x0a, 0x07, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x12, 0x12, 0x2e, 0x76, 0x32, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x76, 0x32, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x55, 0x0a, 0x08, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, 0x23, 0x2e, 0x76, 0x32, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x76, 0x32, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x43, 0x0a, 0x06, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x12, 0x21, 0x2e, 0x76, 0x32, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x42, 0x49, 0x5a, 0x47, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x76, 0x6d, 0x77, 0x61, 0x72, 0x65, 0x2d, 0x74, 0x61, 0x6e, 0x7a, 0x75, 0x2f, 0x76, 0x65, 0x6c, 0x65, 0x72, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x2f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2f, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x69, 0x74, 0x65, 0x6d, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x76, 0x32, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( file_backupitemaction_v2_BackupItemAction_proto_rawDescOnce sync.Once file_backupitemaction_v2_BackupItemAction_proto_rawDescData = file_backupitemaction_v2_BackupItemAction_proto_rawDesc ) func file_backupitemaction_v2_BackupItemAction_proto_rawDescGZIP() []byte { file_backupitemaction_v2_BackupItemAction_proto_rawDescOnce.Do(func() { file_backupitemaction_v2_BackupItemAction_proto_rawDescData = protoimpl.X.CompressGZIP(file_backupitemaction_v2_BackupItemAction_proto_rawDescData) }) return file_backupitemaction_v2_BackupItemAction_proto_rawDescData } var file_backupitemaction_v2_BackupItemAction_proto_msgTypes = make([]protoimpl.MessageInfo, 7) var file_backupitemaction_v2_BackupItemAction_proto_goTypes = []interface{}{ (*ExecuteRequest)(nil), // 0: v2.ExecuteRequest (*ExecuteResponse)(nil), // 1: v2.ExecuteResponse (*BackupItemActionAppliesToRequest)(nil), // 2: v2.BackupItemActionAppliesToRequest (*BackupItemActionAppliesToResponse)(nil), // 3: v2.BackupItemActionAppliesToResponse (*BackupItemActionProgressRequest)(nil), // 4: v2.BackupItemActionProgressRequest (*BackupItemActionProgressResponse)(nil), // 5: v2.BackupItemActionProgressResponse (*BackupItemActionCancelRequest)(nil), // 6: v2.BackupItemActionCancelRequest (*generated.ResourceIdentifier)(nil), // 7: generated.ResourceIdentifier (*generated.ResourceSelector)(nil), // 8: generated.ResourceSelector (*generated.OperationProgress)(nil), // 9: generated.OperationProgress (*emptypb.Empty)(nil), // 10: google.protobuf.Empty } var file_backupitemaction_v2_BackupItemAction_proto_depIdxs = []int32{ 7, // 0: v2.ExecuteResponse.additionalItems:type_name -> generated.ResourceIdentifier 7, // 1: v2.ExecuteResponse.postOperationItems:type_name -> generated.ResourceIdentifier 8, // 2: v2.BackupItemActionAppliesToResponse.ResourceSelector:type_name -> generated.ResourceSelector 9, // 3: v2.BackupItemActionProgressResponse.progress:type_name -> generated.OperationProgress 2, // 4: v2.BackupItemAction.AppliesTo:input_type -> v2.BackupItemActionAppliesToRequest 0, // 5: v2.BackupItemAction.Execute:input_type -> v2.ExecuteRequest 4, // 6: v2.BackupItemAction.Progress:input_type -> v2.BackupItemActionProgressRequest 6, // 7: v2.BackupItemAction.Cancel:input_type -> v2.BackupItemActionCancelRequest 3, // 8: v2.BackupItemAction.AppliesTo:output_type -> v2.BackupItemActionAppliesToResponse 1, // 9: v2.BackupItemAction.Execute:output_type -> v2.ExecuteResponse 5, // 10: v2.BackupItemAction.Progress:output_type -> v2.BackupItemActionProgressResponse 10, // 11: v2.BackupItemAction.Cancel:output_type -> google.protobuf.Empty 8, // [8:12] is the sub-list for method output_type 4, // [4:8] is the sub-list for method input_type 4, // [4:4] is the sub-list for extension type_name 4, // [4:4] is the sub-list for extension extendee 0, // [0:4] is the sub-list for field type_name } func init() { file_backupitemaction_v2_BackupItemAction_proto_init() } func file_backupitemaction_v2_BackupItemAction_proto_init() { if File_backupitemaction_v2_BackupItemAction_proto != nil { return } if !protoimpl.UnsafeEnabled { file_backupitemaction_v2_BackupItemAction_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ExecuteRequest); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_backupitemaction_v2_BackupItemAction_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ExecuteResponse); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_backupitemaction_v2_BackupItemAction_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*BackupItemActionAppliesToRequest); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_backupitemaction_v2_BackupItemAction_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*BackupItemActionAppliesToResponse); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_backupitemaction_v2_BackupItemAction_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*BackupItemActionProgressRequest); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_backupitemaction_v2_BackupItemAction_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*BackupItemActionProgressResponse); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_backupitemaction_v2_BackupItemAction_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*BackupItemActionCancelRequest); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_backupitemaction_v2_BackupItemAction_proto_rawDesc, NumEnums: 0, NumMessages: 7, NumExtensions: 0, NumServices: 1, }, GoTypes: file_backupitemaction_v2_BackupItemAction_proto_goTypes, DependencyIndexes: file_backupitemaction_v2_BackupItemAction_proto_depIdxs, MessageInfos: file_backupitemaction_v2_BackupItemAction_proto_msgTypes, }.Build() File_backupitemaction_v2_BackupItemAction_proto = out.File file_backupitemaction_v2_BackupItemAction_proto_rawDesc = nil file_backupitemaction_v2_BackupItemAction_proto_goTypes = nil file_backupitemaction_v2_BackupItemAction_proto_depIdxs = nil } ================================================ FILE: pkg/plugin/generated/backupitemaction/v2/BackupItemAction_grpc.pb.go ================================================ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.3.0 // - protoc v4.25.2 // source: backupitemaction/v2/BackupItemAction.proto package v2 import ( context "context" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" emptypb "google.golang.org/protobuf/types/known/emptypb" ) // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. // Requires gRPC-Go v1.32.0 or later. const _ = grpc.SupportPackageIsVersion7 const ( BackupItemAction_AppliesTo_FullMethodName = "/v2.BackupItemAction/AppliesTo" BackupItemAction_Execute_FullMethodName = "/v2.BackupItemAction/Execute" BackupItemAction_Progress_FullMethodName = "/v2.BackupItemAction/Progress" BackupItemAction_Cancel_FullMethodName = "/v2.BackupItemAction/Cancel" ) // BackupItemActionClient is the client API for BackupItemAction service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. type BackupItemActionClient interface { AppliesTo(ctx context.Context, in *BackupItemActionAppliesToRequest, opts ...grpc.CallOption) (*BackupItemActionAppliesToResponse, error) Execute(ctx context.Context, in *ExecuteRequest, opts ...grpc.CallOption) (*ExecuteResponse, error) Progress(ctx context.Context, in *BackupItemActionProgressRequest, opts ...grpc.CallOption) (*BackupItemActionProgressResponse, error) Cancel(ctx context.Context, in *BackupItemActionCancelRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) } type backupItemActionClient struct { cc grpc.ClientConnInterface } func NewBackupItemActionClient(cc grpc.ClientConnInterface) BackupItemActionClient { return &backupItemActionClient{cc} } func (c *backupItemActionClient) AppliesTo(ctx context.Context, in *BackupItemActionAppliesToRequest, opts ...grpc.CallOption) (*BackupItemActionAppliesToResponse, error) { out := new(BackupItemActionAppliesToResponse) err := c.cc.Invoke(ctx, BackupItemAction_AppliesTo_FullMethodName, in, out, opts...) if err != nil { return nil, err } return out, nil } func (c *backupItemActionClient) Execute(ctx context.Context, in *ExecuteRequest, opts ...grpc.CallOption) (*ExecuteResponse, error) { out := new(ExecuteResponse) err := c.cc.Invoke(ctx, BackupItemAction_Execute_FullMethodName, in, out, opts...) if err != nil { return nil, err } return out, nil } func (c *backupItemActionClient) Progress(ctx context.Context, in *BackupItemActionProgressRequest, opts ...grpc.CallOption) (*BackupItemActionProgressResponse, error) { out := new(BackupItemActionProgressResponse) err := c.cc.Invoke(ctx, BackupItemAction_Progress_FullMethodName, in, out, opts...) if err != nil { return nil, err } return out, nil } func (c *backupItemActionClient) Cancel(ctx context.Context, in *BackupItemActionCancelRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { out := new(emptypb.Empty) err := c.cc.Invoke(ctx, BackupItemAction_Cancel_FullMethodName, in, out, opts...) if err != nil { return nil, err } return out, nil } // BackupItemActionServer is the server API for BackupItemAction service. // All implementations should embed UnimplementedBackupItemActionServer // for forward compatibility type BackupItemActionServer interface { AppliesTo(context.Context, *BackupItemActionAppliesToRequest) (*BackupItemActionAppliesToResponse, error) Execute(context.Context, *ExecuteRequest) (*ExecuteResponse, error) Progress(context.Context, *BackupItemActionProgressRequest) (*BackupItemActionProgressResponse, error) Cancel(context.Context, *BackupItemActionCancelRequest) (*emptypb.Empty, error) } // UnimplementedBackupItemActionServer should be embedded to have forward compatible implementations. type UnimplementedBackupItemActionServer struct { } func (UnimplementedBackupItemActionServer) AppliesTo(context.Context, *BackupItemActionAppliesToRequest) (*BackupItemActionAppliesToResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method AppliesTo not implemented") } func (UnimplementedBackupItemActionServer) Execute(context.Context, *ExecuteRequest) (*ExecuteResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method Execute not implemented") } func (UnimplementedBackupItemActionServer) Progress(context.Context, *BackupItemActionProgressRequest) (*BackupItemActionProgressResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method Progress not implemented") } func (UnimplementedBackupItemActionServer) Cancel(context.Context, *BackupItemActionCancelRequest) (*emptypb.Empty, error) { return nil, status.Errorf(codes.Unimplemented, "method Cancel not implemented") } // UnsafeBackupItemActionServer may be embedded to opt out of forward compatibility for this service. // Use of this interface is not recommended, as added methods to BackupItemActionServer will // result in compilation errors. type UnsafeBackupItemActionServer interface { mustEmbedUnimplementedBackupItemActionServer() } func RegisterBackupItemActionServer(s grpc.ServiceRegistrar, srv BackupItemActionServer) { s.RegisterService(&BackupItemAction_ServiceDesc, srv) } func _BackupItemAction_AppliesTo_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(BackupItemActionAppliesToRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(BackupItemActionServer).AppliesTo(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: BackupItemAction_AppliesTo_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(BackupItemActionServer).AppliesTo(ctx, req.(*BackupItemActionAppliesToRequest)) } return interceptor(ctx, in, info, handler) } func _BackupItemAction_Execute_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(ExecuteRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(BackupItemActionServer).Execute(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: BackupItemAction_Execute_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(BackupItemActionServer).Execute(ctx, req.(*ExecuteRequest)) } return interceptor(ctx, in, info, handler) } func _BackupItemAction_Progress_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(BackupItemActionProgressRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(BackupItemActionServer).Progress(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: BackupItemAction_Progress_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(BackupItemActionServer).Progress(ctx, req.(*BackupItemActionProgressRequest)) } return interceptor(ctx, in, info, handler) } func _BackupItemAction_Cancel_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(BackupItemActionCancelRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(BackupItemActionServer).Cancel(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: BackupItemAction_Cancel_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(BackupItemActionServer).Cancel(ctx, req.(*BackupItemActionCancelRequest)) } return interceptor(ctx, in, info, handler) } // BackupItemAction_ServiceDesc is the grpc.ServiceDesc for BackupItemAction service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) var BackupItemAction_ServiceDesc = grpc.ServiceDesc{ ServiceName: "v2.BackupItemAction", HandlerType: (*BackupItemActionServer)(nil), Methods: []grpc.MethodDesc{ { MethodName: "AppliesTo", Handler: _BackupItemAction_AppliesTo_Handler, }, { MethodName: "Execute", Handler: _BackupItemAction_Execute_Handler, }, { MethodName: "Progress", Handler: _BackupItemAction_Progress_Handler, }, { MethodName: "Cancel", Handler: _BackupItemAction_Cancel_Handler, }, }, Streams: []grpc.StreamDesc{}, Metadata: "backupitemaction/v2/BackupItemAction.proto", } ================================================ FILE: pkg/plugin/generated/itemblockaction/v1/ItemBlockAction.pb.go ================================================ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.33.0 // protoc v4.25.2 // source: itemblockaction/v1/ItemBlockAction.proto package v1 import ( generated "github.com/vmware-tanzu/velero/pkg/plugin/generated" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" sync "sync" ) const ( // Verify that this generated code is sufficiently up-to-date. _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) // Verify that runtime/protoimpl is sufficiently up-to-date. _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) type ItemBlockActionAppliesToRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` } func (x *ItemBlockActionAppliesToRequest) Reset() { *x = ItemBlockActionAppliesToRequest{} if protoimpl.UnsafeEnabled { mi := &file_itemblockaction_v1_ItemBlockAction_proto_msgTypes[0] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *ItemBlockActionAppliesToRequest) String() string { return protoimpl.X.MessageStringOf(x) } func (*ItemBlockActionAppliesToRequest) ProtoMessage() {} func (x *ItemBlockActionAppliesToRequest) ProtoReflect() protoreflect.Message { mi := &file_itemblockaction_v1_ItemBlockAction_proto_msgTypes[0] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use ItemBlockActionAppliesToRequest.ProtoReflect.Descriptor instead. func (*ItemBlockActionAppliesToRequest) Descriptor() ([]byte, []int) { return file_itemblockaction_v1_ItemBlockAction_proto_rawDescGZIP(), []int{0} } func (x *ItemBlockActionAppliesToRequest) GetPlugin() string { if x != nil { return x.Plugin } return "" } type ItemBlockActionAppliesToResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields ResourceSelector *generated.ResourceSelector `protobuf:"bytes,1,opt,name=ResourceSelector,proto3" json:"ResourceSelector,omitempty"` } func (x *ItemBlockActionAppliesToResponse) Reset() { *x = ItemBlockActionAppliesToResponse{} if protoimpl.UnsafeEnabled { mi := &file_itemblockaction_v1_ItemBlockAction_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *ItemBlockActionAppliesToResponse) String() string { return protoimpl.X.MessageStringOf(x) } func (*ItemBlockActionAppliesToResponse) ProtoMessage() {} func (x *ItemBlockActionAppliesToResponse) ProtoReflect() protoreflect.Message { mi := &file_itemblockaction_v1_ItemBlockAction_proto_msgTypes[1] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use ItemBlockActionAppliesToResponse.ProtoReflect.Descriptor instead. func (*ItemBlockActionAppliesToResponse) Descriptor() ([]byte, []int) { return file_itemblockaction_v1_ItemBlockAction_proto_rawDescGZIP(), []int{1} } func (x *ItemBlockActionAppliesToResponse) GetResourceSelector() *generated.ResourceSelector { if x != nil { return x.ResourceSelector } return nil } type ItemBlockActionGetRelatedItemsRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` Item []byte `protobuf:"bytes,2,opt,name=item,proto3" json:"item,omitempty"` Backup []byte `protobuf:"bytes,3,opt,name=backup,proto3" json:"backup,omitempty"` } func (x *ItemBlockActionGetRelatedItemsRequest) Reset() { *x = ItemBlockActionGetRelatedItemsRequest{} if protoimpl.UnsafeEnabled { mi := &file_itemblockaction_v1_ItemBlockAction_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *ItemBlockActionGetRelatedItemsRequest) String() string { return protoimpl.X.MessageStringOf(x) } func (*ItemBlockActionGetRelatedItemsRequest) ProtoMessage() {} func (x *ItemBlockActionGetRelatedItemsRequest) ProtoReflect() protoreflect.Message { mi := &file_itemblockaction_v1_ItemBlockAction_proto_msgTypes[2] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use ItemBlockActionGetRelatedItemsRequest.ProtoReflect.Descriptor instead. func (*ItemBlockActionGetRelatedItemsRequest) Descriptor() ([]byte, []int) { return file_itemblockaction_v1_ItemBlockAction_proto_rawDescGZIP(), []int{2} } func (x *ItemBlockActionGetRelatedItemsRequest) GetPlugin() string { if x != nil { return x.Plugin } return "" } func (x *ItemBlockActionGetRelatedItemsRequest) GetItem() []byte { if x != nil { return x.Item } return nil } func (x *ItemBlockActionGetRelatedItemsRequest) GetBackup() []byte { if x != nil { return x.Backup } return nil } type ItemBlockActionGetRelatedItemsResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields RelatedItems []*generated.ResourceIdentifier `protobuf:"bytes,1,rep,name=relatedItems,proto3" json:"relatedItems,omitempty"` } func (x *ItemBlockActionGetRelatedItemsResponse) Reset() { *x = ItemBlockActionGetRelatedItemsResponse{} if protoimpl.UnsafeEnabled { mi := &file_itemblockaction_v1_ItemBlockAction_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *ItemBlockActionGetRelatedItemsResponse) String() string { return protoimpl.X.MessageStringOf(x) } func (*ItemBlockActionGetRelatedItemsResponse) ProtoMessage() {} func (x *ItemBlockActionGetRelatedItemsResponse) ProtoReflect() protoreflect.Message { mi := &file_itemblockaction_v1_ItemBlockAction_proto_msgTypes[3] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use ItemBlockActionGetRelatedItemsResponse.ProtoReflect.Descriptor instead. func (*ItemBlockActionGetRelatedItemsResponse) Descriptor() ([]byte, []int) { return file_itemblockaction_v1_ItemBlockAction_proto_rawDescGZIP(), []int{3} } func (x *ItemBlockActionGetRelatedItemsResponse) GetRelatedItems() []*generated.ResourceIdentifier { if x != nil { return x.RelatedItems } return nil } var File_itemblockaction_v1_ItemBlockAction_proto protoreflect.FileDescriptor var file_itemblockaction_v1_ItemBlockAction_proto_rawDesc = []byte{ 0x0a, 0x28, 0x69, 0x74, 0x65, 0x6d, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x76, 0x31, 0x2f, 0x49, 0x74, 0x65, 0x6d, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x02, 0x76, 0x31, 0x1a, 0x0c, 0x53, 0x68, 0x61, 0x72, 0x65, 0x64, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x39, 0x0a, 0x1f, 0x49, 0x74, 0x65, 0x6d, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x22, 0x6b, 0x0a, 0x20, 0x49, 0x74, 0x65, 0x6d, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x47, 0x0a, 0x10, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x10, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x22, 0x6b, 0x0a, 0x25, 0x49, 0x74, 0x65, 0x6d, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x47, 0x65, 0x74, 0x52, 0x65, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x22, 0x6b, 0x0a, 0x26, 0x49, 0x74, 0x65, 0x6d, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x47, 0x65, 0x74, 0x52, 0x65, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x41, 0x0a, 0x0c, 0x72, 0x65, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x52, 0x0c, 0x72, 0x65, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x32, 0xd3, 0x01, 0x0a, 0x0f, 0x49, 0x74, 0x65, 0x6d, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x56, 0x0a, 0x09, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x12, 0x23, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x74, 0x65, 0x6d, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x74, 0x65, 0x6d, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x68, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x52, 0x65, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x12, 0x29, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x74, 0x65, 0x6d, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x47, 0x65, 0x74, 0x52, 0x65, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x74, 0x65, 0x6d, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x47, 0x65, 0x74, 0x52, 0x65, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x48, 0x5a, 0x46, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x76, 0x6d, 0x77, 0x61, 0x72, 0x65, 0x2d, 0x74, 0x61, 0x6e, 0x7a, 0x75, 0x2f, 0x76, 0x65, 0x6c, 0x65, 0x72, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x2f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2f, 0x69, 0x74, 0x65, 0x6d, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( file_itemblockaction_v1_ItemBlockAction_proto_rawDescOnce sync.Once file_itemblockaction_v1_ItemBlockAction_proto_rawDescData = file_itemblockaction_v1_ItemBlockAction_proto_rawDesc ) func file_itemblockaction_v1_ItemBlockAction_proto_rawDescGZIP() []byte { file_itemblockaction_v1_ItemBlockAction_proto_rawDescOnce.Do(func() { file_itemblockaction_v1_ItemBlockAction_proto_rawDescData = protoimpl.X.CompressGZIP(file_itemblockaction_v1_ItemBlockAction_proto_rawDescData) }) return file_itemblockaction_v1_ItemBlockAction_proto_rawDescData } var file_itemblockaction_v1_ItemBlockAction_proto_msgTypes = make([]protoimpl.MessageInfo, 4) var file_itemblockaction_v1_ItemBlockAction_proto_goTypes = []interface{}{ (*ItemBlockActionAppliesToRequest)(nil), // 0: v1.ItemBlockActionAppliesToRequest (*ItemBlockActionAppliesToResponse)(nil), // 1: v1.ItemBlockActionAppliesToResponse (*ItemBlockActionGetRelatedItemsRequest)(nil), // 2: v1.ItemBlockActionGetRelatedItemsRequest (*ItemBlockActionGetRelatedItemsResponse)(nil), // 3: v1.ItemBlockActionGetRelatedItemsResponse (*generated.ResourceSelector)(nil), // 4: generated.ResourceSelector (*generated.ResourceIdentifier)(nil), // 5: generated.ResourceIdentifier } var file_itemblockaction_v1_ItemBlockAction_proto_depIdxs = []int32{ 4, // 0: v1.ItemBlockActionAppliesToResponse.ResourceSelector:type_name -> generated.ResourceSelector 5, // 1: v1.ItemBlockActionGetRelatedItemsResponse.relatedItems:type_name -> generated.ResourceIdentifier 0, // 2: v1.ItemBlockAction.AppliesTo:input_type -> v1.ItemBlockActionAppliesToRequest 2, // 3: v1.ItemBlockAction.GetRelatedItems:input_type -> v1.ItemBlockActionGetRelatedItemsRequest 1, // 4: v1.ItemBlockAction.AppliesTo:output_type -> v1.ItemBlockActionAppliesToResponse 3, // 5: v1.ItemBlockAction.GetRelatedItems:output_type -> v1.ItemBlockActionGetRelatedItemsResponse 4, // [4:6] is the sub-list for method output_type 2, // [2:4] is the sub-list for method input_type 2, // [2:2] is the sub-list for extension type_name 2, // [2:2] is the sub-list for extension extendee 0, // [0:2] is the sub-list for field type_name } func init() { file_itemblockaction_v1_ItemBlockAction_proto_init() } func file_itemblockaction_v1_ItemBlockAction_proto_init() { if File_itemblockaction_v1_ItemBlockAction_proto != nil { return } if !protoimpl.UnsafeEnabled { file_itemblockaction_v1_ItemBlockAction_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ItemBlockActionAppliesToRequest); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_itemblockaction_v1_ItemBlockAction_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ItemBlockActionAppliesToResponse); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_itemblockaction_v1_ItemBlockAction_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ItemBlockActionGetRelatedItemsRequest); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_itemblockaction_v1_ItemBlockAction_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ItemBlockActionGetRelatedItemsResponse); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_itemblockaction_v1_ItemBlockAction_proto_rawDesc, NumEnums: 0, NumMessages: 4, NumExtensions: 0, NumServices: 1, }, GoTypes: file_itemblockaction_v1_ItemBlockAction_proto_goTypes, DependencyIndexes: file_itemblockaction_v1_ItemBlockAction_proto_depIdxs, MessageInfos: file_itemblockaction_v1_ItemBlockAction_proto_msgTypes, }.Build() File_itemblockaction_v1_ItemBlockAction_proto = out.File file_itemblockaction_v1_ItemBlockAction_proto_rawDesc = nil file_itemblockaction_v1_ItemBlockAction_proto_goTypes = nil file_itemblockaction_v1_ItemBlockAction_proto_depIdxs = nil } ================================================ FILE: pkg/plugin/generated/itemblockaction/v1/ItemBlockAction_grpc.pb.go ================================================ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.3.0 // - protoc v4.25.2 // source: itemblockaction/v1/ItemBlockAction.proto package v1 import ( context "context" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" ) // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. // Requires gRPC-Go v1.32.0 or later. const _ = grpc.SupportPackageIsVersion7 const ( ItemBlockAction_AppliesTo_FullMethodName = "/v1.ItemBlockAction/AppliesTo" ItemBlockAction_GetRelatedItems_FullMethodName = "/v1.ItemBlockAction/GetRelatedItems" ) // ItemBlockActionClient is the client API for ItemBlockAction service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. type ItemBlockActionClient interface { AppliesTo(ctx context.Context, in *ItemBlockActionAppliesToRequest, opts ...grpc.CallOption) (*ItemBlockActionAppliesToResponse, error) GetRelatedItems(ctx context.Context, in *ItemBlockActionGetRelatedItemsRequest, opts ...grpc.CallOption) (*ItemBlockActionGetRelatedItemsResponse, error) } type itemBlockActionClient struct { cc grpc.ClientConnInterface } func NewItemBlockActionClient(cc grpc.ClientConnInterface) ItemBlockActionClient { return &itemBlockActionClient{cc} } func (c *itemBlockActionClient) AppliesTo(ctx context.Context, in *ItemBlockActionAppliesToRequest, opts ...grpc.CallOption) (*ItemBlockActionAppliesToResponse, error) { out := new(ItemBlockActionAppliesToResponse) err := c.cc.Invoke(ctx, ItemBlockAction_AppliesTo_FullMethodName, in, out, opts...) if err != nil { return nil, err } return out, nil } func (c *itemBlockActionClient) GetRelatedItems(ctx context.Context, in *ItemBlockActionGetRelatedItemsRequest, opts ...grpc.CallOption) (*ItemBlockActionGetRelatedItemsResponse, error) { out := new(ItemBlockActionGetRelatedItemsResponse) err := c.cc.Invoke(ctx, ItemBlockAction_GetRelatedItems_FullMethodName, in, out, opts...) if err != nil { return nil, err } return out, nil } // ItemBlockActionServer is the server API for ItemBlockAction service. // All implementations should embed UnimplementedItemBlockActionServer // for forward compatibility type ItemBlockActionServer interface { AppliesTo(context.Context, *ItemBlockActionAppliesToRequest) (*ItemBlockActionAppliesToResponse, error) GetRelatedItems(context.Context, *ItemBlockActionGetRelatedItemsRequest) (*ItemBlockActionGetRelatedItemsResponse, error) } // UnimplementedItemBlockActionServer should be embedded to have forward compatible implementations. type UnimplementedItemBlockActionServer struct { } func (UnimplementedItemBlockActionServer) AppliesTo(context.Context, *ItemBlockActionAppliesToRequest) (*ItemBlockActionAppliesToResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method AppliesTo not implemented") } func (UnimplementedItemBlockActionServer) GetRelatedItems(context.Context, *ItemBlockActionGetRelatedItemsRequest) (*ItemBlockActionGetRelatedItemsResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method GetRelatedItems not implemented") } // UnsafeItemBlockActionServer may be embedded to opt out of forward compatibility for this service. // Use of this interface is not recommended, as added methods to ItemBlockActionServer will // result in compilation errors. type UnsafeItemBlockActionServer interface { mustEmbedUnimplementedItemBlockActionServer() } func RegisterItemBlockActionServer(s grpc.ServiceRegistrar, srv ItemBlockActionServer) { s.RegisterService(&ItemBlockAction_ServiceDesc, srv) } func _ItemBlockAction_AppliesTo_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(ItemBlockActionAppliesToRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(ItemBlockActionServer).AppliesTo(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: ItemBlockAction_AppliesTo_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ItemBlockActionServer).AppliesTo(ctx, req.(*ItemBlockActionAppliesToRequest)) } return interceptor(ctx, in, info, handler) } func _ItemBlockAction_GetRelatedItems_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(ItemBlockActionGetRelatedItemsRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(ItemBlockActionServer).GetRelatedItems(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: ItemBlockAction_GetRelatedItems_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ItemBlockActionServer).GetRelatedItems(ctx, req.(*ItemBlockActionGetRelatedItemsRequest)) } return interceptor(ctx, in, info, handler) } // ItemBlockAction_ServiceDesc is the grpc.ServiceDesc for ItemBlockAction service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) var ItemBlockAction_ServiceDesc = grpc.ServiceDesc{ ServiceName: "v1.ItemBlockAction", HandlerType: (*ItemBlockActionServer)(nil), Methods: []grpc.MethodDesc{ { MethodName: "AppliesTo", Handler: _ItemBlockAction_AppliesTo_Handler, }, { MethodName: "GetRelatedItems", Handler: _ItemBlockAction_GetRelatedItems_Handler, }, }, Streams: []grpc.StreamDesc{}, Metadata: "itemblockaction/v1/ItemBlockAction.proto", } ================================================ FILE: pkg/plugin/generated/restoreitemaction/v2/RestoreItemAction.pb.go ================================================ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.33.0 // protoc v4.25.2 // source: restoreitemaction/v2/RestoreItemAction.proto package v2 import ( generated "github.com/vmware-tanzu/velero/pkg/plugin/generated" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" durationpb "google.golang.org/protobuf/types/known/durationpb" emptypb "google.golang.org/protobuf/types/known/emptypb" reflect "reflect" sync "sync" ) const ( // Verify that this generated code is sufficiently up-to-date. _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) // Verify that runtime/protoimpl is sufficiently up-to-date. _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) type RestoreItemActionExecuteRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` Item []byte `protobuf:"bytes,2,opt,name=item,proto3" json:"item,omitempty"` Restore []byte `protobuf:"bytes,3,opt,name=restore,proto3" json:"restore,omitempty"` ItemFromBackup []byte `protobuf:"bytes,4,opt,name=itemFromBackup,proto3" json:"itemFromBackup,omitempty"` } func (x *RestoreItemActionExecuteRequest) Reset() { *x = RestoreItemActionExecuteRequest{} if protoimpl.UnsafeEnabled { mi := &file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[0] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *RestoreItemActionExecuteRequest) String() string { return protoimpl.X.MessageStringOf(x) } func (*RestoreItemActionExecuteRequest) ProtoMessage() {} func (x *RestoreItemActionExecuteRequest) ProtoReflect() protoreflect.Message { mi := &file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[0] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use RestoreItemActionExecuteRequest.ProtoReflect.Descriptor instead. func (*RestoreItemActionExecuteRequest) Descriptor() ([]byte, []int) { return file_restoreitemaction_v2_RestoreItemAction_proto_rawDescGZIP(), []int{0} } func (x *RestoreItemActionExecuteRequest) GetPlugin() string { if x != nil { return x.Plugin } return "" } func (x *RestoreItemActionExecuteRequest) GetItem() []byte { if x != nil { return x.Item } return nil } func (x *RestoreItemActionExecuteRequest) GetRestore() []byte { if x != nil { return x.Restore } return nil } func (x *RestoreItemActionExecuteRequest) GetItemFromBackup() []byte { if x != nil { return x.ItemFromBackup } return nil } type RestoreItemActionExecuteResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Item []byte `protobuf:"bytes,1,opt,name=item,proto3" json:"item,omitempty"` AdditionalItems []*generated.ResourceIdentifier `protobuf:"bytes,2,rep,name=additionalItems,proto3" json:"additionalItems,omitempty"` SkipRestore bool `protobuf:"varint,3,opt,name=skipRestore,proto3" json:"skipRestore,omitempty"` OperationID string `protobuf:"bytes,4,opt,name=operationID,proto3" json:"operationID,omitempty"` WaitForAdditionalItems bool `protobuf:"varint,5,opt,name=waitForAdditionalItems,proto3" json:"waitForAdditionalItems,omitempty"` AdditionalItemsReadyTimeout *durationpb.Duration `protobuf:"bytes,6,opt,name=additionalItemsReadyTimeout,proto3" json:"additionalItemsReadyTimeout,omitempty"` } func (x *RestoreItemActionExecuteResponse) Reset() { *x = RestoreItemActionExecuteResponse{} if protoimpl.UnsafeEnabled { mi := &file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *RestoreItemActionExecuteResponse) String() string { return protoimpl.X.MessageStringOf(x) } func (*RestoreItemActionExecuteResponse) ProtoMessage() {} func (x *RestoreItemActionExecuteResponse) ProtoReflect() protoreflect.Message { mi := &file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[1] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use RestoreItemActionExecuteResponse.ProtoReflect.Descriptor instead. func (*RestoreItemActionExecuteResponse) Descriptor() ([]byte, []int) { return file_restoreitemaction_v2_RestoreItemAction_proto_rawDescGZIP(), []int{1} } func (x *RestoreItemActionExecuteResponse) GetItem() []byte { if x != nil { return x.Item } return nil } func (x *RestoreItemActionExecuteResponse) GetAdditionalItems() []*generated.ResourceIdentifier { if x != nil { return x.AdditionalItems } return nil } func (x *RestoreItemActionExecuteResponse) GetSkipRestore() bool { if x != nil { return x.SkipRestore } return false } func (x *RestoreItemActionExecuteResponse) GetOperationID() string { if x != nil { return x.OperationID } return "" } func (x *RestoreItemActionExecuteResponse) GetWaitForAdditionalItems() bool { if x != nil { return x.WaitForAdditionalItems } return false } func (x *RestoreItemActionExecuteResponse) GetAdditionalItemsReadyTimeout() *durationpb.Duration { if x != nil { return x.AdditionalItemsReadyTimeout } return nil } type RestoreItemActionAppliesToRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` } func (x *RestoreItemActionAppliesToRequest) Reset() { *x = RestoreItemActionAppliesToRequest{} if protoimpl.UnsafeEnabled { mi := &file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *RestoreItemActionAppliesToRequest) String() string { return protoimpl.X.MessageStringOf(x) } func (*RestoreItemActionAppliesToRequest) ProtoMessage() {} func (x *RestoreItemActionAppliesToRequest) ProtoReflect() protoreflect.Message { mi := &file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[2] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use RestoreItemActionAppliesToRequest.ProtoReflect.Descriptor instead. func (*RestoreItemActionAppliesToRequest) Descriptor() ([]byte, []int) { return file_restoreitemaction_v2_RestoreItemAction_proto_rawDescGZIP(), []int{2} } func (x *RestoreItemActionAppliesToRequest) GetPlugin() string { if x != nil { return x.Plugin } return "" } type RestoreItemActionAppliesToResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields ResourceSelector *generated.ResourceSelector `protobuf:"bytes,1,opt,name=ResourceSelector,proto3" json:"ResourceSelector,omitempty"` } func (x *RestoreItemActionAppliesToResponse) Reset() { *x = RestoreItemActionAppliesToResponse{} if protoimpl.UnsafeEnabled { mi := &file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *RestoreItemActionAppliesToResponse) String() string { return protoimpl.X.MessageStringOf(x) } func (*RestoreItemActionAppliesToResponse) ProtoMessage() {} func (x *RestoreItemActionAppliesToResponse) ProtoReflect() protoreflect.Message { mi := &file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[3] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use RestoreItemActionAppliesToResponse.ProtoReflect.Descriptor instead. func (*RestoreItemActionAppliesToResponse) Descriptor() ([]byte, []int) { return file_restoreitemaction_v2_RestoreItemAction_proto_rawDescGZIP(), []int{3} } func (x *RestoreItemActionAppliesToResponse) GetResourceSelector() *generated.ResourceSelector { if x != nil { return x.ResourceSelector } return nil } type RestoreItemActionProgressRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` OperationID string `protobuf:"bytes,2,opt,name=operationID,proto3" json:"operationID,omitempty"` Restore []byte `protobuf:"bytes,3,opt,name=restore,proto3" json:"restore,omitempty"` } func (x *RestoreItemActionProgressRequest) Reset() { *x = RestoreItemActionProgressRequest{} if protoimpl.UnsafeEnabled { mi := &file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *RestoreItemActionProgressRequest) String() string { return protoimpl.X.MessageStringOf(x) } func (*RestoreItemActionProgressRequest) ProtoMessage() {} func (x *RestoreItemActionProgressRequest) ProtoReflect() protoreflect.Message { mi := &file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[4] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use RestoreItemActionProgressRequest.ProtoReflect.Descriptor instead. func (*RestoreItemActionProgressRequest) Descriptor() ([]byte, []int) { return file_restoreitemaction_v2_RestoreItemAction_proto_rawDescGZIP(), []int{4} } func (x *RestoreItemActionProgressRequest) GetPlugin() string { if x != nil { return x.Plugin } return "" } func (x *RestoreItemActionProgressRequest) GetOperationID() string { if x != nil { return x.OperationID } return "" } func (x *RestoreItemActionProgressRequest) GetRestore() []byte { if x != nil { return x.Restore } return nil } type RestoreItemActionProgressResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Progress *generated.OperationProgress `protobuf:"bytes,1,opt,name=progress,proto3" json:"progress,omitempty"` } func (x *RestoreItemActionProgressResponse) Reset() { *x = RestoreItemActionProgressResponse{} if protoimpl.UnsafeEnabled { mi := &file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *RestoreItemActionProgressResponse) String() string { return protoimpl.X.MessageStringOf(x) } func (*RestoreItemActionProgressResponse) ProtoMessage() {} func (x *RestoreItemActionProgressResponse) ProtoReflect() protoreflect.Message { mi := &file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[5] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use RestoreItemActionProgressResponse.ProtoReflect.Descriptor instead. func (*RestoreItemActionProgressResponse) Descriptor() ([]byte, []int) { return file_restoreitemaction_v2_RestoreItemAction_proto_rawDescGZIP(), []int{5} } func (x *RestoreItemActionProgressResponse) GetProgress() *generated.OperationProgress { if x != nil { return x.Progress } return nil } type RestoreItemActionCancelRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` OperationID string `protobuf:"bytes,2,opt,name=operationID,proto3" json:"operationID,omitempty"` Restore []byte `protobuf:"bytes,3,opt,name=restore,proto3" json:"restore,omitempty"` } func (x *RestoreItemActionCancelRequest) Reset() { *x = RestoreItemActionCancelRequest{} if protoimpl.UnsafeEnabled { mi := &file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *RestoreItemActionCancelRequest) String() string { return protoimpl.X.MessageStringOf(x) } func (*RestoreItemActionCancelRequest) ProtoMessage() {} func (x *RestoreItemActionCancelRequest) ProtoReflect() protoreflect.Message { mi := &file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[6] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use RestoreItemActionCancelRequest.ProtoReflect.Descriptor instead. func (*RestoreItemActionCancelRequest) Descriptor() ([]byte, []int) { return file_restoreitemaction_v2_RestoreItemAction_proto_rawDescGZIP(), []int{6} } func (x *RestoreItemActionCancelRequest) GetPlugin() string { if x != nil { return x.Plugin } return "" } func (x *RestoreItemActionCancelRequest) GetOperationID() string { if x != nil { return x.OperationID } return "" } func (x *RestoreItemActionCancelRequest) GetRestore() []byte { if x != nil { return x.Restore } return nil } type RestoreItemActionItemsReadyRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` Restore []byte `protobuf:"bytes,2,opt,name=restore,proto3" json:"restore,omitempty"` AdditionalItems []*generated.ResourceIdentifier `protobuf:"bytes,3,rep,name=additionalItems,proto3" json:"additionalItems,omitempty"` } func (x *RestoreItemActionItemsReadyRequest) Reset() { *x = RestoreItemActionItemsReadyRequest{} if protoimpl.UnsafeEnabled { mi := &file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *RestoreItemActionItemsReadyRequest) String() string { return protoimpl.X.MessageStringOf(x) } func (*RestoreItemActionItemsReadyRequest) ProtoMessage() {} func (x *RestoreItemActionItemsReadyRequest) ProtoReflect() protoreflect.Message { mi := &file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[7] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use RestoreItemActionItemsReadyRequest.ProtoReflect.Descriptor instead. func (*RestoreItemActionItemsReadyRequest) Descriptor() ([]byte, []int) { return file_restoreitemaction_v2_RestoreItemAction_proto_rawDescGZIP(), []int{7} } func (x *RestoreItemActionItemsReadyRequest) GetPlugin() string { if x != nil { return x.Plugin } return "" } func (x *RestoreItemActionItemsReadyRequest) GetRestore() []byte { if x != nil { return x.Restore } return nil } func (x *RestoreItemActionItemsReadyRequest) GetAdditionalItems() []*generated.ResourceIdentifier { if x != nil { return x.AdditionalItems } return nil } type RestoreItemActionItemsReadyResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Ready bool `protobuf:"varint,1,opt,name=ready,proto3" json:"ready,omitempty"` } func (x *RestoreItemActionItemsReadyResponse) Reset() { *x = RestoreItemActionItemsReadyResponse{} if protoimpl.UnsafeEnabled { mi := &file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *RestoreItemActionItemsReadyResponse) String() string { return protoimpl.X.MessageStringOf(x) } func (*RestoreItemActionItemsReadyResponse) ProtoMessage() {} func (x *RestoreItemActionItemsReadyResponse) ProtoReflect() protoreflect.Message { mi := &file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[8] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use RestoreItemActionItemsReadyResponse.ProtoReflect.Descriptor instead. func (*RestoreItemActionItemsReadyResponse) Descriptor() ([]byte, []int) { return file_restoreitemaction_v2_RestoreItemAction_proto_rawDescGZIP(), []int{8} } func (x *RestoreItemActionItemsReadyResponse) GetReady() bool { if x != nil { return x.Ready } return false } var File_restoreitemaction_v2_RestoreItemAction_proto protoreflect.FileDescriptor var file_restoreitemaction_v2_RestoreItemAction_proto_rawDesc = []byte{ 0x0a, 0x2c, 0x72, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x69, 0x74, 0x65, 0x6d, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x76, 0x32, 0x2f, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x02, 0x76, 0x32, 0x1a, 0x0c, 0x53, 0x68, 0x61, 0x72, 0x65, 0x64, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x8f, 0x01, 0x0a, 0x1f, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x72, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x26, 0x0a, 0x0e, 0x69, 0x74, 0x65, 0x6d, 0x46, 0x72, 0x6f, 0x6d, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0e, 0x69, 0x74, 0x65, 0x6d, 0x46, 0x72, 0x6f, 0x6d, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x22, 0xd8, 0x02, 0x0a, 0x20, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x12, 0x47, 0x0a, 0x0f, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x52, 0x0f, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x12, 0x20, 0x0a, 0x0b, 0x73, 0x6b, 0x69, 0x70, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0b, 0x73, 0x6b, 0x69, 0x70, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x12, 0x36, 0x0a, 0x16, 0x77, 0x61, 0x69, 0x74, 0x46, 0x6f, 0x72, 0x41, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x16, 0x77, 0x61, 0x69, 0x74, 0x46, 0x6f, 0x72, 0x41, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x12, 0x5b, 0x0a, 0x1b, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x52, 0x65, 0x61, 0x64, 0x79, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x1b, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x52, 0x65, 0x61, 0x64, 0x79, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x22, 0x3b, 0x0a, 0x21, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x22, 0x6d, 0x0a, 0x22, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x47, 0x0a, 0x10, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x10, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x22, 0x76, 0x0a, 0x20, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x20, 0x0a, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x72, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x22, 0x5d, 0x0a, 0x21, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x38, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x22, 0x74, 0x0a, 0x1e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x20, 0x0a, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x72, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x22, 0x9f, 0x01, 0x0a, 0x22, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x72, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x47, 0x0a, 0x0f, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x52, 0x0f, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x22, 0x3b, 0x0a, 0x23, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x32, 0xd0, 0x03, 0x0a, 0x11, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x5a, 0x0a, 0x09, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x12, 0x25, 0x2e, 0x76, 0x32, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x76, 0x32, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x07, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x12, 0x23, 0x2e, 0x76, 0x32, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x76, 0x32, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x57, 0x0a, 0x08, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, 0x24, 0x2e, 0x76, 0x32, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x76, 0x32, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x44, 0x0a, 0x06, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x12, 0x22, 0x2e, 0x76, 0x32, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x6a, 0x0a, 0x17, 0x41, 0x72, 0x65, 0x41, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x52, 0x65, 0x61, 0x64, 0x79, 0x12, 0x26, 0x2e, 0x76, 0x32, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x76, 0x32, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x4a, 0x5a, 0x48, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x76, 0x6d, 0x77, 0x61, 0x72, 0x65, 0x2d, 0x74, 0x61, 0x6e, 0x7a, 0x75, 0x2f, 0x76, 0x65, 0x6c, 0x65, 0x72, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x2f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2f, 0x72, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x69, 0x74, 0x65, 0x6d, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x76, 0x32, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( file_restoreitemaction_v2_RestoreItemAction_proto_rawDescOnce sync.Once file_restoreitemaction_v2_RestoreItemAction_proto_rawDescData = file_restoreitemaction_v2_RestoreItemAction_proto_rawDesc ) func file_restoreitemaction_v2_RestoreItemAction_proto_rawDescGZIP() []byte { file_restoreitemaction_v2_RestoreItemAction_proto_rawDescOnce.Do(func() { file_restoreitemaction_v2_RestoreItemAction_proto_rawDescData = protoimpl.X.CompressGZIP(file_restoreitemaction_v2_RestoreItemAction_proto_rawDescData) }) return file_restoreitemaction_v2_RestoreItemAction_proto_rawDescData } var file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes = make([]protoimpl.MessageInfo, 9) var file_restoreitemaction_v2_RestoreItemAction_proto_goTypes = []interface{}{ (*RestoreItemActionExecuteRequest)(nil), // 0: v2.RestoreItemActionExecuteRequest (*RestoreItemActionExecuteResponse)(nil), // 1: v2.RestoreItemActionExecuteResponse (*RestoreItemActionAppliesToRequest)(nil), // 2: v2.RestoreItemActionAppliesToRequest (*RestoreItemActionAppliesToResponse)(nil), // 3: v2.RestoreItemActionAppliesToResponse (*RestoreItemActionProgressRequest)(nil), // 4: v2.RestoreItemActionProgressRequest (*RestoreItemActionProgressResponse)(nil), // 5: v2.RestoreItemActionProgressResponse (*RestoreItemActionCancelRequest)(nil), // 6: v2.RestoreItemActionCancelRequest (*RestoreItemActionItemsReadyRequest)(nil), // 7: v2.RestoreItemActionItemsReadyRequest (*RestoreItemActionItemsReadyResponse)(nil), // 8: v2.RestoreItemActionItemsReadyResponse (*generated.ResourceIdentifier)(nil), // 9: generated.ResourceIdentifier (*durationpb.Duration)(nil), // 10: google.protobuf.Duration (*generated.ResourceSelector)(nil), // 11: generated.ResourceSelector (*generated.OperationProgress)(nil), // 12: generated.OperationProgress (*emptypb.Empty)(nil), // 13: google.protobuf.Empty } var file_restoreitemaction_v2_RestoreItemAction_proto_depIdxs = []int32{ 9, // 0: v2.RestoreItemActionExecuteResponse.additionalItems:type_name -> generated.ResourceIdentifier 10, // 1: v2.RestoreItemActionExecuteResponse.additionalItemsReadyTimeout:type_name -> google.protobuf.Duration 11, // 2: v2.RestoreItemActionAppliesToResponse.ResourceSelector:type_name -> generated.ResourceSelector 12, // 3: v2.RestoreItemActionProgressResponse.progress:type_name -> generated.OperationProgress 9, // 4: v2.RestoreItemActionItemsReadyRequest.additionalItems:type_name -> generated.ResourceIdentifier 2, // 5: v2.RestoreItemAction.AppliesTo:input_type -> v2.RestoreItemActionAppliesToRequest 0, // 6: v2.RestoreItemAction.Execute:input_type -> v2.RestoreItemActionExecuteRequest 4, // 7: v2.RestoreItemAction.Progress:input_type -> v2.RestoreItemActionProgressRequest 6, // 8: v2.RestoreItemAction.Cancel:input_type -> v2.RestoreItemActionCancelRequest 7, // 9: v2.RestoreItemAction.AreAdditionalItemsReady:input_type -> v2.RestoreItemActionItemsReadyRequest 3, // 10: v2.RestoreItemAction.AppliesTo:output_type -> v2.RestoreItemActionAppliesToResponse 1, // 11: v2.RestoreItemAction.Execute:output_type -> v2.RestoreItemActionExecuteResponse 5, // 12: v2.RestoreItemAction.Progress:output_type -> v2.RestoreItemActionProgressResponse 13, // 13: v2.RestoreItemAction.Cancel:output_type -> google.protobuf.Empty 8, // 14: v2.RestoreItemAction.AreAdditionalItemsReady:output_type -> v2.RestoreItemActionItemsReadyResponse 10, // [10:15] is the sub-list for method output_type 5, // [5:10] is the sub-list for method input_type 5, // [5:5] is the sub-list for extension type_name 5, // [5:5] is the sub-list for extension extendee 0, // [0:5] is the sub-list for field type_name } func init() { file_restoreitemaction_v2_RestoreItemAction_proto_init() } func file_restoreitemaction_v2_RestoreItemAction_proto_init() { if File_restoreitemaction_v2_RestoreItemAction_proto != nil { return } if !protoimpl.UnsafeEnabled { file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*RestoreItemActionExecuteRequest); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*RestoreItemActionExecuteResponse); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*RestoreItemActionAppliesToRequest); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*RestoreItemActionAppliesToResponse); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*RestoreItemActionProgressRequest); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*RestoreItemActionProgressResponse); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*RestoreItemActionCancelRequest); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*RestoreItemActionItemsReadyRequest); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*RestoreItemActionItemsReadyResponse); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_restoreitemaction_v2_RestoreItemAction_proto_rawDesc, NumEnums: 0, NumMessages: 9, NumExtensions: 0, NumServices: 1, }, GoTypes: file_restoreitemaction_v2_RestoreItemAction_proto_goTypes, DependencyIndexes: file_restoreitemaction_v2_RestoreItemAction_proto_depIdxs, MessageInfos: file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes, }.Build() File_restoreitemaction_v2_RestoreItemAction_proto = out.File file_restoreitemaction_v2_RestoreItemAction_proto_rawDesc = nil file_restoreitemaction_v2_RestoreItemAction_proto_goTypes = nil file_restoreitemaction_v2_RestoreItemAction_proto_depIdxs = nil } ================================================ FILE: pkg/plugin/generated/restoreitemaction/v2/RestoreItemAction_grpc.pb.go ================================================ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.3.0 // - protoc v4.25.2 // source: restoreitemaction/v2/RestoreItemAction.proto package v2 import ( context "context" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" emptypb "google.golang.org/protobuf/types/known/emptypb" ) // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. // Requires gRPC-Go v1.32.0 or later. const _ = grpc.SupportPackageIsVersion7 const ( RestoreItemAction_AppliesTo_FullMethodName = "/v2.RestoreItemAction/AppliesTo" RestoreItemAction_Execute_FullMethodName = "/v2.RestoreItemAction/Execute" RestoreItemAction_Progress_FullMethodName = "/v2.RestoreItemAction/Progress" RestoreItemAction_Cancel_FullMethodName = "/v2.RestoreItemAction/Cancel" RestoreItemAction_AreAdditionalItemsReady_FullMethodName = "/v2.RestoreItemAction/AreAdditionalItemsReady" ) // RestoreItemActionClient is the client API for RestoreItemAction service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. type RestoreItemActionClient interface { AppliesTo(ctx context.Context, in *RestoreItemActionAppliesToRequest, opts ...grpc.CallOption) (*RestoreItemActionAppliesToResponse, error) Execute(ctx context.Context, in *RestoreItemActionExecuteRequest, opts ...grpc.CallOption) (*RestoreItemActionExecuteResponse, error) Progress(ctx context.Context, in *RestoreItemActionProgressRequest, opts ...grpc.CallOption) (*RestoreItemActionProgressResponse, error) Cancel(ctx context.Context, in *RestoreItemActionCancelRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) AreAdditionalItemsReady(ctx context.Context, in *RestoreItemActionItemsReadyRequest, opts ...grpc.CallOption) (*RestoreItemActionItemsReadyResponse, error) } type restoreItemActionClient struct { cc grpc.ClientConnInterface } func NewRestoreItemActionClient(cc grpc.ClientConnInterface) RestoreItemActionClient { return &restoreItemActionClient{cc} } func (c *restoreItemActionClient) AppliesTo(ctx context.Context, in *RestoreItemActionAppliesToRequest, opts ...grpc.CallOption) (*RestoreItemActionAppliesToResponse, error) { out := new(RestoreItemActionAppliesToResponse) err := c.cc.Invoke(ctx, RestoreItemAction_AppliesTo_FullMethodName, in, out, opts...) if err != nil { return nil, err } return out, nil } func (c *restoreItemActionClient) Execute(ctx context.Context, in *RestoreItemActionExecuteRequest, opts ...grpc.CallOption) (*RestoreItemActionExecuteResponse, error) { out := new(RestoreItemActionExecuteResponse) err := c.cc.Invoke(ctx, RestoreItemAction_Execute_FullMethodName, in, out, opts...) if err != nil { return nil, err } return out, nil } func (c *restoreItemActionClient) Progress(ctx context.Context, in *RestoreItemActionProgressRequest, opts ...grpc.CallOption) (*RestoreItemActionProgressResponse, error) { out := new(RestoreItemActionProgressResponse) err := c.cc.Invoke(ctx, RestoreItemAction_Progress_FullMethodName, in, out, opts...) if err != nil { return nil, err } return out, nil } func (c *restoreItemActionClient) Cancel(ctx context.Context, in *RestoreItemActionCancelRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { out := new(emptypb.Empty) err := c.cc.Invoke(ctx, RestoreItemAction_Cancel_FullMethodName, in, out, opts...) if err != nil { return nil, err } return out, nil } func (c *restoreItemActionClient) AreAdditionalItemsReady(ctx context.Context, in *RestoreItemActionItemsReadyRequest, opts ...grpc.CallOption) (*RestoreItemActionItemsReadyResponse, error) { out := new(RestoreItemActionItemsReadyResponse) err := c.cc.Invoke(ctx, RestoreItemAction_AreAdditionalItemsReady_FullMethodName, in, out, opts...) if err != nil { return nil, err } return out, nil } // RestoreItemActionServer is the server API for RestoreItemAction service. // All implementations should embed UnimplementedRestoreItemActionServer // for forward compatibility type RestoreItemActionServer interface { AppliesTo(context.Context, *RestoreItemActionAppliesToRequest) (*RestoreItemActionAppliesToResponse, error) Execute(context.Context, *RestoreItemActionExecuteRequest) (*RestoreItemActionExecuteResponse, error) Progress(context.Context, *RestoreItemActionProgressRequest) (*RestoreItemActionProgressResponse, error) Cancel(context.Context, *RestoreItemActionCancelRequest) (*emptypb.Empty, error) AreAdditionalItemsReady(context.Context, *RestoreItemActionItemsReadyRequest) (*RestoreItemActionItemsReadyResponse, error) } // UnimplementedRestoreItemActionServer should be embedded to have forward compatible implementations. type UnimplementedRestoreItemActionServer struct { } func (UnimplementedRestoreItemActionServer) AppliesTo(context.Context, *RestoreItemActionAppliesToRequest) (*RestoreItemActionAppliesToResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method AppliesTo not implemented") } func (UnimplementedRestoreItemActionServer) Execute(context.Context, *RestoreItemActionExecuteRequest) (*RestoreItemActionExecuteResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method Execute not implemented") } func (UnimplementedRestoreItemActionServer) Progress(context.Context, *RestoreItemActionProgressRequest) (*RestoreItemActionProgressResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method Progress not implemented") } func (UnimplementedRestoreItemActionServer) Cancel(context.Context, *RestoreItemActionCancelRequest) (*emptypb.Empty, error) { return nil, status.Errorf(codes.Unimplemented, "method Cancel not implemented") } func (UnimplementedRestoreItemActionServer) AreAdditionalItemsReady(context.Context, *RestoreItemActionItemsReadyRequest) (*RestoreItemActionItemsReadyResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method AreAdditionalItemsReady not implemented") } // UnsafeRestoreItemActionServer may be embedded to opt out of forward compatibility for this service. // Use of this interface is not recommended, as added methods to RestoreItemActionServer will // result in compilation errors. type UnsafeRestoreItemActionServer interface { mustEmbedUnimplementedRestoreItemActionServer() } func RegisterRestoreItemActionServer(s grpc.ServiceRegistrar, srv RestoreItemActionServer) { s.RegisterService(&RestoreItemAction_ServiceDesc, srv) } func _RestoreItemAction_AppliesTo_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(RestoreItemActionAppliesToRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(RestoreItemActionServer).AppliesTo(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: RestoreItemAction_AppliesTo_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(RestoreItemActionServer).AppliesTo(ctx, req.(*RestoreItemActionAppliesToRequest)) } return interceptor(ctx, in, info, handler) } func _RestoreItemAction_Execute_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(RestoreItemActionExecuteRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(RestoreItemActionServer).Execute(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: RestoreItemAction_Execute_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(RestoreItemActionServer).Execute(ctx, req.(*RestoreItemActionExecuteRequest)) } return interceptor(ctx, in, info, handler) } func _RestoreItemAction_Progress_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(RestoreItemActionProgressRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(RestoreItemActionServer).Progress(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: RestoreItemAction_Progress_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(RestoreItemActionServer).Progress(ctx, req.(*RestoreItemActionProgressRequest)) } return interceptor(ctx, in, info, handler) } func _RestoreItemAction_Cancel_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(RestoreItemActionCancelRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(RestoreItemActionServer).Cancel(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: RestoreItemAction_Cancel_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(RestoreItemActionServer).Cancel(ctx, req.(*RestoreItemActionCancelRequest)) } return interceptor(ctx, in, info, handler) } func _RestoreItemAction_AreAdditionalItemsReady_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(RestoreItemActionItemsReadyRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(RestoreItemActionServer).AreAdditionalItemsReady(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: RestoreItemAction_AreAdditionalItemsReady_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(RestoreItemActionServer).AreAdditionalItemsReady(ctx, req.(*RestoreItemActionItemsReadyRequest)) } return interceptor(ctx, in, info, handler) } // RestoreItemAction_ServiceDesc is the grpc.ServiceDesc for RestoreItemAction service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) var RestoreItemAction_ServiceDesc = grpc.ServiceDesc{ ServiceName: "v2.RestoreItemAction", HandlerType: (*RestoreItemActionServer)(nil), Methods: []grpc.MethodDesc{ { MethodName: "AppliesTo", Handler: _RestoreItemAction_AppliesTo_Handler, }, { MethodName: "Execute", Handler: _RestoreItemAction_Execute_Handler, }, { MethodName: "Progress", Handler: _RestoreItemAction_Progress_Handler, }, { MethodName: "Cancel", Handler: _RestoreItemAction_Cancel_Handler, }, { MethodName: "AreAdditionalItemsReady", Handler: _RestoreItemAction_AreAdditionalItemsReady_Handler, }, }, Streams: []grpc.StreamDesc{}, Metadata: "restoreitemaction/v2/RestoreItemAction.proto", } ================================================ FILE: pkg/plugin/mocks/manager.go ================================================ // Code generated by mockery v2.43.2. DO NOT EDIT. package mocks import ( mock "github.com/stretchr/testify/mock" itemblockactionv1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/itemblockaction/v1" restoreitemactionv1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/restoreitemaction/v1" restoreitemactionv2 "github.com/vmware-tanzu/velero/pkg/plugin/velero/restoreitemaction/v2" v1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/backupitemaction/v1" v2 "github.com/vmware-tanzu/velero/pkg/plugin/velero/backupitemaction/v2" velero "github.com/vmware-tanzu/velero/pkg/plugin/velero" volumesnapshotterv1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/volumesnapshotter/v1" ) // Manager is an autogenerated mock type for the Manager type type Manager struct { mock.Mock } // CleanupClients provides a mock function with given fields: func (_m *Manager) CleanupClients() { _m.Called() } // GetBackupItemAction provides a mock function with given fields: name func (_m *Manager) GetBackupItemAction(name string) (v1.BackupItemAction, error) { ret := _m.Called(name) if len(ret) == 0 { panic("no return value specified for GetBackupItemAction") } var r0 v1.BackupItemAction var r1 error if rf, ok := ret.Get(0).(func(string) (v1.BackupItemAction, error)); ok { return rf(name) } if rf, ok := ret.Get(0).(func(string) v1.BackupItemAction); ok { r0 = rf(name) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(v1.BackupItemAction) } } if rf, ok := ret.Get(1).(func(string) error); ok { r1 = rf(name) } else { r1 = ret.Error(1) } return r0, r1 } // GetBackupItemActionV2 provides a mock function with given fields: name func (_m *Manager) GetBackupItemActionV2(name string) (v2.BackupItemAction, error) { ret := _m.Called(name) if len(ret) == 0 { panic("no return value specified for GetBackupItemActionV2") } var r0 v2.BackupItemAction var r1 error if rf, ok := ret.Get(0).(func(string) (v2.BackupItemAction, error)); ok { return rf(name) } if rf, ok := ret.Get(0).(func(string) v2.BackupItemAction); ok { r0 = rf(name) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(v2.BackupItemAction) } } if rf, ok := ret.Get(1).(func(string) error); ok { r1 = rf(name) } else { r1 = ret.Error(1) } return r0, r1 } // GetBackupItemActions provides a mock function with given fields: func (_m *Manager) GetBackupItemActions() ([]v1.BackupItemAction, error) { ret := _m.Called() if len(ret) == 0 { panic("no return value specified for GetBackupItemActions") } var r0 []v1.BackupItemAction var r1 error if rf, ok := ret.Get(0).(func() ([]v1.BackupItemAction, error)); ok { return rf() } if rf, ok := ret.Get(0).(func() []v1.BackupItemAction); ok { r0 = rf() } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]v1.BackupItemAction) } } if rf, ok := ret.Get(1).(func() error); ok { r1 = rf() } else { r1 = ret.Error(1) } return r0, r1 } // GetBackupItemActionsV2 provides a mock function with given fields: func (_m *Manager) GetBackupItemActionsV2() ([]v2.BackupItemAction, error) { ret := _m.Called() if len(ret) == 0 { panic("no return value specified for GetBackupItemActionsV2") } var r0 []v2.BackupItemAction var r1 error if rf, ok := ret.Get(0).(func() ([]v2.BackupItemAction, error)); ok { return rf() } if rf, ok := ret.Get(0).(func() []v2.BackupItemAction); ok { r0 = rf() } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]v2.BackupItemAction) } } if rf, ok := ret.Get(1).(func() error); ok { r1 = rf() } else { r1 = ret.Error(1) } return r0, r1 } // GetDeleteItemAction provides a mock function with given fields: name func (_m *Manager) GetDeleteItemAction(name string) (velero.DeleteItemAction, error) { ret := _m.Called(name) if len(ret) == 0 { panic("no return value specified for GetDeleteItemAction") } var r0 velero.DeleteItemAction var r1 error if rf, ok := ret.Get(0).(func(string) (velero.DeleteItemAction, error)); ok { return rf(name) } if rf, ok := ret.Get(0).(func(string) velero.DeleteItemAction); ok { r0 = rf(name) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(velero.DeleteItemAction) } } if rf, ok := ret.Get(1).(func(string) error); ok { r1 = rf(name) } else { r1 = ret.Error(1) } return r0, r1 } // GetDeleteItemActions provides a mock function with given fields: func (_m *Manager) GetDeleteItemActions() ([]velero.DeleteItemAction, error) { ret := _m.Called() if len(ret) == 0 { panic("no return value specified for GetDeleteItemActions") } var r0 []velero.DeleteItemAction var r1 error if rf, ok := ret.Get(0).(func() ([]velero.DeleteItemAction, error)); ok { return rf() } if rf, ok := ret.Get(0).(func() []velero.DeleteItemAction); ok { r0 = rf() } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]velero.DeleteItemAction) } } if rf, ok := ret.Get(1).(func() error); ok { r1 = rf() } else { r1 = ret.Error(1) } return r0, r1 } // GetItemBlockAction provides a mock function with given fields: name func (_m *Manager) GetItemBlockAction(name string) (itemblockactionv1.ItemBlockAction, error) { ret := _m.Called(name) if len(ret) == 0 { panic("no return value specified for GetItemBlockAction") } var r0 itemblockactionv1.ItemBlockAction var r1 error if rf, ok := ret.Get(0).(func(string) (itemblockactionv1.ItemBlockAction, error)); ok { return rf(name) } if rf, ok := ret.Get(0).(func(string) itemblockactionv1.ItemBlockAction); ok { r0 = rf(name) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(itemblockactionv1.ItemBlockAction) } } if rf, ok := ret.Get(1).(func(string) error); ok { r1 = rf(name) } else { r1 = ret.Error(1) } return r0, r1 } // GetItemBlockActions provides a mock function with given fields: func (_m *Manager) GetItemBlockActions() ([]itemblockactionv1.ItemBlockAction, error) { ret := _m.Called() if len(ret) == 0 { panic("no return value specified for GetItemBlockActions") } var r0 []itemblockactionv1.ItemBlockAction var r1 error if rf, ok := ret.Get(0).(func() ([]itemblockactionv1.ItemBlockAction, error)); ok { return rf() } if rf, ok := ret.Get(0).(func() []itemblockactionv1.ItemBlockAction); ok { r0 = rf() } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]itemblockactionv1.ItemBlockAction) } } if rf, ok := ret.Get(1).(func() error); ok { r1 = rf() } else { r1 = ret.Error(1) } return r0, r1 } // GetObjectStore provides a mock function with given fields: name func (_m *Manager) GetObjectStore(name string) (velero.ObjectStore, error) { ret := _m.Called(name) if len(ret) == 0 { panic("no return value specified for GetObjectStore") } var r0 velero.ObjectStore var r1 error if rf, ok := ret.Get(0).(func(string) (velero.ObjectStore, error)); ok { return rf(name) } if rf, ok := ret.Get(0).(func(string) velero.ObjectStore); ok { r0 = rf(name) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(velero.ObjectStore) } } if rf, ok := ret.Get(1).(func(string) error); ok { r1 = rf(name) } else { r1 = ret.Error(1) } return r0, r1 } // GetRestoreItemAction provides a mock function with given fields: name func (_m *Manager) GetRestoreItemAction(name string) (restoreitemactionv1.RestoreItemAction, error) { ret := _m.Called(name) if len(ret) == 0 { panic("no return value specified for GetRestoreItemAction") } var r0 restoreitemactionv1.RestoreItemAction var r1 error if rf, ok := ret.Get(0).(func(string) (restoreitemactionv1.RestoreItemAction, error)); ok { return rf(name) } if rf, ok := ret.Get(0).(func(string) restoreitemactionv1.RestoreItemAction); ok { r0 = rf(name) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(restoreitemactionv1.RestoreItemAction) } } if rf, ok := ret.Get(1).(func(string) error); ok { r1 = rf(name) } else { r1 = ret.Error(1) } return r0, r1 } // GetRestoreItemActionV2 provides a mock function with given fields: name func (_m *Manager) GetRestoreItemActionV2(name string) (restoreitemactionv2.RestoreItemAction, error) { ret := _m.Called(name) if len(ret) == 0 { panic("no return value specified for GetRestoreItemActionV2") } var r0 restoreitemactionv2.RestoreItemAction var r1 error if rf, ok := ret.Get(0).(func(string) (restoreitemactionv2.RestoreItemAction, error)); ok { return rf(name) } if rf, ok := ret.Get(0).(func(string) restoreitemactionv2.RestoreItemAction); ok { r0 = rf(name) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(restoreitemactionv2.RestoreItemAction) } } if rf, ok := ret.Get(1).(func(string) error); ok { r1 = rf(name) } else { r1 = ret.Error(1) } return r0, r1 } // GetRestoreItemActions provides a mock function with given fields: func (_m *Manager) GetRestoreItemActions() ([]restoreitemactionv1.RestoreItemAction, error) { ret := _m.Called() if len(ret) == 0 { panic("no return value specified for GetRestoreItemActions") } var r0 []restoreitemactionv1.RestoreItemAction var r1 error if rf, ok := ret.Get(0).(func() ([]restoreitemactionv1.RestoreItemAction, error)); ok { return rf() } if rf, ok := ret.Get(0).(func() []restoreitemactionv1.RestoreItemAction); ok { r0 = rf() } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]restoreitemactionv1.RestoreItemAction) } } if rf, ok := ret.Get(1).(func() error); ok { r1 = rf() } else { r1 = ret.Error(1) } return r0, r1 } // GetRestoreItemActionsV2 provides a mock function with given fields: func (_m *Manager) GetRestoreItemActionsV2() ([]restoreitemactionv2.RestoreItemAction, error) { ret := _m.Called() if len(ret) == 0 { panic("no return value specified for GetRestoreItemActionsV2") } var r0 []restoreitemactionv2.RestoreItemAction var r1 error if rf, ok := ret.Get(0).(func() ([]restoreitemactionv2.RestoreItemAction, error)); ok { return rf() } if rf, ok := ret.Get(0).(func() []restoreitemactionv2.RestoreItemAction); ok { r0 = rf() } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]restoreitemactionv2.RestoreItemAction) } } if rf, ok := ret.Get(1).(func() error); ok { r1 = rf() } else { r1 = ret.Error(1) } return r0, r1 } // GetVolumeSnapshotter provides a mock function with given fields: name func (_m *Manager) GetVolumeSnapshotter(name string) (volumesnapshotterv1.VolumeSnapshotter, error) { ret := _m.Called(name) if len(ret) == 0 { panic("no return value specified for GetVolumeSnapshotter") } var r0 volumesnapshotterv1.VolumeSnapshotter var r1 error if rf, ok := ret.Get(0).(func(string) (volumesnapshotterv1.VolumeSnapshotter, error)); ok { return rf(name) } if rf, ok := ret.Get(0).(func(string) volumesnapshotterv1.VolumeSnapshotter); ok { r0 = rf(name) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(volumesnapshotterv1.VolumeSnapshotter) } } if rf, ok := ret.Get(1).(func(string) error); ok { r1 = rf(name) } else { r1 = ret.Error(1) } return r0, r1 } // NewManager creates a new instance of Manager. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. func NewManager(t interface { mock.TestingT Cleanup(func()) }) *Manager { mock := &Manager{} mock.Mock.Test(t) t.Cleanup(func() { mock.AssertExpectations(t) }) return mock } ================================================ FILE: pkg/plugin/mocks/process_factory.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ // Code generated by mockery v1.0.0. DO NOT EDIT. package mocks import ( logrus "github.com/sirupsen/logrus" mock "github.com/stretchr/testify/mock" process "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt/process" ) // ProcessFactory is an autogenerated mock type for the ProcessFactory type type ProcessFactory struct { mock.Mock } // newProcess provides a mock function with given fields: command, logger, logLevel func (_m *ProcessFactory) newProcess(command string, logger logrus.FieldLogger, logLevel logrus.Level) (process.Process, error) { ret := _m.Called(command, logger, logLevel) var r0 process.Process if rf, ok := ret.Get(0).(func(string, logrus.FieldLogger, logrus.Level) process.Process); ok { r0 = rf(command, logger, logLevel) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(process.Process) } } var r1 error if rf, ok := ret.Get(1).(func(string, logrus.FieldLogger, logrus.Level) error); ok { r1 = rf(command, logger, logLevel) } else { r1 = ret.Error(1) } return r0, r1 } ================================================ FILE: pkg/plugin/proto/BackupItemAction.proto ================================================ syntax = "proto3"; package generated; option go_package = "github.com/vmware-tanzu/velero/pkg/plugin/generated"; import "Shared.proto"; message ExecuteRequest { string plugin = 1; bytes item = 2; bytes backup = 3; } message ExecuteResponse { bytes item = 1; repeated ResourceIdentifier additionalItems = 2; } service BackupItemAction { rpc AppliesTo(BackupItemActionAppliesToRequest) returns (BackupItemActionAppliesToResponse); rpc Execute(ExecuteRequest) returns (ExecuteResponse); } message BackupItemActionAppliesToRequest { string plugin = 1; } message BackupItemActionAppliesToResponse { ResourceSelector ResourceSelector = 1; } ================================================ FILE: pkg/plugin/proto/DeleteItemAction.proto ================================================ syntax = "proto3"; package generated; option go_package = "github.com/vmware-tanzu/velero/pkg/plugin/generated"; import "Shared.proto"; message DeleteItemActionExecuteRequest { string plugin = 1; bytes item = 2; bytes backup = 3; } service DeleteItemAction { rpc AppliesTo(DeleteItemActionAppliesToRequest) returns (DeleteItemActionAppliesToResponse); rpc Execute(DeleteItemActionExecuteRequest) returns (Empty); } message DeleteItemActionAppliesToRequest { string plugin = 1; } message DeleteItemActionAppliesToResponse { ResourceSelector ResourceSelector = 1; } ================================================ FILE: pkg/plugin/proto/ObjectStore.proto ================================================ syntax = "proto3"; package generated; option go_package = "github.com/vmware-tanzu/velero/pkg/plugin/generated"; import "Shared.proto"; message PutObjectRequest { string plugin = 1; string bucket = 2; string key = 3; bytes body = 4; } message ObjectExistsRequest { string plugin = 1; string bucket = 2; string key = 3; } message ObjectExistsResponse { bool exists = 1; } message GetObjectRequest { string plugin = 1; string bucket = 2; string key = 3; } message Bytes { bytes data = 1; } message ListCommonPrefixesRequest { string plugin = 1; string bucket = 2; string delimiter = 3; string prefix = 4; } message ListCommonPrefixesResponse { repeated string prefixes = 1; } message ListObjectsRequest { string plugin = 1; string bucket = 2; string prefix = 3; } message ListObjectsResponse { repeated string keys = 1; } message DeleteObjectRequest { string plugin = 1; string bucket = 2; string key = 3; } message CreateSignedURLRequest { string plugin = 1; string bucket = 2; string key = 3; int64 ttl = 4; } message CreateSignedURLResponse { string url = 1; } message ObjectStoreInitRequest { string plugin = 1; map config = 2; } service ObjectStore { rpc Init(ObjectStoreInitRequest) returns (Empty); rpc PutObject(stream PutObjectRequest) returns (Empty); rpc ObjectExists(ObjectExistsRequest) returns (ObjectExistsResponse); rpc GetObject(GetObjectRequest) returns (stream Bytes); rpc ListCommonPrefixes(ListCommonPrefixesRequest) returns (ListCommonPrefixesResponse); rpc ListObjects(ListObjectsRequest) returns (ListObjectsResponse); rpc DeleteObject(DeleteObjectRequest) returns (Empty); rpc CreateSignedURL(CreateSignedURLRequest) returns (CreateSignedURLResponse); } ================================================ FILE: pkg/plugin/proto/PluginLister.proto ================================================ syntax = "proto3"; package generated; option go_package = "github.com/vmware-tanzu/velero/pkg/plugin/generated"; import "Shared.proto"; message PluginIdentifier { string command = 1; string kind = 2; string name = 3; } message ListPluginsResponse { repeated PluginIdentifier plugins = 1; } service PluginLister { rpc ListPlugins(Empty) returns (ListPluginsResponse); } ================================================ FILE: pkg/plugin/proto/RestoreItemAction.proto ================================================ syntax = "proto3"; package generated; option go_package = "github.com/vmware-tanzu/velero/pkg/plugin/generated"; import "Shared.proto"; message RestoreItemActionExecuteRequest { string plugin = 1; bytes item = 2; bytes restore = 3; bytes itemFromBackup = 4; } message RestoreItemActionExecuteResponse { bytes item = 1; repeated ResourceIdentifier additionalItems = 2; bool skipRestore = 3; } service RestoreItemAction { rpc AppliesTo(RestoreItemActionAppliesToRequest) returns (RestoreItemActionAppliesToResponse); rpc Execute(RestoreItemActionExecuteRequest) returns (RestoreItemActionExecuteResponse); } message RestoreItemActionAppliesToRequest { string plugin = 1; } message RestoreItemActionAppliesToResponse { ResourceSelector ResourceSelector = 1; } ================================================ FILE: pkg/plugin/proto/Shared.proto ================================================ syntax = "proto3"; package generated; option go_package = "github.com/vmware-tanzu/velero/pkg/plugin/generated"; import "google/protobuf/timestamp.proto"; message Empty {} message Stack { repeated StackFrame frames = 1; } message StackFrame { string file = 1; int32 line = 2; string function = 3; } message ResourceIdentifier { string group = 1; string resource = 2; string namespace = 3; string name = 4; } message ResourceSelector { repeated string includedNamespaces = 1; repeated string excludedNamespaces = 2; repeated string includedResources = 3; repeated string excludedResources = 4; string selector = 5; } message OperationProgress { bool completed = 1; string err = 2; int64 nCompleted = 3; int64 nTotal = 4; string operationUnits = 5; string description = 6; google.protobuf.Timestamp started = 7; google.protobuf.Timestamp updated = 8; } ================================================ FILE: pkg/plugin/proto/VolumeSnapshotter.proto ================================================ syntax = "proto3"; package generated; option go_package = "github.com/vmware-tanzu/velero/pkg/plugin/generated"; import "Shared.proto"; message CreateVolumeRequest { string plugin = 1; string snapshotID = 2; string volumeType = 3; string volumeAZ = 4; int64 iops = 5; } message CreateVolumeResponse { string volumeID = 1; } message GetVolumeInfoRequest { string plugin = 1; string volumeID = 2; string volumeAZ = 3; } message GetVolumeInfoResponse { string volumeType = 1; int64 iops = 2; } message CreateSnapshotRequest { string plugin = 1; string volumeID = 2; string volumeAZ = 3; map tags = 4; } message CreateSnapshotResponse { string snapshotID = 1; } message DeleteSnapshotRequest { string plugin = 1; string snapshotID = 2; } message GetVolumeIDRequest { string plugin = 1; bytes persistentVolume = 2; } message GetVolumeIDResponse { string volumeID = 1; } message SetVolumeIDRequest { string plugin = 1; bytes persistentVolume = 2; string volumeID = 3; } message SetVolumeIDResponse { bytes persistentVolume = 1; } message VolumeSnapshotterInitRequest { string plugin = 1; map config = 2; } service VolumeSnapshotter { rpc Init(VolumeSnapshotterInitRequest) returns (Empty); rpc CreateVolumeFromSnapshot(CreateVolumeRequest) returns (CreateVolumeResponse); rpc GetVolumeInfo(GetVolumeInfoRequest) returns (GetVolumeInfoResponse); rpc CreateSnapshot(CreateSnapshotRequest) returns (CreateSnapshotResponse); rpc DeleteSnapshot(DeleteSnapshotRequest) returns (Empty); rpc GetVolumeID(GetVolumeIDRequest) returns (GetVolumeIDResponse); rpc SetVolumeID(SetVolumeIDRequest) returns (SetVolumeIDResponse); } ================================================ FILE: pkg/plugin/proto/backupitemaction/v2/BackupItemAction.proto ================================================ syntax = "proto3"; package v2; option go_package = "github.com/vmware-tanzu/velero/pkg/plugin/generated/backupitemaction/v2"; import "Shared.proto"; import "google/protobuf/empty.proto"; message ExecuteRequest { string plugin = 1; bytes item = 2; bytes backup = 3; } message ExecuteResponse { bytes item = 1; repeated generated.ResourceIdentifier additionalItems = 2; string operationID = 3; repeated generated.ResourceIdentifier postOperationItems = 4; } service BackupItemAction { rpc AppliesTo(BackupItemActionAppliesToRequest) returns (BackupItemActionAppliesToResponse); rpc Execute(ExecuteRequest) returns (ExecuteResponse); rpc Progress(BackupItemActionProgressRequest) returns (BackupItemActionProgressResponse); rpc Cancel(BackupItemActionCancelRequest) returns (google.protobuf.Empty); } message BackupItemActionAppliesToRequest { string plugin = 1; } message BackupItemActionAppliesToResponse { generated.ResourceSelector ResourceSelector = 1; } message BackupItemActionProgressRequest { string plugin = 1; string operationID = 2; bytes backup = 3; } message BackupItemActionProgressResponse { generated.OperationProgress progress = 1; } message BackupItemActionCancelRequest { string plugin = 1; string operationID = 2; bytes backup = 3; } ================================================ FILE: pkg/plugin/proto/itemblockaction/v1/ItemBlockAction.proto ================================================ syntax = "proto3"; package v1; option go_package = "github.com/vmware-tanzu/velero/pkg/plugin/generated/itemblockaction/v1"; import "Shared.proto"; service ItemBlockAction { rpc AppliesTo(ItemBlockActionAppliesToRequest) returns (ItemBlockActionAppliesToResponse); rpc GetRelatedItems(ItemBlockActionGetRelatedItemsRequest) returns (ItemBlockActionGetRelatedItemsResponse); } message ItemBlockActionAppliesToRequest { string plugin = 1; } message ItemBlockActionAppliesToResponse { generated.ResourceSelector ResourceSelector = 1; } message ItemBlockActionGetRelatedItemsRequest { string plugin = 1; bytes item = 2; bytes backup = 3; } message ItemBlockActionGetRelatedItemsResponse { repeated generated.ResourceIdentifier relatedItems = 1; } ================================================ FILE: pkg/plugin/proto/restoreitemaction/v2/RestoreItemAction.proto ================================================ syntax = "proto3"; package v2; option go_package = "github.com/vmware-tanzu/velero/pkg/plugin/generated/restoreitemaction/v2"; import "Shared.proto"; import "google/protobuf/empty.proto"; import "google/protobuf/duration.proto"; message RestoreItemActionExecuteRequest { string plugin = 1; bytes item = 2; bytes restore = 3; bytes itemFromBackup = 4; } message RestoreItemActionExecuteResponse { bytes item = 1; repeated generated.ResourceIdentifier additionalItems = 2; bool skipRestore = 3; string operationID = 4; bool waitForAdditionalItems = 5; google.protobuf.Duration additionalItemsReadyTimeout = 6; } service RestoreItemAction { rpc AppliesTo(RestoreItemActionAppliesToRequest) returns (RestoreItemActionAppliesToResponse); rpc Execute(RestoreItemActionExecuteRequest) returns (RestoreItemActionExecuteResponse); rpc Progress(RestoreItemActionProgressRequest) returns (RestoreItemActionProgressResponse); rpc Cancel(RestoreItemActionCancelRequest) returns (google.protobuf.Empty); rpc AreAdditionalItemsReady(RestoreItemActionItemsReadyRequest) returns (RestoreItemActionItemsReadyResponse); } message RestoreItemActionAppliesToRequest { string plugin = 1; } message RestoreItemActionAppliesToResponse { generated.ResourceSelector ResourceSelector = 1; } message RestoreItemActionProgressRequest { string plugin = 1; string operationID = 2; bytes restore = 3; } message RestoreItemActionProgressResponse { generated.OperationProgress progress = 1; } message RestoreItemActionCancelRequest { string plugin = 1; string operationID = 2; bytes restore = 3; } message RestoreItemActionItemsReadyRequest { string plugin = 1; bytes restore = 2; repeated generated.ResourceIdentifier additionalItems = 3; } message RestoreItemActionItemsReadyResponse { bool ready = 1; } ================================================ FILE: pkg/plugin/utils/volumehelper/volume_policy_helper.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package volumehelper import ( "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" crclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/internal/resourcepolicies" "github.com/vmware-tanzu/velero/internal/volumehelper" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/util/boolptr" ) // ShouldPerformSnapshotWithBackup is used for third-party plugins. // It supports to check whether the PVC or PodVolume should be backed // up on demand. On the other hand, the volumeHelperImpl assume there // is a VolumeHelper instance initialized before calling the // ShouldPerformXXX functions. // // Deprecated: Use ShouldPerformSnapshotWithVolumeHelper instead for better performance. // ShouldPerformSnapshotWithVolumeHelper allows passing a pre-created VolumeHelper with // an internal PVC-to-Pod cache, which avoids O(N*M) complexity when there are many PVCs and pods. // See issue #9179 for details. func ShouldPerformSnapshotWithBackup( unstructured runtime.Unstructured, groupResource schema.GroupResource, backup velerov1api.Backup, crClient crclient.Client, logger logrus.FieldLogger, ) (bool, error) { return ShouldPerformSnapshotWithVolumeHelper( unstructured, groupResource, backup, crClient, logger, nil, // no cached VolumeHelper, will create one ) } // ShouldPerformSnapshotWithVolumeHelper is like ShouldPerformSnapshotWithBackup // but accepts an optional VolumeHelper. If vh is non-nil, it will be used directly, // avoiding the overhead of creating a new VolumeHelper on each call. // This is useful for BIA plugins that process multiple PVCs during a single backup // and want to reuse the same VolumeHelper (with its internal cache) across calls. func ShouldPerformSnapshotWithVolumeHelper( unstructured runtime.Unstructured, groupResource schema.GroupResource, backup velerov1api.Backup, crClient crclient.Client, logger logrus.FieldLogger, vh volumehelper.VolumeHelper, ) (bool, error) { // If a VolumeHelper is provided, use it directly if vh != nil { return vh.ShouldPerformSnapshot(unstructured, groupResource) } // Otherwise, create a new VolumeHelper (original behavior for third-party plugins) resourcePolicies, err := resourcepolicies.GetResourcePoliciesFromBackup( backup, crClient, logger, ) if err != nil { return false, err } //nolint:staticcheck // Intentional use of deprecated function for backwards compatibility volumeHelperImpl := volumehelper.NewVolumeHelperImpl( resourcePolicies, backup.Spec.SnapshotVolumes, logger, crClient, boolptr.IsSetToTrue(backup.Spec.DefaultVolumesToFsBackup), true, ) return volumeHelperImpl.ShouldPerformSnapshot(unstructured, groupResource) } ================================================ FILE: pkg/plugin/utils/volumehelper/volume_policy_helper_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package volumehelper import ( "testing" "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "github.com/vmware-tanzu/velero/internal/volumehelper" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/kuberesource" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func TestShouldPerformSnapshotWithBackup(t *testing.T) { tests := []struct { name string pvc *corev1api.PersistentVolumeClaim pv *corev1api.PersistentVolume backup *velerov1api.Backup wantSnapshot bool wantError bool }{ { name: "Returns true when snapshotVolumes not set", pvc: &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: "test-pvc", Namespace: "default", }, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "test-pv", }, Status: corev1api.PersistentVolumeClaimStatus{ Phase: corev1api.ClaimBound, }, }, pv: &corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "test-pv", }, Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ CSI: &corev1api.CSIPersistentVolumeSource{ Driver: "test-driver", }, }, ClaimRef: &corev1api.ObjectReference{ Namespace: "default", Name: "test-pvc", }, }, }, backup: &velerov1api.Backup{ ObjectMeta: metav1.ObjectMeta{ Name: "test-backup", Namespace: "velero", }, }, wantSnapshot: true, wantError: false, }, { name: "Returns false when snapshotVolumes is false", pvc: &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: "test-pvc", Namespace: "default", }, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "test-pv", }, Status: corev1api.PersistentVolumeClaimStatus{ Phase: corev1api.ClaimBound, }, }, pv: &corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "test-pv", }, Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ CSI: &corev1api.CSIPersistentVolumeSource{ Driver: "test-driver", }, }, ClaimRef: &corev1api.ObjectReference{ Namespace: "default", Name: "test-pvc", }, }, }, backup: &velerov1api.Backup{ ObjectMeta: metav1.ObjectMeta{ Name: "test-backup", Namespace: "velero", }, Spec: velerov1api.BackupSpec{ SnapshotVolumes: boolPtr(false), }, }, wantSnapshot: false, wantError: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { // Create fake client with PV and PVC client := velerotest.NewFakeControllerRuntimeClient(t, tt.pv, tt.pvc) // Convert PVC to unstructured pvcMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tt.pvc) require.NoError(t, err) unstructuredPVC := &unstructured.Unstructured{Object: pvcMap} logger := logrus.New() // Call the function under test - this is the wrapper for third-party plugins result, err := ShouldPerformSnapshotWithBackup( unstructuredPVC, kuberesource.PersistentVolumeClaims, *tt.backup, client, logger, ) if tt.wantError { require.Error(t, err) } else { require.NoError(t, err) require.Equal(t, tt.wantSnapshot, result) } }) } } func boolPtr(b bool) *bool { return &b } func TestShouldPerformSnapshotWithVolumeHelper(t *testing.T) { tests := []struct { name string pvc *corev1api.PersistentVolumeClaim pv *corev1api.PersistentVolume backup *velerov1api.Backup wantSnapshot bool wantError bool }{ { name: "Returns true with nil VolumeHelper when snapshotVolumes not set", pvc: &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: "test-pvc", Namespace: "default", }, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "test-pv", }, Status: corev1api.PersistentVolumeClaimStatus{ Phase: corev1api.ClaimBound, }, }, pv: &corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "test-pv", }, Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ CSI: &corev1api.CSIPersistentVolumeSource{ Driver: "test-driver", }, }, ClaimRef: &corev1api.ObjectReference{ Namespace: "default", Name: "test-pvc", }, }, }, backup: &velerov1api.Backup{ ObjectMeta: metav1.ObjectMeta{ Name: "test-backup", Namespace: "velero", }, }, wantSnapshot: true, wantError: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { // Create fake client with PV client := velerotest.NewFakeControllerRuntimeClient(t, tt.pv, tt.pvc) // Convert PVC to unstructured pvcMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tt.pvc) require.NoError(t, err) unstructuredPVC := &unstructured.Unstructured{Object: pvcMap} logger := logrus.New() // Call the function under test with nil VolumeHelper // This exercises the fallback path that creates a new VolumeHelper per call result, err := ShouldPerformSnapshotWithVolumeHelper( unstructuredPVC, kuberesource.PersistentVolumeClaims, *tt.backup, client, logger, nil, // Pass nil for VolumeHelper - exercises fallback path ) if tt.wantError { require.Error(t, err) } else { require.NoError(t, err) require.Equal(t, tt.wantSnapshot, result) } }) } } // TestShouldPerformSnapshotWithNonNilVolumeHelper tests the ShouldPerformSnapshotWithVolumeHelper // function when a pre-created VolumeHelper is passed. This exercises the cached path used // by BIA plugins for better performance. func TestShouldPerformSnapshotWithNonNilVolumeHelper(t *testing.T) { pvc := &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: "test-pvc", Namespace: "default", }, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "test-pv", }, Status: corev1api.PersistentVolumeClaimStatus{ Phase: corev1api.ClaimBound, }, } pv := &corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "test-pv", }, Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ CSI: &corev1api.CSIPersistentVolumeSource{ Driver: "test-driver", }, }, ClaimRef: &corev1api.ObjectReference{ Namespace: "default", Name: "test-pvc", }, }, } backup := &velerov1api.Backup{ ObjectMeta: metav1.ObjectMeta{ Name: "test-backup", Namespace: "velero", }, Spec: velerov1api.BackupSpec{ IncludedNamespaces: []string{"default"}, }, } // Create fake client with PV and PVC client := velerotest.NewFakeControllerRuntimeClient(t, pv, pvc) logger := logrus.New() // Create VolumeHelper using the internal function with namespace caching vh, err := volumehelper.NewVolumeHelperImplWithNamespaces( nil, // no resource policies for this test nil, // snapshotVolumes not set logger, client, false, // defaultVolumesToFSBackup true, // backupExcludePVC []string{"default"}, ) require.NoError(t, err) require.NotNil(t, vh) // Convert PVC to unstructured pvcMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(pvc) require.NoError(t, err) unstructuredPVC := &unstructured.Unstructured{Object: pvcMap} // Call with non-nil VolumeHelper - exercises the cached path result, err := ShouldPerformSnapshotWithVolumeHelper( unstructuredPVC, kuberesource.PersistentVolumeClaims, *backup, client, logger, vh, // Pass non-nil VolumeHelper - exercises cached path ) require.NoError(t, err) require.True(t, result, "Should return true for snapshot when snapshotVolumes not set") } ================================================ FILE: pkg/plugin/velero/backupitemaction/v1/backup_item_action.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v1 import ( "k8s.io/apimachinery/pkg/runtime" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/plugin/velero" ) // BackupItemAction is an actor that performs an operation on an individual item being backed up. type BackupItemAction interface { // AppliesTo returns information about which resources this action should be invoked for. // A BackupItemAction's Execute function will only be invoked on items that match the returned // selector. A zero-valued ResourceSelector matches all resources. AppliesTo() (velero.ResourceSelector, error) // Execute allows the ItemAction to perform arbitrary logic with the item being backed up, // including mutating the item itself prior to backup. The item (unmodified or modified) // should be returned, along with an optional slice of ResourceIdentifiers specifying // additional related items that should be backed up. Execute(item runtime.Unstructured, backup *api.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, error) } ================================================ FILE: pkg/plugin/velero/backupitemaction/v2/backup_item_action.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v2 import ( "fmt" "k8s.io/apimachinery/pkg/runtime" "github.com/pkg/errors" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/plugin/velero" ) // BackupItemAction is an actor that performs an operation on an individual item being backed up. type BackupItemAction interface { // Name returns the name of this BIA. Plugins which implement this interface must define Name, // but its content is unimportant, as it won't actually be called via RPC. Velero's plugin infrastructure // will implement this directly rather than delegating to the RPC plugin in order to return the name // that the plugin was registered under. The plugins must implement the method to complete the interface. Name() string // AppliesTo returns information about which resources this action should be invoked for. // A BackupItemAction's Execute function will only be invoked on items that match the returned // selector. A zero-valued ResourceSelector matches all resources. AppliesTo() (velero.ResourceSelector, error) // Execute allows the BackupItemAction to perform arbitrary logic with the item being backed up, // including mutating the item itself prior to backup. The item (unmodified or modified) // should be returned, along with an optional slice of ResourceIdentifiers specifying // additional related items that should be backed up now, an optional operationID for actions which // initiate (asynchronous) operations, and a second slice of ResourceIdentifiers specifying related items // which should be backed up after all operations have completed. This last field will be // ignored if operationID is empty, and should not be filled in unless the resource must be updated in the // backup after operations complete (i.e. some of the item's Kubernetes metadata will be updated // during the operation which will be required during restore) // Note that (async) operations are not supported for items being backed up during Finalize phases, // so a plugin should not return an OperationID if the backup phase is "Finalizing" // or "FinalizingPartiallyFailed". The plugin should check the incoming // backup.Status.Phase before initiating operations, since the backup has already passed the waiting // for plugin operations phase. Plugins being called during Finalize will only be called for resources // that were returned as postOperationItems. Execute(item runtime.Unstructured, backup *api.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) // Progress allows the BackupItemAction to report on progress of an asynchronous action. // For the passed-in operation, the plugin will return an OperationProgress struct, indicating // whether the operation has completed, whether there were any errors, a plugin-specific // indication of how much of the operation is done (items completed out of items-to-complete), // and started/updated timestamps Progress(operationID string, backup *api.Backup) (velero.OperationProgress, error) // Cancel allows the BackupItemAction to cancel an asynchronous action (if possible). // Velero will call this if the wait timeout for asynchronous actions has been reached. // If operation cancel is not supported, then the plugin just needs to return. No error // return is expected in this case, since cancellation is optional here. Cancel(operationID string, backup *api.Backup) error } func AsyncOperationsNotSupportedError() error { return errors.New("Plugin does not support asynchronous operations") } func InvalidOperationIDError(operationID string) error { return fmt.Errorf("operation ID %v is invalid", operationID) } ================================================ FILE: pkg/plugin/velero/delete_item_action.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package velero import ( "k8s.io/apimachinery/pkg/runtime" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) // DeleteItemAction is an actor that performs an operation on an individual item being restored. type DeleteItemAction interface { // AppliesTo returns information about which resources this action should be invoked for. // A DeleteItemAction's Execute function will only be invoked on items that match the returned // selector. A zero-valued ResourceSelector matches all resources. AppliesTo() (ResourceSelector, error) // Execute allows the ItemAction to perform arbitrary logic with the item being deleted. // An error should be returned if there were problems with the deletion process, but the // overall deletion process cannot be stopped. // Returned errors are logged. Execute(input *DeleteItemActionExecuteInput) error } // DeleteItemActionExecuteInput contains the input parameters for the ItemAction's Execute function. type DeleteItemActionExecuteInput struct { // Item is the item taken from the pristine backed up version of resource. Item runtime.Unstructured // Backup is the representation of the restore resource processed by Velero. Backup *velerov1api.Backup } ================================================ FILE: pkg/plugin/velero/itemblockaction/v1/item_block_action.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v1 import ( "k8s.io/apimachinery/pkg/runtime" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/plugin/velero" ) // ItemBlockAction is an action that returns a list of related items that must be backed up // along with the current item (and not in a separate parallel backup thread). type ItemBlockAction interface { // Name returns the name of this IBA. Plugins which implement this interface must define Name, // but its content is unimportant, as it won't actually be called via RPC. Velero's plugin infrastructure // will implement this directly rather than delegating to the RPC plugin in order to return the name // that the plugin was registered under. The plugins must implement the method to complete the interface. Name() string // AppliesTo returns information about which resources this action should be invoked for. // A ItemBlockAction's GetRelatedItems function will only be invoked on items that match the returned // selector. A zero-valued ResourceSelector matches all resources. AppliesTo() (velero.ResourceSelector, error) // GetRelatedItems allows the ItemBlockAction to identify related items which must be backed up // along with the current item. In many cases, these will be the same items that a related // BackupItemAction's Execute method will return as additionalItems, but there may be differences. // For example, items that are newly-created in the BIA Execute and don't yet exist at backup // start will *not* be returned here. GetRelatedItems(item runtime.Unstructured, backup *api.Backup) ([]velero.ResourceIdentifier, error) } ================================================ FILE: pkg/plugin/velero/mocks/DeleteItemAction.go ================================================ // Code generated by mockery v2.1.0. DO NOT EDIT. package mocks import ( mock "github.com/stretchr/testify/mock" velero "github.com/vmware-tanzu/velero/pkg/plugin/velero" ) // DeleteItemAction is an autogenerated mock type for the DeleteItemAction type type DeleteItemAction struct { mock.Mock } // AppliesTo provides a mock function with given fields: func (_m *DeleteItemAction) AppliesTo() (velero.ResourceSelector, error) { ret := _m.Called() var r0 velero.ResourceSelector if rf, ok := ret.Get(0).(func() velero.ResourceSelector); ok { r0 = rf() } else { r0 = ret.Get(0).(velero.ResourceSelector) } var r1 error if rf, ok := ret.Get(1).(func() error); ok { r1 = rf() } else { r1 = ret.Error(1) } return r0, r1 } // Execute provides a mock function with given fields: input func (_m *DeleteItemAction) Execute(input *velero.DeleteItemActionExecuteInput) error { ret := _m.Called(input) var r0 error if rf, ok := ret.Get(0).(func(*velero.DeleteItemActionExecuteInput) error); ok { r0 = rf(input) } else { r0 = ret.Error(0) } return r0 } ================================================ FILE: pkg/plugin/velero/mocks/backupitemaction/v1/BackupItemAction.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ // Code generated by mockery v1.0.0. DO NOT EDIT. package v1 import ( mock "github.com/stretchr/testify/mock" runtime "k8s.io/apimachinery/pkg/runtime" velero "github.com/vmware-tanzu/velero/pkg/plugin/velero" velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) // BackupItemAction is an autogenerated mock type for the BackupItemAction type type BackupItemAction struct { mock.Mock } // AppliesTo provides a mock function with given fields: func (_m *BackupItemAction) AppliesTo() (velero.ResourceSelector, error) { ret := _m.Called() var r0 velero.ResourceSelector if rf, ok := ret.Get(0).(func() velero.ResourceSelector); ok { r0 = rf() } else { r0 = ret.Get(0).(velero.ResourceSelector) } var r1 error if rf, ok := ret.Get(1).(func() error); ok { r1 = rf() } else { r1 = ret.Error(1) } return r0, r1 } // Execute provides a mock function with given fields: item, backup func (_m *BackupItemAction) Execute(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, error) { ret := _m.Called(item, backup) var r0 runtime.Unstructured if rf, ok := ret.Get(0).(func(runtime.Unstructured, *velerov1.Backup) runtime.Unstructured); ok { r0 = rf(item, backup) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(runtime.Unstructured) } } var r1 []velero.ResourceIdentifier if rf, ok := ret.Get(1).(func(runtime.Unstructured, *velerov1.Backup) []velero.ResourceIdentifier); ok { r1 = rf(item, backup) } else { if ret.Get(1) != nil { r1 = ret.Get(1).([]velero.ResourceIdentifier) } } var r2 error if rf, ok := ret.Get(2).(func(runtime.Unstructured, *velerov1.Backup) error); ok { r2 = rf(item, backup) } else { r2 = ret.Error(2) } return r0, r1, r2 } ================================================ FILE: pkg/plugin/velero/mocks/backupitemaction/v2/BackupItemAction.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ // Code generated by mockery v2.16.0. DO NOT EDIT. package v2 import ( mock "github.com/stretchr/testify/mock" runtime "k8s.io/apimachinery/pkg/runtime" v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velero "github.com/vmware-tanzu/velero/pkg/plugin/velero" ) // BackupItemAction is an autogenerated mock type for the BackupItemAction type type BackupItemAction struct { mock.Mock } // AppliesTo provides a mock function with given fields: func (_m *BackupItemAction) AppliesTo() (velero.ResourceSelector, error) { ret := _m.Called() var r0 velero.ResourceSelector if rf, ok := ret.Get(0).(func() velero.ResourceSelector); ok { r0 = rf() } else { r0 = ret.Get(0).(velero.ResourceSelector) } var r1 error if rf, ok := ret.Get(1).(func() error); ok { r1 = rf() } else { r1 = ret.Error(1) } return r0, r1 } // Cancel provides a mock function with given fields: operationID, backup func (_m *BackupItemAction) Cancel(operationID string, backup *v1.Backup) error { ret := _m.Called(operationID, backup) var r0 error if rf, ok := ret.Get(0).(func(string, *v1.Backup) error); ok { r0 = rf(operationID, backup) } else { r0 = ret.Error(0) } return r0 } // Execute provides a mock function with given fields: item, backup func (_m *BackupItemAction) Execute(item runtime.Unstructured, backup *v1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { ret := _m.Called(item, backup) var r0 runtime.Unstructured if rf, ok := ret.Get(0).(func(runtime.Unstructured, *v1.Backup) runtime.Unstructured); ok { r0 = rf(item, backup) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(runtime.Unstructured) } } var r1 []velero.ResourceIdentifier if rf, ok := ret.Get(1).(func(runtime.Unstructured, *v1.Backup) []velero.ResourceIdentifier); ok { r1 = rf(item, backup) } else { if ret.Get(1) != nil { r1 = ret.Get(1).([]velero.ResourceIdentifier) } } var r2 string if rf, ok := ret.Get(2).(func(runtime.Unstructured, *v1.Backup) string); ok { r2 = rf(item, backup) } else { r2 = ret.Get(2).(string) } var r3 []velero.ResourceIdentifier if rf, ok := ret.Get(3).(func(runtime.Unstructured, *v1.Backup) []velero.ResourceIdentifier); ok { r3 = rf(item, backup) } else { if ret.Get(3) != nil { r3 = ret.Get(3).([]velero.ResourceIdentifier) } } var r4 error if rf, ok := ret.Get(4).(func(runtime.Unstructured, *v1.Backup) error); ok { r4 = rf(item, backup) } else { r4 = ret.Error(4) } return r0, r1, r2, r3, r4 } // Name provides a mock function with given fields: func (_m *BackupItemAction) Name() string { ret := _m.Called() var r0 string if rf, ok := ret.Get(0).(func() string); ok { r0 = rf() } else { r0 = ret.Get(0).(string) } return r0 } // Progress provides a mock function with given fields: operationID, backup func (_m *BackupItemAction) Progress(operationID string, backup *v1.Backup) (velero.OperationProgress, error) { ret := _m.Called(operationID, backup) var r0 velero.OperationProgress if rf, ok := ret.Get(0).(func(string, *v1.Backup) velero.OperationProgress); ok { r0 = rf(operationID, backup) } else { r0 = ret.Get(0).(velero.OperationProgress) } var r1 error if rf, ok := ret.Get(1).(func(string, *v1.Backup) error); ok { r1 = rf(operationID, backup) } else { r1 = ret.Error(1) } return r0, r1 } type mockConstructorTestingTNewBackupItemAction interface { mock.TestingT Cleanup(func()) } // NewBackupItemAction creates a new instance of BackupItemAction. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. func NewBackupItemAction(t mockConstructorTestingTNewBackupItemAction) *BackupItemAction { mock := &BackupItemAction{} mock.Mock.Test(t) t.Cleanup(func() { mock.AssertExpectations(t) }) return mock } ================================================ FILE: pkg/plugin/velero/mocks/itemblockaction/v1/ItemBlockAction.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ // Code generated by mockery v2.43.2. DO NOT EDIT. package v1 import ( mock "github.com/stretchr/testify/mock" runtime "k8s.io/apimachinery/pkg/runtime" velero "github.com/vmware-tanzu/velero/pkg/plugin/velero" velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) // ItemBlockAction is an autogenerated mock type for the ItemBlockAction type type ItemBlockAction struct { mock.Mock } // AppliesTo provides a mock function with given fields: func (_m *ItemBlockAction) AppliesTo() (velero.ResourceSelector, error) { ret := _m.Called() if len(ret) == 0 { panic("no return value specified for AppliesTo") } var r0 velero.ResourceSelector var r1 error if rf, ok := ret.Get(0).(func() (velero.ResourceSelector, error)); ok { return rf() } if rf, ok := ret.Get(0).(func() velero.ResourceSelector); ok { r0 = rf() } else { r0 = ret.Get(0).(velero.ResourceSelector) } if rf, ok := ret.Get(1).(func() error); ok { r1 = rf() } else { r1 = ret.Error(1) } return r0, r1 } // GetRelatedItems provides a mock function with given fields: item, backup func (_m *ItemBlockAction) GetRelatedItems(item runtime.Unstructured, backup *velerov1.Backup) ([]velero.ResourceIdentifier, error) { ret := _m.Called(item, backup) if len(ret) == 0 { panic("no return value specified for GetRelatedItems") } var r0 []velero.ResourceIdentifier var r1 error if rf, ok := ret.Get(0).(func(runtime.Unstructured, *velerov1.Backup) ([]velero.ResourceIdentifier, error)); ok { return rf(item, backup) } if rf, ok := ret.Get(0).(func(runtime.Unstructured, *velerov1.Backup) []velero.ResourceIdentifier); ok { r0 = rf(item, backup) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]velero.ResourceIdentifier) } } if rf, ok := ret.Get(1).(func(runtime.Unstructured, *velerov1.Backup) error); ok { r1 = rf(item, backup) } else { r1 = ret.Error(1) } return r0, r1 } // Name provides a mock function with given fields: func (_m *ItemBlockAction) Name() string { ret := _m.Called() if len(ret) == 0 { panic("no return value specified for Name") } var r0 string if rf, ok := ret.Get(0).(func() string); ok { r0 = rf() } else { r0 = ret.Get(0).(string) } return r0 } // NewItemBlockAction creates a new instance of ItemBlockAction. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. func NewItemBlockAction(t interface { mock.TestingT Cleanup(func()) }) *ItemBlockAction { mock := &ItemBlockAction{} mock.Mock.Test(t) t.Cleanup(func() { mock.AssertExpectations(t) }) return mock } ================================================ FILE: pkg/plugin/velero/mocks/object_store.go ================================================ // Code generated by mockery v1.0.0. DO NOT EDIT. package mocks import io "io" import mock "github.com/stretchr/testify/mock" import time "time" // ObjectStore is an autogenerated mock type for the ObjectStore type type ObjectStore struct { mock.Mock } // CreateSignedURL provides a mock function with given fields: bucket, key, ttl func (_m *ObjectStore) CreateSignedURL(bucket string, key string, ttl time.Duration) (string, error) { ret := _m.Called(bucket, key, ttl) var r0 string if rf, ok := ret.Get(0).(func(string, string, time.Duration) string); ok { r0 = rf(bucket, key, ttl) } else { r0 = ret.Get(0).(string) } var r1 error if rf, ok := ret.Get(1).(func(string, string, time.Duration) error); ok { r1 = rf(bucket, key, ttl) } else { r1 = ret.Error(1) } return r0, r1 } // DeleteObject provides a mock function with given fields: bucket, key func (_m *ObjectStore) DeleteObject(bucket string, key string) error { ret := _m.Called(bucket, key) var r0 error if rf, ok := ret.Get(0).(func(string, string) error); ok { r0 = rf(bucket, key) } else { r0 = ret.Error(0) } return r0 } // GetObject provides a mock function with given fields: bucket, key func (_m *ObjectStore) GetObject(bucket string, key string) (io.ReadCloser, error) { ret := _m.Called(bucket, key) var r0 io.ReadCloser if rf, ok := ret.Get(0).(func(string, string) io.ReadCloser); ok { r0 = rf(bucket, key) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(io.ReadCloser) } } var r1 error if rf, ok := ret.Get(1).(func(string, string) error); ok { r1 = rf(bucket, key) } else { r1 = ret.Error(1) } return r0, r1 } // Init provides a mock function with given fields: config func (_m *ObjectStore) Init(config map[string]string) error { ret := _m.Called(config) var r0 error if rf, ok := ret.Get(0).(func(map[string]string) error); ok { r0 = rf(config) } else { r0 = ret.Error(0) } return r0 } // ListCommonPrefixes provides a mock function with given fields: bucket, prefix, delimiter func (_m *ObjectStore) ListCommonPrefixes(bucket string, prefix string, delimiter string) ([]string, error) { ret := _m.Called(bucket, prefix, delimiter) var r0 []string if rf, ok := ret.Get(0).(func(string, string, string) []string); ok { r0 = rf(bucket, prefix, delimiter) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]string) } } var r1 error if rf, ok := ret.Get(1).(func(string, string, string) error); ok { r1 = rf(bucket, prefix, delimiter) } else { r1 = ret.Error(1) } return r0, r1 } // ListObjects provides a mock function with given fields: bucket, prefix func (_m *ObjectStore) ListObjects(bucket string, prefix string) ([]string, error) { ret := _m.Called(bucket, prefix) var r0 []string if rf, ok := ret.Get(0).(func(string, string) []string); ok { r0 = rf(bucket, prefix) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]string) } } var r1 error if rf, ok := ret.Get(1).(func(string, string) error); ok { r1 = rf(bucket, prefix) } else { r1 = ret.Error(1) } return r0, r1 } // ObjectExists provides a mock function with given fields: bucket, key func (_m *ObjectStore) ObjectExists(bucket string, key string) (bool, error) { ret := _m.Called(bucket, key) var r0 bool if rf, ok := ret.Get(0).(func(string, string) bool); ok { r0 = rf(bucket, key) } else { r0 = ret.Get(0).(bool) } var r1 error if rf, ok := ret.Get(1).(func(string, string) error); ok { r1 = rf(bucket, key) } else { r1 = ret.Error(1) } return r0, r1 } // PutObject provides a mock function with given fields: bucket, key, body func (_m *ObjectStore) PutObject(bucket string, key string, body io.Reader) error { ret := _m.Called(bucket, key, body) var r0 error if rf, ok := ret.Get(0).(func(string, string, io.Reader) error); ok { r0 = rf(bucket, key, body) } else { r0 = ret.Error(0) } return r0 } ================================================ FILE: pkg/plugin/velero/mocks/restoreitemaction/v1/RestoreItemAction.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ // Code generated by mockery v1.0.0. DO NOT EDIT. package v1 import ( mock "github.com/stretchr/testify/mock" velero "github.com/vmware-tanzu/velero/pkg/plugin/velero" ) // RestoreItemAction is an autogenerated mock type for the RestoreItemAction type type RestoreItemAction struct { mock.Mock } // AppliesTo provides a mock function with given fields: func (_m *RestoreItemAction) AppliesTo() (velero.ResourceSelector, error) { ret := _m.Called() var r0 velero.ResourceSelector if rf, ok := ret.Get(0).(func() velero.ResourceSelector); ok { r0 = rf() } else { r0 = ret.Get(0).(velero.ResourceSelector) } var r1 error if rf, ok := ret.Get(1).(func() error); ok { r1 = rf() } else { r1 = ret.Error(1) } return r0, r1 } // Execute provides a mock function with given fields: input func (_m *RestoreItemAction) Execute(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { ret := _m.Called(input) var r0 *velero.RestoreItemActionExecuteOutput if rf, ok := ret.Get(0).(func(*velero.RestoreItemActionExecuteInput) *velero.RestoreItemActionExecuteOutput); ok { r0 = rf(input) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(*velero.RestoreItemActionExecuteOutput) } } var r1 error if rf, ok := ret.Get(1).(func(*velero.RestoreItemActionExecuteInput) error); ok { r1 = rf(input) } else { r1 = ret.Error(1) } return r0, r1 } ================================================ FILE: pkg/plugin/velero/mocks/restoreitemaction/v2/RestoreItemAction.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ // Code generated by mockery v2.16.0. DO NOT EDIT. package v2 import ( mock "github.com/stretchr/testify/mock" v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velero "github.com/vmware-tanzu/velero/pkg/plugin/velero" ) // RestoreItemAction is an autogenerated mock type for the RestoreItemAction type type RestoreItemAction struct { mock.Mock } // AppliesTo provides a mock function with given fields: func (_m *RestoreItemAction) AppliesTo() (velero.ResourceSelector, error) { ret := _m.Called() var r0 velero.ResourceSelector if rf, ok := ret.Get(0).(func() velero.ResourceSelector); ok { r0 = rf() } else { r0 = ret.Get(0).(velero.ResourceSelector) } var r1 error if rf, ok := ret.Get(1).(func() error); ok { r1 = rf() } else { r1 = ret.Error(1) } return r0, r1 } // AreAdditionalItemsReady provides a mock function with given fields: AdditionalItems, restore func (_m *RestoreItemAction) AreAdditionalItemsReady(additionalItems []velero.ResourceIdentifier, restore *v1.Restore) (bool, error) { ret := _m.Called(additionalItems, restore) var r0 bool if rf, ok := ret.Get(0).(func([]velero.ResourceIdentifier, *v1.Restore) bool); ok { r0 = rf(additionalItems, restore) } else { r0 = ret.Get(0).(bool) } var r1 error if rf, ok := ret.Get(1).(func([]velero.ResourceIdentifier, *v1.Restore) error); ok { r1 = rf(additionalItems, restore) } else { r1 = ret.Error(1) } return r0, r1 } // Cancel provides a mock function with given fields: operationID, restore func (_m *RestoreItemAction) Cancel(operationID string, restore *v1.Restore) error { ret := _m.Called(operationID, restore) var r0 error if rf, ok := ret.Get(0).(func(string, *v1.Restore) error); ok { r0 = rf(operationID, restore) } else { r0 = ret.Error(0) } return r0 } // Execute provides a mock function with given fields: input func (_m *RestoreItemAction) Execute(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { ret := _m.Called(input) var r0 *velero.RestoreItemActionExecuteOutput if rf, ok := ret.Get(0).(func(*velero.RestoreItemActionExecuteInput) *velero.RestoreItemActionExecuteOutput); ok { r0 = rf(input) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(*velero.RestoreItemActionExecuteOutput) } } var r1 error if rf, ok := ret.Get(1).(func(*velero.RestoreItemActionExecuteInput) error); ok { r1 = rf(input) } else { r1 = ret.Error(1) } return r0, r1 } // Name provides a mock function with given fields: func (_m *RestoreItemAction) Name() string { ret := _m.Called() var r0 string if rf, ok := ret.Get(0).(func() string); ok { r0 = rf() } else { r0 = ret.Get(0).(string) } return r0 } // Progress provides a mock function with given fields: operationID, restore func (_m *RestoreItemAction) Progress(operationID string, restore *v1.Restore) (velero.OperationProgress, error) { ret := _m.Called(operationID, restore) var r0 velero.OperationProgress if rf, ok := ret.Get(0).(func(string, *v1.Restore) velero.OperationProgress); ok { r0 = rf(operationID, restore) } else { r0 = ret.Get(0).(velero.OperationProgress) } var r1 error if rf, ok := ret.Get(1).(func(string, *v1.Restore) error); ok { r1 = rf(operationID, restore) } else { r1 = ret.Error(1) } return r0, r1 } type mockConstructorTestingTNewRestoreItemAction interface { mock.TestingT Cleanup(func()) } // NewRestoreItemAction creates a new instance of RestoreItemAction. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. func NewRestoreItemAction(t mockConstructorTestingTNewRestoreItemAction) *RestoreItemAction { mock := &RestoreItemAction{} mock.Mock.Test(t) t.Cleanup(func() { mock.AssertExpectations(t) }) return mock } ================================================ FILE: pkg/plugin/velero/mocks/volumesnapshotter/v1/VolumeSnapshotter.go ================================================ // Code generated by mockery v1.0.0. DO NOT EDIT. package mocks import mock "github.com/stretchr/testify/mock" import runtime "k8s.io/apimachinery/pkg/runtime" // VolumeSnapshotter is an autogenerated mock type for the VolumeSnapshotter type type VolumeSnapshotter struct { mock.Mock } // CreateSnapshot provides a mock function with given fields: volumeID, volumeAZ, tags func (_m *VolumeSnapshotter) CreateSnapshot(volumeID string, volumeAZ string, tags map[string]string) (string, error) { ret := _m.Called(volumeID, volumeAZ, tags) var r0 string if rf, ok := ret.Get(0).(func(string, string, map[string]string) string); ok { r0 = rf(volumeID, volumeAZ, tags) } else { r0 = ret.Get(0).(string) } var r1 error if rf, ok := ret.Get(1).(func(string, string, map[string]string) error); ok { r1 = rf(volumeID, volumeAZ, tags) } else { r1 = ret.Error(1) } return r0, r1 } // CreateVolumeFromSnapshot provides a mock function with given fields: snapshotID, volumeType, volumeAZ, iops func (_m *VolumeSnapshotter) CreateVolumeFromSnapshot(snapshotID string, volumeType string, volumeAZ string, iops *int64) (string, error) { ret := _m.Called(snapshotID, volumeType, volumeAZ, iops) var r0 string if rf, ok := ret.Get(0).(func(string, string, string, *int64) string); ok { r0 = rf(snapshotID, volumeType, volumeAZ, iops) } else { r0 = ret.Get(0).(string) } var r1 error if rf, ok := ret.Get(1).(func(string, string, string, *int64) error); ok { r1 = rf(snapshotID, volumeType, volumeAZ, iops) } else { r1 = ret.Error(1) } return r0, r1 } // DeleteSnapshot provides a mock function with given fields: snapshotID func (_m *VolumeSnapshotter) DeleteSnapshot(snapshotID string) error { ret := _m.Called(snapshotID) var r0 error if rf, ok := ret.Get(0).(func(string) error); ok { r0 = rf(snapshotID) } else { r0 = ret.Error(0) } return r0 } // GetVolumeID provides a mock function with given fields: pv func (_m *VolumeSnapshotter) GetVolumeID(pv runtime.Unstructured) (string, error) { ret := _m.Called(pv) var r0 string if rf, ok := ret.Get(0).(func(runtime.Unstructured) string); ok { r0 = rf(pv) } else { r0 = ret.Get(0).(string) } var r1 error if rf, ok := ret.Get(1).(func(runtime.Unstructured) error); ok { r1 = rf(pv) } else { r1 = ret.Error(1) } return r0, r1 } // GetVolumeInfo provides a mock function with given fields: volumeID, volumeAZ func (_m *VolumeSnapshotter) GetVolumeInfo(volumeID string, volumeAZ string) (string, *int64, error) { ret := _m.Called(volumeID, volumeAZ) var r0 string if rf, ok := ret.Get(0).(func(string, string) string); ok { r0 = rf(volumeID, volumeAZ) } else { r0 = ret.Get(0).(string) } var r1 *int64 if rf, ok := ret.Get(1).(func(string, string) *int64); ok { r1 = rf(volumeID, volumeAZ) } else { if ret.Get(1) != nil { r1 = ret.Get(1).(*int64) } } var r2 error if rf, ok := ret.Get(2).(func(string, string) error); ok { r2 = rf(volumeID, volumeAZ) } else { r2 = ret.Error(2) } return r0, r1, r2 } // Init provides a mock function with given fields: config func (_m *VolumeSnapshotter) Init(config map[string]string) error { ret := _m.Called(config) var r0 error if rf, ok := ret.Get(0).(func(map[string]string) error); ok { r0 = rf(config) } else { r0 = ret.Error(0) } return r0 } // SetVolumeID provides a mock function with given fields: pv, volumeID func (_m *VolumeSnapshotter) SetVolumeID(pv runtime.Unstructured, volumeID string) (runtime.Unstructured, error) { ret := _m.Called(pv, volumeID) var r0 runtime.Unstructured if rf, ok := ret.Get(0).(func(runtime.Unstructured, string) runtime.Unstructured); ok { r0 = rf(pv, volumeID) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(runtime.Unstructured) } } var r1 error if rf, ok := ret.Get(1).(func(runtime.Unstructured, string) error); ok { r1 = rf(pv, volumeID) } else { r1 = ret.Error(1) } return r0, r1 } ================================================ FILE: pkg/plugin/velero/object_store.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package velero import ( "io" "time" ) // ObjectStore exposes basic object-storage operations required // by Velero. type ObjectStore interface { // Init prepares the ObjectStore for usage using the provided map of // configuration key-value pairs. It returns an error if the ObjectStore // cannot be initialized from the provided config. Init(config map[string]string) error // PutObject creates a new object using the data in body within the specified // object storage bucket with the given key. PutObject(bucket, key string, body io.Reader) error // ObjectExists checks if there is an object with the given key in the object storage bucket. ObjectExists(bucket, key string) (bool, error) // GetObject retrieves the object with the given key from the specified // bucket in object storage. GetObject(bucket, key string) (io.ReadCloser, error) // ListCommonPrefixes gets a list of all object key prefixes that start with // the specified prefix and stop at the next instance of the provided delimiter. // // For example, if the bucket contains the following keys: // a-prefix/foo-1/bar // a-prefix/foo-1/baz // a-prefix/foo-2/baz // some-other-prefix/foo-3/bar // and the provided prefix arg is "a-prefix/", and the delimiter is "/", // this will return the slice {"a-prefix/foo-1/", "a-prefix/foo-2/"}. ListCommonPrefixes(bucket, prefix, delimiter string) ([]string, error) // ListObjects gets a list of all keys in the specified bucket // that have the given prefix. ListObjects(bucket, prefix string) ([]string, error) // DeleteObject removes the object with the specified key from the given // bucket. DeleteObject(bucket, key string) error // CreateSignedURL creates a pre-signed URL for the given bucket and key that expires after ttl. CreateSignedURL(bucket, key string, ttl time.Duration) (string, error) } ================================================ FILE: pkg/plugin/velero/restore_item_action_shared.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package velero import ( "time" "k8s.io/apimachinery/pkg/runtime" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) // RestoreItemActionExecuteInput contains the input parameters for the ItemAction's Execute function. type RestoreItemActionExecuteInput struct { // Item is the item being restored. It is likely different from the pristine backed up version // (metadata reset, changed by various restore item action plugins, etc.). Item runtime.Unstructured // ItemFromBackup is the item taken from the pristine backed up version of resource. ItemFromBackup runtime.Unstructured // Restore is the representation of the restore resource processed by Velero. Restore *api.Restore } // RestoreItemActionExecuteOutput contains the output variables for the ItemAction's Execution function. type RestoreItemActionExecuteOutput struct { // UpdatedItem is the item being restored mutated by ItemAction. UpdatedItem runtime.Unstructured // AdditionalItems is a list of additional related items that should // be restored. AdditionalItems []ResourceIdentifier // SkipRestore tells velero to stop executing further actions // on this item, and skip the restore step. When this field's // value is true, AdditionalItems will be ignored. SkipRestore bool // v2 and later // OperationID is an identifier which indicates an ongoing asynchronous action which Velero will // continue to monitor after restoring this item. If left blank, then there is no ongoing operation. OperationID string // v2 and later // WaitForAdditionalItems determines whether velero will wait // until AreAdditionalItemsReady returns true before restoring // this item. If this field's value is true, then after restoring // the returned AdditionalItems, velero will not restore this item // until AreAdditionalItemsReady returns true or the timeout is // reached. Otherwise, AreAdditionalItemsReady is not called. WaitForAdditionalItems bool // v2 and later // AdditionalItemsReadyTimeout will override serverConfig.additionalItemsReadyTimeout // if specified. This value specifies how long velero will wait // for additional items to be ready before moving on. AdditionalItemsReadyTimeout time.Duration } // NewRestoreItemActionExecuteOutput creates a new RestoreItemActionExecuteOutput func NewRestoreItemActionExecuteOutput(item runtime.Unstructured) *RestoreItemActionExecuteOutput { return &RestoreItemActionExecuteOutput{ UpdatedItem: item, } } // WithoutRestore returns SkipRestore for RestoreItemActionExecuteOutput func (r *RestoreItemActionExecuteOutput) WithoutRestore() *RestoreItemActionExecuteOutput { r.SkipRestore = true return r } // WithOperationID returns RestoreItemActionExecuteOutput with OperationID set. func (r *RestoreItemActionExecuteOutput) WithOperationID(operationID string) *RestoreItemActionExecuteOutput { r.OperationID = operationID return r } // WithItemsWait returns RestoreItemActionExecuteOutput with WaitForAdditionalItems set to true. func (r *RestoreItemActionExecuteOutput) WithItemsWait() *RestoreItemActionExecuteOutput { r.WaitForAdditionalItems = true return r } ================================================ FILE: pkg/plugin/velero/restoreitemaction/v1/restore_item_action.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v1 import ( "github.com/vmware-tanzu/velero/pkg/plugin/velero" ) // RestoreItemAction is an actor that performs an operation on an individual item being restored. type RestoreItemAction interface { // AppliesTo returns information about which resources this action should be invoked for. // A RestoreItemAction's Execute function will only be invoked on items that match the returned // selector. A zero-valued ResourceSelector matches all resources. AppliesTo() (velero.ResourceSelector, error) // Execute allows the ItemAction to perform arbitrary logic with the item being restored, // including mutating the item itself prior to restore. The item (unmodified or modified) // should be returned, along with an optional slice of ResourceIdentifiers specifying additional // related items that should be restored, a warning (which will be logged but will not prevent // the item from being restored) or error (which will be logged and will prevent the item // from being restored) if applicable. Execute(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) } ================================================ FILE: pkg/plugin/velero/restoreitemaction/v2/restore_item_action.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v2 import ( "fmt" "github.com/pkg/errors" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/plugin/velero" ) // RestoreItemAction is an actor that performs an operation on an individual item being restored. type RestoreItemAction interface { // Name returns the name of this RIA. Plugins which implement this interface must define Name, // but its content is unimportant, as it won't actually be called via RPC. Velero's plugin infrastructure // will implement this directly rather than delegating to the RPC plugin in order to return the name // that the plugin was registered under. The plugins must implement the method to complete the interface. Name() string // AppliesTo returns information about which resources this action should be invoked for. // A RestoreItemAction's Execute function will only be invoked on items that match the returned // selector. A zero-valued ResourceSelector matches all resources. AppliesTo() (velero.ResourceSelector, error) // Execute allows the ItemAction to perform arbitrary logic with the item being restored, // including mutating the item itself prior to restore. The return struct includes: // The item (unmodified or modified), an optional slice of ResourceIdentifiers // specifying additional related items that should be restored, an optional OperationID, // a bool (waitForAdditionalItems) specifying whether Velero should wait until restored additional // items are ready before restoring this resource, and an optional timeout for the additional items // wait period. An error is returned if the action fails. Execute(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) // Progress allows the RestoreItemAction to report on progress of an asynchronous action. // For the passed-in operation, the plugin will return an OperationProgress struct, indicating // whether the operation has completed, whether there were any errors, a plugin-specific // indication of how much of the operation is done (items completed out of items-to-complete), // and started/updated timestamps Progress(operationID string, restore *api.Restore) (velero.OperationProgress, error) // Cancel allows the RestoreItemAction to cancel an asynchronous action (if possible). // Velero will call this if the wait timeout for asynchronous actions has been reached. // If operation cancel is not supported, then the plugin just needs to return. No error // return is expected in this case, since cancellation is optional here. Cancel(operationID string, restore *api.Restore) error // AreAdditionalItemsReady allows the ItemAction to communicate whether the passed-in // slice of AdditionalItems (previously returned by Execute()) // are ready. Returns true if all items are ready, and false // otherwise. The second return value is to report errors AreAdditionalItemsReady(additionalItems []velero.ResourceIdentifier, restore *api.Restore) (bool, error) } func AsyncOperationsNotSupportedError() error { return errors.New("Plugin does not support asynchronous operations") } func InvalidOperationIDError(operationID string) error { return fmt.Errorf("operation ID %v is invalid", operationID) } ================================================ FILE: pkg/plugin/velero/shared.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ // Package velero contains the interfaces necessary to implement // all of the Velero plugins. Users create their own binary containing // implementations of the plugin kinds in this package. Multiple // plugins of any type can be implemented. package velero import ( "time" "k8s.io/apimachinery/pkg/runtime/schema" ) // ResourceSelector is a collection of included/excluded namespaces, // included/excluded resources, and a label-selector that can be used // to match a set of items from a cluster. type ResourceSelector struct { // IncludedNamespaces is a slice of namespace names to match. All // namespaces in this slice, except those in ExcludedNamespaces, // will be matched. A nil/empty slice matches all namespaces. IncludedNamespaces []string // ExcludedNamespaces is a slice of namespace names to exclude. // All namespaces in IncludedNamespaces, *except* those in // this slice, will be matched. ExcludedNamespaces []string // IncludedResources is a slice of resources to match. Resources may be specified // as full names (e.g. "services"), abbreviations (e.g. "svc"), or with the // groups they are in (e.g. "ingresses.extensions"). All resources in this slice, // except those in ExcludedResources, will be matched. A nil/empty slice matches // all resources. IncludedResources []string // ExcludedResources is a slice of resources to exclude. Resources may be specified // as full names (e.g. "services"), abbreviations (e.g. "svc"), or with the // groups they are in (e.g. "ingresses.extensions"). All resources in IncludedResources, // *except* those in this slice, will be matched. ExcludedResources []string // LabelSelector is a string representation of a selector to apply // when matching resources. See "k8s.io/apimachinery/pkg/labels".Parse() // for details on syntax. LabelSelector string } // Applicable allows actions and plugins to specify which resources they should be invoked for type Applicable interface { // AppliesTo returns information about which resources this Responder should be invoked for. AppliesTo() (ResourceSelector, error) } // ResourceIdentifier describes a single item by its group, resource, namespace, and name. type ResourceIdentifier struct { schema.GroupResource Namespace string Name string } func (in *ResourceIdentifier) DeepCopy() *ResourceIdentifier { if in == nil { return nil } out := new(ResourceIdentifier) in.DeepCopyInto(out) return out } func (in *ResourceIdentifier) DeepCopyInto(out *ResourceIdentifier) { *out = *in out.GroupResource = in.GroupResource } // OperationProgress describes progress of an asynchronous plugin operation. type OperationProgress struct { // True when the operation has completed, either successfully or with a failure Completed bool // Set when the operation has failed Err string // Quantity completed so far and the total quantity associated with the operation // in OperationUnits. For data mover and volume snapshotter use cases, this will // usually be in bytes. On successful completion, NCompleted and NTotal should be // the same NCompleted, NTotal int64 // Units represented by NCompleted and NTotal -- for data mover and item // snapshotters, this will usually be bytes. OperationUnits string // Optional description of operation progress (i.e. "Current phase: Running") Description string // When the operation was started and when the last update was seen. Not all // systems retain when the upload was begun, return Time 0 (time.Unix(0, 0)) // if unknown. Started, Updated time.Time } ================================================ FILE: pkg/plugin/velero/volumesnapshotter/v1/volume_snapshotter.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v1 import ( "k8s.io/apimachinery/pkg/runtime" ) // VolumeSnapshotter defines the operations needed by Velero to // take snapshots of persistent volumes during backup, and to restore // persistent volumes from snapshots during restore. type VolumeSnapshotter interface { // Init prepares the VolumeSnapshotter for usage using the provided map of // configuration key-value pairs. It returns an error if the VolumeSnapshotter // cannot be initialized from the provided config. Init(config map[string]string) error // CreateVolumeFromSnapshot creates a new volume in the specified // availability zone, initialized from the provided snapshot, // and with the specified type and IOPS (if using provisioned IOPS). CreateVolumeFromSnapshot(snapshotID, volumeType, volumeAZ string, iops *int64) (volumeID string, err error) // GetVolumeID returns the cloud provider specific identifier for the PersistentVolume. GetVolumeID(pv runtime.Unstructured) (string, error) // SetVolumeID sets the cloud provider specific identifier for the PersistentVolume. SetVolumeID(pv runtime.Unstructured, volumeID string) (runtime.Unstructured, error) // GetVolumeInfo returns the type and IOPS (if using provisioned IOPS) for // the specified volume in the given availability zone. GetVolumeInfo(volumeID, volumeAZ string) (string, *int64, error) // CreateSnapshot creates a snapshot of the specified volume, and applies the provided // set of tags to the snapshot. CreateSnapshot(volumeID, volumeAZ string, tags map[string]string) (snapshotID string, err error) // DeleteSnapshot deletes the specified volume snapshot. DeleteSnapshot(snapshotID string) error } ================================================ FILE: pkg/podexec/pod_command_executor.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package podexec import ( "bytes" "context" "net/url" "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" kscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "k8s.io/client-go/tools/remotecommand" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) const defaultTimeout = 30 * time.Second // PodCommandExecutor is capable of executing a command in a container in a pod. type PodCommandExecutor interface { // ExecutePodCommand executes a command in a container in a pod. If the command takes longer than // the specified timeout, an error is returned. ExecutePodCommand(log logrus.FieldLogger, item map[string]any, namespace, name, hookName string, hook *api.ExecHook) error } type poster interface { Post() *rest.Request } type defaultPodCommandExecutor struct { restClientConfig *rest.Config restClient poster streamExecutorFactory streamExecutorFactory } // NewPodCommandExecutor creates a new PodCommandExecutor. func NewPodCommandExecutor(restClientConfig *rest.Config, restClient poster) PodCommandExecutor { return &defaultPodCommandExecutor{ restClientConfig: restClientConfig, restClient: restClient, streamExecutorFactory: &defaultStreamExecutorFactory{}, } } // ExecutePodCommand uses the pod exec API to execute a command in a container in a pod. If the // command takes longer than the specified timeout, an error is returned (NOTE: it is not currently // possible to ensure the command is terminated when the timeout occurs, so it may continue to run // in the background). func (e *defaultPodCommandExecutor) ExecutePodCommand(log logrus.FieldLogger, item map[string]any, namespace, name, hookName string, hook *api.ExecHook) error { if item == nil { return errors.New("item is required") } if namespace == "" { return errors.New("namespace is required") } if name == "" { return errors.New("name is required") } if hookName == "" { return errors.New("hookName is required") } if hook == nil { return errors.New("hook is required") } localHook := *hook pod := new(corev1api.Pod) if err := runtime.DefaultUnstructuredConverter.FromUnstructured(item, pod); err != nil { return errors.WithStack(err) } if localHook.Container == "" { if err := setDefaultHookContainer(pod, &localHook); err != nil { return err } } else if err := ensureContainerExists(pod, localHook.Container); err != nil { return err } if len(localHook.Command) == 0 { return errors.New("command is required") } switch localHook.OnError { case api.HookErrorModeFail, api.HookErrorModeContinue: // use the specified value default: // default to fail localHook.OnError = api.HookErrorModeFail } if localHook.Timeout.Duration == 0 { localHook.Timeout.Duration = defaultTimeout } hookLog := log.WithFields( logrus.Fields{ "hookName": hookName, "hookContainer": localHook.Container, "hookCommand": localHook.Command, "hookOnError": localHook.OnError, "hookTimeout": localHook.Timeout, }, ) if pod.Status.Phase == corev1api.PodSucceeded || pod.Status.Phase == corev1api.PodFailed { hookLog.Infof("Pod entered phase %s before some post-backup exec hooks ran", pod.Status.Phase) return nil } hookLog.Info("running exec hook") req := e.restClient.Post(). Resource("pods"). Namespace(namespace). Name(name). SubResource("exec") req.VersionedParams(&corev1api.PodExecOptions{ Container: localHook.Container, Command: localHook.Command, Stdout: true, Stderr: true, }, kscheme.ParameterCodec) executor, err := e.streamExecutorFactory.NewSPDYExecutor(e.restClientConfig, "POST", req.URL()) if err != nil { return err } var stdout, stderr bytes.Buffer streamOptions := remotecommand.StreamOptions{ Stdout: &stdout, Stderr: &stderr, } errCh := make(chan error) go func() { err = executor.StreamWithContext(context.Background(), streamOptions) errCh <- err }() var timeoutCh <-chan time.Time if localHook.Timeout.Duration > 0 { timer := time.NewTimer(localHook.Timeout.Duration) defer timer.Stop() timeoutCh = timer.C } select { case err = <-errCh: case <-timeoutCh: return errors.Errorf("timed out after %v", localHook.Timeout.Duration) } hookLog.Infof("stdout: %s", stdout.String()) hookLog.Infof("stderr: %s", stderr.String()) return err } func ensureContainerExists(pod *corev1api.Pod, container string) error { for _, c := range pod.Spec.Containers { if c.Name == container { return nil } } return errors.Errorf("no such container: %q", container) } func setDefaultHookContainer(pod *corev1api.Pod, hook *api.ExecHook) error { if len(pod.Spec.Containers) < 1 { return errors.New("need at least 1 container") } hook.Container = pod.Spec.Containers[0].Name return nil } type streamExecutorFactory interface { NewSPDYExecutor(config *rest.Config, method string, url *url.URL) (remotecommand.Executor, error) } type defaultStreamExecutorFactory struct{} func (f *defaultStreamExecutorFactory) NewSPDYExecutor(config *rest.Config, method string, url *url.URL) (remotecommand.Executor, error) { return remotecommand.NewSPDYExecutor(config, method, url) } ================================================ FILE: pkg/podexec/pod_command_executor_test.go ================================================ /* Copyright 2017, 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package podexec import ( "bytes" "context" "fmt" "net/url" "strings" "testing" "time" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/rest" "k8s.io/client-go/tools/remotecommand" v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func TestNewPodCommandExecutor(t *testing.T) { restClientConfig := &rest.Config{Host: "foo"} poster := &mockPoster{} pce := NewPodCommandExecutor(restClientConfig, poster).(*defaultPodCommandExecutor) assert.Equal(t, restClientConfig, pce.restClientConfig) assert.Equal(t, poster, pce.restClient) assert.Equal(t, &defaultStreamExecutorFactory{}, pce.streamExecutorFactory) } func TestExecutePodCommandMissingInputs(t *testing.T) { tests := []struct { name string item map[string]any podNamespace string podName string hookName string hook *v1.ExecHook }{ { name: "missing item", }, { name: "missing pod namespace", item: map[string]any{}, }, { name: "missing pod name", item: map[string]any{}, podNamespace: "ns", }, { name: "missing hookName", item: map[string]any{}, podNamespace: "ns", podName: "pod", }, { name: "missing hook", item: map[string]any{}, podNamespace: "ns", podName: "pod", hookName: "hook", }, { name: "container not found", item: velerotest.UnstructuredOrDie(`{"kind":"Pod","spec":{"containers":[{"name":"foo"}]}}`).Object, podNamespace: "ns", podName: "pod", hookName: "hook", hook: &v1.ExecHook{ Container: "missing", }, }, { name: "command missing", item: velerotest.UnstructuredOrDie(`{"kind":"Pod","spec":{"containers":[{"name":"foo"}]}}`).Object, podNamespace: "ns", podName: "pod", hookName: "hook", hook: &v1.ExecHook{ Container: "foo", }, }, { name: "hook's container is not overwritten by pod", item: velerotest.UnstructuredOrDie(`{"kind":"Pod","spec":{"containers":[{"name":"foo"}]}}`).Object, podNamespace: "ns", podName: "pod", hookName: "hook", hook: &v1.ExecHook{ Container: "", }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { pod := new(corev1api.Pod) hookPodContainerNotSame := false if err := runtime.DefaultUnstructuredConverter.FromUnstructured(test.item, pod); err != nil { assert.Error(t, err) } if (len(pod.Spec.Containers) > 0) && (pod.Spec.Containers[0].Name != test.hook.Container) { hookPodContainerNotSame = true } e := &defaultPodCommandExecutor{} err := e.ExecutePodCommand(velerotest.NewLogger(), test.item, test.podNamespace, test.podName, test.hookName, test.hook) if hookPodContainerNotSame && test.hook.Container == pod.Spec.Containers[0].Name { require.Error(t, fmt.Errorf("hook exec container is overwritten")) } assert.Error(t, err) }) } } func TestExecutePodCommand(t *testing.T) { tests := []struct { name string containerName string expectedContainerName string command []string errorMode v1.HookErrorMode expectedErrorMode v1.HookErrorMode timeout time.Duration expectedTimeout time.Duration hookError error expectedError string }{ { name: "validate defaults", command: []string{"some", "command"}, expectedContainerName: "foo", expectedErrorMode: v1.HookErrorModeFail, expectedTimeout: 30 * time.Second, }, { name: "use specified values", command: []string{"some", "command"}, containerName: "bar", expectedContainerName: "bar", errorMode: v1.HookErrorModeContinue, expectedErrorMode: v1.HookErrorModeContinue, timeout: 10 * time.Second, expectedTimeout: 10 * time.Second, }, { name: "hook error", command: []string{"some", "command"}, expectedContainerName: "foo", expectedErrorMode: v1.HookErrorModeFail, expectedTimeout: 30 * time.Second, hookError: errors.New("hook error"), expectedError: "hook error", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { hook := v1.ExecHook{ Container: test.containerName, Command: test.command, OnError: test.errorMode, Timeout: metav1.Duration{Duration: test.timeout}, } pod, err := velerotest.GetAsMap(` { "metadata": { "namespace": "namespace", "name": "name" }, "spec": { "containers": [ {"name": "foo"}, {"name": "bar"} ] } }`) require.NoError(t, err) clientConfig := &rest.Config{} poster := &mockPoster{} defer poster.AssertExpectations(t) podCommandExecutor := NewPodCommandExecutor(clientConfig, poster).(*defaultPodCommandExecutor) streamExecutorFactory := &mockStreamExecutorFactory{} defer streamExecutorFactory.AssertExpectations(t) podCommandExecutor.streamExecutorFactory = streamExecutorFactory baseURL, _ := url.Parse("https://some.server") contentConfig := rest.ClientContentConfig{ GroupVersion: schema.GroupVersion{Group: "", Version: "v1"}, } poster.On("Post").Return(rest.NewRequestWithClient(baseURL, "/api/v1", contentConfig, nil)) streamExecutor := &mockStreamExecutor{} defer streamExecutor.AssertExpectations(t) expectedCommand := strings.Join(test.command, "&command=") expectedURL, _ := url.Parse( fmt.Sprintf("https://some.server/api/v1/namespaces/namespace/pods/name/exec?command=%s&container=%s&stderr=true&stdout=true", expectedCommand, test.expectedContainerName), ) streamExecutorFactory.On("NewSPDYExecutor", clientConfig, "POST", expectedURL).Return(streamExecutor, nil) var stdout, stderr bytes.Buffer expectedStreamOptions := remotecommand.StreamOptions{ Stdout: &stdout, Stderr: &stderr, } streamExecutor.On("StreamWithContext", mock.Anything, expectedStreamOptions).Return(test.hookError) err = podCommandExecutor.ExecutePodCommand(velerotest.NewLogger(), pod, "namespace", "name", "hookName", &hook) if test.expectedError != "" { assert.EqualError(t, err, test.expectedError) return } require.NoError(t, err) }) } } func TestEnsureContainerExists(t *testing.T) { pod := &corev1api.Pod{ Spec: corev1api.PodSpec{ Containers: []corev1api.Container{ { Name: "foo", }, }, }, } err := ensureContainerExists(pod, "bar") require.EqualError(t, err, `no such container: "bar"`) err = ensureContainerExists(pod, "foo") assert.NoError(t, err) } func TestPodCompeted(t *testing.T) { pod := &corev1api.Pod{ Spec: corev1api.PodSpec{ Containers: []corev1api.Container{ { Name: "foo", }, }, }, Status: corev1api.PodStatus{ Phase: corev1api.PodSucceeded, }, } obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(pod) require.NoError(t, err) clientConfig := &rest.Config{} poster := &mockPoster{} defer poster.AssertExpectations(t) podCommandExecutor := NewPodCommandExecutor(clientConfig, poster).(*defaultPodCommandExecutor) hook := v1.ExecHook{ Container: "foo", Command: []string{"some", "command"}, } err = podCommandExecutor.ExecutePodCommand(velerotest.NewLogger(), obj, "namespace", "name", "hookName", &hook) require.NoError(t, err) } type mockStreamExecutorFactory struct { mock.Mock } func (f *mockStreamExecutorFactory) NewSPDYExecutor(config *rest.Config, method string, url *url.URL) (remotecommand.Executor, error) { args := f.Called(config, method, url) return args.Get(0).(remotecommand.Executor), args.Error(1) } type mockStreamExecutor struct { mock.Mock remotecommand.Executor } func (e *mockStreamExecutor) StreamWithContext(ctx context.Context, options remotecommand.StreamOptions) error { args := e.Called(ctx, options) return args.Error(0) } type mockPoster struct { mock.Mock } func (p *mockPoster) Post() *rest.Request { args := p.Called() return args.Get(0).(*rest.Request) } ================================================ FILE: pkg/podvolume/backup_micro_service.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package podvolume import ( "context" "encoding/json" "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" "sigs.k8s.io/controller-runtime/pkg/client" cachetool "k8s.io/client-go/tools/cache" "sigs.k8s.io/controller-runtime/pkg/cache" "github.com/vmware-tanzu/velero/internal/credentials" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/datapath" "github.com/vmware-tanzu/velero/pkg/repository" "github.com/vmware-tanzu/velero/pkg/uploader" "github.com/vmware-tanzu/velero/pkg/util/kube" apierrors "k8s.io/apimachinery/pkg/api/errors" ) const ( podVolumeRequestor = "snapshot-pod-volume" ) // BackupMicroService process data mover backups inside the backup pod type BackupMicroService struct { ctx context.Context client client.Client kubeClient kubernetes.Interface repoEnsurer *repository.Ensurer credentialGetter *credentials.CredentialGetter logger logrus.FieldLogger dataPathMgr *datapath.Manager eventRecorder kube.EventRecorder namespace string pvbName string pvb *velerov1api.PodVolumeBackup sourceTargetPath datapath.AccessPoint resultSignal chan dataPathResult pvbInformer cache.Informer pvbHandler cachetool.ResourceEventHandlerRegistration nodeName string } type dataPathResult struct { err error result string } func NewBackupMicroService(ctx context.Context, client client.Client, kubeClient kubernetes.Interface, pvbName string, namespace string, nodeName string, sourceTargetPath datapath.AccessPoint, dataPathMgr *datapath.Manager, repoEnsurer *repository.Ensurer, cred *credentials.CredentialGetter, pvbInformer cache.Informer, log logrus.FieldLogger) *BackupMicroService { return &BackupMicroService{ ctx: ctx, client: client, kubeClient: kubeClient, credentialGetter: cred, logger: log, repoEnsurer: repoEnsurer, dataPathMgr: dataPathMgr, namespace: namespace, pvbName: pvbName, sourceTargetPath: sourceTargetPath, nodeName: nodeName, resultSignal: make(chan dataPathResult), pvbInformer: pvbInformer, } } func (r *BackupMicroService) Init() error { r.eventRecorder = kube.NewEventRecorder(r.kubeClient, r.client.Scheme(), r.pvbName, r.nodeName, r.logger) handler, err := r.pvbInformer.AddEventHandler( cachetool.ResourceEventHandlerFuncs{ UpdateFunc: func(oldObj any, newObj any) { oldPvb := oldObj.(*velerov1api.PodVolumeBackup) newPvb := newObj.(*velerov1api.PodVolumeBackup) if newPvb.Name != r.pvbName { return } if newPvb.Status.Phase != velerov1api.PodVolumeBackupPhaseInProgress { return } if newPvb.Spec.Cancel && !oldPvb.Spec.Cancel { r.cancelPodVolumeBackup(newPvb) } }, }, ) if err != nil { return errors.Wrap(err, "error adding PVB handler") } r.pvbHandler = handler return err } func (r *BackupMicroService) RunCancelableDataPath(ctx context.Context) (string, error) { log := r.logger.WithFields(logrus.Fields{ "PVB": r.pvbName, }) pvb := &velerov1api.PodVolumeBackup{} err := wait.PollUntilContextCancel(ctx, 500*time.Millisecond, true, func(ctx context.Context) (bool, error) { err := r.client.Get(ctx, types.NamespacedName{ Namespace: r.namespace, Name: r.pvbName, }, pvb) if apierrors.IsNotFound(err) { return false, nil } if err != nil { return true, errors.Wrapf(err, "error to get PVB %s", r.pvbName) } if pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseInProgress { return true, nil } else { return false, nil } }) if err != nil { log.WithError(err).Error("Failed to wait PVB") return "", errors.Wrap(err, "error waiting for PVB") } r.pvb = pvb log.Info("Run cancelable PVB") callbacks := datapath.Callbacks{ OnCompleted: r.OnDataPathCompleted, OnFailed: r.OnDataPathFailed, OnCancelled: r.OnDataPathCancelled, OnProgress: r.OnDataPathProgress, } fsBackup, err := r.dataPathMgr.CreateFileSystemBR(pvb.Name, podVolumeRequestor, ctx, r.client, pvb.Namespace, callbacks, log) if err != nil { return "", errors.Wrap(err, "error to create data path") } log.Debug("Async fs br created") if err := fsBackup.Init(ctx, &datapath.FSBRInitParam{ BSLName: pvb.Spec.BackupStorageLocation, SourceNamespace: pvb.Spec.Pod.Namespace, UploaderType: pvb.Spec.UploaderType, RepositoryType: velerov1api.BackupRepositoryTypeKopia, RepoIdentifier: "", RepositoryEnsurer: r.repoEnsurer, CredentialGetter: r.credentialGetter, }); err != nil { return "", errors.Wrap(err, "error to initialize data path") } log.Info("Async fs br init") tags := map[string]string{} if err := fsBackup.StartBackup(r.sourceTargetPath, pvb.Spec.UploaderSettings, &datapath.FSBRStartParam{ RealSource: GetRealSource(pvb), ParentSnapshot: "", ForceFull: false, Tags: tags, }); err != nil { return "", errors.Wrap(err, "error starting data path backup") } log.Info("Async fs backup data path started") r.eventRecorder.Event(pvb, false, datapath.EventReasonStarted, "Data path for %s started", pvb.Name) result := "" select { case <-ctx.Done(): err = errors.New("timed out waiting for fs backup to complete") break case res := <-r.resultSignal: err = res.err result = res.result break } if err != nil { log.WithError(err).Error("Async fs backup was not completed") } r.eventRecorder.EndingEvent(pvb, false, datapath.EventReasonStopped, "Data path for %s stopped", pvb.Name) return result, err } func (r *BackupMicroService) Shutdown() { r.eventRecorder.Shutdown() r.closeDataPath(r.ctx, r.pvbName) if r.pvbHandler != nil { if err := r.pvbInformer.RemoveEventHandler(r.pvbHandler); err != nil { r.logger.WithError(err).Warn("Failed to remove pod handler") } } } var funcMarshal = json.Marshal func (r *BackupMicroService) OnDataPathCompleted(ctx context.Context, namespace string, pvbName string, result datapath.Result) { log := r.logger.WithField("PVB", pvbName) backupBytes, err := funcMarshal(result.Backup) if err != nil { log.WithError(err).Errorf("Failed to marshal backup result %v", result.Backup) r.resultSignal <- dataPathResult{ err: errors.Wrapf(err, "Failed to marshal backup result %v", result.Backup), } } else { r.eventRecorder.Event(r.pvb, false, datapath.EventReasonCompleted, string(backupBytes)) r.resultSignal <- dataPathResult{ result: string(backupBytes), } } log.Info("Async fs backup completed") } func (r *BackupMicroService) OnDataPathFailed(ctx context.Context, namespace string, pvbName string, err error) { log := r.logger.WithField("PVB", pvbName) log.WithError(err).Error("Async fs backup data path failed") r.eventRecorder.Event(r.pvb, false, datapath.EventReasonFailed, "Data path for PVB %s failed, error %v", r.pvbName, err) r.resultSignal <- dataPathResult{ err: errors.Wrapf(err, "Data path for PVB %s failed", r.pvbName), } } func (r *BackupMicroService) OnDataPathCancelled(ctx context.Context, namespace string, pvbName string) { log := r.logger.WithField("PVB", pvbName) log.Warn("Async fs backup data path canceled") r.eventRecorder.Event(r.pvb, false, datapath.EventReasonCancelled, "Data path for PVB %s canceled", pvbName) r.resultSignal <- dataPathResult{ err: errors.New(datapath.ErrCancelled), } } func (r *BackupMicroService) OnDataPathProgress(ctx context.Context, namespace string, pvbName string, progress *uploader.Progress) { log := r.logger.WithFields(logrus.Fields{ "PVB": pvbName, }) progressBytes, err := funcMarshal(progress) if err != nil { log.WithError(err).Errorf("Failed to marshal progress %v", progress) return } r.eventRecorder.Event(r.pvb, false, datapath.EventReasonProgress, string(progressBytes)) } func (r *BackupMicroService) closeDataPath(ctx context.Context, duName string) { fsBackup := r.dataPathMgr.GetAsyncBR(duName) if fsBackup != nil { fsBackup.Close(ctx) } r.dataPathMgr.RemoveAsyncBR(duName) } func (r *BackupMicroService) cancelPodVolumeBackup(pvb *velerov1api.PodVolumeBackup) { r.logger.WithField("PVB", pvb.Name).Info("PVB is being canceled") r.eventRecorder.Event(pvb, false, datapath.EventReasonCancelling, "Canceling for PVB %s", pvb.Name) fsBackup := r.dataPathMgr.GetAsyncBR(pvb.Name) if fsBackup == nil { r.OnDataPathCancelled(r.ctx, pvb.GetNamespace(), pvb.GetName()) } else { fsBackup.Cancel() } } ================================================ FILE: pkg/podvolume/backup_micro_service_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package podvolume import ( "context" "fmt" "sync" "testing" "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/runtime" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/datapath" "github.com/vmware-tanzu/velero/pkg/uploader" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" clientFake "sigs.k8s.io/controller-runtime/pkg/client/fake" velerotest "github.com/vmware-tanzu/velero/pkg/test" kbclient "sigs.k8s.io/controller-runtime/pkg/client" datapathmockes "github.com/vmware-tanzu/velero/pkg/datapath/mocks" ) type backupMsTestHelper struct { eventReason string eventMsg string marshalErr error marshalBytes []byte withEvent bool eventLock sync.Mutex } func (bt *backupMsTestHelper) Event(_ runtime.Object, _ bool, reason string, message string, a ...any) { bt.eventLock.Lock() defer bt.eventLock.Unlock() bt.withEvent = true bt.eventReason = reason bt.eventMsg = fmt.Sprintf(message, a...) } func (bt *backupMsTestHelper) EndingEvent(_ runtime.Object, _ bool, reason string, message string, a ...any) { bt.eventLock.Lock() defer bt.eventLock.Unlock() bt.withEvent = true bt.eventReason = reason bt.eventMsg = fmt.Sprintf(message, a...) } func (bt *backupMsTestHelper) Shutdown() {} func (bt *backupMsTestHelper) Marshal(v any) ([]byte, error) { if bt.marshalErr != nil { return nil, bt.marshalErr } return bt.marshalBytes, nil } func (bt *backupMsTestHelper) EventReason() string { bt.eventLock.Lock() defer bt.eventLock.Unlock() return bt.eventReason } func (bt *backupMsTestHelper) EventMessage() string { bt.eventLock.Lock() defer bt.eventLock.Unlock() return bt.eventMsg } func TestOnDataPathFailed(t *testing.T) { pvbName := "fake-pvb" bt := &backupMsTestHelper{} bs := &BackupMicroService{ pvbName: pvbName, dataPathMgr: datapath.NewManager(1), eventRecorder: bt, resultSignal: make(chan dataPathResult), logger: velerotest.NewLogger(), } expectedErr := "Data path for PVB fake-pvb failed: fake-error" expectedEventReason := datapath.EventReasonFailed expectedEventMsg := "Data path for PVB fake-pvb failed, error fake-error" go bs.OnDataPathFailed(t.Context(), velerov1api.DefaultNamespace, pvbName, errors.New("fake-error")) result := <-bs.resultSignal require.EqualError(t, result.err, expectedErr) assert.Equal(t, expectedEventReason, bt.EventReason()) assert.Equal(t, expectedEventMsg, bt.EventMessage()) } func TestOnDataPathCancelled(t *testing.T) { pvbName := "fake-pvb" bt := &backupMsTestHelper{} bs := &BackupMicroService{ pvbName: pvbName, dataPathMgr: datapath.NewManager(1), eventRecorder: bt, resultSignal: make(chan dataPathResult), logger: velerotest.NewLogger(), } expectedErr := datapath.ErrCancelled expectedEventReason := datapath.EventReasonCancelled expectedEventMsg := "Data path for PVB fake-pvb canceled" go bs.OnDataPathCancelled(t.Context(), velerov1api.DefaultNamespace, pvbName) result := <-bs.resultSignal require.EqualError(t, result.err, expectedErr) assert.Equal(t, expectedEventReason, bt.EventReason()) assert.Equal(t, expectedEventMsg, bt.EventMessage()) } func TestOnDataPathCompleted(t *testing.T) { tests := []struct { name string expectedErr string expectedEventReason string expectedEventMsg string marshalErr error marshallStr string }{ { name: "marshal fail", marshalErr: errors.New("fake-marshal-error"), expectedErr: "Failed to marshal backup result { false { } 0 0}: fake-marshal-error", }, { name: "succeed", marshallStr: "fake-complete-string", expectedEventReason: datapath.EventReasonCompleted, expectedEventMsg: "fake-complete-string", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { pvbName := "fake-pvb" bt := &backupMsTestHelper{ marshalErr: test.marshalErr, marshalBytes: []byte(test.marshallStr), } bs := &BackupMicroService{ dataPathMgr: datapath.NewManager(1), eventRecorder: bt, resultSignal: make(chan dataPathResult), logger: velerotest.NewLogger(), } funcMarshal = bt.Marshal go bs.OnDataPathCompleted(t.Context(), velerov1api.DefaultNamespace, pvbName, datapath.Result{}) result := <-bs.resultSignal if test.marshalErr != nil { assert.EqualError(t, result.err, test.expectedErr) } else { require.NoError(t, result.err) assert.Equal(t, test.expectedEventReason, bt.EventReason()) assert.Equal(t, test.expectedEventMsg, bt.EventMessage()) } }) } } func TestOnDataPathProgress(t *testing.T) { tests := []struct { name string expectedErr string expectedEventReason string expectedEventMsg string marshalErr error marshallStr string }{ { name: "marshal fail", marshalErr: errors.New("fake-marshal-error"), expectedErr: "Failed to marshal backup result", }, { name: "succeed", marshallStr: "fake-progress-string", expectedEventReason: datapath.EventReasonProgress, expectedEventMsg: "fake-progress-string", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { pvbName := "fake-pvb" bt := &backupMsTestHelper{ marshalErr: test.marshalErr, marshalBytes: []byte(test.marshallStr), } bs := &BackupMicroService{ dataPathMgr: datapath.NewManager(1), eventRecorder: bt, logger: velerotest.NewLogger(), } funcMarshal = bt.Marshal bs.OnDataPathProgress(t.Context(), velerov1api.DefaultNamespace, pvbName, &uploader.Progress{}) if test.marshalErr != nil { assert.False(t, bt.withEvent) } else { assert.True(t, bt.withEvent) assert.Equal(t, test.expectedEventReason, bt.EventReason()) assert.Equal(t, test.expectedEventMsg, bt.EventMessage()) } }) } } func TestCancelPodVolumeBackup(t *testing.T) { tests := []struct { name string expectedEventReason string expectedEventMsg string expectedErr string }{ { name: "no fs backup", expectedEventReason: datapath.EventReasonCancelled, expectedEventMsg: "Data path for PVB fake-pvb canceled", expectedErr: datapath.ErrCancelled, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { pvbName := "fake-pvb" pvb := builder.ForPodVolumeBackup(velerov1api.DefaultNamespace, pvbName).Result() bt := &backupMsTestHelper{} bs := &BackupMicroService{ dataPathMgr: datapath.NewManager(1), eventRecorder: bt, resultSignal: make(chan dataPathResult), logger: velerotest.NewLogger(), } go bs.cancelPodVolumeBackup(pvb) result := <-bs.resultSignal require.EqualError(t, result.err, test.expectedErr) assert.True(t, bt.withEvent) assert.Equal(t, test.expectedEventReason, bt.EventReason()) assert.Equal(t, test.expectedEventMsg, bt.EventMessage()) }) } } func TestRunCancelableDataPath(t *testing.T) { pvbName := "fake-pvb" pvb := builder.ForPodVolumeBackup(velerov1api.DefaultNamespace, pvbName).Phase(velerov1api.PodVolumeBackupPhaseNew).Result() pvbInProgress := builder.ForPodVolumeBackup(velerov1api.DefaultNamespace, pvbName).Phase(velerov1api.PodVolumeBackupPhaseInProgress).Result() ctxTimeout, cancel := context.WithTimeout(t.Context(), time.Second) tests := []struct { name string ctx context.Context result *dataPathResult dataPathMgr *datapath.Manager kubeClientObj []runtime.Object initErr error startErr error dataPathStarted bool expectedEventMsg string expectedErr string }{ { name: "no pvb", ctx: ctxTimeout, expectedErr: "error waiting for PVB: context deadline exceeded", }, { name: "pvb not in in-progress", ctx: ctxTimeout, kubeClientObj: []runtime.Object{pvb}, expectedErr: "error waiting for PVB: context deadline exceeded", }, { name: "create data path fail", ctx: t.Context(), kubeClientObj: []runtime.Object{pvbInProgress}, dataPathMgr: datapath.NewManager(0), expectedErr: "error to create data path: Concurrent number exceeds", }, { name: "init data path fail", ctx: t.Context(), kubeClientObj: []runtime.Object{pvbInProgress}, initErr: errors.New("fake-init-error"), expectedErr: "error to initialize data path: fake-init-error", }, { name: "start data path fail", ctx: t.Context(), kubeClientObj: []runtime.Object{pvbInProgress}, startErr: errors.New("fake-start-error"), expectedErr: "error starting data path backup: fake-start-error", }, { name: "data path timeout", ctx: ctxTimeout, kubeClientObj: []runtime.Object{pvbInProgress}, dataPathStarted: true, expectedEventMsg: fmt.Sprintf("Data path for %s stopped", pvbName), expectedErr: "timed out waiting for fs backup to complete", }, { name: "data path returns error", ctx: t.Context(), kubeClientObj: []runtime.Object{pvbInProgress}, dataPathStarted: true, result: &dataPathResult{ err: errors.New("fake-data-path-error"), }, expectedEventMsg: fmt.Sprintf("Data path for %s stopped", pvbName), expectedErr: "fake-data-path-error", }, { name: "succeed", ctx: t.Context(), kubeClientObj: []runtime.Object{pvbInProgress}, dataPathStarted: true, result: &dataPathResult{ result: "fake-succeed-result", }, expectedEventMsg: fmt.Sprintf("Data path for %s stopped", pvbName), }, } scheme := runtime.NewScheme() velerov1api.AddToScheme(scheme) for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeClientBuilder := clientFake.NewClientBuilder() fakeClientBuilder = fakeClientBuilder.WithScheme(scheme) fakeClient := fakeClientBuilder.WithRuntimeObjects(test.kubeClientObj...).Build() bt := &backupMsTestHelper{} bs := &BackupMicroService{ namespace: velerov1api.DefaultNamespace, pvbName: pvbName, ctx: t.Context(), client: fakeClient, dataPathMgr: datapath.NewManager(1), eventRecorder: bt, resultSignal: make(chan dataPathResult), logger: velerotest.NewLogger(), } if test.ctx != nil { bs.ctx = test.ctx } if test.dataPathMgr != nil { bs.dataPathMgr = test.dataPathMgr } datapath.FSBRCreator = func(string, string, kbclient.Client, string, datapath.Callbacks, logrus.FieldLogger) datapath.AsyncBR { fsBR := datapathmockes.NewAsyncBR(t) if test.initErr != nil { fsBR.On("Init", mock.Anything, mock.Anything).Return(test.initErr) } if test.startErr != nil { fsBR.On("Init", mock.Anything, mock.Anything).Return(nil) fsBR.On("StartBackup", mock.Anything, mock.Anything, mock.Anything).Return(test.startErr) } if test.dataPathStarted { fsBR.On("Init", mock.Anything, mock.Anything).Return(nil) fsBR.On("StartBackup", mock.Anything, mock.Anything, mock.Anything).Return(nil) } return fsBR } if test.result != nil { go func() { time.Sleep(time.Millisecond * 500) bs.resultSignal <- *test.result }() } result, err := bs.RunCancelableDataPath(test.ctx) if test.expectedErr != "" { require.EqualError(t, err, test.expectedErr) } else { require.NoError(t, err) assert.Equal(t, test.result.result, result) } if test.expectedEventMsg != "" { assert.True(t, bt.withEvent) assert.Equal(t, test.expectedEventMsg, bt.EventMessage()) } }) } cancel() } ================================================ FILE: pkg/podvolume/backupper.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package podvolume import ( "context" "fmt" "sync" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/tools/cache" ctrlcache "sigs.k8s.io/controller-runtime/pkg/cache" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/internal/resourcepolicies" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" veleroclient "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/label" "github.com/vmware-tanzu/velero/pkg/nodeagent" "github.com/vmware-tanzu/velero/pkg/podvolume/configs" "github.com/vmware-tanzu/velero/pkg/repository" "github.com/vmware-tanzu/velero/pkg/uploader" uploaderutil "github.com/vmware-tanzu/velero/pkg/uploader/util" "github.com/vmware-tanzu/velero/pkg/util/boolptr" "github.com/vmware-tanzu/velero/pkg/util/kube" ) const ( indexNamePod = "POD" pvbKeyPattern = "%s+%s+%s" ) // Backupper can execute pod volume backups of volumes in a pod. type Backupper interface { // BackupPodVolumes backs up all specified volumes in a pod. BackupPodVolumes(backup *velerov1api.Backup, pod *corev1api.Pod, volumesToBackup []string, resPolicies *resourcepolicies.Policies, log logrus.FieldLogger) ([]*velerov1api.PodVolumeBackup, *PVCBackupSummary, []error) WaitAllPodVolumesProcessed(log logrus.FieldLogger) []*velerov1api.PodVolumeBackup GetPodVolumeBackupByPodAndVolume(podNamespace, podName, volume string) (*velerov1api.PodVolumeBackup, error) ListPodVolumeBackupsByPod(podNamespace, podName string) ([]*velerov1api.PodVolumeBackup, error) } type backupper struct { ctx context.Context repoLocker *repository.RepoLocker repoEnsurer *repository.Ensurer crClient ctrlclient.Client uploaderType string pvbInformer ctrlcache.Informer handlerRegistration cache.ResourceEventHandlerRegistration wg sync.WaitGroup // pvbIndexer holds all PVBs created by this backuper and is capable to search // the PVBs based on specific properties quickly because of the embedded indexes. // The statuses of the PVBs are got updated when Informer receives update events. pvbIndexer cache.Indexer } type skippedPVC struct { PVC *corev1api.PersistentVolumeClaim Reason string } // PVCBackupSummary is a summary for which PVCs are skipped, which are backed up after each execution of the Backupper // The scope should be within one pod, so the volume name is the key for the maps type PVCBackupSummary struct { Backedup map[string]*corev1api.PersistentVolumeClaim Skipped map[string]*skippedPVC pvcMap map[string]*corev1api.PersistentVolumeClaim } func NewPVCBackupSummary() *PVCBackupSummary { return &PVCBackupSummary{ Backedup: make(map[string]*corev1api.PersistentVolumeClaim), Skipped: make(map[string]*skippedPVC), pvcMap: make(map[string]*corev1api.PersistentVolumeClaim), } } func (pbs *PVCBackupSummary) addBackedup(volumeName string) { if pvc, ok := pbs.pvcMap[volumeName]; ok { pbs.Backedup[volumeName] = pvc delete(pbs.Skipped, volumeName) } } func (pbs *PVCBackupSummary) addSkipped(volumeName string, reason string) { if pvc, ok := pbs.pvcMap[volumeName]; ok { if _, ok2 := pbs.Backedup[volumeName]; !ok2 { // if it's not backed up, add it to skipped pbs.Skipped[volumeName] = &skippedPVC{ PVC: pvc, Reason: reason, } } } } func podIndexFunc(obj any) ([]string, error) { pvb, ok := obj.(*velerov1api.PodVolumeBackup) if !ok { return nil, errors.Errorf("expected PodVolumeBackup, but got %T", obj) } if pvb == nil { return nil, errors.New("PodVolumeBackup is nil") } return []string{cache.NewObjectName(pvb.Spec.Pod.Namespace, pvb.Spec.Pod.Name).String()}, nil } // the PVB's name is auto-generated when creating the PVB, we cannot get the name or uid before creating it. // So we cannot use namespace&name or uid as the key because we need to insert PVB into the indexer before creating it in API server func podVolumeBackupKey(obj any) (string, error) { pvb, ok := obj.(*velerov1api.PodVolumeBackup) if !ok { return "", fmt.Errorf("expected PodVolumeBackup, but got %T", obj) } return fmt.Sprintf(pvbKeyPattern, pvb.Spec.Pod.Namespace, pvb.Spec.Pod.Name, pvb.Spec.Volume), nil } func newBackupper( ctx context.Context, log logrus.FieldLogger, repoLocker *repository.RepoLocker, repoEnsurer *repository.Ensurer, pvbInformer ctrlcache.Informer, crClient ctrlclient.Client, uploaderType string, backup *velerov1api.Backup, ) *backupper { b := &backupper{ ctx: ctx, repoLocker: repoLocker, repoEnsurer: repoEnsurer, crClient: crClient, uploaderType: uploaderType, pvbInformer: pvbInformer, wg: sync.WaitGroup{}, pvbIndexer: cache.NewIndexer(podVolumeBackupKey, cache.Indexers{ indexNamePod: podIndexFunc, }), } b.handlerRegistration, _ = pvbInformer.AddEventHandler( cache.ResourceEventHandlerFuncs{ UpdateFunc: func(_, obj any) { pvb, ok := obj.(*velerov1api.PodVolumeBackup) if !ok { log.Errorf("expected PodVolumeBackup, but got %T", obj) return } if pvb.GetLabels()[velerov1api.BackupUIDLabel] != string(backup.UID) { return } if pvb.Status.Phase != velerov1api.PodVolumeBackupPhaseCompleted && pvb.Status.Phase != velerov1api.PodVolumeBackupPhaseFailed && pvb.Status.Phase != velerov1api.PodVolumeBackupPhaseCanceled { return } statusChangedToFinal := true existObj, exist, err := b.pvbIndexer.Get(pvb) if err == nil && exist { existPVB, ok := existObj.(*velerov1api.PodVolumeBackup) // the PVB in the indexer is already in final status, no need to call WaitGroup.Done() if ok && (existPVB.Status.Phase == velerov1api.PodVolumeBackupPhaseCompleted || existPVB.Status.Phase == velerov1api.PodVolumeBackupPhaseFailed || pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseCanceled) { statusChangedToFinal = false } } // the Indexer inserts PVB directly if the PVB to be updated doesn't exist if err := b.pvbIndexer.Update(pvb); err != nil { log.WithError(err).Errorf("failed to update PVB %s/%s in indexer", pvb.Namespace, pvb.Name) } // call WaitGroup.Done() once only when the PVB changes to final status the first time. // This avoid the cases that the handler gets multiple update events whose PVBs are all in final status // which causes panic with "negative WaitGroup counter" error if statusChangedToFinal { b.wg.Done() } }, }, ) return b } func resultsKey(ns, name string) string { return fmt.Sprintf("%s/%s", ns, name) } func (b *backupper) getMatchAction(resPolicies *resourcepolicies.Policies, pvc *corev1api.PersistentVolumeClaim, volume *corev1api.Volume) (*resourcepolicies.Action, error) { if pvc != nil { // Ignore err, if the PV is not available (Pending/Lost PVC or PV fetch failed) - try matching with PVC only // GetPVForPVC returns nil for all error cases pv, _ := kube.GetPVForPVC(pvc, b.crClient) vfd := resourcepolicies.NewVolumeFilterData(pv, nil, pvc) return resPolicies.GetMatchAction(vfd) } if volume != nil { vfd := resourcepolicies.NewVolumeFilterData(nil, volume, pvc) return resPolicies.GetMatchAction(vfd) } return nil, errors.Errorf("failed to check resource policies for empty volume") } var funcGetRepositoryType = getRepositoryType func (b *backupper) BackupPodVolumes(backup *velerov1api.Backup, pod *corev1api.Pod, volumesToBackup []string, resPolicies *resourcepolicies.Policies, log logrus.FieldLogger) ([]*velerov1api.PodVolumeBackup, *PVCBackupSummary, []error) { if len(volumesToBackup) == 0 { return nil, nil, nil } log.Infof("pod %s/%s has volumes to backup: %v", pod.Namespace, pod.Name, volumesToBackup) var ( pvcSummary = NewPVCBackupSummary() podVolumes = make(map[string]corev1api.Volume) errs = []error{} ) // put the pod's volumes and the PVC associated in maps for efficient lookup below for _, podVolume := range pod.Spec.Volumes { podVolumes[podVolume.Name] = podVolume if podVolume.PersistentVolumeClaim != nil { pvc := new(corev1api.PersistentVolumeClaim) err := b.crClient.Get(context.TODO(), ctrlclient.ObjectKey{Namespace: pod.Namespace, Name: podVolume.PersistentVolumeClaim.ClaimName}, pvc) if err != nil { errs = append(errs, errors.Wrap(err, "error getting persistent volume claim for volume")) continue } pvcSummary.pvcMap[podVolume.Name] = pvc } } if msg, err := uploader.ValidateUploaderType(b.uploaderType); err != nil { skipAllPodVolumes(pod, volumesToBackup, err, pvcSummary, log) return nil, pvcSummary, []error{err} } else if msg != "" { log.Warn(msg) } if err := kube.IsPodRunning(pod); err != nil { skipAllPodVolumes(pod, volumesToBackup, err, pvcSummary, log) return nil, pvcSummary, nil } err := nodeagent.IsRunningInNode(b.ctx, backup.Namespace, pod.Spec.NodeName, b.crClient) if err != nil { skipAllPodVolumes(pod, volumesToBackup, err, pvcSummary, log) return nil, pvcSummary, []error{err} } repositoryType := funcGetRepositoryType(b.uploaderType) if repositoryType == "" { err := errors.Errorf("empty repository type, uploader %s", b.uploaderType) skipAllPodVolumes(pod, volumesToBackup, err, pvcSummary, log) return nil, pvcSummary, []error{err} } repo, err := b.repoEnsurer.EnsureRepo(b.ctx, backup.Namespace, pod.Namespace, backup.Spec.StorageLocation, repositoryType) if err != nil { skipAllPodVolumes(pod, volumesToBackup, err, pvcSummary, log) return nil, pvcSummary, []error{err} } // get a single non-exclusive lock since we'll wait for all individual // backups to be complete before releasing it. b.repoLocker.Lock(repo.Name) defer b.repoLocker.Unlock(repo.Name) var ( podVolumeBackups []*velerov1api.PodVolumeBackup mountedPodVolumes = sets.Set[string]{} attachedPodDevices = sets.Set[string]{} ) for _, container := range pod.Spec.Containers { for _, volumeMount := range container.VolumeMounts { mountedPodVolumes.Insert(volumeMount.Name) } for _, volumeDevice := range container.VolumeDevices { attachedPodDevices.Insert(volumeDevice.Name) } } repoIdentifier := "" if repositoryType == velerov1api.BackupRepositoryTypeRestic { repoIdentifier = repo.Spec.ResticIdentifier } for _, volumeName := range volumesToBackup { volume, ok := podVolumes[volumeName] if !ok { log.Warnf("No volume named %s found in pod %s/%s, skipping", volumeName, pod.Namespace, pod.Name) continue } var pvc *corev1api.PersistentVolumeClaim if volume.PersistentVolumeClaim != nil { pvc, ok = pvcSummary.pvcMap[volumeName] if !ok { // there should have been error happened retrieving the PVC and it's recorded already continue } } if resPolicies != nil { if action, err := b.getMatchAction(resPolicies, pvc, &volume); err != nil { errs = append(errs, errors.Wrapf(err, "error getting pv for pvc %s", pvc.Spec.VolumeName)) continue } else if action != nil && action.Type == resourcepolicies.Skip { log.Infof("skip backup of volume %s for the matched resource policies", volumeName) pvcSummary.addSkipped(volumeName, "matched action is 'skip' in chosen resource policies") continue } } // hostPath volumes are not supported because they're not mounted into /var/lib/kubelet/pods, so our // daemonset pod has no way to access their data. isHostPath, err := isHostPathVolume(&volume, pvc, b.crClient) if err != nil { errs = append(errs, errors.Wrap(err, "error checking if volume is a hostPath volume")) continue } if isHostPath { log.Warnf("Volume %s in pod %s/%s is a hostPath volume which is not supported for pod volume backup, skipping", volumeName, pod.Namespace, pod.Name) continue } // check if volume is a block volume if attachedPodDevices.Has(volumeName) { msg := fmt.Sprintf("volume %s declared in pod %s/%s is a block volume. Block volumes are not supported for fs backup, skipping", volumeName, pod.Namespace, pod.Name) log.Warn(msg) pvcSummary.addSkipped(volumeName, msg) continue } // volumes that are not mounted by any container should not be backed up, because // its directory is not created if !mountedPodVolumes.Has(volumeName) { msg := fmt.Sprintf("volume %s is declared in pod %s/%s but not mounted by any container, skipping", volumeName, pod.Namespace, pod.Name) log.Warn(msg) pvcSummary.addSkipped(volumeName, msg) continue } volumeBackup := newPodVolumeBackup(backup, pod, volume, repoIdentifier, b.uploaderType, pvc) // the PVB must be added into the indexer before creating it in API server otherwise unexpected behavior may happen: // the PVB may be handled very quickly by the controller and the informer handler will insert the PVB before "b.pvbIndexer.Add(volumeBackup)" runs, // this causes the PVB inserted by "b.pvbIndexer.Add(volumeBackup)" overrides the PVB in the indexer while the PVB inserted by "b.pvbIndexer.Add(volumeBackup)" // contains empty "Status" if err := b.pvbIndexer.Add(volumeBackup); err != nil { errs = append(errs, errors.Wrapf(err, "failed to add PodVolumeBackup %s/%s to indexer", volumeBackup.Namespace, volumeBackup.Name)) continue } // similar with above: the PVB may be handled very quickly by the controller and the informer handler will call "b.wg.Done()" before "b.wg.Add(1)" runs which causes panic // see https://github.com/vmware-tanzu/velero/issues/8657 b.wg.Add(1) if err := veleroclient.CreateRetryGenerateName(b.crClient, b.ctx, volumeBackup); err != nil { b.wg.Done() errs = append(errs, err) continue } podVolumeBackups = append(podVolumeBackups, volumeBackup) pvcSummary.addBackedup(volumeName) } return podVolumeBackups, pvcSummary, errs } func (b *backupper) WaitAllPodVolumesProcessed(log logrus.FieldLogger) []*velerov1api.PodVolumeBackup { defer func() { if err := b.pvbInformer.RemoveEventHandler(b.handlerRegistration); err != nil { log.Debugf("failed to remove the event handler for PVB: %v", err) } }() log.Info("Waiting for completion of PVB") var podVolumeBackups []*velerov1api.PodVolumeBackup // if no pod volume backups are tracked, return directly to avoid issue mentioned in // https://github.com/vmware-tanzu/velero/issues/8723 if len(b.pvbIndexer.List()) == 0 { return podVolumeBackups } done := make(chan struct{}) go func() { defer close(done) b.wg.Wait() }() select { case <-b.ctx.Done(): log.Error("timed out waiting for all PodVolumeBackups to complete") case <-done: for _, obj := range b.pvbIndexer.List() { pvb, ok := obj.(*velerov1api.PodVolumeBackup) if !ok { log.Errorf("expected PodVolumeBackup, but got %T", obj) continue } podVolumeBackups = append(podVolumeBackups, pvb) if pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseFailed { log.Errorf("pod volume backup failed: %s", pvb.Status.Message) } else if pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseCanceled { log.Errorf("pod volume backup canceled: %s", pvb.Status.Message) } } } return podVolumeBackups } func (b *backupper) GetPodVolumeBackupByPodAndVolume(podNamespace, podName, volume string) (*velerov1api.PodVolumeBackup, error) { obj, exist, err := b.pvbIndexer.GetByKey(fmt.Sprintf(pvbKeyPattern, podNamespace, podName, volume)) if err != nil { return nil, err } if !exist { return nil, nil } pvb, ok := obj.(*velerov1api.PodVolumeBackup) if !ok { return nil, errors.Errorf("expected PodVolumeBackup, but got %T", obj) } return pvb, nil } func (b *backupper) ListPodVolumeBackupsByPod(podNamespace, podName string) ([]*velerov1api.PodVolumeBackup, error) { objs, err := b.pvbIndexer.ByIndex(indexNamePod, cache.NewObjectName(podNamespace, podName).String()) if err != nil { return nil, err } var pvbs []*velerov1api.PodVolumeBackup for _, obj := range objs { pvb, ok := obj.(*velerov1api.PodVolumeBackup) if !ok { return nil, errors.Errorf("expected PodVolumeBackup, but got %T", obj) } pvbs = append(pvbs, pvb) } return pvbs, nil } func skipAllPodVolumes(pod *corev1api.Pod, volumesToBackup []string, err error, pvcSummary *PVCBackupSummary, log logrus.FieldLogger) { for _, volumeName := range volumesToBackup { log.WithError(err).Warnf("Skip pod volume %s", volumeName) pvcSummary.addSkipped(volumeName, fmt.Sprintf("encountered a problem with backing up the PVC of pod %s/%s: %v", pod.Namespace, pod.Name, err)) } } // isHostPathVolume returns true if the volume is either a hostPath pod volume or a persistent // volume claim on a hostPath persistent volume, or false otherwise. func isHostPathVolume(volume *corev1api.Volume, pvc *corev1api.PersistentVolumeClaim, crClient ctrlclient.Client) (bool, error) { if volume.HostPath != nil { return true, nil } if pvc == nil || pvc.Spec.VolumeName == "" { return false, nil } pv := new(corev1api.PersistentVolume) err := crClient.Get(context.TODO(), ctrlclient.ObjectKey{Name: pvc.Spec.VolumeName}, pv) if err != nil { return false, errors.WithStack(err) } return pv.Spec.HostPath != nil, nil } func newPodVolumeBackup(backup *velerov1api.Backup, pod *corev1api.Pod, volume corev1api.Volume, repoIdentifier, uploaderType string, pvc *corev1api.PersistentVolumeClaim) *velerov1api.PodVolumeBackup { pvb := &velerov1api.PodVolumeBackup{ ObjectMeta: metav1.ObjectMeta{ Namespace: backup.Namespace, GenerateName: backup.Name + "-", OwnerReferences: []metav1.OwnerReference{ { APIVersion: velerov1api.SchemeGroupVersion.String(), Kind: "Backup", Name: backup.Name, UID: backup.UID, Controller: boolptr.True(), }, }, Labels: map[string]string{ velerov1api.BackupNameLabel: label.GetValidName(backup.Name), velerov1api.BackupUIDLabel: string(backup.UID), }, }, Spec: velerov1api.PodVolumeBackupSpec{ Node: pod.Spec.NodeName, Pod: corev1api.ObjectReference{ Kind: "Pod", Namespace: pod.Namespace, Name: pod.Name, UID: pod.UID, }, Volume: volume.Name, Tags: map[string]string{ "backup": backup.Name, "backup-uid": string(backup.UID), "pod": pod.Name, "pod-uid": string(pod.UID), "ns": pod.Namespace, "volume": volume.Name, }, BackupStorageLocation: backup.Spec.StorageLocation, RepoIdentifier: repoIdentifier, UploaderType: uploaderType, }, } if pvc != nil { // this annotation is used in pkg/restore to identify if a PVC // has a pod volume backup. pvb.Annotations = map[string]string{ configs.PVCNameAnnotation: pvc.Name, } // this label is used by the pod volume backup controller to tell // if a pod volume backup is for a PVC. pvb.Labels[velerov1api.PVCUIDLabel] = string(pvc.UID) // this tag is not used by velero, but useful for debugging. pvb.Spec.Tags["pvc-uid"] = string(pvc.UID) } if backup.Spec.UploaderConfig != nil { pvb.Spec.UploaderSettings = uploaderutil.StoreBackupConfig(backup.Spec.UploaderConfig) } return pvb } ================================================ FILE: pkg/podvolume/backupper_factory.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package podvolume import ( "context" "github.com/pkg/errors" "github.com/sirupsen/logrus" "k8s.io/client-go/tools/cache" ctrlcache "sigs.k8s.io/controller-runtime/pkg/cache" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/repository" ) // BackupperFactory can construct pod volumes backuppers. type BackupperFactory interface { // NewBackupper returns a pod volumes backupper for use during a single Velero backup. NewBackupper(context.Context, logrus.FieldLogger, *velerov1api.Backup, string) (Backupper, error) } func NewBackupperFactory( repoLocker *repository.RepoLocker, repoEnsurer *repository.Ensurer, crClient ctrlclient.Client, pvbInformer ctrlcache.Informer, log logrus.FieldLogger, ) BackupperFactory { return &backupperFactory{ repoLocker: repoLocker, repoEnsurer: repoEnsurer, crClient: crClient, pvbInformer: pvbInformer, log: log, } } type backupperFactory struct { repoLocker *repository.RepoLocker repoEnsurer *repository.Ensurer crClient ctrlclient.Client pvbInformer ctrlcache.Informer log logrus.FieldLogger } func (bf *backupperFactory) NewBackupper(ctx context.Context, log logrus.FieldLogger, backup *velerov1api.Backup, uploaderType string) (Backupper, error) { b := newBackupper(ctx, log, bf.repoLocker, bf.repoEnsurer, bf.pvbInformer, bf.crClient, uploaderType, backup) if !cache.WaitForCacheSync(ctx.Done(), bf.pvbInformer.HasSynced) { return nil, errors.New("timed out waiting for caches to sync") } return b, nil } ================================================ FILE: pkg/podvolume/backupper_test.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package podvolume import ( "bytes" "context" "fmt" "testing" "time" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" ctrlfake "sigs.k8s.io/controller-runtime/pkg/client/fake" clientTesting "k8s.io/client-go/testing" "k8s.io/client-go/tools/cache" "github.com/vmware-tanzu/velero/internal/resourcepolicies" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/repository" velerotest "github.com/vmware-tanzu/velero/pkg/test" "github.com/vmware-tanzu/velero/pkg/util/kube" ) func TestIsHostPathVolume(t *testing.T) { // hostPath pod volume vol := &corev1api.Volume{ VolumeSource: corev1api.VolumeSource{ HostPath: &corev1api.HostPathVolumeSource{}, }, } isHostPath, err := isHostPathVolume(vol, nil, nil) require.NoError(t, err) assert.True(t, isHostPath) // non-hostPath pod volume vol = &corev1api.Volume{ VolumeSource: corev1api.VolumeSource{ EmptyDir: &corev1api.EmptyDirVolumeSource{}, }, } isHostPath, err = isHostPathVolume(vol, nil, nil) require.NoError(t, err) assert.False(t, isHostPath) // PVC that doesn't have a PV vol = &corev1api.Volume{ VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "pvc-1", }, }, } pvc := &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: "ns-1", Name: "pvc-1", }, } isHostPath, err = isHostPathVolume(vol, pvc, nil) require.NoError(t, err) assert.False(t, isHostPath) // PVC that claims a non-hostPath PV vol = &corev1api.Volume{ VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "pvc-1", }, }, } pvc = &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: "ns-1", Name: "pvc-1", }, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "pv-1", }, } pv := &corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "pv-1", }, Spec: corev1api.PersistentVolumeSpec{}, } crClient1 := velerotest.NewFakeControllerRuntimeClient(t, pv) isHostPath, err = isHostPathVolume(vol, pvc, crClient1) require.NoError(t, err) assert.False(t, isHostPath) // PVC that claims a hostPath PV vol = &corev1api.Volume{ VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "pvc-1", }, }, } pvc = &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: "ns-1", Name: "pvc-1", }, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "pv-1", }, } pv = &corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "pv-1", }, Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ HostPath: &corev1api.HostPathVolumeSource{}, }, }, } crClient2 := velerotest.NewFakeControllerRuntimeClient(t, pv) isHostPath, err = isHostPathVolume(vol, pvc, crClient2) require.NoError(t, err) assert.True(t, isHostPath) } func Test_backupper_BackupPodVolumes_log_test(t *testing.T) { type args struct { backup *velerov1api.Backup pod *corev1api.Pod volumesToBackup []string resPolicies *resourcepolicies.Policies } tests := []struct { name string args args wantLog string }{ { name: "backup pod volumes should log volume names", args: args{ backup: &velerov1api.Backup{ ObjectMeta: metav1.ObjectMeta{ Name: "backup-1", Namespace: "ns-1", }, }, pod: &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "pod-1", Namespace: "ns-1", }, Spec: corev1api.PodSpec{ Volumes: []corev1api.Volume{ { Name: "vol-1", }, { Name: "vol-2", }, }, }, }, volumesToBackup: []string{"vol-1", "vol-2"}, resPolicies: nil, }, wantLog: "pod ns-1/pod-1 has volumes to backup: [vol-1 vol-2]", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { b := &backupper{ ctx: t.Context(), } logOutput := bytes.Buffer{} var log = logrus.New() log.SetOutput(&logOutput) b.BackupPodVolumes(tt.args.backup, tt.args.pod, tt.args.volumesToBackup, tt.args.resPolicies, log) fmt.Println(logOutput.String()) assert.Contains(t, logOutput.String(), tt.wantLog) }) } } type reactor struct { verb string resource string reactorFunc clientTesting.ReactionFunc } func createBackupRepoObj() *velerov1api.BackupRepository { bkRepoObj := repository.NewBackupRepository(velerov1api.DefaultNamespace, repository.BackupRepositoryKey{ VolumeNamespace: "fake-ns", BackupLocation: "fake-bsl", RepositoryType: "kopia", }) bkRepoObj.Status.Phase = velerov1api.BackupRepositoryPhaseReady return bkRepoObj } func createPodObj(running bool, withVolume bool, withVolumeMounted bool, volumeNum int) *corev1api.Pod { podObj := builder.ForPod("fake-ns", "fake-pod").Result() podObj.Spec.NodeName = "fake-node-name" if running { podObj.Status.Phase = corev1api.PodRunning } if withVolume { for i := 0; i < volumeNum; i++ { podObj.Spec.Volumes = append(podObj.Spec.Volumes, corev1api.Volume{ Name: fmt.Sprintf("fake-volume-%d", i+1), VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: fmt.Sprintf("fake-pvc-%d", i+1), }, }, }) } if withVolumeMounted { volumeMount := []corev1api.VolumeMount{} for i := 0; i < volumeNum; i++ { volumeMount = append(volumeMount, corev1api.VolumeMount{ Name: fmt.Sprintf("fake-volume-%d", i+1), }) } podObj.Spec.Containers = []corev1api.Container{ { Name: "fake-container", VolumeMounts: volumeMount, }, } } } return podObj } func createNodeAgentPodObj(running bool) *corev1api.Pod { podObj := builder.ForPod(velerov1api.DefaultNamespace, "fake-node-agent").Result() podObj.Labels = map[string]string{"role": "node-agent"} if running { podObj.Status.Phase = corev1api.PodRunning podObj.Spec.NodeName = "fake-node-name" } return podObj } func createPVObj(index int, withHostPath bool) *corev1api.PersistentVolume { pvObj := builder.ForPersistentVolume(fmt.Sprintf("fake-pv-%d", index)).Result() if withHostPath { pvObj.Spec.HostPath = &corev1api.HostPathVolumeSource{Path: "fake-host-path"} } return pvObj } func createPVCObj(index int) *corev1api.PersistentVolumeClaim { pvcObj := builder.ForPersistentVolumeClaim("fake-ns", fmt.Sprintf("fake-pvc-%d", index)).VolumeName(fmt.Sprintf("fake-pv-%d", index)).Result() return pvcObj } func createPVBObj(fail bool, withSnapshot bool, index int, uploaderType string) *velerov1api.PodVolumeBackup { pvbObj := builder.ForPodVolumeBackup(velerov1api.DefaultNamespace, fmt.Sprintf("fake-pvb-%d", index)). PodName("fake-pod").PodNamespace("fake-ns").Volume(fmt.Sprintf("fake-volume-%d", index)).Result() if fail { pvbObj.Status.Phase = velerov1api.PodVolumeBackupPhaseFailed pvbObj.Status.Message = "fake-message" } else { pvbObj.Status.Phase = velerov1api.PodVolumeBackupPhaseCompleted } if withSnapshot { pvbObj.Status.SnapshotID = fmt.Sprintf("fake-snapshot-id-%d", index) } pvbObj.Spec.UploaderType = uploaderType return pvbObj } func createNodeObj() *corev1api.Node { return builder.ForNode("fake-node-name").Labels(map[string]string{"kubernetes.io/os": "linux"}).Result() } func TestBackupPodVolumes(t *testing.T) { scheme := runtime.NewScheme() require.NoError(t, velerov1api.AddToScheme(scheme)) require.NoError(t, corev1api.AddToScheme(scheme)) log := logrus.New() tests := []struct { name string bsl string uploaderType string volumes []string sourcePod *corev1api.Pod kubeClientObj []runtime.Object ctlClientObj []runtime.Object veleroClientObj []runtime.Object veleroReactors []reactor runtimeScheme *runtime.Scheme pvbs int mockGetRepositoryType bool errs []string }{ { name: "empty volume list", }, { name: "wrong uploader type", volumes: []string{ "fake-volume-1", "fake-volume-2", }, sourcePod: createPodObj(true, false, false, 2), kubeClientObj: []runtime.Object{ createNodeAgentPodObj(true), }, uploaderType: "fake-uploader-type", errs: []string{ "invalid uploader type 'fake-uploader-type', valid type: 'kopia'", }, }, { name: "pod is not running", volumes: []string{ "fake-volume-1", "fake-volume-2", }, kubeClientObj: []runtime.Object{ createNodeAgentPodObj(true), }, ctlClientObj: []runtime.Object{ createBackupRepoObj(), }, runtimeScheme: scheme, sourcePod: createPodObj(false, false, false, 2), uploaderType: "kopia", bsl: "fake-bsl", }, { name: "node-agent pod is not running in node", volumes: []string{ "fake-volume-1", "fake-volume-2", }, sourcePod: createPodObj(true, false, false, 2), kubeClientObj: []runtime.Object{ createNodeObj(), }, uploaderType: "kopia", errs: []string{ "daemonset pod not found in running state in node fake-node-name", }, }, { name: "wrong repository type", volumes: []string{ "fake-volume-1", "fake-volume-2", }, sourcePod: createPodObj(true, false, false, 2), kubeClientObj: []runtime.Object{ createNodeAgentPodObj(true), createNodeObj(), }, uploaderType: "kopia", mockGetRepositoryType: true, errs: []string{ "empty repository type, uploader kopia", }, }, { name: "ensure repo fail", volumes: []string{ "fake-volume-1", "fake-volume-2", }, sourcePod: createPodObj(true, false, false, 2), kubeClientObj: []runtime.Object{ createNodeAgentPodObj(true), createNodeObj(), }, uploaderType: "kopia", errs: []string{ "wrong parameters, namespace \"fake-ns\", backup storage location \"\", repository type \"kopia\"", }, }, { name: "volume not found in pod", volumes: []string{ "fake-volume-1", "fake-volume-2", }, sourcePod: createPodObj(true, false, false, 2), kubeClientObj: []runtime.Object{ createNodeAgentPodObj(true), createNodeObj(), }, ctlClientObj: []runtime.Object{ createBackupRepoObj(), }, runtimeScheme: scheme, uploaderType: "kopia", bsl: "fake-bsl", }, { name: "PVC not found", volumes: []string{ "fake-volume-1", "fake-volume-2", }, sourcePod: createPodObj(true, true, false, 2), kubeClientObj: []runtime.Object{ createNodeAgentPodObj(true), createNodeObj(), }, ctlClientObj: []runtime.Object{ createBackupRepoObj(), }, runtimeScheme: scheme, uploaderType: "kopia", bsl: "fake-bsl", errs: []string{ "error getting persistent volume claim for volume: persistentvolumeclaims \"fake-pvc-1\" not found", "error getting persistent volume claim for volume: persistentvolumeclaims \"fake-pvc-2\" not found", }, }, { name: "check host path fail", volumes: []string{ "fake-volume-1", "fake-volume-2", }, sourcePod: createPodObj(true, true, false, 2), kubeClientObj: []runtime.Object{ createNodeAgentPodObj(true), createNodeObj(), createPVCObj(1), createPVCObj(2), }, ctlClientObj: []runtime.Object{ createBackupRepoObj(), }, runtimeScheme: scheme, uploaderType: "kopia", bsl: "fake-bsl", errs: []string{ "error checking if volume is a hostPath volume: persistentvolumes \"fake-pv-1\" not found", "error checking if volume is a hostPath volume: persistentvolumes \"fake-pv-2\" not found", }, }, { name: "host path volume should be skipped", volumes: []string{ "fake-volume-1", "fake-volume-2", }, sourcePod: createPodObj(true, true, false, 2), kubeClientObj: []runtime.Object{ createNodeAgentPodObj(true), createNodeObj(), createPVCObj(1), createPVCObj(2), createPVObj(1, true), createPVObj(2, true), }, ctlClientObj: []runtime.Object{ createBackupRepoObj(), }, runtimeScheme: scheme, uploaderType: "kopia", bsl: "fake-bsl", errs: []string{}, }, { name: "volume not mounted by pod should be skipped", volumes: []string{ "fake-volume-1", "fake-volume-2", }, sourcePod: createPodObj(true, true, false, 2), kubeClientObj: []runtime.Object{ createNodeAgentPodObj(true), createNodeObj(), createPVCObj(1), createPVCObj(2), createPVObj(1, false), createPVObj(2, false), }, ctlClientObj: []runtime.Object{ createBackupRepoObj(), }, runtimeScheme: scheme, uploaderType: "kopia", bsl: "fake-bsl", errs: []string{}, }, { name: "return completed pvbs", volumes: []string{ "fake-volume-1", }, sourcePod: createPodObj(true, true, true, 1), kubeClientObj: []runtime.Object{ createNodeAgentPodObj(true), createNodeObj(), createPVCObj(1), createPVObj(1, false), }, ctlClientObj: []runtime.Object{ createBackupRepoObj(), }, runtimeScheme: scheme, uploaderType: "kopia", bsl: "fake-bsl", pvbs: 1, errs: []string{}, }, } // TODO add more verification around PVCBackupSummary returned by "BackupPodVolumes" for _, test := range tests { t.Run(test.name, func(t *testing.T) { ctx := t.Context() fakeClientBuilder := ctrlfake.NewClientBuilder() if test.runtimeScheme != nil { fakeClientBuilder = fakeClientBuilder.WithScheme(test.runtimeScheme) } objList := append(test.ctlClientObj, test.veleroClientObj...) objList = append(objList, test.kubeClientObj...) fakeCtrlClient := fakeClientBuilder.WithRuntimeObjects(objList...).Build() fakeCRWatchClient := velerotest.NewFakeControllerRuntimeWatchClient(t, test.kubeClientObj...) lw := kube.InternalLW{ Client: fakeCRWatchClient, Namespace: velerov1api.DefaultNamespace, ObjectList: new(velerov1api.PodVolumeBackupList), } pvbInformer := cache.NewSharedIndexInformer(&lw, &velerov1api.PodVolumeBackup{}, 0, cache.Indexers{}) go pvbInformer.Run(ctx.Done()) require.True(t, cache.WaitForCacheSync(ctx.Done(), pvbInformer.HasSynced)) ensurer := repository.NewEnsurer(fakeCtrlClient, velerotest.NewLogger(), time.Millisecond) backupObj := builder.ForBackup(velerov1api.DefaultNamespace, "fake-backup").Result() backupObj.Spec.StorageLocation = test.bsl factory := NewBackupperFactory(repository.NewRepoLocker(), ensurer, fakeCtrlClient, pvbInformer, velerotest.NewLogger()) bp, err := factory.NewBackupper(ctx, log, backupObj, test.uploaderType) require.NoError(t, err) if test.mockGetRepositoryType { funcGetRepositoryType = func(string) string { return "" } } else { funcGetRepositoryType = getRepositoryType } pvbs, _, errs := bp.BackupPodVolumes(backupObj, test.sourcePod, test.volumes, nil, velerotest.NewLogger()) if test.errs == nil { require.NoError(t, err) } else { for i := 0; i < len(errs); i++ { require.EqualError(t, errs[i], test.errs[i]) } } assert.Len(t, pvbs, test.pvbs) }) } } func TestGetPodVolumeBackupByPodAndVolume(t *testing.T) { backupper := &backupper{ pvbIndexer: cache.NewIndexer(podVolumeBackupKey, cache.Indexers{ indexNamePod: podIndexFunc, }), } obj := &velerov1api.PodVolumeBackup{ Spec: velerov1api.PodVolumeBackupSpec{ Pod: corev1api.ObjectReference{ Kind: "Pod", Namespace: "default", Name: "pod", }, Volume: "volume", }, } err := backupper.pvbIndexer.Add(obj) require.NoError(t, err) // incorrect pod namespace pvb, err := backupper.GetPodVolumeBackupByPodAndVolume("invalid-namespace", "pod", "volume") require.NoError(t, err) assert.Nil(t, pvb) // incorrect pod name pvb, err = backupper.GetPodVolumeBackupByPodAndVolume("default", "invalid-pod", "volume") require.NoError(t, err) assert.Nil(t, pvb) // incorrect volume pvb, err = backupper.GetPodVolumeBackupByPodAndVolume("default", "pod", "invalid-volume") require.NoError(t, err) assert.Nil(t, pvb) // correct pod namespace, name and volume pvb, err = backupper.GetPodVolumeBackupByPodAndVolume("default", "pod", "volume") require.NoError(t, err) assert.NotNil(t, pvb) } func TestListPodVolumeBackupsByPodp(t *testing.T) { backupper := &backupper{ pvbIndexer: cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{ indexNamePod: podIndexFunc, }), } obj1 := &velerov1api.PodVolumeBackup{ ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", Name: "pvb1", }, Spec: velerov1api.PodVolumeBackupSpec{ Pod: corev1api.ObjectReference{ Kind: "Pod", Namespace: "default", Name: "pod", }, }, } obj2 := &velerov1api.PodVolumeBackup{ ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", Name: "pvb2", }, Spec: velerov1api.PodVolumeBackupSpec{ Pod: corev1api.ObjectReference{ Kind: "Pod", Namespace: "default", Name: "pod", }, }, } err := backupper.pvbIndexer.Add(obj1) require.NoError(t, err) err = backupper.pvbIndexer.Add(obj2) require.NoError(t, err) // not exist PVBs pvbs, err := backupper.ListPodVolumeBackupsByPod("invalid-namespace", "invalid-name") require.NoError(t, err) assert.Empty(t, pvbs) // exist PVBs pvbs, err = backupper.ListPodVolumeBackupsByPod("default", "pod") require.NoError(t, err) assert.Len(t, pvbs, 2) } type logHook struct { entry *logrus.Entry } func (l *logHook) Levels() []logrus.Level { return []logrus.Level{logrus.ErrorLevel} } func (l *logHook) Fire(entry *logrus.Entry) error { l.entry = entry return nil } func TestWaitAllPodVolumesProcessed(t *testing.T) { timeoutCtx, cancelFunc := context.WithCancel(t.Context()) cancelFunc() log := logrus.New() pvb := builder.ForPodVolumeBackup(velerov1api.DefaultNamespace, "pvb"). PodNamespace("pod-namespace").PodName("pod-name").Volume("volume").Result() cases := []struct { name string ctx context.Context pvb *velerov1api.PodVolumeBackup statusToBeUpdated *velerov1api.PodVolumeBackupStatus expectedErr string expectedPVBPhase velerov1api.PodVolumeBackupPhase }{ { name: "contains no pvb should report no error", ctx: timeoutCtx, }, { name: "context canceled", ctx: timeoutCtx, pvb: pvb, expectedErr: "timed out waiting for all PodVolumeBackups to complete", }, { name: "failed pvbs", ctx: t.Context(), pvb: pvb, statusToBeUpdated: &velerov1api.PodVolumeBackupStatus{ Phase: velerov1api.PodVolumeBackupPhaseFailed, Message: "failed", }, expectedPVBPhase: velerov1api.PodVolumeBackupPhaseFailed, expectedErr: "pod volume backup failed: failed", }, { name: "completed pvbs", ctx: t.Context(), pvb: pvb, statusToBeUpdated: &velerov1api.PodVolumeBackupStatus{ Phase: velerov1api.PodVolumeBackupPhaseCompleted, Message: "completed", }, expectedPVBPhase: velerov1api.PodVolumeBackupPhaseCompleted, }, } for _, c := range cases { var objs []ctrlclient.Object if c.pvb != nil { objs = append(objs, c.pvb) } scheme := runtime.NewScheme() velerov1api.AddToScheme(scheme) client := ctrlfake.NewClientBuilder().WithScheme(scheme).WithObjects(objs...).Build() lw := kube.InternalLW{ Client: client, Namespace: velerov1api.DefaultNamespace, ObjectList: new(velerov1api.PodVolumeBackupList), } informer := cache.NewSharedIndexInformer(&lw, &velerov1api.PodVolumeBackup{}, 0, cache.Indexers{}) ctx := t.Context() go informer.Run(ctx.Done()) require.True(t, cache.WaitForCacheSync(ctx.Done(), informer.HasSynced)) logger := logrus.New() logHook := &logHook{} logger.Hooks.Add(logHook) backuper := newBackupper(c.ctx, log, nil, nil, informer, nil, "", &velerov1api.Backup{}) if c.pvb != nil { require.NoError(t, backuper.pvbIndexer.Add(c.pvb)) backuper.wg.Add(1) } if c.statusToBeUpdated != nil { pvb := &velerov1api.PodVolumeBackup{} err := client.Get(t.Context(), ctrlclient.ObjectKey{Namespace: c.pvb.Namespace, Name: c.pvb.Name}, pvb) require.NoError(t, err) pvb.Status = *c.statusToBeUpdated err = client.Update(t.Context(), pvb) require.NoError(t, err) } pvbs := backuper.WaitAllPodVolumesProcessed(logger) if c.expectedErr != "" { assert.Equal(t, c.expectedErr, logHook.entry.Message) } else { assert.Nil(t, logHook.entry) } if c.expectedPVBPhase != "" { require.Len(t, pvbs, 1) assert.Equal(t, c.expectedPVBPhase, pvbs[0].Status.Phase) } } } func TestPVCBackupSummary(t *testing.T) { pbs := NewPVCBackupSummary() pbs.pvcMap["vol-1"] = builder.ForPersistentVolumeClaim("ns-1", "pvc-1").VolumeName("pv-1").Result() pbs.pvcMap["vol-2"] = builder.ForPersistentVolumeClaim("ns-2", "pvc-2").VolumeName("pv-2").Result() // it won't be added if the volme is not in the pvc map. pbs.addSkipped("vol-3", "whatever reason") assert.Empty(t, pbs.Skipped) pbs.addBackedup("vol-3") assert.Empty(t, pbs.Backedup) // only can be added as skipped when it's not in backedup set pbs.addBackedup("vol-1") assert.Len(t, pbs.Backedup, 1) assert.Equal(t, "pvc-1", pbs.Backedup["vol-1"].Name) pbs.addSkipped("vol-1", "whatever reason") assert.Empty(t, pbs.Skipped) pbs.addSkipped("vol-2", "vol-2 has to be skipped") assert.Len(t, pbs.Skipped, 1) assert.Equal(t, "pvc-2", pbs.Skipped["vol-2"].PVC.Name) // adding a vol as backedup removes it from skipped set pbs.addBackedup("vol-2") assert.Empty(t, pbs.Skipped) assert.Len(t, pbs.Backedup, 2) } func TestGetMatchAction_PendingPVC(t *testing.T) { // Create resource policies that skip Pending/Lost PVCs resPolicies := &resourcepolicies.ResourcePolicies{ Version: "v1", VolumePolicies: []resourcepolicies.VolumePolicy{ { Conditions: map[string]any{ "pvcPhase": []string{"Pending", "Lost"}, }, Action: resourcepolicies.Action{ Type: resourcepolicies.Skip, }, }, }, } policies := &resourcepolicies.Policies{} err := policies.BuildPolicy(resPolicies) require.NoError(t, err) testCases := []struct { name string pvc *corev1api.PersistentVolumeClaim volume *corev1api.Volume pv *corev1api.PersistentVolume expectedAction *resourcepolicies.Action expectError bool }{ { name: "Pending PVC with pvcPhase skip policy should return skip action", pvc: builder.ForPersistentVolumeClaim("ns", "pending-pvc"). StorageClass("test-sc"). Phase(corev1api.ClaimPending). Result(), volume: &corev1api.Volume{ Name: "test-volume", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "pending-pvc", }, }, }, pv: nil, expectedAction: &resourcepolicies.Action{Type: resourcepolicies.Skip}, expectError: false, }, { name: "Lost PVC with pvcPhase skip policy should return skip action", pvc: builder.ForPersistentVolumeClaim("ns", "lost-pvc"). StorageClass("test-sc"). Phase(corev1api.ClaimLost). Result(), volume: &corev1api.Volume{ Name: "test-volume", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "lost-pvc", }, }, }, pv: nil, expectedAction: &resourcepolicies.Action{Type: resourcepolicies.Skip}, expectError: false, }, { name: "Bound PVC with matching PV should not match pvcPhase policy", pvc: builder.ForPersistentVolumeClaim("ns", "bound-pvc"). StorageClass("test-sc"). VolumeName("test-pv"). Phase(corev1api.ClaimBound). Result(), volume: &corev1api.Volume{ Name: "test-volume", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "bound-pvc", }, }, }, pv: builder.ForPersistentVolume("test-pv").StorageClass("test-sc").Result(), expectedAction: nil, expectError: false, }, { name: "Pending PVC with no matching policy should return nil action", pvc: builder.ForPersistentVolumeClaim("ns", "pending-pvc-no-match"). StorageClass("test-sc"). Phase(corev1api.ClaimPending). Result(), volume: &corev1api.Volume{ Name: "test-volume", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "pending-pvc-no-match", }, }, }, pv: nil, expectedAction: &resourcepolicies.Action{Type: resourcepolicies.Skip}, // Will match the pvcPhase policy expectError: false, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { // Build fake client with PV if present var objs []runtime.Object if tc.pv != nil { objs = append(objs, tc.pv) } fakeClient := velerotest.NewFakeControllerRuntimeClient(t, objs...) b := &backupper{ crClient: fakeClient, } action, err := b.getMatchAction(policies, tc.pvc, tc.volume) if tc.expectError { require.Error(t, err) } else { require.NoError(t, err) } if tc.expectedAction == nil { assert.Nil(t, action) } else { require.NotNil(t, action) assert.Equal(t, tc.expectedAction.Type, action.Type) } }) } } func TestGetMatchAction_PVCWithoutPVLookupError(t *testing.T) { // Test that when a PVC has a VolumeName but the PV doesn't exist, // the function ignores the error and tries to match with PVC only resPolicies := &resourcepolicies.ResourcePolicies{ Version: "v1", VolumePolicies: []resourcepolicies.VolumePolicy{ { Conditions: map[string]any{ "pvcPhase": []string{"Pending"}, }, Action: resourcepolicies.Action{ Type: resourcepolicies.Skip, }, }, }, } policies := &resourcepolicies.Policies{} err := policies.BuildPolicy(resPolicies) require.NoError(t, err) // Pending PVC without a matching PV in the cluster pvc := builder.ForPersistentVolumeClaim("ns", "pending-pvc"). StorageClass("test-sc"). Phase(corev1api.ClaimPending). Result() volume := &corev1api.Volume{ Name: "test-volume", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "pending-pvc", }, }, } // Empty client - no PV exists fakeClient := velerotest.NewFakeControllerRuntimeClient(t) b := &backupper{ crClient: fakeClient, } // Should succeed even though PV lookup would fail // because the function ignores PV lookup errors and uses PVC-only matching action, err := b.getMatchAction(policies, pvc, volume) require.NoError(t, err) require.NotNil(t, action) assert.Equal(t, resourcepolicies.Skip, action.Type) } ================================================ FILE: pkg/podvolume/configs/configs.go ================================================ package configs const ( // PVCNameAnnotation is the key for the annotation added to // pod volume backups when they're for a PVC. PVCNameAnnotation = "velero.io/pvc-name" // DefaultVolumesToFsBackup specifies whether pod volume backup should be used, by default, to // take backup of all pod volumes. DefaultVolumesToFsBackup = false ) ================================================ FILE: pkg/podvolume/mocks/restorer.go ================================================ // Code generated by mockery v2.42.2. DO NOT EDIT. package mocks import ( mock "github.com/stretchr/testify/mock" podvolume "github.com/vmware-tanzu/velero/pkg/podvolume" volume "github.com/vmware-tanzu/velero/internal/volume" ) // Restorer is an autogenerated mock type for the Restorer type type Restorer struct { mock.Mock } // RestorePodVolumes provides a mock function with given fields: _a0, _a1 func (_m *Restorer) RestorePodVolumes(_a0 podvolume.RestoreData, _a1 *volume.RestoreVolumeInfoTracker) []error { ret := _m.Called(_a0, _a1) if len(ret) == 0 { panic("no return value specified for RestorePodVolumes") } var r0 []error if rf, ok := ret.Get(0).(func(podvolume.RestoreData, *volume.RestoreVolumeInfoTracker) []error); ok { r0 = rf(_a0, _a1) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]error) } } return r0 } // NewRestorer creates a new instance of Restorer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. func NewRestorer(t interface { mock.TestingT Cleanup(func()) }) *Restorer { mock := &Restorer{} mock.Mock.Test(t) t.Cleanup(func() { mock.AssertExpectations(t) }) return mock } ================================================ FILE: pkg/podvolume/restore_micro_service.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package podvolume import ( "context" "fmt" "os" "path/filepath" "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/internal/credentials" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/datapath" "github.com/vmware-tanzu/velero/pkg/repository" "github.com/vmware-tanzu/velero/pkg/uploader" "github.com/vmware-tanzu/velero/pkg/util/kube" cachetool "k8s.io/client-go/tools/cache" ) // RestoreMicroService process data mover restores inside the restore pod type RestoreMicroService struct { ctx context.Context client client.Client kubeClient kubernetes.Interface repoEnsurer *repository.Ensurer credentialGetter *credentials.CredentialGetter logger logrus.FieldLogger dataPathMgr *datapath.Manager eventRecorder kube.EventRecorder namespace string pvrName string pvr *velerov1api.PodVolumeRestore sourceTargetPath datapath.AccessPoint resultSignal chan dataPathResult pvrInformer cache.Informer pvrHandler cachetool.ResourceEventHandlerRegistration nodeName string cacheDir string } func NewRestoreMicroService(ctx context.Context, client client.Client, kubeClient kubernetes.Interface, pvrName string, namespace string, nodeName string, sourceTargetPath datapath.AccessPoint, dataPathMgr *datapath.Manager, repoEnsurer *repository.Ensurer, cred *credentials.CredentialGetter, pvrInformer cache.Informer, cacheDir string, log logrus.FieldLogger) *RestoreMicroService { return &RestoreMicroService{ ctx: ctx, client: client, kubeClient: kubeClient, credentialGetter: cred, logger: log, repoEnsurer: repoEnsurer, dataPathMgr: dataPathMgr, namespace: namespace, pvrName: pvrName, sourceTargetPath: sourceTargetPath, nodeName: nodeName, resultSignal: make(chan dataPathResult), pvrInformer: pvrInformer, cacheDir: cacheDir, } } func (r *RestoreMicroService) Init() error { r.eventRecorder = kube.NewEventRecorder(r.kubeClient, r.client.Scheme(), r.pvrName, r.nodeName, r.logger) handler, err := r.pvrInformer.AddEventHandler( cachetool.ResourceEventHandlerFuncs{ UpdateFunc: func(oldObj any, newObj any) { oldPvr := oldObj.(*velerov1api.PodVolumeRestore) newPvr := newObj.(*velerov1api.PodVolumeRestore) if newPvr.Name != r.pvrName { return } if newPvr.Status.Phase != velerov1api.PodVolumeRestorePhaseInProgress { return } if newPvr.Spec.Cancel && !oldPvr.Spec.Cancel { r.cancelPodVolumeRestore(newPvr) } }, }, ) if err != nil { return errors.Wrap(err, "error adding PVR handler") } r.pvrHandler = handler return err } func (r *RestoreMicroService) RunCancelableDataPath(ctx context.Context) (string, error) { log := r.logger.WithFields(logrus.Fields{ "PVR": r.pvrName, }) pvr := &velerov1api.PodVolumeRestore{} err := wait.PollUntilContextCancel(ctx, 500*time.Millisecond, true, func(ctx context.Context) (bool, error) { err := r.client.Get(ctx, types.NamespacedName{ Namespace: r.namespace, Name: r.pvrName, }, pvr) if apierrors.IsNotFound(err) { return false, nil } if err != nil { return true, errors.Wrapf(err, "error to get PVR %s", r.pvrName) } if pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseInProgress { return true, nil } else { return false, nil } }) if err != nil { log.WithError(err).Error("Failed to wait PVR") return "", errors.Wrap(err, "error waiting for PVR") } r.pvr = pvr log.Info("Run cancelable PVR") callbacks := datapath.Callbacks{ OnCompleted: r.OnPvrCompleted, OnFailed: r.OnPvrFailed, OnCancelled: r.OnPvrCancelled, OnProgress: r.OnPvrProgress, } fsRestore, err := r.dataPathMgr.CreateFileSystemBR(pvr.Name, podVolumeRequestor, ctx, r.client, pvr.Namespace, callbacks, log) if err != nil { return "", errors.Wrap(err, "error to create data path") } log.Debug("Async fs br created") if err := fsRestore.Init(ctx, &datapath.FSBRInitParam{ BSLName: pvr.Spec.BackupStorageLocation, SourceNamespace: pvr.Spec.SourceNamespace, UploaderType: pvr.Spec.UploaderType, RepositoryType: velerov1api.BackupRepositoryTypeKopia, RepoIdentifier: "", RepositoryEnsurer: r.repoEnsurer, CredentialGetter: r.credentialGetter, CacheDir: r.cacheDir, }); err != nil { return "", errors.Wrap(err, "error to initialize data path") } log.Info("Async fs br init") if err := fsRestore.StartRestore(pvr.Spec.SnapshotID, r.sourceTargetPath, pvr.Spec.UploaderSettings); err != nil { return "", errors.Wrap(err, "error starting data path restore") } log.Info("Async fs restore data path started") r.eventRecorder.Event(pvr, false, datapath.EventReasonStarted, "Data path for %s started", pvr.Name) result := "" select { case <-ctx.Done(): err = errors.New("timed out waiting for fs restore to complete") break case res := <-r.resultSignal: err = res.err result = res.result break } if err != nil { log.WithError(err).Error("Async fs restore was not completed") } r.eventRecorder.EndingEvent(pvr, false, datapath.EventReasonStopped, "Data path for %s stopped", pvr.Name) return result, err } func (r *RestoreMicroService) Shutdown() { r.eventRecorder.Shutdown() r.closeDataPath(r.ctx, r.pvrName) if r.pvrHandler != nil { if err := r.pvrInformer.RemoveEventHandler(r.pvrHandler); err != nil { r.logger.WithError(err).Warn("Failed to remove pod handler") } } } var funcWriteCompletionMark = writeCompletionMark func (r *RestoreMicroService) OnPvrCompleted(ctx context.Context, namespace string, pvrName string, result datapath.Result) { log := r.logger.WithField("PVR", pvrName) err := funcWriteCompletionMark(r.pvr, result.Restore, log) if err != nil { log.WithError(err).Warnf("Failed to write completion mark, restored pod may failed to start") } restoreBytes, err := funcMarshal(result.Restore) if err != nil { log.WithError(err).Errorf("Failed to marshal restore result %v", result.Restore) r.recordPvrFailed(fmt.Sprintf("error marshaling restore result %v", result.Restore), err) } else { r.eventRecorder.Event(r.pvr, false, datapath.EventReasonCompleted, string(restoreBytes)) r.resultSignal <- dataPathResult{ result: string(restoreBytes), } } log.Info("Async fs restore data path completed") } func (r *RestoreMicroService) recordPvrFailed(msg string, err error) { evtMsg := fmt.Sprintf("%s, error %v", msg, err) r.eventRecorder.Event(r.pvr, false, datapath.EventReasonFailed, evtMsg) r.resultSignal <- dataPathResult{ err: errors.Wrap(err, msg), } } func (r *RestoreMicroService) OnPvrFailed(ctx context.Context, namespace string, pvrName string, err error) { log := r.logger.WithField("PVR", pvrName) log.WithError(err).Error("Async fs restore data path failed") r.recordPvrFailed(fmt.Sprintf("Data path for PVR %s failed", pvrName), err) } func (r *RestoreMicroService) OnPvrCancelled(ctx context.Context, namespace string, pvrName string) { log := r.logger.WithField("PVR", pvrName) log.Warn("Async fs restore data path canceled") r.eventRecorder.Event(r.pvr, false, datapath.EventReasonCancelled, "Data path for PVR %s canceled", pvrName) r.resultSignal <- dataPathResult{ err: errors.New(datapath.ErrCancelled), } } func (r *RestoreMicroService) OnPvrProgress(ctx context.Context, namespace string, pvrName string, progress *uploader.Progress) { log := r.logger.WithFields(logrus.Fields{ "PVR": pvrName, }) progressBytes, err := funcMarshal(progress) if err != nil { log.WithError(err).Errorf("Failed to marshal progress %v", progress) return } r.eventRecorder.Event(r.pvr, false, datapath.EventReasonProgress, string(progressBytes)) } func (r *RestoreMicroService) closeDataPath(ctx context.Context, pvrName string) { fsRestore := r.dataPathMgr.GetAsyncBR(pvrName) if fsRestore != nil { fsRestore.Close(ctx) } r.dataPathMgr.RemoveAsyncBR(pvrName) } func (r *RestoreMicroService) cancelPodVolumeRestore(pvr *velerov1api.PodVolumeRestore) { r.logger.WithField("PVR", pvr.Name).Info("PVR is being canceled") r.eventRecorder.Event(pvr, false, datapath.EventReasonCancelling, "Canceling for PVR %s", pvr.Name) fsBackup := r.dataPathMgr.GetAsyncBR(pvr.Name) if fsBackup == nil { r.OnPvrCancelled(r.ctx, pvr.GetNamespace(), pvr.GetName()) } else { fsBackup.Cancel() } } var funcRemoveAll = os.RemoveAll var funcMkdirAll = os.MkdirAll var funcWriteFile = os.WriteFile func writeCompletionMark(pvr *velerov1api.PodVolumeRestore, result datapath.RestoreResult, log logrus.FieldLogger) error { volumePath := result.Target.ByPath if volumePath == "" { return errors.New("target volume is empty in restore result") } // Remove the .velero directory from the restored volume (it may contain done files from previous restores // of this volume, which we don't want to carry over). If this fails for any reason, log and continue, since // this is non-essential cleanup (the done files are named based on restore UID and the init container looks // for the one specific to the restore being executed). if err := funcRemoveAll(filepath.Join(volumePath, ".velero")); err != nil { log.WithError(err).Warnf("Failed to remove .velero directory from directory %s", volumePath) } if len(pvr.OwnerReferences) == 0 { return errors.New("error finding restore UID") } restoreUID := pvr.OwnerReferences[0].UID // Create the .velero directory within the volume dir so we can write a done file // for this restore. if err := funcMkdirAll(filepath.Join(volumePath, ".velero"), 0755); err != nil { return errors.Wrapf(err, "error creating .velero directory for done file") } // Write a done file with name= into the just-created .velero dir // within the volume. The velero init container on the pod is waiting // for this file to exist in each restored volume before completing. if err := funcWriteFile(filepath.Join(volumePath, ".velero", string(restoreUID)), nil, 0644); err != nil { return errors.Wrapf(err, "error writing done file") } return nil } ================================================ FILE: pkg/podvolume/restore_micro_service_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package podvolume import ( "context" "fmt" "os" "sync" "testing" "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/runtime" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/datapath" "github.com/vmware-tanzu/velero/pkg/uploader" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" clientFake "sigs.k8s.io/controller-runtime/pkg/client/fake" velerotest "github.com/vmware-tanzu/velero/pkg/test" kbclient "sigs.k8s.io/controller-runtime/pkg/client" datapathmockes "github.com/vmware-tanzu/velero/pkg/datapath/mocks" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) type restoreMsTestHelper struct { eventReason string eventMsg string marshalErr error marshalBytes []byte withEvent bool eventLock sync.Mutex writeCompletionErr error } func (rt *restoreMsTestHelper) Event(_ runtime.Object, _ bool, reason string, message string, a ...any) { rt.eventLock.Lock() defer rt.eventLock.Unlock() rt.withEvent = true rt.eventReason = reason rt.eventMsg = fmt.Sprintf(message, a...) } func (rt *restoreMsTestHelper) EndingEvent(_ runtime.Object, _ bool, reason string, message string, a ...any) { rt.eventLock.Lock() defer rt.eventLock.Unlock() rt.withEvent = true rt.eventReason = reason rt.eventMsg = fmt.Sprintf(message, a...) } func (rt *restoreMsTestHelper) Shutdown() {} func (rt *restoreMsTestHelper) Marshal(v any) ([]byte, error) { if rt.marshalErr != nil { return nil, rt.marshalErr } return rt.marshalBytes, nil } func (rt *restoreMsTestHelper) EventReason() string { rt.eventLock.Lock() defer rt.eventLock.Unlock() return rt.eventReason } func (rt *restoreMsTestHelper) EventMessage() string { rt.eventLock.Lock() defer rt.eventLock.Unlock() return rt.eventMsg } func (rt *restoreMsTestHelper) WriteCompletionMark(*velerov1api.PodVolumeRestore, datapath.RestoreResult, logrus.FieldLogger) error { return rt.writeCompletionErr } func TestOnPvrFailed(t *testing.T) { pvrName := "fake-pvr" rt := &restoreMsTestHelper{} rs := &RestoreMicroService{ pvrName: pvrName, dataPathMgr: datapath.NewManager(1), eventRecorder: rt, resultSignal: make(chan dataPathResult), logger: velerotest.NewLogger(), } expectedErr := "Data path for PVR fake-pvr failed: fake-error" expectedEventReason := datapath.EventReasonFailed expectedEventMsg := "Data path for PVR fake-pvr failed, error fake-error" go rs.OnPvrFailed(t.Context(), velerov1api.DefaultNamespace, pvrName, errors.New("fake-error")) result := <-rs.resultSignal require.EqualError(t, result.err, expectedErr) assert.Equal(t, expectedEventReason, rt.EventReason()) assert.Equal(t, expectedEventMsg, rt.EventMessage()) } func TestPvrCancelled(t *testing.T) { pvrName := "fake-pvr" rt := &restoreMsTestHelper{} rs := RestoreMicroService{ pvrName: pvrName, dataPathMgr: datapath.NewManager(1), eventRecorder: rt, resultSignal: make(chan dataPathResult), logger: velerotest.NewLogger(), } expectedErr := datapath.ErrCancelled expectedEventReason := datapath.EventReasonCancelled expectedEventMsg := "Data path for PVR fake-pvr canceled" go rs.OnPvrCancelled(t.Context(), velerov1api.DefaultNamespace, pvrName) result := <-rs.resultSignal require.EqualError(t, result.err, expectedErr) assert.Equal(t, expectedEventReason, rt.EventReason()) assert.Equal(t, expectedEventMsg, rt.EventMessage()) } func TestOnPvrCompleted(t *testing.T) { tests := []struct { name string expectedErr string expectedEventReason string expectedEventMsg string marshalErr error marshallStr string writeCompletionErr error expectedLog string }{ { name: "marshal fail", marshalErr: errors.New("fake-marshal-error"), expectedErr: "error marshaling restore result {{ } 0}: fake-marshal-error", }, { name: "succeed", marshallStr: "fake-complete-string", expectedEventReason: datapath.EventReasonCompleted, expectedEventMsg: "fake-complete-string", }, { name: "succeed but write completion mark fail", marshallStr: "fake-complete-string", writeCompletionErr: errors.New("fake-write-completion-error"), expectedEventReason: datapath.EventReasonCompleted, expectedEventMsg: "fake-complete-string", expectedLog: "Failed to write completion mark, restored pod may failed to start", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { pvrName := "fake-pvr" rt := &restoreMsTestHelper{ marshalErr: test.marshalErr, marshalBytes: []byte(test.marshallStr), writeCompletionErr: test.writeCompletionErr, } logBuffer := []string{} rs := &RestoreMicroService{ dataPathMgr: datapath.NewManager(1), eventRecorder: rt, resultSignal: make(chan dataPathResult), logger: velerotest.NewMultipleLogger(&logBuffer), } funcMarshal = rt.Marshal funcWriteCompletionMark = rt.WriteCompletionMark go rs.OnPvrCompleted(t.Context(), velerov1api.DefaultNamespace, pvrName, datapath.Result{}) result := <-rs.resultSignal if test.marshalErr != nil { assert.EqualError(t, result.err, test.expectedErr) } else { require.NoError(t, result.err) assert.Equal(t, test.expectedEventReason, rt.EventReason()) assert.Equal(t, test.expectedEventMsg, rt.EventMessage()) if test.expectedLog != "" { assert.Contains(t, logBuffer[0], test.expectedLog) } } }) } } func TestOnPvrProgress(t *testing.T) { tests := []struct { name string expectedErr string expectedEventReason string expectedEventMsg string marshalErr error marshallStr string }{ { name: "marshal fail", marshalErr: errors.New("fake-marshal-error"), expectedErr: "Failed to marshal restore result", }, { name: "succeed", marshallStr: "fake-progress-string", expectedEventReason: datapath.EventReasonProgress, expectedEventMsg: "fake-progress-string", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { pvrName := "fake-pvr" rt := &restoreMsTestHelper{ marshalErr: test.marshalErr, marshalBytes: []byte(test.marshallStr), } rs := &RestoreMicroService{ dataPathMgr: datapath.NewManager(1), eventRecorder: rt, logger: velerotest.NewLogger(), } funcMarshal = rt.Marshal rs.OnPvrProgress(t.Context(), velerov1api.DefaultNamespace, pvrName, &uploader.Progress{}) if test.marshalErr != nil { assert.False(t, rt.withEvent) } else { assert.True(t, rt.withEvent) assert.Equal(t, test.expectedEventReason, rt.EventReason()) assert.Equal(t, test.expectedEventMsg, rt.EventMessage()) } }) } } func TestCancelPodVolumeRestore(t *testing.T) { tests := []struct { name string expectedEventReason string expectedEventMsg string expectedErr string }{ { name: "no fs restore", expectedEventReason: datapath.EventReasonCancelled, expectedEventMsg: "Data path for PVR fake-pvr canceled", expectedErr: datapath.ErrCancelled, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { pvrName := "fake-pvr" pvr := builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Result() rt := &restoreMsTestHelper{} rs := &RestoreMicroService{ dataPathMgr: datapath.NewManager(1), eventRecorder: rt, resultSignal: make(chan dataPathResult), logger: velerotest.NewLogger(), } go rs.cancelPodVolumeRestore(pvr) result := <-rs.resultSignal require.EqualError(t, result.err, test.expectedErr) assert.True(t, rt.withEvent) assert.Equal(t, test.expectedEventReason, rt.EventReason()) assert.Equal(t, test.expectedEventMsg, rt.EventMessage()) }) } } func TestRunCancelableDataPathRestore(t *testing.T) { pvrName := "fake-pvr" pvr := builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseNew).Result() pvrInProgress := builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseInProgress).Result() ctxTimeout, cancel := context.WithTimeout(t.Context(), time.Second) tests := []struct { name string ctx context.Context result *dataPathResult dataPathMgr *datapath.Manager kubeClientObj []runtime.Object initErr error startErr error dataPathStarted bool expectedEventMsg string expectedErr string }{ { name: "no pvr", ctx: ctxTimeout, expectedErr: "error waiting for PVR: context deadline exceeded", }, { name: "pvr not in in-progress", ctx: ctxTimeout, kubeClientObj: []runtime.Object{pvr}, expectedErr: "error waiting for PVR: context deadline exceeded", }, { name: "create data path fail", ctx: t.Context(), kubeClientObj: []runtime.Object{pvrInProgress}, dataPathMgr: datapath.NewManager(0), expectedErr: "error to create data path: Concurrent number exceeds", }, { name: "init data path fail", ctx: t.Context(), kubeClientObj: []runtime.Object{pvrInProgress}, initErr: errors.New("fake-init-error"), expectedErr: "error to initialize data path: fake-init-error", }, { name: "start data path fail", ctx: t.Context(), kubeClientObj: []runtime.Object{pvrInProgress}, startErr: errors.New("fake-start-error"), expectedErr: "error starting data path restore: fake-start-error", }, { name: "data path timeout", ctx: ctxTimeout, kubeClientObj: []runtime.Object{pvrInProgress}, dataPathStarted: true, expectedEventMsg: fmt.Sprintf("Data path for %s stopped", pvrName), expectedErr: "timed out waiting for fs restore to complete", }, { name: "data path returns error", ctx: t.Context(), kubeClientObj: []runtime.Object{pvrInProgress}, dataPathStarted: true, result: &dataPathResult{ err: errors.New("fake-data-path-error"), }, expectedEventMsg: fmt.Sprintf("Data path for %s stopped", pvrName), expectedErr: "fake-data-path-error", }, { name: "succeed", ctx: t.Context(), kubeClientObj: []runtime.Object{pvrInProgress}, dataPathStarted: true, result: &dataPathResult{ result: "fake-succeed-result", }, expectedEventMsg: fmt.Sprintf("Data path for %s stopped", pvrName), }, } scheme := runtime.NewScheme() velerov1api.AddToScheme(scheme) for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeClientBuilder := clientFake.NewClientBuilder() fakeClientBuilder = fakeClientBuilder.WithScheme(scheme) fakeClient := fakeClientBuilder.WithRuntimeObjects(test.kubeClientObj...).Build() rt := &restoreMsTestHelper{} rs := &RestoreMicroService{ namespace: velerov1api.DefaultNamespace, pvrName: pvrName, ctx: t.Context(), client: fakeClient, dataPathMgr: datapath.NewManager(1), eventRecorder: rt, resultSignal: make(chan dataPathResult), logger: velerotest.NewLogger(), } if test.ctx != nil { rs.ctx = test.ctx } if test.dataPathMgr != nil { rs.dataPathMgr = test.dataPathMgr } datapath.FSBRCreator = func(string, string, kbclient.Client, string, datapath.Callbacks, logrus.FieldLogger) datapath.AsyncBR { fsBR := datapathmockes.NewAsyncBR(t) if test.initErr != nil { fsBR.On("Init", mock.Anything, mock.Anything).Return(test.initErr) } if test.startErr != nil { fsBR.On("Init", mock.Anything, mock.Anything).Return(nil) fsBR.On("StartRestore", mock.Anything, mock.Anything, mock.Anything).Return(test.startErr) } if test.dataPathStarted { fsBR.On("Init", mock.Anything, mock.Anything).Return(nil) fsBR.On("StartRestore", mock.Anything, mock.Anything, mock.Anything).Return(nil) } return fsBR } if test.result != nil { go func() { time.Sleep(time.Millisecond * 500) rs.resultSignal <- *test.result }() } result, err := rs.RunCancelableDataPath(test.ctx) if test.expectedErr != "" { require.EqualError(t, err, test.expectedErr) } else { require.NoError(t, err) assert.Equal(t, test.result.result, result) } if test.expectedEventMsg != "" { assert.True(t, rt.withEvent) assert.Equal(t, test.expectedEventMsg, rt.EventMessage()) } }) } cancel() } func TestWriteCompletionMark(t *testing.T) { tests := []struct { name string pvr *velerov1api.PodVolumeRestore result datapath.RestoreResult funcRemoveAll func(string) error funcMkdirAll func(string, os.FileMode) error funcWriteFile func(string, []byte, os.FileMode) error expectedErr string expectedLog string }{ { name: "no volume path", result: datapath.RestoreResult{}, expectedErr: "target volume is empty in restore result", }, { name: "no owner reference", result: datapath.RestoreResult{ Target: datapath.AccessPoint{ ByPath: "fake-volume-path", }, }, pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, "fake-pvr").Result(), funcRemoveAll: func(string) error { return nil }, expectedErr: "error finding restore UID", }, { name: "mkdir fail", result: datapath.RestoreResult{ Target: datapath.AccessPoint{ ByPath: "fake-volume-path", }, }, pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, "fake-pvr").OwnerReference([]metav1.OwnerReference{ { UID: "fake-uid", }, }).Result(), funcRemoveAll: func(string) error { return nil }, funcMkdirAll: func(string, os.FileMode) error { return errors.New("fake-mk-dir-error") }, expectedErr: "error creating .velero directory for done file: fake-mk-dir-error", }, { name: "write file fail", result: datapath.RestoreResult{ Target: datapath.AccessPoint{ ByPath: "fake-volume-path", }, }, pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, "fake-pvr").OwnerReference([]metav1.OwnerReference{ { UID: "fake-uid", }, }).Result(), funcRemoveAll: func(string) error { return nil }, funcMkdirAll: func(string, os.FileMode) error { return nil }, funcWriteFile: func(string, []byte, os.FileMode) error { return errors.New("fake-write-file-error") }, expectedErr: "error writing done file: fake-write-file-error", }, { name: "succeed", result: datapath.RestoreResult{ Target: datapath.AccessPoint{ ByPath: "fake-volume-path", }, }, pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, "fake-pvr").OwnerReference([]metav1.OwnerReference{ { UID: "fake-uid", }, }).Result(), funcRemoveAll: func(string) error { return nil }, funcMkdirAll: func(string, os.FileMode) error { return nil }, funcWriteFile: func(string, []byte, os.FileMode) error { return nil }, }, { name: "succeed but previous dir is not removed", result: datapath.RestoreResult{ Target: datapath.AccessPoint{ ByPath: "fake-volume-path", }, }, pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, "fake-pvr").OwnerReference([]metav1.OwnerReference{ { UID: "fake-uid", }, }).Result(), funcRemoveAll: func(string) error { return errors.New("fake-remove-dir-error") }, funcMkdirAll: func(string, os.FileMode) error { return nil }, funcWriteFile: func(string, []byte, os.FileMode) error { return nil }, expectedLog: "Failed to remove .velero directory from directory fake-volume-path", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { if test.funcRemoveAll != nil { funcRemoveAll = test.funcRemoveAll } if test.funcMkdirAll != nil { funcMkdirAll = test.funcMkdirAll } if test.funcWriteFile != nil { funcWriteFile = test.funcWriteFile } logBuffer := "" err := writeCompletionMark(test.pvr, test.result, velerotest.NewSingleLogger(&logBuffer)) if test.expectedErr == "" { require.NoError(t, err) } else { require.EqualError(t, err, test.expectedErr) } if test.expectedLog != "" { assert.Contains(t, logBuffer, test.expectedLog) } }) } } ================================================ FILE: pkg/podvolume/restorer.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package podvolume import ( "context" "sync" "time" "github.com/vmware-tanzu/velero/internal/volume" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" ctrlcache "sigs.k8s.io/controller-runtime/pkg/cache" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" veleroclient "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/label" "github.com/vmware-tanzu/velero/pkg/nodeagent" "github.com/vmware-tanzu/velero/pkg/repository" uploaderutil "github.com/vmware-tanzu/velero/pkg/uploader/util" "github.com/vmware-tanzu/velero/pkg/util/boolptr" "github.com/vmware-tanzu/velero/pkg/util/kube" ) type RestoreData struct { Restore *velerov1api.Restore Pod *corev1api.Pod PodVolumeBackups []*velerov1api.PodVolumeBackup SourceNamespace, BackupLocation string } // Restorer can execute pod volume restores of volumes in a pod. type Restorer interface { // RestorePodVolumes restores all annotated volumes in a pod. RestorePodVolumes(RestoreData, *volume.RestoreVolumeInfoTracker) []error } type restorer struct { ctx context.Context repoLocker *repository.RepoLocker repoEnsurer *repository.Ensurer kubeClient kubernetes.Interface crClient ctrlclient.Client resultsLock sync.Mutex results map[string]chan *velerov1api.PodVolumeRestore nodeAgentCheck chan error log logrus.FieldLogger } func newRestorer( ctx context.Context, repoLocker *repository.RepoLocker, repoEnsurer *repository.Ensurer, pvrInformer ctrlcache.Informer, kubeClient kubernetes.Interface, crClient ctrlclient.Client, restore *velerov1api.Restore, log logrus.FieldLogger, ) *restorer { r := &restorer{ ctx: ctx, repoLocker: repoLocker, repoEnsurer: repoEnsurer, kubeClient: kubeClient, crClient: crClient, results: make(map[string]chan *velerov1api.PodVolumeRestore), log: log, } _, _ = pvrInformer.AddEventHandler( cache.ResourceEventHandlerFuncs{ UpdateFunc: func(oldObj, newObj any) { pvr := newObj.(*velerov1api.PodVolumeRestore) pvrOld := oldObj.(*velerov1api.PodVolumeRestore) if pvr.GetLabels()[velerov1api.RestoreUIDLabel] != string(restore.UID) { return } if pvr.Status.Phase == pvrOld.Status.Phase { return } if pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseCompleted || pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseFailed || pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseCanceled { r.resultsLock.Lock() defer r.resultsLock.Unlock() resChan, ok := r.results[resultsKey(pvr.Spec.Pod.Namespace, pvr.Spec.Pod.Name)] if !ok { log.Errorf("No results channel found for pod %s/%s to send pod volume restore %s/%s on", pvr.Spec.Pod.Namespace, pvr.Spec.Pod.Name, pvr.Namespace, pvr.Name) return } resChan <- pvr } }, }, ) return r } func (r *restorer) RestorePodVolumes(data RestoreData, tracker *volume.RestoreVolumeInfoTracker) []error { volumesToRestore := getVolumeBackupInfoForPod(data.PodVolumeBackups, data.Pod, data.SourceNamespace) if len(volumesToRestore) == 0 { return nil } if err := nodeagent.IsRunningOnLinux(r.ctx, r.kubeClient, data.Restore.Namespace); err != nil { return []error{errors.Wrapf(err, "error to check node agent status")} } repositoryType, err := getVolumesRepositoryType(volumesToRestore) if err != nil { return []error{err} } repo, err := r.repoEnsurer.EnsureRepo(r.ctx, data.Restore.Namespace, data.SourceNamespace, data.BackupLocation, repositoryType) if err != nil { return []error{err} } // get a single non-exclusive lock since we'll wait for all individual // restores to be complete before releasing it. r.repoLocker.Lock(repo.Name) defer r.repoLocker.Unlock(repo.Name) resultsChan := make(chan *velerov1api.PodVolumeRestore) r.resultsLock.Lock() r.results[resultsKey(data.Pod.Namespace, data.Pod.Name)] = resultsChan r.resultsLock.Unlock() r.nodeAgentCheck = make(chan error) var ( errs []error numRestores int podVolumes = make(map[string]corev1api.Volume) ) // put the pod's volumes in a map for efficient lookup below for _, podVolume := range data.Pod.Spec.Volumes { podVolumes[podVolume.Name] = podVolume } repoIdentifier := "" if repositoryType == velerov1api.BackupRepositoryTypeRestic { repoIdentifier = repo.Spec.ResticIdentifier } for volume, backupInfo := range volumesToRestore { volumeObj, ok := podVolumes[volume] var pvc *corev1api.PersistentVolumeClaim if ok { if volumeObj.PersistentVolumeClaim != nil { pvc = new(corev1api.PersistentVolumeClaim) err := r.crClient.Get(context.TODO(), ctrlclient.ObjectKey{Namespace: data.Pod.Namespace, Name: volumeObj.PersistentVolumeClaim.ClaimName}, pvc) if err != nil { errs = append(errs, errors.Wrap(err, "error getting persistent volume claim for volume")) continue } } } volumeRestore := newPodVolumeRestore(data.Restore, data.Pod, data.BackupLocation, volume, backupInfo.snapshotID, backupInfo.snapshotSize, repoIdentifier, backupInfo.uploaderType, data.SourceNamespace, pvc) if err := veleroclient.CreateRetryGenerateName(r.crClient, r.ctx, volumeRestore); err != nil { errs = append(errs, errors.WithStack(err)) continue } numRestores++ } checkCtx, checkCancel := context.WithCancel(context.Background()) go func() { nodeName := "" checkFunc := func(ctx context.Context) (bool, error) { newObj, err := r.kubeClient.CoreV1().Pods(data.Pod.Namespace).Get(ctx, data.Pod.Name, metav1.GetOptions{}) if err != nil { return false, err } nodeName = newObj.Spec.NodeName err = kube.IsPodScheduled(newObj) if err != nil { r.log.WithField("error", err).Debugf("Pod %s/%s is not scheduled yet", newObj.GetNamespace(), newObj.GetName()) return false, nil } return true, nil } err := wait.PollUntilContextTimeout(checkCtx, time.Millisecond*500, time.Minute*10, true, checkFunc) if wait.Interrupted(err) { r.log.WithError(err).Error("Restoring pod is not scheduled until timeout or cancel, disengage") } else if err != nil { r.log.WithError(err).Error("Failed to check node-agent pod status, disengage") } else { err = nodeagent.IsRunningInNode(checkCtx, data.Restore.Namespace, nodeName, r.crClient) if err != nil { r.log.WithField("node", nodeName).WithError(err).Error("node-agent pod is not running in node, abort the restore") r.nodeAgentCheck <- errors.Wrapf(err, "node-agent pod is not running in node %s", nodeName) } } }() ForEachVolume: for i := 0; i < numRestores; i++ { select { case <-r.ctx.Done(): errs = append(errs, errors.New("timed out waiting for all PodVolumeRestores to complete")) break ForEachVolume case res := <-resultsChan: if res.Status.Phase == velerov1api.PodVolumeRestorePhaseFailed { errs = append(errs, errors.Errorf("pod volume restore failed: %s", res.Status.Message)) } else if res.Status.Phase == velerov1api.PodVolumeRestorePhaseCanceled { errs = append(errs, errors.Errorf("pod volume restore canceled: %s", res.Status.Message)) } tracker.TrackPodVolume(res) case err := <-r.nodeAgentCheck: errs = append(errs, err) break ForEachVolume } } // This is to prevent the case that resultsChan is signaled before nodeAgentCheck though this is unlikely possible. // One possible case is that the CR is edited and set to an ending state manually, either completed or failed. // In this case, we must notify the check routine to stop. checkCancel() r.resultsLock.Lock() delete(r.results, resultsKey(data.Pod.Namespace, data.Pod.Name)) r.resultsLock.Unlock() return errs } func newPodVolumeRestore(restore *velerov1api.Restore, pod *corev1api.Pod, backupLocation, volume, snapshot string, size int64, repoIdentifier, uploaderType, sourceNamespace string, pvc *corev1api.PersistentVolumeClaim) *velerov1api.PodVolumeRestore { pvr := &velerov1api.PodVolumeRestore{ ObjectMeta: metav1.ObjectMeta{ Namespace: restore.Namespace, GenerateName: restore.Name + "-", OwnerReferences: []metav1.OwnerReference{ { APIVersion: velerov1api.SchemeGroupVersion.String(), Kind: "Restore", Name: restore.Name, UID: restore.UID, Controller: boolptr.True(), }, }, Labels: map[string]string{ velerov1api.RestoreNameLabel: label.GetValidName(restore.Name), velerov1api.RestoreUIDLabel: string(restore.UID), velerov1api.PodUIDLabel: string(pod.UID), }, }, Spec: velerov1api.PodVolumeRestoreSpec{ Pod: corev1api.ObjectReference{ Kind: "Pod", Namespace: pod.Namespace, Name: pod.Name, UID: pod.UID, }, Volume: volume, SnapshotID: snapshot, SnapshotSize: size, BackupStorageLocation: backupLocation, RepoIdentifier: repoIdentifier, UploaderType: uploaderType, SourceNamespace: sourceNamespace, }, } if pvc != nil { // this label is not used by velero, but useful for debugging. pvr.Labels[velerov1api.PVCUIDLabel] = string(pvc.UID) } if restore.Spec.UploaderConfig != nil { pvr.Spec.UploaderSettings = uploaderutil.StoreRestoreConfig(restore.Spec.UploaderConfig) } return pvr } func getVolumesRepositoryType(volumes map[string]volumeBackupInfo) (string, error) { if len(volumes) == 0 { return "", errors.New("empty volume list") } // the podVolumeBackups list come from one backup. In one backup, it is impossible that volumes are // backed up by different uploaders or to different repositories. Asserting this ensures one repo only, // which will simplify the following logics repositoryType := "" for _, backupInfo := range volumes { if backupInfo.repositoryType == "" { return "", errors.Errorf("empty repository type found among volume snapshots, snapshot ID %s, uploader %s", backupInfo.snapshotID, backupInfo.uploaderType) } if repositoryType == "" { repositoryType = backupInfo.repositoryType } else if repositoryType != backupInfo.repositoryType { return "", errors.Errorf("multiple repository type in one backup, current type %s, differential one [type %s, snapshot ID %s, uploader %s]", repositoryType, backupInfo.repositoryType, backupInfo.snapshotID, backupInfo.uploaderType) } } return repositoryType, nil } ================================================ FILE: pkg/podvolume/restorer_factory.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package podvolume import ( "context" "github.com/pkg/errors" "github.com/sirupsen/logrus" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" ctrlcache "sigs.k8s.io/controller-runtime/pkg/cache" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/repository" ) // RestorerFactory can construct pod volumes restorers. type RestorerFactory interface { // NewRestorer returns a pod volumes restorer for use during a single Velero restore. NewRestorer(context.Context, *velerov1api.Restore) (Restorer, error) } func NewRestorerFactory(repoLocker *repository.RepoLocker, repoEnsurer *repository.Ensurer, kubeClient kubernetes.Interface, crClient ctrlclient.Client, pvrInformer ctrlcache.Informer, log logrus.FieldLogger) RestorerFactory { return &restorerFactory{ repoLocker: repoLocker, repoEnsurer: repoEnsurer, kubeClient: kubeClient, crClient: crClient, pvrInformer: pvrInformer, log: log, } } type restorerFactory struct { repoLocker *repository.RepoLocker repoEnsurer *repository.Ensurer kubeClient kubernetes.Interface crClient ctrlclient.Client pvrInformer ctrlcache.Informer log logrus.FieldLogger } func (rf *restorerFactory) NewRestorer(ctx context.Context, restore *velerov1api.Restore) (Restorer, error) { r := newRestorer(ctx, rf.repoLocker, rf.repoEnsurer, rf.pvrInformer, rf.kubeClient, rf.crClient, restore, rf.log) if !cache.WaitForCacheSync(ctx.Done(), rf.pvrInformer.HasSynced) { return nil, errors.New("timed out waiting for cache to sync") } return r, nil } ================================================ FILE: pkg/podvolume/restorer_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package podvolume import ( "context" "fmt" "testing" "time" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" appsv1api "k8s.io/api/apps/v1" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes" kubefake "k8s.io/client-go/kubernetes/fake" "k8s.io/client-go/tools/cache" "github.com/vmware-tanzu/velero/internal/volume" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/repository" velerotest "github.com/vmware-tanzu/velero/pkg/test" "github.com/vmware-tanzu/velero/pkg/util/kube" ) func TestGetVolumesRepositoryType(t *testing.T) { testCases := []struct { name string volumes map[string]volumeBackupInfo expected string expectedErr string prefixOnly bool }{ { name: "empty volume", expectedErr: "empty volume list", }, { name: "empty repository type, first one", volumes: map[string]volumeBackupInfo{ "volume1": {"fake-snapshot-id-1", 0, "fake-uploader-1", ""}, "volume2": {"", 0, "", "fake-type"}, }, expectedErr: "empty repository type found among volume snapshots, snapshot ID fake-snapshot-id-1, uploader fake-uploader-1", }, { name: "empty repository type, last one", volumes: map[string]volumeBackupInfo{ "volume1": {"", 0, "", "fake-type"}, "volume2": {"", 0, "", "fake-type"}, "volume3": {"fake-snapshot-id-3", 0, "fake-uploader-3", ""}, }, expectedErr: "empty repository type found among volume snapshots, snapshot ID fake-snapshot-id-3, uploader fake-uploader-3", }, { name: "empty repository type, middle one", volumes: map[string]volumeBackupInfo{ "volume1": {"", 0, "", "fake-type"}, "volume2": {"fake-snapshot-id-2", 0, "fake-uploader-2", ""}, "volume3": {"", 0, "", "fake-type"}, }, expectedErr: "empty repository type found among volume snapshots, snapshot ID fake-snapshot-id-2, uploader fake-uploader-2", }, { name: "mismatch repository type", volumes: map[string]volumeBackupInfo{ "volume1": {"", 0, "", "fake-type1"}, "volume2": {"fake-snapshot-id-2", 0, "fake-uploader-2", "fake-type2"}, }, prefixOnly: true, expectedErr: "multiple repository type in one backup", }, { name: "success", volumes: map[string]volumeBackupInfo{ "volume1": {"", 0, "", "fake-type"}, "volume2": {"", 0, "", "fake-type"}, "volume3": {"", 0, "", "fake-type"}, }, expected: "fake-type", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { actual, err := getVolumesRepositoryType(tc.volumes) assert.Equal(t, tc.expected, actual) if err != nil { if tc.prefixOnly { errMsg := err.Error() if len(errMsg) >= len(tc.expectedErr) { errMsg = errMsg[0:len(tc.expectedErr)] } assert.Equal(t, tc.expectedErr, errMsg) } else { assert.EqualError(t, err, tc.expectedErr) } } }) } } func createNodeAgentDaemonset() *appsv1api.DaemonSet { ds := &appsv1api.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Name: "node-agent", Namespace: velerov1api.DefaultNamespace, }, } return ds } func createPVRObj(fail bool, index int) *velerov1api.PodVolumeRestore { pvrObj := &velerov1api.PodVolumeRestore{ TypeMeta: metav1.TypeMeta{ APIVersion: velerov1api.SchemeGroupVersion.String(), Kind: "PodVolumeRestore", }, ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-ns", Name: fmt.Sprintf("fake-pvr-%d", index), }, } if fail { pvrObj.Status.Phase = velerov1api.PodVolumeRestorePhaseFailed pvrObj.Status.Message = "fake-message" } else { pvrObj.Status.Phase = velerov1api.PodVolumeRestorePhaseCompleted } return pvrObj } type expectError struct { err string prefixOnly bool } func TestRestorePodVolumes(t *testing.T) { scheme := runtime.NewScheme() velerov1api.AddToScheme(scheme) corev1api.AddToScheme(scheme) ctxWithCancel, cancel := context.WithCancel(t.Context()) defer cancel() failedPVR := createPVRObj(true, 1) completedPVR := createPVRObj(false, 1) tests := []struct { name string ctx context.Context bsl string kubeClientObj []runtime.Object ctlClientObj []runtime.Object veleroClientObj []runtime.Object veleroReactors []reactor runtimeScheme *runtime.Scheme retPVRs []*velerov1api.PodVolumeRestore pvbs []*velerov1api.PodVolumeBackup restoredPod *corev1api.Pod sourceNamespace string errs []expectError }{ { name: "no volume to restore", pvbs: []*velerov1api.PodVolumeBackup{}, restoredPod: createPodObj(false, false, false, 1), }, { name: "node-agent is not running", pvbs: []*velerov1api.PodVolumeBackup{ createPVBObj(true, true, 1, "kopia"), createPVBObj(true, true, 2, "kopia"), }, restoredPod: createPodObj(false, false, false, 2), sourceNamespace: "fake-ns", errs: []expectError{ { err: "error to check node agent status: daemonset not found", }, }, }, { name: "get repository type fail", pvbs: []*velerov1api.PodVolumeBackup{ createPVBObj(true, true, 1, "restic"), createPVBObj(true, true, 2, "kopia"), }, kubeClientObj: []runtime.Object{ createNodeAgentDaemonset(), }, restoredPod: createPodObj(false, false, false, 2), sourceNamespace: "fake-ns", errs: []expectError{ { err: "multiple repository type in one backup", prefixOnly: true, }, }, }, { name: "ensure repo fail", pvbs: []*velerov1api.PodVolumeBackup{ createPVBObj(true, true, 1, "kopia"), createPVBObj(true, true, 2, "kopia"), }, kubeClientObj: []runtime.Object{ createNodeAgentDaemonset(), }, restoredPod: createPodObj(false, false, false, 2), sourceNamespace: "fake-ns", runtimeScheme: scheme, errs: []expectError{ { err: "wrong parameters, namespace \"fake-ns\", backup storage location \"\", repository type \"kopia\"", }, }, }, { name: "get pvc fail", pvbs: []*velerov1api.PodVolumeBackup{ createPVBObj(true, true, 1, "kopia"), createPVBObj(true, true, 2, "kopia"), }, kubeClientObj: []runtime.Object{ createNodeAgentDaemonset(), }, ctlClientObj: []runtime.Object{ createBackupRepoObj(), }, restoredPod: createPodObj(true, true, true, 2), sourceNamespace: "fake-ns", bsl: "fake-bsl", runtimeScheme: scheme, errs: []expectError{ { err: "error getting persistent volume claim for volume: persistentvolumeclaims \"fake-pvc-1\" not found", }, { err: "error getting persistent volume claim for volume: persistentvolumeclaims \"fake-pvc-2\" not found", }, }, }, { name: "create pvb fail", ctx: ctxWithCancel, pvbs: []*velerov1api.PodVolumeBackup{ createPVBObj(true, true, 1, "kopia"), }, kubeClientObj: []runtime.Object{ createNodeAgentDaemonset(), createPVCObj(1), }, ctlClientObj: []runtime.Object{ createBackupRepoObj(), }, restoredPod: createPodObj(true, true, true, 1), sourceNamespace: "fake-ns", bsl: "fake-bsl", runtimeScheme: scheme, errs: []expectError{ { err: "timed out waiting for all PodVolumeRestores to complete", }, }, }, { name: "create pvb fail", pvbs: []*velerov1api.PodVolumeBackup{ createPVBObj(true, true, 1, "kopia"), }, kubeClientObj: []runtime.Object{ createNodeAgentDaemonset(), createPVCObj(1), }, ctlClientObj: []runtime.Object{ createBackupRepoObj(), }, restoredPod: createPodObj(true, true, true, 1), sourceNamespace: "fake-ns", bsl: "fake-bsl", runtimeScheme: scheme, retPVRs: []*velerov1api.PodVolumeRestore{ failedPVR, }, errs: []expectError{ { err: "pod volume restore failed: fake-message", }, }, }, { name: "node-agent pod is not running", pvbs: []*velerov1api.PodVolumeBackup{ createPVBObj(true, true, 1, "kopia"), }, kubeClientObj: []runtime.Object{ createNodeAgentDaemonset(), createNodeObj(), createPVCObj(1), createPodObj(true, true, true, 1), }, ctlClientObj: []runtime.Object{ createBackupRepoObj(), }, restoredPod: createPodObj(true, true, true, 1), sourceNamespace: "fake-ns", bsl: "fake-bsl", runtimeScheme: scheme, errs: []expectError{ { err: "node-agent pod is not running in node fake-node-name: daemonset pod not found in running state in node fake-node-name", }, }, }, { name: "complete", pvbs: []*velerov1api.PodVolumeBackup{ createPVBObj(true, true, 1, "kopia"), }, kubeClientObj: []runtime.Object{ createNodeAgentDaemonset(), createNodeObj(), createPVCObj(1), createPodObj(true, true, true, 1), createNodeAgentPodObj(true), }, ctlClientObj: []runtime.Object{ createBackupRepoObj(), }, restoredPod: createPodObj(true, true, true, 1), sourceNamespace: "fake-ns", bsl: "fake-bsl", runtimeScheme: scheme, retPVRs: []*velerov1api.PodVolumeRestore{ completedPVR, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { ctx := t.Context() if test.ctx != nil { ctx = test.ctx } objClient := append(test.ctlClientObj, test.kubeClientObj...) objClient = append(objClient, test.veleroClientObj...) fakeCRClient := velerotest.NewFakeControllerRuntimeClient(t, objClient...) fakeKubeClient := kubefake.NewSimpleClientset(test.kubeClientObj...) var kubeClient kubernetes.Interface = fakeKubeClient fakeCRWatchClient := velerotest.NewFakeControllerRuntimeWatchClient(t, test.kubeClientObj...) lw := kube.InternalLW{ Client: fakeCRWatchClient, Namespace: velerov1api.DefaultNamespace, ObjectList: new(velerov1api.PodVolumeRestoreList), } pvrInformer := cache.NewSharedIndexInformer(&lw, &velerov1api.PodVolumeBackup{}, 0, cache.Indexers{}) go pvrInformer.Run(ctx.Done()) require.True(t, cache.WaitForCacheSync(ctx.Done(), pvrInformer.HasSynced)) ensurer := repository.NewEnsurer(fakeCRClient, velerotest.NewLogger(), time.Millisecond) restoreObj := builder.ForRestore(velerov1api.DefaultNamespace, "fake-restore").Result() factory := NewRestorerFactory(repository.NewRepoLocker(), ensurer, kubeClient, fakeCRClient, pvrInformer, velerotest.NewLogger()) rs, err := factory.NewRestorer(ctx, restoreObj) require.NoError(t, err) go func() { if test.ctx != nil { time.Sleep(time.Second) cancel() } else if test.retPVRs != nil { time.Sleep(time.Second) for _, pvr := range test.retPVRs { rs.(*restorer).results[resultsKey(test.restoredPod.Namespace, test.restoredPod.Name)] <- pvr } } }() errs := rs.RestorePodVolumes(RestoreData{ Restore: restoreObj, Pod: test.restoredPod, PodVolumeBackups: test.pvbs, SourceNamespace: test.sourceNamespace, BackupLocation: test.bsl, }, volume.NewRestoreVolInfoTracker(restoreObj, logrus.New(), fakeCRClient)) if errs == nil { assert.Nil(t, test.errs) } else { for i := 0; i < len(errs); i++ { if test.errs[i].prefixOnly { errMsg := errs[i].Error() if len(errMsg) >= len(test.errs[i].err) { errMsg = errMsg[0:len(test.errs[i].err)] } assert.Equal(t, test.errs[i].err, errMsg) } else { for i := 0; i < len(errs); i++ { j := 0 for ; j < len(test.errs); j++ { err := errs[i].Error() if err == test.errs[j].err { break } } assert.Less(t, j, len(test.errs)) } } } } }) } } ================================================ FILE: pkg/podvolume/snaphost_tracker_test.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package podvolume import ( "testing" "github.com/stretchr/testify/assert" "github.com/vmware-tanzu/velero/pkg/builder" ) func TestOptoutVolume(t *testing.T) { pod := builder.ForPod("ns-1", "pod-1").Volumes( builder.ForVolume("pod-vol-1").PersistentVolumeClaimSource("pvc-1").Result(), builder.ForVolume("pod-vol-2").PersistentVolumeClaimSource("pvc-2").Result(), ).Result() tracker := NewTracker() tracker.Optout(pod, "pod-vol-1") ok, pn := tracker.OptedoutByPod("ns-1", "pvc-1") assert.True(t, ok) assert.Equal(t, "pod-1", pn) // if a volume is tracked for opted out, it can't be tracked as "tracked" or "taken" tracker.Track(pod, "pod-vol-1") tracker.Track(pod, "pod-vol-2") assert.False(t, tracker.Has("ns-1", "pvc-1")) assert.True(t, tracker.Has("ns-1", "pvc-2")) tracker.Take(pod, "pod-vol-1") tracker.Take(pod, "pod-vol-2") ok1, _ := tracker.TakenForPodVolume(pod, "pod-vol-1") assert.False(t, ok1) ok2, _ := tracker.TakenForPodVolume(pod, "pod-vol-2") assert.True(t, ok2) } func TestABC(t *testing.T) { tracker := NewTracker() v1, v2 := tracker.OptedoutByPod("a", "b") t.Logf("v1: %v, v2: %v", v1, v2) } ================================================ FILE: pkg/podvolume/snapshot_tracker.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package podvolume import ( "fmt" "sync" corev1api "k8s.io/api/core/v1" ) // Tracker keeps track of persistent volume claims that have been handled // via pod volume backup. type Tracker struct { pvcs map[string]pvcSnapshotStatus pvcPod map[string]string *sync.RWMutex } type pvcSnapshotStatus int const ( pvcSnapshotStatusNotTracked pvcSnapshotStatus = -1 pvcSnapshotStatusTracked pvcSnapshotStatus = iota pvcSnapshotStatusTaken pvcSnapshotStatusOptedout ) func NewTracker() *Tracker { return &Tracker{ pvcs: make(map[string]pvcSnapshotStatus), // key: pvc ns/name, value: pod name pvcPod: make(map[string]string), RWMutex: &sync.RWMutex{}, } } // Track indicates a volume from a pod should be snapshotted by pod volume backup. func (t *Tracker) Track(pod *corev1api.Pod, volumeName string) { t.recordStatus(pod, volumeName, pvcSnapshotStatusTracked, pvcSnapshotStatusNotTracked) } // Take indicates a volume from a pod has been taken by pod volume backup. func (t *Tracker) Take(pod *corev1api.Pod, volumeName string) { t.recordStatus(pod, volumeName, pvcSnapshotStatusTaken, pvcSnapshotStatusTracked) } // Optout indicates a volume from a pod has been opted out by pod's annotation func (t *Tracker) Optout(pod *corev1api.Pod, volumeName string) { t.recordStatus(pod, volumeName, pvcSnapshotStatusOptedout, pvcSnapshotStatusNotTracked) } // OptedoutByPod returns true if the PVC with the specified namespace and name has been opted out by the pod. The // second return value is the name of the pod which has the annotation that opted out the volume/pvc func (t *Tracker) OptedoutByPod(namespace, name string) (bool, string) { t.RLock() defer t.RUnlock() status, found := t.pvcs[key(namespace, name)] if !found || status != pvcSnapshotStatusOptedout { return false, "" } return true, t.pvcPod[key(namespace, name)] } // if the volume is a PVC, record the status and the related pod func (t *Tracker) recordStatus(pod *corev1api.Pod, volumeName string, status pvcSnapshotStatus, preReqStatus pvcSnapshotStatus) { t.Lock() defer t.Unlock() for _, volume := range pod.Spec.Volumes { if volume.Name == volumeName { if volume.PersistentVolumeClaim != nil { t.pvcPod[key(pod.Namespace, volume.PersistentVolumeClaim.ClaimName)] = pod.Name currStatus, ok := t.pvcs[key(pod.Namespace, volume.PersistentVolumeClaim.ClaimName)] if !ok { currStatus = pvcSnapshotStatusNotTracked } if currStatus == preReqStatus { t.pvcs[key(pod.Namespace, volume.PersistentVolumeClaim.ClaimName)] = status } } break } } } // Has returns true if the PVC with the specified namespace and name has been tracked. func (t *Tracker) Has(namespace, name string) bool { t.RLock() defer t.RUnlock() status, found := t.pvcs[key(namespace, name)] return found && (status == pvcSnapshotStatusTracked || status == pvcSnapshotStatusTaken) } // TakenForPodVolume returns true and the PVC's name if the pod volume with the specified name uses a // PVC and that PVC has been taken by pod volume backup. func (t *Tracker) TakenForPodVolume(pod *corev1api.Pod, volume string) (bool, string) { t.RLock() defer t.RUnlock() for _, podVolume := range pod.Spec.Volumes { if podVolume.Name != volume { continue } if podVolume.PersistentVolumeClaim == nil { return false, "" } status, found := t.pvcs[key(pod.Namespace, podVolume.PersistentVolumeClaim.ClaimName)] if !found { return false, "" } if status != pvcSnapshotStatusTaken { return false, "" } return true, podVolume.PersistentVolumeClaim.ClaimName } return false, "" } func key(namespace, name string) string { return fmt.Sprintf("%s/%s", namespace, name) } ================================================ FILE: pkg/podvolume/util.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package podvolume import ( "fmt" "strings" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/podvolume/configs" repotypes "github.com/vmware-tanzu/velero/pkg/repository/types" "github.com/vmware-tanzu/velero/pkg/uploader" ) const ( // Deprecated. // // TODO(2.0): remove podAnnotationPrefix = "snapshot.velero.io/" ) // volumeBackupInfo describes the backup info of a volume backed up by PodVolumeBackups type volumeBackupInfo struct { snapshotID string snapshotSize int64 uploaderType string repositoryType string } // GetVolumeBackupsForPod returns a map, of volume name -> snapshot id, // of the PodVolumeBackups that exist for the provided pod. func GetVolumeBackupsForPod(podVolumeBackups []*velerov1api.PodVolumeBackup, pod *corev1api.Pod, sourcePodNs string) map[string]string { volumeBkInfo := getVolumeBackupInfoForPod(podVolumeBackups, pod, sourcePodNs) if volumeBkInfo == nil { return nil } volumes := make(map[string]string) for k, v := range volumeBkInfo { volumes[k] = v.snapshotID } return volumes } // GetPvbRepositoryType returns the repositoryType according to the PVB information func GetPvbRepositoryType(pvb *velerov1api.PodVolumeBackup) string { return getRepositoryType(pvb.Spec.UploaderType) } // GetPvrRepositoryType returns the repositoryType according to the PVR information func GetPvrRepositoryType(pvr *velerov1api.PodVolumeRestore) string { return getRepositoryType(pvr.Spec.UploaderType) } // getVolumeBackupInfoForPod returns a map, of volume name -> VolumeBackupInfo, // of the PodVolumeBackups that exist for the provided pod. func getVolumeBackupInfoForPod(podVolumeBackups []*velerov1api.PodVolumeBackup, pod *corev1api.Pod, sourcePodNs string) map[string]volumeBackupInfo { volumes := make(map[string]volumeBackupInfo) for _, pvb := range podVolumeBackups { if !isPVBMatchPod(pvb, pod.GetName(), sourcePodNs) { continue } // skip PVBs without a snapshot ID since there's nothing // to restore (they could be failed, or for empty volumes). if pvb.Status.SnapshotID == "" { continue } // If the volume came from a projected or DownwardAPI source, skip its restore. // This allows backups affected by https://github.com/vmware-tanzu/velero/issues/3863 // or https://github.com/vmware-tanzu/velero/issues/4053 to be restored successfully. if volumeHasNonRestorableSource(pvb.Spec.Volume, pod.Spec.Volumes) { continue } volumes[pvb.Spec.Volume] = volumeBackupInfo{ snapshotID: pvb.Status.SnapshotID, snapshotSize: pvb.Status.Progress.TotalBytes, uploaderType: getUploaderTypeOrDefault(pvb.Spec.UploaderType), repositoryType: getRepositoryType(pvb.Spec.UploaderType), } } if len(volumes) > 0 { return volumes } fromAnnntation := getPodSnapshotAnnotations(pod) if fromAnnntation == nil { return nil } for k, v := range fromAnnntation { volumes[k] = volumeBackupInfo{v, 0, uploader.ResticType, velerov1api.BackupRepositoryTypeRestic} } return volumes } // GetSnapshotIdentifier returns the snapshots represented by SnapshotIdentifier for the given PVBs func GetSnapshotIdentifier(podVolumeBackups *velerov1api.PodVolumeBackupList) map[string][]repotypes.SnapshotIdentifier { res := map[string][]repotypes.SnapshotIdentifier{} for _, item := range podVolumeBackups.Items { if item.Status.SnapshotID == "" { continue } if res[item.Spec.Pod.Namespace] == nil { res[item.Spec.Pod.Namespace] = []repotypes.SnapshotIdentifier{} } snapshots := res[item.Spec.Pod.Namespace] snapshots = append(snapshots, repotypes.SnapshotIdentifier{ VolumeNamespace: item.Spec.Pod.Namespace, BackupStorageLocation: item.Spec.BackupStorageLocation, SnapshotID: item.Status.SnapshotID, RepositoryType: getRepositoryType(item.Spec.UploaderType), UploaderType: item.Spec.UploaderType, Source: item.Status.Path, RepoIdentifier: item.Spec.RepoIdentifier, }) res[item.Spec.Pod.Namespace] = snapshots } return res } func GetRealSource(pvb *velerov1api.PodVolumeBackup) string { pvcName := "" if pvb.Annotations != nil { pvcName = pvb.Annotations[configs.PVCNameAnnotation] } if pvcName != "" { return fmt.Sprintf("%s/%s/%s", pvb.Spec.Pod.Namespace, pvb.Spec.Pod.Name, pvcName) } else { return fmt.Sprintf("%s/%s/%s", pvb.Spec.Pod.Namespace, pvb.Spec.Pod.Name, pvb.Spec.Volume) } } func getUploaderTypeOrDefault(uploaderType string) string { if uploaderType != "" { return uploaderType } return uploader.ResticType } // getRepositoryType returns the hardcode repositoryType for different backup methods - Restic or Kopia,uploaderType // indicates the method. // For Restic backup method, it is always hardcode to BackupRepositoryTypeRestic, never changed. // For Kopia backup method, this means we hardcode repositoryType as BackupRepositoryTypeKopia for Unified Repo, // at present (Kopia backup method is using Unified Repo). However, it doesn't mean we could deduce repositoryType // from uploaderType for Unified Repo. // TODO: post v1.10, refactor this function for Kopia backup method. In future, when we have multiple implementations of // Unified Repo (besides Kopia), we will add the repositoryType to BSL, because by then, we are not able to hardcode // the repositoryType to BackupRepositoryTypeKopia for Unified Repo. func getRepositoryType(uploaderType string) string { switch uploaderType { case "", uploader.ResticType: return velerov1api.BackupRepositoryTypeRestic case uploader.KopiaType: return velerov1api.BackupRepositoryTypeKopia default: return "" } } func isPVBMatchPod(pvb *velerov1api.PodVolumeBackup, podName string, namespace string) bool { return podName == pvb.Spec.Pod.Name && namespace == pvb.Spec.Pod.Namespace } // volumeHasNonRestorableSource checks if the given volume exists in the list of podVolumes // and returns true if the volume's source is not restorable. This is true for volumes with // a Projected or DownwardAPI source. func volumeHasNonRestorableSource(volumeName string, podVolumes []corev1api.Volume) bool { var volume corev1api.Volume for _, v := range podVolumes { if v.Name == volumeName { volume = v break } } return volume.Projected != nil || volume.DownwardAPI != nil } // getPodSnapshotAnnotations returns a map, of volume name -> snapshot id, // of all snapshots for this pod. // TODO(2.0) to remove // Deprecated: we will stop using pod annotations to record pod volume snapshot IDs after they're taken, // therefore we won't need to check if these annotations exist. func getPodSnapshotAnnotations(obj metav1.Object) map[string]string { var res map[string]string insertSafe := func(k, v string) { if res == nil { res = make(map[string]string) } res[k] = v } for k, v := range obj.GetAnnotations() { if strings.HasPrefix(k, podAnnotationPrefix) { insertSafe(k[len(podAnnotationPrefix):], v) } } return res } ================================================ FILE: pkg/podvolume/util_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package podvolume import ( "testing" "github.com/stretchr/testify/assert" corev1api "k8s.io/api/core/v1" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" ) func TestGetVolumeBackupsForPod(t *testing.T) { tests := []struct { name string podVolumeBackups []*velerov1api.PodVolumeBackup podVolumes []corev1api.Volume podAnnotations map[string]string podName string sourcePodNs string expected map[string]string }{ { name: "nil annotations results in no volume backups returned", podAnnotations: nil, expected: nil, }, { name: "empty annotations results in no volume backups returned", podAnnotations: make(map[string]string), expected: nil, }, { name: "pod annotations with no snapshot annotation prefix results in no volume backups returned", podAnnotations: map[string]string{"foo": "bar"}, expected: nil, }, { name: "pod annotation with only snapshot annotation prefix, results in volume backup with empty volume key", podAnnotations: map[string]string{podAnnotationPrefix: "snapshotID"}, expected: map[string]string{"": "snapshotID"}, }, { name: "pod annotation with snapshot annotation prefix results in volume backup with volume name and snapshot ID", podAnnotations: map[string]string{podAnnotationPrefix + "volume": "snapshotID"}, expected: map[string]string{"volume": "snapshotID"}, }, { name: "only pod annotations with snapshot annotation prefix are considered", podAnnotations: map[string]string{"x": "y", podAnnotationPrefix + "volume1": "snapshot1", podAnnotationPrefix + "volume2": "snapshot2"}, expected: map[string]string{"volume1": "snapshot1", "volume2": "snapshot2"}, }, { name: "pod annotations are not considered if PVBs are provided", podVolumeBackups: []*velerov1api.PodVolumeBackup{ builder.ForPodVolumeBackup("velero", "pvb-1").PodName("TestPod").PodNamespace("TestNS").SnapshotID("snapshot1").Volume("pvbtest1-foo").Result(), builder.ForPodVolumeBackup("velero", "pvb-2").PodName("TestPod").PodNamespace("TestNS").SnapshotID("snapshot2").Volume("pvbtest2-abc").Result(), }, podName: "TestPod", sourcePodNs: "TestNS", podAnnotations: map[string]string{"x": "y", podAnnotationPrefix + "foo": "bar", podAnnotationPrefix + "abc": "123"}, expected: map[string]string{"pvbtest1-foo": "snapshot1", "pvbtest2-abc": "snapshot2"}, }, { name: "volume backups are returned even if no pod annotations are present", podVolumeBackups: []*velerov1api.PodVolumeBackup{ builder.ForPodVolumeBackup("velero", "pvb-1").PodName("TestPod").PodNamespace("TestNS").SnapshotID("snapshot1").Volume("pvbtest1-foo").Result(), builder.ForPodVolumeBackup("velero", "pvb-2").PodName("TestPod").PodNamespace("TestNS").SnapshotID("snapshot2").Volume("pvbtest2-abc").Result(), }, podName: "TestPod", sourcePodNs: "TestNS", expected: map[string]string{"pvbtest1-foo": "snapshot1", "pvbtest2-abc": "snapshot2"}, }, { name: "only volumes from PVBs with snapshot IDs are returned", podVolumeBackups: []*velerov1api.PodVolumeBackup{ builder.ForPodVolumeBackup("velero", "pvb-1").PodName("TestPod").PodNamespace("TestNS").SnapshotID("snapshot1").Volume("pvbtest1-foo").Result(), builder.ForPodVolumeBackup("velero", "pvb-2").PodName("TestPod").PodNamespace("TestNS").SnapshotID("snapshot2").Volume("pvbtest2-abc").Result(), builder.ForPodVolumeBackup("velero", "pvb-3").PodName("TestPod").PodNamespace("TestNS").Volume("pvbtest3-foo").Result(), builder.ForPodVolumeBackup("velero", "pvb-4").PodName("TestPod").PodNamespace("TestNS").Volume("pvbtest4-abc").Result(), }, podName: "TestPod", sourcePodNs: "TestNS", expected: map[string]string{"pvbtest1-foo": "snapshot1", "pvbtest2-abc": "snapshot2"}, }, { name: "only volumes from PVBs for the given pod are returned", podVolumeBackups: []*velerov1api.PodVolumeBackup{ builder.ForPodVolumeBackup("velero", "pvb-1").PodName("TestPod").PodNamespace("TestNS").SnapshotID("snapshot1").Volume("pvbtest1-foo").Result(), builder.ForPodVolumeBackup("velero", "pvb-2").PodName("TestPod").PodNamespace("TestNS").SnapshotID("snapshot2").Volume("pvbtest2-abc").Result(), builder.ForPodVolumeBackup("velero", "pvb-3").PodName("TestAnotherPod").SnapshotID("snapshot3").Volume("pvbtest3-xyz").Result(), }, podName: "TestPod", sourcePodNs: "TestNS", expected: map[string]string{"pvbtest1-foo": "snapshot1", "pvbtest2-abc": "snapshot2"}, }, { name: "only volumes from PVBs which match the pod name and source pod namespace are returned", podVolumeBackups: []*velerov1api.PodVolumeBackup{ builder.ForPodVolumeBackup("velero", "pvb-1").PodName("TestPod").PodNamespace("TestNS").SnapshotID("snapshot1").Volume("pvbtest1-foo").Result(), builder.ForPodVolumeBackup("velero", "pvb-2").PodName("TestAnotherPod").PodNamespace("TestNS").SnapshotID("snapshot2").Volume("pvbtest2-abc").Result(), builder.ForPodVolumeBackup("velero", "pvb-3").PodName("TestPod").PodNamespace("TestAnotherNS").SnapshotID("snapshot3").Volume("pvbtest3-xyz").Result(), }, podName: "TestPod", sourcePodNs: "TestNS", expected: map[string]string{"pvbtest1-foo": "snapshot1"}, }, { name: "volumes from PVBs that correspond to a pod volume from a projected source are not returned", podVolumeBackups: []*velerov1api.PodVolumeBackup{ builder.ForPodVolumeBackup("velero", "pvb-1").PodName("TestPod").PodNamespace("TestNS").SnapshotID("snapshot1").Volume("pvb-non-projected").Result(), builder.ForPodVolumeBackup("velero", "pvb-1").PodName("TestPod").PodNamespace("TestNS").SnapshotID("snapshot2").Volume("pvb-projected").Result(), }, podVolumes: []corev1api.Volume{ { Name: "pvb-non-projected", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{}, }, }, { Name: "pvb-projected", VolumeSource: corev1api.VolumeSource{ Projected: &corev1api.ProjectedVolumeSource{}, }, }, }, podName: "TestPod", sourcePodNs: "TestNS", expected: map[string]string{"pvb-non-projected": "snapshot1"}, }, { name: "volumes from PVBs that correspond to a pod volume from a DownwardAPI source are not returned", podVolumeBackups: []*velerov1api.PodVolumeBackup{ builder.ForPodVolumeBackup("velero", "pvb-1").PodName("TestPod").PodNamespace("TestNS").SnapshotID("snapshot1").Volume("pvb-non-downwardapi").Result(), builder.ForPodVolumeBackup("velero", "pvb-1").PodName("TestPod").PodNamespace("TestNS").SnapshotID("snapshot2").Volume("pvb-downwardapi").Result(), }, podVolumes: []corev1api.Volume{ { Name: "pvb-non-downwardapi", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{}, }, }, { Name: "pvb-downwardapi", VolumeSource: corev1api.VolumeSource{ DownwardAPI: &corev1api.DownwardAPIVolumeSource{}, }, }, }, podName: "TestPod", sourcePodNs: "TestNS", expected: map[string]string{"pvb-non-downwardapi": "snapshot1"}, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { pod := &corev1api.Pod{} pod.Annotations = test.podAnnotations pod.Name = test.podName pod.Spec.Volumes = test.podVolumes res := GetVolumeBackupsForPod(test.podVolumeBackups, pod, test.sourcePodNs) assert.Equal(t, test.expected, res) }) } } func TestVolumeHasNonRestorableSource(t *testing.T) { testCases := []struct { name string volumeName string podVolumes []corev1api.Volume expected bool }{ { name: "volume name not in list of volumes", volumeName: "missing-volume", podVolumes: []corev1api.Volume{ { Name: "restorable", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{}, }, }, { Name: "projected", VolumeSource: corev1api.VolumeSource{ Projected: &corev1api.ProjectedVolumeSource{}, }, }, { Name: "downwardapi", VolumeSource: corev1api.VolumeSource{ DownwardAPI: &corev1api.DownwardAPIVolumeSource{}, }, }, }, expected: false, }, { name: "volume name in list of volumes but not projected or DownwardAPI", volumeName: "restorable", podVolumes: []corev1api.Volume{ { Name: "restorable", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{}, }, }, { Name: "projected", VolumeSource: corev1api.VolumeSource{ Projected: &corev1api.ProjectedVolumeSource{}, }, }, { Name: "downwardapi", VolumeSource: corev1api.VolumeSource{ DownwardAPI: &corev1api.DownwardAPIVolumeSource{}, }, }, }, expected: false, }, { name: "volume name in list of volumes and projected", volumeName: "projected", podVolumes: []corev1api.Volume{ { Name: "restorable", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{}, }, }, { Name: "projected", VolumeSource: corev1api.VolumeSource{ Projected: &corev1api.ProjectedVolumeSource{}, }, }, { Name: "downwardapi", VolumeSource: corev1api.VolumeSource{ DownwardAPI: &corev1api.DownwardAPIVolumeSource{}, }, }, }, expected: true, }, { name: "volume name in list of volumes and is a DownwardAPI volume", volumeName: "downwardapi", podVolumes: []corev1api.Volume{ { Name: "restorable", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{}, }, }, { Name: "projected", VolumeSource: corev1api.VolumeSource{ Projected: &corev1api.ProjectedVolumeSource{}, }, }, { Name: "downwardapi", VolumeSource: corev1api.VolumeSource{ DownwardAPI: &corev1api.DownwardAPIVolumeSource{}, }, }, }, expected: true, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { actual := volumeHasNonRestorableSource(tc.volumeName, tc.podVolumes) assert.Equal(t, tc.expected, actual) }) } } func TestGetRealSource(t *testing.T) { testCases := []struct { name string pvb *velerov1api.PodVolumeBackup expected string }{ { name: "pvb with empty annotation", pvb: builder.ForPodVolumeBackup("fake-ns", "fake-name").PodNamespace("fake-pod-ns").PodName("fake-pod-name").Volume("fake-volume").Result(), expected: "fake-pod-ns/fake-pod-name/fake-volume", }, { name: "pvb without pvc name annotation", pvb: builder.ForPodVolumeBackup("fake-ns", "fake-name").PodNamespace("fake-pod-ns").PodName("fake-pod-name").Volume("fake-volume").Annotations(map[string]string{}).Result(), expected: "fake-pod-ns/fake-pod-name/fake-volume", }, { name: "pvb with pvc name annotation", pvb: builder.ForPodVolumeBackup("fake-ns", "fake-name").PodNamespace("fake-pod-ns").PodName("fake-pod-name").Volume("fake-volume").Annotations(map[string]string{"velero.io/pvc-name": "fake-pvc-name"}).Result(), expected: "fake-pod-ns/fake-pod-name/fake-pvc-name", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { actual := GetRealSource(tc.pvb) assert.Equal(t, tc.expected, actual) }) } } ================================================ FILE: pkg/repository/backup_repo_op.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package repository import ( "context" "fmt" "github.com/pkg/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/label" ) // A BackupRepositoryKey uniquely identify a backup repository type BackupRepositoryKey struct { VolumeNamespace string BackupLocation string RepositoryType string } var ( errBackupRepoNotFound = errors.New("backup repository not found") errBackupRepoNotProvisioned = errors.New("backup repository not provisioned") ) func repoLabelsFromKey(key BackupRepositoryKey) labels.Set { return map[string]string{ velerov1api.VolumeNamespaceLabel: label.GetValidName(key.VolumeNamespace), velerov1api.StorageLocationLabel: label.GetValidName(key.BackupLocation), velerov1api.RepositoryTypeLabel: label.GetValidName(key.RepositoryType), } } // GetBackupRepository gets a backup repository through BackupRepositoryKey and ensure ready if required. func GetBackupRepository(ctx context.Context, cli client.Client, namespace string, key BackupRepositoryKey, options ...bool) (*velerov1api.BackupRepository, error) { var ensureReady = true if len(options) > 0 { ensureReady = options[0] } selector := labels.SelectorFromSet(repoLabelsFromKey(key)) backupRepoList := &velerov1api.BackupRepositoryList{} err := cli.List(ctx, backupRepoList, &client.ListOptions{ Namespace: namespace, LabelSelector: selector, }) if err != nil { return nil, errors.Wrap(err, "error getting backup repository list") } if len(backupRepoList.Items) == 0 { return nil, errBackupRepoNotFound } if len(backupRepoList.Items) > 1 { return nil, errors.Errorf("more than one BackupRepository found for workload namespace %q, backup storage location %q, repository type %q", key.VolumeNamespace, key.BackupLocation, key.RepositoryType) } repo := &backupRepoList.Items[0] if ensureReady { if repo.Status.Phase == velerov1api.BackupRepositoryPhaseNotReady { return nil, errors.Errorf("backup repository is not ready: %s", repo.Status.Message) } if repo.Status.Phase == "" || repo.Status.Phase == velerov1api.BackupRepositoryPhaseNew { return nil, errBackupRepoNotProvisioned } } return repo, nil } func NewBackupRepository(namespace string, key BackupRepositoryKey) *velerov1api.BackupRepository { return &velerov1api.BackupRepository{ ObjectMeta: metav1.ObjectMeta{ Namespace: namespace, Name: fmt.Sprintf("%s-%s-%s", key.VolumeNamespace, key.BackupLocation, key.RepositoryType), Labels: repoLabelsFromKey(key), }, Spec: velerov1api.BackupRepositorySpec{ VolumeNamespace: key.VolumeNamespace, BackupStorageLocation: key.BackupLocation, RepositoryType: key.RepositoryType, }, } } func isBackupRepositoryNotFoundError(err error) bool { return err == errBackupRepoNotFound } func isBackupRepositoryNotProvisionedError(err error) bool { return err == errBackupRepoNotProvisioned } ================================================ FILE: pkg/repository/backup_repo_op_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package repository import ( "fmt" "github.com/stretchr/testify/assert" "testing" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func buildBackupRepo(key BackupRepositoryKey, phase velerov1api.BackupRepositoryPhase, seqNum string) velerov1api.BackupRepository { return velerov1api.BackupRepository{ Spec: velerov1api.BackupRepositorySpec{ResticIdentifier: ""}, TypeMeta: metav1.TypeMeta{ APIVersion: velerov1api.SchemeGroupVersion.String(), Kind: "BackupRepository", }, ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: fmt.Sprintf("%s-%s-%s-%s", key.VolumeNamespace, key.BackupLocation, key.RepositoryType, seqNum), Labels: map[string]string{ velerov1api.StorageLocationLabel: key.BackupLocation, velerov1api.VolumeNamespaceLabel: key.VolumeNamespace, velerov1api.RepositoryTypeLabel: key.RepositoryType, }, }, Status: velerov1api.BackupRepositoryStatus{ Phase: phase, }, } } func buildBackupRepoPointer(key BackupRepositoryKey, phase velerov1api.BackupRepositoryPhase, seqNum string) *velerov1api.BackupRepository { value := buildBackupRepo(key, phase, seqNum) return &value } func TestGetBackupRepository(t *testing.T) { testCases := []struct { name string backupRepositories []velerov1api.BackupRepository ensureReady bool backupRepositoryKey BackupRepositoryKey expected *velerov1api.BackupRepository expectedErr string }{ { name: "repository not found", expectedErr: "backup repository not found", }, { name: "found more than one repository", backupRepositories: []velerov1api.BackupRepository{ buildBackupRepo(BackupRepositoryKey{"fake-volume-ns", "fake-bsl", "fake-repository-type"}, velerov1api.BackupRepositoryPhaseReady, "01"), buildBackupRepo(BackupRepositoryKey{"fake-volume-ns", "fake-bsl", "fake-repository-type"}, velerov1api.BackupRepositoryPhaseReady, "02")}, backupRepositoryKey: BackupRepositoryKey{"fake-volume-ns", "fake-bsl", "fake-repository-type"}, expectedErr: "more than one BackupRepository found for workload namespace \"fake-volume-ns\", backup storage location \"fake-bsl\", repository type \"fake-repository-type\"", }, { name: "repository not ready, not expect ready", backupRepositories: []velerov1api.BackupRepository{ buildBackupRepo(BackupRepositoryKey{"fake-volume-ns-01", "fake-bsl-01", "fake-repository-type-01"}, velerov1api.BackupRepositoryPhaseReady, "01"), buildBackupRepo(BackupRepositoryKey{"fake-volume-ns-02", "fake-bsl-02", "fake-repository-type-02"}, velerov1api.BackupRepositoryPhaseNotReady, "02")}, backupRepositoryKey: BackupRepositoryKey{"fake-volume-ns-02", "fake-bsl-02", "fake-repository-type-02"}, expected: buildBackupRepoPointer(BackupRepositoryKey{"fake-volume-ns-02", "fake-bsl-02", "fake-repository-type-02"}, velerov1api.BackupRepositoryPhaseNotReady, "02"), }, { name: "repository is new, not expect ready", backupRepositories: []velerov1api.BackupRepository{ buildBackupRepo(BackupRepositoryKey{"fake-volume-ns-01", "fake-bsl-01", "fake-repository-type-01"}, velerov1api.BackupRepositoryPhaseReady, "01"), buildBackupRepo(BackupRepositoryKey{"fake-volume-ns-02", "fake-bsl-02", "fake-repository-type-02"}, velerov1api.BackupRepositoryPhaseNew, "02")}, backupRepositoryKey: BackupRepositoryKey{"fake-volume-ns-02", "fake-bsl-02", "fake-repository-type-02"}, expected: buildBackupRepoPointer(BackupRepositoryKey{"fake-volume-ns-02", "fake-bsl-02", "fake-repository-type-02"}, velerov1api.BackupRepositoryPhaseNew, "02"), }, { name: "repository state is empty, not expect ready", backupRepositories: []velerov1api.BackupRepository{ buildBackupRepo(BackupRepositoryKey{"fake-volume-ns-01", "fake-bsl-01", "fake-repository-type-01"}, velerov1api.BackupRepositoryPhaseReady, "01"), buildBackupRepo(BackupRepositoryKey{"fake-volume-ns-02", "fake-bsl-02", "fake-repository-type-02"}, "", "02")}, backupRepositoryKey: BackupRepositoryKey{"fake-volume-ns-02", "fake-bsl-02", "fake-repository-type-02"}, expected: buildBackupRepoPointer(BackupRepositoryKey{"fake-volume-ns-02", "fake-bsl-02", "fake-repository-type-02"}, "", "02"), }, { name: "repository not ready, expect ready", backupRepositories: []velerov1api.BackupRepository{ buildBackupRepo(BackupRepositoryKey{"fake-volume-ns-01", "fake-bsl-01", "fake-repository-type-01"}, velerov1api.BackupRepositoryPhaseReady, "01"), buildBackupRepo(BackupRepositoryKey{"fake-volume-ns-02", "fake-bsl-02", "fake-repository-type-02"}, velerov1api.BackupRepositoryPhaseNotReady, "02")}, backupRepositoryKey: BackupRepositoryKey{"fake-volume-ns-02", "fake-bsl-02", "fake-repository-type-02"}, ensureReady: true, expectedErr: "backup repository is not ready: ", }, { name: "repository is new, expect ready", backupRepositories: []velerov1api.BackupRepository{ buildBackupRepo(BackupRepositoryKey{"fake-volume-ns-01", "fake-bsl-01", "fake-repository-type-01"}, velerov1api.BackupRepositoryPhaseReady, "01"), buildBackupRepo(BackupRepositoryKey{"fake-volume-ns-02", "fake-bsl-02", "fake-repository-type-02"}, velerov1api.BackupRepositoryPhaseNew, "02")}, backupRepositoryKey: BackupRepositoryKey{"fake-volume-ns-02", "fake-bsl-02", "fake-repository-type-02"}, ensureReady: true, expectedErr: "backup repository not provisioned", }, { name: "repository state is empty, expect ready", backupRepositories: []velerov1api.BackupRepository{ buildBackupRepo(BackupRepositoryKey{"fake-volume-ns-01", "fake-bsl-01", "fake-repository-type-01"}, velerov1api.BackupRepositoryPhaseReady, "01"), buildBackupRepo(BackupRepositoryKey{"fake-volume-ns-02", "fake-bsl-02", "fake-repository-type-02"}, "", "02")}, backupRepositoryKey: BackupRepositoryKey{"fake-volume-ns-02", "fake-bsl-02", "fake-repository-type-02"}, ensureReady: true, expectedErr: "backup repository not provisioned", }, { name: "repository ready, expect ready", backupRepositories: []velerov1api.BackupRepository{ buildBackupRepo(BackupRepositoryKey{"fake-volume-ns-01", "fake-bsl-01", "fake-repository-type-01"}, velerov1api.BackupRepositoryPhaseNotReady, "01"), buildBackupRepo(BackupRepositoryKey{"fake-volume-ns-02", "fake-bsl-02", "fake-repository-type-02"}, velerov1api.BackupRepositoryPhaseReady, "02")}, backupRepositoryKey: BackupRepositoryKey{"fake-volume-ns-02", "fake-bsl-02", "fake-repository-type-02"}, ensureReady: true, expected: buildBackupRepoPointer(BackupRepositoryKey{"fake-volume-ns-02", "fake-bsl-02", "fake-repository-type-02"}, velerov1api.BackupRepositoryPhaseReady, "02"), }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { clientBuilder := velerotest.NewFakeControllerRuntimeClientBuilder(t) clientBuilder.WithLists(&velerov1api.BackupRepositoryList{ Items: tc.backupRepositories, }) fakeClient := clientBuilder.Build() backupRepo, err := GetBackupRepository(t.Context(), fakeClient, velerov1api.DefaultNamespace, tc.backupRepositoryKey, tc.ensureReady) if backupRepo != nil && tc.expected != nil { backupRepo.ResourceVersion = tc.expected.ResourceVersion require.Equal(t, *tc.expected, *backupRepo) } else { require.Equal(t, tc.expected, backupRepo) } if tc.expectedErr == "" { assert.NoError(t, err) } else { assert.EqualError(t, err, tc.expectedErr) } }) } } ================================================ FILE: pkg/repository/config/aws.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ //nolint:gosec // Internal usage. No need to check. package config import ( "context" "fmt" "os" "time" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/credentials/stscreds" "github.com/aws/aws-sdk-go-v2/service/sts" "github.com/aws/aws-sdk-go-v2/aws" awsconfig "github.com/aws/aws-sdk-go-v2/config" s3manager "github.com/aws/aws-sdk-go-v2/feature/s3/manager" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/pkg/errors" ) // getS3CredentialsFunc is used to make testing more convenient var getS3CredentialsFunc = GetS3Credentials const ( // AWS specific environment variable awsProfileEnvVar = "AWS_PROFILE" awsKeyIDEnvVar = "AWS_ACCESS_KEY_ID" awsSecretKeyEnvVar = "AWS_SECRET_ACCESS_KEY" awsSessTokenEnvVar = "AWS_SESSION_TOKEN" awsProfileKey = "profile" awsCredentialsFileEnvVar = "AWS_SHARED_CREDENTIALS_FILE" awsConfigFileEnvVar = "AWS_CONFIG_FILE" awsDefaultProfile = "default" ) // GetS3ResticEnvVars gets the environment variables that restic // relies on (AWS_PROFILE) based on info in the provided object // storage location config map. func GetS3ResticEnvVars(config map[string]string) (map[string]string, error) { result := make(map[string]string) if credentialsFile, ok := config[CredentialsFileKey]; ok { result[awsCredentialsFileEnvVar] = credentialsFile } if profile, ok := config[awsProfileKey]; ok { result[awsProfileEnvVar] = profile } // GetS3ResticEnvVars reads the AWS config, from files and envs // if needed assumes the role and returns the session credentials // setting these variables emulates what would happen for example when using kube2iam if creds, err := getS3CredentialsFunc(config); err == nil && creds != nil { result[awsKeyIDEnvVar] = creds.AccessKeyID result[awsSecretKeyEnvVar] = creds.SecretAccessKey result[awsSessTokenEnvVar] = creds.SessionToken result[awsCredentialsFileEnvVar] = "" result[awsProfileEnvVar] = "" // profile is not needed since we have the credentials from profile via GetS3Credentials result[awsConfigFileEnvVar] = "" } return result, nil } // GetS3Credentials gets the S3 credential values according to the information // of the provided config or the system's environment variables func GetS3Credentials(config map[string]string) (*aws.Credentials, error) { var opts []func(*awsconfig.LoadOptions) error credentialsFile := config[CredentialsFileKey] if credentialsFile == "" { credentialsFile = os.Getenv(awsCredentialsFileEnvVar) } if credentialsFile != "" { opts = append(opts, awsconfig.WithSharedCredentialsFiles([]string{credentialsFile}), // To support the existing use case where config file is passed // as credentials of a BSL awsconfig.WithSharedConfigFiles([]string{credentialsFile})) } opts = append(opts, awsconfig.WithSharedConfigProfile(config[awsProfileKey])) cfg, err := awsconfig.LoadDefaultConfig(context.Background(), opts...) if err != nil { return nil, err } if credentialsFile != "" && os.Getenv("AWS_WEB_IDENTITY_TOKEN_FILE") != "" && os.Getenv("AWS_ROLE_ARN") != "" { // Reset the config to use the credentials from the credentials/config file profile := config[awsProfileKey] if profile == "" { profile = awsDefaultProfile } sfp, err := awsconfig.LoadSharedConfigProfile(context.Background(), profile, func(o *awsconfig.LoadSharedConfigOptions) { o.ConfigFiles = []string{credentialsFile} o.CredentialsFiles = []string{credentialsFile} }) if err != nil { return nil, fmt.Errorf("error loading config profile '%s': %v", profile, err) } if err := resolveCredsFromProfile(&cfg, &sfp); err != nil { return nil, fmt.Errorf("error resolving creds from profile '%s': %v", profile, err) } } creds, err := cfg.Credentials.Retrieve(context.Background()) return &creds, err } // GetAWSBucketRegion returns the AWS region that a bucket is in, or an error // if the region cannot be determined. // It will use us-east-1 as hinting server and requires config param to use as credentials func GetAWSBucketRegion(bucket string, config map[string]string) (string, error) { cfg, err := awsconfig.LoadDefaultConfig(context.Background(), awsconfig.WithCredentialsProvider( aws.CredentialsProviderFunc( func(context.Context) (aws.Credentials, error) { s3creds, err := GetS3Credentials(config) if s3creds == nil { return aws.Credentials{}, err } return *s3creds, err }, ), )) if err != nil { return "", errors.WithStack(err) } client := s3.NewFromConfig(cfg) region, err := s3manager.GetBucketRegion(context.Background(), client, bucket, func(o *s3.Options) { o.Region = "us-east-1" }) if err != nil { return "", errors.WithStack(err) } if region == "" { return "", errors.New("unable to determine bucket's region") } return region, nil } func resolveCredsFromProfile(cfg *aws.Config, sharedConfig *awsconfig.SharedConfig) error { var err error switch { case sharedConfig.Source != nil: // Assume IAM role with credentials source from a different profile. err = resolveCredsFromProfile(cfg, sharedConfig.Source) case sharedConfig.Credentials.HasKeys(): // Static Credentials from Shared Config/Credentials file. cfg.Credentials = credentials.StaticCredentialsProvider{ Value: sharedConfig.Credentials, } } if err != nil { return err } if len(sharedConfig.RoleARN) > 0 { credsFromAssumeRole(cfg, sharedConfig) } return nil } func credsFromAssumeRole(cfg *aws.Config, sharedCfg *awsconfig.SharedConfig) { optFns := []func(*stscreds.AssumeRoleOptions){ func(options *stscreds.AssumeRoleOptions) { options.RoleSessionName = sharedCfg.RoleSessionName if sharedCfg.RoleDurationSeconds != nil { if *sharedCfg.RoleDurationSeconds/time.Minute > 15 { options.Duration = *sharedCfg.RoleDurationSeconds } } if len(sharedCfg.ExternalID) > 0 { options.ExternalID = aws.String(sharedCfg.ExternalID) } if len(sharedCfg.MFASerial) != 0 { options.SerialNumber = aws.String(sharedCfg.MFASerial) } }, } cfg.Credentials = stscreds.NewAssumeRoleProvider(sts.NewFromConfig(*cfg), sharedCfg.RoleARN, optFns...) } ================================================ FILE: pkg/repository/config/aws_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package config import ( "os" "reflect" "testing" "github.com/aws/aws-sdk-go-v2/aws" "github.com/stretchr/testify/require" ) func TestGetS3ResticEnvVars(t *testing.T) { testCases := []struct { name string config map[string]string expected map[string]string getS3Credentials func(config map[string]string) (*aws.Credentials, error) }{ { name: "when config is empty, no env vars are returned", config: map[string]string{}, expected: map[string]string{}, getS3Credentials: func(config map[string]string) (*aws.Credentials, error) { return nil, nil }, }, { name: "when config contains profile key, profile env var is set with profile value", config: map[string]string{ "profile": "profile-value", }, expected: map[string]string{ "AWS_PROFILE": "profile-value", }, }, { name: "when config contains credentials file key, credentials file env var is set with credentials file value", config: map[string]string{ "credentialsFile": "/tmp/credentials/path/to/secret", }, expected: map[string]string{ "AWS_SHARED_CREDENTIALS_FILE": "/tmp/credentials/path/to/secret", }, getS3Credentials: func(config map[string]string) (*aws.Credentials, error) { return nil, nil }, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { // Mock GetS3Credentials if tc.getS3Credentials != nil { getS3CredentialsFunc = tc.getS3Credentials } else { getS3CredentialsFunc = GetS3Credentials } actual, err := GetS3ResticEnvVars(tc.config) require.NoError(t, err) // Avoid direct comparison of expected and actual to prevent exposing secrets. // This may occur if the test doesn't set getS3Credentials func correctly. if !reflect.DeepEqual(tc.expected, actual) { t.Errorf("Expected and actual results do not match for test case %q", tc.name) for key, value := range actual { if expVal, err := tc.expected[key]; !err || expVal != value { if actualVal, ok := actual[key]; !ok { t.Errorf("Key %q is missing in actual result", key) } else if expVal != actualVal { t.Errorf("Key %q: expected value %q", key, expVal) } } } } }) } } func TestGetS3CredentialsCorrectlyUseProfile(t *testing.T) { type args struct { config map[string]string secretFileContents string } tests := []struct { name string args args want *aws.Credentials wantErr bool }{ { name: "Test GetS3Credentials use profile correctly", args: args{ config: map[string]string{ "profile": "some-profile", }, secretFileContents: `[default] aws_access_key_id = default-access-key-id aws_secret_access_key = default-secret-access-key [profile some-profile] aws_access_key_id = some-profile-access-key-id aws_secret_access_key = some-profile-secret-access-key `, }, want: &aws.Credentials{ AccessKeyID: "some-profile-access-key-id", SecretAccessKey: "some-profile-secret-access-key", }, }, { name: "Test GetS3Credentials default to default profile", args: args{ config: map[string]string{}, secretFileContents: `[default] aws_access_key_id = default-access-key-id aws_secret_access_key = default-secret-access-key [profile some-profile] aws_access_key_id = some-profile-access-key-id aws_secret_access_key = some-profile-secret-access-key `, }, want: &aws.Credentials{ AccessKeyID: "default-access-key-id", SecretAccessKey: "default-secret-access-key", }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { // Ensure env variables do not set AWS config entries t.Setenv("AWS_ACCESS_KEY_ID", "") t.Setenv("AWS_SECRET_ACCESS_KEY", "") t.Setenv("AWS_SHARED_CREDENTIALS_FILE", "") tmpFile, err := os.CreateTemp(t.TempDir(), "velero-test-aws-credentials") defer os.Remove(tmpFile.Name()) if err != nil { t.Errorf("GetS3Credentials() error = %v", err) return } // write the contents of the secret file to the temp file _, err = tmpFile.WriteString(tt.args.secretFileContents) if err != nil { t.Errorf("GetS3Credentials() error = %v", err) return } tt.args.config["credentialsFile"] = tmpFile.Name() got, err := GetS3Credentials(tt.args.config) if (err != nil) != tt.wantErr { t.Errorf("GetS3Credentials() error = %v, wantErr %v", err, tt.wantErr) return } if !reflect.DeepEqual(got.AccessKeyID, tt.want.AccessKeyID) { t.Errorf("GetS3Credentials() want %v", tt.want.AccessKeyID) } if !reflect.DeepEqual(got.SecretAccessKey, tt.want.SecretAccessKey) { t.Errorf("GetS3Credentials() want %v", tt.want.SecretAccessKey) } }) } } ================================================ FILE: pkg/repository/config/azure.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package config import ( "github.com/pkg/errors" "github.com/vmware-tanzu/velero/pkg/util/azure" ) // GetAzureResticEnvVars gets the environment variables that restic // relies on (AZURE_ACCOUNT_NAME and AZURE_ACCOUNT_KEY) based // on info in the provided object storage location config map. func GetAzureResticEnvVars(config map[string]string) (map[string]string, error) { storageAccount := config[azure.BSLConfigStorageAccount] if storageAccount == "" { return nil, errors.New("storageAccount is required in the BSL") } creds, err := azure.LoadCredentials(config) if err != nil { return nil, err } // restic doesn't support Azure AD, set it as false config[azure.BSLConfigUseAAD] = "false" credentials, err := azure.GetStorageAccountCredentials(config, creds) if err != nil { return nil, err } return map[string]string{ "AZURE_ACCOUNT_NAME": storageAccount, "AZURE_ACCOUNT_KEY": credentials[azure.CredentialKeyStorageAccountAccessKey], }, nil } ================================================ FILE: pkg/repository/config/azure_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package config import ( "os" "path/filepath" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/vmware-tanzu/velero/pkg/util/azure" ) func TestGetAzureResticEnvVars(t *testing.T) { config := map[string]string{} // no storage account specified _, err := GetAzureResticEnvVars(config) require.Error(t, err) // specify storage account access key name := filepath.Join(os.TempDir(), "credential") file, err := os.Create(name) require.NoError(t, err) defer file.Close() defer os.Remove(name) _, err = file.WriteString("AccessKey: accesskey") require.NoError(t, err) config[azure.BSLConfigStorageAccount] = "account01" config[azure.BSLConfigStorageAccountAccessKeyName] = "AccessKey" config["credentialsFile"] = name envs, err := GetAzureResticEnvVars(config) require.NoError(t, err) assert.Equal(t, "account01", envs["AZURE_ACCOUNT_NAME"]) assert.Equal(t, "accesskey", envs["AZURE_ACCOUNT_KEY"]) } ================================================ FILE: pkg/repository/config/config.go ================================================ /* Copyright 2018, 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package config import ( "fmt" "path" "strings" "github.com/pkg/errors" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/persistence" ) type BackendType string const ( AWSBackend BackendType = "velero.io/aws" AzureBackend BackendType = "velero.io/azure" GCPBackend BackendType = "velero.io/gcp" FSBackend BackendType = "velero.io/fs" // CredentialsFileKey is the key within a BSL config that is checked to see if // the BSL is using its own credentials, rather than those in the environment CredentialsFileKey = "credentialsFile" ) // this func is assigned to a package-level variable so it can be // replaced when unit-testing var getAWSBucketRegion = GetAWSBucketRegion // getRepoPrefix returns the prefix of the value of the --repo flag for // restic commands, i.e. everything except the "/". func getRepoPrefix(location *velerov1api.BackupStorageLocation) (string, error) { var bucket, prefix string if location.Spec.ObjectStorage != nil { layout := persistence.NewObjectStoreLayout(location.Spec.ObjectStorage.Prefix) bucket = location.Spec.ObjectStorage.Bucket prefix = layout.GetResticDir() } backendType := GetBackendType(location.Spec.Provider, location.Spec.Config) if repoPrefix := location.Spec.Config["resticRepoPrefix"]; repoPrefix != "" { return repoPrefix, nil } switch backendType { case AWSBackend: var url string // non-AWS, S3-compatible object store if s3Url := location.Spec.Config["s3Url"]; s3Url != "" { url = strings.TrimSuffix(s3Url, "/") } else { var err error region := location.Spec.Config["region"] if region == "" { region, err = getAWSBucketRegion(bucket, location.Spec.Config) } if err != nil { return "", errors.Wrapf(err, "failed to detect the region via bucket: %s", bucket) } url = fmt.Sprintf("s3-%s.amazonaws.com", region) } return fmt.Sprintf("s3:%s/%s", url, path.Join(bucket, prefix)), nil case AzureBackend: return fmt.Sprintf("azure:%s:/%s", bucket, prefix), nil case GCPBackend: return fmt.Sprintf("gs:%s:/%s", bucket, prefix), nil } return "", errors.Errorf("invalid backend type %s, provider %s", backendType, location.Spec.Provider) } // GetBackendType returns a backend type that is known by Velero. // If the provider doesn't indicate a known backend type, but the endpoint is // specified, Velero regards it as a S3 compatible object store and return AWSBackend as the type. func GetBackendType(provider string, config map[string]string) BackendType { if !strings.Contains(provider, "/") { provider = "velero.io/" + provider } bt := BackendType(provider) if IsBackendTypeValid(bt) { return bt } else if config != nil && config["s3Url"] != "" { return AWSBackend } else { return bt } } func IsBackendTypeValid(backendType BackendType) bool { return (backendType == AWSBackend || backendType == AzureBackend || backendType == GCPBackend || backendType == FSBackend) } // GetRepoIdentifier returns the string to be used as the value of the --repo flag in // restic commands for the given repository. func GetRepoIdentifier(location *velerov1api.BackupStorageLocation, name string) (string, error) { prefix, err := getRepoPrefix(location) if err != nil { return "", err } return fmt.Sprintf("%s/%s", strings.TrimSuffix(prefix, "/"), name), nil } ================================================ FILE: pkg/repository/config/config_test.go ================================================ /* Copyright 2018, 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package config import ( "testing" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) func TestGetRepoIdentifier(t *testing.T) { testCases := []struct { name string bsl *velerov1api.BackupStorageLocation repoName string getAWSBucketRegion func(s string, config map[string]string) (string, error) expected string expectedErr string }{ { name: "error is returned if BSL uses unsupported provider and resticRepoPrefix is not set", bsl: &velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Provider: "unsupported-provider", StorageType: velerov1api.StorageType{ ObjectStorage: &velerov1api.ObjectStorageLocation{ Bucket: "bucket-2", Prefix: "prefix-2", }, }, }, }, repoName: "repo-1", expectedErr: "invalid backend type velero.io/unsupported-provider, provider unsupported-provider", }, { name: "resticRepoPrefix in BSL config is used if set", bsl: &velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Provider: "custom-repo-identifier", Config: map[string]string{ "resticRepoPrefix": "custom:prefix:/restic", }, StorageType: velerov1api.StorageType{ ObjectStorage: &velerov1api.ObjectStorageLocation{ Bucket: "bucket", Prefix: "prefix", }, }, }, }, repoName: "repo-1", expected: "custom:prefix:/restic/repo-1", }, { name: "s3Url in BSL config is used", bsl: &velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Provider: "custom-repo-identifier", Config: map[string]string{ "s3Url": "s3Url", }, StorageType: velerov1api.StorageType{ ObjectStorage: &velerov1api.ObjectStorageLocation{ Bucket: "bucket", Prefix: "prefix", }, }, }, }, repoName: "repo-1", expected: "s3:s3Url/bucket/prefix/restic/repo-1", }, { name: "s3.amazonaws.com URL format is used if region cannot be determined for AWS BSL", bsl: &velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Provider: "aws", StorageType: velerov1api.StorageType{ ObjectStorage: &velerov1api.ObjectStorageLocation{ Bucket: "bucket", }, }, }, }, repoName: "repo-1", getAWSBucketRegion: func(s string, config map[string]string) (string, error) { return "", errors.New("no region found") }, expected: "", expectedErr: "failed to detect the region via bucket: bucket: no region found", }, { name: "s3.s3-.amazonaws.com URL format is used if region can be determined for AWS BSL", bsl: &velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Provider: "aws", StorageType: velerov1api.StorageType{ ObjectStorage: &velerov1api.ObjectStorageLocation{ Bucket: "bucket", }, }, }, }, repoName: "repo-1", getAWSBucketRegion: func(string, map[string]string) (string, error) { return "eu-west-1", nil }, expected: "s3:s3-eu-west-1.amazonaws.com/bucket/restic/repo-1", }, { name: "prefix is included in repo identifier if set for AWS BSL", bsl: &velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Provider: "aws", StorageType: velerov1api.StorageType{ ObjectStorage: &velerov1api.ObjectStorageLocation{ Bucket: "bucket", Prefix: "prefix", }, }, }, }, repoName: "repo-1", getAWSBucketRegion: func(s string, config map[string]string) (string, error) { return "eu-west-1", nil }, expected: "s3:s3-eu-west-1.amazonaws.com/bucket/prefix/restic/repo-1", }, { name: "s3Url is used in repo identifier if set for AWS BSL", bsl: &velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Provider: "aws", Config: map[string]string{ "s3Url": "alternate-url", }, StorageType: velerov1api.StorageType{ ObjectStorage: &velerov1api.ObjectStorageLocation{ Bucket: "bucket", Prefix: "prefix", }, }, }, }, repoName: "repo-1", getAWSBucketRegion: func(s string, config map[string]string) (string, error) { return "eu-west-1", nil }, expected: "s3:alternate-url/bucket/prefix/restic/repo-1", }, { name: "region is used in repo identifier if set for AWS BSL", bsl: &velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Provider: "aws", Config: map[string]string{ "region": "us-west-1", }, StorageType: velerov1api.StorageType{ ObjectStorage: &velerov1api.ObjectStorageLocation{ Bucket: "bucket", Prefix: "prefix", }, }, }, }, repoName: "aws-repo", getAWSBucketRegion: func(s string, config map[string]string) (string, error) { return "eu-west-1", nil }, expected: "s3:s3-us-west-1.amazonaws.com/bucket/prefix/restic/aws-repo", }, { name: "trailing slash in s3Url is not included in repo identifier for AWS BSL", bsl: &velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Provider: "aws", Config: map[string]string{ "s3Url": "alternate-url-with-trailing-slash/", }, StorageType: velerov1api.StorageType{ ObjectStorage: &velerov1api.ObjectStorageLocation{ Bucket: "bucket", Prefix: "prefix", }, }, }, }, repoName: "aws-repo", getAWSBucketRegion: func(s string, config map[string]string) (string, error) { return "eu-west-1", nil }, expected: "s3:alternate-url-with-trailing-slash/bucket/prefix/restic/aws-repo", }, { name: "repo identifier includes bucket and prefix for Azure BSL", bsl: &velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Provider: "azure", StorageType: velerov1api.StorageType{ ObjectStorage: &velerov1api.ObjectStorageLocation{ Bucket: "azure-bucket", Prefix: "azure-prefix", }, }, }, }, repoName: "azure-repo", expected: "azure:azure-bucket:/azure-prefix/restic/azure-repo", }, { name: "repo identifier includes bucket and prefix for GCP BSL", bsl: &velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Provider: "gcp", StorageType: velerov1api.StorageType{ ObjectStorage: &velerov1api.ObjectStorageLocation{ Bucket: "gcp-bucket", Prefix: "gcp-prefix", }, }, }, }, repoName: "gcp-repo", expected: "gs:gcp-bucket:/gcp-prefix/restic/gcp-repo", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { getAWSBucketRegion = tc.getAWSBucketRegion id, err := GetRepoIdentifier(tc.bsl, tc.repoName) assert.Equal(t, tc.expected, id) if tc.expectedErr == "" { assert.NoError(t, err) } else { require.EqualError(t, err, tc.expectedErr) assert.Empty(t, id) } }) } } ================================================ FILE: pkg/repository/config/gcp.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ //nolint:gosec // Internal usage. No need to check. package config import "os" const ( // GCP specific environment variable gcpCredentialsFileEnvVar = "GOOGLE_APPLICATION_CREDENTIALS" ) // GetGCPResticEnvVars gets the environment variables that restic relies // on based on info in the provided object storage location config map. func GetGCPResticEnvVars(config map[string]string) (map[string]string, error) { result := make(map[string]string) if credentialsFile, ok := config[CredentialsFileKey]; ok { result[gcpCredentialsFileEnvVar] = credentialsFile } return result, nil } // GetGCPCredentials gets the credential file required by a GCP bucket connection, // if the provided config doean't have the value, get it from system's environment variables func GetGCPCredentials(config map[string]string) string { if credentialsFile, ok := config[CredentialsFileKey]; ok { return credentialsFile } return os.Getenv(gcpCredentialsFileEnvVar) } ================================================ FILE: pkg/repository/config/gcp_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package config import ( "testing" "github.com/stretchr/testify/require" ) func TestGetGCPResticEnvVars(t *testing.T) { testCases := []struct { name string config map[string]string expected map[string]string }{ { name: "when config is empty, no env vars are returned", config: map[string]string{}, expected: map[string]string{}, }, { name: "when config contains credentials file key, credentials file env var is set with credentials file value", config: map[string]string{ "credentialsFile": "/tmp/credentials/path/to/secret", }, expected: map[string]string{ "GOOGLE_APPLICATION_CREDENTIALS": "/tmp/credentials/path/to/secret", }, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { actual, err := GetGCPResticEnvVars(tc.config) require.NoError(t, err) require.Equal(t, tc.expected, actual) }) } } ================================================ FILE: pkg/repository/ensurer.go ================================================ /* Copyright 2018, 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package repository import ( "context" "sync" "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/util/wait" "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" ) // Ensurer ensures that backup repositories are created and ready. type Ensurer struct { log logrus.FieldLogger repoClient client.Client // repoLocksMu synchronizes reads/writes to the repoLocks map itself // since maps are not threadsafe. repoLocksMu sync.Mutex repoLocks map[BackupRepositoryKey]*sync.Mutex resourceTimeout time.Duration } func NewEnsurer(repoClient client.Client, log logrus.FieldLogger, resourceTimeout time.Duration) *Ensurer { return &Ensurer{ log: log, repoClient: repoClient, repoLocks: make(map[BackupRepositoryKey]*sync.Mutex), resourceTimeout: resourceTimeout, } } func (r *Ensurer) EnsureRepo(ctx context.Context, namespace, volumeNamespace, backupLocation, repositoryType string) (*velerov1api.BackupRepository, error) { if volumeNamespace == "" || backupLocation == "" || repositoryType == "" { return nil, errors.Errorf("wrong parameters, namespace %q, backup storage location %q, repository type %q", volumeNamespace, backupLocation, repositoryType) } backupRepoKey := BackupRepositoryKey{volumeNamespace, backupLocation, repositoryType} log := r.log.WithField("volumeNamespace", volumeNamespace).WithField("backupLocation", backupLocation).WithField("repositoryType", repositoryType) // The BackupRepository is labeled with BackupRepositoryKey. // This function searches for an existing BackupRepository by BackupRepositoryKey label. // If it doesn't exist, it creates a new one and wait for its readiness. // // The BackupRepository is also named as BackupRepositoryKey. // It creates a BackupRepository with deterministic name // so that this function could support multiple thread calling by leveraging API server's optimistic lock mechanism. // Therefore, the name must be unique for a BackupRepository. // Don't use name to filter/search BackupRepository, since it may be changed in future, use label instead. log.Debug("Acquiring lock") repoMu := r.repoLock(backupRepoKey) repoMu.Lock() defer func() { repoMu.Unlock() log.Debug("Released lock") }() _, err := GetBackupRepository(ctx, r.repoClient, namespace, backupRepoKey, false) if err == nil { log.Info("Founding existing repo") return r.waitBackupRepository(ctx, namespace, backupRepoKey) } else if isBackupRepositoryNotFoundError(err) { log.Info("No repository found, creating one") // no repo found: create one and wait for it to be ready return r.createBackupRepositoryAndWait(ctx, namespace, backupRepoKey) } else { return nil, errors.WithStack(err) } } func (r *Ensurer) repoLock(key BackupRepositoryKey) *sync.Mutex { r.repoLocksMu.Lock() defer r.repoLocksMu.Unlock() if r.repoLocks[key] == nil { r.repoLocks[key] = new(sync.Mutex) } return r.repoLocks[key] } func (r *Ensurer) createBackupRepositoryAndWait(ctx context.Context, namespace string, backupRepoKey BackupRepositoryKey) (*velerov1api.BackupRepository, error) { toCreate := NewBackupRepository(namespace, backupRepoKey) if err := r.repoClient.Create(ctx, toCreate, &client.CreateOptions{}); err != nil && !apierrors.IsAlreadyExists(err) { return nil, errors.Wrap(err, "unable to create backup repository resource") } return r.waitBackupRepository(ctx, namespace, backupRepoKey) } func (r *Ensurer) waitBackupRepository(ctx context.Context, namespace string, backupRepoKey BackupRepositoryKey) (*velerov1api.BackupRepository, error) { var repo *velerov1api.BackupRepository var checkErr error checkFunc := func(ctx context.Context) (bool, error) { found, err := GetBackupRepository(ctx, r.repoClient, namespace, backupRepoKey, true) if err == nil { repo = found return true, nil } else if isBackupRepositoryNotFoundError(err) || isBackupRepositoryNotProvisionedError(err) { checkErr = err return false, nil } else { return false, err } } err := wait.PollUntilContextTimeout(ctx, time.Millisecond*500, r.resourceTimeout, true, checkFunc) if err != nil { if err == context.DeadlineExceeded { // if deadline is exceeded, return the error from the last check instead of the wait error return nil, errors.Wrap(checkErr, "failed to wait BackupRepository, timeout exceeded") } // if the error is not deadline exceeded, return the error from the wait return nil, errors.Wrap(err, "failed to wait BackupRepository, errored early") } return repo, nil } ================================================ FILE: pkg/repository/ensurer_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package repository import ( "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client/fake" velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func TestEnsureRepo(t *testing.T) { bkRepoObjReady := NewBackupRepository(velerov1.DefaultNamespace, BackupRepositoryKey{ VolumeNamespace: "fake-ns", BackupLocation: "fake-bsl", RepositoryType: "fake-repo-type", }) bkRepoObjReady.Status.Phase = velerov1.BackupRepositoryPhaseReady bkRepoObjNotReady := NewBackupRepository(velerov1.DefaultNamespace, BackupRepositoryKey{ VolumeNamespace: "fake-ns", BackupLocation: "fake-bsl", RepositoryType: "fake-repo-type", }) scheme := runtime.NewScheme() velerov1.AddToScheme(scheme) tests := []struct { name string namespace string bsl string repositoryType string kubeClientObj []runtime.Object runtimeScheme *runtime.Scheme expectedRepo *velerov1.BackupRepository err string }{ { name: "namespace is empty", bsl: "fake-bsl", repositoryType: "fake-repo-type", err: "wrong parameters, namespace \"\", backup storage location \"fake-bsl\", repository type \"fake-repo-type\"", }, { name: "bsl is empty", namespace: "fake-ns", repositoryType: "fake-repo-type", err: "wrong parameters, namespace \"fake-ns\", backup storage location \"\", repository type \"fake-repo-type\"", }, { name: "repositoryType is empty", namespace: "fake-ns", bsl: "fake-bsl", err: "wrong parameters, namespace \"fake-ns\", backup storage location \"fake-bsl\", repository type \"\"", }, { name: "get repo fail", namespace: "fake-ns", bsl: "fake-bsl", repositoryType: "fake-repo-type", err: "error getting backup repository list: no kind is registered for the type v1.BackupRepositoryList in scheme", }, { name: "success on existing repo", namespace: "fake-ns", bsl: "fake-bsl", repositoryType: "fake-repo-type", kubeClientObj: []runtime.Object{ bkRepoObjReady, }, runtimeScheme: scheme, expectedRepo: bkRepoObjReady, }, { name: "wait existing repo fail", namespace: "fake-ns", bsl: "fake-bsl", repositoryType: "fake-repo-type", kubeClientObj: []runtime.Object{ bkRepoObjNotReady, }, runtimeScheme: scheme, err: "failed to wait BackupRepository, timeout exceeded: backup repository not provisioned", }, { name: "create fail", namespace: "fake-ns", bsl: "fake-bsl", repositoryType: "fake-repo-type", runtimeScheme: scheme, err: "failed to wait BackupRepository, timeout exceeded: backup repository not provisioned", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeClientBuilder := fake.NewClientBuilder() if test.runtimeScheme != nil { fakeClientBuilder = fakeClientBuilder.WithScheme(test.runtimeScheme) } fakeClient := fakeClientBuilder.WithRuntimeObjects(test.kubeClientObj...).Build() ensurer := NewEnsurer(fakeClient, velerotest.NewLogger(), time.Millisecond) repo, err := ensurer.EnsureRepo(t.Context(), velerov1.DefaultNamespace, test.namespace, test.bsl, test.repositoryType) if err != nil { require.ErrorContains(t, err, test.err) } else { require.NoError(t, err) } assert.Equal(t, test.expectedRepo, repo) }) } } func TestCreateBackupRepositoryAndWait(t *testing.T) { existingRepoReady := NewBackupRepository(velerov1.DefaultNamespace, BackupRepositoryKey{ VolumeNamespace: "fake-ns", BackupLocation: "fake-bsl", RepositoryType: "fake-repo-type", }) existingRepoReady.Status.Phase = velerov1.BackupRepositoryPhaseReady existingRepoNotReady := NewBackupRepository(velerov1.DefaultNamespace, BackupRepositoryKey{ VolumeNamespace: "fake-ns", BackupLocation: "fake-bsl", RepositoryType: "fake-repo-type", }) key := BackupRepositoryKey{ VolumeNamespace: "fake-ns", BackupLocation: "fake-bsl", RepositoryType: "fake-repo-type", } existingRepoWithUnexpectedName := &velerov1.BackupRepository{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1.DefaultNamespace, Name: "ake-ns-fake-bsl-fake-repo-type-xxx00", Labels: repoLabelsFromKey(key), }, Spec: velerov1.BackupRepositorySpec{ VolumeNamespace: key.VolumeNamespace, BackupStorageLocation: key.BackupLocation, RepositoryType: key.RepositoryType, }, } scheme := runtime.NewScheme() velerov1.AddToScheme(scheme) tests := []struct { name string namespace string bsl string repositoryType string kubeClientObj []runtime.Object runtimeScheme *runtime.Scheme expectedRepo *velerov1.BackupRepository err string }{ { name: "create fail", namespace: "fake-ns", bsl: "fake-bsl", repositoryType: "fake-repo-type", err: "unable to create backup repository resource: no kind is registered for the type v1.BackupRepository in scheme", }, { name: "get repo fail", namespace: "fake-ns", bsl: "fake-bsl", repositoryType: "fake-repo-type", kubeClientObj: []runtime.Object{ existingRepoWithUnexpectedName, }, runtimeScheme: scheme, err: "failed to wait BackupRepository, errored early: more than one BackupRepository found for workload namespace \"fake-ns\", backup storage location \"fake-bsl\", repository type \"fake-repo-type\"", }, { name: "wait repo fail", namespace: "fake-ns", bsl: "fake-bsl", repositoryType: "fake-repo-type", runtimeScheme: scheme, err: "failed to wait BackupRepository, timeout exceeded: backup repository not provisioned", }, { name: "repo already exists and ready", namespace: "fake-ns", bsl: "fake-bsl", repositoryType: "fake-repo-type", kubeClientObj: []runtime.Object{ existingRepoReady, }, runtimeScheme: scheme, expectedRepo: existingRepoReady, }, { name: "repo already exists but not ready", namespace: "fake-ns", bsl: "fake-bsl", repositoryType: "fake-repo-type", kubeClientObj: []runtime.Object{ existingRepoNotReady, }, runtimeScheme: scheme, err: "failed to wait BackupRepository, timeout exceeded: backup repository not provisioned", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeClientBuilder := fake.NewClientBuilder() if test.runtimeScheme != nil { fakeClientBuilder = fakeClientBuilder.WithScheme(test.runtimeScheme) } fakeClient := fakeClientBuilder.WithRuntimeObjects(test.kubeClientObj...).Build() ensurer := NewEnsurer(fakeClient, velerotest.NewLogger(), time.Millisecond) repo, err := ensurer.createBackupRepositoryAndWait(t.Context(), velerov1.DefaultNamespace, BackupRepositoryKey{ VolumeNamespace: test.namespace, BackupLocation: test.bsl, RepositoryType: test.repositoryType, }) if err != nil { require.ErrorContains(t, err, test.err) } else { require.NoError(t, err) } assert.Equal(t, test.expectedRepo, repo) }) } } ================================================ FILE: pkg/repository/keys/keys.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ //nolint:gosec // Internal call. No need to check. package keys import ( "context" "github.com/pkg/errors" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" "github.com/vmware-tanzu/velero/pkg/builder" ) const ( credentialsSecretName = "velero-repo-credentials" credentialsKey = "repository-password" encryptionKey = "static-passw0rd" ) func EnsureCommonRepositoryKey(secretClient corev1client.SecretsGetter, namespace string) error { _, err := secretClient.Secrets(namespace).Get(context.TODO(), credentialsSecretName, metav1.GetOptions{}) if err != nil && !apierrors.IsNotFound(err) { return errors.WithStack(err) } if err == nil { return nil } // if we got here, we got an IsNotFound error, so we need to create the key secret := &corev1api.Secret{ ObjectMeta: metav1.ObjectMeta{ Namespace: namespace, Name: credentialsSecretName, }, Type: corev1api.SecretTypeOpaque, Data: map[string][]byte{ credentialsKey: []byte(encryptionKey), }, } if _, err = secretClient.Secrets(namespace).Create(context.TODO(), secret, metav1.CreateOptions{}); err != nil { return errors.Wrapf(err, "error creating %s secret", credentialsSecretName) } return nil } // RepoKeySelector returns the SecretKeySelector which can be used to fetch // the backup repository key. func RepoKeySelector() *corev1api.SecretKeySelector { // For now, all backup repos share the same key so we don't need the repoName to fetch it. // When we move to full-backup encryption, we'll likely have a separate key per backup repo // (all within the Velero server's namespace) so RepoKeySelector will need to select the key // for that repo. return builder.ForSecretKeySelector(credentialsSecretName, credentialsKey).Result() } ================================================ FILE: pkg/repository/keys/keys_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package keys import ( "testing" "github.com/stretchr/testify/require" ) func TestRepoKeySelector(t *testing.T) { selector := RepoKeySelector() require.Equal(t, credentialsSecretName, selector.Name) require.Equal(t, credentialsKey, selector.Key) } ================================================ FILE: pkg/repository/locker.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package repository import "sync" // RepoLocker manages exclusive/non-exclusive locks for // operations against backup repositories. The semantics // of exclusive/non-exclusive locks are the same as for // a sync.RWMutex, where a non-exclusive lock is equivalent // to a read lock, and an exclusive lock is equivalent to // a write lock. type RepoLocker struct { mu sync.Mutex locks map[string]*sync.RWMutex } func NewRepoLocker() *RepoLocker { return &RepoLocker{ locks: make(map[string]*sync.RWMutex), } } // LockExclusive acquires an exclusive lock for the specified // repository. This function blocks until no other locks exist // for the repo. func (rl *RepoLocker) LockExclusive(name string) { rl.ensureLock(name).Lock() } // Lock acquires a non-exclusive lock for the specified // repository. This function blocks until no exclusive // locks exist for the repo. func (rl *RepoLocker) Lock(name string) { rl.ensureLock(name).RLock() } // UnlockExclusive releases an exclusive lock for the repo. func (rl *RepoLocker) UnlockExclusive(name string) { rl.ensureLock(name).Unlock() } // Unlock releases a non-exclusive lock for the repo. func (rl *RepoLocker) Unlock(name string) { rl.ensureLock(name).RUnlock() } func (rl *RepoLocker) ensureLock(name string) *sync.RWMutex { rl.mu.Lock() defer rl.mu.Unlock() if _, ok := rl.locks[name]; !ok { rl.locks[name] = new(sync.RWMutex) } return rl.locks[name] } ================================================ FILE: pkg/repository/maintenance/maintenance.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package maintenance import ( "context" "encoding/json" "fmt" "math" "sort" "strings" "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" appsv1api "k8s.io/api/apps/v1" batchv1api "k8s.io/api/batch/v1" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/constant" velerolabel "github.com/vmware-tanzu/velero/pkg/label" velerotypes "github.com/vmware-tanzu/velero/pkg/types" "github.com/vmware-tanzu/velero/pkg/util" "github.com/vmware-tanzu/velero/pkg/util/kube" "github.com/vmware-tanzu/velero/pkg/util/logging" veleroutil "github.com/vmware-tanzu/velero/pkg/util/velero" ) const ( RepositoryNameLabel = "velero.io/repo-name" GlobalKeyForRepoMaintenanceJobCM = "global" TerminationLogIndicator = "Repo maintenance error: " DefaultKeepLatestMaintenanceJobs = 3 DefaultMaintenanceJobCPURequest = "0" DefaultMaintenanceJobCPULimit = "0" DefaultMaintenanceJobMemRequest = "0" DefaultMaintenanceJobMemLimit = "0" ) func GenerateJobName(repo string) string { millisecond := time.Now().UTC().UnixMilli() // millisecond jobName := fmt.Sprintf("%s-maintain-job-%d", repo, millisecond) if len(jobName) > 63 { // k8s job name length limit jobName = fmt.Sprintf("repo-maintain-job-%d", millisecond) } return jobName } // DeleteOldJobs deletes old maintenance jobs and keeps the latest N jobs func DeleteOldJobs(cli client.Client, repo velerov1api.BackupRepository, keep int, logger logrus.FieldLogger) error { logger.Infof("Start to delete old maintenance jobs. %d jobs will be kept.", keep) // Get the maintenance job list by label jobList := &batchv1api.JobList{} err := cli.List( context.TODO(), jobList, &client.ListOptions{ Namespace: repo.Namespace, LabelSelector: labels.SelectorFromSet( map[string]string{ RepositoryNameLabel: velerolabel.ReturnNameOrHash(repo.Name), }, ), }, ) if err != nil { return err } // Delete old maintenance jobs if len(jobList.Items) > keep { sort.Slice(jobList.Items, func(i, j int) bool { return jobList.Items[i].CreationTimestamp.Before(&jobList.Items[j].CreationTimestamp) }) for i := 0; i < len(jobList.Items)-keep; i++ { err = cli.Delete(context.TODO(), &jobList.Items[i], client.PropagationPolicy(metav1.DeletePropagationBackground)) if err != nil { return err } } } return nil } var waitCompletionBackOff = wait.Backoff{ Duration: time.Minute * 20, Steps: math.MaxInt, Factor: 2, Cap: time.Hour * 12, } // waitForJobComplete wait for completion of the specified job and update the latest job object func waitForJobComplete(ctx context.Context, client client.Client, ns string, job string, logger logrus.FieldLogger) (*batchv1api.Job, error) { var ret *batchv1api.Job backOff := waitCompletionBackOff startTime := time.Now() nextCheckpoint := startTime.Add(backOff.Step()) err := wait.PollUntilContextCancel(ctx, time.Second, true, func(ctx context.Context) (bool, error) { updated := &batchv1api.Job{} err := client.Get(ctx, types.NamespacedName{Namespace: ns, Name: job}, updated) if err != nil && !apierrors.IsNotFound(err) { return false, err } ret = updated if updated.Status.Succeeded > 0 { return true, nil } if updated.Status.Failed > 0 { return true, nil } now := time.Now() if now.After(nextCheckpoint) { logger.Warnf("Repo maintenance job %s has lasted %v minutes", job, now.Sub(startTime).Minutes()) nextCheckpoint = now.Add(backOff.Step()) } return false, nil }) return ret, err } func getResultFromJob(cli client.Client, job *batchv1api.Job) (string, error) { // Get the maintenance job related pod by label selector podList := &corev1api.PodList{} err := cli.List(context.TODO(), podList, client.InNamespace(job.Namespace), client.MatchingLabels(map[string]string{"job-name": job.Name})) if err != nil { return "", err } if len(podList.Items) == 0 { return "", errors.Errorf("no pod found for job %s", job.Name) } // we only have one maintenance pod for the job pod := podList.Items[0] statuses := pod.Status.ContainerStatuses if len(statuses) == 0 { return "", errors.Errorf("no container statuses found for job %s", job.Name) } // we only have one maintenance container terminated := statuses[0].State.Terminated if terminated == nil { return "", errors.Errorf("container for job %s is not terminated", job.Name) } if terminated.Message == "" { return "", nil } idx := strings.Index(terminated.Message, TerminationLogIndicator) if idx == -1 { return "", errors.New("error to locate repo maintenance error indicator from termination message") } if idx+len(TerminationLogIndicator) >= len(terminated.Message) { return "", errors.New("nothing after repo maintenance error indicator in termination message") } return terminated.Message[idx+len(TerminationLogIndicator):], nil } // getJobConfig is called to get the Maintenance Job Config for the // BackupRepository specified by the repo parameter. // // Params: // // ctx: the Go context used for controller-runtime client. // client: the controller-runtime client. // logger: the logger. // veleroNamespace: the Velero-installed namespace. It's used to retrieve the BackupRepository. // repoMaintenanceJobConfig: the repository maintenance job ConfigMap name. // repo: the BackupRepository needs to run the maintenance Job. func getJobConfig( ctx context.Context, client client.Client, logger logrus.FieldLogger, veleroNamespace string, repoMaintenanceJobConfig string, repo *velerov1api.BackupRepository, ) (*velerotypes.JobConfigs, error) { var cm corev1api.ConfigMap if err := client.Get( ctx, types.NamespacedName{ Namespace: veleroNamespace, Name: repoMaintenanceJobConfig, }, &cm, ); err != nil { if apierrors.IsNotFound(err) { return nil, nil } else { return nil, errors.Wrapf( err, "fail to get repo maintenance job configs %s", repoMaintenanceJobConfig) } } if cm.Data == nil { return nil, errors.Errorf("data is not available in config map %s", repoMaintenanceJobConfig) } // Generate the BackupRepository key. // If using the BackupRepository name as the is more intuitive, // but the BackupRepository generation is dynamic. We cannot assume // they are ready when installing Velero. // Instead we use the volume source namespace, BSL name, and the uploader // type to represent the BackupRepository. The combination of those three // keys can identify a unique BackupRepository. repoJobConfigKey := repo.Spec.VolumeNamespace + "-" + repo.Spec.BackupStorageLocation + "-" + repo.Spec.RepositoryType var result *velerotypes.JobConfigs if _, ok := cm.Data[repoJobConfigKey]; ok { logger.Debugf("Find the repo maintenance config %s for repo %s", repoJobConfigKey, repo.Name) result = new(velerotypes.JobConfigs) if err := json.Unmarshal([]byte(cm.Data[repoJobConfigKey]), result); err != nil { return nil, errors.Wrapf( err, "fail to unmarshal configs from %s's key %s", repoMaintenanceJobConfig, repoJobConfigKey) } } if _, ok := cm.Data[GlobalKeyForRepoMaintenanceJobCM]; ok { logger.Debugf("Find the global repo maintenance config for repo %s", repo.Name) if result == nil { result = new(velerotypes.JobConfigs) } globalResult := new(velerotypes.JobConfigs) if err := json.Unmarshal([]byte(cm.Data[GlobalKeyForRepoMaintenanceJobCM]), globalResult); err != nil { return nil, errors.Wrapf( err, "fail to unmarshal configs from %s's key %s", repoMaintenanceJobConfig, GlobalKeyForRepoMaintenanceJobCM) } if result.PodResources == nil && globalResult.PodResources != nil { result.PodResources = globalResult.PodResources } if len(result.LoadAffinities) == 0 { result.LoadAffinities = globalResult.LoadAffinities } if result.KeepLatestMaintenanceJobs == nil && globalResult.KeepLatestMaintenanceJobs != nil { result.KeepLatestMaintenanceJobs = globalResult.KeepLatestMaintenanceJobs } // Priority class is only read from global config, not per-repository if globalResult.PriorityClassName != "" { result.PriorityClassName = globalResult.PriorityClassName } // Pod's labels are only read from global config, not per-repository if len(globalResult.PodLabels) > 0 { result.PodLabels = globalResult.PodLabels } // Pod's annotations are only read from global config, not per-repository if len(globalResult.PodAnnotations) > 0 { result.PodAnnotations = globalResult.PodAnnotations } } logger.Debugf("Configuration content for repository %s is %+v", repo.Name, result) return result, nil } // GetKeepLatestMaintenanceJobs returns the configured number of maintenance jobs to keep from the JobConfigs. // Because the CLI configured Job kept number is deprecated, // if not configured in the ConfigMap, it returns default value to indicate using the fallback value. func GetKeepLatestMaintenanceJobs( ctx context.Context, client client.Client, logger logrus.FieldLogger, veleroNamespace string, repoMaintenanceJobConfig string, repo *velerov1api.BackupRepository, ) (int, error) { if repoMaintenanceJobConfig == "" { return DefaultKeepLatestMaintenanceJobs, nil } config, err := getJobConfig(ctx, client, logger, veleroNamespace, repoMaintenanceJobConfig, repo) if err != nil { return DefaultKeepLatestMaintenanceJobs, err } if config != nil && config.KeepLatestMaintenanceJobs != nil { return *config.KeepLatestMaintenanceJobs, nil } return DefaultKeepLatestMaintenanceJobs, nil } // WaitJobComplete waits the completion of the specified maintenance job and return the BackupRepositoryMaintenanceStatus func WaitJobComplete(cli client.Client, ctx context.Context, jobName, ns string, logger logrus.FieldLogger) (velerov1api.BackupRepositoryMaintenanceStatus, error) { log := logger.WithField("job name", jobName) maintenanceJob, err := waitForJobComplete(ctx, cli, ns, jobName, logger) if err != nil { return velerov1api.BackupRepositoryMaintenanceStatus{}, errors.Wrap(err, "error to wait for maintenance job complete") } log.Infof("Maintenance repo complete, succeeded %v, failed %v", maintenanceJob.Status.Succeeded, maintenanceJob.Status.Failed) result := "" if maintenanceJob.Status.Failed > 0 { if r, err := getResultFromJob(cli, maintenanceJob); err != nil { log.WithError(err).Warn("Failed to get maintenance job result") result = "Repo maintenance failed but result is not retrieveable" } else { result = r } } return composeStatusFromJob(maintenanceJob, result), nil } // WaitAllJobsComplete checks all the incomplete maintenance jobs of the specified repo and wait for them to complete, // and then return the maintenance jobs' status in the range of limit func WaitAllJobsComplete(ctx context.Context, cli client.Client, repo *velerov1api.BackupRepository, limit int, log logrus.FieldLogger) ([]velerov1api.BackupRepositoryMaintenanceStatus, error) { jobList := &batchv1api.JobList{} err := cli.List( context.TODO(), jobList, &client.ListOptions{ Namespace: repo.Namespace, LabelSelector: labels.SelectorFromSet( map[string]string{ RepositoryNameLabel: velerolabel.ReturnNameOrHash(repo.Name), }, ), }, ) if err != nil { return nil, errors.Wrapf(err, "error listing maintenance job for repo %s", repo.Name) } if len(jobList.Items) == 0 { return nil, nil } sort.Slice(jobList.Items, func(i, j int) bool { return jobList.Items[i].CreationTimestamp.Time.Before(jobList.Items[j].CreationTimestamp.Time) }) history := []velerov1api.BackupRepositoryMaintenanceStatus{} startPos := len(jobList.Items) - limit if startPos < 0 { startPos = 0 } for i := startPos; i < len(jobList.Items); i++ { job := &jobList.Items[i] if job.Status.Succeeded == 0 && job.Status.Failed == 0 { log.Infof("Waiting for maintenance job %s to complete", job.Name) updated, err := waitForJobComplete(ctx, cli, job.Namespace, job.Name, log) if err != nil { return nil, errors.Wrapf(err, "error waiting maintenance job[%s] complete", job.Name) } job = updated } message := "" if job.Status.Failed > 0 { if msg, err := getResultFromJob(cli, job); err != nil { log.WithError(err).Warnf("Failed to get result of maintenance job %s", job.Name) message = fmt.Sprintf("Repo maintenance failed but result is not retrieveable, err: %v", err) } else { message = msg } } history = append(history, composeStatusFromJob(job, message)) } return history, nil } // StartNewJob creates a new maintenance job func StartNewJob( cli client.Client, ctx context.Context, repo *velerov1api.BackupRepository, repoMaintenanceJobConfig string, logLevel logrus.Level, logFormat *logging.FormatFlag, logger logrus.FieldLogger, ) (string, error) { bsl := &velerov1api.BackupStorageLocation{} if err := cli.Get(ctx, client.ObjectKey{Namespace: repo.Namespace, Name: repo.Spec.BackupStorageLocation}, bsl); err != nil { return "", errors.WithStack(err) } log := logger.WithFields(logrus.Fields{ "BSL name": bsl.Name, "repo type": repo.Spec.RepositoryType, "repo name": repo.Name, "repo UID": repo.UID, }) jobConfig, err := getJobConfig( ctx, cli, log, repo.Namespace, repoMaintenanceJobConfig, repo, ) if err != nil { log.Warnf("Fail to find the ConfigMap %s to build maintenance job with error: %s. Use default value.", repo.Namespace+"/"+repoMaintenanceJobConfig, err.Error(), ) } log.Info("Starting maintenance repo") maintenanceJob, err := buildJob(cli, ctx, repo, bsl.Name, jobConfig, logLevel, logFormat, log) if err != nil { return "", errors.Wrap(err, "error to build maintenance job") } log = log.WithField("job", fmt.Sprintf("%s/%s", maintenanceJob.Namespace, maintenanceJob.Name)) if err := cli.Create(ctx, maintenanceJob); err != nil { return "", errors.Wrap(err, "error to create maintenance job") } log.Info("Repo maintenance job started") return maintenanceJob.Name, nil } // buildTolerationsForMaintenanceJob builds the tolerations for maintenance jobs. // It includes the required Windows toleration for backward compatibility and filters // tolerations from the Velero deployment to only include those with keys that are // in the ThirdPartyTolerations allowlist, following the same pattern as labels and annotations. func buildTolerationsForMaintenanceJob(deployment *appsv1api.Deployment) []corev1api.Toleration { // Start with the Windows toleration for backward compatibility windowsToleration := corev1api.Toleration{ Key: "os", Operator: "Equal", Effect: "NoSchedule", Value: "windows", } result := []corev1api.Toleration{windowsToleration} // Filter tolerations from the Velero deployment to only include allowed ones // Only tolerations that exist on the deployment AND have keys in the allowlist are inherited deploymentTolerations := veleroutil.GetTolerationsFromVeleroServer(deployment) for _, k := range util.ThirdPartyTolerations { for _, toleration := range deploymentTolerations { if toleration.Key == k { result = append(result, toleration) break // Only add the first matching toleration for each allowed key } } } return result } func getPriorityClassName(ctx context.Context, cli client.Client, config *velerotypes.JobConfigs, logger logrus.FieldLogger) string { // Use the priority class name from the global job configuration if available // Note: Priority class is only read from global config, not per-repository if config != nil && config.PriorityClassName != "" { // Validate that the priority class exists in the cluster if err := kube.ValidatePriorityClassWithClient(ctx, cli, config.PriorityClassName); err != nil { if apierrors.IsNotFound(err) { logger.Warnf("Priority class %q not found in cluster. Job creation may fail if the priority class doesn't exist when jobs are scheduled.", config.PriorityClassName) } else { logger.WithError(err).Warnf("Failed to validate priority class %q", config.PriorityClassName) } // Still return the priority class name to let Kubernetes handle the error return config.PriorityClassName } logger.Infof("Validated priority class %q exists in cluster", config.PriorityClassName) return config.PriorityClassName } return "" } func buildJob( cli client.Client, ctx context.Context, repo *velerov1api.BackupRepository, bslName string, config *velerotypes.JobConfigs, logLevel logrus.Level, logFormat *logging.FormatFlag, logger logrus.FieldLogger, ) (*batchv1api.Job, error) { // Get the Velero server deployment deployment := &appsv1api.Deployment{} err := cli.Get(ctx, types.NamespacedName{Name: "velero", Namespace: repo.Namespace}, deployment) if err != nil { return nil, err } // Get the environment variables from the Velero server deployment envVars := veleroutil.GetEnvVarsFromVeleroServer(deployment) // Get the referenced storage from the Velero server deployment envFromSources := veleroutil.GetEnvFromSourcesFromVeleroServer(deployment) // Get the volume mounts from the Velero server deployment volumeMounts := veleroutil.GetVolumeMountsFromVeleroServer(deployment) // Get the volumes from the Velero server deployment volumes := veleroutil.GetVolumesFromVeleroServer(deployment) // Get the service account from the Velero server deployment serviceAccount := veleroutil.GetServiceAccountFromVeleroServer(deployment) // Get the security context from the Velero server deployment securityContext := veleroutil.GetContainerSecurityContextsFromVeleroServer(deployment) // Get the pod security context from the Velero server deployment podSecurityContext := veleroutil.GetPodSecurityContextsFromVeleroServer(deployment) imagePullSecrets := veleroutil.GetImagePullSecretsFromVeleroServer(deployment) // Get image image := veleroutil.GetVeleroServerImage(deployment) // Set resource limits and requests cpuRequest := DefaultMaintenanceJobCPURequest memRequest := DefaultMaintenanceJobMemRequest ephemeralStorageRequest := constant.DefaultEphemeralStorageRequest cpuLimit := DefaultMaintenanceJobCPULimit memLimit := DefaultMaintenanceJobMemLimit ephemeralStorageLimit := constant.DefaultEphemeralStorageLimit if config != nil && config.PodResources != nil { cpuRequest = config.PodResources.CPURequest memRequest = config.PodResources.MemoryRequest cpuLimit = config.PodResources.CPULimit memLimit = config.PodResources.MemoryLimit // To make the PodResources ConfigMap without ephemeral storage request/limit backward compatible, // need to avoid set value as empty, because empty string will cause parsing error. if config.PodResources.EphemeralStorageRequest != "" { ephemeralStorageRequest = config.PodResources.EphemeralStorageRequest } if config.PodResources.EphemeralStorageLimit != "" { ephemeralStorageLimit = config.PodResources.EphemeralStorageLimit } } resources, err := kube.ParseResourceRequirements( cpuRequest, memRequest, ephemeralStorageRequest, cpuLimit, memLimit, ephemeralStorageLimit, ) if err != nil { return nil, errors.Wrap(err, "failed to parse resource requirements for maintenance job") } podLabels := map[string]string{ RepositoryNameLabel: velerolabel.ReturnNameOrHash(repo.Name), } if config != nil && len(config.PodLabels) > 0 { for k, v := range config.PodLabels { podLabels[k] = v } } else { for _, k := range util.ThirdPartyLabels { if v := veleroutil.GetVeleroServerLabelValue(deployment, k); v != "" { podLabels[k] = v } } } podAnnotations := map[string]string{} if config != nil && len(config.PodAnnotations) > 0 { for k, v := range config.PodAnnotations { podAnnotations[k] = v } } else { for _, k := range util.ThirdPartyAnnotations { if v := veleroutil.GetVeleroServerAnnotationValue(deployment, k); v != "" { podAnnotations[k] = v } } } // Set arguments args := []string{"repo-maintenance"} args = append(args, fmt.Sprintf("--repo-name=%s", repo.Spec.VolumeNamespace)) args = append(args, fmt.Sprintf("--repo-type=%s", repo.Spec.RepositoryType)) args = append(args, fmt.Sprintf("--backup-storage-location=%s", bslName)) args = append(args, fmt.Sprintf("--log-level=%s", logLevel.String())) args = append(args, fmt.Sprintf("--log-format=%s", logFormat.String())) // build the maintenance job job := &batchv1api.Job{ ObjectMeta: metav1.ObjectMeta{ Name: GenerateJobName(repo.Name), Namespace: repo.Namespace, Labels: map[string]string{ RepositoryNameLabel: velerolabel.ReturnNameOrHash(repo.Name), }, }, Spec: batchv1api.JobSpec{ BackoffLimit: new(int32), // Never retry Template: corev1api.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Name: "velero-repo-maintenance-pod", Labels: podLabels, Annotations: podAnnotations, }, Spec: corev1api.PodSpec{ Containers: []corev1api.Container{ { Name: "velero-repo-maintenance-container", Image: image, Command: []string{ "/velero", }, Args: args, ImagePullPolicy: corev1api.PullIfNotPresent, Env: envVars, EnvFrom: envFromSources, VolumeMounts: volumeMounts, Resources: resources, SecurityContext: securityContext, TerminationMessagePolicy: corev1api.TerminationMessageFallbackToLogsOnError, }, }, PriorityClassName: getPriorityClassName(ctx, cli, config, logger), RestartPolicy: corev1api.RestartPolicyNever, SecurityContext: podSecurityContext, Volumes: volumes, ServiceAccountName: serviceAccount, Tolerations: buildTolerationsForMaintenanceJob(deployment), ImagePullSecrets: imagePullSecrets, }, }, }, } if config != nil && len(config.LoadAffinities) > 0 { affinity := kube.ToSystemAffinity(config.LoadAffinities[0], nil) job.Spec.Template.Spec.Affinity = affinity } return job, nil } func composeStatusFromJob(job *batchv1api.Job, message string) velerov1api.BackupRepositoryMaintenanceStatus { result := velerov1api.BackupRepositoryMaintenanceSucceeded if job.Status.Failed > 0 { result = velerov1api.BackupRepositoryMaintenanceFailed } return velerov1api.BackupRepositoryMaintenanceStatus{ Result: result, StartTimestamp: &job.CreationTimestamp, CompleteTimestamp: job.Status.CompletionTime, Message: message, } } ================================================ FILE: pkg/repository/maintenance/maintenance_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package maintenance import ( "context" "fmt" "math" "strings" "testing" "time" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" appsv1api "k8s.io/api/apps/v1" batchv1api "k8s.io/api/batch/v1" corev1api "k8s.io/api/core/v1" schedulingv1 "k8s.io/api/scheduling/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/wait" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" velerolabel "github.com/vmware-tanzu/velero/pkg/label" "github.com/vmware-tanzu/velero/pkg/repository/provider" velerotest "github.com/vmware-tanzu/velero/pkg/test" velerotypes "github.com/vmware-tanzu/velero/pkg/types" "github.com/vmware-tanzu/velero/pkg/util/boolptr" "github.com/vmware-tanzu/velero/pkg/util/kube" "github.com/vmware-tanzu/velero/pkg/util/logging" ) func TestGenerateJobName(t *testing.T) { testCases := []struct { repo string expectedStart string }{ { repo: "example", expectedStart: "example-maintain-job-", }, { repo: strings.Repeat("a", 60), expectedStart: "repo-maintain-job-", }, } for _, tc := range testCases { t.Run(tc.repo, func(t *testing.T) { // Call the function to test jobName := GenerateJobName(tc.repo) // Check if the generated job name starts with the expected prefix if !strings.HasPrefix(jobName, tc.expectedStart) { t.Errorf("generated job name does not start with expected prefix") } // Check if the length of the generated job name exceeds the Kubernetes limit if len(jobName) > 63 { t.Errorf("generated job name exceeds Kubernetes limit") } }) } } func TestDeleteOldJobs(t *testing.T) { // Set up test repo and keep value repo := &velerov1api.BackupRepository{ ObjectMeta: metav1.ObjectMeta{ Name: "label with more than 63 characters should be modified", Namespace: velerov1api.DefaultNamespace, }, } keep := 1 jobArray := []client.Object{ &batchv1api.Job{ ObjectMeta: metav1.ObjectMeta{ Name: "job-0", Namespace: velerov1api.DefaultNamespace, Labels: map[string]string{RepositoryNameLabel: velerolabel.ReturnNameOrHash(repo.Name)}, }, Spec: batchv1api.JobSpec{}, }, &batchv1api.Job{ ObjectMeta: metav1.ObjectMeta{ Name: "job-1", Namespace: velerov1api.DefaultNamespace, Labels: map[string]string{RepositoryNameLabel: velerolabel.ReturnNameOrHash(repo.Name)}, }, Spec: batchv1api.JobSpec{}, }, } newJob := &batchv1api.Job{ ObjectMeta: metav1.ObjectMeta{ Name: "job-new", Namespace: velerov1api.DefaultNamespace, Labels: map[string]string{RepositoryNameLabel: velerolabel.ReturnNameOrHash(repo.Name)}, }, Spec: batchv1api.JobSpec{}, } // Create a fake Kubernetes client with 2 jobs. scheme := runtime.NewScheme() _ = batchv1api.AddToScheme(scheme) cli := fake.NewClientBuilder().WithScheme(scheme).WithObjects(jobArray...).Build() // Create a new job require.NoError(t, cli.Create(t.Context(), newJob)) // Call the function require.NoError(t, DeleteOldJobs(cli, *repo, keep, velerotest.NewLogger())) // Get the remaining jobs jobList := &batchv1api.JobList{} require.NoError(t, cli.List(t.Context(), jobList, client.MatchingLabels(map[string]string{RepositoryNameLabel: repo.Name}))) // We expect the number of jobs to be equal to 'keep' assert.Len(t, jobList.Items, keep) // Only the new created job should be left. assert.Equal(t, jobList.Items[0].Name, newJob.Name) } func TestWaitForJobComplete(t *testing.T) { // Set up test job job := &batchv1api.Job{ ObjectMeta: metav1.ObjectMeta{ Name: "test-job", Namespace: "default", }, Status: batchv1api.JobStatus{}, } schemeFail := runtime.NewScheme() scheme := runtime.NewScheme() batchv1api.AddToScheme(scheme) waitCompletionBackOff1 := wait.Backoff{ Duration: time.Second, Steps: math.MaxInt, Factor: 2, Cap: time.Second * 12, } waitCompletionBackOff2 := wait.Backoff{ Duration: time.Second, Steps: math.MaxInt, Factor: 2, Cap: time.Second * 2, } // Define test cases tests := []struct { description string // Test case description kubeClientObj []runtime.Object runtimeScheme *runtime.Scheme jobStatus batchv1api.JobStatus // Job status to set for the test logBackOff wait.Backoff updateAfter time.Duration expectedLogs int expectError bool // Whether an error is expected }{ { description: "wait error", runtimeScheme: schemeFail, expectError: true, }, { description: "Job Succeeded", runtimeScheme: scheme, kubeClientObj: []runtime.Object{ job, }, jobStatus: batchv1api.JobStatus{ Succeeded: 1, }, expectError: false, }, { description: "Job Failed", runtimeScheme: scheme, kubeClientObj: []runtime.Object{ job, }, jobStatus: batchv1api.JobStatus{ Failed: 1, }, expectError: false, }, { description: "Log backoff not to cap", runtimeScheme: scheme, kubeClientObj: []runtime.Object{ job, }, logBackOff: waitCompletionBackOff1, updateAfter: time.Second * 8, expectedLogs: 3, }, { description: "Log backoff to cap", runtimeScheme: scheme, kubeClientObj: []runtime.Object{ job, }, logBackOff: waitCompletionBackOff2, updateAfter: time.Second * 6, expectedLogs: 3, }, } // Run tests for _, tc := range tests { t.Run(tc.description, func(t *testing.T) { // Set the job status job.Status = tc.jobStatus // Create a fake Kubernetes client fakeClientBuilder := fake.NewClientBuilder() fakeClientBuilder = fakeClientBuilder.WithScheme(tc.runtimeScheme) fakeClient := fakeClientBuilder.WithRuntimeObjects(tc.kubeClientObj...).Build() buffer := []string{} logger := velerotest.NewMultipleLogger(&buffer) waitCompletionBackOff = tc.logBackOff if tc.updateAfter != 0 { go func() { time.Sleep(tc.updateAfter) original := job.DeepCopy() job.Status.Succeeded = 1 err := fakeClient.Status().Patch(t.Context(), job, client.MergeFrom(original)) require.NoError(t, err) }() } // Call the function _, err := waitForJobComplete(t.Context(), fakeClient, job.Namespace, job.Name, logger) // Check if the error matches the expectation if tc.expectError { require.Error(t, err) } else { require.NoError(t, err) } assert.LessOrEqual(t, len(buffer), tc.expectedLogs) }) } } func TestGetResultFromJob(t *testing.T) { // Set up test job job := &batchv1api.Job{ ObjectMeta: metav1.ObjectMeta{ Name: "test-job", Namespace: "default", }, } // Set up test pod with no status pod := &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "test-pod", Namespace: "default", Labels: map[string]string{"job-name": job.Name}, }, } // Create a fake Kubernetes client cli := fake.NewClientBuilder().Build() // test an error should be returned result, err := getResultFromJob(cli, job) require.EqualError(t, err, "no pod found for job test-job") assert.Empty(t, result) cli = fake.NewClientBuilder().WithObjects(job, pod).Build() // test an error should be returned result, err = getResultFromJob(cli, job) require.EqualError(t, err, "no container statuses found for job test-job") assert.Empty(t, result) // Set a non-terminated container status to the pod pod.Status = corev1api.PodStatus{ ContainerStatuses: []corev1api.ContainerStatus{ { State: corev1api.ContainerState{}, }, }, } // Test an error should be returned cli = fake.NewClientBuilder().WithObjects(job, pod).Build() result, err = getResultFromJob(cli, job) require.EqualError(t, err, "container for job test-job is not terminated") assert.Empty(t, result) // Set a terminated container status to the pod pod.Status = corev1api.PodStatus{ ContainerStatuses: []corev1api.ContainerStatus{ { State: corev1api.ContainerState{ Terminated: &corev1api.ContainerStateTerminated{}, }, }, }, } // This call should return the termination message with no error cli = fake.NewClientBuilder().WithObjects(job, pod).Build() result, err = getResultFromJob(cli, job) require.NoError(t, err) assert.Empty(t, result) // Set a terminated container status with invalidate message to the pod pod.Status = corev1api.PodStatus{ ContainerStatuses: []corev1api.ContainerStatus{ { State: corev1api.ContainerState{ Terminated: &corev1api.ContainerStateTerminated{ Message: "fake-message", }, }, }, }, } cli = fake.NewClientBuilder().WithObjects(job, pod).Build() result, err = getResultFromJob(cli, job) require.EqualError(t, err, "error to locate repo maintenance error indicator from termination message") assert.Empty(t, result) // Set a terminated container status with empty maintenance error to the pod pod.Status = corev1api.PodStatus{ ContainerStatuses: []corev1api.ContainerStatus{ { State: corev1api.ContainerState{ Terminated: &corev1api.ContainerStateTerminated{ Message: "Repo maintenance error: ", }, }, }, }, } cli = fake.NewClientBuilder().WithObjects(job, pod).Build() result, err = getResultFromJob(cli, job) require.EqualError(t, err, "nothing after repo maintenance error indicator in termination message") assert.Empty(t, result) // Set a terminated container status with maintenance error to the pod pod.Status = corev1api.PodStatus{ ContainerStatuses: []corev1api.ContainerStatus{ { State: corev1api.ContainerState{ Terminated: &corev1api.ContainerStateTerminated{ Message: "Repo maintenance error: fake-error", }, }, }, }, } cli = fake.NewClientBuilder().WithObjects(job, pod).Build() result, err = getResultFromJob(cli, job) require.NoError(t, err) assert.Equal(t, "fake-error", result) } func TestGetJobConfig(t *testing.T) { keepLatestMaintenanceJobs := 1 ctx := t.Context() logger := logrus.New() veleroNamespace := "velero" repoMaintenanceJobConfig := "repo-maintenance-job-config" repo := &velerov1api.BackupRepository{ ObjectMeta: metav1.ObjectMeta{ Namespace: veleroNamespace, Name: repoMaintenanceJobConfig, }, Spec: velerov1api.BackupRepositorySpec{ BackupStorageLocation: "default", RepositoryType: "kopia", VolumeNamespace: "test", }, } testCases := []struct { name string repoJobConfig *corev1api.ConfigMap expectedConfig *velerotypes.JobConfigs expectedError error }{ { name: "Config not exist", expectedConfig: nil, expectedError: nil, }, { name: "Invalid JSON", repoJobConfig: &corev1api.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Namespace: veleroNamespace, Name: repoMaintenanceJobConfig, }, Data: map[string]string{ "test-default-kopia": "{\"cpuRequest:\"100m\"}", }, }, expectedConfig: nil, expectedError: fmt.Errorf("fail to unmarshal configs from %s", repoMaintenanceJobConfig), }, { name: "Find config specific for BackupRepository", repoJobConfig: &corev1api.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Namespace: veleroNamespace, Name: repoMaintenanceJobConfig, }, Data: map[string]string{ "test-default-kopia": "{\"podResources\":{\"cpuRequest\":\"100m\",\"cpuLimit\":\"200m\",\"memoryRequest\":\"100Mi\",\"memoryLimit\":\"200Mi\"},\"loadAffinity\":[{\"nodeSelector\":{\"matchExpressions\":[{\"key\":\"cloud.google.com/machine-family\",\"operator\":\"In\",\"values\":[\"e2\"]}]}}]}", }, }, expectedConfig: &velerotypes.JobConfigs{ PodResources: &kube.PodResources{ CPURequest: "100m", CPULimit: "200m", MemoryRequest: "100Mi", MemoryLimit: "200Mi", }, LoadAffinities: []*kube.LoadAffinity{ { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { Key: "cloud.google.com/machine-family", Operator: metav1.LabelSelectorOpIn, Values: []string{"e2"}, }, }, }, }, }, }, expectedError: nil, }, { name: "Find config specific for global", repoJobConfig: &corev1api.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Namespace: veleroNamespace, Name: repoMaintenanceJobConfig, }, Data: map[string]string{ GlobalKeyForRepoMaintenanceJobCM: "{\"podResources\":{\"cpuRequest\":\"50m\",\"cpuLimit\":\"100m\",\"memoryRequest\":\"50Mi\",\"memoryLimit\":\"100Mi\"},\"loadAffinity\":[{\"nodeSelector\":{\"matchExpressions\":[{\"key\":\"cloud.google.com/machine-family\",\"operator\":\"In\",\"values\":[\"n2\"]}]}}]}", }, }, expectedConfig: &velerotypes.JobConfigs{ PodResources: &kube.PodResources{ CPURequest: "50m", CPULimit: "100m", MemoryRequest: "50Mi", MemoryLimit: "100Mi", }, LoadAffinities: []*kube.LoadAffinity{ { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { Key: "cloud.google.com/machine-family", Operator: metav1.LabelSelectorOpIn, Values: []string{"n2"}, }, }, }, }, }, }, expectedError: nil, }, { name: "Specific config supersede global config", repoJobConfig: &corev1api.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Namespace: veleroNamespace, Name: repoMaintenanceJobConfig, }, Data: map[string]string{ GlobalKeyForRepoMaintenanceJobCM: "{\"keepLatestMaintenanceJobs\":1,\"podResources\":{\"cpuRequest\":\"50m\",\"cpuLimit\":\"100m\",\"memoryRequest\":\"50Mi\",\"memoryLimit\":\"100Mi\"},\"loadAffinity\":[{\"nodeSelector\":{\"matchExpressions\":[{\"key\":\"cloud.google.com/machine-family\",\"operator\":\"In\",\"values\":[\"n2\"]}]}}]}", "test-default-kopia": "{\"podResources\":{\"cpuRequest\":\"100m\",\"cpuLimit\":\"200m\",\"memoryRequest\":\"100Mi\",\"memoryLimit\":\"200Mi\"},\"loadAffinity\":[{\"nodeSelector\":{\"matchExpressions\":[{\"key\":\"cloud.google.com/machine-family\",\"operator\":\"In\",\"values\":[\"e2\"]}]}}]}", }, }, expectedConfig: &velerotypes.JobConfigs{ KeepLatestMaintenanceJobs: &keepLatestMaintenanceJobs, PodResources: &kube.PodResources{ CPURequest: "100m", CPULimit: "200m", MemoryRequest: "100Mi", MemoryLimit: "200Mi", }, LoadAffinities: []*kube.LoadAffinity{ { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { Key: "cloud.google.com/machine-family", Operator: metav1.LabelSelectorOpIn, Values: []string{"e2"}, }, }, }, }, }, }, expectedError: nil, }, { name: "Configs only exist in global section should supersede specific config", repoJobConfig: &corev1api.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Namespace: veleroNamespace, Name: repoMaintenanceJobConfig, }, Data: map[string]string{ GlobalKeyForRepoMaintenanceJobCM: "{\"keepLatestMaintenanceJobs\":1,\"podResources\":{\"cpuRequest\":\"50m\",\"cpuLimit\":\"100m\",\"memoryRequest\":\"50Mi\",\"memoryLimit\":\"100Mi\"},\"loadAffinity\":[{\"nodeSelector\":{\"matchExpressions\":[{\"key\":\"cloud.google.com/machine-family\",\"operator\":\"In\",\"values\":[\"n2\"]}]}}],\"priorityClassName\":\"global-priority\",\"podAnnotations\":{\"global-key\":\"global-value\"},\"podLabels\":{\"global-key\":\"global-value\"}}", "test-default-kopia": "{\"podResources\":{\"cpuRequest\":\"100m\",\"cpuLimit\":\"200m\",\"memoryRequest\":\"100Mi\",\"memoryLimit\":\"200Mi\"},\"loadAffinity\":[{\"nodeSelector\":{\"matchExpressions\":[{\"key\":\"cloud.google.com/machine-family\",\"operator\":\"In\",\"values\":[\"e2\"]}]}}],\"priorityClassName\":\"specific-priority\",\"podAnnotations\":{\"specific-key\":\"specific-value\"},\"podLabels\":{\"specific-key\":\"specific-value\"}}", }, }, expectedConfig: &velerotypes.JobConfigs{ KeepLatestMaintenanceJobs: &keepLatestMaintenanceJobs, PodResources: &kube.PodResources{ CPURequest: "100m", CPULimit: "200m", MemoryRequest: "100Mi", MemoryLimit: "200Mi", }, LoadAffinities: []*kube.LoadAffinity{ { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { Key: "cloud.google.com/machine-family", Operator: metav1.LabelSelectorOpIn, Values: []string{"e2"}, }, }, }, }, }, PriorityClassName: "global-priority", PodAnnotations: map[string]string{"global-key": "global-value"}, PodLabels: map[string]string{"global-key": "global-value"}, }, expectedError: nil, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { var fakeClient client.Client if tc.repoJobConfig != nil { fakeClient = velerotest.NewFakeControllerRuntimeClient(t, tc.repoJobConfig) } else { fakeClient = velerotest.NewFakeControllerRuntimeClient(t) } jobConfig, err := getJobConfig( ctx, fakeClient, logger, veleroNamespace, repoMaintenanceJobConfig, repo, ) if tc.expectedError != nil { require.ErrorContains(t, err, tc.expectedError.Error()) } else { require.NoError(t, err) } require.Equal(t, tc.expectedConfig, jobConfig) }) } } func TestWaitAllJobsComplete(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), time.Second*2) veleroNamespace := "velero" repo := &velerov1api.BackupRepository{ ObjectMeta: metav1.ObjectMeta{ Namespace: veleroNamespace, Name: "label with more than 63 characters should be modified", }, Spec: velerov1api.BackupRepositorySpec{ BackupStorageLocation: "default", RepositoryType: "kopia", VolumeNamespace: "test", }, } now := time.Now().Round(time.Second) jobOtherLabel := &batchv1api.Job{ ObjectMeta: metav1.ObjectMeta{ Name: "job1", Namespace: veleroNamespace, Labels: map[string]string{RepositoryNameLabel: "other-repo"}, CreationTimestamp: metav1.Time{Time: now}, }, } jobIncomplete := &batchv1api.Job{ ObjectMeta: metav1.ObjectMeta{ Name: "job1", Namespace: veleroNamespace, Labels: map[string]string{RepositoryNameLabel: velerolabel.ReturnNameOrHash(repo.Name)}, CreationTimestamp: metav1.Time{Time: now}, }, } jobSucceeded1 := &batchv1api.Job{ ObjectMeta: metav1.ObjectMeta{ Name: "job1", Namespace: veleroNamespace, Labels: map[string]string{RepositoryNameLabel: velerolabel.ReturnNameOrHash(repo.Name)}, CreationTimestamp: metav1.Time{Time: now}, }, Status: batchv1api.JobStatus{ StartTime: &metav1.Time{Time: now}, CompletionTime: &metav1.Time{Time: now.Add(time.Hour)}, Succeeded: 1, }, } jobPodSucceeded1 := builder.ForPod(veleroNamespace, "job1").Labels(map[string]string{"job-name": "job1"}).ContainerStatuses(&corev1api.ContainerStatus{ State: corev1api.ContainerState{ Terminated: &corev1api.ContainerStateTerminated{}, }, }).Result() jobFailed1 := &batchv1api.Job{ ObjectMeta: metav1.ObjectMeta{ Name: "job2", Namespace: veleroNamespace, Labels: map[string]string{RepositoryNameLabel: velerolabel.ReturnNameOrHash(repo.Name)}, CreationTimestamp: metav1.Time{Time: now.Add(time.Hour)}, }, Status: batchv1api.JobStatus{ StartTime: &metav1.Time{Time: now.Add(time.Hour)}, Failed: 1, }, } jobPodFailed1 := builder.ForPod(veleroNamespace, "job2").Labels(map[string]string{"job-name": "job2"}).ContainerStatuses(&corev1api.ContainerStatus{ State: corev1api.ContainerState{ Terminated: &corev1api.ContainerStateTerminated{ Message: "Repo maintenance error: fake-message-2", }, }, }).Result() jobSucceeded2 := &batchv1api.Job{ ObjectMeta: metav1.ObjectMeta{ Name: "job3", Namespace: veleroNamespace, Labels: map[string]string{RepositoryNameLabel: velerolabel.ReturnNameOrHash(repo.Name)}, CreationTimestamp: metav1.Time{Time: now.Add(time.Hour * 2)}, }, Status: batchv1api.JobStatus{ StartTime: &metav1.Time{Time: now.Add(time.Hour * 2)}, CompletionTime: &metav1.Time{Time: now.Add(time.Hour * 3)}, Succeeded: 1, }, } jobPodSucceeded2 := builder.ForPod(veleroNamespace, "job3").Labels(map[string]string{"job-name": "job3"}).ContainerStatuses(&corev1api.ContainerStatus{ State: corev1api.ContainerState{ Terminated: &corev1api.ContainerStateTerminated{}, }, }).Result() jobSucceeded3 := &batchv1api.Job{ ObjectMeta: metav1.ObjectMeta{ Name: "job4", Namespace: veleroNamespace, Labels: map[string]string{RepositoryNameLabel: velerolabel.ReturnNameOrHash(repo.Name)}, CreationTimestamp: metav1.Time{Time: now.Add(time.Hour * 3)}, }, Status: batchv1api.JobStatus{ StartTime: &metav1.Time{Time: now.Add(time.Hour * 3)}, CompletionTime: &metav1.Time{Time: now.Add(time.Hour * 4)}, Succeeded: 1, }, } jobPodSucceeded3 := builder.ForPod(veleroNamespace, "job4").Labels(map[string]string{"job-name": "job4"}).ContainerStatuses(&corev1api.ContainerStatus{ State: corev1api.ContainerState{ Terminated: &corev1api.ContainerStateTerminated{}, }, }).Result() schemeFail := runtime.NewScheme() scheme := runtime.NewScheme() batchv1api.AddToScheme(scheme) corev1api.AddToScheme(scheme) testCases := []struct { name string ctx context.Context kubeClientObj []runtime.Object runtimeScheme *runtime.Scheme expectedStatus []velerov1api.BackupRepositoryMaintenanceStatus expectedError string }{ { name: "list job error", runtimeScheme: schemeFail, expectedError: "error listing maintenance job for repo label with more than 63 characters should be modified: no kind is registered for the type v1.JobList in scheme", }, { name: "job not exist", runtimeScheme: scheme, }, { name: "no matching job", runtimeScheme: scheme, kubeClientObj: []runtime.Object{ jobOtherLabel, }, }, { name: "wait complete error", ctx: ctx, runtimeScheme: scheme, kubeClientObj: []runtime.Object{ jobIncomplete, }, expectedError: "error waiting maintenance job[job1] complete: context deadline exceeded", }, { name: "get result error on succeeded job", ctx: t.Context(), runtimeScheme: scheme, kubeClientObj: []runtime.Object{ jobSucceeded1, }, expectedStatus: []velerov1api.BackupRepositoryMaintenanceStatus{ { Result: velerov1api.BackupRepositoryMaintenanceSucceeded, StartTimestamp: &metav1.Time{Time: now}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour)}, }, }, }, { name: "get result error on failed job", ctx: t.Context(), runtimeScheme: scheme, kubeClientObj: []runtime.Object{ jobFailed1, }, expectedStatus: []velerov1api.BackupRepositoryMaintenanceStatus{ { Result: velerov1api.BackupRepositoryMaintenanceFailed, StartTimestamp: &metav1.Time{Time: now.Add(time.Hour)}, Message: "Repo maintenance failed but result is not retrieveable, err: no pod found for job job2", }, }, }, { name: "less than limit", ctx: t.Context(), runtimeScheme: scheme, kubeClientObj: []runtime.Object{ jobFailed1, jobSucceeded1, jobPodSucceeded1, jobPodFailed1, }, expectedStatus: []velerov1api.BackupRepositoryMaintenanceStatus{ { Result: velerov1api.BackupRepositoryMaintenanceSucceeded, StartTimestamp: &metav1.Time{Time: now}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour)}, }, { Result: velerov1api.BackupRepositoryMaintenanceFailed, StartTimestamp: &metav1.Time{Time: now.Add(time.Hour)}, Message: "fake-message-2", }, }, }, { name: "equal to limit", ctx: t.Context(), runtimeScheme: scheme, kubeClientObj: []runtime.Object{ jobSucceeded2, jobFailed1, jobSucceeded1, jobPodSucceeded1, jobPodFailed1, jobPodSucceeded2, }, expectedStatus: []velerov1api.BackupRepositoryMaintenanceStatus{ { Result: velerov1api.BackupRepositoryMaintenanceSucceeded, StartTimestamp: &metav1.Time{Time: now}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour)}, }, { Result: velerov1api.BackupRepositoryMaintenanceFailed, StartTimestamp: &metav1.Time{Time: now.Add(time.Hour)}, Message: "fake-message-2", }, { Result: velerov1api.BackupRepositoryMaintenanceSucceeded, StartTimestamp: &metav1.Time{Time: now.Add(time.Hour * 2)}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour * 3)}, }, }, }, { name: "more than limit", ctx: t.Context(), runtimeScheme: scheme, kubeClientObj: []runtime.Object{ jobSucceeded3, jobSucceeded2, jobFailed1, jobSucceeded1, jobPodSucceeded1, jobPodFailed1, jobPodSucceeded2, jobPodSucceeded3, }, expectedStatus: []velerov1api.BackupRepositoryMaintenanceStatus{ { Result: velerov1api.BackupRepositoryMaintenanceFailed, StartTimestamp: &metav1.Time{Time: now.Add(time.Hour)}, Message: "fake-message-2", }, { Result: velerov1api.BackupRepositoryMaintenanceSucceeded, StartTimestamp: &metav1.Time{Time: now.Add(time.Hour * 2)}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour * 3)}, }, { Result: velerov1api.BackupRepositoryMaintenanceSucceeded, StartTimestamp: &metav1.Time{Time: now.Add(time.Hour * 3)}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour * 4)}, }, }, }, } for _, test := range testCases { t.Run(test.name, func(t *testing.T) { fakeClientBuilder := fake.NewClientBuilder() fakeClientBuilder = fakeClientBuilder.WithScheme(test.runtimeScheme) fakeClient := fakeClientBuilder.WithRuntimeObjects(test.kubeClientObj...).Build() history, err := WaitAllJobsComplete(test.ctx, fakeClient, repo, 3, velerotest.NewLogger()) if test.expectedError != "" { require.ErrorContains(t, err, test.expectedError) } else { require.NoError(t, err) } assert.Len(t, history, len(test.expectedStatus)) for i := 0; i < len(test.expectedStatus); i++ { assert.Equal(t, test.expectedStatus[i].Result, history[i].Result) assert.Equal(t, test.expectedStatus[i].Message, history[i].Message) assert.Equal(t, test.expectedStatus[i].StartTimestamp.Time, history[i].StartTimestamp.Time) if test.expectedStatus[i].CompleteTimestamp == nil { assert.Nil(t, history[i].CompleteTimestamp) } else { assert.Equal(t, test.expectedStatus[i].CompleteTimestamp.Time, history[i].CompleteTimestamp.Time) } } }) } cancel() } func TestBuildJob(t *testing.T) { deploy := appsv1api.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: "velero", Namespace: "velero", }, Spec: appsv1api.DeploymentSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ SecurityContext: &corev1api.PodSecurityContext{ RunAsNonRoot: boolptr.True(), }, Containers: []corev1api.Container{ { Name: "velero-repo-maintenance-container", Image: "velero-image", SecurityContext: &corev1api.SecurityContext{ RunAsNonRoot: boolptr.True(), }, Env: []corev1api.EnvVar{ { Name: "test-name", Value: "test-value", }, }, EnvFrom: []corev1api.EnvFromSource{ { ConfigMapRef: &corev1api.ConfigMapEnvSource{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "test-configmap", }, }, }, { SecretRef: &corev1api.SecretEnvSource{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "test-secret", }, }, }, }, }, }, ImagePullSecrets: []corev1api.LocalObjectReference{ { Name: "imagePullSecret1", }, }, }, }, }, } deploy2 := deploy.DeepCopy() deploy2.Spec.Template.Labels = map[string]string{"azure.workload.identity/use": "fake-label-value"} deploy2.Spec.Template.Spec.SecurityContext = nil deploy2.Spec.Template.Spec.Containers[0].SecurityContext = nil testCases := []struct { name string m *velerotypes.JobConfigs deploy *appsv1api.Deployment logLevel logrus.Level logFormat *logging.FormatFlag expectedJobName string expectedError bool expectedEnv []corev1api.EnvVar expectedEnvFrom []corev1api.EnvFromSource expectedPodLabel map[string]string expectedPodAnnotation map[string]string expectedSecurityContext *corev1api.SecurityContext expectedPodSecurityContext *corev1api.PodSecurityContext expectedImagePullSecrets []corev1api.LocalObjectReference backupRepository *velerov1api.BackupRepository }{ { name: "Valid maintenance job without third party labels", m: &velerotypes.JobConfigs{ PodResources: &kube.PodResources{ CPURequest: "100m", MemoryRequest: "128Mi", CPULimit: "200m", MemoryLimit: "256Mi", }, }, deploy: &deploy, logLevel: logrus.InfoLevel, logFormat: logging.NewFormatFlag(), expectedJobName: "test-123-maintain-job", expectedError: false, expectedEnv: []corev1api.EnvVar{ { Name: "test-name", Value: "test-value", }, }, expectedEnvFrom: []corev1api.EnvFromSource{ { ConfigMapRef: &corev1api.ConfigMapEnvSource{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "test-configmap", }, }, }, { SecretRef: &corev1api.SecretEnvSource{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "test-secret", }, }, }, }, expectedPodLabel: map[string]string{ RepositoryNameLabel: "test-123", }, expectedSecurityContext: &corev1api.SecurityContext{ RunAsNonRoot: boolptr.True(), }, expectedPodSecurityContext: &corev1api.PodSecurityContext{ RunAsNonRoot: boolptr.True(), }, expectedImagePullSecrets: []corev1api.LocalObjectReference{ { Name: "imagePullSecret1", }, }, }, { name: "Valid maintenance job with third party labels", m: &velerotypes.JobConfigs{ PodResources: &kube.PodResources{ CPURequest: "100m", MemoryRequest: "128Mi", CPULimit: "200m", MemoryLimit: "256Mi", }, }, deploy: deploy2, logLevel: logrus.InfoLevel, logFormat: logging.NewFormatFlag(), expectedJobName: "test-123-maintain-job", expectedError: false, expectedEnv: []corev1api.EnvVar{ { Name: "test-name", Value: "test-value", }, }, expectedEnvFrom: []corev1api.EnvFromSource{ { ConfigMapRef: &corev1api.ConfigMapEnvSource{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "test-configmap", }, }, }, { SecretRef: &corev1api.SecretEnvSource{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "test-secret", }, }, }, }, expectedPodLabel: map[string]string{ RepositoryNameLabel: "test-123", "azure.workload.identity/use": "fake-label-value", }, expectedSecurityContext: nil, expectedPodSecurityContext: nil, expectedImagePullSecrets: []corev1api.LocalObjectReference{ { Name: "imagePullSecret1", }, }, }, { name: "Error getting Velero server deployment", m: &velerotypes.JobConfigs{ PodResources: &kube.PodResources{ CPURequest: "100m", MemoryRequest: "128Mi", CPULimit: "200m", MemoryLimit: "256Mi", }, }, logLevel: logrus.InfoLevel, logFormat: logging.NewFormatFlag(), expectedJobName: "", expectedError: true, }, { name: "Valid maintenance job customized labels and annotations", m: &velerotypes.JobConfigs{ PodResources: &kube.PodResources{ CPURequest: "100m", MemoryRequest: "128Mi", CPULimit: "200m", MemoryLimit: "256Mi", }, PodLabels: map[string]string{ "global-label-1": "global-label-value-1", "global-label-2": "global-label-value-2", }, PodAnnotations: map[string]string{ "global-annotation-1": "global-annotation-value-1", "global-annotation-2": "global-annotation-value-2", }, }, deploy: deploy2, logLevel: logrus.InfoLevel, logFormat: logging.NewFormatFlag(), expectedError: false, expectedJobName: "test-123-maintain-job", expectedEnv: []corev1api.EnvVar{ { Name: "test-name", Value: "test-value", }, }, expectedEnvFrom: []corev1api.EnvFromSource{ { ConfigMapRef: &corev1api.ConfigMapEnvSource{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "test-configmap", }, }, }, { SecretRef: &corev1api.SecretEnvSource{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "test-secret", }, }, }, }, expectedPodLabel: map[string]string{ "global-label-1": "global-label-value-1", "global-label-2": "global-label-value-2", RepositoryNameLabel: "test-123", }, expectedPodAnnotation: map[string]string{ "global-annotation-1": "global-annotation-value-1", "global-annotation-2": "global-annotation-value-2", }, expectedSecurityContext: nil, expectedPodSecurityContext: nil, expectedImagePullSecrets: []corev1api.LocalObjectReference{ { Name: "imagePullSecret1", }, }, }, { name: "Valid maintenance job with third party labels and BackupRepository name longer than 63", m: &velerotypes.JobConfigs{ PodResources: &kube.PodResources{ CPURequest: "100m", MemoryRequest: "128Mi", CPULimit: "200m", MemoryLimit: "256Mi", }, }, deploy: deploy2, logLevel: logrus.InfoLevel, logFormat: logging.NewFormatFlag(), expectedError: false, expectedEnv: []corev1api.EnvVar{ { Name: "test-name", Value: "test-value", }, }, expectedEnvFrom: []corev1api.EnvFromSource{ { ConfigMapRef: &corev1api.ConfigMapEnvSource{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "test-configmap", }, }, }, { SecretRef: &corev1api.SecretEnvSource{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "test-secret", }, }, }, }, expectedPodLabel: map[string]string{ RepositoryNameLabel: velerolabel.ReturnNameOrHash("label with more than 63 characters should be modified"), "azure.workload.identity/use": "fake-label-value", }, expectedSecurityContext: nil, expectedPodSecurityContext: nil, expectedImagePullSecrets: []corev1api.LocalObjectReference{ { Name: "imagePullSecret1", }, }, backupRepository: &velerov1api.BackupRepository{ ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", Name: "label with more than 63 characters should be modified", }, Spec: velerov1api.BackupRepositorySpec{ VolumeNamespace: "test-123", RepositoryType: "kopia", }, }, }, } param := provider.RepoParam{ BackupRepo: &velerov1api.BackupRepository{ ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", Name: "test-123", }, Spec: velerov1api.BackupRepositorySpec{ VolumeNamespace: "test-123", RepositoryType: "kopia", }, }, BackupLocation: &velerov1api.BackupStorageLocation{ ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", Name: "test-location", }, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { if tc.backupRepository != nil { param.BackupRepo = tc.backupRepository } // Create a fake clientset with resources objs := []runtime.Object{param.BackupLocation, param.BackupRepo} if tc.deploy != nil { objs = append(objs, tc.deploy) } scheme := runtime.NewScheme() _ = appsv1api.AddToScheme(scheme) _ = velerov1api.AddToScheme(scheme) cli := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(objs...).Build() // Call the function to test job, err := buildJob( cli, t.Context(), param.BackupRepo, param.BackupLocation.Name, tc.m, tc.logLevel, tc.logFormat, logrus.New(), ) // Check the error if tc.expectedError { require.Error(t, err) assert.Nil(t, job) } else { require.NoError(t, err) assert.NotNil(t, job) assert.Contains(t, job.Name, tc.expectedJobName) assert.Equal(t, param.BackupRepo.Namespace, job.Namespace) assert.Equal(t, param.BackupRepo.Name, job.Labels[RepositoryNameLabel]) assert.Equal(t, param.BackupRepo.Name, job.Spec.Template.ObjectMeta.Labels[RepositoryNameLabel]) // Check container assert.Len(t, job.Spec.Template.Spec.Containers, 1) container := job.Spec.Template.Spec.Containers[0] assert.Equal(t, "velero-repo-maintenance-container", container.Name) assert.Equal(t, "velero-image", container.Image) assert.Equal(t, corev1api.PullIfNotPresent, container.ImagePullPolicy) // Check container env assert.Equal(t, tc.expectedEnv, container.Env) assert.Equal(t, tc.expectedEnvFrom, container.EnvFrom) // Check security context assert.Equal(t, tc.expectedPodSecurityContext, job.Spec.Template.Spec.SecurityContext) assert.Equal(t, tc.expectedSecurityContext, container.SecurityContext) // Check resources expectedResources := corev1api.ResourceRequirements{ Requests: corev1api.ResourceList{ corev1api.ResourceCPU: resource.MustParse(tc.m.PodResources.CPURequest), corev1api.ResourceMemory: resource.MustParse(tc.m.PodResources.MemoryRequest), }, Limits: corev1api.ResourceList{ corev1api.ResourceCPU: resource.MustParse(tc.m.PodResources.CPULimit), corev1api.ResourceMemory: resource.MustParse(tc.m.PodResources.MemoryLimit), }, } assert.Equal(t, expectedResources, container.Resources) // Check args expectedArgs := []string{ "repo-maintenance", fmt.Sprintf("--repo-name=%s", param.BackupRepo.Spec.VolumeNamespace), fmt.Sprintf("--repo-type=%s", param.BackupRepo.Spec.RepositoryType), fmt.Sprintf("--backup-storage-location=%s", param.BackupLocation.Name), fmt.Sprintf("--log-level=%s", tc.logLevel.String()), fmt.Sprintf("--log-format=%s", tc.logFormat.String()), } assert.Equal(t, expectedArgs, container.Args) assert.Equal(t, tc.expectedPodLabel, job.Spec.Template.Labels) assert.Equal(t, tc.expectedImagePullSecrets, job.Spec.Template.Spec.ImagePullSecrets) } }) } } func TestGetKeepLatestMaintenanceJobs(t *testing.T) { tests := []struct { name string repoMaintenanceJobConfig string configMap *corev1api.ConfigMap repo *velerov1api.BackupRepository expectedValue int expectError bool }{ { name: "no config map name provided", repoMaintenanceJobConfig: "", configMap: nil, repo: mockBackupRepo(), expectedValue: 3, expectError: false, }, { name: "config map not found", repoMaintenanceJobConfig: "non-existent-config", configMap: nil, repo: mockBackupRepo(), expectedValue: 3, expectError: false, }, { name: "config map with global keepLatestMaintenanceJobs", repoMaintenanceJobConfig: "repo-job-config", configMap: &corev1api.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", Name: "repo-job-config", }, Data: map[string]string{ "global": `{"keepLatestMaintenanceJobs": 5}`, }, }, repo: mockBackupRepo(), expectedValue: 5, expectError: false, }, { name: "config map with specific repo keepLatestMaintenanceJobs overriding global", repoMaintenanceJobConfig: "repo-job-config", configMap: &corev1api.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", Name: "repo-job-config", }, Data: map[string]string{ "global": `{"keepLatestMaintenanceJobs": 5}`, "test-ns-default-kopia": `{"keepLatestMaintenanceJobs": 10}`, }, }, repo: mockBackupRepo(), expectedValue: 10, expectError: false, }, { name: "config map with no keepLatestMaintenanceJobs specified", repoMaintenanceJobConfig: "repo-job-config", configMap: &corev1api.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", Name: "repo-job-config", }, Data: map[string]string{ "global": `{"podResources": {"cpuRequest": "100m"}}`, }, }, repo: mockBackupRepo(), expectedValue: 3, expectError: false, }, { name: "config map with invalid JSON", repoMaintenanceJobConfig: "repo-job-config", configMap: &corev1api.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", Name: "repo-job-config", }, Data: map[string]string{ "global": `{"keepLatestMaintenanceJobs": invalid}`, }, }, repo: mockBackupRepo(), expectedValue: 3, expectError: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { scheme := runtime.NewScheme() corev1api.AddToScheme(scheme) var objects []runtime.Object if test.configMap != nil { objects = append(objects, test.configMap) } client := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(objects...).Build() logger := velerotest.NewLogger() result, err := GetKeepLatestMaintenanceJobs( t.Context(), client, logger, "velero", test.repoMaintenanceJobConfig, test.repo, ) if test.expectError { require.Error(t, err) } else { require.NoError(t, err) assert.Equal(t, test.expectedValue, result) } }) } } func mockBackupRepo() *velerov1api.BackupRepository { return &velerov1api.BackupRepository{ ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", Name: "test-repo", }, Spec: velerov1api.BackupRepositorySpec{ VolumeNamespace: "test-ns", BackupStorageLocation: "default", RepositoryType: "kopia", }, } } func TestGetPriorityClassName(t *testing.T) { testCases := []struct { name string config *velerotypes.JobConfigs priorityClassExists bool expectedValue string expectedLogContains string expectedLogLevel string }{ { name: "empty priority class name should return empty string", config: &velerotypes.JobConfigs{PriorityClassName: ""}, expectedValue: "", expectedLogContains: "", }, { name: "nil config should return empty string", config: nil, expectedValue: "", expectedLogContains: "", }, { name: "existing priority class should log info and return name", config: &velerotypes.JobConfigs{PriorityClassName: "high-priority"}, priorityClassExists: true, expectedValue: "high-priority", expectedLogContains: "Validated priority class \\\"high-priority\\\" exists in cluster", expectedLogLevel: "info", }, { name: "non-existing priority class should log warning and still return name", config: &velerotypes.JobConfigs{PriorityClassName: "missing-priority"}, priorityClassExists: false, expectedValue: "missing-priority", expectedLogContains: "Priority class \\\"missing-priority\\\" not found in cluster", expectedLogLevel: "warning", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { // Create a new scheme and add necessary API types localScheme := runtime.NewScheme() err := schedulingv1.AddToScheme(localScheme) require.NoError(t, err) // Create fake client builder clientBuilder := fake.NewClientBuilder().WithScheme(localScheme) // Add priority class if it should exist if tc.priorityClassExists { priorityClass := &schedulingv1.PriorityClass{ ObjectMeta: metav1.ObjectMeta{ Name: tc.config.PriorityClassName, }, Value: 1000, } clientBuilder = clientBuilder.WithObjects(priorityClass) } client := clientBuilder.Build() // Capture logs var logBuffer strings.Builder logger := logrus.New() logger.SetOutput(&logBuffer) logger.SetLevel(logrus.InfoLevel) // Call the function result := getPriorityClassName(t.Context(), client, tc.config, logger) // Verify the result assert.Equal(t, tc.expectedValue, result) // Verify log output logOutput := logBuffer.String() if tc.expectedLogContains != "" { assert.Contains(t, logOutput, tc.expectedLogContains) } // Verify log level if tc.expectedLogLevel == "warning" { assert.Contains(t, logOutput, "level=warning") } else if tc.expectedLogLevel == "info" { assert.Contains(t, logOutput, "level=info") } }) } } func TestBuildJobWithPriorityClassName(t *testing.T) { testCases := []struct { name string priorityClassName string expectedValue string }{ { name: "with priority class name", priorityClassName: "high-priority", expectedValue: "high-priority", }, { name: "without priority class name", priorityClassName: "", expectedValue: "", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { // Create a new scheme and add necessary API types localScheme := runtime.NewScheme() err := velerov1api.AddToScheme(localScheme) require.NoError(t, err) err = appsv1api.AddToScheme(localScheme) require.NoError(t, err) err = batchv1api.AddToScheme(localScheme) require.NoError(t, err) err = schedulingv1.AddToScheme(localScheme) require.NoError(t, err) // Create a fake client client := fake.NewClientBuilder().WithScheme(localScheme).Build() // Create a deployment with the specified priority class name deployment := &appsv1api.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: "velero", Namespace: "velero", }, Spec: appsv1api.DeploymentSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ Containers: []corev1api.Container{ { Name: "velero", Image: "velero/velero:latest", }, }, PriorityClassName: tc.priorityClassName, }, }, }, } // Create a backup repository repo := &velerov1api.BackupRepository{ ObjectMeta: metav1.ObjectMeta{ Name: "test-repo", Namespace: "velero", }, Spec: velerov1api.BackupRepositorySpec{ VolumeNamespace: "velero", BackupStorageLocation: "default", }, } // Create the deployment in the fake client err = client.Create(t.Context(), deployment) require.NoError(t, err) // Create minimal job configs and resources jobConfig := &velerotypes.JobConfigs{ PriorityClassName: tc.priorityClassName, } logLevel := logrus.InfoLevel logFormat := logging.NewFormatFlag() logFormat.Set("text") // Call buildJob job, err := buildJob(client, t.Context(), repo, "default", jobConfig, logLevel, logFormat, logrus.New()) require.NoError(t, err) // Verify the priority class name is set correctly assert.Equal(t, tc.expectedValue, job.Spec.Template.Spec.PriorityClassName) }) } } func TestBuildTolerationsForMaintenanceJob(t *testing.T) { windowsToleration := corev1api.Toleration{ Key: "os", Operator: "Equal", Effect: "NoSchedule", Value: "windows", } testCases := []struct { name string deploymentTolerations []corev1api.Toleration expectedTolerations []corev1api.Toleration }{ { name: "no tolerations should only include Windows toleration", deploymentTolerations: nil, expectedTolerations: []corev1api.Toleration{ windowsToleration, }, }, { name: "empty tolerations should only include Windows toleration", deploymentTolerations: []corev1api.Toleration{}, expectedTolerations: []corev1api.Toleration{ windowsToleration, }, }, { name: "non-allowed toleration should not be inherited", deploymentTolerations: []corev1api.Toleration{ { Key: "vng-ondemand", Operator: "Equal", Effect: "NoSchedule", Value: "amd64", }, }, expectedTolerations: []corev1api.Toleration{ windowsToleration, }, }, { name: "allowed toleration should be inherited", deploymentTolerations: []corev1api.Toleration{ { Key: "kubernetes.azure.com/scalesetpriority", Operator: "Equal", Effect: "NoSchedule", Value: "spot", }, }, expectedTolerations: []corev1api.Toleration{ windowsToleration, { Key: "kubernetes.azure.com/scalesetpriority", Operator: "Equal", Effect: "NoSchedule", Value: "spot", }, }, }, { name: "mixed allowed and non-allowed tolerations should only inherit allowed", deploymentTolerations: []corev1api.Toleration{ { Key: "vng-ondemand", // not in allowlist Operator: "Equal", Effect: "NoSchedule", Value: "amd64", }, { Key: "CriticalAddonsOnly", // in allowlist Operator: "Exists", Effect: "NoSchedule", }, { Key: "custom-key", // not in allowlist Operator: "Equal", Effect: "NoSchedule", Value: "custom-value", }, }, expectedTolerations: []corev1api.Toleration{ windowsToleration, { Key: "CriticalAddonsOnly", Operator: "Exists", Effect: "NoSchedule", }, }, }, { name: "multiple allowed tolerations should all be inherited", deploymentTolerations: []corev1api.Toleration{ { Key: "kubernetes.azure.com/scalesetpriority", Operator: "Equal", Effect: "NoSchedule", Value: "spot", }, { Key: "CriticalAddonsOnly", Operator: "Exists", Effect: "NoSchedule", }, }, expectedTolerations: []corev1api.Toleration{ windowsToleration, { Key: "kubernetes.azure.com/scalesetpriority", Operator: "Equal", Effect: "NoSchedule", Value: "spot", }, { Key: "CriticalAddonsOnly", Operator: "Exists", Effect: "NoSchedule", }, }, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { // Create a deployment with the specified tolerations deployment := &appsv1api.Deployment{ Spec: appsv1api.DeploymentSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ Tolerations: tc.deploymentTolerations, }, }, }, } result := buildTolerationsForMaintenanceJob(deployment) assert.Equal(t, tc.expectedTolerations, result) }) } } func TestBuildJobWithTolerationsInheritance(t *testing.T) { // Define allowed tolerations that would be set on Velero deployment allowedTolerations := []corev1api.Toleration{ { Key: "kubernetes.azure.com/scalesetpriority", Operator: "Equal", Effect: "NoSchedule", Value: "spot", }, { Key: "CriticalAddonsOnly", Operator: "Exists", Effect: "NoSchedule", }, } // Mixed tolerations (allowed and non-allowed) mixedTolerations := []corev1api.Toleration{ { Key: "vng-ondemand", // not in allowlist Operator: "Equal", Effect: "NoSchedule", Value: "amd64", }, { Key: "CriticalAddonsOnly", // in allowlist Operator: "Exists", Effect: "NoSchedule", }, } // Windows toleration that should always be present windowsToleration := corev1api.Toleration{ Key: "os", Operator: "Equal", Effect: "NoSchedule", Value: "windows", } testCases := []struct { name string deploymentTolerations []corev1api.Toleration expectedTolerations []corev1api.Toleration }{ { name: "no tolerations on deployment should only have Windows toleration", deploymentTolerations: nil, expectedTolerations: []corev1api.Toleration{ windowsToleration, }, }, { name: "allowed tolerations should be inherited along with Windows toleration", deploymentTolerations: allowedTolerations, expectedTolerations: []corev1api.Toleration{ windowsToleration, { Key: "kubernetes.azure.com/scalesetpriority", Operator: "Equal", Effect: "NoSchedule", Value: "spot", }, { Key: "CriticalAddonsOnly", Operator: "Exists", Effect: "NoSchedule", }, }, }, { name: "mixed tolerations should only inherit allowed ones", deploymentTolerations: mixedTolerations, expectedTolerations: []corev1api.Toleration{ windowsToleration, { Key: "CriticalAddonsOnly", Operator: "Exists", Effect: "NoSchedule", }, }, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { // Create a new scheme and add necessary API types localScheme := runtime.NewScheme() err := velerov1api.AddToScheme(localScheme) require.NoError(t, err) err = appsv1api.AddToScheme(localScheme) require.NoError(t, err) err = batchv1api.AddToScheme(localScheme) require.NoError(t, err) // Create a deployment with the specified tolerations deployment := &appsv1api.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: "velero", Namespace: "velero", }, Spec: appsv1api.DeploymentSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ Containers: []corev1api.Container{ { Name: "velero", Image: "velero/velero:latest", }, }, Tolerations: tc.deploymentTolerations, }, }, }, } // Create a backup repository repo := &velerov1api.BackupRepository{ ObjectMeta: metav1.ObjectMeta{ Name: "test-repo", Namespace: "velero", }, Spec: velerov1api.BackupRepositorySpec{ VolumeNamespace: "velero", BackupStorageLocation: "default", }, } // Create fake client and add the deployment client := fake.NewClientBuilder().WithScheme(localScheme).WithObjects(deployment).Build() // Create minimal job configs and resources jobConfig := &velerotypes.JobConfigs{} logLevel := logrus.InfoLevel logFormat := logging.NewFormatFlag() logFormat.Set("text") // Call buildJob job, err := buildJob(client, t.Context(), repo, "default", jobConfig, logLevel, logFormat, logrus.New()) require.NoError(t, err) // Verify the tolerations are set correctly assert.Equal(t, tc.expectedTolerations, job.Spec.Template.Spec.Tolerations) }) } } ================================================ FILE: pkg/repository/manager/manager.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package repository import ( "context" "fmt" "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/internal/credentials" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/repository" "github.com/vmware-tanzu/velero/pkg/repository/provider" "github.com/vmware-tanzu/velero/pkg/util/filesystem" ) // Manager manages backup repositories. type Manager interface { // InitRepo initializes a repo with the specified name and identifier. InitRepo(repo *velerov1api.BackupRepository) error // ConnectToRepo tries to connect to the specified repo, and returns an error if it fails. // This is intended to be used to ensure that the repo exists/can be authenticated to. ConnectToRepo(repo *velerov1api.BackupRepository) error // PrepareRepo tries to connect to the specific repo first, if it fails because of the // repo is not initialized, it turns to initialize the repo PrepareRepo(repo *velerov1api.BackupRepository) error // PruneRepo deletes unused data from a repo. PruneRepo(repo *velerov1api.BackupRepository) error // UnlockRepo removes stale locks from a repo. UnlockRepo(repo *velerov1api.BackupRepository) error // Forget removes a snapshot from the list of // available snapshots in a repo. Forget(context.Context, *velerov1api.BackupRepository, string) error // BatchForget removes a list of snapshots from the list of // available snapshots in a repo. BatchForget(context.Context, *velerov1api.BackupRepository, []string) []error // DefaultMaintenanceFrequency returns the default maintenance frequency from the specific repo DefaultMaintenanceFrequency(*velerov1api.BackupRepository) (time.Duration, error) // ClientSideCacheLimit returns the max cache size required on client side ClientSideCacheLimit(*velerov1api.BackupRepository) (int64, error) } // ConfigProvider defines the methods to get configurations of a backup repository type ConfigManager interface { // DefaultMaintenanceFrequency returns the default maintenance frequency from the specific repo DefaultMaintenanceFrequency(string) (time.Duration, error) // ClientSideCacheLimit returns the max cache size required on client side ClientSideCacheLimit(string, map[string]string) (int64, error) } type manager struct { namespace string providers map[string]provider.Provider // client is the Velero controller manager's client. // It's limited to resources in the Velero namespace. client client.Client repoLocker *repository.RepoLocker fileSystem filesystem.Interface log logrus.FieldLogger } type configManager struct { providers map[string]provider.ConfigProvider log logrus.FieldLogger } // NewManager create a new repository manager. func NewManager( namespace string, client client.Client, repoLocker *repository.RepoLocker, credentialFileStore credentials.FileStore, credentialSecretStore credentials.SecretStore, log logrus.FieldLogger, ) Manager { mgr := &manager{ namespace: namespace, client: client, providers: map[string]provider.Provider{}, repoLocker: repoLocker, fileSystem: filesystem.NewFileSystem(), log: log, } mgr.providers[velerov1api.BackupRepositoryTypeRestic] = provider.NewResticRepositoryProvider(credentials.CredentialGetter{ FromFile: credentialFileStore, FromSecret: credentialSecretStore, }, mgr.fileSystem, mgr.log) mgr.providers[velerov1api.BackupRepositoryTypeKopia] = provider.NewUnifiedRepoProvider(credentials.CredentialGetter{ FromFile: credentialFileStore, FromSecret: credentialSecretStore, }, velerov1api.BackupRepositoryTypeKopia, mgr.log) return mgr } // NewConfigManager create a new repository config manager. func NewConfigManager( log logrus.FieldLogger, ) ConfigManager { mgr := &configManager{ providers: map[string]provider.ConfigProvider{}, log: log, } mgr.providers[velerov1api.BackupRepositoryTypeKopia] = provider.NewUnifiedRepoConfigProvider(velerov1api.BackupRepositoryTypeKopia, mgr.log) return mgr } func (m *manager) InitRepo(repo *velerov1api.BackupRepository) error { m.repoLocker.LockExclusive(repo.Name) defer m.repoLocker.UnlockExclusive(repo.Name) prd, err := m.getRepositoryProvider(repo) if err != nil { return errors.WithStack(err) } param, err := m.assembleRepoParam(repo) if err != nil { return errors.WithStack(err) } return prd.InitRepo(context.Background(), param) } func (m *manager) ConnectToRepo(repo *velerov1api.BackupRepository) error { m.repoLocker.Lock(repo.Name) defer m.repoLocker.Unlock(repo.Name) prd, err := m.getRepositoryProvider(repo) if err != nil { return errors.WithStack(err) } param, err := m.assembleRepoParam(repo) if err != nil { return errors.WithStack(err) } return prd.ConnectToRepo(context.Background(), param) } func (m *manager) PrepareRepo(repo *velerov1api.BackupRepository) error { m.repoLocker.Lock(repo.Name) defer m.repoLocker.Unlock(repo.Name) prd, err := m.getRepositoryProvider(repo) if err != nil { return errors.WithStack(err) } param, err := m.assembleRepoParam(repo) if err != nil { return errors.WithStack(err) } return prd.PrepareRepo(context.Background(), param) } func (m *manager) PruneRepo(repo *velerov1api.BackupRepository) error { m.repoLocker.LockExclusive(repo.Name) defer m.repoLocker.UnlockExclusive(repo.Name) prd, err := m.getRepositoryProvider(repo) if err != nil { return errors.WithStack(err) } param, err := m.assembleRepoParam(repo) if err != nil { return errors.WithStack(err) } if err := prd.BoostRepoConnect(context.Background(), param); err != nil { return errors.WithStack(err) } return prd.PruneRepo(context.Background(), param) } func (m *manager) UnlockRepo(repo *velerov1api.BackupRepository) error { m.repoLocker.Lock(repo.Name) defer m.repoLocker.Unlock(repo.Name) prd, err := m.getRepositoryProvider(repo) if err != nil { return errors.WithStack(err) } param, err := m.assembleRepoParam(repo) if err != nil { return errors.WithStack(err) } return prd.EnsureUnlockRepo(context.Background(), param) } func (m *manager) Forget(ctx context.Context, repo *velerov1api.BackupRepository, snapshot string) error { m.repoLocker.LockExclusive(repo.Name) defer m.repoLocker.UnlockExclusive(repo.Name) prd, err := m.getRepositoryProvider(repo) if err != nil { return errors.WithStack(err) } param, err := m.assembleRepoParam(repo) if err != nil { return errors.WithStack(err) } if err := prd.BoostRepoConnect(context.Background(), param); err != nil { return errors.WithStack(err) } return prd.Forget(context.Background(), snapshot, param) } func (m *manager) BatchForget(ctx context.Context, repo *velerov1api.BackupRepository, snapshots []string) []error { m.repoLocker.LockExclusive(repo.Name) defer m.repoLocker.UnlockExclusive(repo.Name) prd, err := m.getRepositoryProvider(repo) if err != nil { return []error{errors.WithStack(err)} } param, err := m.assembleRepoParam(repo) if err != nil { return []error{errors.WithStack(err)} } if err := prd.BoostRepoConnect(context.Background(), param); err != nil { return []error{errors.WithStack(err)} } return prd.BatchForget(context.Background(), snapshots, param) } func (m *manager) DefaultMaintenanceFrequency(repo *velerov1api.BackupRepository) (time.Duration, error) { prd, err := m.getRepositoryProvider(repo) if err != nil { return 0, errors.WithStack(err) } return prd.DefaultMaintenanceFrequency(), nil } func (m *manager) ClientSideCacheLimit(repo *velerov1api.BackupRepository) (int64, error) { prd, err := m.getRepositoryProvider(repo) if err != nil { return 0, errors.WithStack(err) } return prd.ClientSideCacheLimit(repo.Spec.RepositoryConfig), nil } func (m *manager) getRepositoryProvider(repo *velerov1api.BackupRepository) (provider.Provider, error) { switch repo.Spec.RepositoryType { case "", velerov1api.BackupRepositoryTypeRestic: return m.providers[velerov1api.BackupRepositoryTypeRestic], nil case velerov1api.BackupRepositoryTypeKopia: return m.providers[velerov1api.BackupRepositoryTypeKopia], nil default: return nil, fmt.Errorf("failed to get provider for repository %s", repo.Spec.RepositoryType) } } func (m *manager) assembleRepoParam(repo *velerov1api.BackupRepository) (provider.RepoParam, error) { bsl := &velerov1api.BackupStorageLocation{} if err := m.client.Get(context.Background(), client.ObjectKey{Namespace: m.namespace, Name: repo.Spec.BackupStorageLocation}, bsl); err != nil { return provider.RepoParam{}, errors.WithStack(err) } return provider.RepoParam{ BackupLocation: bsl, BackupRepo: repo, }, nil } func (cm *configManager) DefaultMaintenanceFrequency(repoType string) (time.Duration, error) { prd, err := cm.getRepositoryProvider(repoType) if err != nil { return 0, errors.WithStack(err) } return prd.DefaultMaintenanceFrequency(), nil } func (cm *configManager) ClientSideCacheLimit(repoType string, repoOption map[string]string) (int64, error) { prd, err := cm.getRepositoryProvider(repoType) if err != nil { return 0, errors.WithStack(err) } return prd.ClientSideCacheLimit(repoOption), nil } func (cm *configManager) getRepositoryProvider(repoType string) (provider.ConfigProvider, error) { switch repoType { case velerov1api.BackupRepositoryTypeKopia: return cm.providers[velerov1api.BackupRepositoryTypeKopia], nil default: return nil, fmt.Errorf("failed to get provider for repository %s", repoType) } } ================================================ FILE: pkg/repository/manager/manager_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package repository import ( "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" kbclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) func TestGetRepositoryProvider(t *testing.T) { var fakeClient kbclient.Client mgr := NewManager("", fakeClient, nil, nil, nil, nil).(*manager) repo := &velerov1.BackupRepository{} // empty repository type provider, err := mgr.getRepositoryProvider(repo) require.NoError(t, err) assert.NotNil(t, provider) // valid repository type repo.Spec.RepositoryType = velerov1.BackupRepositoryTypeRestic provider, err = mgr.getRepositoryProvider(repo) require.NoError(t, err) assert.NotNil(t, provider) // invalid repository type repo.Spec.RepositoryType = "unknown" _, err = mgr.getRepositoryProvider(repo) require.Error(t, err) } func TestGetRepositoryConfigProvider(t *testing.T) { mgr := NewConfigManager(nil).(*configManager) // empty repository type _, err := mgr.getRepositoryProvider("") require.Error(t, err) // valid repository type provider, err := mgr.getRepositoryProvider(velerov1.BackupRepositoryTypeKopia) require.NoError(t, err) assert.NotNil(t, provider) // invalid repository type _, err = mgr.getRepositoryProvider(velerov1.BackupRepositoryTypeRestic) require.Error(t, err) } ================================================ FILE: pkg/repository/mocks/ConfigManager.go ================================================ // Code generated by mockery v2.53.2. DO NOT EDIT. package mocks import ( mock "github.com/stretchr/testify/mock" time "time" ) // ConfigManager is an autogenerated mock type for the ConfigManager type type ConfigManager struct { mock.Mock } // ClientSideCacheLimit provides a mock function with given fields: repoType, repoOption func (_m *ConfigManager) ClientSideCacheLimit(repoType string, repoOption map[string]string) (int64, error) { ret := _m.Called(repoType, repoOption) if len(ret) == 0 { panic("no return value specified for ClientSideCacheLimit") } var r0 int64 var r1 error if rf, ok := ret.Get(0).(func(string, map[string]string) (int64, error)); ok { return rf(repoType, repoOption) } if rf, ok := ret.Get(0).(func(string, map[string]string) int64); ok { r0 = rf(repoType, repoOption) } else { r0 = ret.Get(0).(int64) } if rf, ok := ret.Get(1).(func(string, map[string]string) error); ok { r1 = rf(repoType, repoOption) } else { r1 = ret.Error(1) } return r0, r1 } // DefaultMaintenanceFrequency provides a mock function with given fields: repoType func (_m *ConfigManager) DefaultMaintenanceFrequency(repoType string) (time.Duration, error) { ret := _m.Called(repoType) if len(ret) == 0 { panic("no return value specified for DefaultMaintenanceFrequency") } var r0 time.Duration var r1 error if rf, ok := ret.Get(0).(func(string) (time.Duration, error)); ok { return rf(repoType) } if rf, ok := ret.Get(0).(func(string) time.Duration); ok { r0 = rf(repoType) } else { r0 = ret.Get(0).(time.Duration) } if rf, ok := ret.Get(1).(func(string) error); ok { r1 = rf(repoType) } else { r1 = ret.Error(1) } return r0, r1 } // NewConfigManager creates a new instance of ConfigManager. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. func NewConfigManager(t interface { mock.TestingT Cleanup(func()) }) *ConfigManager { mock := &ConfigManager{} mock.Mock.Test(t) t.Cleanup(func() { mock.AssertExpectations(t) }) return mock } ================================================ FILE: pkg/repository/mocks/Manager.go ================================================ // Code generated by mockery v2.53.2. DO NOT EDIT. package mocks import ( context "context" mock "github.com/stretchr/testify/mock" time "time" v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) // Manager is an autogenerated mock type for the Manager type type Manager struct { mock.Mock } // BatchForget provides a mock function with given fields: _a0, _a1, _a2 func (_m *Manager) BatchForget(_a0 context.Context, _a1 *v1.BackupRepository, _a2 []string) []error { ret := _m.Called(_a0, _a1, _a2) if len(ret) == 0 { panic("no return value specified for BatchForget") } var r0 []error if rf, ok := ret.Get(0).(func(context.Context, *v1.BackupRepository, []string) []error); ok { r0 = rf(_a0, _a1, _a2) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]error) } } return r0 } // ClientSideCacheLimit provides a mock function with given fields: _a0 func (_m *Manager) ClientSideCacheLimit(_a0 *v1.BackupRepository) (int64, error) { ret := _m.Called(_a0) if len(ret) == 0 { panic("no return value specified for ClientSideCacheLimit") } var r0 int64 var r1 error if rf, ok := ret.Get(0).(func(*v1.BackupRepository) (int64, error)); ok { return rf(_a0) } if rf, ok := ret.Get(0).(func(*v1.BackupRepository) int64); ok { r0 = rf(_a0) } else { r0 = ret.Get(0).(int64) } if rf, ok := ret.Get(1).(func(*v1.BackupRepository) error); ok { r1 = rf(_a0) } else { r1 = ret.Error(1) } return r0, r1 } // ConnectToRepo provides a mock function with given fields: repo func (_m *Manager) ConnectToRepo(repo *v1.BackupRepository) error { ret := _m.Called(repo) if len(ret) == 0 { panic("no return value specified for ConnectToRepo") } var r0 error if rf, ok := ret.Get(0).(func(*v1.BackupRepository) error); ok { r0 = rf(repo) } else { r0 = ret.Error(0) } return r0 } // DefaultMaintenanceFrequency provides a mock function with given fields: _a0 func (_m *Manager) DefaultMaintenanceFrequency(_a0 *v1.BackupRepository) (time.Duration, error) { ret := _m.Called(_a0) if len(ret) == 0 { panic("no return value specified for DefaultMaintenanceFrequency") } var r0 time.Duration var r1 error if rf, ok := ret.Get(0).(func(*v1.BackupRepository) (time.Duration, error)); ok { return rf(_a0) } if rf, ok := ret.Get(0).(func(*v1.BackupRepository) time.Duration); ok { r0 = rf(_a0) } else { r0 = ret.Get(0).(time.Duration) } if rf, ok := ret.Get(1).(func(*v1.BackupRepository) error); ok { r1 = rf(_a0) } else { r1 = ret.Error(1) } return r0, r1 } // Forget provides a mock function with given fields: _a0, _a1, _a2 func (_m *Manager) Forget(_a0 context.Context, _a1 *v1.BackupRepository, _a2 string) error { ret := _m.Called(_a0, _a1, _a2) if len(ret) == 0 { panic("no return value specified for Forget") } var r0 error if rf, ok := ret.Get(0).(func(context.Context, *v1.BackupRepository, string) error); ok { r0 = rf(_a0, _a1, _a2) } else { r0 = ret.Error(0) } return r0 } // InitRepo provides a mock function with given fields: repo func (_m *Manager) InitRepo(repo *v1.BackupRepository) error { ret := _m.Called(repo) if len(ret) == 0 { panic("no return value specified for InitRepo") } var r0 error if rf, ok := ret.Get(0).(func(*v1.BackupRepository) error); ok { r0 = rf(repo) } else { r0 = ret.Error(0) } return r0 } // PrepareRepo provides a mock function with given fields: repo func (_m *Manager) PrepareRepo(repo *v1.BackupRepository) error { ret := _m.Called(repo) if len(ret) == 0 { panic("no return value specified for PrepareRepo") } var r0 error if rf, ok := ret.Get(0).(func(*v1.BackupRepository) error); ok { r0 = rf(repo) } else { r0 = ret.Error(0) } return r0 } // PruneRepo provides a mock function with given fields: repo func (_m *Manager) PruneRepo(repo *v1.BackupRepository) error { ret := _m.Called(repo) if len(ret) == 0 { panic("no return value specified for PruneRepo") } var r0 error if rf, ok := ret.Get(0).(func(*v1.BackupRepository) error); ok { r0 = rf(repo) } else { r0 = ret.Error(0) } return r0 } // UnlockRepo provides a mock function with given fields: repo func (_m *Manager) UnlockRepo(repo *v1.BackupRepository) error { ret := _m.Called(repo) if len(ret) == 0 { panic("no return value specified for UnlockRepo") } var r0 error if rf, ok := ret.Get(0).(func(*v1.BackupRepository) error); ok { r0 = rf(repo) } else { r0 = ret.Error(0) } return r0 } // NewManager creates a new instance of Manager. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. func NewManager(t interface { mock.TestingT Cleanup(func()) }) *Manager { mock := &Manager{} mock.Mock.Test(t) t.Cleanup(func() { mock.AssertExpectations(t) }) return mock } ================================================ FILE: pkg/repository/mocks/RepositoryWriter.go ================================================ // Code generated by mockery v2.39.1. DO NOT EDIT. package mocks import ( context "context" index "github.com/kopia/kopia/repo/content/index" manifest "github.com/kopia/kopia/repo/manifest" mock "github.com/stretchr/testify/mock" object "github.com/kopia/kopia/repo/object" repo "github.com/kopia/kopia/repo" time "time" ) // RepositoryWriter is an autogenerated mock type for the RepositoryWriter type type RepositoryWriter struct { mock.Mock } // ClientOptions provides a mock function with given fields: func (_m *RepositoryWriter) ClientOptions() repo.ClientOptions { ret := _m.Called() if len(ret) == 0 { panic("no return value specified for ClientOptions") } var r0 repo.ClientOptions if rf, ok := ret.Get(0).(func() repo.ClientOptions); ok { r0 = rf() } else { r0 = ret.Get(0).(repo.ClientOptions) } return r0 } // Close provides a mock function with given fields: ctx func (_m *RepositoryWriter) Close(ctx context.Context) error { ret := _m.Called(ctx) if len(ret) == 0 { panic("no return value specified for Close") } var r0 error if rf, ok := ret.Get(0).(func(context.Context) error); ok { r0 = rf(ctx) } else { r0 = ret.Error(0) } return r0 } // ConcatenateObjects provides a mock function with given fields: ctx, objectIDs, opt func (_m *RepositoryWriter) ConcatenateObjects(ctx context.Context, objectIDs []object.ID, opt repo.ConcatenateOptions) (object.ID, error) { ret := _m.Called(ctx, objectIDs, opt) if len(ret) == 0 { panic("no return value specified for ConcatenateObjects") } var r0 object.ID var r1 error if rf, ok := ret.Get(0).(func(context.Context, []object.ID, repo.ConcatenateOptions) (object.ID, error)); ok { return rf(ctx, objectIDs, opt) } if rf, ok := ret.Get(0).(func(context.Context, []object.ID, repo.ConcatenateOptions) object.ID); ok { r0 = rf(ctx, objectIDs, opt) } else { r0 = ret.Get(0).(object.ID) } if rf, ok := ret.Get(1).(func(context.Context, []object.ID, repo.ConcatenateOptions) error); ok { r1 = rf(ctx, objectIDs, opt) } else { r1 = ret.Error(1) } return r0, r1 } // ContentInfo provides a mock function with given fields: ctx, contentID func (_m *RepositoryWriter) ContentInfo(ctx context.Context, contentID index.ID) (index.Info, error) { ret := _m.Called(ctx, contentID) if len(ret) == 0 { panic("no return value specified for ContentInfo") } var r0 index.Info var r1 error if rf, ok := ret.Get(0).(func(context.Context, index.ID) (index.Info, error)); ok { return rf(ctx, contentID) } if rf, ok := ret.Get(0).(func(context.Context, index.ID) index.Info); ok { r0 = rf(ctx, contentID) } else { r0 = ret.Get(0).(index.Info) } if rf, ok := ret.Get(1).(func(context.Context, index.ID) error); ok { r1 = rf(ctx, contentID) } else { r1 = ret.Error(1) } return r0, r1 } // DeleteManifest provides a mock function with given fields: ctx, id func (_m *RepositoryWriter) DeleteManifest(ctx context.Context, id manifest.ID) error { ret := _m.Called(ctx, id) if len(ret) == 0 { panic("no return value specified for DeleteManifest") } var r0 error if rf, ok := ret.Get(0).(func(context.Context, manifest.ID) error); ok { r0 = rf(ctx, id) } else { r0 = ret.Error(0) } return r0 } // FindManifests provides a mock function with given fields: ctx, labels func (_m *RepositoryWriter) FindManifests(ctx context.Context, labels map[string]string) ([]*manifest.EntryMetadata, error) { ret := _m.Called(ctx, labels) if len(ret) == 0 { panic("no return value specified for FindManifests") } var r0 []*manifest.EntryMetadata var r1 error if rf, ok := ret.Get(0).(func(context.Context, map[string]string) ([]*manifest.EntryMetadata, error)); ok { return rf(ctx, labels) } if rf, ok := ret.Get(0).(func(context.Context, map[string]string) []*manifest.EntryMetadata); ok { r0 = rf(ctx, labels) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]*manifest.EntryMetadata) } } if rf, ok := ret.Get(1).(func(context.Context, map[string]string) error); ok { r1 = rf(ctx, labels) } else { r1 = ret.Error(1) } return r0, r1 } // Flush provides a mock function with given fields: ctx func (_m *RepositoryWriter) Flush(ctx context.Context) error { ret := _m.Called(ctx) if len(ret) == 0 { panic("no return value specified for Flush") } var r0 error if rf, ok := ret.Get(0).(func(context.Context) error); ok { r0 = rf(ctx) } else { r0 = ret.Error(0) } return r0 } // GetManifest provides a mock function with given fields: ctx, id, data func (_m *RepositoryWriter) GetManifest(ctx context.Context, id manifest.ID, data interface{}) (*manifest.EntryMetadata, error) { ret := _m.Called(ctx, id, data) if len(ret) == 0 { panic("no return value specified for GetManifest") } var r0 *manifest.EntryMetadata var r1 error if rf, ok := ret.Get(0).(func(context.Context, manifest.ID, interface{}) (*manifest.EntryMetadata, error)); ok { return rf(ctx, id, data) } if rf, ok := ret.Get(0).(func(context.Context, manifest.ID, interface{}) *manifest.EntryMetadata); ok { r0 = rf(ctx, id, data) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(*manifest.EntryMetadata) } } if rf, ok := ret.Get(1).(func(context.Context, manifest.ID, interface{}) error); ok { r1 = rf(ctx, id, data) } else { r1 = ret.Error(1) } return r0, r1 } // NewObjectWriter provides a mock function with given fields: ctx, opt func (_m *RepositoryWriter) NewObjectWriter(ctx context.Context, opt object.WriterOptions) object.Writer { ret := _m.Called(ctx, opt) if len(ret) == 0 { panic("no return value specified for NewObjectWriter") } var r0 object.Writer if rf, ok := ret.Get(0).(func(context.Context, object.WriterOptions) object.Writer); ok { r0 = rf(ctx, opt) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(object.Writer) } } return r0 } // NewWriter provides a mock function with given fields: ctx, opt func (_m *RepositoryWriter) NewWriter(ctx context.Context, opt repo.WriteSessionOptions) (context.Context, repo.RepositoryWriter, error) { ret := _m.Called(ctx, opt) if len(ret) == 0 { panic("no return value specified for NewWriter") } var r0 context.Context var r1 repo.RepositoryWriter var r2 error if rf, ok := ret.Get(0).(func(context.Context, repo.WriteSessionOptions) (context.Context, repo.RepositoryWriter, error)); ok { return rf(ctx, opt) } if rf, ok := ret.Get(0).(func(context.Context, repo.WriteSessionOptions) context.Context); ok { r0 = rf(ctx, opt) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(context.Context) } } if rf, ok := ret.Get(1).(func(context.Context, repo.WriteSessionOptions) repo.RepositoryWriter); ok { r1 = rf(ctx, opt) } else { if ret.Get(1) != nil { r1 = ret.Get(1).(repo.RepositoryWriter) } } if rf, ok := ret.Get(2).(func(context.Context, repo.WriteSessionOptions) error); ok { r2 = rf(ctx, opt) } else { r2 = ret.Error(2) } return r0, r1, r2 } // OnSuccessfulFlush provides a mock function with given fields: callback func (_m *RepositoryWriter) OnSuccessfulFlush(callback repo.RepositoryWriterCallback) { _m.Called(callback) } // OpenObject provides a mock function with given fields: ctx, id func (_m *RepositoryWriter) OpenObject(ctx context.Context, id object.ID) (object.Reader, error) { ret := _m.Called(ctx, id) if len(ret) == 0 { panic("no return value specified for OpenObject") } var r0 object.Reader var r1 error if rf, ok := ret.Get(0).(func(context.Context, object.ID) (object.Reader, error)); ok { return rf(ctx, id) } if rf, ok := ret.Get(0).(func(context.Context, object.ID) object.Reader); ok { r0 = rf(ctx, id) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(object.Reader) } } if rf, ok := ret.Get(1).(func(context.Context, object.ID) error); ok { r1 = rf(ctx, id) } else { r1 = ret.Error(1) } return r0, r1 } // PrefetchContents provides a mock function with given fields: ctx, contentIDs, hint func (_m *RepositoryWriter) PrefetchContents(ctx context.Context, contentIDs []index.ID, hint string) []index.ID { ret := _m.Called(ctx, contentIDs, hint) if len(ret) == 0 { panic("no return value specified for PrefetchContents") } var r0 []index.ID if rf, ok := ret.Get(0).(func(context.Context, []index.ID, string) []index.ID); ok { r0 = rf(ctx, contentIDs, hint) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]index.ID) } } return r0 } // PrefetchObjects provides a mock function with given fields: ctx, objectIDs, hint func (_m *RepositoryWriter) PrefetchObjects(ctx context.Context, objectIDs []object.ID, hint string) ([]index.ID, error) { ret := _m.Called(ctx, objectIDs, hint) if len(ret) == 0 { panic("no return value specified for PrefetchObjects") } var r0 []index.ID var r1 error if rf, ok := ret.Get(0).(func(context.Context, []object.ID, string) ([]index.ID, error)); ok { return rf(ctx, objectIDs, hint) } if rf, ok := ret.Get(0).(func(context.Context, []object.ID, string) []index.ID); ok { r0 = rf(ctx, objectIDs, hint) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]index.ID) } } if rf, ok := ret.Get(1).(func(context.Context, []object.ID, string) error); ok { r1 = rf(ctx, objectIDs, hint) } else { r1 = ret.Error(1) } return r0, r1 } // PutManifest provides a mock function with given fields: ctx, labels, payload func (_m *RepositoryWriter) PutManifest(ctx context.Context, labels map[string]string, payload interface{}) (manifest.ID, error) { ret := _m.Called(ctx, labels, payload) if len(ret) == 0 { panic("no return value specified for PutManifest") } var r0 manifest.ID var r1 error if rf, ok := ret.Get(0).(func(context.Context, map[string]string, interface{}) (manifest.ID, error)); ok { return rf(ctx, labels, payload) } if rf, ok := ret.Get(0).(func(context.Context, map[string]string, interface{}) manifest.ID); ok { r0 = rf(ctx, labels, payload) } else { r0 = ret.Get(0).(manifest.ID) } if rf, ok := ret.Get(1).(func(context.Context, map[string]string, interface{}) error); ok { r1 = rf(ctx, labels, payload) } else { r1 = ret.Error(1) } return r0, r1 } // Refresh provides a mock function with given fields: ctx func (_m *RepositoryWriter) Refresh(ctx context.Context) error { ret := _m.Called(ctx) if len(ret) == 0 { panic("no return value specified for Refresh") } var r0 error if rf, ok := ret.Get(0).(func(context.Context) error); ok { r0 = rf(ctx) } else { r0 = ret.Error(0) } return r0 } // ReplaceManifests provides a mock function with given fields: ctx, labels, payload func (_m *RepositoryWriter) ReplaceManifests(ctx context.Context, labels map[string]string, payload interface{}) (manifest.ID, error) { ret := _m.Called(ctx, labels, payload) if len(ret) == 0 { panic("no return value specified for ReplaceManifests") } var r0 manifest.ID var r1 error if rf, ok := ret.Get(0).(func(context.Context, map[string]string, interface{}) (manifest.ID, error)); ok { return rf(ctx, labels, payload) } if rf, ok := ret.Get(0).(func(context.Context, map[string]string, interface{}) manifest.ID); ok { r0 = rf(ctx, labels, payload) } else { r0 = ret.Get(0).(manifest.ID) } if rf, ok := ret.Get(1).(func(context.Context, map[string]string, interface{}) error); ok { r1 = rf(ctx, labels, payload) } else { r1 = ret.Error(1) } return r0, r1 } // Time provides a mock function with given fields: func (_m *RepositoryWriter) Time() time.Time { ret := _m.Called() if len(ret) == 0 { panic("no return value specified for Time") } var r0 time.Time if rf, ok := ret.Get(0).(func() time.Time); ok { r0 = rf() } else { r0 = ret.Get(0).(time.Time) } return r0 } // UpdateDescription provides a mock function with given fields: d func (_m *RepositoryWriter) UpdateDescription(d string) { _m.Called(d) } // VerifyObject provides a mock function with given fields: ctx, id func (_m *RepositoryWriter) VerifyObject(ctx context.Context, id object.ID) ([]index.ID, error) { ret := _m.Called(ctx, id) if len(ret) == 0 { panic("no return value specified for VerifyObject") } var r0 []index.ID var r1 error if rf, ok := ret.Get(0).(func(context.Context, object.ID) ([]index.ID, error)); ok { return rf(ctx, id) } if rf, ok := ret.Get(0).(func(context.Context, object.ID) []index.ID); ok { r0 = rf(ctx, id) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]index.ID) } } if rf, ok := ret.Get(1).(func(context.Context, object.ID) error); ok { r1 = rf(ctx, id) } else { r1 = ret.Error(1) } return r0, r1 } // NewRepositoryWriter creates a new instance of RepositoryWriter. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. func NewRepositoryWriter(t interface { mock.TestingT Cleanup(func()) }) *RepositoryWriter { mock := &RepositoryWriter{} mock.Mock.Test(t) t.Cleanup(func() { mock.AssertExpectations(t) }) return mock } ================================================ FILE: pkg/repository/provider/provider.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package provider import ( "context" "time" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) // RepoParam includes the parameters to manipulate a backup repository type RepoParam struct { BackupLocation *velerov1api.BackupStorageLocation BackupRepo *velerov1api.BackupRepository CacheDir string } // Provider defines the methods to manipulate a backup repository type Provider interface { ConfigProvider // InitRepo is to initialize a repository from a new storage place InitRepo(ctx context.Context, param RepoParam) error // ConnectToRepo is to establish the connection to a // storage place that a repository is already initialized ConnectToRepo(ctx context.Context, param RepoParam) error // PrepareRepo is a combination of InitRepo and ConnectToRepo, // it may do initializing + connecting, connecting only if the repository // is already initialized, or do nothing if the repository is already connected PrepareRepo(ctx context.Context, param RepoParam) error // BoostRepoConnect is used to re-ensure the local connection to the repo, // so that the followed operations could succeed in some environment reset // scenarios, for example, pod restart BoostRepoConnect(ctx context.Context, param RepoParam) error // PruneRepo does a full prune/maintenance of the repository PruneRepo(ctx context.Context, param RepoParam) error // EnsureUnlockRepo esures to remove any stale file locks in the storage EnsureUnlockRepo(ctx context.Context, param RepoParam) error // Forget is to delete a snapshot from the repository Forget(ctx context.Context, snapshotID string, param RepoParam) error // BatchForget is to delete a list of snapshots from the repository BatchForget(ctx context.Context, snapshotIDs []string, param RepoParam) []error } // ConfigProvider defines the methods to get configurations of a backup repository type ConfigProvider interface { // DefaultMaintenanceFrequency returns the default frequency to run maintenance DefaultMaintenanceFrequency() time.Duration // ClientSideCacheLimit returns the max cache size required on client side ClientSideCacheLimit(repoOption map[string]string) int64 } ================================================ FILE: pkg/repository/provider/restic.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package provider import ( "context" "strings" "time" "github.com/sirupsen/logrus" "github.com/vmware-tanzu/velero/internal/credentials" "github.com/vmware-tanzu/velero/pkg/repository/restic" "github.com/vmware-tanzu/velero/pkg/util/filesystem" ) func NewResticRepositoryProvider(credGetter credentials.CredentialGetter, fs filesystem.Interface, log logrus.FieldLogger) Provider { return &resticRepositoryProvider{ svc: restic.NewRepositoryService(credGetter, fs, log), } } type resticRepositoryProvider struct { svc *restic.RepositoryService } func (r *resticRepositoryProvider) InitRepo(ctx context.Context, param RepoParam) error { return r.svc.InitRepo(param.BackupLocation, param.BackupRepo) } func (r *resticRepositoryProvider) ConnectToRepo(ctx context.Context, param RepoParam) error { return r.svc.ConnectToRepo(param.BackupLocation, param.BackupRepo) } func (r *resticRepositoryProvider) PrepareRepo(ctx context.Context, param RepoParam) error { if err := r.ConnectToRepo(ctx, param); err != nil { // If the repository has not yet been initialized, the error message will always include // the following string. This is the only scenario where we should try to initialize it. // Other errors (e.g. "already locked") should be returned as-is since the repository // does already exist, but it can't be connected to. if strings.Contains(err.Error(), "Is there a repository at the following location?") { return r.InitRepo(ctx, param) } return err } return nil } func (r *resticRepositoryProvider) BoostRepoConnect(ctx context.Context, param RepoParam) error { return nil } func (r *resticRepositoryProvider) PruneRepo(ctx context.Context, param RepoParam) error { return r.svc.PruneRepo(param.BackupLocation, param.BackupRepo) } func (r *resticRepositoryProvider) EnsureUnlockRepo(ctx context.Context, param RepoParam) error { return r.svc.UnlockRepo(param.BackupLocation, param.BackupRepo) } func (r *resticRepositoryProvider) Forget(ctx context.Context, snapshotID string, param RepoParam) error { return r.svc.Forget(param.BackupLocation, param.BackupRepo, snapshotID) } func (r *resticRepositoryProvider) BatchForget(ctx context.Context, snapshotIDs []string, param RepoParam) []error { errs := []error{} for _, snapshot := range snapshotIDs { err := r.Forget(ctx, snapshot, param) if err != nil { errs = append(errs, err) } } return errs } func (r *resticRepositoryProvider) DefaultMaintenanceFrequency() time.Duration { return r.svc.DefaultMaintenanceFrequency() } func (r *resticRepositoryProvider) ClientSideCacheLimit(repoOption map[string]string) int64 { return 0 } ================================================ FILE: pkg/repository/provider/unified_repo.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package provider import ( "context" "encoding/base64" "fmt" "maps" "net/url" "path" "strconv" "strings" "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/vmware-tanzu/velero/internal/credentials" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" repoconfig "github.com/vmware-tanzu/velero/pkg/repository/config" repokey "github.com/vmware-tanzu/velero/pkg/repository/keys" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" reposervice "github.com/vmware-tanzu/velero/pkg/repository/udmrepo/service" ) type unifiedRepoProvider struct { credentialGetter credentials.CredentialGetter workPath string repoService udmrepo.BackupRepoService repoBackend string log logrus.FieldLogger } type unifiedRepoConfigProvider struct { repoService udmrepo.BackupRepoService repoBackend string log logrus.FieldLogger } // this func is assigned to a package-level variable so it can be // replaced when unit-testing var getS3Credentials = repoconfig.GetS3Credentials var getGCPCredentials = repoconfig.GetGCPCredentials var getS3BucketRegion = repoconfig.GetAWSBucketRegion type localFuncTable struct { getStorageVariables func(*velerov1api.BackupStorageLocation, string, string, map[string]string, credentials.CredentialGetter) (map[string]string, error) getStorageCredentials func(*velerov1api.BackupStorageLocation, credentials.FileStore) (map[string]string, error) } var funcTable = localFuncTable{ getStorageVariables: getStorageVariables, getStorageCredentials: getStorageCredentials, } const ( repoOpDescMaintain = "repo maintenance" repoOpDescForget = "forget" repoConnectDesc = "unified repo" ) // NewUnifiedRepoProvider creates the service provider for Unified Repo func NewUnifiedRepoProvider( credentialGetter credentials.CredentialGetter, repoBackend string, log logrus.FieldLogger, ) Provider { repo := unifiedRepoProvider{ credentialGetter: credentialGetter, repoBackend: repoBackend, log: log, } repo.repoService = createRepoService(repoBackend, log) return &repo } func NewUnifiedRepoConfigProvider( repoBackend string, log logrus.FieldLogger, ) ConfigProvider { repo := unifiedRepoConfigProvider{ repoBackend: repoBackend, log: log, } repo.repoService = createRepoService(repoBackend, log) return &repo } func (urp *unifiedRepoProvider) InitRepo(ctx context.Context, param RepoParam) error { log := urp.log.WithFields(logrus.Fields{ "BSL name": param.BackupLocation.Name, "repo name": param.BackupRepo.Name, "repo UID": param.BackupRepo.UID, }) log.Debug("Start to init repo") if param.BackupLocation.Spec.AccessMode == velerov1api.BackupStorageLocationAccessModeReadOnly { return errors.Errorf("cannot create new backup repo for read-only backup storage location %s/%s", param.BackupLocation.Namespace, param.BackupLocation.Name) } repoOption, err := udmrepo.NewRepoOptions( udmrepo.WithPassword(urp, param), udmrepo.WithConfigFile(urp.workPath, string(param.BackupRepo.UID)), udmrepo.WithGenOptions( map[string]string{ udmrepo.GenOptionOwnerName: udmrepo.GetRepoUser(), udmrepo.GenOptionOwnerDomain: udmrepo.GetRepoDomain(), }, ), udmrepo.WithStoreOptions(urp, param), udmrepo.WithDescription(repoConnectDesc), ) if err != nil { return errors.Wrap(err, "error to get repo options") } err = urp.repoService.Create(ctx, *repoOption) if err != nil { return errors.Wrap(err, "error to init backup repo") } log.Debug("Init repo complete") return nil } func (urp *unifiedRepoProvider) ConnectToRepo(ctx context.Context, param RepoParam) error { log := urp.log.WithFields(logrus.Fields{ "BSL name": param.BackupLocation.Name, "repo name": param.BackupRepo.Name, "repo UID": param.BackupRepo.UID, }) log.Debug("Start to connect repo") repoOption, err := udmrepo.NewRepoOptions( udmrepo.WithPassword(urp, param), udmrepo.WithConfigFile(urp.workPath, string(param.BackupRepo.UID)), udmrepo.WithGenOptions( map[string]string{ udmrepo.GenOptionOwnerName: udmrepo.GetRepoUser(), udmrepo.GenOptionOwnerDomain: udmrepo.GetRepoDomain(), }, ), udmrepo.WithStoreOptions(urp, param), udmrepo.WithDescription(repoConnectDesc), ) if err != nil { return errors.Wrap(err, "error to get repo options") } err = urp.repoService.Connect(ctx, *repoOption) if err != nil { return errors.Wrap(err, "error to connect backup repo") } log.Debug("Connect repo complete") return nil } func (urp *unifiedRepoProvider) PrepareRepo(ctx context.Context, param RepoParam) error { log := urp.log.WithFields(logrus.Fields{ "BSL name": param.BackupLocation.Name, "repo name": param.BackupRepo.Name, "repo UID": param.BackupRepo.UID, }) log.Info("Start to prepare repo") repoOption, err := udmrepo.NewRepoOptions( udmrepo.WithPassword(urp, param), udmrepo.WithConfigFile(urp.workPath, string(param.BackupRepo.UID)), udmrepo.WithGenOptions( map[string]string{ udmrepo.GenOptionOwnerName: udmrepo.GetRepoUser(), udmrepo.GenOptionOwnerDomain: udmrepo.GetRepoDomain(), }, ), udmrepo.WithStoreOptions(urp, param), udmrepo.WithDescription(repoConnectDesc), ) if err != nil { return errors.Wrap(err, "error to get repo options") } if created, err := urp.repoService.IsCreated(ctx, *repoOption); err != nil { return errors.Wrap(err, "error to check backup repo") } else if created { log.Info("Repo has already been initialized") return nil } if param.BackupLocation.Spec.AccessMode == velerov1api.BackupStorageLocationAccessModeReadOnly { return errors.Errorf("cannot create new backup repo for read-only backup storage location %s/%s", param.BackupLocation.Namespace, param.BackupLocation.Name) } err = urp.repoService.Create(ctx, *repoOption) if err != nil { return errors.Wrap(err, "error to create backup repo") } log.Info("Prepare repo complete") return nil } func (urp *unifiedRepoProvider) BoostRepoConnect(ctx context.Context, param RepoParam) error { log := urp.log.WithFields(logrus.Fields{ "BSL name": param.BackupLocation.Name, "repo name": param.BackupRepo.Name, "repo UID": param.BackupRepo.UID, }) log.Debug("Start to boost repo connect") repoOption, err := udmrepo.NewRepoOptions( udmrepo.WithPassword(urp, param), udmrepo.WithConfigFile(urp.workPath, string(param.BackupRepo.UID)), udmrepo.WithDescription(repoConnectDesc), ) if err != nil { return errors.Wrap(err, "error to get repo options") } bkRepo, err := urp.repoService.Open(ctx, *repoOption) if err == nil { if c := bkRepo.Close(ctx); c != nil { log.WithError(c).Error("Failed to close repo") } return nil } return urp.ConnectToRepo(ctx, param) } func (urp *unifiedRepoProvider) PruneRepo(ctx context.Context, param RepoParam) error { log := urp.log.WithFields(logrus.Fields{ "BSL name": param.BackupLocation.Name, "repo name": param.BackupRepo.Name, "repo UID": param.BackupRepo.UID, }) log.Debug("Start to prune repo") repoOption, err := udmrepo.NewRepoOptions( udmrepo.WithPassword(urp, param), udmrepo.WithConfigFile(urp.workPath, string(param.BackupRepo.UID)), udmrepo.WithDescription(repoOpDescMaintain), ) if err != nil { return errors.Wrap(err, "error to get repo options") } err = urp.repoService.Maintain(ctx, *repoOption) if err != nil { return errors.Wrap(err, "error to prune backup repo") } log.Debug("Prune repo complete") return nil } func (urp *unifiedRepoProvider) EnsureUnlockRepo(ctx context.Context, param RepoParam) error { return nil } func (urp *unifiedRepoProvider) Forget(ctx context.Context, snapshotID string, param RepoParam) error { log := urp.log.WithFields(logrus.Fields{ "BSL name": param.BackupLocation.Name, "repo name": param.BackupRepo.Name, "repo UID": param.BackupRepo.UID, "snapshotID": snapshotID, }) log.Debug("Start to forget snapshot") repoOption, err := udmrepo.NewRepoOptions( udmrepo.WithPassword(urp, param), udmrepo.WithConfigFile(urp.workPath, string(param.BackupRepo.UID)), udmrepo.WithDescription(repoOpDescForget), ) if err != nil { return errors.Wrap(err, "error to get repo options") } bkRepo, err := urp.repoService.Open(ctx, *repoOption) if err != nil { return errors.Wrap(err, "error to open backup repo") } defer func() { c := bkRepo.Close(ctx) if c != nil { log.WithError(c).Error("Failed to close repo") } }() err = bkRepo.DeleteManifest(ctx, udmrepo.ID(snapshotID)) if err != nil { return errors.Wrap(err, "error to delete manifest") } err = bkRepo.Flush(ctx) if err != nil { return errors.Wrap(err, "error to flush repo") } log.Debug("Forget snapshot complete") return nil } func (urp *unifiedRepoProvider) BatchForget(ctx context.Context, snapshotIDs []string, param RepoParam) []error { log := urp.log.WithFields(logrus.Fields{ "BSL name": param.BackupLocation.Name, "repo name": param.BackupRepo.Name, "repo UID": param.BackupRepo.UID, "snapshotIDs": snapshotIDs, }) log.Debug("Start to batch forget snapshot") repoOption, err := udmrepo.NewRepoOptions( udmrepo.WithPassword(urp, param), udmrepo.WithConfigFile(urp.workPath, string(param.BackupRepo.UID)), udmrepo.WithDescription(repoOpDescForget), ) if err != nil { return []error{errors.Wrap(err, "error to get repo options")} } bkRepo, err := urp.repoService.Open(ctx, *repoOption) if err != nil { return []error{errors.Wrap(err, "error to open backup repo")} } defer func() { c := bkRepo.Close(ctx) if c != nil { log.WithError(c).Error("Failed to close repo") } }() errs := []error{} for _, snapshotID := range snapshotIDs { err = bkRepo.DeleteManifest(ctx, udmrepo.ID(snapshotID)) if err != nil { errs = append(errs, errors.Wrapf(err, "error to delete manifest %s", snapshotID)) } } err = bkRepo.Flush(ctx) if err != nil { return []error{errors.Wrap(err, "error to flush repo")} } log.Debug("Forget snapshot complete") return errs } func (urp *unifiedRepoProvider) DefaultMaintenanceFrequency() time.Duration { return urp.repoService.DefaultMaintenanceFrequency() } func (urp *unifiedRepoProvider) ClientSideCacheLimit(repoOption map[string]string) int64 { return urp.repoService.ClientSideCacheLimit(repoOption) } func (urp *unifiedRepoProvider) GetPassword(param any) (string, error) { _, ok := param.(RepoParam) if !ok { return "", errors.Errorf("invalid parameter, expect %T, actual %T", RepoParam{}, param) } repoPassword, err := getRepoPassword(urp.credentialGetter.FromSecret) if err != nil { return "", errors.Wrap(err, "error to get repo password") } return repoPassword, nil } func (urp *unifiedRepoProvider) GetStoreType(param any) (string, error) { repoParam, ok := param.(RepoParam) if !ok { return "", errors.Errorf("invalid parameter, expect %T, actual %T", RepoParam{}, param) } return getStorageType(repoParam.BackupLocation), nil } func (urp *unifiedRepoProvider) GetStoreOptions(param any) (map[string]string, error) { repoParam, ok := param.(RepoParam) if !ok { return map[string]string{}, errors.Errorf("invalid parameter, expect %T, actual %T", RepoParam{}, param) } storeVar, err := funcTable.getStorageVariables(repoParam.BackupLocation, urp.repoBackend, repoParam.BackupRepo.Spec.VolumeNamespace, repoParam.BackupRepo.Spec.RepositoryConfig, urp.credentialGetter) if err != nil { return map[string]string{}, errors.Wrap(err, "error to get storage variables") } storeCred, err := funcTable.getStorageCredentials(repoParam.BackupLocation, urp.credentialGetter.FromFile) if err != nil { return map[string]string{}, errors.Wrap(err, "error to get repo credentials") } storeOptions := make(map[string]string) maps.Copy(storeOptions, storeVar) maps.Copy(storeOptions, storeCred) if repoParam.CacheDir != "" { storeOptions[udmrepo.StoreOptionCacheDir] = repoParam.CacheDir } return storeOptions, nil } func (urcp *unifiedRepoConfigProvider) DefaultMaintenanceFrequency() time.Duration { return urcp.repoService.DefaultMaintenanceFrequency() } func (urcp *unifiedRepoConfigProvider) ClientSideCacheLimit(repoOption map[string]string) int64 { return urcp.repoService.ClientSideCacheLimit(repoOption) } func getRepoPassword(secretStore credentials.SecretStore) (string, error) { if secretStore == nil { return "", errors.New("invalid credentials interface") } rawPass, err := secretStore.Get(repokey.RepoKeySelector()) if err != nil { return "", errors.Wrap(err, "error to get password") } return strings.TrimSpace(rawPass), nil } func getStorageType(backupLocation *velerov1api.BackupStorageLocation) string { backendType := repoconfig.GetBackendType(backupLocation.Spec.Provider, backupLocation.Spec.Config) switch backendType { case repoconfig.AWSBackend: return udmrepo.StorageTypeS3 case repoconfig.AzureBackend: return udmrepo.StorageTypeAzure case repoconfig.GCPBackend: return udmrepo.StorageTypeGcs case repoconfig.FSBackend: return udmrepo.StorageTypeFs default: return "" } } func getStorageCredentials(backupLocation *velerov1api.BackupStorageLocation, credentialsFileStore credentials.FileStore) (map[string]string, error) { result := make(map[string]string) var err error if credentialsFileStore == nil { return map[string]string{}, errors.New("invalid credentials interface") } backendType := repoconfig.GetBackendType(backupLocation.Spec.Provider, backupLocation.Spec.Config) if !repoconfig.IsBackendTypeValid(backendType) { return map[string]string{}, errors.New("invalid storage provider") } config := backupLocation.Spec.Config if config == nil { config = map[string]string{} } if backupLocation.Spec.Credential != nil { config[repoconfig.CredentialsFileKey], err = credentialsFileStore.Path(backupLocation.Spec.Credential) if err != nil { return map[string]string{}, errors.Wrap(err, "error get credential file in bsl") } } switch backendType { case repoconfig.AWSBackend: credValue, err := getS3Credentials(config) if err != nil { return map[string]string{}, errors.Wrap(err, "error get s3 credentials") } if credValue != nil { result[udmrepo.StoreOptionS3KeyID] = credValue.AccessKeyID result[udmrepo.StoreOptionS3Provider] = credValue.Source result[udmrepo.StoreOptionS3SecretKey] = credValue.SecretAccessKey result[udmrepo.StoreOptionS3Token] = credValue.SessionToken } case repoconfig.AzureBackend: if config[repoconfig.CredentialsFileKey] != "" { result[repoconfig.CredentialsFileKey] = config[repoconfig.CredentialsFileKey] } case repoconfig.GCPBackend: result[udmrepo.StoreOptionCredentialFile] = getGCPCredentials(config) } return result, nil } // Translates user specified options (backupRepoConfig) to internal parameters // so we would accept only the options that are well defined in the internal system. // Users' inputs should not be treated as safe any time. // We remove the unnecessary parameters and keep the modules/logics below safe func getStorageVariables(backupLocation *velerov1api.BackupStorageLocation, repoBackend string, repoName string, backupRepoConfig map[string]string, credGetter credentials.CredentialGetter) (map[string]string, error) { result := make(map[string]string) backendType := repoconfig.GetBackendType(backupLocation.Spec.Provider, backupLocation.Spec.Config) if !repoconfig.IsBackendTypeValid(backendType) { return map[string]string{}, errors.New("invalid storage provider") } config := backupLocation.Spec.Config if config == nil { config = map[string]string{} } bucket := strings.Trim(config["bucket"], "/") prefix := strings.Trim(config["prefix"], "/") if backupLocation.Spec.ObjectStorage != nil { bucket = strings.Trim(backupLocation.Spec.ObjectStorage.Bucket, "/") prefix = strings.Trim(backupLocation.Spec.ObjectStorage.Prefix, "/") } prefix = path.Join(prefix, repoBackend, repoName) + "/" region := config["region"] if backendType == repoconfig.AWSBackend { s3URL := config["s3Url"] disableTLS := false var err error if s3URL == "" { if region == "" { region, err = getS3BucketRegion(bucket, config) if err != nil { return map[string]string{}, errors.Wrap(err, "error get s3 bucket region") } } s3URL = fmt.Sprintf("s3-%s.amazonaws.com", region) disableTLS = false } else { url, err := url.Parse(s3URL) if err != nil { return map[string]string{}, errors.Wrapf(err, "error to parse s3Url %s", s3URL) } if url.Path != "" && url.Path != "/" { return map[string]string{}, errors.Errorf("path is not expected in s3Url %s", s3URL) } s3URL = url.Host disableTLS = url.Scheme == "http" } result[udmrepo.StoreOptionS3Endpoint] = strings.Trim(s3URL, "/") result[udmrepo.StoreOptionS3DisableTLSVerify] = config["insecureSkipTLSVerify"] result[udmrepo.StoreOptionS3DisableTLS] = strconv.FormatBool(disableTLS) } else if backendType == repoconfig.AzureBackend { for k, v := range config { result[k] = v } } result[udmrepo.StoreOptionOssBucket] = bucket result[udmrepo.StoreOptionPrefix] = prefix if backupLocation.Spec.ObjectStorage != nil { var caCertData []byte // Try CACertRef first (new method), then fall back to CACert (deprecated) if backupLocation.Spec.ObjectStorage.CACertRef != nil { caCertString, err := credGetter.FromSecret.Get(backupLocation.Spec.ObjectStorage.CACertRef) if err != nil { return nil, errors.Wrap(err, "error getting CA certificate from secret") } caCertData = []byte(caCertString) } else if backupLocation.Spec.ObjectStorage.CACert != nil { caCertData = backupLocation.Spec.ObjectStorage.CACert } if caCertData != nil { result[udmrepo.StoreOptionCACert] = base64.StdEncoding.EncodeToString(caCertData) } } result[udmrepo.StoreOptionOssRegion] = strings.Trim(region, "/") result[udmrepo.StoreOptionFsPath] = config["fspath"] // We remove the unnecessary parameters and keep the modules/logics below safe if backupRepoConfig != nil { // range of valid params to keep, everything else will be discarded. validParams := []string{ udmrepo.StoreOptionCacheLimit, udmrepo.StoreOptionKeyFullMaintenanceInterval, } for _, param := range validParams { if v, found := backupRepoConfig[param]; found { result[param] = v } } } return result, nil } func createRepoService(repoBackend string, log logrus.FieldLogger) udmrepo.BackupRepoService { return reposervice.Create(repoBackend, log) } ================================================ FILE: pkg/repository/provider/unified_repo_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package provider import ( "context" "encoding/base64" "errors" "testing" "github.com/aws/aws-sdk-go-v2/aws" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" velerocredentials "github.com/vmware-tanzu/velero/internal/credentials" credmock "github.com/vmware-tanzu/velero/internal/credentials/mocks" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" reposervicenmocks "github.com/vmware-tanzu/velero/pkg/repository/udmrepo/mocks" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func TestGetStorageCredentials(t *testing.T) { testCases := []struct { name string backupLocation velerov1api.BackupStorageLocation credFileStore *credmock.FileStore credStoreError error credStorePath string getS3Credentials func(map[string]string) (*aws.Credentials, error) getGCPCredentials func(map[string]string) string expected map[string]string expectedErr string }{ { name: "invalid credentials file store interface", expected: map[string]string{}, expectedErr: "invalid credentials interface", }, { name: "invalid provider", backupLocation: velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Provider: "invalid-provider", }, }, credFileStore: new(credmock.FileStore), expected: map[string]string{}, expectedErr: "invalid storage provider", }, { name: "credential section exists in BSL, file store fail", backupLocation: velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Provider: "aws", Credential: &corev1api.SecretKeySelector{}, }, }, credFileStore: new(credmock.FileStore), credStoreError: errors.New("fake error"), expected: map[string]string{}, expectedErr: "error get credential file in bsl: fake error", }, { name: "aws, Credential section not exists in BSL", backupLocation: velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Provider: "velero.io/aws", Config: map[string]string{ "credentialsFile": "credentials-from-config-map", }, }, }, getS3Credentials: func(config map[string]string) (*aws.Credentials, error) { return &aws.Credentials{ AccessKeyID: "from: " + config["credentialsFile"], }, nil }, credFileStore: new(credmock.FileStore), expected: map[string]string{ "accessKeyID": "from: credentials-from-config-map", "providerName": "", "secretAccessKey": "", "sessionToken": "", }, }, { name: "aws, Credential section exists in BSL", backupLocation: velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Provider: "velero.io/aws", Config: map[string]string{ "credentialsFile": "credentials-from-config-map", }, Credential: &corev1api.SecretKeySelector{}, }, }, credFileStore: new(credmock.FileStore), credStorePath: "credentials-from-credential-key", getS3Credentials: func(config map[string]string) (*aws.Credentials, error) { return &aws.Credentials{ AccessKeyID: "from: " + config["credentialsFile"], }, nil }, expected: map[string]string{ "accessKeyID": "from: credentials-from-credential-key", "providerName": "", "secretAccessKey": "", "sessionToken": "", }, }, { name: "aws, get credentials fail", backupLocation: velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Provider: "velero.io/aws", Config: map[string]string{ "credentialsFile": "credentials-from-config-map", }, }, }, getS3Credentials: func(config map[string]string) (*aws.Credentials, error) { return nil, errors.New("fake error") }, credFileStore: new(credmock.FileStore), expected: map[string]string{}, expectedErr: "error get s3 credentials: fake error", }, { name: "aws, credential file not exist", backupLocation: velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Provider: "velero.io/aws", Config: map[string]string{}, }, }, getS3Credentials: func(config map[string]string) (*aws.Credentials, error) { return nil, nil }, credFileStore: new(credmock.FileStore), expected: map[string]string{}, }, { name: "azure", backupLocation: velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Provider: "velero.io/azure", Credential: &corev1api.SecretKeySelector{}, }, }, credFileStore: new(credmock.FileStore), expected: map[string]string{}, }, { name: "gcp, Credential section not exists in BSL", backupLocation: velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Provider: "velero.io/gcp", Config: map[string]string{ "credentialsFile": "credentials-from-config-map", }, }, }, getGCPCredentials: func(config map[string]string) string { return "credentials-from-config-map" }, credFileStore: new(credmock.FileStore), expected: map[string]string{ "credFile": "credentials-from-config-map", }, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { getS3Credentials = tc.getS3Credentials getGCPCredentials = tc.getGCPCredentials var fileStore velerocredentials.FileStore if tc.credFileStore != nil { tc.credFileStore.On("Path", mock.Anything, mock.Anything).Return(tc.credStorePath, tc.credStoreError) fileStore = tc.credFileStore } actual, err := getStorageCredentials(&tc.backupLocation, fileStore) require.Equal(t, tc.expected, actual) if tc.expectedErr == "" { assert.NoError(t, err) } else { assert.EqualError(t, err, tc.expectedErr) } }) } } func TestGetStorageVariables(t *testing.T) { testCases := []struct { name string backupLocation velerov1api.BackupStorageLocation credFileStore *credmock.FileStore repoName string repoBackend string repoConfig map[string]string getS3BucketRegion func(bucket string, config map[string]string) (string, error) expected map[string]string expectedErr string }{ { name: "invalid provider", backupLocation: velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Provider: "invalid-provider", }, }, expected: map[string]string{}, expectedErr: "invalid storage provider", }, { name: "aws, ObjectStorage section not exists in BSL, s3Url exist, https", backupLocation: velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Provider: "velero.io/aws", Config: map[string]string{ "bucket": "fake-bucket", "prefix": "fake-prefix", "region": "fake-region/", "s3Url": "https://fake-url/", "insecureSkipTLSVerify": "true", }, }, }, repoBackend: "fake-repo-type", expected: map[string]string{ "bucket": "fake-bucket", "prefix": "fake-prefix/fake-repo-type/", "region": "fake-region", "fspath": "", "endpoint": "fake-url", "doNotUseTLS": "false", "skipTLSVerify": "true", }, }, { name: "aws, ObjectStorage section not exists in BSL, s3Url exist, invalid", backupLocation: velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Provider: "velero.io/aws", Config: map[string]string{ "bucket": "fake-bucket", "prefix": "fake-prefix", "region": "fake-region/", "s3Url": "https://fake-url/fake-path", "insecureSkipTLSVerify": "true", }, }, }, repoBackend: "fake-repo-type", expected: map[string]string{}, expectedErr: "path is not expected in s3Url https://fake-url/fake-path", }, { name: "aws, ObjectStorage section not exists in BSL, s3Url not exist", backupLocation: velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Provider: "velero.io/aws", Config: map[string]string{ "bucket": "fake-bucket", "prefix": "fake-prefix", "insecureSkipTLSVerify": "false", }, }, }, getS3BucketRegion: func(bucket string, config map[string]string) (string, error) { return "region from bucket: " + bucket, nil }, repoBackend: "fake-repo-type", expected: map[string]string{ "bucket": "fake-bucket", "prefix": "fake-prefix/fake-repo-type/", "region": "region from bucket: fake-bucket", "fspath": "", "endpoint": "s3-region from bucket: fake-bucket.amazonaws.com", "doNotUseTLS": "false", "skipTLSVerify": "false", }, }, { name: "aws, ObjectStorage section not exists in BSL, s3Url not exist, get region fail", backupLocation: velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Provider: "velero.io/aws", Config: map[string]string{}, }, }, getS3BucketRegion: func(bucket string, config map[string]string) (string, error) { return "", errors.New("fake error") }, expected: map[string]string{}, expectedErr: "error get s3 bucket region: fake error", }, { name: "aws, ObjectStorage section exists in BSL, s3Url exist, http", backupLocation: velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Provider: "velero.io/aws", Config: map[string]string{ "bucket": "fake-bucket-config", "prefix": "fake-prefix-config", "region": "fake-region", "s3Url": "http://fake-url/", "insecureSkipTLSVerify": "false", }, StorageType: velerov1api.StorageType{ ObjectStorage: &velerov1api.ObjectStorageLocation{ Bucket: "fake-bucket-object-store", Prefix: "fake-prefix-object-store", }, }, }, }, getS3BucketRegion: func(bucket string, config map[string]string) (string, error) { return "region from bucket: " + bucket, nil }, repoBackend: "fake-repo-type", expected: map[string]string{ "bucket": "fake-bucket-object-store", "prefix": "fake-prefix-object-store/fake-repo-type/", "region": "fake-region", "fspath": "", "endpoint": "fake-url", "doNotUseTLS": "true", "skipTLSVerify": "false", }, }, { name: "aws, ObjectStorage section exists in BSL, s3Url exist, https, custom CA exist", backupLocation: velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Provider: "velero.io/aws", Config: map[string]string{ "bucket": "fake-bucket-config", "prefix": "fake-prefix-config", "region": "fake-region", "s3Url": "https://fake-url/", "insecureSkipTLSVerify": "false", }, StorageType: velerov1api.StorageType{ ObjectStorage: &velerov1api.ObjectStorageLocation{ Bucket: "fake-bucket-object-store", Prefix: "fake-prefix-object-store", CACert: []byte{0x01, 0x02, 0x03, 0x04, 0x05}, }, }, }, }, getS3BucketRegion: func(bucket string, config map[string]string) (string, error) { return "region from bucket: " + bucket, nil }, repoBackend: "fake-repo-type", expected: map[string]string{ "bucket": "fake-bucket-object-store", "prefix": "fake-prefix-object-store/fake-repo-type/", "region": "fake-region", "fspath": "", "endpoint": "fake-url", "doNotUseTLS": "false", "skipTLSVerify": "false", "caCert": base64.StdEncoding.EncodeToString([]byte{0x01, 0x02, 0x03, 0x04, 0x05}), }, }, { name: "azure", backupLocation: velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Provider: "velero.io/azure", Config: map[string]string{ "bucket": "fake-bucket-config", "prefix": "fake-prefix-config", "region": "fake-region", "fspath": "", }, StorageType: velerov1api.StorageType{ ObjectStorage: &velerov1api.ObjectStorageLocation{ Bucket: "fake-bucket-object-store", Prefix: "fake-prefix-object-store", }, }, }, }, credFileStore: new(credmock.FileStore), repoBackend: "fake-repo-type", expected: map[string]string{ "bucket": "fake-bucket-object-store", "prefix": "fake-prefix-object-store/fake-repo-type/", "region": "fake-region", "fspath": "", }, }, { name: "fs", backupLocation: velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Provider: "velero.io/fs", Config: map[string]string{ "fspath": "fake-path", "prefix": "fake-prefix", }, }, }, repoBackend: "fake-repo-type", expected: map[string]string{ "fspath": "fake-path", "bucket": "", "prefix": "fake-prefix/fake-repo-type/", "region": "", }, }, { name: "fs with repo config", backupLocation: velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Provider: "velero.io/fs", Config: map[string]string{ "fspath": "fake-path", "prefix": "fake-prefix", }, }, }, repoBackend: "fake-repo-type", repoConfig: map[string]string{ udmrepo.StoreOptionCacheLimit: "1000", }, expected: map[string]string{ "fspath": "fake-path", "bucket": "", "prefix": "fake-prefix/fake-repo-type/", "region": "", "cacheLimitMB": "1000", }, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { getS3BucketRegion = tc.getS3BucketRegion actual, err := getStorageVariables(&tc.backupLocation, tc.repoBackend, tc.repoName, tc.repoConfig, velerocredentials.CredentialGetter{}) require.Equal(t, tc.expected, actual) if tc.expectedErr == "" { assert.NoError(t, err) } else { assert.EqualError(t, err, tc.expectedErr) } }) } } func TestGetRepoPassword(t *testing.T) { testCases := []struct { name string getter *credmock.SecretStore credStoreReturn string credStoreError error cached string expected string expectedErr string }{ { name: "invalid secret interface", expectedErr: "invalid credentials interface", }, { name: "error from secret interface", getter: new(credmock.SecretStore), credStoreError: errors.New("fake error"), expectedErr: "error to get password: fake error", }, { name: "secret with whitespace", getter: new(credmock.SecretStore), credStoreReturn: " fake-passwor d ", expected: "fake-passwor d", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { var secretStore velerocredentials.SecretStore if tc.getter != nil { tc.getter.On("Get", mock.Anything, mock.Anything).Return(tc.credStoreReturn, tc.credStoreError) secretStore = tc.getter } urp := unifiedRepoProvider{ credentialGetter: velerocredentials.CredentialGetter{ FromSecret: secretStore, }, } password, err := getRepoPassword(urp.credentialGetter.FromSecret) require.Equal(t, tc.expected, password) if tc.expectedErr == "" { assert.NoError(t, err) } else { assert.EqualError(t, err, tc.expectedErr) } }) } } func TestGetStoreOptions(t *testing.T) { testCases := []struct { name string funcTable localFuncTable repoParam any expected map[string]string expectedErr string }{ { name: "wrong param type", repoParam: struct{}{}, expected: map[string]string{}, expectedErr: "invalid parameter, expect provider.RepoParam, actual struct {}", }, { name: "get storage variable fail", repoParam: RepoParam{ BackupLocation: &velerov1api.BackupStorageLocation{}, BackupRepo: &velerov1api.BackupRepository{}, }, funcTable: localFuncTable{ getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, map[string]string, velerocredentials.CredentialGetter) (map[string]string, error) { return map[string]string{}, errors.New("fake-error-2") }, }, expected: map[string]string{}, expectedErr: "error to get storage variables: fake-error-2", }, { name: "get storage credentials fail", repoParam: RepoParam{ BackupLocation: &velerov1api.BackupStorageLocation{}, BackupRepo: &velerov1api.BackupRepository{}, }, funcTable: localFuncTable{ getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, map[string]string, velerocredentials.CredentialGetter) (map[string]string, error) { return map[string]string{}, nil }, getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) { return map[string]string{}, errors.New("fake-error-3") }, }, expected: map[string]string{}, expectedErr: "error to get repo credentials: fake-error-3", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { funcTable = tc.funcTable urp := unifiedRepoProvider{} options, err := urp.GetStoreOptions(tc.repoParam) require.Equal(t, tc.expected, options) if tc.expectedErr == "" { assert.NoError(t, err) } else { assert.EqualError(t, err, tc.expectedErr) } }) } } func TestPrepareRepo(t *testing.T) { bsl := velerov1api.BackupStorageLocation{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-bsl", Namespace: velerov1api.DefaultNamespace, }, } testCases := []struct { name string funcTable localFuncTable getter *credmock.SecretStore repoService *reposervicenmocks.BackupRepoService retFuncCreate func(context.Context, udmrepo.RepoOptions) error retFuncCheck func(context.Context, udmrepo.RepoOptions) (bool, error) credStoreReturn string credStoreError error readOnlyBSL bool expectedErr string }{ { name: "get repo option fail", repoService: new(reposervicenmocks.BackupRepoService), expectedErr: "error to get repo options: error to get repo password: invalid credentials interface", }, { name: "get repo option fail, get password fail", getter: new(credmock.SecretStore), repoService: new(reposervicenmocks.BackupRepoService), credStoreError: errors.New("fake-password-error"), expectedErr: "error to get repo options: error to get repo password: error to get password: fake-password-error", }, { name: "get repo option fail, get store options fail", getter: new(credmock.SecretStore), repoService: new(reposervicenmocks.BackupRepoService), credStoreReturn: "fake-password", funcTable: localFuncTable{ getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, map[string]string, velerocredentials.CredentialGetter) (map[string]string, error) { return map[string]string{}, errors.New("fake-store-option-error") }, }, expectedErr: "error to get repo options: error to get storage variables: fake-store-option-error", }, { name: "check error", getter: new(credmock.SecretStore), credStoreReturn: "fake-password", funcTable: localFuncTable{ getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, map[string]string, velerocredentials.CredentialGetter) (map[string]string, error) { return map[string]string{}, nil }, getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) { return map[string]string{}, nil }, }, repoService: new(reposervicenmocks.BackupRepoService), retFuncCheck: func(ctx context.Context, repoOption udmrepo.RepoOptions) (bool, error) { return false, errors.New("fake-error") }, expectedErr: "error to check backup repo: fake-error", }, { name: "already initialized", getter: new(credmock.SecretStore), credStoreReturn: "fake-password", funcTable: localFuncTable{ getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, map[string]string, velerocredentials.CredentialGetter) (map[string]string, error) { return map[string]string{}, nil }, getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) { return map[string]string{}, nil }, }, repoService: new(reposervicenmocks.BackupRepoService), retFuncCheck: func(ctx context.Context, repoOption udmrepo.RepoOptions) (bool, error) { return true, nil }, retFuncCreate: func(ctx context.Context, repoOption udmrepo.RepoOptions) error { return errors.New("fake-error") }, }, { name: "bsl is readonly", readOnlyBSL: true, getter: new(credmock.SecretStore), credStoreReturn: "fake-password", funcTable: localFuncTable{ getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, map[string]string, velerocredentials.CredentialGetter) (map[string]string, error) { return map[string]string{}, nil }, getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) { return map[string]string{}, nil }, }, repoService: new(reposervicenmocks.BackupRepoService), retFuncCheck: func(ctx context.Context, repoOption udmrepo.RepoOptions) (bool, error) { return false, nil }, retFuncCreate: func(ctx context.Context, repoOption udmrepo.RepoOptions) error { return errors.New("fake-error-2") }, expectedErr: "cannot create new backup repo for read-only backup storage location velero/fake-bsl", }, { name: "create fail", getter: new(credmock.SecretStore), credStoreReturn: "fake-password", funcTable: localFuncTable{ getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, map[string]string, velerocredentials.CredentialGetter) (map[string]string, error) { return map[string]string{}, nil }, getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) { return map[string]string{}, nil }, }, repoService: new(reposervicenmocks.BackupRepoService), retFuncCheck: func(ctx context.Context, repoOption udmrepo.RepoOptions) (bool, error) { return false, nil }, retFuncCreate: func(ctx context.Context, repoOption udmrepo.RepoOptions) error { return errors.New("fake-error-1") }, expectedErr: "error to create backup repo: fake-error-1", }, { name: "initialize error", getter: new(credmock.SecretStore), credStoreReturn: "fake-password", funcTable: localFuncTable{ getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, map[string]string, velerocredentials.CredentialGetter) (map[string]string, error) { return map[string]string{}, nil }, getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) { return map[string]string{}, nil }, }, repoService: new(reposervicenmocks.BackupRepoService), retFuncCheck: func(ctx context.Context, repoOption udmrepo.RepoOptions) (bool, error) { return false, nil }, retFuncCreate: func(ctx context.Context, repoOption udmrepo.RepoOptions) error { return errors.New("fake-error-2") }, expectedErr: "error to create backup repo: fake-error-2", }, { name: "initialize succeed", getter: new(credmock.SecretStore), credStoreReturn: "fake-password", funcTable: localFuncTable{ getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, map[string]string, velerocredentials.CredentialGetter) (map[string]string, error) { return map[string]string{}, nil }, getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) { return map[string]string{}, nil }, }, repoService: new(reposervicenmocks.BackupRepoService), retFuncCheck: func(ctx context.Context, repoOption udmrepo.RepoOptions) (bool, error) { return false, nil }, retFuncCreate: func(ctx context.Context, repoOption udmrepo.RepoOptions) error { return nil }, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { funcTable = tc.funcTable var secretStore velerocredentials.SecretStore if tc.getter != nil { tc.getter.On("Get", mock.Anything, mock.Anything).Return(tc.credStoreReturn, tc.credStoreError) secretStore = tc.getter } urp := unifiedRepoProvider{ credentialGetter: velerocredentials.CredentialGetter{ FromSecret: secretStore, }, repoService: tc.repoService, log: velerotest.NewLogger(), } tc.repoService.On("IsCreated", mock.Anything, mock.Anything).Return(tc.retFuncCheck) tc.repoService.On("Create", mock.Anything, mock.Anything, mock.Anything).Return(tc.retFuncCreate) if tc.readOnlyBSL { bsl.Spec.AccessMode = velerov1api.BackupStorageLocationAccessModeReadOnly } else { bsl.Spec.AccessMode = velerov1api.BackupStorageLocationAccessModeReadWrite } err := urp.PrepareRepo(t.Context(), RepoParam{ BackupLocation: &bsl, BackupRepo: &velerov1api.BackupRepository{}, }) if tc.expectedErr == "" { assert.NoError(t, err) } else { assert.EqualError(t, err, tc.expectedErr) } }) } } func TestForget(t *testing.T) { var backupRepo *reposervicenmocks.BackupRepo testCases := []struct { name string funcTable localFuncTable getter *credmock.SecretStore repoService *reposervicenmocks.BackupRepoService backupRepo *reposervicenmocks.BackupRepo retFuncOpen []any retFuncDelete any retFuncFlush any credStoreReturn string credStoreError error expectedErr string }{ { name: "get repo option fail", expectedErr: "error to get repo options: error to get repo password: invalid credentials interface", }, { name: "repo open fail", getter: new(credmock.SecretStore), credStoreReturn: "fake-password", funcTable: localFuncTable{ getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, map[string]string, velerocredentials.CredentialGetter) (map[string]string, error) { return map[string]string{}, nil }, getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) { return map[string]string{}, nil }, }, repoService: new(reposervicenmocks.BackupRepoService), retFuncOpen: []any{ func(context.Context, udmrepo.RepoOptions) udmrepo.BackupRepo { return backupRepo }, func(context.Context, udmrepo.RepoOptions) error { return errors.New("fake-error-2") }, }, expectedErr: "error to open backup repo: fake-error-2", }, { name: "delete fail", getter: new(credmock.SecretStore), credStoreReturn: "fake-password", funcTable: localFuncTable{ getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, map[string]string, velerocredentials.CredentialGetter) (map[string]string, error) { return map[string]string{}, nil }, getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) { return map[string]string{}, nil }, }, repoService: new(reposervicenmocks.BackupRepoService), backupRepo: new(reposervicenmocks.BackupRepo), retFuncOpen: []any{ func(context.Context, udmrepo.RepoOptions) udmrepo.BackupRepo { return backupRepo }, func(context.Context, udmrepo.RepoOptions) error { return nil }, }, retFuncDelete: func(context.Context, udmrepo.ID) error { return errors.New("fake-error-3") }, expectedErr: "error to delete manifest: fake-error-3", }, { name: "flush fail", getter: new(credmock.SecretStore), credStoreReturn: "fake-password", funcTable: localFuncTable{ getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, map[string]string, velerocredentials.CredentialGetter) (map[string]string, error) { return map[string]string{}, nil }, getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) { return map[string]string{}, nil }, }, repoService: new(reposervicenmocks.BackupRepoService), backupRepo: new(reposervicenmocks.BackupRepo), retFuncOpen: []any{ func(context.Context, udmrepo.RepoOptions) udmrepo.BackupRepo { return backupRepo }, func(context.Context, udmrepo.RepoOptions) error { return nil }, }, retFuncDelete: func(context.Context, udmrepo.ID) error { return nil }, retFuncFlush: func(context.Context) error { return errors.New("fake-error-4") }, expectedErr: "error to flush repo: fake-error-4", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { funcTable = tc.funcTable var secretStore velerocredentials.SecretStore if tc.getter != nil { tc.getter.On("Get", mock.Anything, mock.Anything).Return(tc.credStoreReturn, tc.credStoreError) secretStore = tc.getter } urp := unifiedRepoProvider{ credentialGetter: velerocredentials.CredentialGetter{ FromSecret: secretStore, }, repoService: tc.repoService, log: velerotest.NewLogger(), } backupRepo = tc.backupRepo if tc.repoService != nil { tc.repoService.On("Open", mock.Anything, mock.Anything).Return(tc.retFuncOpen[0], tc.retFuncOpen[1]) } if tc.backupRepo != nil { backupRepo.On("DeleteManifest", mock.Anything, mock.Anything).Return(tc.retFuncDelete) backupRepo.On("Flush", mock.Anything).Return(tc.retFuncFlush) backupRepo.On("Close", mock.Anything).Return(nil) } err := urp.Forget(t.Context(), "", RepoParam{ BackupLocation: &velerov1api.BackupStorageLocation{}, BackupRepo: &velerov1api.BackupRepository{}, }) if tc.expectedErr == "" { assert.NoError(t, err) } else { assert.EqualError(t, err, tc.expectedErr) } }) } } func TestBatchForget(t *testing.T) { var backupRepo *reposervicenmocks.BackupRepo testCases := []struct { name string funcTable localFuncTable getter *credmock.SecretStore repoService *reposervicenmocks.BackupRepoService backupRepo *reposervicenmocks.BackupRepo retFuncOpen []any retFuncDelete any retFuncFlush any credStoreReturn string credStoreError error snapshots []string expectedErr []string }{ { name: "get repo option fail", expectedErr: []string{"error to get repo options: error to get repo password: invalid credentials interface"}, }, { name: "repo open fail", getter: new(credmock.SecretStore), credStoreReturn: "fake-password", funcTable: localFuncTable{ getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, map[string]string, velerocredentials.CredentialGetter) (map[string]string, error) { return map[string]string{}, nil }, getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) { return map[string]string{}, nil }, }, repoService: new(reposervicenmocks.BackupRepoService), retFuncOpen: []any{ func(context.Context, udmrepo.RepoOptions) udmrepo.BackupRepo { return backupRepo }, func(context.Context, udmrepo.RepoOptions) error { return errors.New("fake-error-2") }, }, expectedErr: []string{"error to open backup repo: fake-error-2"}, }, { name: "delete fail", getter: new(credmock.SecretStore), credStoreReturn: "fake-password", funcTable: localFuncTable{ getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, map[string]string, velerocredentials.CredentialGetter) (map[string]string, error) { return map[string]string{}, nil }, getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) { return map[string]string{}, nil }, }, repoService: new(reposervicenmocks.BackupRepoService), backupRepo: new(reposervicenmocks.BackupRepo), retFuncOpen: []any{ func(context.Context, udmrepo.RepoOptions) udmrepo.BackupRepo { return backupRepo }, func(context.Context, udmrepo.RepoOptions) error { return nil }, }, retFuncDelete: func(context.Context, udmrepo.ID) error { return errors.New("fake-error-3") }, snapshots: []string{"snapshot-1", "snapshot-2"}, expectedErr: []string{"error to delete manifest snapshot-1: fake-error-3", "error to delete manifest snapshot-2: fake-error-3"}, }, { name: "flush fail", getter: new(credmock.SecretStore), credStoreReturn: "fake-password", funcTable: localFuncTable{ getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, map[string]string, velerocredentials.CredentialGetter) (map[string]string, error) { return map[string]string{}, nil }, getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) { return map[string]string{}, nil }, }, repoService: new(reposervicenmocks.BackupRepoService), backupRepo: new(reposervicenmocks.BackupRepo), retFuncOpen: []any{ func(context.Context, udmrepo.RepoOptions) udmrepo.BackupRepo { return backupRepo }, func(context.Context, udmrepo.RepoOptions) error { return nil }, }, retFuncDelete: func(context.Context, udmrepo.ID) error { return nil }, retFuncFlush: func(context.Context) error { return errors.New("fake-error-4") }, expectedErr: []string{"error to flush repo: fake-error-4"}, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { funcTable = tc.funcTable var secretStore velerocredentials.SecretStore if tc.getter != nil { tc.getter.On("Get", mock.Anything, mock.Anything).Return(tc.credStoreReturn, tc.credStoreError) secretStore = tc.getter } urp := unifiedRepoProvider{ credentialGetter: velerocredentials.CredentialGetter{ FromSecret: secretStore, }, repoService: tc.repoService, log: velerotest.NewLogger(), } backupRepo = tc.backupRepo if tc.repoService != nil { tc.repoService.On("Open", mock.Anything, mock.Anything).Return(tc.retFuncOpen[0], tc.retFuncOpen[1]) } if tc.backupRepo != nil { backupRepo.On("DeleteManifest", mock.Anything, mock.Anything).Return(tc.retFuncDelete) backupRepo.On("Flush", mock.Anything).Return(tc.retFuncFlush) backupRepo.On("Close", mock.Anything).Return(nil) } errs := urp.BatchForget(t.Context(), tc.snapshots, RepoParam{ BackupLocation: &velerov1api.BackupStorageLocation{}, BackupRepo: &velerov1api.BackupRepository{}, }) if tc.expectedErr == nil { assert.Empty(t, errs) } else { assert.Len(t, errs, len(tc.expectedErr)) for i := range tc.expectedErr { assert.EqualError(t, errs[i], tc.expectedErr[i]) } } }) } } func TestInitRepo(t *testing.T) { bsl := velerov1api.BackupStorageLocation{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-bsl", Namespace: velerov1api.DefaultNamespace, }, } testCases := []struct { name string funcTable localFuncTable getter *credmock.SecretStore repoService *reposervicenmocks.BackupRepoService retFuncInit any credStoreReturn string credStoreError error readOnlyBSL bool expectedErr string }{ { name: "bsl is readonly", readOnlyBSL: true, expectedErr: "cannot create new backup repo for read-only backup storage location velero/fake-bsl", }, { name: "get repo option fail", expectedErr: "error to get repo options: error to get repo password: invalid credentials interface", }, { name: "repo init fail", getter: new(credmock.SecretStore), credStoreReturn: "fake-password", funcTable: localFuncTable{ getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, map[string]string, velerocredentials.CredentialGetter) (map[string]string, error) { return map[string]string{}, nil }, getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) { return map[string]string{}, nil }, }, repoService: new(reposervicenmocks.BackupRepoService), retFuncInit: func(context.Context, udmrepo.RepoOptions) error { return errors.New("fake-error-1") }, expectedErr: "error to init backup repo: fake-error-1", }, { name: "succeed", getter: new(credmock.SecretStore), credStoreReturn: "fake-password", funcTable: localFuncTable{ getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, map[string]string, velerocredentials.CredentialGetter) (map[string]string, error) { return map[string]string{}, nil }, getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) { return map[string]string{}, nil }, }, repoService: new(reposervicenmocks.BackupRepoService), retFuncInit: func(context.Context, udmrepo.RepoOptions) error { return nil }, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { funcTable = tc.funcTable var secretStore velerocredentials.SecretStore if tc.getter != nil { tc.getter.On("Get", mock.Anything, mock.Anything).Return(tc.credStoreReturn, tc.credStoreError) secretStore = tc.getter } urp := unifiedRepoProvider{ credentialGetter: velerocredentials.CredentialGetter{ FromSecret: secretStore, }, repoService: tc.repoService, log: velerotest.NewLogger(), } if tc.repoService != nil { tc.repoService.On("Create", mock.Anything, mock.Anything).Return(tc.retFuncInit) } if tc.readOnlyBSL { bsl.Spec.AccessMode = velerov1api.BackupStorageLocationAccessModeReadOnly } else { bsl.Spec.AccessMode = velerov1api.BackupStorageLocationAccessModeReadWrite } err := urp.InitRepo(t.Context(), RepoParam{ BackupLocation: &bsl, BackupRepo: &velerov1api.BackupRepository{}, }) if tc.expectedErr == "" { assert.NoError(t, err) } else { assert.EqualError(t, err, tc.expectedErr) } }) } } func TestConnectToRepo(t *testing.T) { testCases := []struct { name string funcTable localFuncTable getter *credmock.SecretStore repoService *reposervicenmocks.BackupRepoService retFuncInit any credStoreReturn string credStoreError error expectedErr string }{ { name: "get repo option fail", expectedErr: "error to get repo options: error to get repo password: invalid credentials interface", }, { name: "repo init fail", getter: new(credmock.SecretStore), credStoreReturn: "fake-password", funcTable: localFuncTable{ getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, map[string]string, velerocredentials.CredentialGetter) (map[string]string, error) { return map[string]string{}, nil }, getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) { return map[string]string{}, nil }, }, repoService: new(reposervicenmocks.BackupRepoService), retFuncInit: func(context.Context, udmrepo.RepoOptions) error { return errors.New("fake-error-1") }, expectedErr: "error to connect backup repo: fake-error-1", }, { name: "succeed", getter: new(credmock.SecretStore), credStoreReturn: "fake-password", funcTable: localFuncTable{ getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, map[string]string, velerocredentials.CredentialGetter) (map[string]string, error) { return map[string]string{}, nil }, getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) { return map[string]string{}, nil }, }, repoService: new(reposervicenmocks.BackupRepoService), retFuncInit: func(context.Context, udmrepo.RepoOptions) error { return nil }, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { funcTable = tc.funcTable var secretStore velerocredentials.SecretStore if tc.getter != nil { tc.getter.On("Get", mock.Anything, mock.Anything).Return(tc.credStoreReturn, tc.credStoreError) secretStore = tc.getter } urp := unifiedRepoProvider{ credentialGetter: velerocredentials.CredentialGetter{ FromSecret: secretStore, }, repoService: tc.repoService, log: velerotest.NewLogger(), } if tc.repoService != nil { tc.repoService.On("Connect", mock.Anything, mock.Anything).Return(tc.retFuncInit) } err := urp.ConnectToRepo(t.Context(), RepoParam{ BackupLocation: &velerov1api.BackupStorageLocation{}, BackupRepo: &velerov1api.BackupRepository{}, }) if tc.expectedErr == "" { assert.NoError(t, err) } else { assert.EqualError(t, err, tc.expectedErr) } }) } } func TestBoostRepoConnect(t *testing.T) { var backupRepo *reposervicenmocks.BackupRepo testCases := []struct { name string funcTable localFuncTable getter *credmock.SecretStore repoService *reposervicenmocks.BackupRepoService backupRepo *reposervicenmocks.BackupRepo retFuncInit any retFuncOpen []any credStoreReturn string credStoreError error expectedErr string }{ { name: "get repo option fail", expectedErr: "error to get repo options: error to get repo password: invalid credentials interface", }, { name: "repo not opened and connect fail", getter: new(credmock.SecretStore), credStoreReturn: "fake-password", funcTable: localFuncTable{ getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, map[string]string, velerocredentials.CredentialGetter) (map[string]string, error) { return map[string]string{}, nil }, getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) { return map[string]string{}, nil }, }, repoService: new(reposervicenmocks.BackupRepoService), retFuncOpen: []any{ func(context.Context, udmrepo.RepoOptions) udmrepo.BackupRepo { return backupRepo }, func(context.Context, udmrepo.RepoOptions) error { return errors.New("fake-error-1") }, }, retFuncInit: func(context.Context, udmrepo.RepoOptions) error { return errors.New("fake-error-2") }, expectedErr: "error to connect backup repo: fake-error-2", }, { name: "repo not opened and connect succeed", getter: new(credmock.SecretStore), credStoreReturn: "fake-password", funcTable: localFuncTable{ getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, map[string]string, velerocredentials.CredentialGetter) (map[string]string, error) { return map[string]string{}, nil }, getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) { return map[string]string{}, nil }, }, repoService: new(reposervicenmocks.BackupRepoService), retFuncOpen: []any{ func(context.Context, udmrepo.RepoOptions) udmrepo.BackupRepo { return backupRepo }, func(context.Context, udmrepo.RepoOptions) error { return errors.New("fake-error-1") }, }, retFuncInit: func(context.Context, udmrepo.RepoOptions) error { return nil }, }, { name: "repo is opened", getter: new(credmock.SecretStore), credStoreReturn: "fake-password", funcTable: localFuncTable{ getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, map[string]string, velerocredentials.CredentialGetter) (map[string]string, error) { return map[string]string{}, nil }, getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) { return map[string]string{}, nil }, }, repoService: new(reposervicenmocks.BackupRepoService), backupRepo: new(reposervicenmocks.BackupRepo), retFuncOpen: []any{ func(context.Context, udmrepo.RepoOptions) udmrepo.BackupRepo { return backupRepo }, func(context.Context, udmrepo.RepoOptions) error { return nil }, }, retFuncInit: func(context.Context, udmrepo.RepoOptions) error { return nil }, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { funcTable = tc.funcTable var secretStore velerocredentials.SecretStore if tc.getter != nil { tc.getter.On("Get", mock.Anything, mock.Anything).Return(tc.credStoreReturn, tc.credStoreError) secretStore = tc.getter } urp := unifiedRepoProvider{ credentialGetter: velerocredentials.CredentialGetter{ FromSecret: secretStore, }, repoService: tc.repoService, log: velerotest.NewLogger(), } backupRepo = tc.backupRepo if tc.repoService != nil { tc.repoService.On("Open", mock.Anything, mock.Anything).Return(tc.retFuncOpen[0], tc.retFuncOpen[1]) tc.repoService.On("Connect", mock.Anything, mock.Anything).Return(tc.retFuncInit) } if tc.backupRepo != nil { backupRepo.On("Close", mock.Anything).Return(nil) } err := urp.BoostRepoConnect(t.Context(), RepoParam{ BackupLocation: &velerov1api.BackupStorageLocation{}, BackupRepo: &velerov1api.BackupRepository{}, }) if tc.expectedErr == "" { assert.NoError(t, err) } else { assert.EqualError(t, err, tc.expectedErr) } }) } } func TestPruneRepo(t *testing.T) { testCases := []struct { name string funcTable localFuncTable getter *credmock.SecretStore repoService *reposervicenmocks.BackupRepoService retFuncMaintain any credStoreReturn string credStoreError error expectedErr string }{ { name: "get repo option fail", expectedErr: "error to get repo options: error to get repo password: invalid credentials interface", }, { name: "repo maintain fail", getter: new(credmock.SecretStore), credStoreReturn: "fake-password", funcTable: localFuncTable{ getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, map[string]string, velerocredentials.CredentialGetter) (map[string]string, error) { return map[string]string{}, nil }, getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) { return map[string]string{}, nil }, }, repoService: new(reposervicenmocks.BackupRepoService), retFuncMaintain: func(context.Context, udmrepo.RepoOptions) error { return errors.New("fake-error-1") }, expectedErr: "error to prune backup repo: fake-error-1", }, { name: "succeed", getter: new(credmock.SecretStore), credStoreReturn: "fake-password", funcTable: localFuncTable{ getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, map[string]string, velerocredentials.CredentialGetter) (map[string]string, error) { return map[string]string{}, nil }, getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) { return map[string]string{}, nil }, }, repoService: new(reposervicenmocks.BackupRepoService), retFuncMaintain: func(context.Context, udmrepo.RepoOptions) error { return nil }, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { funcTable = tc.funcTable var secretStore velerocredentials.SecretStore if tc.getter != nil { tc.getter.On("Get", mock.Anything, mock.Anything).Return(tc.credStoreReturn, tc.credStoreError) secretStore = tc.getter } urp := unifiedRepoProvider{ credentialGetter: velerocredentials.CredentialGetter{ FromSecret: secretStore, }, repoService: tc.repoService, log: velerotest.NewLogger(), } if tc.repoService != nil { tc.repoService.On("Maintain", mock.Anything, mock.Anything).Return(tc.retFuncMaintain) } err := urp.PruneRepo(t.Context(), RepoParam{ BackupLocation: &velerov1api.BackupStorageLocation{}, BackupRepo: &velerov1api.BackupRepository{}, }) if tc.expectedErr == "" { assert.NoError(t, err) } else { assert.EqualError(t, err, tc.expectedErr) } }) } } func TestGetStorageType(t *testing.T) { testCases := []struct { name string backupLocation *velerov1api.BackupStorageLocation expectedRet string }{ { name: "wrong backend type", backupLocation: &velerov1api.BackupStorageLocation{}, }, { name: "aws provider", backupLocation: &velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Provider: "velero.io/aws", }, }, expectedRet: "s3", }, { name: "azure provider", backupLocation: &velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Provider: "velero.io/azure", }, }, expectedRet: "azure", }, { name: "gcp provider", backupLocation: &velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Provider: "velero.io/gcp", }, }, expectedRet: "gcs", }, { name: "fs provider", backupLocation: &velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Provider: "velero.io/fs", }, }, expectedRet: "filesystem", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { ret := getStorageType(tc.backupLocation) assert.Equal(t, tc.expectedRet, ret) }) } } ================================================ FILE: pkg/repository/restic/repository.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package restic import ( "os" "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/vmware-tanzu/velero/internal/credentials" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" repokey "github.com/vmware-tanzu/velero/pkg/repository/keys" "github.com/vmware-tanzu/velero/pkg/restic" veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" "github.com/vmware-tanzu/velero/pkg/util/filesystem" ) func NewRepositoryService(credGetter credentials.CredentialGetter, fs filesystem.Interface, log logrus.FieldLogger) *RepositoryService { return &RepositoryService{ credGetter: credGetter, fileSystem: fs, log: log, } } type RepositoryService struct { credGetter credentials.CredentialGetter fileSystem filesystem.Interface log logrus.FieldLogger } func (r *RepositoryService) InitRepo(bsl *velerov1api.BackupStorageLocation, repo *velerov1api.BackupRepository) error { return r.exec(restic.InitCommand(repo.Spec.ResticIdentifier), bsl) } func (r *RepositoryService) ConnectToRepo(bsl *velerov1api.BackupStorageLocation, repo *velerov1api.BackupRepository) error { snapshotsCmd := restic.SnapshotsCommand(repo.Spec.ResticIdentifier) // use the '--latest=1' flag to minimize the amount of data fetched since // we're just validating that the repo exists and can be authenticated // to. // "--last" is replaced by "--latest=1" in restic v0.12.1 snapshotsCmd.ExtraFlags = append(snapshotsCmd.ExtraFlags, "--latest=1") return r.exec(snapshotsCmd, bsl) } func (r *RepositoryService) PruneRepo(bsl *velerov1api.BackupStorageLocation, repo *velerov1api.BackupRepository) error { return r.exec(restic.PruneCommand(repo.Spec.ResticIdentifier), bsl) } func (r *RepositoryService) UnlockRepo(bsl *velerov1api.BackupStorageLocation, repo *velerov1api.BackupRepository) error { return r.exec(restic.UnlockCommand(repo.Spec.ResticIdentifier), bsl) } func (r *RepositoryService) Forget(bsl *velerov1api.BackupStorageLocation, repo *velerov1api.BackupRepository, snapshotID string) error { return r.exec(restic.ForgetCommand(repo.Spec.ResticIdentifier, snapshotID), bsl) } func (r *RepositoryService) DefaultMaintenanceFrequency() time.Duration { return restic.DefaultMaintenanceFrequency } func (r *RepositoryService) exec(cmd *restic.Command, bsl *velerov1api.BackupStorageLocation) error { file, err := r.credGetter.FromFile.Path(repokey.RepoKeySelector()) if err != nil { return err } // ignore error since there's nothing we can do and it's a temp file. defer os.Remove(file) cmd.PasswordFile = file // if there's a caCert on the ObjectStorage, write it to disk so that it can be passed to restic var caCertFile string if bsl.Spec.ObjectStorage != nil { var caCertData []byte // Try CACertRef first (new method), then fall back to CACert (deprecated) if bsl.Spec.ObjectStorage.CACertRef != nil { caCertString, err := r.credGetter.FromSecret.Get(bsl.Spec.ObjectStorage.CACertRef) if err != nil { return errors.Wrap(err, "error getting CA certificate from secret") } caCertData = []byte(caCertString) } else if bsl.Spec.ObjectStorage.CACert != nil { caCertData = bsl.Spec.ObjectStorage.CACert } if caCertData != nil { caCertFile, err = restic.TempCACertFile(caCertData, bsl.Name, r.fileSystem) if err != nil { return errors.Wrap(err, "error creating temp cacert file") } // ignore error since there's nothing we can do and it's a temp file. defer os.Remove(caCertFile) } } cmd.CACertFile = caCertFile // CmdEnv uses credGetter.FromFile (not FromSecret) to get cloud provider credentials. // FromFile materializes the BSL's Credential secret to a file path that cloud SDKs // can read (e.g., AWS_SHARED_CREDENTIALS_FILE). This is different from caCertRef above, // which uses FromSecret to read the CA certificate data directly into memory, then // writes it to a temp file because restic CLI only accepts file paths (--cacert flag). env, err := restic.CmdEnv(bsl, r.credGetter.FromFile) if err != nil { return err } cmd.Env = env // #4820: restrieve insecureSkipTLSVerify from BSL configuration for // AWS plugin. If nothing is return, that means insecureSkipTLSVerify // is not enable for Restic command. skipTLSRet := restic.GetInsecureSkipTLSVerifyFromBSL(bsl, r.log) if len(skipTLSRet) > 0 { cmd.ExtraFlags = append(cmd.ExtraFlags, skipTLSRet) } stdout, stderr, err := veleroexec.RunCommandWithLog(cmd.Cmd(), r.log) r.log.WithFields(logrus.Fields{ "repository": cmd.RepoName(), "command": cmd.String(), "stdout": stdout, "stderr": stderr, }).Debugf("Ran restic command") if err != nil { return errors.Wrapf(err, "error running command=%s, stdout=%s, stderr=%s", cmd.String(), stdout, stderr) } return nil } ================================================ FILE: pkg/repository/types/snapshotidentifier.go ================================================ package types // SnapshotIdentifier uniquely identifies a snapshot // taken by Velero. type SnapshotIdentifier struct { // VolumeNamespace is the namespace of the pod/volume that // the snapshot is for. VolumeNamespace string `json:"volumeNamespace"` // BackupStorageLocation is the backup's storage location // name. BackupStorageLocation string `json:"backupStorageLocation"` // SnapshotID is the short ID of the snapshot. SnapshotID string `json:"snapshotID"` // RepositoryType is the type of the repository where the // snapshot is stored RepositoryType string `json:"repositoryType"` // Source is the source of the data saved in the repo by the snapshot Source string `json:"source"` // UploaderType is the type of uploader which saved the snapshot data UploaderType string `json:"uploaderType"` // RepoIdentifier is the identifier of the repository where the // snapshot is stored RepoIdentifier string `json:"repoIdentifier"` } ================================================ FILE: pkg/repository/udmrepo/kopialib/backend/azure/azure_storage_wrapper.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package azure import ( "context" "github.com/kopia/kopia/repo/blob" "github.com/kopia/kopia/repo/blob/azure" "github.com/kopia/kopia/repo/blob/throttling" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo/kopialib/backend/logging" azureutil "github.com/vmware-tanzu/velero/pkg/util/azure" ) const ( storageType = "azure" ) func init() { blob.AddSupportedStorage(storageType, Option{}, NewStorage) } type Option struct { Config map[string]string `json:"config" kopia:"sensitive"` Limits throttling.Limits } type Storage struct { blob.Storage Option *Option } func (s *Storage) ConnectionInfo() blob.ConnectionInfo { return blob.ConnectionInfo{ Type: storageType, Config: s.Option, } } func NewStorage(ctx context.Context, option *Option, isCreate bool) (blob.Storage, error) { cfg := option.Config // Get logger from context logger := logging.LoggerFromContext(ctx) client, _, err := azureutil.NewStorageClient(logger, cfg) if err != nil { return nil, err } opt := &azure.Options{ Container: cfg[udmrepo.StoreOptionOssBucket], Prefix: cfg[udmrepo.StoreOptionPrefix], Limits: option.Limits, } azStorage, err := azure.NewWithClient(ctx, opt, client) if err != nil { return nil, err } logger.Info("Successfully created Azure storage backend") return &Storage{ Option: option, Storage: azStorage, }, nil } ================================================ FILE: pkg/repository/udmrepo/kopialib/backend/azure.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backend import ( "context" "github.com/sirupsen/logrus" "github.com/kopia/kopia/repo/blob" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo/kopialib/backend/azure" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo/kopialib/backend/logging" ) type AzureBackend struct { option azure.Option } func (c *AzureBackend) Setup(ctx context.Context, flags map[string]string, logger logrus.FieldLogger) error { if flags[udmrepo.StoreOptionCACert] != "" { flags["caCertEncoded"] = "true" } c.option = azure.Option{ Config: flags, Limits: setupLimits(ctx, flags), } return nil } func (c *AzureBackend) Connect(ctx context.Context, isCreate bool, logger logrus.FieldLogger) (blob.Storage, error) { ctx = logging.WithLogger(ctx, logger) return azure.NewStorage(ctx, &c.option, false) } ================================================ FILE: pkg/repository/udmrepo/kopialib/backend/azure_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backend import ( "testing" velerotest "github.com/vmware-tanzu/velero/pkg/test" "github.com/kopia/kopia/repo/blob/throttling" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" ) func TestAzureSetup(t *testing.T) { backend := AzureBackend{} logger := velerotest.NewLogger() flags := map[string]string{ "key": "value", udmrepo.ThrottleOptionReadOps: "100", udmrepo.ThrottleOptionUploadBytes: "200", } limits := throttling.Limits{ ReadsPerSecond: 100, UploadBytesPerSecond: 200, } err := backend.Setup(t.Context(), flags, logger) require.NoError(t, err) assert.Equal(t, flags, backend.option.Config) assert.Equal(t, limits, backend.option.Limits) } ================================================ FILE: pkg/repository/udmrepo/kopialib/backend/backend.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backend import ( "context" "errors" "github.com/sirupsen/logrus" "github.com/kopia/kopia/repo/blob" ) var ErrStoreNotExist = errors.New("store does not exist") // Store defines the methods for Kopia to establish a connection to // the backend storage type Store interface { // Setup setups the variables to a specific backend storage Setup(ctx context.Context, flags map[string]string, logger logrus.FieldLogger) error // Connect connects to a specific backend storage with the storage variables Connect(ctx context.Context, isCreate bool, logger logrus.FieldLogger) (blob.Storage, error) } ================================================ FILE: pkg/repository/udmrepo/kopialib/backend/common.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backend import ( "context" "time" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/blob" "github.com/kopia/kopia/repo/blob/throttling" "github.com/kopia/kopia/repo/content" "github.com/kopia/kopia/repo/encryption" "github.com/kopia/kopia/repo/format" "github.com/kopia/kopia/repo/hashing" "github.com/kopia/kopia/repo/splitter" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" ) const ( DefaultCacheLimitMB = 5000 maxCacheDurationSecond = 30 ) func setupLimits(ctx context.Context, flags map[string]string) throttling.Limits { return throttling.Limits{ DownloadBytesPerSecond: optionalHaveFloat64(ctx, udmrepo.ThrottleOptionDownloadBytes, flags), ListsPerSecond: optionalHaveFloat64(ctx, udmrepo.ThrottleOptionListOps, flags), ReadsPerSecond: optionalHaveFloat64(ctx, udmrepo.ThrottleOptionReadOps, flags), UploadBytesPerSecond: optionalHaveFloat64(ctx, udmrepo.ThrottleOptionUploadBytes, flags), WritesPerSecond: optionalHaveFloat64(ctx, udmrepo.ThrottleOptionWriteOps, flags), } } // SetupNewRepositoryOptions setups the options when creating a new Kopia repository func SetupNewRepositoryOptions(ctx context.Context, flags map[string]string) repo.NewRepositoryOptions { return repo.NewRepositoryOptions{ BlockFormat: format.ContentFormat{ Hash: optionalHaveStringWithDefault(udmrepo.StoreOptionGenHashAlgo, flags, hashing.DefaultAlgorithm), Encryption: optionalHaveStringWithDefault(udmrepo.StoreOptionGenEncryptAlgo, flags, encryption.DefaultAlgorithm), }, ObjectFormat: format.ObjectFormat{ Splitter: optionalHaveStringWithDefault(udmrepo.StoreOptionGenSplitAlgo, flags, splitter.DefaultAlgorithm), }, RetentionMode: blob.RetentionMode(optionalHaveString(udmrepo.StoreOptionGenRetentionMode, flags)), RetentionPeriod: optionalHaveDuration(ctx, udmrepo.StoreOptionGenRetentionPeriod, flags), } } // SetupConnectOptions setups the options when connecting to an existing Kopia repository func SetupConnectOptions(ctx context.Context, repoOptions udmrepo.RepoOptions) repo.ConnectOptions { cacheLimit := optionalHaveIntWithDefault(ctx, udmrepo.StoreOptionCacheLimit, repoOptions.StorageOptions, DefaultCacheLimitMB) << 20 cacheDir := optionalHaveString(udmrepo.StoreOptionCacheDir, repoOptions.StorageOptions) // 80% for data cache and 20% for metadata cache and align to KB dataCacheLimit := (cacheLimit / 5 * 4) >> 10 metadataCacheLimit := (cacheLimit / 5) >> 10 return repo.ConnectOptions{ CachingOptions: content.CachingOptions{ CacheDirectory: cacheDir, // softLimit 80% ContentCacheSizeBytes: (dataCacheLimit / 5 * 4) << 10, MetadataCacheSizeBytes: (metadataCacheLimit / 5 * 4) << 10, // hardLimit 100% ContentCacheSizeLimitBytes: dataCacheLimit << 10, MetadataCacheSizeLimitBytes: metadataCacheLimit << 10, MaxListCacheDuration: content.DurationSeconds(time.Duration(maxCacheDurationSecond) * time.Second), }, ClientOptions: repo.ClientOptions{ Hostname: optionalHaveString(udmrepo.GenOptionOwnerDomain, repoOptions.GeneralOptions), Username: optionalHaveString(udmrepo.GenOptionOwnerName, repoOptions.GeneralOptions), ReadOnly: optionalHaveBool(ctx, udmrepo.StoreOptionGenReadOnly, repoOptions.GeneralOptions), Description: repoOptions.Description, }, } } func RepoOwnerFromRepoOptions(repoOptions udmrepo.RepoOptions) string { hostname := optionalHaveStringWithDefault(udmrepo.GenOptionOwnerDomain, repoOptions.GeneralOptions, udmrepo.GetRepoDomain()) username := optionalHaveStringWithDefault(udmrepo.GenOptionOwnerName, repoOptions.GeneralOptions, udmrepo.GetRepoUser()) return username + "@" + hostname } ================================================ FILE: pkg/repository/udmrepo/kopialib/backend/common_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backend import ( "testing" "time" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/content" "github.com/kopia/kopia/repo/encryption" "github.com/kopia/kopia/repo/format" "github.com/kopia/kopia/repo/hashing" "github.com/kopia/kopia/repo/splitter" "github.com/stretchr/testify/assert" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" ) func TestSetupNewRepositoryOptions(t *testing.T) { testCases := []struct { name string flags map[string]string expected repo.NewRepositoryOptions }{ { name: "with hash algo", flags: map[string]string{ udmrepo.StoreOptionGenHashAlgo: "fake-hash", }, expected: repo.NewRepositoryOptions{ BlockFormat: format.ContentFormat{ Hash: "fake-hash", Encryption: encryption.DefaultAlgorithm, }, ObjectFormat: format.ObjectFormat{ Splitter: splitter.DefaultAlgorithm, }, }, }, { name: "with encrypt algo", flags: map[string]string{ udmrepo.StoreOptionGenEncryptAlgo: "fake-encrypt", }, expected: repo.NewRepositoryOptions{ BlockFormat: format.ContentFormat{ Hash: hashing.DefaultAlgorithm, Encryption: "fake-encrypt", }, ObjectFormat: format.ObjectFormat{ Splitter: splitter.DefaultAlgorithm, }, }, }, { name: "with splitter algo", flags: map[string]string{ udmrepo.StoreOptionGenSplitAlgo: "fake-splitter", }, expected: repo.NewRepositoryOptions{ BlockFormat: format.ContentFormat{ Hash: hashing.DefaultAlgorithm, Encryption: encryption.DefaultAlgorithm, }, ObjectFormat: format.ObjectFormat{ Splitter: "fake-splitter", }, }, }, { name: "with retention algo", flags: map[string]string{ udmrepo.StoreOptionGenRetentionMode: "fake-retention-mode", }, expected: repo.NewRepositoryOptions{ BlockFormat: format.ContentFormat{ Hash: hashing.DefaultAlgorithm, Encryption: encryption.DefaultAlgorithm, }, ObjectFormat: format.ObjectFormat{ Splitter: splitter.DefaultAlgorithm, }, RetentionMode: "fake-retention-mode", }, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { ret := SetupNewRepositoryOptions(t.Context(), tc.flags) assert.Equal(t, tc.expected, ret) }) } } func TestSetupConnectOptions(t *testing.T) { defaultCacheOption := content.CachingOptions{ ContentCacheSizeBytes: 3200 << 20, MetadataCacheSizeBytes: 800 << 20, ContentCacheSizeLimitBytes: 4000 << 20, MetadataCacheSizeLimitBytes: 1000 << 20, MaxListCacheDuration: content.DurationSeconds(time.Duration(30) * time.Second), } testCases := []struct { name string repoOptions udmrepo.RepoOptions expected repo.ConnectOptions }{ { name: "with domain", repoOptions: udmrepo.RepoOptions{ GeneralOptions: map[string]string{ udmrepo.GenOptionOwnerDomain: "fake-domain", }, }, expected: repo.ConnectOptions{ CachingOptions: defaultCacheOption, ClientOptions: repo.ClientOptions{ Hostname: "fake-domain", }, }, }, { name: "with username", repoOptions: udmrepo.RepoOptions{ GeneralOptions: map[string]string{ udmrepo.GenOptionOwnerName: "fake-user", }, }, expected: repo.ConnectOptions{ CachingOptions: defaultCacheOption, ClientOptions: repo.ClientOptions{ Username: "fake-user", }, }, }, { name: "with wrong readonly", repoOptions: udmrepo.RepoOptions{ GeneralOptions: map[string]string{ udmrepo.StoreOptionGenReadOnly: "fake-bool", }, }, expected: repo.ConnectOptions{ CachingOptions: defaultCacheOption, ClientOptions: repo.ClientOptions{}, }, }, { name: "with correct readonly", repoOptions: udmrepo.RepoOptions{ GeneralOptions: map[string]string{ udmrepo.StoreOptionGenReadOnly: "true", }, }, expected: repo.ConnectOptions{ CachingOptions: defaultCacheOption, ClientOptions: repo.ClientOptions{ ReadOnly: true, }, }, }, { name: "with description", repoOptions: udmrepo.RepoOptions{ Description: "fake-description", }, expected: repo.ConnectOptions{ CachingOptions: defaultCacheOption, ClientOptions: repo.ClientOptions{ Description: "fake-description", }, }, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { ret := SetupConnectOptions(t.Context(), tc.repoOptions) assert.Equal(t, tc.expected, ret) }) } } ================================================ FILE: pkg/repository/udmrepo/kopialib/backend/file_system.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backend import ( "context" "os" "path/filepath" "github.com/sirupsen/logrus" "github.com/kopia/kopia/repo/blob" "github.com/kopia/kopia/repo/blob/filesystem" "github.com/pkg/errors" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo/kopialib/backend/logging" ) type FsBackend struct { options filesystem.Options } const ( defaultFileMode = 0o600 defaultDirMode = 0o700 ) func (c *FsBackend) Setup(ctx context.Context, flags map[string]string, logger logrus.FieldLogger) error { path, err := mustHaveString(udmrepo.StoreOptionFsPath, flags) if err != nil { return err } prefix := optionalHaveString(udmrepo.StoreOptionPrefix, flags) c.options.Path = filepath.Join(path, prefix) c.options.FileMode = defaultFileMode c.options.DirectoryMode = defaultDirMode ctx = logging.WithLogger(ctx, logger) c.options.Limits = setupLimits(ctx, flags) return nil } func (c *FsBackend) Connect(ctx context.Context, isCreate bool, logger logrus.FieldLogger) (blob.Storage, error) { if !filepath.IsAbs(c.options.Path) { return nil, errors.Errorf("filesystem repository path is not absolute, path: %s", c.options.Path) } if !isCreate { if _, err := os.Stat(c.options.Path); err != nil { return nil, ErrStoreNotExist } } ctx = logging.WithLogger(ctx, logger) return filesystem.New(ctx, &c.options, isCreate) } ================================================ FILE: pkg/repository/udmrepo/kopialib/backend/file_system_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backend import ( "testing" velerotest "github.com/vmware-tanzu/velero/pkg/test" "github.com/kopia/kopia/repo/blob/filesystem" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" ) func TestFSSetup(t *testing.T) { testCases := []struct { name string flags map[string]string expectedOptions filesystem.Options expectedErr string }{ { name: "must have fs path", flags: map[string]string{}, expectedErr: "key " + udmrepo.StoreOptionFsPath + " not found", }, { name: "with fs path only", flags: map[string]string{ udmrepo.StoreOptionFsPath: "fake/path", }, expectedOptions: filesystem.Options{ Path: "fake/path", FileMode: 0o600, DirectoryMode: 0o700, }, }, { name: "with prefix", flags: map[string]string{ udmrepo.StoreOptionFsPath: "fake/path", udmrepo.StoreOptionPrefix: "fake-prefix", }, expectedOptions: filesystem.Options{ Path: "fake/path/fake-prefix", FileMode: 0o600, DirectoryMode: 0o700, }, }, } logger := velerotest.NewLogger() for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { fsFlags := FsBackend{} err := fsFlags.Setup(t.Context(), tc.flags, logger) if tc.expectedErr == "" { require.NoError(t, err) assert.Equal(t, tc.expectedOptions, fsFlags.options) } else { assert.EqualError(t, err, tc.expectedErr) } }) } } ================================================ FILE: pkg/repository/udmrepo/kopialib/backend/gcs.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backend import ( "context" "github.com/sirupsen/logrus" "github.com/kopia/kopia/repo/blob" "github.com/kopia/kopia/repo/blob/gcs" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo/kopialib/backend/logging" ) type GCSBackend struct { options gcs.Options } func (c *GCSBackend) Setup(ctx context.Context, flags map[string]string, logger logrus.FieldLogger) error { var err error c.options.BucketName, err = mustHaveString(udmrepo.StoreOptionOssBucket, flags) if err != nil { return err } c.options.ServiceAccountCredentialsFile, err = mustHaveString(udmrepo.StoreOptionCredentialFile, flags) if err != nil { return err } c.options.Prefix = optionalHaveString(udmrepo.StoreOptionPrefix, flags) c.options.ReadOnly = optionalHaveBool(ctx, udmrepo.StoreOptionGcsReadonly, flags) ctx = logging.WithLogger(ctx, logger) c.options.Limits = setupLimits(ctx, flags) return nil } func (c *GCSBackend) Connect(ctx context.Context, isCreate bool, logger logrus.FieldLogger) (blob.Storage, error) { ctx = logging.WithLogger(ctx, logger) return gcs.New(ctx, &c.options, false) } ================================================ FILE: pkg/repository/udmrepo/kopialib/backend/gcs_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backend import ( "testing" velerotest "github.com/vmware-tanzu/velero/pkg/test" "github.com/kopia/kopia/repo/blob/gcs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" ) func TestGcsSetup(t *testing.T) { testCases := []struct { name string flags map[string]string expectedOptions gcs.Options expectedErr string }{ { name: "must have bucket name", flags: map[string]string{}, expectedErr: "key " + udmrepo.StoreOptionOssBucket + " not found", }, { name: "must have credential file", flags: map[string]string{ udmrepo.StoreOptionOssBucket: "fake-bucket", }, expectedErr: "key " + udmrepo.StoreOptionCredentialFile + " not found", }, { name: "with prefix", flags: map[string]string{ udmrepo.StoreOptionOssBucket: "fake-bucket", udmrepo.StoreOptionCredentialFile: "fake-credential", udmrepo.StoreOptionPrefix: "fake-prefix", }, expectedOptions: gcs.Options{ BucketName: "fake-bucket", ServiceAccountCredentialsFile: "fake-credential", Prefix: "fake-prefix", }, }, { name: "with wrong readonly", flags: map[string]string{ udmrepo.StoreOptionOssBucket: "fake-bucket", udmrepo.StoreOptionCredentialFile: "fake-credential", udmrepo.StoreOptionGcsReadonly: "fake-bool", }, expectedOptions: gcs.Options{ BucketName: "fake-bucket", ServiceAccountCredentialsFile: "fake-credential", }, }, { name: "with correct readonly", flags: map[string]string{ udmrepo.StoreOptionOssBucket: "fake-bucket", udmrepo.StoreOptionCredentialFile: "fake-credential", udmrepo.StoreOptionGcsReadonly: "true", }, expectedOptions: gcs.Options{ BucketName: "fake-bucket", ServiceAccountCredentialsFile: "fake-credential", ReadOnly: true, }, }, } logger := velerotest.NewLogger() for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { gcsFlags := GCSBackend{} err := gcsFlags.Setup(t.Context(), tc.flags, logger) if tc.expectedErr == "" { require.NoError(t, err) assert.Equal(t, tc.expectedOptions, gcsFlags.options) } else { assert.EqualError(t, err, tc.expectedErr) } }) } } ================================================ FILE: pkg/repository/udmrepo/kopialib/backend/logging/context.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package logging import ( "context" "github.com/sirupsen/logrus" ) type ctxKeyLogger struct{} // WithLogger returns a new context with the provided logger. func WithLogger(ctx context.Context, logger logrus.FieldLogger) context.Context { return context.WithValue(ctx, ctxKeyLogger{}, logger) } // LoggerFromContext retrieves the logger from the context, or returns a default logger if none found. func LoggerFromContext(ctx context.Context) logrus.FieldLogger { if logger, ok := ctx.Value(ctxKeyLogger{}).(logrus.FieldLogger); ok && logger != nil { return logger } return logrus.New() } ================================================ FILE: pkg/repository/udmrepo/kopialib/backend/mocks/Logger.go ================================================ // Code generated by mockery v2.22.1. DO NOT EDIT. package mocks import ( mock "github.com/stretchr/testify/mock" zapcore "go.uber.org/zap/zapcore" ) // Core is an autogenerated mock type for the Core type type Core struct { mock.Mock } // Check provides a mock function with given fields: _a0, _a1 func (_m *Core) Check(_a0 zapcore.Entry, _a1 *zapcore.CheckedEntry) *zapcore.CheckedEntry { ret := _m.Called(_a0, _a1) var r0 *zapcore.CheckedEntry if rf, ok := ret.Get(0).(func(zapcore.Entry, *zapcore.CheckedEntry) *zapcore.CheckedEntry); ok { r0 = rf(_a0, _a1) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(*zapcore.CheckedEntry) } } return r0 } // Enabled provides a mock function with given fields: _a0 func (_m *Core) Enabled(_a0 zapcore.Level) bool { ret := _m.Called(_a0) var r0 bool if rf, ok := ret.Get(0).(func(zapcore.Level) bool); ok { r0 = rf(_a0) } else { r0 = ret.Get(0).(bool) } return r0 } // Sync provides a mock function with given fields: func (_m *Core) Sync() error { ret := _m.Called() var r0 error if rf, ok := ret.Get(0).(func() error); ok { r0 = rf() } else { r0 = ret.Error(0) } return r0 } // With provides a mock function with given fields: _a0 func (_m *Core) With(_a0 []zapcore.Field) zapcore.Core { ret := _m.Called(_a0) var r0 zapcore.Core if rf, ok := ret.Get(0).(func([]zapcore.Field) zapcore.Core); ok { r0 = rf(_a0) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(zapcore.Core) } } return r0 } // Write provides a mock function with given fields: _a0, _a1 func (_m *Core) Write(_a0 zapcore.Entry, _a1 []zapcore.Field) error { ret := _m.Called(_a0, _a1) var r0 error if rf, ok := ret.Get(0).(func(zapcore.Entry, []zapcore.Field) error); ok { r0 = rf(_a0, _a1) } else { r0 = ret.Error(0) } return r0 } type mockConstructorTestingTNewCore interface { mock.TestingT Cleanup(func()) } // NewCore creates a new instance of Core. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. func NewCore(t mockConstructorTestingTNewCore) *Core { mock := &Core{} mock.Mock.Test(t) t.Cleanup(func() { mock.AssertExpectations(t) }) return mock } ================================================ FILE: pkg/repository/udmrepo/kopialib/backend/mocks/Reader.go ================================================ // Code generated by mockery v2.22.1. DO NOT EDIT. package mocks import mock "github.com/stretchr/testify/mock" // Reader is an autogenerated mock type for the Reader type type Reader struct { mock.Mock } // Close provides a mock function with given fields: func (_m *Reader) Close() error { ret := _m.Called() var r0 error if rf, ok := ret.Get(0).(func() error); ok { r0 = rf() } else { r0 = ret.Error(0) } return r0 } // Length provides a mock function with given fields: func (_m *Reader) Length() int64 { ret := _m.Called() var r0 int64 if rf, ok := ret.Get(0).(func() int64); ok { r0 = rf() } else { r0 = ret.Get(0).(int64) } return r0 } // Read provides a mock function with given fields: p func (_m *Reader) Read(p []byte) (int, error) { ret := _m.Called(p) var r0 int var r1 error if rf, ok := ret.Get(0).(func([]byte) (int, error)); ok { return rf(p) } if rf, ok := ret.Get(0).(func([]byte) int); ok { r0 = rf(p) } else { r0 = ret.Get(0).(int) } if rf, ok := ret.Get(1).(func([]byte) error); ok { r1 = rf(p) } else { r1 = ret.Error(1) } return r0, r1 } // Seek provides a mock function with given fields: offset, whence func (_m *Reader) Seek(offset int64, whence int) (int64, error) { ret := _m.Called(offset, whence) var r0 int64 var r1 error if rf, ok := ret.Get(0).(func(int64, int) (int64, error)); ok { return rf(offset, whence) } if rf, ok := ret.Get(0).(func(int64, int) int64); ok { r0 = rf(offset, whence) } else { r0 = ret.Get(0).(int64) } if rf, ok := ret.Get(1).(func(int64, int) error); ok { r1 = rf(offset, whence) } else { r1 = ret.Error(1) } return r0, r1 } type mockConstructorTestingTNewReader interface { mock.TestingT Cleanup(func()) } // NewReader creates a new instance of Reader. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. func NewReader(t mockConstructorTestingTNewReader) *Reader { mock := &Reader{} mock.Mock.Test(t) t.Cleanup(func() { mock.AssertExpectations(t) }) return mock } ================================================ FILE: pkg/repository/udmrepo/kopialib/backend/mocks/Storage.go ================================================ // Code generated by mockery v2.22.1. DO NOT EDIT. package mocks import ( context "context" blob "github.com/kopia/kopia/repo/blob" mock "github.com/stretchr/testify/mock" ) // Storage is an autogenerated mock type for the Storage type type Storage struct { mock.Mock } // Close provides a mock function with given fields: ctx func (_m *Storage) Close(ctx context.Context) error { ret := _m.Called(ctx) var r0 error if rf, ok := ret.Get(0).(func(context.Context) error); ok { r0 = rf(ctx) } else { r0 = ret.Error(0) } return r0 } // ConnectionInfo provides a mock function with given fields: func (_m *Storage) ConnectionInfo() blob.ConnectionInfo { ret := _m.Called() var r0 blob.ConnectionInfo if rf, ok := ret.Get(0).(func() blob.ConnectionInfo); ok { r0 = rf() } else { r0 = ret.Get(0).(blob.ConnectionInfo) } return r0 } // DeleteBlob provides a mock function with given fields: ctx, blobID func (_m *Storage) DeleteBlob(ctx context.Context, blobID blob.ID) error { ret := _m.Called(ctx, blobID) var r0 error if rf, ok := ret.Get(0).(func(context.Context, blob.ID) error); ok { r0 = rf(ctx, blobID) } else { r0 = ret.Error(0) } return r0 } // DisplayName provides a mock function with given fields: func (_m *Storage) DisplayName() string { ret := _m.Called() var r0 string if rf, ok := ret.Get(0).(func() string); ok { r0 = rf() } else { r0 = ret.Get(0).(string) } return r0 } // ExtendBlobRetention provides a mock function with given fields: ctx, blobID, opts func (_m *Storage) ExtendBlobRetention(ctx context.Context, blobID blob.ID, opts blob.ExtendOptions) error { ret := _m.Called(ctx, blobID, opts) var r0 error if rf, ok := ret.Get(0).(func(context.Context, blob.ID, blob.ExtendOptions) error); ok { r0 = rf(ctx, blobID, opts) } else { r0 = ret.Error(0) } return r0 } // FlushCaches provides a mock function with given fields: ctx func (_m *Storage) FlushCaches(ctx context.Context) error { ret := _m.Called(ctx) var r0 error if rf, ok := ret.Get(0).(func(context.Context) error); ok { r0 = rf(ctx) } else { r0 = ret.Error(0) } return r0 } // GetBlob provides a mock function with given fields: ctx, blobID, offset, length, output func (_m *Storage) GetBlob(ctx context.Context, blobID blob.ID, offset int64, length int64, output blob.OutputBuffer) error { ret := _m.Called(ctx, blobID, offset, length, output) var r0 error if rf, ok := ret.Get(0).(func(context.Context, blob.ID, int64, int64, blob.OutputBuffer) error); ok { r0 = rf(ctx, blobID, offset, length, output) } else { r0 = ret.Error(0) } return r0 } // GetCapacity provides a mock function with given fields: ctx func (_m *Storage) GetCapacity(ctx context.Context) (blob.Capacity, error) { ret := _m.Called(ctx) var r0 blob.Capacity var r1 error if rf, ok := ret.Get(0).(func(context.Context) (blob.Capacity, error)); ok { return rf(ctx) } if rf, ok := ret.Get(0).(func(context.Context) blob.Capacity); ok { r0 = rf(ctx) } else { r0 = ret.Get(0).(blob.Capacity) } if rf, ok := ret.Get(1).(func(context.Context) error); ok { r1 = rf(ctx) } else { r1 = ret.Error(1) } return r0, r1 } // GetMetadata provides a mock function with given fields: ctx, blobID func (_m *Storage) GetMetadata(ctx context.Context, blobID blob.ID) (blob.Metadata, error) { ret := _m.Called(ctx, blobID) var r0 blob.Metadata var r1 error if rf, ok := ret.Get(0).(func(context.Context, blob.ID) (blob.Metadata, error)); ok { return rf(ctx, blobID) } if rf, ok := ret.Get(0).(func(context.Context, blob.ID) blob.Metadata); ok { r0 = rf(ctx, blobID) } else { r0 = ret.Get(0).(blob.Metadata) } if rf, ok := ret.Get(1).(func(context.Context, blob.ID) error); ok { r1 = rf(ctx, blobID) } else { r1 = ret.Error(1) } return r0, r1 } // IsReadOnly provides a mock function with given fields: func (_m *Storage) IsReadOnly() bool { ret := _m.Called() var r0 bool if rf, ok := ret.Get(0).(func() bool); ok { r0 = rf() } else { r0 = ret.Get(0).(bool) } return r0 } // ListBlobs provides a mock function with given fields: ctx, blobIDPrefix, cb func (_m *Storage) ListBlobs(ctx context.Context, blobIDPrefix blob.ID, cb func(blob.Metadata) error) error { ret := _m.Called(ctx, blobIDPrefix, cb) var r0 error if rf, ok := ret.Get(0).(func(context.Context, blob.ID, func(blob.Metadata) error) error); ok { r0 = rf(ctx, blobIDPrefix, cb) } else { r0 = ret.Error(0) } return r0 } // PutBlob provides a mock function with given fields: ctx, blobID, data, opts func (_m *Storage) PutBlob(ctx context.Context, blobID blob.ID, data blob.Bytes, opts blob.PutOptions) error { ret := _m.Called(ctx, blobID, data, opts) var r0 error if rf, ok := ret.Get(0).(func(context.Context, blob.ID, blob.Bytes, blob.PutOptions) error); ok { r0 = rf(ctx, blobID, data, opts) } else { r0 = ret.Error(0) } return r0 } type mockConstructorTestingTNewStorage interface { mock.TestingT Cleanup(func()) } // NewStorage creates a new instance of Storage. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. func NewStorage(t mockConstructorTestingTNewStorage) *Storage { mock := &Storage{} mock.Mock.Test(t) t.Cleanup(func() { mock.AssertExpectations(t) }) return mock } ================================================ FILE: pkg/repository/udmrepo/kopialib/backend/mocks/Store.go ================================================ // Code generated by mockery v2.14.0. DO NOT EDIT. package mocks import ( context "context" "github.com/sirupsen/logrus" blob "github.com/kopia/kopia/repo/blob" mock "github.com/stretchr/testify/mock" ) // Store is an autogenerated mock type for the Store type type Store struct { mock.Mock } // Connect provides a mock function with given fields: ctx, isCreate func (_m *Store) Connect(ctx context.Context, isCreate bool, logger logrus.FieldLogger) (blob.Storage, error) { ret := _m.Called(ctx, isCreate) var r0 blob.Storage if rf, ok := ret.Get(0).(func(context.Context, bool) blob.Storage); ok { r0 = rf(ctx, isCreate) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(blob.Storage) } } var r1 error if rf, ok := ret.Get(1).(func(context.Context, bool) error); ok { r1 = rf(ctx, isCreate) } else { r1 = ret.Error(1) } return r0, r1 } // Setup provides a mock function with given fields: ctx, flags func (_m *Store) Setup(ctx context.Context, flags map[string]string, logger logrus.FieldLogger) error { ret := _m.Called(ctx, flags) var r0 error if rf, ok := ret.Get(0).(func(context.Context, map[string]string) error); ok { r0 = rf(ctx, flags) } else { r0 = ret.Error(0) } return r0 } type mockConstructorTestingTNewStore interface { mock.TestingT Cleanup(func()) } // NewStore creates a new instance of Store. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. func NewStore(t mockConstructorTestingTNewStore) *Store { mock := &Store{} mock.Mock.Test(t) t.Cleanup(func() { mock.AssertExpectations(t) }) return mock } ================================================ FILE: pkg/repository/udmrepo/kopialib/backend/mocks/Writer.go ================================================ // Code generated by mockery v2.22.1. DO NOT EDIT. package mocks import ( object "github.com/kopia/kopia/repo/object" mock "github.com/stretchr/testify/mock" ) // Writer is an autogenerated mock type for the Writer type type Writer struct { mock.Mock } // Checkpoint provides a mock function with given fields: func (_m *Writer) Checkpoint() (object.ID, error) { ret := _m.Called() var r0 object.ID var r1 error if rf, ok := ret.Get(0).(func() (object.ID, error)); ok { return rf() } if rf, ok := ret.Get(0).(func() object.ID); ok { r0 = rf() } else { r0 = ret.Get(0).(object.ID) } if rf, ok := ret.Get(1).(func() error); ok { r1 = rf() } else { r1 = ret.Error(1) } return r0, r1 } // Close provides a mock function with given fields: func (_m *Writer) Close() error { ret := _m.Called() var r0 error if rf, ok := ret.Get(0).(func() error); ok { r0 = rf() } else { r0 = ret.Error(0) } return r0 } // Result provides a mock function with given fields: func (_m *Writer) Result() (object.ID, error) { ret := _m.Called() var r0 object.ID var r1 error if rf, ok := ret.Get(0).(func() (object.ID, error)); ok { return rf() } if rf, ok := ret.Get(0).(func() object.ID); ok { r0 = rf() } else { r0 = ret.Get(0).(object.ID) } if rf, ok := ret.Get(1).(func() error); ok { r1 = rf() } else { r1 = ret.Error(1) } return r0, r1 } // Write provides a mock function with given fields: p func (_m *Writer) Write(p []byte) (int, error) { ret := _m.Called(p) var r0 int var r1 error if rf, ok := ret.Get(0).(func([]byte) (int, error)); ok { return rf(p) } if rf, ok := ret.Get(0).(func([]byte) int); ok { r0 = rf(p) } else { r0 = ret.Get(0).(int) } if rf, ok := ret.Get(1).(func([]byte) error); ok { r1 = rf(p) } else { r1 = ret.Error(1) } return r0, r1 } type mockConstructorTestingTNewWriter interface { mock.TestingT Cleanup(func()) } // NewWriter creates a new instance of Writer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. func NewWriter(t mockConstructorTestingTNewWriter) *Writer { mock := &Writer{} mock.Mock.Test(t) t.Cleanup(func() { mock.AssertExpectations(t) }) return mock } ================================================ FILE: pkg/repository/udmrepo/kopialib/backend/mocks/repository.go ================================================ // Code generated by mockery; DO NOT EDIT. // github.com/vektra/mockery // template: testify package mocks import ( "context" "time" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/content" "github.com/kopia/kopia/repo/manifest" "github.com/kopia/kopia/repo/object" mock "github.com/stretchr/testify/mock" ) // NewMockRepository creates a new instance of MockRepository. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. func NewMockRepository(t interface { mock.TestingT Cleanup(func()) }) *MockRepository { mock := &MockRepository{} mock.Mock.Test(t) t.Cleanup(func() { mock.AssertExpectations(t) }) return mock } // MockRepository is an autogenerated mock type for the Repository type type MockRepository struct { mock.Mock } type MockRepository_Expecter struct { mock *mock.Mock } func (_m *MockRepository) EXPECT() *MockRepository_Expecter { return &MockRepository_Expecter{mock: &_m.Mock} } // ClientOptions provides a mock function for the type MockRepository func (_mock *MockRepository) ClientOptions() repo.ClientOptions { ret := _mock.Called() if len(ret) == 0 { panic("no return value specified for ClientOptions") } var r0 repo.ClientOptions if returnFunc, ok := ret.Get(0).(func() repo.ClientOptions); ok { r0 = returnFunc() } else { r0 = ret.Get(0).(repo.ClientOptions) } return r0 } // MockRepository_ClientOptions_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ClientOptions' type MockRepository_ClientOptions_Call struct { *mock.Call } // ClientOptions is a helper method to define mock.On call func (_e *MockRepository_Expecter) ClientOptions() *MockRepository_ClientOptions_Call { return &MockRepository_ClientOptions_Call{Call: _e.mock.On("ClientOptions")} } func (_c *MockRepository_ClientOptions_Call) Run(run func()) *MockRepository_ClientOptions_Call { _c.Call.Run(func(args mock.Arguments) { run() }) return _c } func (_c *MockRepository_ClientOptions_Call) Return(clientOptions repo.ClientOptions) *MockRepository_ClientOptions_Call { _c.Call.Return(clientOptions) return _c } func (_c *MockRepository_ClientOptions_Call) RunAndReturn(run func() repo.ClientOptions) *MockRepository_ClientOptions_Call { _c.Call.Return(run) return _c } // Close provides a mock function for the type MockRepository func (_mock *MockRepository) Close(ctx context.Context) error { ret := _mock.Called(ctx) if len(ret) == 0 { panic("no return value specified for Close") } var r0 error if returnFunc, ok := ret.Get(0).(func(context.Context) error); ok { r0 = returnFunc(ctx) } else { r0 = ret.Error(0) } return r0 } // MockRepository_Close_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Close' type MockRepository_Close_Call struct { *mock.Call } // Close is a helper method to define mock.On call // - ctx context.Context func (_e *MockRepository_Expecter) Close(ctx interface{}) *MockRepository_Close_Call { return &MockRepository_Close_Call{Call: _e.mock.On("Close", ctx)} } func (_c *MockRepository_Close_Call) Run(run func(ctx context.Context)) *MockRepository_Close_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { arg0 = args[0].(context.Context) } run( arg0, ) }) return _c } func (_c *MockRepository_Close_Call) Return(err error) *MockRepository_Close_Call { _c.Call.Return(err) return _c } func (_c *MockRepository_Close_Call) RunAndReturn(run func(ctx context.Context) error) *MockRepository_Close_Call { _c.Call.Return(run) return _c } // ContentInfo provides a mock function for the type MockRepository func (_mock *MockRepository) ContentInfo(ctx context.Context, contentID content.ID) (content.Info, error) { ret := _mock.Called(ctx, contentID) if len(ret) == 0 { panic("no return value specified for ContentInfo") } var r0 content.Info var r1 error if returnFunc, ok := ret.Get(0).(func(context.Context, content.ID) (content.Info, error)); ok { return returnFunc(ctx, contentID) } if returnFunc, ok := ret.Get(0).(func(context.Context, content.ID) content.Info); ok { r0 = returnFunc(ctx, contentID) } else { r0 = ret.Get(0).(content.Info) } if returnFunc, ok := ret.Get(1).(func(context.Context, content.ID) error); ok { r1 = returnFunc(ctx, contentID) } else { r1 = ret.Error(1) } return r0, r1 } // MockRepository_ContentInfo_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ContentInfo' type MockRepository_ContentInfo_Call struct { *mock.Call } // ContentInfo is a helper method to define mock.On call // - ctx context.Context // - contentID content.ID func (_e *MockRepository_Expecter) ContentInfo(ctx interface{}, contentID interface{}) *MockRepository_ContentInfo_Call { return &MockRepository_ContentInfo_Call{Call: _e.mock.On("ContentInfo", ctx, contentID)} } func (_c *MockRepository_ContentInfo_Call) Run(run func(ctx context.Context, contentID content.ID)) *MockRepository_ContentInfo_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { arg0 = args[0].(context.Context) } var arg1 content.ID if args[1] != nil { arg1 = args[1].(content.ID) } run( arg0, arg1, ) }) return _c } func (_c *MockRepository_ContentInfo_Call) Return(v content.Info, err error) *MockRepository_ContentInfo_Call { _c.Call.Return(v, err) return _c } func (_c *MockRepository_ContentInfo_Call) RunAndReturn(run func(ctx context.Context, contentID content.ID) (content.Info, error)) *MockRepository_ContentInfo_Call { _c.Call.Return(run) return _c } // FindManifests provides a mock function for the type MockRepository func (_mock *MockRepository) FindManifests(ctx context.Context, labels map[string]string) ([]*manifest.EntryMetadata, error) { ret := _mock.Called(ctx, labels) if len(ret) == 0 { panic("no return value specified for FindManifests") } var r0 []*manifest.EntryMetadata var r1 error if returnFunc, ok := ret.Get(0).(func(context.Context, map[string]string) ([]*manifest.EntryMetadata, error)); ok { return returnFunc(ctx, labels) } if returnFunc, ok := ret.Get(0).(func(context.Context, map[string]string) []*manifest.EntryMetadata); ok { r0 = returnFunc(ctx, labels) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]*manifest.EntryMetadata) } } if returnFunc, ok := ret.Get(1).(func(context.Context, map[string]string) error); ok { r1 = returnFunc(ctx, labels) } else { r1 = ret.Error(1) } return r0, r1 } // MockRepository_FindManifests_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'FindManifests' type MockRepository_FindManifests_Call struct { *mock.Call } // FindManifests is a helper method to define mock.On call // - ctx context.Context // - labels map[string]string func (_e *MockRepository_Expecter) FindManifests(ctx interface{}, labels interface{}) *MockRepository_FindManifests_Call { return &MockRepository_FindManifests_Call{Call: _e.mock.On("FindManifests", ctx, labels)} } func (_c *MockRepository_FindManifests_Call) Run(run func(ctx context.Context, labels map[string]string)) *MockRepository_FindManifests_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { arg0 = args[0].(context.Context) } var arg1 map[string]string if args[1] != nil { arg1 = args[1].(map[string]string) } run( arg0, arg1, ) }) return _c } func (_c *MockRepository_FindManifests_Call) Return(entryMetadatas []*manifest.EntryMetadata, err error) *MockRepository_FindManifests_Call { _c.Call.Return(entryMetadatas, err) return _c } func (_c *MockRepository_FindManifests_Call) RunAndReturn(run func(ctx context.Context, labels map[string]string) ([]*manifest.EntryMetadata, error)) *MockRepository_FindManifests_Call { _c.Call.Return(run) return _c } // GetManifest provides a mock function for the type MockRepository func (_mock *MockRepository) GetManifest(ctx context.Context, id manifest.ID, data any) (*manifest.EntryMetadata, error) { ret := _mock.Called(ctx, id, data) if len(ret) == 0 { panic("no return value specified for GetManifest") } var r0 *manifest.EntryMetadata var r1 error if returnFunc, ok := ret.Get(0).(func(context.Context, manifest.ID, any) (*manifest.EntryMetadata, error)); ok { return returnFunc(ctx, id, data) } if returnFunc, ok := ret.Get(0).(func(context.Context, manifest.ID, any) *manifest.EntryMetadata); ok { r0 = returnFunc(ctx, id, data) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(*manifest.EntryMetadata) } } if returnFunc, ok := ret.Get(1).(func(context.Context, manifest.ID, any) error); ok { r1 = returnFunc(ctx, id, data) } else { r1 = ret.Error(1) } return r0, r1 } // MockRepository_GetManifest_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetManifest' type MockRepository_GetManifest_Call struct { *mock.Call } // GetManifest is a helper method to define mock.On call // - ctx context.Context // - id manifest.ID // - data any func (_e *MockRepository_Expecter) GetManifest(ctx interface{}, id interface{}, data interface{}) *MockRepository_GetManifest_Call { return &MockRepository_GetManifest_Call{Call: _e.mock.On("GetManifest", ctx, id, data)} } func (_c *MockRepository_GetManifest_Call) Run(run func(ctx context.Context, id manifest.ID, data any)) *MockRepository_GetManifest_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { arg0 = args[0].(context.Context) } var arg1 manifest.ID if args[1] != nil { arg1 = args[1].(manifest.ID) } var arg2 any if args[2] != nil { arg2 = args[2].(any) } run( arg0, arg1, arg2, ) }) return _c } func (_c *MockRepository_GetManifest_Call) Return(entryMetadata *manifest.EntryMetadata, err error) *MockRepository_GetManifest_Call { _c.Call.Return(entryMetadata, err) return _c } func (_c *MockRepository_GetManifest_Call) RunAndReturn(run func(ctx context.Context, id manifest.ID, data any) (*manifest.EntryMetadata, error)) *MockRepository_GetManifest_Call { _c.Call.Return(run) return _c } // NewWriter provides a mock function for the type MockRepository func (_mock *MockRepository) NewWriter(ctx context.Context, opt repo.WriteSessionOptions) (context.Context, repo.RepositoryWriter, error) { ret := _mock.Called(ctx, opt) if len(ret) == 0 { panic("no return value specified for NewWriter") } var r0 context.Context var r1 repo.RepositoryWriter var r2 error if returnFunc, ok := ret.Get(0).(func(context.Context, repo.WriteSessionOptions) (context.Context, repo.RepositoryWriter, error)); ok { return returnFunc(ctx, opt) } if returnFunc, ok := ret.Get(0).(func(context.Context, repo.WriteSessionOptions) context.Context); ok { r0 = returnFunc(ctx, opt) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(context.Context) } } if returnFunc, ok := ret.Get(1).(func(context.Context, repo.WriteSessionOptions) repo.RepositoryWriter); ok { r1 = returnFunc(ctx, opt) } else { if ret.Get(1) != nil { r1 = ret.Get(1).(repo.RepositoryWriter) } } if returnFunc, ok := ret.Get(2).(func(context.Context, repo.WriteSessionOptions) error); ok { r2 = returnFunc(ctx, opt) } else { r2 = ret.Error(2) } return r0, r1, r2 } // MockRepository_NewWriter_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'NewWriter' type MockRepository_NewWriter_Call struct { *mock.Call } // NewWriter is a helper method to define mock.On call // - ctx context.Context // - opt repo.WriteSessionOptions func (_e *MockRepository_Expecter) NewWriter(ctx interface{}, opt interface{}) *MockRepository_NewWriter_Call { return &MockRepository_NewWriter_Call{Call: _e.mock.On("NewWriter", ctx, opt)} } func (_c *MockRepository_NewWriter_Call) Run(run func(ctx context.Context, opt repo.WriteSessionOptions)) *MockRepository_NewWriter_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { arg0 = args[0].(context.Context) } var arg1 repo.WriteSessionOptions if args[1] != nil { arg1 = args[1].(repo.WriteSessionOptions) } run( arg0, arg1, ) }) return _c } func (_c *MockRepository_NewWriter_Call) Return(context1 context.Context, repositoryWriter repo.RepositoryWriter, err error) *MockRepository_NewWriter_Call { _c.Call.Return(context1, repositoryWriter, err) return _c } func (_c *MockRepository_NewWriter_Call) RunAndReturn(run func(ctx context.Context, opt repo.WriteSessionOptions) (context.Context, repo.RepositoryWriter, error)) *MockRepository_NewWriter_Call { _c.Call.Return(run) return _c } // OpenObject provides a mock function for the type MockRepository func (_mock *MockRepository) OpenObject(ctx context.Context, id object.ID) (object.Reader, error) { ret := _mock.Called(ctx, id) if len(ret) == 0 { panic("no return value specified for OpenObject") } var r0 object.Reader var r1 error if returnFunc, ok := ret.Get(0).(func(context.Context, object.ID) (object.Reader, error)); ok { return returnFunc(ctx, id) } if returnFunc, ok := ret.Get(0).(func(context.Context, object.ID) object.Reader); ok { r0 = returnFunc(ctx, id) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(object.Reader) } } if returnFunc, ok := ret.Get(1).(func(context.Context, object.ID) error); ok { r1 = returnFunc(ctx, id) } else { r1 = ret.Error(1) } return r0, r1 } // MockRepository_OpenObject_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'OpenObject' type MockRepository_OpenObject_Call struct { *mock.Call } // OpenObject is a helper method to define mock.On call // - ctx context.Context // - id object.ID func (_e *MockRepository_Expecter) OpenObject(ctx interface{}, id interface{}) *MockRepository_OpenObject_Call { return &MockRepository_OpenObject_Call{Call: _e.mock.On("OpenObject", ctx, id)} } func (_c *MockRepository_OpenObject_Call) Run(run func(ctx context.Context, id object.ID)) *MockRepository_OpenObject_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { arg0 = args[0].(context.Context) } var arg1 object.ID if args[1] != nil { arg1 = args[1].(object.ID) } run( arg0, arg1, ) }) return _c } func (_c *MockRepository_OpenObject_Call) Return(reader object.Reader, err error) *MockRepository_OpenObject_Call { _c.Call.Return(reader, err) return _c } func (_c *MockRepository_OpenObject_Call) RunAndReturn(run func(ctx context.Context, id object.ID) (object.Reader, error)) *MockRepository_OpenObject_Call { _c.Call.Return(run) return _c } // PrefetchContents provides a mock function for the type MockRepository func (_mock *MockRepository) PrefetchContents(ctx context.Context, contentIDs []content.ID, hint string) []content.ID { ret := _mock.Called(ctx, contentIDs, hint) if len(ret) == 0 { panic("no return value specified for PrefetchContents") } var r0 []content.ID if returnFunc, ok := ret.Get(0).(func(context.Context, []content.ID, string) []content.ID); ok { r0 = returnFunc(ctx, contentIDs, hint) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]content.ID) } } return r0 } // MockRepository_PrefetchContents_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'PrefetchContents' type MockRepository_PrefetchContents_Call struct { *mock.Call } // PrefetchContents is a helper method to define mock.On call // - ctx context.Context // - contentIDs []content.ID // - hint string func (_e *MockRepository_Expecter) PrefetchContents(ctx interface{}, contentIDs interface{}, hint interface{}) *MockRepository_PrefetchContents_Call { return &MockRepository_PrefetchContents_Call{Call: _e.mock.On("PrefetchContents", ctx, contentIDs, hint)} } func (_c *MockRepository_PrefetchContents_Call) Run(run func(ctx context.Context, contentIDs []content.ID, hint string)) *MockRepository_PrefetchContents_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { arg0 = args[0].(context.Context) } var arg1 []content.ID if args[1] != nil { arg1 = args[1].([]content.ID) } var arg2 string if args[2] != nil { arg2 = args[2].(string) } run( arg0, arg1, arg2, ) }) return _c } func (_c *MockRepository_PrefetchContents_Call) Return(vs []content.ID) *MockRepository_PrefetchContents_Call { _c.Call.Return(vs) return _c } func (_c *MockRepository_PrefetchContents_Call) RunAndReturn(run func(ctx context.Context, contentIDs []content.ID, hint string) []content.ID) *MockRepository_PrefetchContents_Call { _c.Call.Return(run) return _c } // PrefetchObjects provides a mock function for the type MockRepository func (_mock *MockRepository) PrefetchObjects(ctx context.Context, objectIDs []object.ID, hint string) ([]content.ID, error) { ret := _mock.Called(ctx, objectIDs, hint) if len(ret) == 0 { panic("no return value specified for PrefetchObjects") } var r0 []content.ID var r1 error if returnFunc, ok := ret.Get(0).(func(context.Context, []object.ID, string) ([]content.ID, error)); ok { return returnFunc(ctx, objectIDs, hint) } if returnFunc, ok := ret.Get(0).(func(context.Context, []object.ID, string) []content.ID); ok { r0 = returnFunc(ctx, objectIDs, hint) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]content.ID) } } if returnFunc, ok := ret.Get(1).(func(context.Context, []object.ID, string) error); ok { r1 = returnFunc(ctx, objectIDs, hint) } else { r1 = ret.Error(1) } return r0, r1 } // MockRepository_PrefetchObjects_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'PrefetchObjects' type MockRepository_PrefetchObjects_Call struct { *mock.Call } // PrefetchObjects is a helper method to define mock.On call // - ctx context.Context // - objectIDs []object.ID // - hint string func (_e *MockRepository_Expecter) PrefetchObjects(ctx interface{}, objectIDs interface{}, hint interface{}) *MockRepository_PrefetchObjects_Call { return &MockRepository_PrefetchObjects_Call{Call: _e.mock.On("PrefetchObjects", ctx, objectIDs, hint)} } func (_c *MockRepository_PrefetchObjects_Call) Run(run func(ctx context.Context, objectIDs []object.ID, hint string)) *MockRepository_PrefetchObjects_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { arg0 = args[0].(context.Context) } var arg1 []object.ID if args[1] != nil { arg1 = args[1].([]object.ID) } var arg2 string if args[2] != nil { arg2 = args[2].(string) } run( arg0, arg1, arg2, ) }) return _c } func (_c *MockRepository_PrefetchObjects_Call) Return(vs []content.ID, err error) *MockRepository_PrefetchObjects_Call { _c.Call.Return(vs, err) return _c } func (_c *MockRepository_PrefetchObjects_Call) RunAndReturn(run func(ctx context.Context, objectIDs []object.ID, hint string) ([]content.ID, error)) *MockRepository_PrefetchObjects_Call { _c.Call.Return(run) return _c } // Refresh provides a mock function for the type MockRepository func (_mock *MockRepository) Refresh(ctx context.Context) error { ret := _mock.Called(ctx) if len(ret) == 0 { panic("no return value specified for Refresh") } var r0 error if returnFunc, ok := ret.Get(0).(func(context.Context) error); ok { r0 = returnFunc(ctx) } else { r0 = ret.Error(0) } return r0 } // MockRepository_Refresh_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Refresh' type MockRepository_Refresh_Call struct { *mock.Call } // Refresh is a helper method to define mock.On call // - ctx context.Context func (_e *MockRepository_Expecter) Refresh(ctx interface{}) *MockRepository_Refresh_Call { return &MockRepository_Refresh_Call{Call: _e.mock.On("Refresh", ctx)} } func (_c *MockRepository_Refresh_Call) Run(run func(ctx context.Context)) *MockRepository_Refresh_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { arg0 = args[0].(context.Context) } run( arg0, ) }) return _c } func (_c *MockRepository_Refresh_Call) Return(err error) *MockRepository_Refresh_Call { _c.Call.Return(err) return _c } func (_c *MockRepository_Refresh_Call) RunAndReturn(run func(ctx context.Context) error) *MockRepository_Refresh_Call { _c.Call.Return(run) return _c } // Time provides a mock function for the type MockRepository func (_mock *MockRepository) Time() time.Time { ret := _mock.Called() if len(ret) == 0 { panic("no return value specified for Time") } var r0 time.Time if returnFunc, ok := ret.Get(0).(func() time.Time); ok { r0 = returnFunc() } else { r0 = ret.Get(0).(time.Time) } return r0 } // MockRepository_Time_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Time' type MockRepository_Time_Call struct { *mock.Call } // Time is a helper method to define mock.On call func (_e *MockRepository_Expecter) Time() *MockRepository_Time_Call { return &MockRepository_Time_Call{Call: _e.mock.On("Time")} } func (_c *MockRepository_Time_Call) Run(run func()) *MockRepository_Time_Call { _c.Call.Run(func(args mock.Arguments) { run() }) return _c } func (_c *MockRepository_Time_Call) Return(time1 time.Time) *MockRepository_Time_Call { _c.Call.Return(time1) return _c } func (_c *MockRepository_Time_Call) RunAndReturn(run func() time.Time) *MockRepository_Time_Call { _c.Call.Return(run) return _c } // UpdateDescription provides a mock function for the type MockRepository func (_mock *MockRepository) UpdateDescription(d string) { _mock.Called(d) return } // MockRepository_UpdateDescription_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UpdateDescription' type MockRepository_UpdateDescription_Call struct { *mock.Call } // UpdateDescription is a helper method to define mock.On call // - d string func (_e *MockRepository_Expecter) UpdateDescription(d interface{}) *MockRepository_UpdateDescription_Call { return &MockRepository_UpdateDescription_Call{Call: _e.mock.On("UpdateDescription", d)} } func (_c *MockRepository_UpdateDescription_Call) Run(run func(d string)) *MockRepository_UpdateDescription_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 string if args[0] != nil { arg0 = args[0].(string) } run( arg0, ) }) return _c } func (_c *MockRepository_UpdateDescription_Call) Return() *MockRepository_UpdateDescription_Call { _c.Call.Return() return _c } func (_c *MockRepository_UpdateDescription_Call) RunAndReturn(run func(d string)) *MockRepository_UpdateDescription_Call { _c.Run(run) return _c } // VerifyObject provides a mock function for the type MockRepository func (_mock *MockRepository) VerifyObject(ctx context.Context, id object.ID) ([]content.ID, error) { ret := _mock.Called(ctx, id) if len(ret) == 0 { panic("no return value specified for VerifyObject") } var r0 []content.ID var r1 error if returnFunc, ok := ret.Get(0).(func(context.Context, object.ID) ([]content.ID, error)); ok { return returnFunc(ctx, id) } if returnFunc, ok := ret.Get(0).(func(context.Context, object.ID) []content.ID); ok { r0 = returnFunc(ctx, id) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]content.ID) } } if returnFunc, ok := ret.Get(1).(func(context.Context, object.ID) error); ok { r1 = returnFunc(ctx, id) } else { r1 = ret.Error(1) } return r0, r1 } // MockRepository_VerifyObject_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'VerifyObject' type MockRepository_VerifyObject_Call struct { *mock.Call } // VerifyObject is a helper method to define mock.On call // - ctx context.Context // - id object.ID func (_e *MockRepository_Expecter) VerifyObject(ctx interface{}, id interface{}) *MockRepository_VerifyObject_Call { return &MockRepository_VerifyObject_Call{Call: _e.mock.On("VerifyObject", ctx, id)} } func (_c *MockRepository_VerifyObject_Call) Run(run func(ctx context.Context, id object.ID)) *MockRepository_VerifyObject_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { arg0 = args[0].(context.Context) } var arg1 object.ID if args[1] != nil { arg1 = args[1].(object.ID) } run( arg0, arg1, ) }) return _c } func (_c *MockRepository_VerifyObject_Call) Return(vs []content.ID, err error) *MockRepository_VerifyObject_Call { _c.Call.Return(vs, err) return _c } func (_c *MockRepository_VerifyObject_Call) RunAndReturn(run func(ctx context.Context, id object.ID) ([]content.ID, error)) *MockRepository_VerifyObject_Call { _c.Call.Return(run) return _c } ================================================ FILE: pkg/repository/udmrepo/kopialib/backend/mocks/repository_writer.go ================================================ // Code generated by mockery; DO NOT EDIT. // github.com/vektra/mockery // template: testify package mocks import ( "context" "time" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/content" "github.com/kopia/kopia/repo/manifest" "github.com/kopia/kopia/repo/object" mock "github.com/stretchr/testify/mock" ) // NewMockRepositoryWriter creates a new instance of MockRepositoryWriter. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. func NewMockRepositoryWriter(t interface { mock.TestingT Cleanup(func()) }) *MockRepositoryWriter { mock := &MockRepositoryWriter{} mock.Mock.Test(t) t.Cleanup(func() { mock.AssertExpectations(t) }) return mock } // MockRepositoryWriter is an autogenerated mock type for the RepositoryWriter type type MockRepositoryWriter struct { mock.Mock } type MockRepositoryWriter_Expecter struct { mock *mock.Mock } func (_m *MockRepositoryWriter) EXPECT() *MockRepositoryWriter_Expecter { return &MockRepositoryWriter_Expecter{mock: &_m.Mock} } // ClientOptions provides a mock function for the type MockRepositoryWriter func (_mock *MockRepositoryWriter) ClientOptions() repo.ClientOptions { ret := _mock.Called() if len(ret) == 0 { panic("no return value specified for ClientOptions") } var r0 repo.ClientOptions if returnFunc, ok := ret.Get(0).(func() repo.ClientOptions); ok { r0 = returnFunc() } else { r0 = ret.Get(0).(repo.ClientOptions) } return r0 } // MockRepositoryWriter_ClientOptions_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ClientOptions' type MockRepositoryWriter_ClientOptions_Call struct { *mock.Call } // ClientOptions is a helper method to define mock.On call func (_e *MockRepositoryWriter_Expecter) ClientOptions() *MockRepositoryWriter_ClientOptions_Call { return &MockRepositoryWriter_ClientOptions_Call{Call: _e.mock.On("ClientOptions")} } func (_c *MockRepositoryWriter_ClientOptions_Call) Run(run func()) *MockRepositoryWriter_ClientOptions_Call { _c.Call.Run(func(args mock.Arguments) { run() }) return _c } func (_c *MockRepositoryWriter_ClientOptions_Call) Return(clientOptions repo.ClientOptions) *MockRepositoryWriter_ClientOptions_Call { _c.Call.Return(clientOptions) return _c } func (_c *MockRepositoryWriter_ClientOptions_Call) RunAndReturn(run func() repo.ClientOptions) *MockRepositoryWriter_ClientOptions_Call { _c.Call.Return(run) return _c } // Close provides a mock function for the type MockRepositoryWriter func (_mock *MockRepositoryWriter) Close(ctx context.Context) error { ret := _mock.Called(ctx) if len(ret) == 0 { panic("no return value specified for Close") } var r0 error if returnFunc, ok := ret.Get(0).(func(context.Context) error); ok { r0 = returnFunc(ctx) } else { r0 = ret.Error(0) } return r0 } // MockRepositoryWriter_Close_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Close' type MockRepositoryWriter_Close_Call struct { *mock.Call } // Close is a helper method to define mock.On call // - ctx context.Context func (_e *MockRepositoryWriter_Expecter) Close(ctx interface{}) *MockRepositoryWriter_Close_Call { return &MockRepositoryWriter_Close_Call{Call: _e.mock.On("Close", ctx)} } func (_c *MockRepositoryWriter_Close_Call) Run(run func(ctx context.Context)) *MockRepositoryWriter_Close_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { arg0 = args[0].(context.Context) } run( arg0, ) }) return _c } func (_c *MockRepositoryWriter_Close_Call) Return(err error) *MockRepositoryWriter_Close_Call { _c.Call.Return(err) return _c } func (_c *MockRepositoryWriter_Close_Call) RunAndReturn(run func(ctx context.Context) error) *MockRepositoryWriter_Close_Call { _c.Call.Return(run) return _c } // ConcatenateObjects provides a mock function for the type MockRepositoryWriter func (_mock *MockRepositoryWriter) ConcatenateObjects(ctx context.Context, objectIDs []object.ID, opt repo.ConcatenateOptions) (object.ID, error) { ret := _mock.Called(ctx, objectIDs, opt) if len(ret) == 0 { panic("no return value specified for ConcatenateObjects") } var r0 object.ID var r1 error if returnFunc, ok := ret.Get(0).(func(context.Context, []object.ID, repo.ConcatenateOptions) (object.ID, error)); ok { return returnFunc(ctx, objectIDs, opt) } if returnFunc, ok := ret.Get(0).(func(context.Context, []object.ID, repo.ConcatenateOptions) object.ID); ok { r0 = returnFunc(ctx, objectIDs, opt) } else { r0 = ret.Get(0).(object.ID) } if returnFunc, ok := ret.Get(1).(func(context.Context, []object.ID, repo.ConcatenateOptions) error); ok { r1 = returnFunc(ctx, objectIDs, opt) } else { r1 = ret.Error(1) } return r0, r1 } // MockRepositoryWriter_ConcatenateObjects_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ConcatenateObjects' type MockRepositoryWriter_ConcatenateObjects_Call struct { *mock.Call } // ConcatenateObjects is a helper method to define mock.On call // - ctx context.Context // - objectIDs []object.ID // - opt repo.ConcatenateOptions func (_e *MockRepositoryWriter_Expecter) ConcatenateObjects(ctx interface{}, objectIDs interface{}, opt interface{}) *MockRepositoryWriter_ConcatenateObjects_Call { return &MockRepositoryWriter_ConcatenateObjects_Call{Call: _e.mock.On("ConcatenateObjects", ctx, objectIDs, opt)} } func (_c *MockRepositoryWriter_ConcatenateObjects_Call) Run(run func(ctx context.Context, objectIDs []object.ID, opt repo.ConcatenateOptions)) *MockRepositoryWriter_ConcatenateObjects_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { arg0 = args[0].(context.Context) } var arg1 []object.ID if args[1] != nil { arg1 = args[1].([]object.ID) } var arg2 repo.ConcatenateOptions if args[2] != nil { arg2 = args[2].(repo.ConcatenateOptions) } run( arg0, arg1, arg2, ) }) return _c } func (_c *MockRepositoryWriter_ConcatenateObjects_Call) Return(iD object.ID, err error) *MockRepositoryWriter_ConcatenateObjects_Call { _c.Call.Return(iD, err) return _c } func (_c *MockRepositoryWriter_ConcatenateObjects_Call) RunAndReturn(run func(ctx context.Context, objectIDs []object.ID, opt repo.ConcatenateOptions) (object.ID, error)) *MockRepositoryWriter_ConcatenateObjects_Call { _c.Call.Return(run) return _c } // ContentInfo provides a mock function for the type MockRepositoryWriter func (_mock *MockRepositoryWriter) ContentInfo(ctx context.Context, contentID content.ID) (content.Info, error) { ret := _mock.Called(ctx, contentID) if len(ret) == 0 { panic("no return value specified for ContentInfo") } var r0 content.Info var r1 error if returnFunc, ok := ret.Get(0).(func(context.Context, content.ID) (content.Info, error)); ok { return returnFunc(ctx, contentID) } if returnFunc, ok := ret.Get(0).(func(context.Context, content.ID) content.Info); ok { r0 = returnFunc(ctx, contentID) } else { r0 = ret.Get(0).(content.Info) } if returnFunc, ok := ret.Get(1).(func(context.Context, content.ID) error); ok { r1 = returnFunc(ctx, contentID) } else { r1 = ret.Error(1) } return r0, r1 } // MockRepositoryWriter_ContentInfo_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ContentInfo' type MockRepositoryWriter_ContentInfo_Call struct { *mock.Call } // ContentInfo is a helper method to define mock.On call // - ctx context.Context // - contentID content.ID func (_e *MockRepositoryWriter_Expecter) ContentInfo(ctx interface{}, contentID interface{}) *MockRepositoryWriter_ContentInfo_Call { return &MockRepositoryWriter_ContentInfo_Call{Call: _e.mock.On("ContentInfo", ctx, contentID)} } func (_c *MockRepositoryWriter_ContentInfo_Call) Run(run func(ctx context.Context, contentID content.ID)) *MockRepositoryWriter_ContentInfo_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { arg0 = args[0].(context.Context) } var arg1 content.ID if args[1] != nil { arg1 = args[1].(content.ID) } run( arg0, arg1, ) }) return _c } func (_c *MockRepositoryWriter_ContentInfo_Call) Return(v content.Info, err error) *MockRepositoryWriter_ContentInfo_Call { _c.Call.Return(v, err) return _c } func (_c *MockRepositoryWriter_ContentInfo_Call) RunAndReturn(run func(ctx context.Context, contentID content.ID) (content.Info, error)) *MockRepositoryWriter_ContentInfo_Call { _c.Call.Return(run) return _c } // DeleteManifest provides a mock function for the type MockRepositoryWriter func (_mock *MockRepositoryWriter) DeleteManifest(ctx context.Context, id manifest.ID) error { ret := _mock.Called(ctx, id) if len(ret) == 0 { panic("no return value specified for DeleteManifest") } var r0 error if returnFunc, ok := ret.Get(0).(func(context.Context, manifest.ID) error); ok { r0 = returnFunc(ctx, id) } else { r0 = ret.Error(0) } return r0 } // MockRepositoryWriter_DeleteManifest_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DeleteManifest' type MockRepositoryWriter_DeleteManifest_Call struct { *mock.Call } // DeleteManifest is a helper method to define mock.On call // - ctx context.Context // - id manifest.ID func (_e *MockRepositoryWriter_Expecter) DeleteManifest(ctx interface{}, id interface{}) *MockRepositoryWriter_DeleteManifest_Call { return &MockRepositoryWriter_DeleteManifest_Call{Call: _e.mock.On("DeleteManifest", ctx, id)} } func (_c *MockRepositoryWriter_DeleteManifest_Call) Run(run func(ctx context.Context, id manifest.ID)) *MockRepositoryWriter_DeleteManifest_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { arg0 = args[0].(context.Context) } var arg1 manifest.ID if args[1] != nil { arg1 = args[1].(manifest.ID) } run( arg0, arg1, ) }) return _c } func (_c *MockRepositoryWriter_DeleteManifest_Call) Return(err error) *MockRepositoryWriter_DeleteManifest_Call { _c.Call.Return(err) return _c } func (_c *MockRepositoryWriter_DeleteManifest_Call) RunAndReturn(run func(ctx context.Context, id manifest.ID) error) *MockRepositoryWriter_DeleteManifest_Call { _c.Call.Return(run) return _c } // FindManifests provides a mock function for the type MockRepositoryWriter func (_mock *MockRepositoryWriter) FindManifests(ctx context.Context, labels map[string]string) ([]*manifest.EntryMetadata, error) { ret := _mock.Called(ctx, labels) if len(ret) == 0 { panic("no return value specified for FindManifests") } var r0 []*manifest.EntryMetadata var r1 error if returnFunc, ok := ret.Get(0).(func(context.Context, map[string]string) ([]*manifest.EntryMetadata, error)); ok { return returnFunc(ctx, labels) } if returnFunc, ok := ret.Get(0).(func(context.Context, map[string]string) []*manifest.EntryMetadata); ok { r0 = returnFunc(ctx, labels) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]*manifest.EntryMetadata) } } if returnFunc, ok := ret.Get(1).(func(context.Context, map[string]string) error); ok { r1 = returnFunc(ctx, labels) } else { r1 = ret.Error(1) } return r0, r1 } // MockRepositoryWriter_FindManifests_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'FindManifests' type MockRepositoryWriter_FindManifests_Call struct { *mock.Call } // FindManifests is a helper method to define mock.On call // - ctx context.Context // - labels map[string]string func (_e *MockRepositoryWriter_Expecter) FindManifests(ctx interface{}, labels interface{}) *MockRepositoryWriter_FindManifests_Call { return &MockRepositoryWriter_FindManifests_Call{Call: _e.mock.On("FindManifests", ctx, labels)} } func (_c *MockRepositoryWriter_FindManifests_Call) Run(run func(ctx context.Context, labels map[string]string)) *MockRepositoryWriter_FindManifests_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { arg0 = args[0].(context.Context) } var arg1 map[string]string if args[1] != nil { arg1 = args[1].(map[string]string) } run( arg0, arg1, ) }) return _c } func (_c *MockRepositoryWriter_FindManifests_Call) Return(entryMetadatas []*manifest.EntryMetadata, err error) *MockRepositoryWriter_FindManifests_Call { _c.Call.Return(entryMetadatas, err) return _c } func (_c *MockRepositoryWriter_FindManifests_Call) RunAndReturn(run func(ctx context.Context, labels map[string]string) ([]*manifest.EntryMetadata, error)) *MockRepositoryWriter_FindManifests_Call { _c.Call.Return(run) return _c } // Flush provides a mock function for the type MockRepositoryWriter func (_mock *MockRepositoryWriter) Flush(ctx context.Context) error { ret := _mock.Called(ctx) if len(ret) == 0 { panic("no return value specified for Flush") } var r0 error if returnFunc, ok := ret.Get(0).(func(context.Context) error); ok { r0 = returnFunc(ctx) } else { r0 = ret.Error(0) } return r0 } // MockRepositoryWriter_Flush_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Flush' type MockRepositoryWriter_Flush_Call struct { *mock.Call } // Flush is a helper method to define mock.On call // - ctx context.Context func (_e *MockRepositoryWriter_Expecter) Flush(ctx interface{}) *MockRepositoryWriter_Flush_Call { return &MockRepositoryWriter_Flush_Call{Call: _e.mock.On("Flush", ctx)} } func (_c *MockRepositoryWriter_Flush_Call) Run(run func(ctx context.Context)) *MockRepositoryWriter_Flush_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { arg0 = args[0].(context.Context) } run( arg0, ) }) return _c } func (_c *MockRepositoryWriter_Flush_Call) Return(err error) *MockRepositoryWriter_Flush_Call { _c.Call.Return(err) return _c } func (_c *MockRepositoryWriter_Flush_Call) RunAndReturn(run func(ctx context.Context) error) *MockRepositoryWriter_Flush_Call { _c.Call.Return(run) return _c } // GetManifest provides a mock function for the type MockRepositoryWriter func (_mock *MockRepositoryWriter) GetManifest(ctx context.Context, id manifest.ID, data any) (*manifest.EntryMetadata, error) { ret := _mock.Called(ctx, id, data) if len(ret) == 0 { panic("no return value specified for GetManifest") } var r0 *manifest.EntryMetadata var r1 error if returnFunc, ok := ret.Get(0).(func(context.Context, manifest.ID, any) (*manifest.EntryMetadata, error)); ok { return returnFunc(ctx, id, data) } if returnFunc, ok := ret.Get(0).(func(context.Context, manifest.ID, any) *manifest.EntryMetadata); ok { r0 = returnFunc(ctx, id, data) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(*manifest.EntryMetadata) } } if returnFunc, ok := ret.Get(1).(func(context.Context, manifest.ID, any) error); ok { r1 = returnFunc(ctx, id, data) } else { r1 = ret.Error(1) } return r0, r1 } // MockRepositoryWriter_GetManifest_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetManifest' type MockRepositoryWriter_GetManifest_Call struct { *mock.Call } // GetManifest is a helper method to define mock.On call // - ctx context.Context // - id manifest.ID // - data any func (_e *MockRepositoryWriter_Expecter) GetManifest(ctx interface{}, id interface{}, data interface{}) *MockRepositoryWriter_GetManifest_Call { return &MockRepositoryWriter_GetManifest_Call{Call: _e.mock.On("GetManifest", ctx, id, data)} } func (_c *MockRepositoryWriter_GetManifest_Call) Run(run func(ctx context.Context, id manifest.ID, data any)) *MockRepositoryWriter_GetManifest_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { arg0 = args[0].(context.Context) } var arg1 manifest.ID if args[1] != nil { arg1 = args[1].(manifest.ID) } var arg2 any if args[2] != nil { arg2 = args[2].(any) } run( arg0, arg1, arg2, ) }) return _c } func (_c *MockRepositoryWriter_GetManifest_Call) Return(entryMetadata *manifest.EntryMetadata, err error) *MockRepositoryWriter_GetManifest_Call { _c.Call.Return(entryMetadata, err) return _c } func (_c *MockRepositoryWriter_GetManifest_Call) RunAndReturn(run func(ctx context.Context, id manifest.ID, data any) (*manifest.EntryMetadata, error)) *MockRepositoryWriter_GetManifest_Call { _c.Call.Return(run) return _c } // NewObjectWriter provides a mock function for the type MockRepositoryWriter func (_mock *MockRepositoryWriter) NewObjectWriter(ctx context.Context, opt object.WriterOptions) object.Writer { ret := _mock.Called(ctx, opt) if len(ret) == 0 { panic("no return value specified for NewObjectWriter") } var r0 object.Writer if returnFunc, ok := ret.Get(0).(func(context.Context, object.WriterOptions) object.Writer); ok { r0 = returnFunc(ctx, opt) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(object.Writer) } } return r0 } // MockRepositoryWriter_NewObjectWriter_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'NewObjectWriter' type MockRepositoryWriter_NewObjectWriter_Call struct { *mock.Call } // NewObjectWriter is a helper method to define mock.On call // - ctx context.Context // - opt object.WriterOptions func (_e *MockRepositoryWriter_Expecter) NewObjectWriter(ctx interface{}, opt interface{}) *MockRepositoryWriter_NewObjectWriter_Call { return &MockRepositoryWriter_NewObjectWriter_Call{Call: _e.mock.On("NewObjectWriter", ctx, opt)} } func (_c *MockRepositoryWriter_NewObjectWriter_Call) Run(run func(ctx context.Context, opt object.WriterOptions)) *MockRepositoryWriter_NewObjectWriter_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { arg0 = args[0].(context.Context) } var arg1 object.WriterOptions if args[1] != nil { arg1 = args[1].(object.WriterOptions) } run( arg0, arg1, ) }) return _c } func (_c *MockRepositoryWriter_NewObjectWriter_Call) Return(writer object.Writer) *MockRepositoryWriter_NewObjectWriter_Call { _c.Call.Return(writer) return _c } func (_c *MockRepositoryWriter_NewObjectWriter_Call) RunAndReturn(run func(ctx context.Context, opt object.WriterOptions) object.Writer) *MockRepositoryWriter_NewObjectWriter_Call { _c.Call.Return(run) return _c } // NewWriter provides a mock function for the type MockRepositoryWriter func (_mock *MockRepositoryWriter) NewWriter(ctx context.Context, opt repo.WriteSessionOptions) (context.Context, repo.RepositoryWriter, error) { ret := _mock.Called(ctx, opt) if len(ret) == 0 { panic("no return value specified for NewWriter") } var r0 context.Context var r1 repo.RepositoryWriter var r2 error if returnFunc, ok := ret.Get(0).(func(context.Context, repo.WriteSessionOptions) (context.Context, repo.RepositoryWriter, error)); ok { return returnFunc(ctx, opt) } if returnFunc, ok := ret.Get(0).(func(context.Context, repo.WriteSessionOptions) context.Context); ok { r0 = returnFunc(ctx, opt) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(context.Context) } } if returnFunc, ok := ret.Get(1).(func(context.Context, repo.WriteSessionOptions) repo.RepositoryWriter); ok { r1 = returnFunc(ctx, opt) } else { if ret.Get(1) != nil { r1 = ret.Get(1).(repo.RepositoryWriter) } } if returnFunc, ok := ret.Get(2).(func(context.Context, repo.WriteSessionOptions) error); ok { r2 = returnFunc(ctx, opt) } else { r2 = ret.Error(2) } return r0, r1, r2 } // MockRepositoryWriter_NewWriter_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'NewWriter' type MockRepositoryWriter_NewWriter_Call struct { *mock.Call } // NewWriter is a helper method to define mock.On call // - ctx context.Context // - opt repo.WriteSessionOptions func (_e *MockRepositoryWriter_Expecter) NewWriter(ctx interface{}, opt interface{}) *MockRepositoryWriter_NewWriter_Call { return &MockRepositoryWriter_NewWriter_Call{Call: _e.mock.On("NewWriter", ctx, opt)} } func (_c *MockRepositoryWriter_NewWriter_Call) Run(run func(ctx context.Context, opt repo.WriteSessionOptions)) *MockRepositoryWriter_NewWriter_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { arg0 = args[0].(context.Context) } var arg1 repo.WriteSessionOptions if args[1] != nil { arg1 = args[1].(repo.WriteSessionOptions) } run( arg0, arg1, ) }) return _c } func (_c *MockRepositoryWriter_NewWriter_Call) Return(context1 context.Context, repositoryWriter repo.RepositoryWriter, err error) *MockRepositoryWriter_NewWriter_Call { _c.Call.Return(context1, repositoryWriter, err) return _c } func (_c *MockRepositoryWriter_NewWriter_Call) RunAndReturn(run func(ctx context.Context, opt repo.WriteSessionOptions) (context.Context, repo.RepositoryWriter, error)) *MockRepositoryWriter_NewWriter_Call { _c.Call.Return(run) return _c } // OnSuccessfulFlush provides a mock function for the type MockRepositoryWriter func (_mock *MockRepositoryWriter) OnSuccessfulFlush(callback repo.RepositoryWriterCallback) { _mock.Called(callback) return } // MockRepositoryWriter_OnSuccessfulFlush_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'OnSuccessfulFlush' type MockRepositoryWriter_OnSuccessfulFlush_Call struct { *mock.Call } // OnSuccessfulFlush is a helper method to define mock.On call // - callback repo.RepositoryWriterCallback func (_e *MockRepositoryWriter_Expecter) OnSuccessfulFlush(callback interface{}) *MockRepositoryWriter_OnSuccessfulFlush_Call { return &MockRepositoryWriter_OnSuccessfulFlush_Call{Call: _e.mock.On("OnSuccessfulFlush", callback)} } func (_c *MockRepositoryWriter_OnSuccessfulFlush_Call) Run(run func(callback repo.RepositoryWriterCallback)) *MockRepositoryWriter_OnSuccessfulFlush_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 repo.RepositoryWriterCallback if args[0] != nil { arg0 = args[0].(repo.RepositoryWriterCallback) } run( arg0, ) }) return _c } func (_c *MockRepositoryWriter_OnSuccessfulFlush_Call) Return() *MockRepositoryWriter_OnSuccessfulFlush_Call { _c.Call.Return() return _c } func (_c *MockRepositoryWriter_OnSuccessfulFlush_Call) RunAndReturn(run func(callback repo.RepositoryWriterCallback)) *MockRepositoryWriter_OnSuccessfulFlush_Call { _c.Run(run) return _c } // OpenObject provides a mock function for the type MockRepositoryWriter func (_mock *MockRepositoryWriter) OpenObject(ctx context.Context, id object.ID) (object.Reader, error) { ret := _mock.Called(ctx, id) if len(ret) == 0 { panic("no return value specified for OpenObject") } var r0 object.Reader var r1 error if returnFunc, ok := ret.Get(0).(func(context.Context, object.ID) (object.Reader, error)); ok { return returnFunc(ctx, id) } if returnFunc, ok := ret.Get(0).(func(context.Context, object.ID) object.Reader); ok { r0 = returnFunc(ctx, id) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(object.Reader) } } if returnFunc, ok := ret.Get(1).(func(context.Context, object.ID) error); ok { r1 = returnFunc(ctx, id) } else { r1 = ret.Error(1) } return r0, r1 } // MockRepositoryWriter_OpenObject_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'OpenObject' type MockRepositoryWriter_OpenObject_Call struct { *mock.Call } // OpenObject is a helper method to define mock.On call // - ctx context.Context // - id object.ID func (_e *MockRepositoryWriter_Expecter) OpenObject(ctx interface{}, id interface{}) *MockRepositoryWriter_OpenObject_Call { return &MockRepositoryWriter_OpenObject_Call{Call: _e.mock.On("OpenObject", ctx, id)} } func (_c *MockRepositoryWriter_OpenObject_Call) Run(run func(ctx context.Context, id object.ID)) *MockRepositoryWriter_OpenObject_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { arg0 = args[0].(context.Context) } var arg1 object.ID if args[1] != nil { arg1 = args[1].(object.ID) } run( arg0, arg1, ) }) return _c } func (_c *MockRepositoryWriter_OpenObject_Call) Return(reader object.Reader, err error) *MockRepositoryWriter_OpenObject_Call { _c.Call.Return(reader, err) return _c } func (_c *MockRepositoryWriter_OpenObject_Call) RunAndReturn(run func(ctx context.Context, id object.ID) (object.Reader, error)) *MockRepositoryWriter_OpenObject_Call { _c.Call.Return(run) return _c } // PrefetchContents provides a mock function for the type MockRepositoryWriter func (_mock *MockRepositoryWriter) PrefetchContents(ctx context.Context, contentIDs []content.ID, hint string) []content.ID { ret := _mock.Called(ctx, contentIDs, hint) if len(ret) == 0 { panic("no return value specified for PrefetchContents") } var r0 []content.ID if returnFunc, ok := ret.Get(0).(func(context.Context, []content.ID, string) []content.ID); ok { r0 = returnFunc(ctx, contentIDs, hint) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]content.ID) } } return r0 } // MockRepositoryWriter_PrefetchContents_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'PrefetchContents' type MockRepositoryWriter_PrefetchContents_Call struct { *mock.Call } // PrefetchContents is a helper method to define mock.On call // - ctx context.Context // - contentIDs []content.ID // - hint string func (_e *MockRepositoryWriter_Expecter) PrefetchContents(ctx interface{}, contentIDs interface{}, hint interface{}) *MockRepositoryWriter_PrefetchContents_Call { return &MockRepositoryWriter_PrefetchContents_Call{Call: _e.mock.On("PrefetchContents", ctx, contentIDs, hint)} } func (_c *MockRepositoryWriter_PrefetchContents_Call) Run(run func(ctx context.Context, contentIDs []content.ID, hint string)) *MockRepositoryWriter_PrefetchContents_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { arg0 = args[0].(context.Context) } var arg1 []content.ID if args[1] != nil { arg1 = args[1].([]content.ID) } var arg2 string if args[2] != nil { arg2 = args[2].(string) } run( arg0, arg1, arg2, ) }) return _c } func (_c *MockRepositoryWriter_PrefetchContents_Call) Return(vs []content.ID) *MockRepositoryWriter_PrefetchContents_Call { _c.Call.Return(vs) return _c } func (_c *MockRepositoryWriter_PrefetchContents_Call) RunAndReturn(run func(ctx context.Context, contentIDs []content.ID, hint string) []content.ID) *MockRepositoryWriter_PrefetchContents_Call { _c.Call.Return(run) return _c } // PrefetchObjects provides a mock function for the type MockRepositoryWriter func (_mock *MockRepositoryWriter) PrefetchObjects(ctx context.Context, objectIDs []object.ID, hint string) ([]content.ID, error) { ret := _mock.Called(ctx, objectIDs, hint) if len(ret) == 0 { panic("no return value specified for PrefetchObjects") } var r0 []content.ID var r1 error if returnFunc, ok := ret.Get(0).(func(context.Context, []object.ID, string) ([]content.ID, error)); ok { return returnFunc(ctx, objectIDs, hint) } if returnFunc, ok := ret.Get(0).(func(context.Context, []object.ID, string) []content.ID); ok { r0 = returnFunc(ctx, objectIDs, hint) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]content.ID) } } if returnFunc, ok := ret.Get(1).(func(context.Context, []object.ID, string) error); ok { r1 = returnFunc(ctx, objectIDs, hint) } else { r1 = ret.Error(1) } return r0, r1 } // MockRepositoryWriter_PrefetchObjects_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'PrefetchObjects' type MockRepositoryWriter_PrefetchObjects_Call struct { *mock.Call } // PrefetchObjects is a helper method to define mock.On call // - ctx context.Context // - objectIDs []object.ID // - hint string func (_e *MockRepositoryWriter_Expecter) PrefetchObjects(ctx interface{}, objectIDs interface{}, hint interface{}) *MockRepositoryWriter_PrefetchObjects_Call { return &MockRepositoryWriter_PrefetchObjects_Call{Call: _e.mock.On("PrefetchObjects", ctx, objectIDs, hint)} } func (_c *MockRepositoryWriter_PrefetchObjects_Call) Run(run func(ctx context.Context, objectIDs []object.ID, hint string)) *MockRepositoryWriter_PrefetchObjects_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { arg0 = args[0].(context.Context) } var arg1 []object.ID if args[1] != nil { arg1 = args[1].([]object.ID) } var arg2 string if args[2] != nil { arg2 = args[2].(string) } run( arg0, arg1, arg2, ) }) return _c } func (_c *MockRepositoryWriter_PrefetchObjects_Call) Return(vs []content.ID, err error) *MockRepositoryWriter_PrefetchObjects_Call { _c.Call.Return(vs, err) return _c } func (_c *MockRepositoryWriter_PrefetchObjects_Call) RunAndReturn(run func(ctx context.Context, objectIDs []object.ID, hint string) ([]content.ID, error)) *MockRepositoryWriter_PrefetchObjects_Call { _c.Call.Return(run) return _c } // PutManifest provides a mock function for the type MockRepositoryWriter func (_mock *MockRepositoryWriter) PutManifest(ctx context.Context, labels map[string]string, payload any) (manifest.ID, error) { ret := _mock.Called(ctx, labels, payload) if len(ret) == 0 { panic("no return value specified for PutManifest") } var r0 manifest.ID var r1 error if returnFunc, ok := ret.Get(0).(func(context.Context, map[string]string, any) (manifest.ID, error)); ok { return returnFunc(ctx, labels, payload) } if returnFunc, ok := ret.Get(0).(func(context.Context, map[string]string, any) manifest.ID); ok { r0 = returnFunc(ctx, labels, payload) } else { r0 = ret.Get(0).(manifest.ID) } if returnFunc, ok := ret.Get(1).(func(context.Context, map[string]string, any) error); ok { r1 = returnFunc(ctx, labels, payload) } else { r1 = ret.Error(1) } return r0, r1 } // MockRepositoryWriter_PutManifest_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'PutManifest' type MockRepositoryWriter_PutManifest_Call struct { *mock.Call } // PutManifest is a helper method to define mock.On call // - ctx context.Context // - labels map[string]string // - payload any func (_e *MockRepositoryWriter_Expecter) PutManifest(ctx interface{}, labels interface{}, payload interface{}) *MockRepositoryWriter_PutManifest_Call { return &MockRepositoryWriter_PutManifest_Call{Call: _e.mock.On("PutManifest", ctx, labels, payload)} } func (_c *MockRepositoryWriter_PutManifest_Call) Run(run func(ctx context.Context, labels map[string]string, payload any)) *MockRepositoryWriter_PutManifest_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { arg0 = args[0].(context.Context) } var arg1 map[string]string if args[1] != nil { arg1 = args[1].(map[string]string) } var arg2 any if args[2] != nil { arg2 = args[2].(any) } run( arg0, arg1, arg2, ) }) return _c } func (_c *MockRepositoryWriter_PutManifest_Call) Return(iD manifest.ID, err error) *MockRepositoryWriter_PutManifest_Call { _c.Call.Return(iD, err) return _c } func (_c *MockRepositoryWriter_PutManifest_Call) RunAndReturn(run func(ctx context.Context, labels map[string]string, payload any) (manifest.ID, error)) *MockRepositoryWriter_PutManifest_Call { _c.Call.Return(run) return _c } // Refresh provides a mock function for the type MockRepositoryWriter func (_mock *MockRepositoryWriter) Refresh(ctx context.Context) error { ret := _mock.Called(ctx) if len(ret) == 0 { panic("no return value specified for Refresh") } var r0 error if returnFunc, ok := ret.Get(0).(func(context.Context) error); ok { r0 = returnFunc(ctx) } else { r0 = ret.Error(0) } return r0 } // MockRepositoryWriter_Refresh_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Refresh' type MockRepositoryWriter_Refresh_Call struct { *mock.Call } // Refresh is a helper method to define mock.On call // - ctx context.Context func (_e *MockRepositoryWriter_Expecter) Refresh(ctx interface{}) *MockRepositoryWriter_Refresh_Call { return &MockRepositoryWriter_Refresh_Call{Call: _e.mock.On("Refresh", ctx)} } func (_c *MockRepositoryWriter_Refresh_Call) Run(run func(ctx context.Context)) *MockRepositoryWriter_Refresh_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { arg0 = args[0].(context.Context) } run( arg0, ) }) return _c } func (_c *MockRepositoryWriter_Refresh_Call) Return(err error) *MockRepositoryWriter_Refresh_Call { _c.Call.Return(err) return _c } func (_c *MockRepositoryWriter_Refresh_Call) RunAndReturn(run func(ctx context.Context) error) *MockRepositoryWriter_Refresh_Call { _c.Call.Return(run) return _c } // ReplaceManifests provides a mock function for the type MockRepositoryWriter func (_mock *MockRepositoryWriter) ReplaceManifests(ctx context.Context, labels map[string]string, payload any) (manifest.ID, error) { ret := _mock.Called(ctx, labels, payload) if len(ret) == 0 { panic("no return value specified for ReplaceManifests") } var r0 manifest.ID var r1 error if returnFunc, ok := ret.Get(0).(func(context.Context, map[string]string, any) (manifest.ID, error)); ok { return returnFunc(ctx, labels, payload) } if returnFunc, ok := ret.Get(0).(func(context.Context, map[string]string, any) manifest.ID); ok { r0 = returnFunc(ctx, labels, payload) } else { r0 = ret.Get(0).(manifest.ID) } if returnFunc, ok := ret.Get(1).(func(context.Context, map[string]string, any) error); ok { r1 = returnFunc(ctx, labels, payload) } else { r1 = ret.Error(1) } return r0, r1 } // MockRepositoryWriter_ReplaceManifests_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ReplaceManifests' type MockRepositoryWriter_ReplaceManifests_Call struct { *mock.Call } // ReplaceManifests is a helper method to define mock.On call // - ctx context.Context // - labels map[string]string // - payload any func (_e *MockRepositoryWriter_Expecter) ReplaceManifests(ctx interface{}, labels interface{}, payload interface{}) *MockRepositoryWriter_ReplaceManifests_Call { return &MockRepositoryWriter_ReplaceManifests_Call{Call: _e.mock.On("ReplaceManifests", ctx, labels, payload)} } func (_c *MockRepositoryWriter_ReplaceManifests_Call) Run(run func(ctx context.Context, labels map[string]string, payload any)) *MockRepositoryWriter_ReplaceManifests_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { arg0 = args[0].(context.Context) } var arg1 map[string]string if args[1] != nil { arg1 = args[1].(map[string]string) } var arg2 any if args[2] != nil { arg2 = args[2].(any) } run( arg0, arg1, arg2, ) }) return _c } func (_c *MockRepositoryWriter_ReplaceManifests_Call) Return(iD manifest.ID, err error) *MockRepositoryWriter_ReplaceManifests_Call { _c.Call.Return(iD, err) return _c } func (_c *MockRepositoryWriter_ReplaceManifests_Call) RunAndReturn(run func(ctx context.Context, labels map[string]string, payload any) (manifest.ID, error)) *MockRepositoryWriter_ReplaceManifests_Call { _c.Call.Return(run) return _c } // Time provides a mock function for the type MockRepositoryWriter func (_mock *MockRepositoryWriter) Time() time.Time { ret := _mock.Called() if len(ret) == 0 { panic("no return value specified for Time") } var r0 time.Time if returnFunc, ok := ret.Get(0).(func() time.Time); ok { r0 = returnFunc() } else { r0 = ret.Get(0).(time.Time) } return r0 } // MockRepositoryWriter_Time_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Time' type MockRepositoryWriter_Time_Call struct { *mock.Call } // Time is a helper method to define mock.On call func (_e *MockRepositoryWriter_Expecter) Time() *MockRepositoryWriter_Time_Call { return &MockRepositoryWriter_Time_Call{Call: _e.mock.On("Time")} } func (_c *MockRepositoryWriter_Time_Call) Run(run func()) *MockRepositoryWriter_Time_Call { _c.Call.Run(func(args mock.Arguments) { run() }) return _c } func (_c *MockRepositoryWriter_Time_Call) Return(time1 time.Time) *MockRepositoryWriter_Time_Call { _c.Call.Return(time1) return _c } func (_c *MockRepositoryWriter_Time_Call) RunAndReturn(run func() time.Time) *MockRepositoryWriter_Time_Call { _c.Call.Return(run) return _c } // UpdateDescription provides a mock function for the type MockRepositoryWriter func (_mock *MockRepositoryWriter) UpdateDescription(d string) { _mock.Called(d) return } // MockRepositoryWriter_UpdateDescription_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UpdateDescription' type MockRepositoryWriter_UpdateDescription_Call struct { *mock.Call } // UpdateDescription is a helper method to define mock.On call // - d string func (_e *MockRepositoryWriter_Expecter) UpdateDescription(d interface{}) *MockRepositoryWriter_UpdateDescription_Call { return &MockRepositoryWriter_UpdateDescription_Call{Call: _e.mock.On("UpdateDescription", d)} } func (_c *MockRepositoryWriter_UpdateDescription_Call) Run(run func(d string)) *MockRepositoryWriter_UpdateDescription_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 string if args[0] != nil { arg0 = args[0].(string) } run( arg0, ) }) return _c } func (_c *MockRepositoryWriter_UpdateDescription_Call) Return() *MockRepositoryWriter_UpdateDescription_Call { _c.Call.Return() return _c } func (_c *MockRepositoryWriter_UpdateDescription_Call) RunAndReturn(run func(d string)) *MockRepositoryWriter_UpdateDescription_Call { _c.Run(run) return _c } // VerifyObject provides a mock function for the type MockRepositoryWriter func (_mock *MockRepositoryWriter) VerifyObject(ctx context.Context, id object.ID) ([]content.ID, error) { ret := _mock.Called(ctx, id) if len(ret) == 0 { panic("no return value specified for VerifyObject") } var r0 []content.ID var r1 error if returnFunc, ok := ret.Get(0).(func(context.Context, object.ID) ([]content.ID, error)); ok { return returnFunc(ctx, id) } if returnFunc, ok := ret.Get(0).(func(context.Context, object.ID) []content.ID); ok { r0 = returnFunc(ctx, id) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]content.ID) } } if returnFunc, ok := ret.Get(1).(func(context.Context, object.ID) error); ok { r1 = returnFunc(ctx, id) } else { r1 = ret.Error(1) } return r0, r1 } // MockRepositoryWriter_VerifyObject_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'VerifyObject' type MockRepositoryWriter_VerifyObject_Call struct { *mock.Call } // VerifyObject is a helper method to define mock.On call // - ctx context.Context // - id object.ID func (_e *MockRepositoryWriter_Expecter) VerifyObject(ctx interface{}, id interface{}) *MockRepositoryWriter_VerifyObject_Call { return &MockRepositoryWriter_VerifyObject_Call{Call: _e.mock.On("VerifyObject", ctx, id)} } func (_c *MockRepositoryWriter_VerifyObject_Call) Run(run func(ctx context.Context, id object.ID)) *MockRepositoryWriter_VerifyObject_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { arg0 = args[0].(context.Context) } var arg1 object.ID if args[1] != nil { arg1 = args[1].(object.ID) } run( arg0, arg1, ) }) return _c } func (_c *MockRepositoryWriter_VerifyObject_Call) Return(vs []content.ID, err error) *MockRepositoryWriter_VerifyObject_Call { _c.Call.Return(vs, err) return _c } func (_c *MockRepositoryWriter_VerifyObject_Call) RunAndReturn(run func(ctx context.Context, id object.ID) ([]content.ID, error)) *MockRepositoryWriter_VerifyObject_Call { _c.Call.Return(run) return _c } ================================================ FILE: pkg/repository/udmrepo/kopialib/backend/s3.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backend import ( "context" "github.com/sirupsen/logrus" "github.com/kopia/kopia/repo/blob" "github.com/kopia/kopia/repo/blob/s3" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo/kopialib/backend/logging" ) type S3Backend struct { options s3.Options } func (c *S3Backend) Setup(ctx context.Context, flags map[string]string, logger logrus.FieldLogger) error { var err error c.options.BucketName, err = mustHaveString(udmrepo.StoreOptionOssBucket, flags) if err != nil { return err } c.options.AccessKeyID = optionalHaveString(udmrepo.StoreOptionS3KeyID, flags) c.options.SecretAccessKey = optionalHaveString(udmrepo.StoreOptionS3SecretKey, flags) c.options.Endpoint = optionalHaveString(udmrepo.StoreOptionS3Endpoint, flags) c.options.Region = optionalHaveString(udmrepo.StoreOptionOssRegion, flags) c.options.Prefix = optionalHaveString(udmrepo.StoreOptionPrefix, flags) c.options.DoNotUseTLS = optionalHaveBool(ctx, udmrepo.StoreOptionS3DisableTLS, flags) c.options.DoNotVerifyTLS = optionalHaveBool(ctx, udmrepo.StoreOptionS3DisableTLSVerify, flags) c.options.SessionToken = optionalHaveString(udmrepo.StoreOptionS3Token, flags) c.options.RootCA = optionalHaveBase64(ctx, udmrepo.StoreOptionCACert, flags) ctx = logging.WithLogger(ctx, logger) c.options.Limits = setupLimits(ctx, flags) return nil } func (c *S3Backend) Connect(ctx context.Context, isCreate bool, logger logrus.FieldLogger) (blob.Storage, error) { ctx = logging.WithLogger(ctx, logger) return s3.New(ctx, &c.options, false) } ================================================ FILE: pkg/repository/udmrepo/kopialib/backend/s3_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backend import ( "testing" velerotest "github.com/vmware-tanzu/velero/pkg/test" "github.com/kopia/kopia/repo/blob/s3" "github.com/stretchr/testify/assert" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" ) func TestS3Setup(t *testing.T) { testCases := []struct { name string flags map[string]string expectedOptions s3.Options expectedErr string }{ { name: "must have bucket name", flags: map[string]string{}, expectedErr: "key " + udmrepo.StoreOptionOssBucket + " not found", }, { name: "with bucket only", flags: map[string]string{ udmrepo.StoreOptionOssBucket: "fake-bucket", }, expectedOptions: s3.Options{ BucketName: "fake-bucket", }, }, { name: "with others", flags: map[string]string{ udmrepo.StoreOptionOssBucket: "fake-bucket", udmrepo.StoreOptionS3KeyID: "fake-ak", udmrepo.StoreOptionS3SecretKey: "fake-sk", udmrepo.StoreOptionS3Endpoint: "fake-endpoint", udmrepo.StoreOptionOssRegion: "fake-region", udmrepo.StoreOptionPrefix: "fake-prefix", udmrepo.StoreOptionS3Token: "fake-token", }, expectedOptions: s3.Options{ BucketName: "fake-bucket", AccessKeyID: "fake-ak", SecretAccessKey: "fake-sk", Endpoint: "fake-endpoint", Region: "fake-region", Prefix: "fake-prefix", SessionToken: "fake-token", }, }, { name: "with wrong tls", flags: map[string]string{ udmrepo.StoreOptionOssBucket: "fake-bucket", udmrepo.StoreOptionS3DisableTLS: "fake-bool", udmrepo.StoreOptionS3DisableTLSVerify: "fake-bool", }, expectedOptions: s3.Options{ BucketName: "fake-bucket", }, }, { name: "with correct tls", flags: map[string]string{ udmrepo.StoreOptionOssBucket: "fake-bucket", udmrepo.StoreOptionS3DisableTLS: "true", udmrepo.StoreOptionS3DisableTLSVerify: "false", }, expectedOptions: s3.Options{ BucketName: "fake-bucket", DoNotUseTLS: true, DoNotVerifyTLS: false, }, }, { name: "with wrong ca", flags: map[string]string{ udmrepo.StoreOptionOssBucket: "fake-bucket", udmrepo.StoreOptionCACert: "fake-base-64", }, expectedOptions: s3.Options{ BucketName: "fake-bucket", }, }, { name: "with correct ca", flags: map[string]string{ udmrepo.StoreOptionOssBucket: "fake-bucket", udmrepo.StoreOptionCACert: "ZmFrZS1jYQ==", }, expectedOptions: s3.Options{ BucketName: "fake-bucket", RootCA: []byte{'f', 'a', 'k', 'e', '-', 'c', 'a'}, }, }, } logger := velerotest.NewLogger() for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { s3Flags := S3Backend{} err := s3Flags.Setup(t.Context(), tc.flags, logger) if tc.expectedErr == "" { assert.NoError(t, err) } else { assert.EqualError(t, err, tc.expectedErr) } }) } } ================================================ FILE: pkg/repository/udmrepo/kopialib/backend/utils.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backend import ( "context" "encoding/base64" "strconv" "time" "github.com/kopia/kopia/repo/logging" "github.com/pkg/errors" ) func mustHaveString(key string, flags map[string]string) (string, error) { if value, exist := flags[key]; exist { return value, nil } return "", errors.Errorf("key %s not found", key) } func optionalHaveString(key string, flags map[string]string) string { return optionalHaveStringWithDefault(key, flags, "") } func optionalHaveBool(ctx context.Context, key string, flags map[string]string) bool { if value, exist := flags[key]; exist { if value != "" { ret, err := strconv.ParseBool(value) if err == nil { return ret } backendLog()(ctx).Errorf("Ignore %s, value [%s] is invalid, err %v", key, value, err) } } return false } func optionalHaveFloat64(ctx context.Context, key string, flags map[string]string) float64 { if value, exist := flags[key]; exist { ret, err := strconv.ParseFloat(value, 64) if err == nil { return ret } backendLog()(ctx).Errorf("Ignore %s, value [%s] is invalid, err %v", key, value, err) } return 0 } func optionalHaveStringWithDefault(key string, flags map[string]string, defValue string) string { if value, exist := flags[key]; exist { return value } return defValue } func optionalHaveDuration(ctx context.Context, key string, flags map[string]string) time.Duration { if value, exist := flags[key]; exist { ret, err := time.ParseDuration(value) if err == nil { return ret } backendLog()(ctx).Errorf("Ignore %s, value [%s] is invalid, err %v", key, value, err) } return 0 } func optionalHaveBase64(ctx context.Context, key string, flags map[string]string) []byte { if value, exist := flags[key]; exist { ret, err := base64.StdEncoding.DecodeString(value) if err == nil { return ret } backendLog()(ctx).Errorf("Ignore %s, value [%s] is invalid, err %v", key, value, err) } return nil } func optionalHaveIntWithDefault(ctx context.Context, key string, flags map[string]string, defValue int64) int64 { if value, exist := flags[key]; exist { if value != "" { ret, err := strconv.ParseInt(value, 10, 64) if err == nil { return ret } backendLog()(ctx).Errorf("Ignore %s, value [%s] is invalid, err %v", key, value, err) } } return defValue } func backendLog() func(ctx context.Context) logging.Logger { return logging.Module("kopialib-bd") } ================================================ FILE: pkg/repository/udmrepo/kopialib/backend/utils_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backend import ( "testing" "github.com/kopia/kopia/repo/logging" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "go.uber.org/zap" "go.uber.org/zap/zapcore" storagemocks "github.com/vmware-tanzu/velero/pkg/repository/udmrepo/kopialib/backend/mocks" ) func TestOptionalHaveBool(t *testing.T) { var expectMsg string testCases := []struct { name string key string flags map[string]string logger *storagemocks.Core retFuncCheck func(mock.Arguments) expectMsg string retValue bool }{ { name: "key not exist", key: "fake-key", flags: map[string]string{}, retValue: false, }, { name: "value valid", key: "fake-key", flags: map[string]string{ "fake-key": "true", }, retValue: true, }, { name: "value invalid", key: "fake-key", flags: map[string]string{ "fake-key": "fake-value", }, logger: new(storagemocks.Core), retFuncCheck: func(args mock.Arguments) { ent := args[0].(zapcore.Entry) if ent.Level == zapcore.ErrorLevel { expectMsg = ent.Message } }, expectMsg: "Ignore fake-key, value [fake-value] is invalid, err strconv.ParseBool: parsing \"fake-value\": invalid syntax", retValue: false, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { if tc.logger != nil { tc.logger.On("Enabled", mock.Anything).Return(true) tc.logger.On("Check", mock.Anything, mock.Anything).Run(tc.retFuncCheck).Return(&zapcore.CheckedEntry{}) } ctx := logging.WithLogger(t.Context(), func(module string) logging.Logger { return zap.New(tc.logger).Sugar() }) retValue := optionalHaveBool(ctx, tc.key, tc.flags) require.Equal(t, retValue, tc.retValue) require.Equal(t, tc.expectMsg, expectMsg) }) } } func TestOptionalHaveIntWithDefault(t *testing.T) { var expectMsg string testCases := []struct { name string key string flags map[string]string defaultValue int64 logger *storagemocks.Core retFuncCheck func(mock.Arguments) expectMsg string retValue int64 }{ { name: "key not exist", key: "fake-key", flags: map[string]string{}, defaultValue: 2000, retValue: 2000, }, { name: "value valid", key: "fake-key", flags: map[string]string{ "fake-key": "1000", }, retValue: 1000, }, { name: "value invalid", key: "fake-key", flags: map[string]string{ "fake-key": "fake-value", }, logger: new(storagemocks.Core), retFuncCheck: func(args mock.Arguments) { ent := args[0].(zapcore.Entry) if ent.Level == zapcore.ErrorLevel { expectMsg = ent.Message } }, expectMsg: "Ignore fake-key, value [fake-value] is invalid, err strconv.ParseInt: parsing \"fake-value\": invalid syntax", defaultValue: 2000, retValue: 2000, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { if tc.logger != nil { tc.logger.On("Enabled", mock.Anything).Return(true) tc.logger.On("Check", mock.Anything, mock.Anything).Run(tc.retFuncCheck).Return(&zapcore.CheckedEntry{}) } ctx := logging.WithLogger(t.Context(), func(module string) logging.Logger { return zap.New(tc.logger).Sugar() }) retValue := optionalHaveIntWithDefault(ctx, tc.key, tc.flags, tc.defaultValue) require.Equal(t, retValue, tc.retValue) require.Equal(t, tc.expectMsg, expectMsg) }) } } ================================================ FILE: pkg/repository/udmrepo/kopialib/lib_repo.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package kopialib import ( "context" "encoding/json" "io" "os" "strings" "sync/atomic" "time" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/compression" "github.com/kopia/kopia/repo/content/index" "github.com/kopia/kopia/repo/maintenance" "github.com/kopia/kopia/repo/manifest" "github.com/kopia/kopia/repo/object" "github.com/kopia/kopia/snapshot/snapshotmaintenance" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/vmware-tanzu/velero/pkg/kopia" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo/kopialib/backend" ) type kopiaRepoService struct { logger logrus.FieldLogger } type kopiaRepository struct { rawRepo repo.Repository rawWriter repo.RepositoryWriter description string uploaded int64 openTime time.Time throttle logThrottle logger logrus.FieldLogger } type kopiaMaintenance struct { mode maintenance.Mode startTime time.Time uploaded int64 throttle logThrottle logger logrus.FieldLogger } type logThrottle struct { lastTime int64 interval time.Duration } type kopiaObjectReader struct { rawReader object.Reader } type kopiaObjectWriter struct { rawWriter object.Writer } type openOptions struct { repoLogger io.Writer } const ( defaultLogInterval = time.Second * 10 defaultMaintainCheckPeriod = time.Hour overwriteFullMaintainInterval = time.Duration(0) overwriteQuickMaintainInterval = time.Duration(0) repoBackend = "kopia" ) var kopiaRepoOpen = repo.Open // NewKopiaRepoService creates an instance of BackupRepoService implemented by Kopia func NewKopiaRepoService(logger logrus.FieldLogger) udmrepo.BackupRepoService { ks := &kopiaRepoService{ logger: logger, } return ks } func (ks *kopiaRepoService) Create(ctx context.Context, repoOption udmrepo.RepoOptions) error { repoCtx := kopia.SetupKopiaLog(ctx, ks.logger) status, err := GetRepositoryStatus(ctx, repoOption, ks.logger) if err != nil { return errors.Wrap(err, "error getting repo status") } if status != RepoStatusSystemNotCreated && status != RepoStatusNotInitialized { return errors.Errorf("unexpected repo status %v", status) } if status == RepoStatusSystemNotCreated { if err := CreateBackupRepo(repoCtx, repoOption, ks.logger); err != nil { return errors.Wrap(err, "error creating backup repo") } } if err := InitializeBackupRepo(ctx, repoOption, ks.logger); err != nil { return errors.Wrap(err, "error initializing backup repo") } return nil } func (ks *kopiaRepoService) Connect(ctx context.Context, repoOption udmrepo.RepoOptions) error { repoCtx := kopia.SetupKopiaLog(ctx, ks.logger) return ConnectBackupRepo(repoCtx, repoOption, ks.logger) } func (ks *kopiaRepoService) IsCreated(ctx context.Context, repoOption udmrepo.RepoOptions) (bool, error) { repoCtx := kopia.SetupKopiaLog(ctx, ks.logger) status, err := GetRepositoryStatus(repoCtx, repoOption, ks.logger) if err != nil { return false, err } if status != RepoStatusCreated { ks.logger.Infof("Repo is not fully created, status %v", status) return false, nil } return true, nil } func (ks *kopiaRepoService) Open(ctx context.Context, repoOption udmrepo.RepoOptions) (udmrepo.BackupRepo, error) { repoConfig := repoOption.ConfigFilePath if repoConfig == "" { return nil, errors.New("invalid config file path") } if _, err := os.Stat(repoConfig); os.IsNotExist(err) { return nil, errors.Wrapf(err, "repo config %s doesn't exist", repoConfig) } repoCtx := kopia.SetupKopiaLog(ctx, ks.logger) r, err := openKopiaRepo(repoCtx, repoConfig, repoOption.RepoPassword, &openOptions{repoLogger: kopia.RepositoryLogger(ks.logger)}) if err != nil { return nil, err } kr := kopiaRepository{ rawRepo: r, openTime: time.Now(), description: repoOption.Description, throttle: logThrottle{ interval: defaultLogInterval, }, logger: ks.logger, } _, kr.rawWriter, err = r.NewWriter(repoCtx, repo.WriteSessionOptions{ Purpose: repoOption.Description, OnUpload: kr.updateProgress, }) if err != nil { if e := r.Close(repoCtx); e != nil { ks.logger.WithError(e).Error("Failed to close raw repository on error") } return nil, errors.Wrap(err, "error to create repo writer") } return &kr, nil } func (ks *kopiaRepoService) Maintain(ctx context.Context, repoOption udmrepo.RepoOptions) error { repoConfig := repoOption.ConfigFilePath if repoConfig == "" { return errors.New("invalid config file path") } if _, err := os.Stat(repoConfig); os.IsNotExist(err) { return errors.Wrapf(err, "repo config %s doesn't exist", repoConfig) } repoCtx := kopia.SetupKopiaLog(ctx, ks.logger) ks.logger.Info("Start to open repo for maintenance, allow index write on load") r, err := openKopiaRepo(repoCtx, repoConfig, repoOption.RepoPassword, &openOptions{repoLogger: kopia.RepositoryLogger(ks.logger)}) if err != nil { return err } ks.logger.Info("Succeeded to open repo for maintenance") defer func() { c := r.Close(repoCtx) if c != nil { ks.logger.WithError(c).Error("Failed to close repo") } }() km := kopiaMaintenance{ mode: maintenance.ModeAuto, startTime: time.Now(), throttle: logThrottle{ interval: defaultLogInterval, }, logger: ks.logger, } if mode, exist := repoOption.GeneralOptions[udmrepo.GenOptionMaintainMode]; exist { if strings.EqualFold(mode, udmrepo.GenOptionMaintainFull) { km.mode = maintenance.ModeFull } else if strings.EqualFold(mode, udmrepo.GenOptionMaintainQuick) { km.mode = maintenance.ModeQuick } } err = repo.DirectWriteSession(repoCtx, r.(repo.DirectRepository), repo.WriteSessionOptions{ Purpose: "UdmRepoMaintenance", OnUpload: km.maintainProgress, }, func(ctx context.Context, dw repo.DirectRepositoryWriter) error { return km.runMaintenance(ctx, dw) }) if err != nil { return errors.Wrap(err, "error to maintain repo") } return nil } func (ks *kopiaRepoService) DefaultMaintenanceFrequency() time.Duration { return defaultMaintainCheckPeriod } func (ks *kopiaRepoService) ClientSideCacheLimit(repoOption map[string]string) int64 { defaultLimit := int64(backend.DefaultCacheLimitMB << 20) if repoOption == nil { return defaultLimit } if v, found := repoOption[repoBackend]; found { var configs map[string]any if err := json.Unmarshal([]byte(v), &configs); err != nil { ks.logger.WithError(err).Warnf("error unmarshalling config data from data %v", v) return defaultLimit } limit := defaultLimit if v, found := configs[udmrepo.StoreOptionCacheLimit]; found { if iv, ok := v.(float64); ok { limit = int64(iv) << 20 } else { ks.logger.Warnf("ignore cache limit from data %v", v) } } return limit } return defaultLimit } func (km *kopiaMaintenance) runMaintenance(ctx context.Context, rep repo.DirectRepositoryWriter) error { err := snapshotmaintenance.Run(kopia.SetupKopiaLog(ctx, km.logger), rep, km.mode, false, maintenance.SafetyFull) if err != nil { return errors.Wrapf(err, "error to run maintenance under mode %s", km.mode) } return nil } // maintainProgress is called when the repository writes a piece of blob data to the storage during the maintenance func (km *kopiaMaintenance) maintainProgress(uploaded int64) { total := atomic.AddInt64(&km.uploaded, uploaded) if km.throttle.shouldLog() { km.logger.WithFields( logrus.Fields{ "Start Time": km.startTime.Format(time.RFC3339Nano), "Current": time.Now().Format(time.RFC3339Nano), }, ).Debugf("Repo maintenance uploaded %d bytes.", total) } } func (kr *kopiaRepository) OpenObject(ctx context.Context, id udmrepo.ID) (udmrepo.ObjectReader, error) { if kr.rawRepo == nil { return nil, errors.New("repo is closed or not open") } objID, err := object.ParseID(string(id)) if err != nil { return nil, errors.Wrapf(err, "error to parse object ID from %v", id) } reader, err := kr.rawRepo.OpenObject(kopia.SetupKopiaLog(ctx, kr.logger), objID) if err != nil { return nil, errors.Wrap(err, "error to open object") } return &kopiaObjectReader{ rawReader: reader, }, nil } func (kr *kopiaRepository) GetManifest(ctx context.Context, id udmrepo.ID, mani *udmrepo.RepoManifest) error { if kr.rawRepo == nil { return errors.New("repo is closed or not open") } metadata, err := kr.rawRepo.GetManifest(kopia.SetupKopiaLog(ctx, kr.logger), manifest.ID(id), mani.Payload) if err != nil { return errors.Wrap(err, "error to get manifest") } mani.Metadata = getManifestEntryFromKopia(metadata) return nil } func (kr *kopiaRepository) FindManifests(ctx context.Context, filter udmrepo.ManifestFilter) ([]*udmrepo.ManifestEntryMetadata, error) { if kr.rawRepo == nil { return nil, errors.New("repo is closed or not open") } metadata, err := kr.rawRepo.FindManifests(kopia.SetupKopiaLog(ctx, kr.logger), filter.Labels) if err != nil { return nil, errors.Wrap(err, "error to find manifests") } return getManifestEntriesFromKopia(metadata), nil } func (kr *kopiaRepository) Time() time.Time { if kr.rawRepo == nil { return time.Time{} } return kr.rawRepo.Time() } func (kr *kopiaRepository) Close(ctx context.Context) error { if kr.rawWriter != nil { err := kr.rawWriter.Close(kopia.SetupKopiaLog(ctx, kr.logger)) if err != nil { return errors.Wrap(err, "error to close repo writer") } kr.rawWriter = nil } if kr.rawRepo != nil { err := kr.rawRepo.Close(kopia.SetupKopiaLog(ctx, kr.logger)) if err != nil { return errors.Wrap(err, "error to close repo") } kr.rawRepo = nil } return nil } func (kr *kopiaRepository) NewObjectWriter(ctx context.Context, opt udmrepo.ObjectWriteOptions) udmrepo.ObjectWriter { if kr.rawWriter == nil { return nil } writer := kr.rawWriter.NewObjectWriter(kopia.SetupKopiaLog(ctx, kr.logger), object.WriterOptions{ Description: opt.Description, Prefix: index.IDPrefix(opt.Prefix), AsyncWrites: opt.AsyncWrites, Compressor: getCompressorForObject(opt), MetadataCompressor: getMetadataCompressor(), }) if writer == nil { return nil } return &kopiaObjectWriter{ rawWriter: writer, } } func (kr *kopiaRepository) PutManifest(ctx context.Context, manifest udmrepo.RepoManifest) (udmrepo.ID, error) { if kr.rawWriter == nil { return "", errors.New("repo writer is closed or not open") } id, err := kr.rawWriter.PutManifest(kopia.SetupKopiaLog(ctx, kr.logger), manifest.Metadata.Labels, manifest.Payload) if err != nil { return "", errors.Wrap(err, "error to put manifest") } return udmrepo.ID(id), nil } func (kr *kopiaRepository) DeleteManifest(ctx context.Context, id udmrepo.ID) error { if kr.rawWriter == nil { return errors.New("repo writer is closed or not open") } err := kr.rawWriter.DeleteManifest(kopia.SetupKopiaLog(ctx, kr.logger), manifest.ID(id)) if err != nil { return errors.Wrap(err, "error to delete manifest") } return nil } func (kr *kopiaRepository) Flush(ctx context.Context) error { if kr.rawWriter == nil { return errors.New("repo writer is closed or not open") } err := kr.rawWriter.Flush(kopia.SetupKopiaLog(ctx, kr.logger)) if err != nil { return errors.Wrap(err, "error to flush repo") } return nil } func (kr *kopiaRepository) GetAdvancedFeatures() udmrepo.AdvancedFeatureInfo { return udmrepo.AdvancedFeatureInfo{ MultiPartBackup: true, } } func (kr *kopiaRepository) ConcatenateObjects(ctx context.Context, objectIDs []udmrepo.ID) (udmrepo.ID, error) { if kr.rawWriter == nil { return "", errors.New("repo writer is closed or not open") } if len(objectIDs) == 0 { return udmrepo.ID(""), errors.New("object list is empty") } rawIDs := []object.ID{} for _, id := range objectIDs { rawID, err := object.ParseID(string(id)) if err != nil { return udmrepo.ID(""), errors.Wrapf(err, "error to parse object ID from %v", id) } rawIDs = append(rawIDs, rawID) } result, err := kr.rawWriter.ConcatenateObjects(ctx, rawIDs, repo.ConcatenateOptions{ Compressor: getMetadataCompressor(), }) if err != nil { return udmrepo.ID(""), errors.Wrap(err, "error to concatenate objects") } return udmrepo.ID(result.String()), nil } // updateProgress is called when the repository writes a piece of blob data to the storage during data write func (kr *kopiaRepository) updateProgress(uploaded int64) { total := atomic.AddInt64(&kr.uploaded, uploaded) if kr.throttle.shouldLog() { kr.logger.WithFields( logrus.Fields{ "Description": kr.description, "Open Time": kr.openTime.Format(time.RFC3339Nano), "Current": time.Now().Format(time.RFC3339Nano), }, ).Debugf("Repo uploaded %d bytes.", total) } } func (kor *kopiaObjectReader) Read(p []byte) (int, error) { if kor.rawReader == nil { return 0, errors.New("object reader is closed or not open") } return kor.rawReader.Read(p) } func (kor *kopiaObjectReader) Seek(offset int64, whence int) (int64, error) { if kor.rawReader == nil { return -1, errors.New("object reader is closed or not open") } return kor.rawReader.Seek(offset, whence) } func (kor *kopiaObjectReader) Close() error { if kor.rawReader == nil { return nil } err := kor.rawReader.Close() if err != nil { return err } kor.rawReader = nil return nil } func (kor *kopiaObjectReader) Length() int64 { if kor.rawReader == nil { return -1 } return kor.rawReader.Length() } func (kow *kopiaObjectWriter) Write(p []byte) (int, error) { if kow.rawWriter == nil { return 0, errors.New("object writer is closed or not open") } return kow.rawWriter.Write(p) } func (kow *kopiaObjectWriter) Seek(offset int64, whence int) (int64, error) { return -1, errors.New("not supported") } func (kow *kopiaObjectWriter) Checkpoint() (udmrepo.ID, error) { if kow.rawWriter == nil { return udmrepo.ID(""), errors.New("object writer is closed or not open") } id, err := kow.rawWriter.Checkpoint() if err != nil { return udmrepo.ID(""), errors.Wrap(err, "error to checkpoint object") } return udmrepo.ID(id.String()), nil } func (kow *kopiaObjectWriter) Result() (udmrepo.ID, error) { if kow.rawWriter == nil { return udmrepo.ID(""), errors.New("object writer is closed or not open") } id, err := kow.rawWriter.Result() if err != nil { return udmrepo.ID(""), errors.Wrap(err, "error to wait object") } return udmrepo.ID(id.String()), nil } func (kow *kopiaObjectWriter) Close() error { if kow.rawWriter == nil { return nil } err := kow.rawWriter.Close() if err != nil { return err } kow.rawWriter = nil return nil } // getCompressorForObject returns the compressor for an object, at present, we don't support compression func getCompressorForObject(_ udmrepo.ObjectWriteOptions) compression.Name { return "" } // getMetadataCompressor returns the compressor for metadata, return kopia's default since we don't support compression func getMetadataCompressor() compression.Name { return "zstd-fastest" } func getManifestEntryFromKopia(mani *manifest.EntryMetadata) *udmrepo.ManifestEntryMetadata { return &udmrepo.ManifestEntryMetadata{ ID: udmrepo.ID(mani.ID), Labels: mani.Labels, Length: int32(mani.Length), ModTime: mani.ModTime, } } func getManifestEntriesFromKopia(mani []*manifest.EntryMetadata) []*udmrepo.ManifestEntryMetadata { var ret []*udmrepo.ManifestEntryMetadata for _, entry := range mani { ret = append(ret, &udmrepo.ManifestEntryMetadata{ ID: udmrepo.ID(entry.ID), Labels: entry.Labels, Length: int32(entry.Length), ModTime: entry.ModTime, }) } return ret } func (lt *logThrottle) shouldLog() bool { nextOutputTime := atomic.LoadInt64(<.lastTime) if nowNano := time.Now().UnixNano(); nowNano > nextOutputTime { if atomic.CompareAndSwapInt64(<.lastTime, nextOutputTime, nowNano+lt.interval.Nanoseconds()) { return true } } return false } func openKopiaRepo(ctx context.Context, configFile string, password string, options *openOptions) (repo.Repository, error) { r, err := kopiaRepoOpen(ctx, configFile, password, &repo.Options{ ContentLogWriter: options.repoLogger, }) if os.IsNotExist(err) { return nil, errors.Wrap(err, "error to open repo, repo doesn't exist") } if err != nil { return nil, errors.Wrap(err, "error to open repo") } return r, nil } ================================================ FILE: pkg/repository/udmrepo/kopialib/lib_repo_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package kopialib import ( "context" "math" "os" "testing" "time" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/manifest" "github.com/kopia/kopia/repo/object" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" repomocks "github.com/vmware-tanzu/velero/pkg/repository/udmrepo/kopialib/backend/mocks" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func TestOpen(t *testing.T) { var directRpo *repomocks.MockRepository testCases := []struct { name string repoOptions udmrepo.RepoOptions returnRepo *repomocks.MockRepository repoOpen func(context.Context, string, string, *repo.Options) (repo.Repository, error) newWriterError error mockClose bool expectedErr string expected *kopiaRepository }{ { name: "invalid config file", expectedErr: "invalid config file path", }, { name: "config file doesn't exist", repoOptions: udmrepo.RepoOptions{ ConfigFilePath: "fake-file", }, expectedErr: "repo config fake-file doesn't exist: stat fake-file: no such file or directory", }, { name: "repo open fail, repo not exist", repoOptions: udmrepo.RepoOptions{ ConfigFilePath: "/tmp", }, repoOpen: func(context.Context, string, string, *repo.Options) (repo.Repository, error) { return nil, os.ErrNotExist }, expectedErr: "error to open repo, repo doesn't exist: file does not exist", }, { name: "repo open fail, other error", repoOptions: udmrepo.RepoOptions{ ConfigFilePath: "/tmp", }, repoOpen: func(context.Context, string, string, *repo.Options) (repo.Repository, error) { return nil, errors.New("fake-repo-open-error") }, expectedErr: "error to open repo: fake-repo-open-error", }, { name: "create repository writer fail", repoOptions: udmrepo.RepoOptions{ ConfigFilePath: "/tmp", }, repoOpen: func(context.Context, string, string, *repo.Options) (repo.Repository, error) { return directRpo, nil }, returnRepo: repomocks.NewMockRepository(t), mockClose: true, newWriterError: errors.New("fake-new-writer-error"), expectedErr: "error to create repo writer: fake-new-writer-error", }, { name: "create repository success", repoOptions: udmrepo.RepoOptions{ ConfigFilePath: "/tmp", Description: "fake-description", }, repoOpen: func(context.Context, string, string, *repo.Options) (repo.Repository, error) { return directRpo, nil }, returnRepo: repomocks.NewMockRepository(t), expected: &kopiaRepository{ description: "fake-description", throttle: logThrottle{ interval: defaultLogInterval, }, }, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { logger := velerotest.NewLogger() service := kopiaRepoService{ logger: logger, } if tc.repoOpen != nil { kopiaRepoOpen = tc.repoOpen } if tc.returnRepo != nil { directRpo = tc.returnRepo } if tc.returnRepo != nil { tc.returnRepo.On("NewWriter", mock.Anything, mock.Anything).Return(nil, nil, tc.newWriterError) if tc.mockClose { tc.returnRepo.On("Close", mock.Anything).Return(nil) } } repo, err := service.Open(t.Context(), tc.repoOptions) if repo != nil { require.Equal(t, tc.expected.description, repo.(*kopiaRepository).description) require.Equal(t, tc.expected.throttle.interval, repo.(*kopiaRepository).throttle.interval) require.Equal(t, repo.(*kopiaRepository).logger, logger) } if tc.expectedErr == "" { assert.NoError(t, err) } else { assert.EqualError(t, err, tc.expectedErr) } }) } } func TestMaintain(t *testing.T) { testCases := []struct { name string repoOptions udmrepo.RepoOptions returnRepo *repomocks.MockRepository returnRepoWriter *repomocks.MockRepositoryWriter repoOpen func(context.Context, string, string, *repo.Options) (repo.Repository, error) newRepoWriterError error findManifestError error expectedErr string }{ { name: "invalid config file", expectedErr: "invalid config file path", }, { name: "config file doesn't exist", repoOptions: udmrepo.RepoOptions{ ConfigFilePath: "fake-file", }, expectedErr: "repo config fake-file doesn't exist: stat fake-file: no such file or directory", }, { name: "repo open fail, repo not exist", repoOptions: udmrepo.RepoOptions{ ConfigFilePath: "/tmp", GeneralOptions: map[string]string{}, }, repoOpen: func(context.Context, string, string, *repo.Options) (repo.Repository, error) { return nil, os.ErrNotExist }, expectedErr: "error to open repo, repo doesn't exist: file does not exist", }, { name: "repo open fail, other error", repoOptions: udmrepo.RepoOptions{ ConfigFilePath: "/tmp", GeneralOptions: map[string]string{}, }, repoOpen: func(context.Context, string, string, *repo.Options) (repo.Repository, error) { return nil, errors.New("fake-repo-open-error") }, expectedErr: "error to open repo: fake-repo-open-error", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { logger := velerotest.NewLogger() ctx := t.Context() service := kopiaRepoService{ logger: logger, } if tc.repoOpen != nil { kopiaRepoOpen = tc.repoOpen } if tc.returnRepo != nil { tc.returnRepo.On("Close", mock.Anything).Return(nil) } err := service.Maintain(ctx, tc.repoOptions) if tc.expectedErr == "" { assert.NoError(t, err) } else { assert.EqualError(t, err, tc.expectedErr) } }) } } func TestShouldLog(t *testing.T) { testCases := []struct { name string lastTime int64 interval time.Duration retValue bool }{ { name: "first time", retValue: true, }, { name: "not run", lastTime: time.Now().Add(time.Hour).UnixNano(), interval: time.Second * 10, }, { name: "not first time, run", lastTime: time.Now().Add(-time.Hour).UnixNano(), interval: time.Second * 10, retValue: true, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { lt := logThrottle{ lastTime: tc.lastTime, interval: tc.interval, } before := lt.lastTime nw := time.Now() s := lt.shouldLog() require.Equal(t, s, tc.retValue) if s { require.GreaterOrEqual(t, lt.lastTime-nw.UnixNano(), lt.interval) } else { require.Equal(t, lt.lastTime, before) } }) } } func TestOpenObject(t *testing.T) { testCases := []struct { name string rawRepo *repomocks.MockRepository objectID string retErr error expectedErr string }{ { name: "raw repo is nil", expectedErr: "repo is closed or not open", }, { name: "objectID is invalid", rawRepo: repomocks.NewMockRepository(t), objectID: "fake-id", expectedErr: "error to parse object ID from fake-id: malformed content ID: \"fake-id\": invalid content prefix", }, { name: "raw open fail", rawRepo: repomocks.NewMockRepository(t), retErr: errors.New("fake-open-error"), expectedErr: "error to open object: fake-open-error", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { kr := &kopiaRepository{} if tc.rawRepo != nil { if tc.retErr != nil { tc.rawRepo.On("OpenObject", mock.Anything, mock.Anything).Return(nil, tc.retErr) } kr.rawRepo = tc.rawRepo } _, err := kr.OpenObject(t.Context(), udmrepo.ID(tc.objectID)) if tc.expectedErr == "" { assert.NoError(t, err) } else { assert.EqualError(t, err, tc.expectedErr) } }) } } func TestGetManifest(t *testing.T) { testCases := []struct { name string rawRepo *repomocks.MockRepository retErr error expectedErr string }{ { name: "raw repo is nil", expectedErr: "repo is closed or not open", }, { name: "raw get fail", rawRepo: repomocks.NewMockRepository(t), retErr: errors.New("fake-get-error"), expectedErr: "error to get manifest: fake-get-error", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { kr := &kopiaRepository{} if tc.rawRepo != nil { if tc.retErr != nil { tc.rawRepo.On("GetManifest", mock.Anything, mock.Anything, mock.Anything).Return(nil, tc.retErr) } kr.rawRepo = tc.rawRepo } err := kr.GetManifest(t.Context(), udmrepo.ID(""), &udmrepo.RepoManifest{}) if tc.expectedErr == "" { assert.NoError(t, err) } else { assert.EqualError(t, err, tc.expectedErr) } }) } } func TestFindManifests(t *testing.T) { testCases := []struct { name string rawRepo *repomocks.MockRepository retErr error expectedErr string }{ { name: "raw repo is nil", expectedErr: "repo is closed or not open", }, { name: "raw find fail", rawRepo: repomocks.NewMockRepository(t), retErr: errors.New("fake-find-error"), expectedErr: "error to find manifests: fake-find-error", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { kr := &kopiaRepository{} if tc.rawRepo != nil { tc.rawRepo.On("FindManifests", mock.Anything, mock.Anything).Return(nil, tc.retErr) kr.rawRepo = tc.rawRepo } _, err := kr.FindManifests(t.Context(), udmrepo.ManifestFilter{}) if tc.expectedErr == "" { assert.NoError(t, err) } else { assert.EqualError(t, err, tc.expectedErr) } }) } } func TestClose(t *testing.T) { testCases := []struct { name string rawRepo *repomocks.MockRepository rawWriter *repomocks.MockRepositoryWriter rawRepoRetErr error rawWriterRetErr error expectedErr string }{ { name: "both nil", }, { name: "writer is not nil", rawWriter: repomocks.NewMockRepositoryWriter(t), }, { name: "repo is not nil", rawRepo: repomocks.NewMockRepository(t), }, { name: "writer close error", rawWriter: repomocks.NewMockRepositoryWriter(t), rawWriterRetErr: errors.New("fake-writer-close-error"), expectedErr: "error to close repo writer: fake-writer-close-error", }, { name: "repo is not nil", rawRepo: repomocks.NewMockRepository(t), rawRepoRetErr: errors.New("fake-repo-close-error"), expectedErr: "error to close repo: fake-repo-close-error", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { kr := &kopiaRepository{} if tc.rawRepo != nil { tc.rawRepo.On("Close", mock.Anything).Return(tc.rawRepoRetErr) kr.rawRepo = tc.rawRepo } if tc.rawWriter != nil { tc.rawWriter.On("Close", mock.Anything).Return(tc.rawWriterRetErr) kr.rawWriter = tc.rawWriter } err := kr.Close(t.Context()) if tc.expectedErr == "" { assert.NoError(t, err) } else { assert.EqualError(t, err, tc.expectedErr) } }) } } func TestPutManifest(t *testing.T) { testCases := []struct { name string rawWriter *repomocks.MockRepositoryWriter rawWriterRetErr error expectedErr string }{ { name: "raw writer is nil", expectedErr: "repo writer is closed or not open", }, { name: "raw put fail", rawWriter: repomocks.NewMockRepositoryWriter(t), rawWriterRetErr: errors.New("fake-writer-put-error"), expectedErr: "error to put manifest: fake-writer-put-error", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { kr := &kopiaRepository{} if tc.rawWriter != nil { tc.rawWriter.On("PutManifest", mock.Anything, mock.Anything, mock.Anything).Return(manifest.ID(""), tc.rawWriterRetErr) kr.rawWriter = tc.rawWriter } _, err := kr.PutManifest(t.Context(), udmrepo.RepoManifest{ Metadata: &udmrepo.ManifestEntryMetadata{}, }) if tc.expectedErr == "" { assert.NoError(t, err) } else { assert.EqualError(t, err, tc.expectedErr) } }) } } func TestDeleteManifest(t *testing.T) { testCases := []struct { name string rawWriter *repomocks.MockRepositoryWriter rawWriterRetErr error expectedErr string }{ { name: "raw writer is nil", expectedErr: "repo writer is closed or not open", }, { name: "raw delete fail", rawWriter: repomocks.NewMockRepositoryWriter(t), rawWriterRetErr: errors.New("fake-writer-delete-error"), expectedErr: "error to delete manifest: fake-writer-delete-error", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { kr := &kopiaRepository{} if tc.rawWriter != nil { tc.rawWriter.On("DeleteManifest", mock.Anything, mock.Anything).Return(tc.rawWriterRetErr) kr.rawWriter = tc.rawWriter } err := kr.DeleteManifest(t.Context(), udmrepo.ID("")) if tc.expectedErr == "" { assert.NoError(t, err) } else { assert.EqualError(t, err, tc.expectedErr) } }) } } func TestFlush(t *testing.T) { testCases := []struct { name string rawWriter *repomocks.MockRepositoryWriter rawWriterRetErr error expectedErr string }{ { name: "raw writer is nil", expectedErr: "repo writer is closed or not open", }, { name: "raw flush fail", rawWriter: repomocks.NewMockRepositoryWriter(t), rawWriterRetErr: errors.New("fake-writer-flush-error"), expectedErr: "error to flush repo: fake-writer-flush-error", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { kr := &kopiaRepository{} if tc.rawWriter != nil { tc.rawWriter.On("Flush", mock.Anything).Return(tc.rawWriterRetErr) kr.rawWriter = tc.rawWriter } err := kr.Flush(t.Context()) if tc.expectedErr == "" { assert.NoError(t, err) } else { assert.EqualError(t, err, tc.expectedErr) } }) } } func TestConcatenateObjects(t *testing.T) { testCases := []struct { name string setWriter bool rawWriter *repomocks.MockRepositoryWriter rawWriterRetErr error objectIDs []udmrepo.ID expectedErr string }{ { name: "writer is nil", expectedErr: "repo writer is closed or not open", }, { name: "empty object list", setWriter: true, expectedErr: "object list is empty", }, { name: "invalid object id", objectIDs: []udmrepo.ID{ "I123456", "fake-id", "I678901", }, setWriter: true, expectedErr: "error to parse object ID from fake-id: malformed content ID: \"fake-id\": invalid content prefix", }, { name: "concatenate error", rawWriter: repomocks.NewMockRepositoryWriter(t), rawWriterRetErr: errors.New("fake-concatenate-error"), objectIDs: []udmrepo.ID{ "I123456", }, setWriter: true, expectedErr: "error to concatenate objects: fake-concatenate-error", }, { name: "succeed", rawWriter: repomocks.NewMockRepositoryWriter(t), objectIDs: []udmrepo.ID{ "I123456", }, setWriter: true, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { kr := &kopiaRepository{} if tc.rawWriter != nil { require.NotNil(t, tc.rawWriter) tc.rawWriter.On("ConcatenateObjects", mock.Anything, mock.Anything, mock.Anything).Return(object.ID{}, tc.rawWriterRetErr) } if tc.setWriter { kr.rawWriter = tc.rawWriter } _, err := kr.ConcatenateObjects(t.Context(), tc.objectIDs) if tc.expectedErr == "" { assert.NoError(t, err) } else { assert.EqualError(t, err, tc.expectedErr) } }) } } func TestNewObjectWriter(t *testing.T) { rawObjWriter := repomocks.NewWriter(t) testCases := []struct { name string rawWriter *repomocks.MockRepositoryWriter rawWriterRet object.Writer expectedRet udmrepo.ObjectWriter }{ { name: "raw writer is nil", }, { name: "new object writer fail", rawWriter: repomocks.NewMockRepositoryWriter(t), }, { name: "succeed", rawWriter: repomocks.NewMockRepositoryWriter(t), rawWriterRet: rawObjWriter, expectedRet: &kopiaObjectWriter{rawWriter: rawObjWriter}, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { kr := &kopiaRepository{} if tc.rawWriter != nil { tc.rawWriter.On("NewObjectWriter", mock.Anything, mock.Anything).Return(tc.rawWriterRet) kr.rawWriter = tc.rawWriter } ret := kr.NewObjectWriter(t.Context(), udmrepo.ObjectWriteOptions{}) assert.Equal(t, tc.expectedRet, ret) }) } } func TestUpdateProgress(t *testing.T) { testCases := []struct { name string progress int64 uploaded int64 throttle logThrottle logMessage string }{ { name: "should not output", throttle: logThrottle{ lastTime: math.MaxInt64, }, }, { name: "should output", progress: 100, uploaded: 200, logMessage: "Repo uploaded 300 bytes.", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { logMessage := "" kr := &kopiaRepository{ logger: velerotest.NewSingleLogger(&logMessage), throttle: tc.throttle, uploaded: tc.uploaded, } kr.updateProgress(tc.progress) if len(tc.logMessage) > 0 { assert.Contains(t, logMessage, tc.logMessage) } else { assert.Empty(t, logMessage) } }) } } func TestReaderRead(t *testing.T) { testCases := []struct { name string rawObjReader *repomocks.Reader rawReaderRetErr error expectedErr string }{ { name: "raw reader is nil", expectedErr: "object reader is closed or not open", }, { name: "raw read fail", rawObjReader: repomocks.NewReader(t), rawReaderRetErr: errors.New("fake-read-error"), expectedErr: "fake-read-error", }, { name: "succeed", rawObjReader: repomocks.NewReader(t), }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { kr := &kopiaObjectReader{} if tc.rawObjReader != nil { tc.rawObjReader.On("Read", mock.Anything).Return(0, tc.rawReaderRetErr) kr.rawReader = tc.rawObjReader } _, err := kr.Read(nil) if tc.expectedErr == "" { assert.NoError(t, err) } else { assert.EqualError(t, err, tc.expectedErr) } }) } } func TestReaderSeek(t *testing.T) { testCases := []struct { name string rawObjReader *repomocks.Reader rawReaderRet int64 rawReaderRetErr error expectedRet int64 expectedErr string }{ { name: "raw reader is nil", expectedErr: "object reader is closed or not open", }, { name: "raw seek fail", rawObjReader: repomocks.NewReader(t), rawReaderRetErr: errors.New("fake-seek-error"), expectedErr: "fake-seek-error", }, { name: "succeed", rawObjReader: repomocks.NewReader(t), rawReaderRet: 100, expectedRet: 100, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { kr := &kopiaObjectReader{} if tc.rawObjReader != nil { tc.rawObjReader.On("Seek", mock.Anything, mock.Anything).Return(tc.rawReaderRet, tc.rawReaderRetErr) kr.rawReader = tc.rawObjReader } ret, err := kr.Seek(0, 0) if tc.expectedErr == "" { require.NoError(t, err) assert.Equal(t, tc.expectedRet, ret) } else { assert.EqualError(t, err, tc.expectedErr) } }) } } func TestReaderClose(t *testing.T) { testCases := []struct { name string rawObjReader *repomocks.Reader rawReaderRetErr error expectedErr string }{ { name: "raw reader is nil", }, { name: "raw close fail", rawObjReader: repomocks.NewReader(t), rawReaderRetErr: errors.New("fake-close-error"), expectedErr: "fake-close-error", }, { name: "succeed", rawObjReader: repomocks.NewReader(t), }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { kr := &kopiaObjectReader{} if tc.rawObjReader != nil { tc.rawObjReader.On("Close").Return(tc.rawReaderRetErr) kr.rawReader = tc.rawObjReader } err := kr.Close() if tc.expectedErr == "" { assert.NoError(t, err) } else { assert.EqualError(t, err, tc.expectedErr) } }) } } func TestReaderLength(t *testing.T) { testCases := []struct { name string rawObjReader *repomocks.Reader rawReaderRet int64 expectedRet int64 }{ { name: "raw reader is nil", expectedRet: -1, }, { name: "raw length fail", rawObjReader: repomocks.NewReader(t), rawReaderRet: 0, expectedRet: 0, }, { name: "succeed", rawObjReader: repomocks.NewReader(t), rawReaderRet: 200, expectedRet: 200, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { kr := &kopiaObjectReader{} if tc.rawObjReader != nil { tc.rawObjReader.On("Length").Return(tc.rawReaderRet) kr.rawReader = tc.rawObjReader } ret := kr.Length() assert.Equal(t, tc.expectedRet, ret) }) } } func TestWriterWrite(t *testing.T) { testCases := []struct { name string rawObjWriter *repomocks.Writer rawWrtierRet int rawWriterRetErr error expectedRet int expectedErr string }{ { name: "raw writer is nil", expectedErr: "object writer is closed or not open", }, { name: "raw read fail", rawObjWriter: repomocks.NewWriter(t), rawWriterRetErr: errors.New("fake-write-error"), expectedErr: "fake-write-error", }, { name: "succeed", rawObjWriter: repomocks.NewWriter(t), rawWrtierRet: 200, expectedRet: 200, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { kr := &kopiaObjectWriter{} if tc.rawObjWriter != nil { tc.rawObjWriter.On("Write", mock.Anything).Return(tc.rawWrtierRet, tc.rawWriterRetErr) kr.rawWriter = tc.rawObjWriter } ret, err := kr.Write(nil) if tc.expectedErr == "" { require.NoError(t, err) assert.Equal(t, tc.expectedRet, ret) } else { assert.EqualError(t, err, tc.expectedErr) } }) } } func TestWriterCheckpoint(t *testing.T) { testCases := []struct { name string rawObjWriter *repomocks.Writer rawWrtierRet object.ID rawWriterRetErr error expectedRet udmrepo.ID expectedErr string }{ { name: "raw writer is nil", expectedErr: "object writer is closed or not open", }, { name: "raw checkpoint fail", rawObjWriter: repomocks.NewWriter(t), rawWriterRetErr: errors.New("fake-checkpoint-error"), expectedErr: "error to checkpoint object: fake-checkpoint-error", }, { name: "succeed", rawObjWriter: repomocks.NewWriter(t), rawWrtierRet: object.ID{}, expectedRet: udmrepo.ID(""), }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { kr := &kopiaObjectWriter{} if tc.rawObjWriter != nil { tc.rawObjWriter.On("Checkpoint").Return(tc.rawWrtierRet, tc.rawWriterRetErr) kr.rawWriter = tc.rawObjWriter } ret, err := kr.Checkpoint() if tc.expectedErr == "" { require.NoError(t, err) assert.Equal(t, tc.expectedRet, ret) } else { assert.EqualError(t, err, tc.expectedErr) } }) } } func TestWriterResult(t *testing.T) { testCases := []struct { name string rawObjWriter *repomocks.Writer rawWrtierRet object.ID rawWriterRetErr error expectedRet udmrepo.ID expectedErr string }{ { name: "raw writer is nil", expectedErr: "object writer is closed or not open", }, { name: "raw result fail", rawObjWriter: repomocks.NewWriter(t), rawWriterRetErr: errors.New("fake-result-error"), expectedErr: "error to wait object: fake-result-error", }, { name: "succeed", rawObjWriter: repomocks.NewWriter(t), rawWrtierRet: object.ID{}, expectedRet: udmrepo.ID(""), }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { kr := &kopiaObjectWriter{} if tc.rawObjWriter != nil { tc.rawObjWriter.On("Result").Return(tc.rawWrtierRet, tc.rawWriterRetErr) kr.rawWriter = tc.rawObjWriter } ret, err := kr.Result() if tc.expectedErr == "" { require.NoError(t, err) assert.Equal(t, tc.expectedRet, ret) } else { assert.EqualError(t, err, tc.expectedErr) } }) } } func TestWriterClose(t *testing.T) { testCases := []struct { name string rawObjWriter *repomocks.Writer rawWriterRetErr error expectedErr string }{ { name: "raw writer is nil", }, { name: "raw close fail", rawObjWriter: repomocks.NewWriter(t), rawWriterRetErr: errors.New("fake-close-error"), expectedErr: "fake-close-error", }, { name: "succeed", rawObjWriter: repomocks.NewWriter(t), }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { kr := &kopiaObjectWriter{} if tc.rawObjWriter != nil { tc.rawObjWriter.On("Close").Return(tc.rawWriterRetErr) kr.rawWriter = tc.rawObjWriter } err := kr.Close() if tc.expectedErr == "" { assert.NoError(t, err) } else { assert.EqualError(t, err, tc.expectedErr) } }) } } func TestMaintainProgress(t *testing.T) { testCases := []struct { name string progress int64 uploaded int64 throttle logThrottle logMessage string }{ { name: "should not output", throttle: logThrottle{ lastTime: math.MaxInt64, }, }, { name: "should output", progress: 100, uploaded: 200, logMessage: "Repo maintenance uploaded 300 bytes.", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { logMessage := "" km := &kopiaMaintenance{ logger: velerotest.NewSingleLogger(&logMessage), throttle: tc.throttle, uploaded: tc.uploaded, } km.maintainProgress(tc.progress) if len(tc.logMessage) > 0 { assert.Contains(t, logMessage, tc.logMessage) } else { assert.Empty(t, logMessage) } }) } } func TestClientSideCacheLimit(t *testing.T) { testCases := []struct { name string repoOption map[string]string expected int64 }{ { name: "nil option", expected: 5000 << 20, }, { name: "no option", repoOption: map[string]string{ "other-repo": "\"enableCompression\": true", }, expected: 5000 << 20, }, { name: "unmarshall fails", repoOption: map[string]string{ "kopia": "wrong-json", }, expected: 5000 << 20, }, { name: "no cache limit", repoOption: map[string]string{ "kopia": "{\"enableCompression\": true}", }, expected: 5000 << 20, }, { name: "wrong cache value type", repoOption: map[string]string{ "kopia": "{\"cacheLimitMB\": \"abcd\",\"enableCompression\": true}", }, expected: 5000 << 20, }, { name: "succeed", repoOption: map[string]string{ "kopia": "{\"cacheLimitMB\": 1,\"enableCompression\": true}", }, expected: 1048576, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { ks := &kopiaRepoService{ logger: velerotest.NewLogger(), } limit := ks.ClientSideCacheLimit(tc.repoOption) assert.Equal(t, tc.expected, limit) }) } } ================================================ FILE: pkg/repository/udmrepo/kopialib/repo_init.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package kopialib import ( "context" "encoding/json" "io" "slices" "strings" "time" "github.com/sirupsen/logrus" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/blob" "github.com/kopia/kopia/repo/format" "github.com/kopia/kopia/repo/maintenance" "github.com/pkg/errors" "github.com/vmware-tanzu/velero/pkg/kopia" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo/kopialib/backend" ) type kopiaBackendStore struct { name string description string store backend.Store } // backendStores lists the supported backend storages at present var backendStores = []kopiaBackendStore{ {udmrepo.StorageTypeAzure, "an Azure blob storage", &backend.AzureBackend{}}, {udmrepo.StorageTypeFs, "a filesystem", &backend.FsBackend{}}, {udmrepo.StorageTypeGcs, "a Google Cloud Storage bucket", &backend.GCSBackend{}}, {udmrepo.StorageTypeS3, "an S3 bucket", &backend.S3Backend{}}, } const udmRepoBlobID = "udmrepo.Repository" type udmRepoMetadata struct { UniqueID []byte `json:"uniqueID"` } type RepoStatus int const ( RepoStatusUnknown = 0 RepoStatusCorrupted = 1 RepoStatusSystemNotCreated = 2 RepoStatusNotInitialized = 3 RepoStatusCreated = 4 ) // CreateBackupRepo creates a Kopia repository and then connect to it. // The storage must be empty, otherwise, it will fail func CreateBackupRepo(ctx context.Context, repoOption udmrepo.RepoOptions, logger logrus.FieldLogger) error { backendStore, err := setupBackendStore(ctx, repoOption.StorageType, repoOption.StorageOptions, logger) if err != nil { return errors.Wrap(err, "error to setup backend storage") } st, err := backendStore.store.Connect(ctx, true, logger) if err != nil { return errors.Wrap(err, "error to connect to storage") } err = createWithStorage(ctx, st, repoOption) if err != nil { return errors.Wrap(err, "error to create repo with storage") } return nil } // ConnectBackupRepo connects to an existing Kopia repository. // If the repository doesn't exist, it will fail func ConnectBackupRepo(ctx context.Context, repoOption udmrepo.RepoOptions, logger logrus.FieldLogger) error { if repoOption.ConfigFilePath == "" { return errors.New("invalid config file path") } st, err := connectStore(ctx, repoOption, logger) if err != nil { return err } err = connectWithStorage(ctx, st, repoOption) if err != nil { return errors.Wrap(err, "error to connect repo with storage") } return nil } func GetRepositoryStatus(ctx context.Context, repoOption udmrepo.RepoOptions, logger logrus.FieldLogger) (RepoStatus, error) { st, err := connectStore(ctx, repoOption, logger) if errors.Is(err, backend.ErrStoreNotExist) { return RepoStatusSystemNotCreated, nil } else if err != nil { return RepoStatusUnknown, err } var formatBytes byteBuffer if err := st.GetBlob(ctx, format.KopiaRepositoryBlobID, 0, -1, &formatBytes); err != nil { if errors.Is(err, blob.ErrBlobNotFound) { logger.Debug("Kopia repository blob is not found") return RepoStatusSystemNotCreated, nil } return RepoStatusUnknown, errors.Wrap(err, "error reading format blob") } repoFmt, err := format.ParseKopiaRepositoryJSON(formatBytes.buffer) if err != nil { return RepoStatusCorrupted, err } var initInfoBytes byteBuffer if err := st.GetBlob(ctx, udmRepoBlobID, 0, -1, &initInfoBytes); err != nil { if errors.Is(err, blob.ErrBlobNotFound) { logger.Debug("Udm repo metadata blob is not found") return RepoStatusNotInitialized, nil } return RepoStatusUnknown, errors.Wrap(err, "error reading udm repo blob") } udmpRepo := &udmRepoMetadata{} if err := json.Unmarshal(initInfoBytes.buffer, udmpRepo); err != nil { return RepoStatusCorrupted, errors.Wrap(err, "invalid udm repo blob") } if !slices.Equal(udmpRepo.UniqueID, repoFmt.UniqueID) { return RepoStatusCorrupted, errors.Errorf("unique ID doesn't match: %v(%v)", udmpRepo.UniqueID, repoFmt.UniqueID) } return RepoStatusCreated, nil } func InitializeBackupRepo(ctx context.Context, repoOption udmrepo.RepoOptions, logger logrus.FieldLogger) error { if repoOption.ConfigFilePath == "" { return errors.New("invalid config file path") } st, err := connectStore(ctx, repoOption, logger) if err != nil { return err } err = connectWithStorage(ctx, st, repoOption) if err != nil { return errors.Wrap(err, "error connecting repo with storage") } err = writeInitParameters(ctx, repoOption, logger) if err != nil { return errors.Wrap(err, "error writing init parameters") } err = writeUdmRepoMetadata(ctx, st) if err != nil { return errors.Wrap(err, "error writing udm repo metadata") } return nil } func writeUdmRepoMetadata(ctx context.Context, st blob.Storage) error { var formatBytes byteBuffer if err := st.GetBlob(ctx, format.KopiaRepositoryBlobID, 0, -1, &formatBytes); err != nil { return errors.Wrap(err, "error reading format blob") } repoFmt, err := format.ParseKopiaRepositoryJSON(formatBytes.buffer) if err != nil { return err } udmpRepo := &udmRepoMetadata{ UniqueID: repoFmt.UniqueID, } bytes, err := json.Marshal(udmpRepo) if err != nil { return errors.Wrap(err, "error marshaling udm repo metadata") } err = st.PutBlob(ctx, udmRepoBlobID, &byteBuffer{bytes}, blob.PutOptions{}) if err != nil { return errors.Wrap(err, "error writing udm repo metadata") } return nil } func connectStore(ctx context.Context, repoOption udmrepo.RepoOptions, logger logrus.FieldLogger) (blob.Storage, error) { backendStore, err := setupBackendStore(ctx, repoOption.StorageType, repoOption.StorageOptions, logger) if err != nil { return nil, errors.Wrap(err, "error to setup backend storage") } st, err := backendStore.store.Connect(ctx, false, logger) if err != nil { return nil, errors.Wrap(err, "error to connect to storage") } return st, nil } func findBackendStore(storage string) *kopiaBackendStore { for _, options := range backendStores { if strings.EqualFold(options.name, storage) { return &options } } return nil } func setupBackendStore(ctx context.Context, storageType string, storageOptions map[string]string, logger logrus.FieldLogger) (*kopiaBackendStore, error) { backendStore := findBackendStore(storageType) if backendStore == nil { return nil, errors.New("error to find storage type") } err := backendStore.store.Setup(ctx, storageOptions, logger) if err != nil { return nil, errors.Wrap(err, "error to setup storage") } return backendStore, nil } func createWithStorage(ctx context.Context, st blob.Storage, repoOption udmrepo.RepoOptions) error { err := ensureEmpty(ctx, st) if err != nil { return errors.Wrap(err, "error to ensure repository storage empty") } options := backend.SetupNewRepositoryOptions(ctx, repoOption.GeneralOptions) if err := repo.Initialize(ctx, st, &options, repoOption.RepoPassword); err != nil { return errors.Wrap(err, "error to initialize repository") } return nil } func connectWithStorage(ctx context.Context, st blob.Storage, repoOption udmrepo.RepoOptions) error { options := backend.SetupConnectOptions(ctx, repoOption) if err := repo.Connect(ctx, repoOption.ConfigFilePath, st, repoOption.RepoPassword, &options); err != nil { return errors.Wrap(err, "error to connect to repository") } return nil } func ensureEmpty(ctx context.Context, s blob.Storage) error { hasDataError := errors.Errorf("has data") err := s.ListBlobs(ctx, "", func(cb blob.Metadata) error { return hasDataError }) if errors.Is(err, hasDataError) { return errors.New("found existing data in storage location") } return errors.Wrap(err, "error to list blobs") } type byteBuffer struct { buffer []byte } type byteBufferReader struct { buffer []byte pos int } func (b *byteBuffer) Write(p []byte) (int, error) { b.buffer = append(b.buffer, p...) return len(p), nil } func (b *byteBuffer) WriteTo(w io.Writer) (int64, error) { n, err := w.Write(b.buffer) return int64(n), err } func (b *byteBuffer) Reset() { b.buffer = nil } func (b *byteBuffer) Length() int { return len(b.buffer) } func (b *byteBuffer) Reader() io.ReadSeekCloser { return &byteBufferReader{buffer: b.buffer} } func (b *byteBufferReader) Close() error { return nil } func (b *byteBufferReader) Read(out []byte) (int, error) { if b.pos == len(b.buffer) { return 0, io.EOF } copied := copy(out, b.buffer[b.pos:]) b.pos += copied return copied, nil } func (b *byteBufferReader) Seek(offset int64, whence int) (int64, error) { newOffset := b.pos switch whence { case io.SeekStart: newOffset = int(offset) case io.SeekCurrent: newOffset += int(offset) case io.SeekEnd: newOffset = len(b.buffer) + int(offset) } if newOffset < 0 || newOffset > len(b.buffer) { return -1, errors.New("invalid seek") } b.pos = newOffset return int64(newOffset), nil } var funcGetParam = maintenance.GetParams func writeInitParameters(ctx context.Context, repoOption udmrepo.RepoOptions, logger logrus.FieldLogger) error { r, err := openKopiaRepo(ctx, repoOption.ConfigFilePath, repoOption.RepoPassword, &openOptions{repoLogger: kopia.RepositoryLogger(logger)}) if err != nil { return err } defer func() { c := r.Close(ctx) if c != nil { logger.WithError(c).Error("Failed to close repo") } }() params, err := funcGetParam(ctx, r) if err != nil { return errors.Wrap(err, "error getting existing maintenance params") } if params.Owner == backend.RepoOwnerFromRepoOptions(repoOption) { logger.Warn("Init parameters already exists, skip") return nil } if params.Owner != "" { logger.Warnf("Overwriting existing init params %v", params) } err = repo.WriteSession(ctx, r, repo.WriteSessionOptions{ Purpose: "set init parameters", }, func(ctx context.Context, w repo.RepositoryWriter) error { p := maintenance.DefaultParams() if overwriteFullMaintainInterval != time.Duration(0) { logger.Infof("Full maintenance interval change from %v to %v", p.FullCycle.Interval, overwriteFullMaintainInterval) p.FullCycle.Interval = overwriteFullMaintainInterval } if overwriteQuickMaintainInterval != time.Duration(0) { logger.Infof("Quick maintenance interval change from %v to %v", p.QuickCycle.Interval, overwriteQuickMaintainInterval) p.QuickCycle.Interval = overwriteQuickMaintainInterval } // the repoOption.StorageOptions are set via // udmrepo.WithStoreOptions -> udmrepo.GetStoreOptions (interface) // -> pkg/repository/provider.GetStoreOptions(param interface{}) -> pkg/repository/provider.getStorageVariables(..., backupRepoConfig) // where backupRepoConfig comes from param.(RepoParam).BackupRepo.Spec.RepositoryConfig map[string]string // where RepositoryConfig comes from pkg/controller/getBackupRepositoryConfig(...) // where it gets a configMap name from pkg/cmd/server/config/Config.BackupRepoConfig // which gets set via velero server flag `backup-repository-configmap` "The name of ConfigMap containing backup repository configurations." // and data stored as json under ConfigMap.Data[repoType] where repoType is BackupRepository.Spec.RepositoryType: either kopia or restic // repoOption.StorageOptions[udmrepo.StoreOptionKeyFullMaintenanceInterval] would for example look like // configMapName.data.kopia: {"fullMaintenanceInterval": "eagerGC"} fullMaintIntervalOption := udmrepo.FullMaintenanceIntervalOptions(repoOption.StorageOptions[udmrepo.StoreOptionKeyFullMaintenanceInterval]) priorMaintInterval := p.FullCycle.Interval switch fullMaintIntervalOption { case udmrepo.FastGC: p.FullCycle.Interval = udmrepo.FastGCInterval case udmrepo.EagerGC: p.FullCycle.Interval = udmrepo.EagerGCInterval case udmrepo.NormalGC: p.FullCycle.Interval = udmrepo.NormalGCInterval case "": // do nothing default: return errors.Errorf("invalid full maintenance interval option %s", fullMaintIntervalOption) } if priorMaintInterval != p.FullCycle.Interval { logger.Infof("Full maintenance interval change from %v to %v", priorMaintInterval, p.FullCycle.Interval) } p.Owner = r.ClientOptions().UsernameAtHost() if err := maintenance.SetParams(ctx, w, &p); err != nil { return errors.Wrap(err, "error to set maintenance params") } return nil }) if err != nil { return errors.Wrap(err, "error to init write repo parameters") } return nil } ================================================ FILE: pkg/repository/udmrepo/kopialib/repo_init_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package kopialib import ( "context" "io" "os" "testing" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/blob" "github.com/kopia/kopia/repo/maintenance" "github.com/kopia/kopia/repo/manifest" velerotest "github.com/vmware-tanzu/velero/pkg/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo/kopialib/backend" repomocks "github.com/vmware-tanzu/velero/pkg/repository/udmrepo/kopialib/backend/mocks" storagemocks "github.com/vmware-tanzu/velero/pkg/repository/udmrepo/kopialib/backend/mocks" "github.com/pkg/errors" ) type comparableError struct { message string } func (ce *comparableError) Error() string { return ce.message } func (ce *comparableError) Is(err error) bool { return err.Error() == ce.message } func TestCreateBackupRepo(t *testing.T) { testCases := []struct { name string backendStore *storagemocks.Store repoOptions udmrepo.RepoOptions connectErr error setupError error returnStore *storagemocks.Storage storeListErr error getBlobErr error listBlobErr error expectedErr string }{ { name: "storage setup fail, invalid type", repoOptions: udmrepo.RepoOptions{ ConfigFilePath: "fake-file", }, expectedErr: "error to setup backend storage: error to find storage type", }, { name: "storage setup fail, backend store steup fail", repoOptions: udmrepo.RepoOptions{ ConfigFilePath: "fake-file", StorageType: udmrepo.StorageTypeAzure, }, backendStore: new(storagemocks.Store), setupError: errors.New("fake-setup-error"), expectedErr: "error to setup backend storage: error to setup storage: fake-setup-error", }, { name: "storage connect fail", repoOptions: udmrepo.RepoOptions{ ConfigFilePath: "fake-file", StorageType: udmrepo.StorageTypeAzure, }, backendStore: new(storagemocks.Store), connectErr: errors.New("fake-connect-error"), expectedErr: "error to connect to storage: fake-connect-error", }, { name: "create repository error, exist blobs", repoOptions: udmrepo.RepoOptions{ ConfigFilePath: "fake-file", StorageType: udmrepo.StorageTypeAzure, }, backendStore: new(storagemocks.Store), returnStore: new(storagemocks.Storage), listBlobErr: &comparableError{ message: "has data", }, expectedErr: "error to create repo with storage: error to ensure repository storage empty: found existing data in storage location", }, { name: "create repository error, error list blobs", repoOptions: udmrepo.RepoOptions{ ConfigFilePath: "fake-file", StorageType: udmrepo.StorageTypeAzure, }, backendStore: new(storagemocks.Store), returnStore: new(storagemocks.Storage), listBlobErr: errors.New("fake-list-blob-error"), expectedErr: "error to create repo with storage: error to ensure repository storage empty: error to list blobs: fake-list-blob-error", }, { name: "create repository error, initialize error", repoOptions: udmrepo.RepoOptions{ ConfigFilePath: "fake-file", StorageType: udmrepo.StorageTypeAzure, }, backendStore: new(storagemocks.Store), returnStore: new(storagemocks.Storage), getBlobErr: errors.New("fake-list-blob-error-01"), expectedErr: "error to create repo with storage: error to initialize repository: unexpected error when checking for format blob: fake-list-blob-error-01", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { logger := velerotest.NewLogger() backendStores = []kopiaBackendStore{ {udmrepo.StorageTypeAzure, "fake store", tc.backendStore}, {udmrepo.StorageTypeFs, "fake store", tc.backendStore}, {udmrepo.StorageTypeGcs, "fake store", tc.backendStore}, {udmrepo.StorageTypeS3, "fake store", tc.backendStore}, } if tc.backendStore != nil { tc.backendStore.On("Connect", mock.Anything, mock.Anything, mock.Anything).Return(tc.returnStore, tc.connectErr) tc.backendStore.On("Setup", mock.Anything, mock.Anything, mock.Anything).Return(tc.setupError) } if tc.returnStore != nil { tc.returnStore.On("ListBlobs", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(tc.listBlobErr) tc.returnStore.On("GetBlob", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(tc.getBlobErr) } err := CreateBackupRepo(t.Context(), tc.repoOptions, logger) if tc.expectedErr == "" { assert.NoError(t, err) } else { assert.EqualError(t, err, tc.expectedErr) } }) } } func TestConnectBackupRepo(t *testing.T) { testCases := []struct { name string backendStore *storagemocks.Store repoOptions udmrepo.RepoOptions connectErr error setupError error returnStore *storagemocks.Storage getBlobErr error expectedErr string }{ { name: "invalid config file", expectedErr: "invalid config file path", }, { name: "storage setup fail, invalid type", repoOptions: udmrepo.RepoOptions{ ConfigFilePath: "fake-file", }, expectedErr: "error to setup backend storage: error to find storage type", }, { name: "storage setup fail, backend store steup fail", repoOptions: udmrepo.RepoOptions{ ConfigFilePath: "fake-file", StorageType: udmrepo.StorageTypeAzure, }, backendStore: new(storagemocks.Store), setupError: errors.New("fake-setup-error"), expectedErr: "error to setup backend storage: error to setup storage: fake-setup-error", }, { name: "storage connect fail", repoOptions: udmrepo.RepoOptions{ ConfigFilePath: "fake-file", StorageType: udmrepo.StorageTypeAzure, }, backendStore: new(storagemocks.Store), connectErr: errors.New("fake-connect-error"), expectedErr: "error to connect to storage: fake-connect-error", }, { name: "connect repository error", repoOptions: udmrepo.RepoOptions{ ConfigFilePath: "fake-file", StorageType: udmrepo.StorageTypeAzure, }, backendStore: new(storagemocks.Store), returnStore: new(storagemocks.Storage), getBlobErr: errors.New("fake-get-blob-error"), expectedErr: "error to connect repo with storage: error to connect to repository: unable to read format blob: fake-get-blob-error", }, } logger := velerotest.NewLogger() for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { backendStores = []kopiaBackendStore{ {udmrepo.StorageTypeAzure, "fake store", tc.backendStore}, {udmrepo.StorageTypeFs, "fake store", tc.backendStore}, {udmrepo.StorageTypeGcs, "fake store", tc.backendStore}, {udmrepo.StorageTypeS3, "fake store", tc.backendStore}, } if tc.backendStore != nil { tc.backendStore.On("Connect", mock.Anything, mock.Anything, mock.Anything).Return(tc.returnStore, tc.connectErr) tc.backendStore.On("Setup", mock.Anything, mock.Anything, mock.Anything).Return(tc.setupError) } if tc.returnStore != nil { tc.returnStore.On("GetBlob", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(tc.getBlobErr) } err := ConnectBackupRepo(t.Context(), tc.repoOptions, logger) if tc.expectedErr == "" { assert.NoError(t, err) } else { assert.EqualError(t, err, tc.expectedErr) } }) } } func TestGetRepositoryStatus(t *testing.T) { testCases := []struct { name string backendStore *storagemocks.Store repoOptions udmrepo.RepoOptions connectErr error setupError error returnStore *storagemocks.Storage retFuncGetBlob func(context.Context, blob.ID, int64, int64, blob.OutputBuffer) error expected RepoStatus expectedErr string }{ { name: "storage setup fail, invalid type", repoOptions: udmrepo.RepoOptions{ ConfigFilePath: "fake-file", }, expected: RepoStatusUnknown, expectedErr: "error to setup backend storage: error to find storage type", }, { name: "storage setup fail, backend store steup fail", repoOptions: udmrepo.RepoOptions{ ConfigFilePath: "fake-file", StorageType: udmrepo.StorageTypeAzure, }, backendStore: new(storagemocks.Store), setupError: errors.New("fake-setup-error"), expected: RepoStatusUnknown, expectedErr: "error to setup backend storage: error to setup storage: fake-setup-error", }, { name: "storage connect fail", repoOptions: udmrepo.RepoOptions{ ConfigFilePath: "fake-file", StorageType: udmrepo.StorageTypeAzure, }, backendStore: new(storagemocks.Store), connectErr: errors.New("fake-connect-error"), expected: RepoStatusUnknown, expectedErr: "error to connect to storage: fake-connect-error", }, { name: "storage not exist", repoOptions: udmrepo.RepoOptions{ ConfigFilePath: "fake-file", StorageType: udmrepo.StorageTypeAzure, }, backendStore: new(storagemocks.Store), connectErr: backend.ErrStoreNotExist, expected: RepoStatusSystemNotCreated, }, { name: "get repo blob error", repoOptions: udmrepo.RepoOptions{ ConfigFilePath: "fake-file", StorageType: udmrepo.StorageTypeAzure, }, backendStore: new(storagemocks.Store), returnStore: new(storagemocks.Storage), retFuncGetBlob: func(context.Context, blob.ID, int64, int64, blob.OutputBuffer) error { return errors.New("fake-get-blob-error") }, expected: RepoStatusUnknown, expectedErr: "error reading format blob: fake-get-blob-error", }, { name: "no repo blob", repoOptions: udmrepo.RepoOptions{ ConfigFilePath: "fake-file", StorageType: udmrepo.StorageTypeAzure, }, backendStore: new(storagemocks.Store), returnStore: new(storagemocks.Storage), retFuncGetBlob: func(context.Context, blob.ID, int64, int64, blob.OutputBuffer) error { return blob.ErrBlobNotFound }, expected: RepoStatusSystemNotCreated, }, { name: "wrong repo format", repoOptions: udmrepo.RepoOptions{ ConfigFilePath: "fake-file", StorageType: udmrepo.StorageTypeAzure, }, backendStore: new(storagemocks.Store), returnStore: new(storagemocks.Storage), retFuncGetBlob: func(ctx context.Context, id blob.ID, offset int64, length int64, output blob.OutputBuffer) error { output.Write([]byte("fake-buffer")) return nil }, expected: RepoStatusCorrupted, expectedErr: "invalid format blob: invalid character 'k' in literal false (expecting 'l')", }, { name: "get udm repo blob error", repoOptions: udmrepo.RepoOptions{ ConfigFilePath: "fake-file", StorageType: udmrepo.StorageTypeAzure, }, backendStore: new(storagemocks.Store), returnStore: new(storagemocks.Storage), retFuncGetBlob: func(ctx context.Context, blobID blob.ID, offset int64, length int64, output blob.OutputBuffer) error { if blobID == udmRepoBlobID { return errors.New("fake-get-blob-error") } else { output.Write([]byte(`{"tool":"","buildVersion":"","buildInfo":"","uniqueID":[],"keyAlgo":"","encryption":""}`)) return nil } }, expected: RepoStatusUnknown, expectedErr: "error reading udm repo blob: fake-get-blob-error", }, { name: "no udm repo blob", repoOptions: udmrepo.RepoOptions{ ConfigFilePath: "fake-file", StorageType: udmrepo.StorageTypeAzure, }, backendStore: new(storagemocks.Store), returnStore: new(storagemocks.Storage), retFuncGetBlob: func(ctx context.Context, blobID blob.ID, offset int64, length int64, output blob.OutputBuffer) error { if blobID == udmRepoBlobID { return blob.ErrBlobNotFound } else { output.Write([]byte(`{"tool":"","buildVersion":"","buildInfo":"","uniqueID":[],"keyAlgo":"","encryption":""}`)) return nil } }, expected: RepoStatusNotInitialized, }, { name: "wrong udm repo metadata", repoOptions: udmrepo.RepoOptions{ ConfigFilePath: "fake-file", StorageType: udmrepo.StorageTypeAzure, }, backendStore: new(storagemocks.Store), returnStore: new(storagemocks.Storage), retFuncGetBlob: func(ctx context.Context, blobID blob.ID, offset int64, length int64, output blob.OutputBuffer) error { if blobID == udmRepoBlobID { output.Write([]byte("fake-buffer")) } else { output.Write([]byte(`{"tool":"","buildVersion":"","buildInfo":"","uniqueID":[],"keyAlgo":"","encryption":""}`)) } return nil }, expected: RepoStatusCorrupted, expectedErr: "invalid udm repo blob: invalid character 'k' in literal false (expecting 'l')", }, { name: "wrong unique id", repoOptions: udmrepo.RepoOptions{ ConfigFilePath: "fake-file", StorageType: udmrepo.StorageTypeAzure, }, backendStore: new(storagemocks.Store), returnStore: new(storagemocks.Storage), retFuncGetBlob: func(ctx context.Context, blobID blob.ID, offset int64, length int64, output blob.OutputBuffer) error { if blobID == udmRepoBlobID { output.Write([]byte(`{"uniqueID":[4,5,6]}`)) } else { output.Write([]byte(`{"tool":"","buildVersion":"","buildInfo":"","uniqueID":[1,2,3],"keyAlgo":"","encryption":""}`)) } return nil }, expected: RepoStatusCorrupted, expectedErr: "unique ID doesn't match: [4 5 6]([1 2 3])", }, { name: "succeed", repoOptions: udmrepo.RepoOptions{ ConfigFilePath: "fake-file", StorageType: udmrepo.StorageTypeAzure, }, backendStore: new(storagemocks.Store), returnStore: new(storagemocks.Storage), retFuncGetBlob: func(ctx context.Context, blobID blob.ID, offset int64, length int64, output blob.OutputBuffer) error { if blobID == udmRepoBlobID { output.Write([]byte(`{"uniqueID":[1,2,3]}`)) } else { output.Write([]byte(`{"tool":"","buildVersion":"","buildInfo":"","uniqueID":[1,2,3],"keyAlgo":"","encryption":""}`)) } return nil }, expected: RepoStatusCreated, }, } logger := velerotest.NewLogger() for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { backendStores = []kopiaBackendStore{ {udmrepo.StorageTypeAzure, "fake store", tc.backendStore}, {udmrepo.StorageTypeFs, "fake store", tc.backendStore}, {udmrepo.StorageTypeGcs, "fake store", tc.backendStore}, {udmrepo.StorageTypeS3, "fake store", tc.backendStore}, } if tc.backendStore != nil { tc.backendStore.On("Connect", mock.Anything, mock.Anything, mock.Anything).Return(tc.returnStore, tc.connectErr) tc.backendStore.On("Setup", mock.Anything, mock.Anything, mock.Anything).Return(tc.setupError) } if tc.returnStore != nil { tc.returnStore.On("GetBlob", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(tc.retFuncGetBlob) } status, err := GetRepositoryStatus(t.Context(), tc.repoOptions, logger) if tc.expectedErr == "" { require.NoError(t, err) } else { require.EqualError(t, err, tc.expectedErr) } assert.Equal(t, tc.expected, status) }) } } func TestWriteInitParameters(t *testing.T) { var directRpo *repomocks.MockRepository assertFullMaintIntervalEqual := func(expected, actual *maintenance.Params) bool { return assert.Equal(t, expected.FullCycle.Interval, actual.FullCycle.Interval) } testCases := []struct { name string repoOptions udmrepo.RepoOptions returnRepo *repomocks.MockRepository returnRepoWriter *repomocks.MockRepositoryWriter repoOpen func(context.Context, string, string, *repo.Options) (repo.Repository, error) newRepoWriterError error replaceManifestError error getParam func(context.Context, repo.Repository) (*maintenance.Params, error) // expected replacemanifest params to be received by maintenance.SetParams, and therefore writeInitParameters expectedReplaceManifestsParams *maintenance.Params // allows for asserting only certain fields are set as expected assertReplaceManifestsParams func(*maintenance.Params, *maintenance.Params) bool mockNewWriter bool mockClientOptions bool expectedErr string }{ { name: "repo open fail, repo not exist", repoOptions: udmrepo.RepoOptions{ ConfigFilePath: "/tmp", GeneralOptions: map[string]string{}, }, repoOpen: func(context.Context, string, string, *repo.Options) (repo.Repository, error) { return nil, os.ErrNotExist }, expectedErr: "error to open repo, repo doesn't exist: file does not exist", }, { name: "repo open fail, other error", repoOptions: udmrepo.RepoOptions{ ConfigFilePath: "/tmp", GeneralOptions: map[string]string{}, }, repoOpen: func(context.Context, string, string, *repo.Options) (repo.Repository, error) { return nil, errors.New("fake-repo-open-error") }, expectedErr: "error to open repo: fake-repo-open-error", }, { name: "get params error", repoOptions: udmrepo.RepoOptions{ ConfigFilePath: "/tmp", GeneralOptions: map[string]string{}, }, repoOpen: func(context.Context, string, string, *repo.Options) (repo.Repository, error) { return directRpo, nil }, getParam: func(context.Context, repo.Repository) (*maintenance.Params, error) { return nil, errors.New("fake-get-param-error") }, returnRepo: repomocks.NewMockRepository(t), expectedErr: "error getting existing maintenance params: fake-get-param-error", }, { name: "existing param with identical owner", repoOptions: udmrepo.RepoOptions{ ConfigFilePath: "/tmp", GeneralOptions: map[string]string{}, }, repoOpen: func(context.Context, string, string, *repo.Options) (repo.Repository, error) { return directRpo, nil }, getParam: func(context.Context, repo.Repository) (*maintenance.Params, error) { return &maintenance.Params{ Owner: "default@default", }, nil }, returnRepo: repomocks.NewMockRepository(t), }, { name: "existing param with different owner", repoOptions: udmrepo.RepoOptions{ ConfigFilePath: "/tmp", GeneralOptions: map[string]string{}, }, repoOpen: func(context.Context, string, string, *repo.Options) (repo.Repository, error) { return directRpo, nil }, getParam: func(context.Context, repo.Repository) (*maintenance.Params, error) { return &maintenance.Params{ Owner: "fake-owner", }, nil }, returnRepo: repomocks.NewMockRepository(t), returnRepoWriter: repomocks.NewMockRepositoryWriter(t), expectedReplaceManifestsParams: &maintenance.Params{ FullCycle: maintenance.CycleParams{ Interval: udmrepo.NormalGCInterval, }, }, mockNewWriter: true, mockClientOptions: true, assertReplaceManifestsParams: assertFullMaintIntervalEqual, }, { name: "write session fail", repoOptions: udmrepo.RepoOptions{ ConfigFilePath: "/tmp", GeneralOptions: map[string]string{}, }, repoOpen: func(context.Context, string, string, *repo.Options) (repo.Repository, error) { return directRpo, nil }, returnRepo: repomocks.NewMockRepository(t), mockNewWriter: true, newRepoWriterError: errors.New("fake-new-writer-error"), expectedErr: "error to init write repo parameters: unable to create writer: fake-new-writer-error", }, { name: "set repo param fail", repoOptions: udmrepo.RepoOptions{ ConfigFilePath: "/tmp", GeneralOptions: map[string]string{}, }, repoOpen: func(context.Context, string, string, *repo.Options) (repo.Repository, error) { return directRpo, nil }, returnRepo: repomocks.NewMockRepository(t), returnRepoWriter: repomocks.NewMockRepositoryWriter(t), mockNewWriter: true, mockClientOptions: true, replaceManifestError: errors.New("fake-replace-manifest-error"), expectedErr: "error to init write repo parameters: error to set maintenance params: put manifest: fake-replace-manifest-error", }, { name: "repo with maintenance interval has expected params", repoOptions: udmrepo.RepoOptions{ ConfigFilePath: "/tmp", StorageOptions: map[string]string{ udmrepo.StoreOptionKeyFullMaintenanceInterval: string(udmrepo.FastGC), }, }, repoOpen: func(context.Context, string, string, *repo.Options) (repo.Repository, error) { return directRpo, nil }, returnRepo: repomocks.NewMockRepository(t), returnRepoWriter: repomocks.NewMockRepositoryWriter(t), mockNewWriter: true, mockClientOptions: true, expectedReplaceManifestsParams: &maintenance.Params{ FullCycle: maintenance.CycleParams{ Interval: udmrepo.FastGCInterval, }, }, assertReplaceManifestsParams: assertFullMaintIntervalEqual, }, { name: "repo with empty maintenance interval has expected params", repoOptions: udmrepo.RepoOptions{ ConfigFilePath: "/tmp", StorageOptions: map[string]string{ udmrepo.StoreOptionKeyFullMaintenanceInterval: string(""), }, }, repoOpen: func(context.Context, string, string, *repo.Options) (repo.Repository, error) { return directRpo, nil }, returnRepo: repomocks.NewMockRepository(t), returnRepoWriter: repomocks.NewMockRepositoryWriter(t), mockNewWriter: true, mockClientOptions: true, expectedReplaceManifestsParams: &maintenance.Params{ FullCycle: maintenance.CycleParams{ Interval: udmrepo.NormalGCInterval, }, }, assertReplaceManifestsParams: assertFullMaintIntervalEqual, }, { name: "repo with invalid maintenance interval has expected errors", repoOptions: udmrepo.RepoOptions{ ConfigFilePath: "/tmp", StorageOptions: map[string]string{ udmrepo.StoreOptionKeyFullMaintenanceInterval: string("foo"), }, }, repoOpen: func(context.Context, string, string, *repo.Options) (repo.Repository, error) { return directRpo, nil }, returnRepo: repomocks.NewMockRepository(t), returnRepoWriter: repomocks.NewMockRepositoryWriter(t), mockNewWriter: true, expectedErr: "error to init write repo parameters: invalid full maintenance interval option foo", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { logger := velerotest.NewLogger() ctx := t.Context() if tc.repoOpen != nil { kopiaRepoOpen = tc.repoOpen } if tc.returnRepo != nil { directRpo = tc.returnRepo } if tc.returnRepo != nil { if tc.mockNewWriter { tc.returnRepo.On("NewWriter", mock.Anything, mock.Anything).Return(ctx, tc.returnRepoWriter, tc.newRepoWriterError) } if tc.mockClientOptions { tc.returnRepo.On("ClientOptions").Return(repo.ClientOptions{}) } tc.returnRepo.On("Close", mock.Anything).Return(nil) } if tc.returnRepoWriter != nil { tc.returnRepoWriter.On("Close", mock.Anything).Return(nil) if tc.replaceManifestError != nil { tc.returnRepoWriter.On("ReplaceManifests", mock.Anything, mock.Anything, mock.Anything).Return(manifest.ID(""), tc.replaceManifestError) } if tc.expectedReplaceManifestsParams != nil { tc.returnRepoWriter.On("ReplaceManifests", mock.Anything, mock.AnythingOfType("map[string]string"), mock.AnythingOfType("*maintenance.Params")).Return(manifest.ID(""), nil) tc.returnRepoWriter.On("Flush", mock.Anything).Return(nil) } } if tc.getParam != nil { funcGetParam = tc.getParam } else { funcGetParam = func(ctx context.Context, rep repo.Repository) (*maintenance.Params, error) { return &maintenance.Params{}, nil } } err := writeInitParameters(ctx, tc.repoOptions, logger) if tc.expectedErr == "" { require.NoError(t, err) } else { require.EqualError(t, err, tc.expectedErr) } if tc.expectedReplaceManifestsParams != nil { actualReplaceManifestsParams, converted := tc.returnRepoWriter.Calls[0].Arguments.Get(2).(*maintenance.Params) assert.True(t, converted) tc.assertReplaceManifestsParams(tc.expectedReplaceManifestsParams, actualReplaceManifestsParams) } }) } } func TestWriteUdmRepoMetadata(t *testing.T) { testCases := []struct { name string retFuncGetBlob func(context.Context, blob.ID, int64, int64, blob.OutputBuffer) error retFuncPutBlob func(context.Context, blob.ID, blob.Bytes, blob.PutOptions) error replaceMetadata *udmRepoMetadata expectedErr string }{ { name: "get repo blob error", retFuncGetBlob: func(context.Context, blob.ID, int64, int64, blob.OutputBuffer) error { return errors.New("fake-get-blob-error") }, expectedErr: "error reading format blob: fake-get-blob-error", }, { name: "wrong repo format", retFuncGetBlob: func(ctx context.Context, id blob.ID, offset int64, length int64, output blob.OutputBuffer) error { output.Write([]byte("fake-buffer")) return nil }, expectedErr: "invalid format blob: invalid character 'k' in literal false (expecting 'l')", }, { name: "put udm repo metadata blob error", retFuncGetBlob: func(ctx context.Context, blobID blob.ID, offset int64, length int64, output blob.OutputBuffer) error { output.Write([]byte(`{"tool":"","buildVersion":"","buildInfo":"","uniqueID":[],"keyAlgo":"","encryption":""}`)) return nil }, retFuncPutBlob: func(context.Context, blob.ID, blob.Bytes, blob.PutOptions) error { return errors.New("fake-put-blob-error") }, expectedErr: "error writing udm repo metadata: fake-put-blob-error", }, { name: "succeed", retFuncGetBlob: func(ctx context.Context, blobID blob.ID, offset int64, length int64, output blob.OutputBuffer) error { output.Write([]byte(`{"tool":"","buildVersion":"","buildInfo":"","uniqueID":[],"keyAlgo":"","encryption":""}`)) return nil }, retFuncPutBlob: func(context.Context, blob.ID, blob.Bytes, blob.PutOptions) error { return nil }, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { storage := new(storagemocks.Storage) if tc.retFuncGetBlob != nil { storage.On("GetBlob", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(tc.retFuncGetBlob) } if tc.retFuncPutBlob != nil { storage.On("PutBlob", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(tc.retFuncPutBlob) } err := writeUdmRepoMetadata(t.Context(), storage) if tc.expectedErr == "" { require.NoError(t, err) } else { require.EqualError(t, err, tc.expectedErr) } }) } } type testRecv struct { buffer []byte } func (r *testRecv) Write(p []byte) (n int, err error) { r.buffer = append(r.buffer, p...) return len(p), nil } func TestByteBuffer(t *testing.T) { buffer := &byteBuffer{} written, err := buffer.Write([]byte("12345")) require.NoError(t, err) require.Equal(t, 5, written) written, err = buffer.Write([]byte("67890")) require.NoError(t, err) require.Equal(t, 5, written) require.Equal(t, 10, buffer.Length()) recv := &testRecv{} copied, err := buffer.WriteTo(recv) require.NoError(t, err) require.Equal(t, int64(10), copied) require.Equal(t, []byte("1234567890"), recv.buffer) buffer.Reset() require.Zero(t, buffer.Length()) } func TestByteBufferReader(t *testing.T) { buffer := &byteBufferReader{buffer: []byte("123456789012345678901234567890")} off, err := buffer.Seek(100, io.SeekStart) require.Equal(t, int64(-1), off) require.EqualError(t, err, "invalid seek") require.Zero(t, buffer.pos) off, err = buffer.Seek(-100, io.SeekEnd) require.Equal(t, int64(-1), off) require.EqualError(t, err, "invalid seek") require.Zero(t, buffer.pos) off, err = buffer.Seek(3, io.SeekCurrent) require.Equal(t, int64(3), off) require.NoError(t, err) require.Equal(t, 3, buffer.pos) output := make([]byte, 6) read, err := buffer.Read(output) require.NoError(t, err) require.Equal(t, 6, read) require.Equal(t, 9, buffer.pos) require.Equal(t, []byte("456789"), output) off, err = buffer.Seek(21, io.SeekStart) require.Equal(t, int64(21), off) require.NoError(t, err) require.Equal(t, 21, buffer.pos) output = make([]byte, 6) read, err = buffer.Read(output) require.NoError(t, err) require.Equal(t, 6, read) require.Equal(t, 27, buffer.pos) require.Equal(t, []byte("234567"), output) output = make([]byte, 6) read, err = buffer.Read(output) require.NoError(t, err) require.Equal(t, 3, read) require.Equal(t, 30, buffer.pos) require.Equal(t, []byte{'8', '9', '0', 0, 0, 0}, output) output = make([]byte, 6) read, err = buffer.Read(output) require.Zero(t, read) require.Equal(t, io.EOF, err) err = buffer.Close() require.NoError(t, err) } ================================================ FILE: pkg/repository/udmrepo/mocks/BackupRepo.go ================================================ // Code generated by mockery v2.39.1. DO NOT EDIT. package mocks import ( context "context" time "time" mock "github.com/stretchr/testify/mock" udmrepo "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" ) // BackupRepo is an autogenerated mock type for the BackupRepo type type BackupRepo struct { mock.Mock } // Close provides a mock function with given fields: ctx func (_m *BackupRepo) Close(ctx context.Context) error { ret := _m.Called(ctx) if len(ret) == 0 { panic("no return value specified for Close") } var r0 error if rf, ok := ret.Get(0).(func(context.Context) error); ok { r0 = rf(ctx) } else { r0 = ret.Error(0) } return r0 } // ConcatenateObjects provides a mock function with given fields: ctx, objectIDs func (_m *BackupRepo) ConcatenateObjects(ctx context.Context, objectIDs []udmrepo.ID) (udmrepo.ID, error) { ret := _m.Called(ctx, objectIDs) if len(ret) == 0 { panic("no return value specified for ConcatenateObjects") } var r0 udmrepo.ID var r1 error if rf, ok := ret.Get(0).(func(context.Context, []udmrepo.ID) (udmrepo.ID, error)); ok { return rf(ctx, objectIDs) } if rf, ok := ret.Get(0).(func(context.Context, []udmrepo.ID) udmrepo.ID); ok { r0 = rf(ctx, objectIDs) } else { r0 = ret.Get(0).(udmrepo.ID) } if rf, ok := ret.Get(1).(func(context.Context, []udmrepo.ID) error); ok { r1 = rf(ctx, objectIDs) } else { r1 = ret.Error(1) } return r0, r1 } // DeleteManifest provides a mock function with given fields: ctx, id func (_m *BackupRepo) DeleteManifest(ctx context.Context, id udmrepo.ID) error { ret := _m.Called(ctx, id) if len(ret) == 0 { panic("no return value specified for DeleteManifest") } var r0 error if rf, ok := ret.Get(0).(func(context.Context, udmrepo.ID) error); ok { r0 = rf(ctx, id) } else { r0 = ret.Error(0) } return r0 } // FindManifests provides a mock function with given fields: ctx, filter func (_m *BackupRepo) FindManifests(ctx context.Context, filter udmrepo.ManifestFilter) ([]*udmrepo.ManifestEntryMetadata, error) { ret := _m.Called(ctx, filter) if len(ret) == 0 { panic("no return value specified for FindManifests") } var r0 []*udmrepo.ManifestEntryMetadata var r1 error if rf, ok := ret.Get(0).(func(context.Context, udmrepo.ManifestFilter) ([]*udmrepo.ManifestEntryMetadata, error)); ok { return rf(ctx, filter) } if rf, ok := ret.Get(0).(func(context.Context, udmrepo.ManifestFilter) []*udmrepo.ManifestEntryMetadata); ok { r0 = rf(ctx, filter) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]*udmrepo.ManifestEntryMetadata) } } if rf, ok := ret.Get(1).(func(context.Context, udmrepo.ManifestFilter) error); ok { r1 = rf(ctx, filter) } else { r1 = ret.Error(1) } return r0, r1 } // Flush provides a mock function with given fields: ctx func (_m *BackupRepo) Flush(ctx context.Context) error { ret := _m.Called(ctx) if len(ret) == 0 { panic("no return value specified for Flush") } var r0 error if rf, ok := ret.Get(0).(func(context.Context) error); ok { r0 = rf(ctx) } else { r0 = ret.Error(0) } return r0 } // GetAdvancedFeatures provides a mock function with given fields: func (_m *BackupRepo) GetAdvancedFeatures() udmrepo.AdvancedFeatureInfo { ret := _m.Called() if len(ret) == 0 { panic("no return value specified for GetAdvancedFeatures") } var r0 udmrepo.AdvancedFeatureInfo if rf, ok := ret.Get(0).(func() udmrepo.AdvancedFeatureInfo); ok { r0 = rf() } else { r0 = ret.Get(0).(udmrepo.AdvancedFeatureInfo) } return r0 } // GetManifest provides a mock function with given fields: ctx, id, mani func (_m *BackupRepo) GetManifest(ctx context.Context, id udmrepo.ID, mani *udmrepo.RepoManifest) error { ret := _m.Called(ctx, id, mani) if len(ret) == 0 { panic("no return value specified for GetManifest") } var r0 error if rf, ok := ret.Get(0).(func(context.Context, udmrepo.ID, *udmrepo.RepoManifest) error); ok { r0 = rf(ctx, id, mani) } else { r0 = ret.Error(0) } return r0 } // NewObjectWriter provides a mock function with given fields: ctx, opt func (_m *BackupRepo) NewObjectWriter(ctx context.Context, opt udmrepo.ObjectWriteOptions) udmrepo.ObjectWriter { ret := _m.Called(ctx, opt) if len(ret) == 0 { panic("no return value specified for NewObjectWriter") } var r0 udmrepo.ObjectWriter if rf, ok := ret.Get(0).(func(context.Context, udmrepo.ObjectWriteOptions) udmrepo.ObjectWriter); ok { r0 = rf(ctx, opt) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(udmrepo.ObjectWriter) } } return r0 } // OpenObject provides a mock function with given fields: ctx, id func (_m *BackupRepo) OpenObject(ctx context.Context, id udmrepo.ID) (udmrepo.ObjectReader, error) { ret := _m.Called(ctx, id) if len(ret) == 0 { panic("no return value specified for OpenObject") } var r0 udmrepo.ObjectReader var r1 error if rf, ok := ret.Get(0).(func(context.Context, udmrepo.ID) (udmrepo.ObjectReader, error)); ok { return rf(ctx, id) } if rf, ok := ret.Get(0).(func(context.Context, udmrepo.ID) udmrepo.ObjectReader); ok { r0 = rf(ctx, id) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(udmrepo.ObjectReader) } } if rf, ok := ret.Get(1).(func(context.Context, udmrepo.ID) error); ok { r1 = rf(ctx, id) } else { r1 = ret.Error(1) } return r0, r1 } // PutManifest provides a mock function with given fields: ctx, mani func (_m *BackupRepo) PutManifest(ctx context.Context, mani udmrepo.RepoManifest) (udmrepo.ID, error) { ret := _m.Called(ctx, mani) if len(ret) == 0 { panic("no return value specified for PutManifest") } var r0 udmrepo.ID var r1 error if rf, ok := ret.Get(0).(func(context.Context, udmrepo.RepoManifest) (udmrepo.ID, error)); ok { return rf(ctx, mani) } if rf, ok := ret.Get(0).(func(context.Context, udmrepo.RepoManifest) udmrepo.ID); ok { r0 = rf(ctx, mani) } else { r0 = ret.Get(0).(udmrepo.ID) } if rf, ok := ret.Get(1).(func(context.Context, udmrepo.RepoManifest) error); ok { r1 = rf(ctx, mani) } else { r1 = ret.Error(1) } return r0, r1 } // Time provides a mock function with given fields: func (_m *BackupRepo) Time() time.Time { ret := _m.Called() if len(ret) == 0 { panic("no return value specified for Time") } var r0 time.Time if rf, ok := ret.Get(0).(func() time.Time); ok { r0 = rf() } else { r0 = ret.Get(0).(time.Time) } return r0 } // NewBackupRepo creates a new instance of BackupRepo. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. func NewBackupRepo(t interface { mock.TestingT Cleanup(func()) }) *BackupRepo { mock := &BackupRepo{} mock.Mock.Test(t) t.Cleanup(func() { mock.AssertExpectations(t) }) return mock } ================================================ FILE: pkg/repository/udmrepo/mocks/BackupRepoService.go ================================================ // Code generated by mockery v2.53.2. DO NOT EDIT. package mocks import ( context "context" time "time" mock "github.com/stretchr/testify/mock" udmrepo "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" ) // BackupRepoService is an autogenerated mock type for the BackupRepoService type type BackupRepoService struct { mock.Mock } // ClientSideCacheLimit provides a mock function with given fields: repoOption func (_m *BackupRepoService) ClientSideCacheLimit(repoOption map[string]string) int64 { ret := _m.Called(repoOption) if len(ret) == 0 { panic("no return value specified for ClientSideCacheLimit") } var r0 int64 if rf, ok := ret.Get(0).(func(map[string]string) int64); ok { r0 = rf(repoOption) } else { r0 = ret.Get(0).(int64) } return r0 } // Connect provides a mock function with given fields: ctx, repoOption func (_m *BackupRepoService) Connect(ctx context.Context, repoOption udmrepo.RepoOptions) error { ret := _m.Called(ctx, repoOption) if len(ret) == 0 { panic("no return value specified for Connect") } var r0 error if rf, ok := ret.Get(0).(func(context.Context, udmrepo.RepoOptions) error); ok { r0 = rf(ctx, repoOption) } else { r0 = ret.Error(0) } return r0 } // Create provides a mock function with given fields: ctx, repoOption func (_m *BackupRepoService) Create(ctx context.Context, repoOption udmrepo.RepoOptions) error { ret := _m.Called(ctx, repoOption) if len(ret) == 0 { panic("no return value specified for Create") } var r0 error if rf, ok := ret.Get(0).(func(context.Context, udmrepo.RepoOptions) error); ok { r0 = rf(ctx, repoOption) } else { r0 = ret.Error(0) } return r0 } // DefaultMaintenanceFrequency provides a mock function with no fields func (_m *BackupRepoService) DefaultMaintenanceFrequency() time.Duration { ret := _m.Called() if len(ret) == 0 { panic("no return value specified for DefaultMaintenanceFrequency") } var r0 time.Duration if rf, ok := ret.Get(0).(func() time.Duration); ok { r0 = rf() } else { r0 = ret.Get(0).(time.Duration) } return r0 } // IsCreated provides a mock function with given fields: ctx, repoOption func (_m *BackupRepoService) IsCreated(ctx context.Context, repoOption udmrepo.RepoOptions) (bool, error) { ret := _m.Called(ctx, repoOption) if len(ret) == 0 { panic("no return value specified for IsCreated") } var r0 bool var r1 error if rf, ok := ret.Get(0).(func(context.Context, udmrepo.RepoOptions) (bool, error)); ok { return rf(ctx, repoOption) } if rf, ok := ret.Get(0).(func(context.Context, udmrepo.RepoOptions) bool); ok { r0 = rf(ctx, repoOption) } else { r0 = ret.Get(0).(bool) } if rf, ok := ret.Get(1).(func(context.Context, udmrepo.RepoOptions) error); ok { r1 = rf(ctx, repoOption) } else { r1 = ret.Error(1) } return r0, r1 } // Maintain provides a mock function with given fields: ctx, repoOption func (_m *BackupRepoService) Maintain(ctx context.Context, repoOption udmrepo.RepoOptions) error { ret := _m.Called(ctx, repoOption) if len(ret) == 0 { panic("no return value specified for Maintain") } var r0 error if rf, ok := ret.Get(0).(func(context.Context, udmrepo.RepoOptions) error); ok { r0 = rf(ctx, repoOption) } else { r0 = ret.Error(0) } return r0 } // Open provides a mock function with given fields: ctx, repoOption func (_m *BackupRepoService) Open(ctx context.Context, repoOption udmrepo.RepoOptions) (udmrepo.BackupRepo, error) { ret := _m.Called(ctx, repoOption) if len(ret) == 0 { panic("no return value specified for Open") } var r0 udmrepo.BackupRepo var r1 error if rf, ok := ret.Get(0).(func(context.Context, udmrepo.RepoOptions) (udmrepo.BackupRepo, error)); ok { return rf(ctx, repoOption) } if rf, ok := ret.Get(0).(func(context.Context, udmrepo.RepoOptions) udmrepo.BackupRepo); ok { r0 = rf(ctx, repoOption) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(udmrepo.BackupRepo) } } if rf, ok := ret.Get(1).(func(context.Context, udmrepo.RepoOptions) error); ok { r1 = rf(ctx, repoOption) } else { r1 = ret.Error(1) } return r0, r1 } // NewBackupRepoService creates a new instance of BackupRepoService. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. func NewBackupRepoService(t interface { mock.TestingT Cleanup(func()) }) *BackupRepoService { mock := &BackupRepoService{} mock.Mock.Test(t) t.Cleanup(func() { mock.AssertExpectations(t) }) return mock } ================================================ FILE: pkg/repository/udmrepo/mocks/ObjectReader.go ================================================ // Code generated by mockery v2.39.1. DO NOT EDIT. package mocks import mock "github.com/stretchr/testify/mock" // ObjectReader is an autogenerated mock type for the ObjectReader type type ObjectReader struct { mock.Mock } // Close provides a mock function with given fields: func (_m *ObjectReader) Close() error { ret := _m.Called() if len(ret) == 0 { panic("no return value specified for Close") } var r0 error if rf, ok := ret.Get(0).(func() error); ok { r0 = rf() } else { r0 = ret.Error(0) } return r0 } // Length provides a mock function with given fields: func (_m *ObjectReader) Length() int64 { ret := _m.Called() if len(ret) == 0 { panic("no return value specified for Length") } var r0 int64 if rf, ok := ret.Get(0).(func() int64); ok { r0 = rf() } else { r0 = ret.Get(0).(int64) } return r0 } // Read provides a mock function with given fields: p func (_m *ObjectReader) Read(p []byte) (int, error) { ret := _m.Called(p) if len(ret) == 0 { panic("no return value specified for Read") } var r0 int var r1 error if rf, ok := ret.Get(0).(func([]byte) (int, error)); ok { return rf(p) } if rf, ok := ret.Get(0).(func([]byte) int); ok { r0 = rf(p) } else { r0 = ret.Get(0).(int) } if rf, ok := ret.Get(1).(func([]byte) error); ok { r1 = rf(p) } else { r1 = ret.Error(1) } return r0, r1 } // Seek provides a mock function with given fields: offset, whence func (_m *ObjectReader) Seek(offset int64, whence int) (int64, error) { ret := _m.Called(offset, whence) if len(ret) == 0 { panic("no return value specified for Seek") } var r0 int64 var r1 error if rf, ok := ret.Get(0).(func(int64, int) (int64, error)); ok { return rf(offset, whence) } if rf, ok := ret.Get(0).(func(int64, int) int64); ok { r0 = rf(offset, whence) } else { r0 = ret.Get(0).(int64) } if rf, ok := ret.Get(1).(func(int64, int) error); ok { r1 = rf(offset, whence) } else { r1 = ret.Error(1) } return r0, r1 } // NewObjectReader creates a new instance of ObjectReader. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. func NewObjectReader(t interface { mock.TestingT Cleanup(func()) }) *ObjectReader { mock := &ObjectReader{} mock.Mock.Test(t) t.Cleanup(func() { mock.AssertExpectations(t) }) return mock } ================================================ FILE: pkg/repository/udmrepo/mocks/ObjectWriter.go ================================================ // Code generated by mockery v2.39.1. DO NOT EDIT. package mocks import ( mock "github.com/stretchr/testify/mock" udmrepo "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" ) // ObjectWriter is an autogenerated mock type for the ObjectWriter type type ObjectWriter struct { mock.Mock } // Checkpoint provides a mock function with given fields: func (_m *ObjectWriter) Checkpoint() (udmrepo.ID, error) { ret := _m.Called() if len(ret) == 0 { panic("no return value specified for Checkpoint") } var r0 udmrepo.ID var r1 error if rf, ok := ret.Get(0).(func() (udmrepo.ID, error)); ok { return rf() } if rf, ok := ret.Get(0).(func() udmrepo.ID); ok { r0 = rf() } else { r0 = ret.Get(0).(udmrepo.ID) } if rf, ok := ret.Get(1).(func() error); ok { r1 = rf() } else { r1 = ret.Error(1) } return r0, r1 } // Close provides a mock function with given fields: func (_m *ObjectWriter) Close() error { ret := _m.Called() if len(ret) == 0 { panic("no return value specified for Close") } var r0 error if rf, ok := ret.Get(0).(func() error); ok { r0 = rf() } else { r0 = ret.Error(0) } return r0 } // Result provides a mock function with given fields: func (_m *ObjectWriter) Result() (udmrepo.ID, error) { ret := _m.Called() if len(ret) == 0 { panic("no return value specified for Result") } var r0 udmrepo.ID var r1 error if rf, ok := ret.Get(0).(func() (udmrepo.ID, error)); ok { return rf() } if rf, ok := ret.Get(0).(func() udmrepo.ID); ok { r0 = rf() } else { r0 = ret.Get(0).(udmrepo.ID) } if rf, ok := ret.Get(1).(func() error); ok { r1 = rf() } else { r1 = ret.Error(1) } return r0, r1 } // Seek provides a mock function with given fields: offset, whence func (_m *ObjectWriter) Seek(offset int64, whence int) (int64, error) { ret := _m.Called(offset, whence) if len(ret) == 0 { panic("no return value specified for Seek") } var r0 int64 var r1 error if rf, ok := ret.Get(0).(func(int64, int) (int64, error)); ok { return rf(offset, whence) } if rf, ok := ret.Get(0).(func(int64, int) int64); ok { r0 = rf(offset, whence) } else { r0 = ret.Get(0).(int64) } if rf, ok := ret.Get(1).(func(int64, int) error); ok { r1 = rf(offset, whence) } else { r1 = ret.Error(1) } return r0, r1 } // Write provides a mock function with given fields: p func (_m *ObjectWriter) Write(p []byte) (int, error) { ret := _m.Called(p) if len(ret) == 0 { panic("no return value specified for Write") } var r0 int var r1 error if rf, ok := ret.Get(0).(func([]byte) (int, error)); ok { return rf(p) } if rf, ok := ret.Get(0).(func([]byte) int); ok { r0 = rf(p) } else { r0 = ret.Get(0).(int) } if rf, ok := ret.Get(1).(func([]byte) error); ok { r1 = rf(p) } else { r1 = ret.Error(1) } return r0, r1 } // NewObjectWriter creates a new instance of ObjectWriter. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. func NewObjectWriter(t interface { mock.TestingT Cleanup(func()) }) *ObjectWriter { mock := &ObjectWriter{} mock.Mock.Test(t) t.Cleanup(func() { mock.AssertExpectations(t) }) return mock } ================================================ FILE: pkg/repository/udmrepo/repo.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package udmrepo import ( "context" "io" "time" ) type ID string // ManifestEntryMetadata is the metadata describing one manifest data type ManifestEntryMetadata struct { ID ID // The ID of the manifest data Length int32 // The data size of the manifest data Labels map[string]string // Labels saved together with the manifest data ModTime time.Time // Modified time of the manifest data } type RepoManifest struct { Payload any // The user data of manifest Metadata *ManifestEntryMetadata // The metadata data of manifest } type ManifestFilter struct { Labels map[string]string } const ( // Below consts descrbe the data type of one object. // Metadata: This type describes how the data is organized. // For a file system backup, the Metadata describes a Dir or File. // For a block backup, the Metadata describes a Disk and its incremental link. ObjectDataTypeUnknown int = 0 ObjectDataTypeMetadata int = 1 ObjectDataTypeData int = 2 // Below consts defines the access mode when creating an object for write ObjectDataAccessModeUnknown int = 0 ObjectDataAccessModeFile int = 1 ObjectDataAccessModeBlock int = 2 ObjectDataBackupModeUnknown int = 0 ObjectDataBackupModeFull int = 1 ObjectDataBackupModeInc int = 2 ) // ObjectWriteOptions defines the options when creating an object for write type ObjectWriteOptions struct { FullPath string // Full logical path of the object DataType int // OBJECT_DATA_TYPE_* Description string // A description of the object, could be empty Prefix ID // A prefix of the name used to save the object AccessMode int // OBJECT_DATA_ACCESS_* BackupMode int // OBJECT_DATA_BACKUP_* AsyncWrites int // Num of async writes for the object, 0 means no async write } type AdvancedFeatureInfo struct { MultiPartBackup bool // if set to true, it means the repo supports multiple-part backup } // BackupRepoService is used to initialize, open or maintain a backup repository type BackupRepoService interface { // Create creates a new backup repository. // repoOption: option to the backup repository and the underlying backup storage. Create(ctx context.Context, repoOption RepoOptions) error // Connect connects to an existing backup repository. // repoOption: option to the backup repository and the underlying backup storage. Connect(ctx context.Context, repoOption RepoOptions) error // IsCreated checks if the backup repository has been created in the underlying backup storage. // repoOption: option to the underlying backup storage IsCreated(ctx context.Context, repoOption RepoOptions) (bool, error) // Open opens an backup repository that has been created/connected. // repoOption: options to open the backup repository and the underlying storage. Open(ctx context.Context, repoOption RepoOptions) (BackupRepo, error) // Maintain is periodically called to maintain the backup repository to eliminate redundant data. // repoOption: options to maintain the backup repository. Maintain(ctx context.Context, repoOption RepoOptions) error // DefaultMaintenanceFrequency returns the defgault frequency of maintenance, callers refer this // frequency to maintain the backup repository to get the best maintenance performance DefaultMaintenanceFrequency() time.Duration // ClientSideCacheLimit returns the max cache size required on client side ClientSideCacheLimit(repoOption map[string]string) int64 } // BackupRepo provides the access to the backup repository type BackupRepo interface { // OpenObject opens an existing object for read. // id: the object's unified identifier. OpenObject(ctx context.Context, id ID) (ObjectReader, error) // GetManifest gets a manifest data from the backup repository. GetManifest(ctx context.Context, id ID, mani *RepoManifest) error // FindManifests gets one or more manifest data that match the given labels FindManifests(ctx context.Context, filter ManifestFilter) ([]*ManifestEntryMetadata, error) // NewObjectWriter creates a new object and return the object's writer interface. // return: A unified identifier of the object on success. NewObjectWriter(ctx context.Context, opt ObjectWriteOptions) ObjectWriter // PutManifest saves a manifest object into the backup repository. PutManifest(ctx context.Context, mani RepoManifest) (ID, error) // DeleteManifest deletes a manifest object from the backup repository. DeleteManifest(ctx context.Context, id ID) error // Flush flushes all the backup repository data Flush(ctx context.Context) error // GetAdvancedFeatures returns the support for advanced features GetAdvancedFeatures() AdvancedFeatureInfo // ConcatenateObjects is for multiple-part backup, it concatenates multiple objects into one object ConcatenateObjects(ctx context.Context, objectIDs []ID) (ID, error) // Time returns the local time of the backup repository. It may be different from the time of the caller Time() time.Time // Close closes the backup repository Close(ctx context.Context) error } type ObjectReader interface { io.ReadCloser io.Seeker // Length returns the logical size of the object Length() int64 } type ObjectWriter interface { io.WriteCloser // Seeker is used in the cases that the object is not written sequentially io.Seeker // Checkpoint is periodically called to preserve the state of data written to the repo so far. // Checkpoint returns a unified identifier that represent the current state. // An empty ID could be returned on success if the backup repository doesn't support this. Checkpoint() (ID, error) // Result waits for the completion of the object write. // Result returns the object's unified identifier after the write completes. Result() (ID, error) } ================================================ FILE: pkg/repository/udmrepo/repo_options.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package udmrepo import ( "maps" "os" "path/filepath" "strings" "time" ) const ( StorageTypeS3 = "s3" StorageTypeAzure = "azure" StorageTypeFs = "filesystem" StorageTypeGcs = "gcs" GenOptionMaintainMode = "mode" GenOptionMaintainFull = "full" GenOptionMaintainQuick = "quick" GenOptionOwnerName = "username" GenOptionOwnerDomain = "domainname" StoreOptionS3KeyID = "accessKeyID" StoreOptionS3Provider = "providerName" StoreOptionS3SecretKey = "secretAccessKey" StoreOptionS3Token = "sessionToken" StoreOptionS3Endpoint = "endpoint" StoreOptionS3DisableTLS = "doNotUseTLS" StoreOptionS3DisableTLSVerify = "skipTLSVerify" StoreOptionFsPath = "fspath" StoreOptionGcsReadonly = "readonly" StoreOptionOssBucket = "bucket" StoreOptionOssRegion = "region" StoreOptionCACert = "caCert" StoreOptionCredentialFile = "credFile" StoreOptionPrefix = "prefix" StoreOptionPrefixName = "unified-repo" StoreOptionGenHashAlgo = "hashAlgo" StoreOptionGenEncryptAlgo = "encryptAlgo" StoreOptionGenSplitAlgo = "splitAlgo" StoreOptionGenRetentionMode = "retentionMode" StoreOptionGenRetentionPeriod = "retentionPeriod" StoreOptionGenReadOnly = "readOnly" StoreOptionCacheLimit = "cacheLimitMB" StoreOptionCacheDir = "cacheDir" ThrottleOptionReadOps = "readOPS" ThrottleOptionWriteOps = "writeOPS" ThrottleOptionListOps = "listOPS" ThrottleOptionUploadBytes = "uploadBytes" ThrottleOptionDownloadBytes = "downloadBytes" // FullMaintenanceInterval will overwrite kopia maintenance interval // options are fastGC for 12 hours, eagerGC for 6 hours, normalGC for 24 hours StoreOptionKeyFullMaintenanceInterval = "fullMaintenanceInterval" FastGC FullMaintenanceIntervalOptions = "fastGC" FastGCInterval time.Duration = 12 * time.Hour EagerGC FullMaintenanceIntervalOptions = "eagerGC" EagerGCInterval time.Duration = 6 * time.Hour NormalGC FullMaintenanceIntervalOptions = "normalGC" NormalGCInterval time.Duration = 24 * time.Hour ) type FullMaintenanceIntervalOptions string const ( defaultUsername = "default" defaultDomain = "default" ) type RepoOptions struct { // StorageType is a repository specific string to identify a backup storage, i.e., "s3", "filesystem" StorageType string // RepoPassword is the backup repository's password, if any RepoPassword string // ConfigFilePath is a custom path to save the repository's configuration, if any ConfigFilePath string // GeneralOptions takes other repository specific options GeneralOptions map[string]string // StorageOptions takes storage specific options StorageOptions map[string]string // Description is a description of the backup repository/backup repository operation. // It is for logging/debugging purpose only and doesn't control any behavior of the backup repository. Description string } // PasswordGetter defines the method to get a repository password. type PasswordGetter interface { GetPassword(param any) (string, error) } // StoreOptionsGetter defines the methods to get the storage related options. type StoreOptionsGetter interface { GetStoreType(param any) (string, error) GetStoreOptions(param any) (map[string]string, error) } // NewRepoOptions creates a new RepoOptions for different purpose func NewRepoOptions(optionFuncs ...func(*RepoOptions) error) (*RepoOptions, error) { options := &RepoOptions{ GeneralOptions: make(map[string]string), StorageOptions: make(map[string]string), } for _, optionFunc := range optionFuncs { err := optionFunc(options) if err != nil { return nil, err } } return options, nil } // WithPassword sets the RepoPassword to RepoOptions, the password is acquired through // the provided interface func WithPassword(getter PasswordGetter, param any) func(*RepoOptions) error { return func(options *RepoOptions) error { password, err := getter.GetPassword(param) if err != nil { return err } options.RepoPassword = password return nil } } // WithConfigFile sets the ConfigFilePath to RepoOptions func WithConfigFile(workPath string, repoID string) func(*RepoOptions) error { return func(options *RepoOptions) error { options.ConfigFilePath = getRepoConfigFile(workPath, repoID) return nil } } // WithGenOptions sets the GeneralOptions to RepoOptions func WithGenOptions(genOptions map[string]string) func(*RepoOptions) error { return func(options *RepoOptions) error { for k, v := range genOptions { options.GeneralOptions[k] = v } return nil } } // WithStoreOptions sets the StorageOptions to RepoOptions, the store options are acquired through // the provided interface func WithStoreOptions(getter StoreOptionsGetter, param any) func(*RepoOptions) error { return func(options *RepoOptions) error { storeType, err := getter.GetStoreType(param) if err != nil { return err } storeOptions, err := getter.GetStoreOptions(param) if err != nil { return err } options.StorageType = storeType maps.Copy(options.StorageOptions, storeOptions) return nil } } // WithDescription sets the Description to RepoOptions func WithDescription(desc string) func(*RepoOptions) error { return func(options *RepoOptions) error { options.Description = desc return nil } } // GetRepoUser returns the default username that is used to manipulate the Unified Repo func GetRepoUser() string { return defaultUsername } // GetRepoDomain returns the default user domain that is used to manipulate the Unified Repo func GetRepoDomain() string { return defaultDomain } func getRepoConfigFile(workPath string, repoID string) string { if workPath == "" { home := os.Getenv("HOME") if home != "" { workPath = filepath.Join(home, "udmrepo") } else { workPath = filepath.Join(os.TempDir(), "udmrepo") } } name := "repo-" + strings.ToLower(repoID) + ".conf" return filepath.Join(workPath, name) } ================================================ FILE: pkg/repository/udmrepo/service/service.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package service import ( "github.com/sirupsen/logrus" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo/kopialib" ) // Create creates an instance of BackupRepoService func Create(repoBackend string, logger logrus.FieldLogger) udmrepo.BackupRepoService { return kopialib.NewKopiaRepoService(logger) } ================================================ FILE: pkg/restic/command.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package restic import ( "fmt" "os" "os/exec" "path/filepath" "strings" ) // Command represents a restic command. type Command struct { Command string RepoIdentifier string PasswordFile string CACertFile string Dir string Args []string ExtraFlags []string Env []string } func (c *Command) RepoName() string { if c.RepoIdentifier == "" { return "" } return c.RepoIdentifier[strings.LastIndex(c.RepoIdentifier, "/")+1:] } // StringSlice returns the command as a slice of strings. func (c *Command) StringSlice() []string { res := []string{"restic"} res = append(res, c.Command, repoFlag(c.RepoIdentifier)) if c.PasswordFile != "" { res = append(res, passwordFlag(c.PasswordFile)) } if c.CACertFile != "" { res = append(res, cacertFlag(c.CACertFile)) } // If VELERO_SCRATCH_DIR is defined, put the restic cache within it. If not, // allow restic to choose the location. This makes running either in-cluster // or local (dev) work properly. if scratch := os.Getenv("VELERO_SCRATCH_DIR"); scratch != "" { res = append(res, cacheDirFlag(filepath.Join(scratch, ".cache", "restic"))) } res = append(res, c.Args...) res = append(res, c.ExtraFlags...) return res } // String returns the command as a string. func (c *Command) String() string { return strings.Join(c.StringSlice(), " ") } // Cmd returns an exec.Cmd for the command. func (c *Command) Cmd() *exec.Cmd { parts := c.StringSlice() cmd := exec.Command(parts[0], parts[1:]...) //nolint:gosec,noctx // Internal call. No need to check the parameter. No to add context for deprecated Restic. cmd.Dir = c.Dir if len(c.Env) > 0 { cmd.Env = c.Env } return cmd } func repoFlag(repoIdentifier string) string { return fmt.Sprintf("--repo=%s", repoIdentifier) } func passwordFlag(file string) string { return fmt.Sprintf("--password-file=%s", file) } func cacheDirFlag(dir string) string { return fmt.Sprintf("--cache-dir=%s", dir) } func cacertFlag(path string) string { return fmt.Sprintf("--cacert=%s", path) } ================================================ FILE: pkg/restic/command_factory.go ================================================ /* Copyright 2018, 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package restic import ( "fmt" "strings" ) // BackupCommand returns a Command for running a restic backup. func BackupCommand(repoIdentifier, passwordFile, path string, tags map[string]string) *Command { // --host flag is provided with a generic value because restic uses the host // to find a parent snapshot, and by default it will be the name of the daemonset pod // where the `restic backup` command is run. If this pod is recreated, we want to continue // taking incremental backups rather than triggering a full one due to a new pod name. return &Command{ Command: "backup", RepoIdentifier: repoIdentifier, PasswordFile: passwordFile, Dir: path, Args: []string{"."}, ExtraFlags: append(backupTagFlags(tags), "--host=velero", "--json"), } } func backupTagFlags(tags map[string]string) []string { var flags []string for k, v := range tags { flags = append(flags, fmt.Sprintf("--tag=%s=%s", k, v)) } return flags } // RestoreCommand returns a Command for running a restic restore. func RestoreCommand(repoIdentifier, passwordFile, snapshotID, target string) *Command { return &Command{ Command: "restore", RepoIdentifier: repoIdentifier, PasswordFile: passwordFile, Dir: target, Args: []string{snapshotID}, ExtraFlags: []string{"--target=."}, } } // GetSnapshotCommand returns a Command for running a restic (get) snapshots. func GetSnapshotCommand(repoIdentifier, passwordFile string, tags map[string]string) *Command { return &Command{ Command: "snapshots", RepoIdentifier: repoIdentifier, PasswordFile: passwordFile, // "--last" is replaced by "--latest=1" in restic v0.12.1 ExtraFlags: []string{"--json", "--latest=1", getSnapshotTagFlag(tags)}, } } func getSnapshotTagFlag(tags map[string]string) string { var tagFilters []string for k, v := range tags { tagFilters = append(tagFilters, fmt.Sprintf("%s=%s", k, v)) } return fmt.Sprintf("--tag=%s", strings.Join(tagFilters, ",")) } func InitCommand(repoIdentifier string) *Command { return &Command{ Command: "init", RepoIdentifier: repoIdentifier, } } func SnapshotsCommand(repoIdentifier string) *Command { return &Command{ Command: "snapshots", RepoIdentifier: repoIdentifier, } } func PruneCommand(repoIdentifier string) *Command { return &Command{ Command: "prune", RepoIdentifier: repoIdentifier, } } func ForgetCommand(repoIdentifier, snapshotID string) *Command { return &Command{ Command: "forget", RepoIdentifier: repoIdentifier, Args: []string{snapshotID}, } } func UnlockCommand(repoIdentifier string) *Command { return &Command{ Command: "unlock", RepoIdentifier: repoIdentifier, } } func StatsCommand(repoIdentifier, passwordFile, snapshotID string) *Command { return &Command{ Command: "stats", RepoIdentifier: repoIdentifier, PasswordFile: passwordFile, Args: []string{snapshotID}, ExtraFlags: []string{"--json"}, } } ================================================ FILE: pkg/restic/command_factory_test.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package restic import ( "sort" "strings" "testing" "github.com/stretchr/testify/assert" ) func TestBackupCommand(t *testing.T) { c := BackupCommand("repo-id", "password-file", "path", map[string]string{"foo": "bar", "c": "d"}) assert.Equal(t, "backup", c.Command) assert.Equal(t, "repo-id", c.RepoIdentifier) assert.Equal(t, "password-file", c.PasswordFile) assert.Equal(t, "path", c.Dir) assert.Equal(t, []string{"."}, c.Args) expected := []string{"--tag=foo=bar", "--tag=c=d", "--host=velero", "--json"} sort.Strings(expected) sort.Strings(c.ExtraFlags) assert.Equal(t, expected, c.ExtraFlags) } func TestRestoreCommand(t *testing.T) { c := RestoreCommand("repo-id", "password-file", "snapshot-id", "target") assert.Equal(t, "restore", c.Command) assert.Equal(t, "repo-id", c.RepoIdentifier) assert.Equal(t, "password-file", c.PasswordFile) assert.Equal(t, "target", c.Dir) assert.Equal(t, []string{"snapshot-id"}, c.Args) assert.Equal(t, []string{"--target=."}, c.ExtraFlags) } func TestGetSnapshotCommand(t *testing.T) { expectedTags := map[string]string{"foo": "bar", "c": "d"} c := GetSnapshotCommand("repo-id", "password-file", expectedTags) assert.Equal(t, "snapshots", c.Command) assert.Equal(t, "repo-id", c.RepoIdentifier) assert.Equal(t, "password-file", c.PasswordFile) // set up expected flag names expectedFlags := []string{"--json", "--latest=1", "--tag"} // for tracking actual flag names actualFlags := []string{} // for tracking actual --tag values as a map actualTags := make(map[string]string) // loop through actual flags for _, flag := range c.ExtraFlags { // split into 2 parts from the first = sign (if any) parts := strings.SplitN(flag, "=", 2) // convert --tag data to a map if parts[0] == "--tag" { actualFlags = append(actualFlags, parts[0]) // split based on , tags := strings.Split(parts[1], ",") // loop through each key-value tag pair for _, tag := range tags { // split the pair on = kvs := strings.Split(tag, "=") // record actual key & value actualTags[kvs[0]] = kvs[1] } } else { actualFlags = append(actualFlags, flag) } } assert.Equal(t, expectedFlags, actualFlags) assert.Equal(t, expectedTags, actualTags) } func TestInitCommand(t *testing.T) { c := InitCommand("repo-id") assert.Equal(t, "init", c.Command) assert.Equal(t, "repo-id", c.RepoIdentifier) } func TestSnapshotsCommand(t *testing.T) { c := SnapshotsCommand("repo-id") assert.Equal(t, "snapshots", c.Command) assert.Equal(t, "repo-id", c.RepoIdentifier) } func TestPruneCommand(t *testing.T) { c := PruneCommand("repo-id") assert.Equal(t, "prune", c.Command) assert.Equal(t, "repo-id", c.RepoIdentifier) } func TestForgetCommand(t *testing.T) { c := ForgetCommand("repo-id", "snapshot-id") assert.Equal(t, "forget", c.Command) assert.Equal(t, "repo-id", c.RepoIdentifier) assert.Equal(t, []string{"snapshot-id"}, c.Args) } func TestStatsCommand(t *testing.T) { c := StatsCommand("repo-id", "password-file", "snapshot-id") assert.Equal(t, "stats", c.Command) assert.Equal(t, "repo-id", c.RepoIdentifier) assert.Equal(t, "password-file", c.PasswordFile) assert.Equal(t, []string{"snapshot-id"}, c.Args) assert.Equal(t, []string{"--json"}, c.ExtraFlags) } ================================================ FILE: pkg/restic/command_test.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package restic import ( "os" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestRepoName(t *testing.T) { c := &Command{RepoIdentifier: ""} assert.Empty(t, c.RepoName()) c.RepoIdentifier = "s3:s3.amazonaws.com/bucket/prefix/repo" assert.Equal(t, "repo", c.RepoName()) c.RepoIdentifier = "azure:bucket:/repo" assert.Equal(t, "repo", c.RepoName()) c.RepoIdentifier = "gs:bucket:/prefix/repo" assert.Equal(t, "repo", c.RepoName()) } func TestStringSlice(t *testing.T) { c := &Command{ Command: "cmd", RepoIdentifier: "repo-id", PasswordFile: "/path/to/password-file", Dir: "/some/pwd", Args: []string{"arg-1", "arg-2"}, ExtraFlags: []string{"--foo=bar"}, } require.NoError(t, os.Unsetenv("VELERO_SCRATCH_DIR")) assert.Equal(t, []string{ "restic", "cmd", "--repo=repo-id", "--password-file=/path/to/password-file", "arg-1", "arg-2", "--foo=bar", }, c.StringSlice()) os.Setenv("VELERO_SCRATCH_DIR", "/foo") assert.Equal(t, []string{ "restic", "cmd", "--repo=repo-id", "--password-file=/path/to/password-file", "--cache-dir=/foo/.cache/restic", "arg-1", "arg-2", "--foo=bar", }, c.StringSlice()) require.NoError(t, os.Unsetenv("VELERO_SCRATCH_DIR")) } func TestString(t *testing.T) { c := &Command{ Command: "cmd", RepoIdentifier: "repo-id", PasswordFile: "/path/to/password-file", Dir: "/some/pwd", Args: []string{"arg-1", "arg-2"}, ExtraFlags: []string{"--foo=bar"}, } require.NoError(t, os.Unsetenv("VELERO_SCRATCH_DIR")) assert.Equal(t, "restic cmd --repo=repo-id --password-file=/path/to/password-file arg-1 arg-2 --foo=bar", c.String()) } func TestCmd(t *testing.T) { c := &Command{ Command: "cmd", RepoIdentifier: "repo-id", PasswordFile: "/path/to/password-file", Dir: "/some/pwd", Args: []string{"arg-1", "arg-2"}, ExtraFlags: []string{"--foo=bar"}, } require.NoError(t, os.Unsetenv("VELERO_SCRATCH_DIR")) execCmd := c.Cmd() assert.Equal(t, c.StringSlice(), execCmd.Args) assert.Equal(t, c.Dir, execCmd.Dir) } ================================================ FILE: pkg/restic/common.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package restic import ( "fmt" "os" "strconv" "strings" "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/vmware-tanzu/velero/internal/credentials" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" repoconfig "github.com/vmware-tanzu/velero/pkg/repository/config" "github.com/vmware-tanzu/velero/pkg/util/filesystem" ) const ( // DefaultMaintenanceFrequency is the default time interval // at which restic prune is run. DefaultMaintenanceFrequency = 7 * 24 * time.Hour // insecureSkipTLSVerifyKey is the flag in BackupStorageLocation's config // to indicate whether to skip TLS verify to setup insecure HTTPS connection. insecureSkipTLSVerifyKey = "insecureSkipTLSVerify" // resticInsecureTLSFlag is the flag for Restic command line to indicate // skip TLS verify on https connection. resticInsecureTLSFlag = "--insecure-tls" ) // TempCACertFile creates a temp file containing a CA bundle // and returns its path. The caller should generally call os.Remove() // to remove the file when done with it. func TempCACertFile(caCert []byte, bsl string, fs filesystem.Interface) (string, error) { file, err := fs.TempFile("", fmt.Sprintf("cacert-%s", bsl)) if err != nil { return "", errors.WithStack(err) } if _, err := file.Write(caCert); err != nil { // nothing we can do about an error closing the file here, and we're // already returning an error about the write failing. file.Close() return "", errors.WithStack(err) } name := file.Name() if err := file.Close(); err != nil { return "", errors.WithStack(err) } return name, nil } // environ is a slice of strings representing the environment, in the form "key=value". type environ []string // Unset a single environment variable. func (e *environ) Unset(key string) { for i := range *e { if strings.HasPrefix((*e)[i], key+"=") { (*e)[i] = (*e)[len(*e)-1] *e = (*e)[:len(*e)-1] break } } } // CmdEnv returns a list of environment variables (in the format var=val) that // should be used when running a restic command for a particular backend provider. // This list is the current environment, plus any provider-specific variables restic needs. func CmdEnv(backupLocation *velerov1api.BackupStorageLocation, credentialFileStore credentials.FileStore) ([]string, error) { var env environ env = os.Environ() customEnv := map[string]string{} var err error config := backupLocation.Spec.Config if config == nil { config = map[string]string{} } if backupLocation.Spec.Credential != nil { credsFile, err := credentialFileStore.Path(backupLocation.Spec.Credential) if err != nil { return []string{}, errors.WithStack(err) } config[repoconfig.CredentialsFileKey] = credsFile } backendType := repoconfig.GetBackendType(backupLocation.Spec.Provider, backupLocation.Spec.Config) switch backendType { case repoconfig.AWSBackend: customEnv, err = repoconfig.GetS3ResticEnvVars(config) if err != nil { return []string{}, err } case repoconfig.AzureBackend: customEnv, err = repoconfig.GetAzureResticEnvVars(config) if err != nil { return []string{}, err } case repoconfig.GCPBackend: customEnv, err = repoconfig.GetGCPResticEnvVars(config) if err != nil { return []string{}, err } } for k, v := range customEnv { env.Unset(k) if v == "" { continue } env = append(env, fmt.Sprintf("%s=%s", k, v)) } return env, nil } // GetInsecureSkipTLSVerifyFromBSL get insecureSkipTLSVerify flag from BSL configuration, // Then return --insecure-tls flag with boolean value as result. func GetInsecureSkipTLSVerifyFromBSL(backupLocation *velerov1api.BackupStorageLocation, logger logrus.FieldLogger) string { result := "" if backupLocation == nil { logger.Info("bsl is nil. return empty.") return result } if insecure, _ := strconv.ParseBool(backupLocation.Spec.Config[insecureSkipTLSVerifyKey]); insecure { logger.Debugf("set --insecure-tls=true for Restic command according to BSL %s config", backupLocation.Name) result = resticInsecureTLSFlag + "=true" return result } return result } ================================================ FILE: pkg/restic/common_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package restic import ( "os" "testing" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func TestTempCACertFile(t *testing.T) { var ( fs = velerotest.NewFakeFileSystem() caCertData = []byte("cacert") ) fileName, err := TempCACertFile(caCertData, "default", fs) require.NoError(t, err) contents, err := fs.ReadFile(fileName) require.NoError(t, err) assert.Equal(t, string(caCertData), string(contents)) os.Remove(fileName) } func TestGetInsecureSkipTLSVerifyFromBSL(t *testing.T) { log := logrus.StandardLogger() tests := []struct { name string backupLocation *velerov1api.BackupStorageLocation logger logrus.FieldLogger expected string }{ { "Test with nil BSL. Should return empty string.", nil, log, "", }, { "Test BSL with no configuration. Should return empty string.", &velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Provider: "azure", }, }, log, "", }, { "Test with AWS BSL's insecureSkipTLSVerify set to false.", &velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Provider: "aws", Config: map[string]string{ "insecureSkipTLSVerify": "false", }, }, }, log, "", }, { "Test with AWS BSL's insecureSkipTLSVerify set to true.", &velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Provider: "aws", Config: map[string]string{ "insecureSkipTLSVerify": "true", }, }, }, log, "--insecure-tls=true", }, { "Test with Azure BSL's insecureSkipTLSVerify set to invalid.", &velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Provider: "azure", Config: map[string]string{ "insecureSkipTLSVerify": "invalid", }, }, }, log, "", }, { "Test with GCP without insecureSkipTLSVerify.", &velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Provider: "gcp", Config: map[string]string{}, }, }, log, "", }, { "Test with AWS without config.", &velerov1api.BackupStorageLocation{ Spec: velerov1api.BackupStorageLocationSpec{ Provider: "aws", }, }, log, "", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { res := GetInsecureSkipTLSVerifyFromBSL(test.backupLocation, test.logger) assert.Equal(t, test.expected, res) }) } } ================================================ FILE: pkg/restic/exec_commands.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package restic import ( "bytes" "encoding/json" "fmt" "strings" "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/vmware-tanzu/velero/pkg/uploader" "github.com/vmware-tanzu/velero/pkg/util/exec" "github.com/vmware-tanzu/velero/pkg/util/filesystem" ) const restoreProgressCheckInterval = 10 * time.Second const backupProgressCheckInterval = 10 * time.Second var fileSystem = filesystem.NewFileSystem() type backupStatusLine struct { MessageType string `json:"message_type"` // seen in status lines TotalBytes int64 `json:"total_bytes"` BytesDone int64 `json:"bytes_done"` // seen in summary line at the end TotalBytesProcessed int64 `json:"total_bytes_processed"` } // GetSnapshotID runs provided 'restic snapshots' command to get the ID of a snapshot // and an error if a unique snapshot cannot be identified. func GetSnapshotID(snapshotIDCmd *Command) (string, error) { stdout, stderr, err := exec.RunCommand(snapshotIDCmd.Cmd()) if err != nil { return "", errors.Wrapf(err, "error running command, stderr=%s", stderr) } type snapshotID struct { ShortID string `json:"short_id"` } var snapshots []snapshotID if err := json.Unmarshal([]byte(stdout), &snapshots); err != nil { return "", errors.Wrap(err, "error unmarshaling restic snapshots result") } if len(snapshots) != 1 { return "", errors.Errorf("expected one matching snapshot by command: %s, got %d", snapshotIDCmd.String(), len(snapshots)) } return snapshots[0].ShortID, nil } // RunBackup runs a `restic backup` command and watches the output to provide // progress updates to the caller. func RunBackup(backupCmd *Command, log logrus.FieldLogger, updater uploader.ProgressUpdater) (string, string, error) { // buffers for copying command stdout/err output into stdoutBuf := new(bytes.Buffer) stderrBuf := new(bytes.Buffer) // create a channel to signal when to end the goroutine scanning for progress // updates quit := make(chan struct{}) cmd := backupCmd.Cmd() cmd.Stdout = stdoutBuf cmd.Stderr = stderrBuf err := cmd.Start() if err != nil { exec.LogErrorAsExitCode(err, log) return stdoutBuf.String(), stderrBuf.String(), err } go func() { ticker := time.NewTicker(backupProgressCheckInterval) for { select { case <-ticker.C: lastLine := getLastLine(stdoutBuf.Bytes()) if len(lastLine) > 0 { stat, err := decodeBackupStatusLine(lastLine) if err != nil { log.WithError(err).Errorf("error getting restic backup progress") } // if the line contains a non-empty bytes_done field, we can update the // caller with the progress if stat.BytesDone != 0 { updater.UpdateProgress(&uploader.Progress{ TotalBytes: stat.TotalBytes, BytesDone: stat.BytesDone, }) } } case <-quit: ticker.Stop() return } } }() err = cmd.Wait() if err != nil { exec.LogErrorAsExitCode(err, log) return stdoutBuf.String(), stderrBuf.String(), err } quit <- struct{}{} summary, err := getSummaryLine(stdoutBuf.Bytes()) if err != nil { return stdoutBuf.String(), stderrBuf.String(), err } stat, err := decodeBackupStatusLine(summary) if err != nil { return stdoutBuf.String(), stderrBuf.String(), err } if stat.MessageType != "summary" { return stdoutBuf.String(), stderrBuf.String(), errors.WithStack(fmt.Errorf("error getting restic backup summary: %s", string(summary))) } // update progress to 100% updater.UpdateProgress(&uploader.Progress{ TotalBytes: stat.TotalBytesProcessed, BytesDone: stat.TotalBytesProcessed, }) return string(summary), stderrBuf.String(), nil } func decodeBackupStatusLine(lastLine []byte) (backupStatusLine, error) { var stat backupStatusLine if err := json.Unmarshal(lastLine, &stat); err != nil { return stat, errors.Wrapf(err, "unable to decode backup JSON line: %s", string(lastLine)) } return stat, nil } // getLastLine returns the last line of a byte array. The string is assumed to // have a newline at the end of it, so this returns the substring between the // last two newlines. func getLastLine(b []byte) []byte { if len(b) == 0 { return []byte("") } // subslice the byte array to ignore the newline at the end of the string lastNewLineIdx := bytes.LastIndex(b[:len(b)-1], []byte("\n")) return b[lastNewLineIdx+1 : len(b)-1] } // getSummaryLine looks for the summary JSON line // (`{"message_type:"summary",...`) in the restic backup command output. Due to // an issue in Restic, this might not always be the last line // (https://github.com/restic/restic/issues/2389). It returns an error if it // can't be found. func getSummaryLine(b []byte) ([]byte, error) { summaryLineIdx := bytes.LastIndex(b, []byte(`{"message_type":"summary"`)) if summaryLineIdx < 0 { return nil, errors.New("unable to find summary in restic backup command output") } // find the end of the summary line newLineIdx := bytes.Index(b[summaryLineIdx:], []byte("\n")) if newLineIdx < 0 { return nil, errors.New("unable to get summary line from restic backup command output") } return b[summaryLineIdx : summaryLineIdx+newLineIdx], nil } // RunRestore runs a `restic restore` command and monitors the volume size to // provide progress updates to the caller. func RunRestore(restoreCmd *Command, log logrus.FieldLogger, updater uploader.ProgressUpdater) (string, string, error) { insecureTLSFlag := "" for _, extraFlag := range restoreCmd.ExtraFlags { if strings.Contains(extraFlag, resticInsecureTLSFlag) { insecureTLSFlag = extraFlag } } snapshotSize, err := getSnapshotSize(restoreCmd.RepoIdentifier, restoreCmd.PasswordFile, restoreCmd.CACertFile, restoreCmd.Args[0], restoreCmd.Env, insecureTLSFlag) if err != nil { return "", "", errors.Wrap(err, "error getting snapshot size") } updater.UpdateProgress(&uploader.Progress{ TotalBytes: snapshotSize, }) // create a channel to signal when to end the goroutine scanning for progress // updates quit := make(chan struct{}) go func() { ticker := time.NewTicker(restoreProgressCheckInterval) for { select { case <-ticker.C: volumeSize, err := getVolumeSize(restoreCmd.Dir) if err != nil { log.WithError(err).Errorf("error getting restic restore progress") } if volumeSize != 0 { updater.UpdateProgress(&uploader.Progress{ TotalBytes: snapshotSize, BytesDone: volumeSize, }) } case <-quit: ticker.Stop() return } } }() stdout, stderr, err := exec.RunCommandWithLog(restoreCmd.Cmd(), log) quit <- struct{}{} // update progress to 100% updater.UpdateProgress(&uploader.Progress{ TotalBytes: snapshotSize, BytesDone: snapshotSize, }) return stdout, stderr, err } func getSnapshotSize(repoIdentifier, passwordFile, caCertFile, snapshotID string, env []string, insecureTLS string) (int64, error) { cmd := StatsCommand(repoIdentifier, passwordFile, snapshotID) cmd.Env = env cmd.CACertFile = caCertFile if len(insecureTLS) > 0 { cmd.ExtraFlags = append(cmd.ExtraFlags, insecureTLS) } stdout, stderr, err := exec.RunCommand(cmd.Cmd()) if err != nil { return 0, errors.Wrapf(err, "error running command, stderr=%s", stderr) } var snapshotStats struct { TotalSize int64 `json:"total_size"` } if err := json.Unmarshal([]byte(stdout), &snapshotStats); err != nil { return 0, errors.Wrapf(err, "error unmarshaling restic stats result, stdout=%s", stdout) } return snapshotStats.TotalSize, nil } func getVolumeSize(path string) (int64, error) { var size int64 files, err := fileSystem.ReadDir(path) if err != nil { return 0, errors.Wrapf(err, "error reading directory %s", path) } for _, file := range files { if file.IsDir() { s, err := getVolumeSize(fmt.Sprintf("%s/%s", path, file.Name())) if err != nil { return 0, err } size += s } else { size += file.Size() } } return size, nil } ================================================ FILE: pkg/restic/exec_commands_test.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package restic import ( "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/vmware-tanzu/velero/pkg/test" "github.com/vmware-tanzu/velero/pkg/util/filesystem" ) func Test_getSummaryLine(t *testing.T) { summaryLine := `{"message_type":"summary","files_new":0,"files_changed":0,"files_unmodified":3,"dirs_new":0,"dirs_changed":0,"dirs_unmodified":0,"data_blobs":0,"tree_blobs":0,"data_added":0,"total_files_processed":3,"total_bytes_processed":13238272000,"total_duration":0.319265105,"snapshot_id":"38515bb5"}` tests := []struct { name string output string wantErr bool }{ {"no summary", `{"message_type":"status","percent_done":0,"total_files":1,"total_bytes":10485760000} {"message_type":"status","percent_done":0,"total_files":3,"files_done":1,"total_bytes":13238272000} `, true}, {"no newline after summary", `{"message_type":"status","percent_done":0,"total_files":1,"total_bytes":10485760000} {"message_type":"status","percent_done":0,"total_files":3,"files_done":1,"total_bytes":13238272000} {"message_type":"summary","files_new":0,"files_changed":0,"files_unmodified":3,"dirs_new":0`, true}, {"summary at end", `{"message_type":"status","percent_done":0,"total_files":1,"total_bytes":10485760000} {"message_type":"status","percent_done":0,"total_files":3,"files_done":1,"total_bytes":13238272000} {"message_type":"status","percent_done":1,"total_files":3,"files_done":3,"total_bytes":13238272000,"bytes_done":13238272000} {"message_type":"summary","files_new":0,"files_changed":0,"files_unmodified":3,"dirs_new":0,"dirs_changed":0,"dirs_unmodified":0,"data_blobs":0,"tree_blobs":0,"data_added":0,"total_files_processed":3,"total_bytes_processed":13238272000,"total_duration":0.319265105,"snapshot_id":"38515bb5"} `, false}, {"summary before status", `{"message_type":"status","percent_done":0,"total_files":1,"total_bytes":10485760000} {"message_type":"status","percent_done":0,"total_files":3,"files_done":1,"total_bytes":13238272000} {"message_type":"summary","files_new":0,"files_changed":0,"files_unmodified":3,"dirs_new":0,"dirs_changed":0,"dirs_unmodified":0,"data_blobs":0,"tree_blobs":0,"data_added":0,"total_files_processed":3,"total_bytes_processed":13238272000,"total_duration":0.319265105,"snapshot_id":"38515bb5"} {"message_type":"status","percent_done":1,"total_files":3,"files_done":3,"total_bytes":13238272000,"bytes_done":13238272000} `, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { summary, err := getSummaryLine([]byte(tt.output)) if tt.wantErr { assert.Error(t, err) } else { require.NoError(t, err) assert.Equal(t, summaryLine, string(summary)) } }) } } func Test_getLastLine(t *testing.T) { tests := []struct { output []byte want string }{ {[]byte(`last line `), "last line"}, {[]byte(`first line second line third line `), "third line"}, {[]byte(""), ""}, {nil, ""}, } for _, tt := range tests { t.Run(tt.want, func(t *testing.T) { assert.Equal(t, []byte(tt.want), getLastLine(tt.output)) }) } } func Test_getVolumeSize(t *testing.T) { files := map[string][]byte{ "/file1.txt": []byte("file1"), "/file2.txt": []byte("file2"), "/file3.txt": []byte("file3"), "/files/file4.txt": []byte("file4"), "/files/nested/file5.txt": []byte("file5"), } fakefs := test.NewFakeFileSystem() var expectedSize int64 for path, content := range files { fakefs.WithFile(path, content) expectedSize += int64(len(content)) } fileSystem = fakefs defer func() { fileSystem = filesystem.NewFileSystem() }() actualSize, err := getVolumeSize("/") require.NoError(t, err) assert.Equal(t, expectedSize, actualSize) } ================================================ FILE: pkg/restore/actions/add_pvc_from_pod_action.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actions import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/plugin/velero" ) type AddPVCFromPodAction struct { logger logrus.FieldLogger } func NewAddPVCFromPodAction(logger logrus.FieldLogger) *AddPVCFromPodAction { return &AddPVCFromPodAction{logger: logger} } func (a *AddPVCFromPodAction) AppliesTo() (velero.ResourceSelector, error) { return velero.ResourceSelector{ IncludedResources: []string{"pods"}, }, nil } func (a *AddPVCFromPodAction) Execute(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { a.logger.Info("Executing AddPVCFromPodAction") var pod corev1api.Pod if err := runtime.DefaultUnstructuredConverter.FromUnstructured(input.Item.UnstructuredContent(), &pod); err != nil { return nil, errors.Wrap(err, "unable to convert unstructured item to pod") } var additionalItems []velero.ResourceIdentifier for _, volume := range pod.Spec.Volumes { if volume.PersistentVolumeClaim == nil { continue } a.logger.Infof("Adding PVC %s/%s as an additional item to restore", pod.Namespace, volume.PersistentVolumeClaim.ClaimName) additionalItems = append(additionalItems, velero.ResourceIdentifier{ GroupResource: kuberesource.PersistentVolumeClaims, Namespace: pod.Namespace, Name: volume.PersistentVolumeClaim.ClaimName, }) } return &velero.RestoreItemActionExecuteOutput{ UpdatedItem: input.Item, AdditionalItems: additionalItems, }, nil } ================================================ FILE: pkg/restore/actions/add_pvc_from_pod_action_test.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actions import ( "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/plugin/velero" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func TestAddPVCFromPodActionExecute(t *testing.T) { tests := []struct { name string item *corev1api.Pod want []velero.ResourceIdentifier }{ { name: "pod with no volumes returns no additional items", item: &corev1api.Pod{}, want: nil, }, { name: "pod with some PVCs returns them as additional items", item: &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: "ns-1", Name: "foo", }, Spec: corev1api.PodSpec{ Volumes: []corev1api.Volume{ { VolumeSource: corev1api.VolumeSource{ EmptyDir: new(corev1api.EmptyDirVolumeSource), }, }, { VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "pvc-1", }, }, }, { VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "pvc-2", }, }, }, { VolumeSource: corev1api.VolumeSource{ HostPath: new(corev1api.HostPathVolumeSource), }, }, { VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "pvc-3", }, }, }, }, }, }, want: []velero.ResourceIdentifier{ {GroupResource: kuberesource.PersistentVolumeClaims, Namespace: "ns-1", Name: "pvc-1"}, {GroupResource: kuberesource.PersistentVolumeClaims, Namespace: "ns-1", Name: "pvc-2"}, {GroupResource: kuberesource.PersistentVolumeClaims, Namespace: "ns-1", Name: "pvc-3"}, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { itemData, err := runtime.DefaultUnstructuredConverter.ToUnstructured(test.item) require.NoError(t, err) action := &AddPVCFromPodAction{logger: velerotest.NewLogger()} input := &velero.RestoreItemActionExecuteInput{ Item: &unstructured.Unstructured{Object: itemData}, } res, err := action.Execute(input) require.NoError(t, err) assert.Equal(t, test.want, res.AdditionalItems) }) } } ================================================ FILE: pkg/restore/actions/admissionwebhook_config_action.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actions import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "github.com/vmware-tanzu/velero/pkg/plugin/velero" ) // AdmissionWebhookConfigurationAction is a RestoreItemAction plugin applicable to mutatingwebhookconfiguration and // validatingwebhookconfiguration to reset the invalid value for "sideEffects" of the webhooks. // More background please refer to https://github.com/vmware-tanzu/velero/issues/3516 type AdmissionWebhookConfigurationAction struct { logger logrus.FieldLogger } // NewAdmissionWebhookConfigurationAction creates a new instance of AdmissionWebhookConfigurationAction func NewAdmissionWebhookConfigurationAction(logger logrus.FieldLogger) *AdmissionWebhookConfigurationAction { return &AdmissionWebhookConfigurationAction{logger: logger} } // AppliesTo implements the RestoreItemAction plugin interface method. func (a *AdmissionWebhookConfigurationAction) AppliesTo() (velero.ResourceSelector, error) { return velero.ResourceSelector{ IncludedResources: []string{"mutatingwebhookconfigurations", "validatingwebhookconfigurations"}, }, nil } // Execute will reset the value of "sideEffects" attribute of each item in the "webhooks" list to "None" if they are invalid values for // v1, such as "Unknown" or "Some" func (a *AdmissionWebhookConfigurationAction) Execute(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { a.logger.Info("Executing ChangeStorageClassAction") defer a.logger.Info("Done executing ChangeStorageClassAction") item := input.Item apiVersion, _, err := unstructured.NestedString(item.UnstructuredContent(), "apiVersion") if err != nil { return nil, errors.Wrap(err, "failed to get the apiVersion from input item") } name, _, _ := unstructured.NestedString(item.UnstructuredContent(), "metadata", "name") logger := a.logger.WithField("resource_name", name) if apiVersion != "admissionregistration.k8s.io/v1" { logger.Infof("unable to handle api version: %s, skip", apiVersion) return velero.NewRestoreItemActionExecuteOutput(input.Item), nil } webhooks, ok, err := unstructured.NestedSlice(item.UnstructuredContent(), "webhooks") if err != nil { return nil, errors.Wrap(err, "failed to get webhooks slice from input item") } if !ok { logger.Info("webhooks is not set, skip") return velero.NewRestoreItemActionExecuteOutput(input.Item), nil } newWebhooks := make([]any, 0) for i := range webhooks { logger2 := logger.WithField("index", i) obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&webhooks[i]) if err != nil { logger2.Errorf("failed to convert the webhook entry, error: %v, it will be dropped", err) continue } s, _, _ := unstructured.NestedString(obj, "sideEffects") if s != "None" && s != "NoneOnDryRun" { logger2.Infof("reset the invalid sideEffects value '%s' to 'None'", s) obj["sideEffects"] = "None" } newWebhooks = append(newWebhooks, obj) } item.UnstructuredContent()["webhooks"] = newWebhooks return velero.NewRestoreItemActionExecuteOutput(item), nil } ================================================ FILE: pkg/restore/actions/admissionwebhook_config_action_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actions import ( "encoding/json" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "github.com/vmware-tanzu/velero/pkg/plugin/velero" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func TestNewAdmissionWebhookConfigurationActionExecute(t *testing.T) { action := NewAdmissionWebhookConfigurationAction(velerotest.NewLogger()) cases := []struct { name string itemJSON string wantErr bool NoneSideEffectsIndex []int // the indexes with sideEffects that arereset to None NotNoneSideEffectsIndex []int // the indexes with sideEffects that are not reset to None }{ { name: "v1 mutatingwebhookconfiguration with sideEffects as Unknown", itemJSON: `{ "apiVersion": "admissionregistration.k8s.io/v1", "kind": "MutatingWebhookConfiguration", "metadata": { "name": "my-test-mutating" }, "webhooks": [ { "clientConfig": { "url": "https://mytest.org" }, "rules": [ { "apiGroups": [ "" ], "apiVersions": [ "v1" ], "operations": [ "CREATE" ], "resources": [ "pods" ], "scope": "Namespaced" } ], "sideEffects": "Unknown" } ] }`, wantErr: false, NoneSideEffectsIndex: []int{0}, }, { name: "v1 validatingwebhookconfiguration with sideEffects as Some", itemJSON: `{ "apiVersion": "admissionregistration.k8s.io/v1", "kind": "ValidatingWebhookConfiguration", "metadata": { "name": "my-test-validating" }, "webhooks": [ { "clientConfig": { "url": "https://mytest.org" }, "rules": [ { "apiGroups": [ "" ], "apiVersions": [ "v1" ], "operations": [ "CREATE" ], "resources": [ "pods" ], "scope": "Namespaced" } ], "sideEffects": "Some" } ] }`, wantErr: false, NoneSideEffectsIndex: []int{0}, }, { name: "v1beta1 validatingwebhookconfiguration with sideEffects as Some, nothing should change", itemJSON: `{ "apiVersion": "admissionregistration.k8s.io/v1beta1", "kind": "ValidatingWebhookConfiguration", "metadata": { "name": "my-test-validating" }, "webhooks": [ { "clientConfig": { "url": "https://mytest.org" }, "rules": [ { "apiGroups": [ "" ], "apiVersions": [ "v1" ], "operations": [ "CREATE" ], "resources": [ "pods" ], "scope": "Namespaced" } ], "sideEffects": "Some" } ] }`, wantErr: false, NotNoneSideEffectsIndex: []int{0}, }, { name: "v1 validatingwebhookconfiguration with multiple invalid sideEffects", itemJSON: `{ "apiVersion": "admissionregistration.k8s.io/v1", "kind": "ValidatingWebhookConfiguration", "metadata": { "name": "my-test-validating" }, "webhooks": [ { "clientConfig": { "url": "https://mytest.org" }, "sideEffects": "Some" }, { "clientConfig": { "url": "https://mytest2.org" }, "sideEffects": "Some" } ] }`, wantErr: false, NoneSideEffectsIndex: []int{0, 1}, }, } for _, tt := range cases { t.Run(tt.name, func(t *testing.T) { o := map[string]any{} json.Unmarshal([]byte(tt.itemJSON), &o) input := &velero.RestoreItemActionExecuteInput{ Item: &unstructured.Unstructured{ Object: o, }, } output, err := action.Execute(input) if tt.wantErr { require.Error(t, err) } else { require.NoError(t, err) } if tt.NoneSideEffectsIndex != nil { wb, _, err := unstructured.NestedSlice(output.UpdatedItem.UnstructuredContent(), "webhooks") require.NoError(t, err) for _, i := range tt.NoneSideEffectsIndex { it, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&wb[i]) require.NoError(t, err) s := it["sideEffects"].(string) assert.Equal(t, "None", s) } } if tt.NotNoneSideEffectsIndex != nil { wb, _, err := unstructured.NestedSlice(output.UpdatedItem.UnstructuredContent(), "webhooks") require.NoError(t, err) for _, i := range tt.NotNoneSideEffectsIndex { it, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&wb[i]) require.NoError(t, err) s := it["sideEffects"].(string) assert.NotEqual(t, "None", s) } } }) } } ================================================ FILE: pkg/restore/actions/apiservice_action.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actions import ( "github.com/sirupsen/logrus" "k8s.io/kube-aggregator/pkg/controllers/autoregister" "github.com/vmware-tanzu/velero/pkg/plugin/velero" ) type APIServiceAction struct { logger logrus.FieldLogger } // NewAPIServiceAction returns an APIServiceAction which is a RestoreItemAction plugin // that will skip the restore of any APIServices which are managed by Kubernetes. This // is determined by looking for the "kube-aggregator.kubernetes.io/automanaged" label on // the APIService. func NewAPIServiceAction(logger logrus.FieldLogger) *APIServiceAction { return &APIServiceAction{logger: logger} } func (a *APIServiceAction) AppliesTo() (velero.ResourceSelector, error) { return velero.ResourceSelector{ IncludedResources: []string{"apiservices"}, LabelSelector: autoregister.AutoRegisterManagedLabel, }, nil } func (a *APIServiceAction) Execute(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { a.logger.Info("Executing APIServiceAction") defer a.logger.Info("Done executing APIServiceAction") a.logger.Infof("Skipping restore of APIService as it is managed by Kubernetes") return velero.NewRestoreItemActionExecuteOutput(input.Item).WithoutRestore(), nil } ================================================ FILE: pkg/restore/actions/apiservice_action_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actions import ( "testing" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" "github.com/vmware-tanzu/velero/pkg/plugin/velero" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func TestAPIServiceActionExecuteSkipsRestore(t *testing.T) { obj := apiregistrationv1.APIService{ ObjectMeta: metav1.ObjectMeta{ Name: "v1.test.velero.io", }, } unstructuredAPIService, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&obj) require.NoError(t, err) action := NewAPIServiceAction(velerotest.NewLogger()) res, err := action.Execute(&velero.RestoreItemActionExecuteInput{ Item: &unstructured.Unstructured{Object: unstructuredAPIService}, ItemFromBackup: &unstructured.Unstructured{Object: unstructuredAPIService}, }) require.NoError(t, err) var apiService apiregistrationv1.APIService require.NoError(t, runtime.DefaultUnstructuredConverter.FromUnstructured(res.UpdatedItem.UnstructuredContent(), &apiService)) require.Equal(t, obj, apiService) require.True(t, res.SkipRestore) } ================================================ FILE: pkg/restore/actions/change_image_name_action.go ================================================ /* Copyright 2022 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actions import ( "context" "fmt" "strings" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" "github.com/vmware-tanzu/velero/pkg/plugin/velero" ) const ( delimiterValue = "," ) // ChangeImageNameAction updates a deployment or Pod's image name // if a mapping is found in the plugin's config map. type ChangeImageNameAction struct { logger logrus.FieldLogger configMapClient corev1client.ConfigMapInterface } // NewChangeImageNameAction is the constructor for ChangeImageNameAction. func NewChangeImageNameAction( logger logrus.FieldLogger, configMapClient corev1client.ConfigMapInterface, ) *ChangeImageNameAction { return &ChangeImageNameAction{ logger: logger, configMapClient: configMapClient, } } // AppliesTo returns the resources that ChangeImageNameAction should // be run for. func (a *ChangeImageNameAction) AppliesTo() (velero.ResourceSelector, error) { return velero.ResourceSelector{ IncludedResources: []string{"deployments", "statefulsets", "daemonsets", "replicasets", "replicationcontrollers", "jobs", "cronjobs", "pods"}, }, nil } // Execute updates the item's spec.containers' image if a mapping is found // in the config map for the plugin. func (a *ChangeImageNameAction) Execute(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { a.logger.Info("Executing ChangeImageNameAction") defer a.logger.Info("Done executing ChangeImageNameAction") opts := metav1.ListOptions{ LabelSelector: fmt.Sprintf("velero.io/plugin-config,%s=%s", "velero.io/change-image-name", common.PluginKindRestoreItemAction), } list, err := a.configMapClient.List(context.TODO(), opts) if err != nil { return nil, errors.WithStack(err) } if len(list.Items) == 0 { return &velero.RestoreItemActionExecuteOutput{ UpdatedItem: input.Item, }, nil } if len(list.Items) > 1 { var items []string for _, item := range list.Items { items = append(items, item.Name) } return nil, errors.Errorf("found more than one ConfigMap matching label selector %q: %v", opts.LabelSelector, items) } config := &list.Items[0] if len(config.Data) == 0 { a.logger.Info("No image name mappings found") return velero.NewRestoreItemActionExecuteOutput(input.Item), nil } obj, ok := input.Item.(*unstructured.Unstructured) if !ok { return nil, errors.Errorf("object was of unexpected type %T", input.Item) } if obj.GetKind() == "Pod" { err = a.replaceImageName(obj, config, "spec", "containers") if err != nil { a.logger.Infof("replace image name meet error: %v", err) return nil, errors.Wrap(err, "error getting item's spec.containers") } err = a.replaceImageName(obj, config, "spec", "initContainers") if err != nil { a.logger.Infof("replace image name meet error: %v", err) return nil, errors.Wrap(err, "error getting item's spec.containers") } } else if obj.GetKind() == "CronJob" { //handle containers err = a.replaceImageName(obj, config, "spec", "jobTemplate", "spec", "template", "spec", "containers") if err != nil { a.logger.Infof("replace image name meet error: %v", err) return nil, errors.Wrap(err, "error getting item's spec.containers") } //handle initContainers err = a.replaceImageName(obj, config, "spec", "jobTemplate", "spec", "template", "spec", "initContainers") if err != nil { a.logger.Infof("replace image name meet error: %v", err) return nil, errors.Wrap(err, "error getting item's spec.containers") } } else { //handle containers err = a.replaceImageName(obj, config, "spec", "template", "spec", "containers") if err != nil { a.logger.Infof("replace image name meet error: %v", err) return nil, errors.Wrap(err, "error getting item's spec.containers") } //handle initContainers err = a.replaceImageName(obj, config, "spec", "template", "spec", "initContainers") if err != nil { a.logger.Infof("replace image name meet error: %v", err) return nil, errors.Wrap(err, "error getting item's spec.containers") } } return velero.NewRestoreItemActionExecuteOutput(obj), nil } func (a *ChangeImageNameAction) replaceImageName(obj *unstructured.Unstructured, config *corev1api.ConfigMap, filed ...string) error { log := a.logger.WithFields(map[string]any{ "kind": obj.GetKind(), "namespace": obj.GetNamespace(), "name": obj.GetName(), }) needUpdateObj := false containers, _, err := unstructured.NestedSlice(obj.UnstructuredContent(), filed...) if err != nil { log.Infof("UnstructuredConverter meet error: %v", err) return errors.Wrap(err, "error getting item's spec.containers") } if len(containers) == 0 { return nil } for i, container := range containers { log.Infoln("container:", container) if image, ok := container.(map[string]any)["image"]; ok { imageName := image.(string) if exists, newImageName, err := a.isImageReplaceRuleExist(log, imageName, config); exists && err == nil { needUpdateObj = true log.Infof("Updating item's image from %s to %s", imageName, newImageName) container.(map[string]any)["image"] = newImageName containers[i] = container } } } if needUpdateObj { if err := unstructured.SetNestedField(obj.UnstructuredContent(), containers, filed...); err != nil { return errors.Wrap(err, "unable to set item's initContainer image") } } return nil } func (a *ChangeImageNameAction) isImageReplaceRuleExist(log *logrus.Entry, oldImageName string, cm *corev1api.ConfigMap) (exists bool, newImageName string, err error) { if oldImageName == "" { log.Infoln("Item has no old image name specified") return false, "", nil } log.Debug("oldImageName: ", oldImageName) //how to use: "" //for current implementation the value can only be "," //e.x: in case your old image name is 1.1.1.1:5000/abc:test //"case1":"1.1.1.1:5000,2.2.2.2:3000" //"case2":"5000,3000" //"case3":"abc:test,edf:test" //"case4":"1.1.1.1:5000/abc:test,2.2.2.2:3000/edf:test" for _, row := range cm.Data { if !strings.Contains(row, delimiterValue) { continue } if strings.Contains(oldImageName, strings.TrimSpace(row[0:strings.Index(row, delimiterValue)])) && len(row[strings.Index(row, delimiterValue):]) > len(delimiterValue) { log.Infoln("match specific case:", row) oldImagePart := strings.TrimSpace(row[0:strings.Index(row, delimiterValue)]) newImagePart := strings.TrimSpace(row[strings.Index(row, delimiterValue)+len(delimiterValue):]) newImageName = strings.Replace(oldImageName, oldImagePart, newImagePart, -1) return true, newImageName, nil } } return false, "", errors.Errorf("No mapping rule found for image: %s", oldImageName) } ================================================ FILE: pkg/restore/actions/change_image_name_action_test.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actions import ( "testing" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes/fake" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/plugin/velero" ) // TestChangeImageRepositoryActionExecute runs the ChangeImageNameAction's Execute // method and validates that the item's image name is modified (or not) as expected. // Validation is done by comparing the result of the Execute method to the test case's // desired result. func TestChangeImageRepositoryActionExecute(t *testing.T) { tests := []struct { name string podOrObj any configMap *corev1api.ConfigMap freshedImageName string imageNameSlice []string want any wantErr error }{ { name: "a valid mapping with spaces for a new image repository is applied correctly", podOrObj: builder.ForPod("default", "pod1").ObjectMeta(). Containers(&corev1api.Container{ Name: "container1", Image: "1.1.1.1:5000/abc:test", }).Result(), configMap: builder.ForConfigMap("velero", "change-image-name"). ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/change-image-name", "RestoreItemAction")). Data("case1", "1.1.1.1:5000 , 2.2.2.2:3000"). Result(), freshedImageName: "2.2.2.2:3000/abc:test", want: "2.2.2.2:3000/abc:test", }, { name: "a valid mapping for a new image repository is applied correctly", podOrObj: builder.ForPod("default", "pod1").ObjectMeta(). Containers(&corev1api.Container{ Name: "container2", Image: "1.1.1.1:5000/abc:test", }).Result(), configMap: builder.ForConfigMap("velero", "change-image-name"). ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/change-image-name", "RestoreItemAction")). Data("specific", "1.1.1.1:5000,2.2.2.2:3000"). Result(), freshedImageName: "2.2.2.2:3000/abc:test", want: "2.2.2.2:3000/abc:test", }, { name: "a valid mapping for a new image name is applied correctly", podOrObj: builder.ForPod("default", "pod1").ObjectMeta(). Containers(&corev1api.Container{ Name: "container3", Image: "1.1.1.1:5000/abc:test", }).Result(), configMap: builder.ForConfigMap("velero", "change-image-name"). ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/change-image-name", "RestoreItemAction")). Data("specific", "abc:test,myproject:latest"). Result(), freshedImageName: "1.1.1.1:5000/myproject:latest", want: "1.1.1.1:5000/myproject:latest", }, { name: "a valid mapping for a new image repository port is applied correctly", podOrObj: builder.ForPod("default", "pod1").ObjectMeta(). Containers(&corev1api.Container{ Name: "container4", Image: "1.1.1.1:5000/abc:test", }).Result(), configMap: builder.ForConfigMap("velero", "change-image-name"). ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/change-image-name", "RestoreItemAction")). Data("specific", "5000,3333"). Result(), freshedImageName: "1.1.1.1:5000/abc:test", want: "1.1.1.1:3333/abc:test", }, { name: "a valid mapping for a new image tag is applied correctly", podOrObj: builder.ForPod("default", "pod1").ObjectMeta(). Containers(&corev1api.Container{ Name: "container5", Image: "1.1.1.1:5000/abc:test", }).Result(), configMap: builder.ForConfigMap("velero", "change-image-name"). ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/change-image-name", "RestoreItemAction")). Data("specific", "test,latest"). Result(), freshedImageName: "1.1.1.1:5000/abc:test", want: "1.1.1.1:5000/abc:latest", }, { name: "image name contains more than one part that matching the replacing words.", podOrObj: builder.ForPod("default", "pod1").ObjectMeta(). Containers(&corev1api.Container{ Name: "container6", Image: "dev/image1:dev", }).Result(), configMap: builder.ForConfigMap("velero", "change-image-name"). ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/change-image-name", "RestoreItemAction")). Data("specific", "dev/,test/"). Result(), freshedImageName: "dev/image1:dev", want: "test/image1:dev", }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { clientset := fake.NewSimpleClientset() a := NewChangeImageNameAction( logrus.StandardLogger(), clientset.CoreV1().ConfigMaps("velero"), ) // set up test data if tc.configMap != nil { _, err := clientset.CoreV1().ConfigMaps(tc.configMap.Namespace).Create(t.Context(), tc.configMap, metav1.CreateOptions{}) require.NoError(t, err) } unstructuredMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.podOrObj) require.NoError(t, err) input := &velero.RestoreItemActionExecuteInput{ Item: &unstructured.Unstructured{ Object: unstructuredMap, }, } // execute method under test res, err := a.Execute(input) // validate for both error and non-error cases switch { case tc.wantErr != nil: require.EqualError(t, err, tc.wantErr.Error()) default: require.NoError(t, err) pod := new(corev1api.Pod) err = runtime.DefaultUnstructuredConverter.FromUnstructured(res.UpdatedItem.UnstructuredContent(), pod) require.NoError(t, err) assert.Equal(t, tc.want, pod.Spec.Containers[0].Image) } }) } } ================================================ FILE: pkg/restore/actions/change_storageclass_action.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actions import ( "context" "github.com/pkg/errors" "github.com/sirupsen/logrus" appsv1api "k8s.io/api/apps/v1" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" storagev1client "k8s.io/client-go/kubernetes/typed/storage/v1" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" "github.com/vmware-tanzu/velero/pkg/plugin/velero" ) // ChangeStorageClassAction updates a PV or PVC's storage class name // if a mapping is found in the plugin's config map. type ChangeStorageClassAction struct { logger logrus.FieldLogger configMapClient corev1client.ConfigMapInterface storageClassClient storagev1client.StorageClassInterface } // NewChangeStorageClassAction is the constructor for ChangeStorageClassAction. func NewChangeStorageClassAction( logger logrus.FieldLogger, configMapClient corev1client.ConfigMapInterface, storageClassClient storagev1client.StorageClassInterface, ) *ChangeStorageClassAction { return &ChangeStorageClassAction{ logger: logger, configMapClient: configMapClient, storageClassClient: storageClassClient, } } // AppliesTo returns the resources that ChangeStorageClassAction should // be run for. func (a *ChangeStorageClassAction) AppliesTo() (velero.ResourceSelector, error) { return velero.ResourceSelector{ IncludedResources: []string{"persistentvolumeclaims", "persistentvolumes", "statefulsets"}, }, nil } // Execute updates the item's spec.storageClassName if a mapping is found // in the config map for the plugin. func (a *ChangeStorageClassAction) Execute(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { a.logger.Info("Executing ChangeStorageClassAction") defer a.logger.Info("Done executing ChangeStorageClassAction") a.logger.Debug("Getting plugin config") config, err := common.GetPluginConfig(common.PluginKindRestoreItemAction, "velero.io/change-storage-class", a.configMapClient) if err != nil { return nil, err } if config == nil || len(config.Data) == 0 { a.logger.Debug("No storage class mappings found") return velero.NewRestoreItemActionExecuteOutput(input.Item), nil } obj, ok := input.Item.(*unstructured.Unstructured) if !ok { return nil, errors.Errorf("object was of unexpected type %T", input.Item) } log := a.logger.WithFields(map[string]any{ "kind": obj.GetKind(), "namespace": obj.GetNamespace(), "name": obj.GetName(), }) // change StatefulSet volumeClaimTemplates storageClassName if obj.GetKind() == "StatefulSet" { sts := new(appsv1api.StatefulSet) if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), sts); err != nil { return nil, err } if len(sts.Spec.VolumeClaimTemplates) > 0 { for index, pvc := range sts.Spec.VolumeClaimTemplates { exists, newStorageClass, err := a.isStorageClassExist(log, pvc.Spec.StorageClassName, config) if err != nil { return nil, err } else if !exists { continue } log.Infof("Updating item's storage class name to %s", newStorageClass) sts.Spec.VolumeClaimTemplates[index].Spec.StorageClassName = &newStorageClass } newObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(sts) if err != nil { return nil, errors.Wrap(err, "convert obj to StatefulSet failed") } obj.Object = newObj } } else { // use the unstructured helpers here since this code is for both PVs and PVCs, and the // field names are the same for both types. storageClass, _, err := unstructured.NestedString(obj.UnstructuredContent(), "spec", "storageClassName") if err != nil { return nil, errors.Wrap(err, "error getting item's spec.storageClassName") } exists, newStorageClass, err := a.isStorageClassExist(log, &storageClass, config) if err != nil { return nil, err } else if !exists { return velero.NewRestoreItemActionExecuteOutput(input.Item), nil } log.Infof("Updating item's storage class name to %s", newStorageClass) if err := unstructured.SetNestedField(obj.UnstructuredContent(), newStorageClass, "spec", "storageClassName"); err != nil { return nil, errors.Wrap(err, "unable to set item's spec.storageClassName") } } return velero.NewRestoreItemActionExecuteOutput(obj), nil } func (a *ChangeStorageClassAction) isStorageClassExist(log *logrus.Entry, storageClass *string, cm *corev1api.ConfigMap) (exists bool, newStorageClass string, err error) { if storageClass == nil || *storageClass == "" { log.Debug("Item has no storage class specified") return false, "", nil } newStorageClass, ok := cm.Data[*storageClass] if !ok { log.Debugf("No mapping found for storage class %s", *storageClass) return false, "", nil } // validate that new storage class exists if _, err := a.storageClassClient.Get(context.TODO(), newStorageClass, metav1.GetOptions{}); err != nil { return false, "", errors.Wrapf(err, "error getting storage class %s from API", newStorageClass) } return true, newStorageClass, nil } ================================================ FILE: pkg/restore/actions/change_storageclass_action_test.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actions import ( "testing" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" storagev1api "k8s.io/api/storage/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes/fake" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/plugin/velero" ) // TestChangeStorageClassActionExecute runs the ChangeStorageClassAction's Execute // method and validates that the item's storage class is modified (or not) as expected. // Validation is done by comparing the result of the Execute method to the test case's // desired result. func TestChangeStorageClassActionExecute(t *testing.T) { tests := []struct { name string pvOrPvcOrSTS any configMap *corev1api.ConfigMap storageClass *storagev1api.StorageClass storageClassSlice []*storagev1api.StorageClass want any wantErr error }{ { name: "a valid mapping for a persistent volume is applied correctly", pvOrPvcOrSTS: builder.ForPersistentVolume("pv-1").StorageClass("storageclass-1").Result(), configMap: builder.ForConfigMap("velero", "change-storage-classs"). ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/change-storage-class", "RestoreItemAction")). Data("storageclass-1", "storageclass-2"). Result(), storageClass: builder.ForStorageClass("storageclass-2").Result(), want: builder.ForPersistentVolume("pv-1").StorageClass("storageclass-2").Result(), }, { name: "a valid mapping for a persistent volume claim is applied correctly", pvOrPvcOrSTS: builder.ForPersistentVolumeClaim("velero", "pvc-1").StorageClass("storageclass-1").Result(), configMap: builder.ForConfigMap("velero", "change-storage-classs"). ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/change-storage-class", "RestoreItemAction")). Data("storageclass-1", "storageclass-2"). Result(), storageClass: builder.ForStorageClass("storageclass-2").Result(), want: builder.ForPersistentVolumeClaim("velero", "pvc-1").StorageClass("storageclass-2").Result(), }, { name: "when no config map exists for the plugin, the item is returned as-is", pvOrPvcOrSTS: builder.ForPersistentVolume("pv-1").StorageClass("storageclass-1").Result(), configMap: builder.ForConfigMap("velero", "change-storage-classs"). ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/some-other-plugin", "RestoreItemAction")). Data("storageclass-1", "storageclass-2"). Result(), want: builder.ForPersistentVolume("pv-1").StorageClass("storageclass-1").Result(), }, { name: "when no storage class mappings exist in the plugin config map, the item is returned as-is", pvOrPvcOrSTS: builder.ForPersistentVolume("pv-1").StorageClass("storageclass-1").Result(), configMap: builder.ForConfigMap("velero", "change-storage-classs"). ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/change-storage-class", "RestoreItemAction")). Result(), want: builder.ForPersistentVolume("pv-1").StorageClass("storageclass-1").Result(), }, { name: "when persistent volume has no storage class, the item is returned as-is", pvOrPvcOrSTS: builder.ForPersistentVolume("pv-1").Result(), configMap: builder.ForConfigMap("velero", "change-storage-classs"). ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/change-storage-class", "RestoreItemAction")). Data("storageclass-1", "storageclass-2"). Result(), want: builder.ForPersistentVolume("pv-1").Result(), }, { name: "when persistent volume claim has no storage class, the item is returned as-is", pvOrPvcOrSTS: builder.ForPersistentVolumeClaim("velero", "pvc-1").Result(), configMap: builder.ForConfigMap("velero", "change-storage-classs"). ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/change-storage-class", "RestoreItemAction")). Data("storageclass-1", "storageclass-2"). Result(), want: builder.ForPersistentVolumeClaim("velero", "pvc-1").Result(), }, { name: "when persistent volume's storage class has no mapping in the config map, the item is returned as-is", pvOrPvcOrSTS: builder.ForPersistentVolume("pv-1").StorageClass("storageclass-1").Result(), configMap: builder.ForConfigMap("velero", "change-storage-classs"). ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/change-storage-class", "RestoreItemAction")). Data("storageclass-3", "storageclass-4"). Result(), want: builder.ForPersistentVolume("pv-1").StorageClass("storageclass-1").Result(), }, { name: "when persistent volume claim's storage class has no mapping in the config map, the item is returned as-is", pvOrPvcOrSTS: builder.ForPersistentVolumeClaim("velero", "pvc-1").StorageClass("storageclass-1").Result(), configMap: builder.ForConfigMap("velero", "change-storage-classs"). ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/change-storage-class", "RestoreItemAction")). Data("storageclass-3", "storageclass-4"). Result(), want: builder.ForPersistentVolumeClaim("velero", "pvc-1").StorageClass("storageclass-1").Result(), }, { name: "when persistent volume's storage class is mapped to a nonexistent storage class, an error is returned", pvOrPvcOrSTS: builder.ForPersistentVolume("pv-1").StorageClass("storageclass-1").Result(), configMap: builder.ForConfigMap("velero", "change-storage-classs"). ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/change-storage-class", "RestoreItemAction")). Data("storageclass-1", "nonexistent-storage-class"). Result(), wantErr: errors.New("error getting storage class nonexistent-storage-class from API: storageclasses.storage.k8s.io \"nonexistent-storage-class\" not found"), }, { name: "when persistent volume claim's storage class is mapped to a nonexistent storage class, an error is returned", pvOrPvcOrSTS: builder.ForPersistentVolumeClaim("velero", "pvc-1").StorageClass("storageclass-1").Result(), configMap: builder.ForConfigMap("velero", "change-storage-classs"). ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/change-storage-class", "RestoreItemAction")). Data("storageclass-1", "nonexistent-storage-class"). Result(), wantErr: errors.New("error getting storage class nonexistent-storage-class from API: storageclasses.storage.k8s.io \"nonexistent-storage-class\" not found"), }, { name: "when statefulset's VolumeClaimTemplates has only one pvc, a valid mapping for a statefulset is applied correctly", pvOrPvcOrSTS: builder.ForStatefulSet("velero", "sts-1").StorageClass("storageclass-1").Result(), configMap: builder.ForConfigMap("velero", "change-storage-classs"). ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/change-storage-class", "RestoreItemAction")). Data("storageclass-1", "storageclass-2"). Result(), storageClass: builder.ForStorageClass("storageclass-2").Result(), want: builder.ForStatefulSet("velero", "sts-1").StorageClass("storageclass-2").Result(), }, { name: "when statefulset's VolumeClaimTemplates has more than one same pvc's storageClassName, a valid mapping for a statefulset is applied correctly", pvOrPvcOrSTS: builder.ForStatefulSet("velero", "sts-1").StorageClass("storageclass-1", "storageclass-1").Result(), configMap: builder.ForConfigMap("velero", "change-storage-classs"). ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/change-storage-class", "RestoreItemAction")). Data("storageclass-1", "storageclass-2", "storageclass-3", "storageclass-4"). Result(), storageClass: builder.ForStorageClass("storageclass-2").Result(), want: builder.ForStatefulSet("velero", "sts-1").StorageClass("storageclass-2", "storageclass-2").Result(), }, { name: "when statefulset's VolumeClaimTemplates has more than one different pvc's storageClassName, a valid mapping for a statefulset is applied correctly", pvOrPvcOrSTS: builder.ForStatefulSet("velero", "sts-1").StorageClass("storageclass-1", "storageclass-2", "storageclass-3").Result(), configMap: builder.ForConfigMap("velero", "change-storage-classs"). ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/change-storage-class", "RestoreItemAction")). Data("storageclass-1", "storageclass-a", "storageclass-2", "storageclass-b", "storageclass-3", "storageclass-c"). Result(), storageClassSlice: builder.ForStorageClassSlice("storageclass-a", "storageclass-b", "storageclass-c").SliceResult(), want: builder.ForStatefulSet("velero", "sts-1").StorageClass("storageclass-a", "storageclass-b", "storageclass-c").Result(), }, { name: "when no config map exists for the plugin, the statefulset item is returned as-is", pvOrPvcOrSTS: builder.ForStatefulSet("velero", "sts-1").StorageClass("storageclass-1").Result(), configMap: builder.ForConfigMap("velero", "change-storage-classs"). ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/some-other-plugin", "RestoreItemAction")). Data("storageclass-1", "storageclass-2"). Result(), want: builder.ForStatefulSet("velero", "sts-1").StorageClass("storageclass-1").Result(), }, { name: "when no storage class mappings exist in the plugin config map, the statefulset item is returned as-is", pvOrPvcOrSTS: builder.ForStatefulSet("velero", "sts-1").StorageClass("storageclass-1").Result(), configMap: builder.ForConfigMap("velero", "change-storage-classs"). ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/change-storage-class", "RestoreItemAction")). Result(), want: builder.ForStatefulSet("velero", "sts-1").StorageClass("storageclass-1").Result(), }, { name: "when persistent volume claim has no storage class, the statefulset item is returned as-is", pvOrPvcOrSTS: builder.ForStatefulSet("velero", "sts-1").Result(), configMap: builder.ForConfigMap("velero", "change-storage-classs"). ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/change-storage-class", "RestoreItemAction")). Result(), want: builder.ForStatefulSet("velero", "sts-1").Result(), }, { name: "when statefulset's storage class has no mapping in the config map, the item is returned as-is", pvOrPvcOrSTS: builder.ForStatefulSet("velero", "sts-1").StorageClass("storageclass-1").Result(), configMap: builder.ForConfigMap("velero", "change-storage-classs"). ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/change-storage-class", "RestoreItemAction")). Data("storageclass-3", "storageclass-4"). Result(), want: builder.ForStatefulSet("velero", "sts-1").StorageClass("storageclass-1").Result(), }, { name: "when statefulset's storage class is mapped to a nonexistent storage class, an error is returned", pvOrPvcOrSTS: builder.ForStatefulSet("velero", "sts-1").StorageClass("storageclass-1").Result(), configMap: builder.ForConfigMap("velero", "change-storage-classs"). ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/change-storage-class", "RestoreItemAction")). Data("storageclass-1", "nonexistent-storage-class"). Result(), wantErr: errors.New("error getting storage class nonexistent-storage-class from API: storageclasses.storage.k8s.io \"nonexistent-storage-class\" not found"), }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { clientset := fake.NewSimpleClientset() a := NewChangeStorageClassAction( logrus.StandardLogger(), clientset.CoreV1().ConfigMaps("velero"), clientset.StorageV1().StorageClasses(), ) // set up test data if tc.configMap != nil { _, err := clientset.CoreV1().ConfigMaps(tc.configMap.Namespace).Create(t.Context(), tc.configMap, metav1.CreateOptions{}) require.NoError(t, err) } if tc.storageClass != nil { _, err := clientset.StorageV1().StorageClasses().Create(t.Context(), tc.storageClass, metav1.CreateOptions{}) require.NoError(t, err) } if tc.storageClassSlice != nil { for _, storageClass := range tc.storageClassSlice { _, err := clientset.StorageV1().StorageClasses().Create(t.Context(), storageClass, metav1.CreateOptions{}) require.NoError(t, err) } } unstructuredMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.pvOrPvcOrSTS) require.NoError(t, err) input := &velero.RestoreItemActionExecuteInput{ Item: &unstructured.Unstructured{ Object: unstructuredMap, }, } // execute method under test res, err := a.Execute(input) // validate for both error and non-error cases switch { case tc.wantErr != nil: require.EqualError(t, err, tc.wantErr.Error()) default: require.NoError(t, err) wantUnstructured, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.want) require.NoError(t, err) assert.Equal(t, &unstructured.Unstructured{Object: wantUnstructured}, res.UpdatedItem) } }) } } ================================================ FILE: pkg/restore/actions/clusterrolebinding_action.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actions import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "github.com/vmware-tanzu/velero/pkg/plugin/velero" ) // ClusterRoleBindingAction handle namespace remappings for role bindings type ClusterRoleBindingAction struct { logger logrus.FieldLogger } func NewClusterRoleBindingAction(logger logrus.FieldLogger) *ClusterRoleBindingAction { return &ClusterRoleBindingAction{logger: logger} } func (a *ClusterRoleBindingAction) AppliesTo() (velero.ResourceSelector, error) { return velero.ResourceSelector{ IncludedResources: []string{"clusterrolebindings"}, }, nil } func (a *ClusterRoleBindingAction) Execute(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { namespaceMapping := input.Restore.Spec.NamespaceMapping if len(namespaceMapping) == 0 { return velero.NewRestoreItemActionExecuteOutput(&unstructured.Unstructured{Object: input.Item.UnstructuredContent()}), nil } clusterRoleBinding := new(rbacv1.ClusterRoleBinding) if err := runtime.DefaultUnstructuredConverter.FromUnstructured(input.Item.UnstructuredContent(), clusterRoleBinding); err != nil { return nil, errors.WithStack(err) } for i, subject := range clusterRoleBinding.Subjects { if newNamespace, ok := namespaceMapping[subject.Namespace]; ok { clusterRoleBinding.Subjects[i].Namespace = newNamespace } } res, err := runtime.DefaultUnstructuredConverter.ToUnstructured(clusterRoleBinding) if err != nil { return nil, errors.WithStack(err) } return velero.NewRestoreItemActionExecuteOutput(&unstructured.Unstructured{Object: res}), nil } ================================================ FILE: pkg/restore/actions/clusterrolebinding_action_test.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actions import ( "sort" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/plugin/velero" "github.com/vmware-tanzu/velero/pkg/test" ) func TestClusterRoleBindingActionAppliesTo(t *testing.T) { action := NewClusterRoleBindingAction(test.NewLogger()) actual, err := action.AppliesTo() require.NoError(t, err) assert.Equal(t, velero.ResourceSelector{IncludedResources: []string{"clusterrolebindings"}}, actual) } func TestClusterRoleBindingActionExecute(t *testing.T) { tests := []struct { name string namespaces []string namespaceMapping map[string]string expected []string }{ { name: "namespace mapping disabled", namespaces: []string{"foo"}, namespaceMapping: map[string]string{}, expected: []string{"foo"}, }, { name: "namespace mapping enabled", namespaces: []string{"foo"}, namespaceMapping: map[string]string{"foo": "bar", "fizz": "buzz"}, expected: []string{"bar"}, }, { name: "namespace mapping enabled, not included namespace remains unchanged", namespaces: []string{"foo", "xyz"}, namespaceMapping: map[string]string{"foo": "bar", "fizz": "buzz"}, expected: []string{"bar", "xyz"}, }, { name: "namespace mapping enabled, not included namespace remains unchanged", namespaces: []string{"foo", "xyz"}, namespaceMapping: map[string]string{"a": "b", "c": "d"}, expected: []string{"foo", "xyz"}, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { subjects := []rbacv1.Subject{} for _, ns := range tc.namespaces { subjects = append(subjects, rbacv1.Subject{ Namespace: ns, }) } clusterRoleBinding := rbacv1.ClusterRoleBinding{ Subjects: subjects, } roleBindingUnstructured, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&clusterRoleBinding) require.NoError(t, err) action := NewClusterRoleBindingAction(test.NewLogger()) res, err := action.Execute(&velero.RestoreItemActionExecuteInput{ Item: &unstructured.Unstructured{Object: roleBindingUnstructured}, ItemFromBackup: &unstructured.Unstructured{Object: roleBindingUnstructured}, Restore: &api.Restore{ Spec: api.RestoreSpec{ NamespaceMapping: tc.namespaceMapping, }, }, }) require.NoError(t, err) var resClusterRoleBinding *rbacv1.ClusterRoleBinding err = runtime.DefaultUnstructuredConverter.FromUnstructured(res.UpdatedItem.UnstructuredContent(), &resClusterRoleBinding) require.NoError(t, err) actual := []string{} for _, subject := range resClusterRoleBinding.Subjects { actual = append(actual, subject.Namespace) } sort.Strings(tc.expected) sort.Strings(actual) assert.Equal(t, tc.expected, actual) }) } } ================================================ FILE: pkg/restore/actions/crd_v1_preserve_unknown_fields_action.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actions import ( "encoding/json" "github.com/pkg/errors" "github.com/sirupsen/logrus" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "github.com/vmware-tanzu/velero/pkg/plugin/velero" ) // CRDV1PreserveUnknownFieldsAction will take a CRD and inspect it for the API version and the PreserveUnknownFields value. // If the API Version is 1 and the PreserveUnknownFields value is True, then the x-preserve-unknown-fields value in the OpenAPIV3 schema will be set to True // and PreserveUnknownFields set to False in order to allow Kubernetes 1.16+ servers to accept the object. type CRDV1PreserveUnknownFieldsAction struct { logger logrus.FieldLogger } func NewCRDV1PreserveUnknownFieldsAction(logger logrus.FieldLogger) *CRDV1PreserveUnknownFieldsAction { return &CRDV1PreserveUnknownFieldsAction{logger: logger} } func (c *CRDV1PreserveUnknownFieldsAction) AppliesTo() (velero.ResourceSelector, error) { return velero.ResourceSelector{ IncludedResources: []string{"customresourcedefinition.apiextensions.k8s.io"}, }, nil } func (c *CRDV1PreserveUnknownFieldsAction) Execute(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { c.logger.Info("Executing CRDV1PreserveUnknownFieldsAction") name, _, err := unstructured.NestedString(input.Item.UnstructuredContent(), "name") if err != nil { return nil, errors.Wrap(err, "could not get CRD name") } log := c.logger.WithField("plugin", "CRDV1PreserveUnknownFieldsAction").WithField("CRD", name) version, _, err := unstructured.NestedString(input.Item.UnstructuredContent(), "apiVersion") if err != nil { return nil, errors.Wrap(err, "could not get CRD version") } // We don't want to "fix" anything in beta CRDs at the moment, just v1 versions with preserveunknownfields = true if version != "apiextensions.k8s.io/v1" { return &velero.RestoreItemActionExecuteOutput{ UpdatedItem: input.Item, }, nil } // Do not use runtime.DefaultUnstructuredConverter.FromUnstructured here because it has a bug when converting integers/whole // numbers in float fields (https://github.com/kubernetes/kubernetes/issues/87675). // Using JSON as a go-between avoids this issue, without adding a bunch of type conversion by using unstructured helper functions // to inspect the fields we want to look at. crd, err := fromUnstructured(input.Item.UnstructuredContent()) if err != nil { return nil, errors.Wrap(err, "unable to convert CRD from unstructured to structured") } // The v1 API doesn't allow the PreserveUnknownFields value to be true, so make sure the schema flag is set instead if crd.Spec.PreserveUnknownFields { // First, change the top-level value since the Kubernetes API server on 1.16+ will generate errors otherwise. log.Debug("Set PreserveUnknownFields to False") crd.Spec.PreserveUnknownFields = false // Make sure all versions are set to preserve unknown fields for _, v := range crd.Spec.Versions { // If the schema fields are nil, there are no nested fields to set, so skip over it for this version. if v.Schema == nil || v.Schema.OpenAPIV3Schema == nil { continue } // Use the address, since the XPreserveUnknownFields value is nil or // a pointer to true (false is not allowed) preserve := true v.Schema.OpenAPIV3Schema.XPreserveUnknownFields = &preserve log.Debugf("Set x-preserve-unknown-fields in Open API for schema version %s", v.Name) } } res, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&crd) if err != nil { return nil, errors.Wrap(err, "unable to convert crd to runtime.Unstructured") } return &velero.RestoreItemActionExecuteOutput{ UpdatedItem: &unstructured.Unstructured{Object: res}, }, nil } func fromUnstructured(unstructured map[string]any) (*apiextv1.CustomResourceDefinition, error) { var crd apiextv1.CustomResourceDefinition js, err := json.Marshal(unstructured) if err != nil { return nil, errors.Wrap(err, "unable to convert unstructured item to JSON") } if err = json.Unmarshal(js, &crd); err != nil { return nil, errors.Wrap(err, "unable to convert JSON to CRD Go type") } return &crd, nil } ================================================ FILE: pkg/restore/actions/crd_v1_preserve_unknown_fields_action_test.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actions import ( "encoding/json" "testing" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/plugin/velero" "github.com/vmware-tanzu/velero/pkg/test" ) func TestExecuteForACRDWithAnIntOnAFloat64FieldShouldWork(t *testing.T) { // ref. reopen of https://github.com/vmware-tanzu/velero/issues/2319 b := builder.ForV1CustomResourceDefinition("test.velero.io") // 5 here is just an int value, it could be any other whole number. schema := builder.ForJSONSchemaPropsBuilder().Maximum(5).Result() b.Version(builder.ForV1CustomResourceDefinitionVersion("v1").Served(true).Storage(true).Schema(schema).Result()) c := b.Result() // Marshall in and out of JSON because the problem doesn't manifest when we use ToUnstructured directly // This should simulate the JSON passing over the wire in an HTTP request/response with a dynamic client js, err := json.Marshal(c) require.NoError(t, err) var u unstructured.Unstructured err = json.Unmarshal(js, &u) require.NoError(t, err) a := NewCRDV1PreserveUnknownFieldsAction(test.NewLogger()) _, err = a.Execute(&velero.RestoreItemActionExecuteInput{Item: &u}) require.NoError(t, err) } ================================================ FILE: pkg/restore/actions/csi/pvc_action.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package csi import ( "context" "encoding/json" "fmt" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" utilrand "k8s.io/apimachinery/pkg/util/rand" crclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" "github.com/vmware-tanzu/velero/pkg/client" kuberesource "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/label" plugincommon "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" "github.com/vmware-tanzu/velero/pkg/plugin/velero" riav2 "github.com/vmware-tanzu/velero/pkg/plugin/velero/restoreitemaction/v2" uploaderUtil "github.com/vmware-tanzu/velero/pkg/uploader/util" "github.com/vmware-tanzu/velero/pkg/util" "github.com/vmware-tanzu/velero/pkg/util/boolptr" ) const ( AnnSelectedNode = "volume.kubernetes.io/selected-node" GenerateNameRandomLength = 5 ) // pvcRestoreItemAction is a restore item action plugin for Velero type pvcRestoreItemAction struct { log logrus.FieldLogger crClient crclient.Client } // AppliesTo returns information indicating that the // PVCRestoreItemAction should be run while restoring PVCs. func (p *pvcRestoreItemAction) AppliesTo() (velero.ResourceSelector, error) { return velero.ResourceSelector{ IncludedResources: []string{"persistentvolumeclaims"}, //TODO: add label selector volumeSnapshotLabel }, nil } // Execute modifies the PVC's spec to use the VolumeSnapshot object as the // data source ensuring that the newly provisioned volume can be pre-populated // with data from the VolumeSnapshot. func (p *pvcRestoreItemAction) Execute( input *velero.RestoreItemActionExecuteInput, ) (*velero.RestoreItemActionExecuteOutput, error) { var pvc, pvcFromBackup corev1api.PersistentVolumeClaim if err := runtime.DefaultUnstructuredConverter.FromUnstructured( input.Item.UnstructuredContent(), &pvc); err != nil { return nil, errors.WithStack(err) } if err := runtime.DefaultUnstructuredConverter.FromUnstructured( input.ItemFromBackup.UnstructuredContent(), &pvcFromBackup); err != nil { return nil, errors.WithStack(err) } logger := p.log.WithFields(logrus.Fields{ "Action": "PVCRestoreItemAction", "PVC": pvc.Namespace + "/" + pvc.Name, "Restore": input.Restore.Namespace + "/" + input.Restore.Name, }) logger.Info("Starting PVCRestoreItemAction for PVC") // If PVC already exists, returns early. if p.isResourceExist(pvc, *input.Restore) { logger.Warnf("PVC already exists. Skip restore this PVC.") return &velero.RestoreItemActionExecuteOutput{ UpdatedItem: input.Item, }, nil } // If cross-namespace restore is configured, change the namespace // for PVC object to be restored newNamespace, ok := input.Restore.Spec.NamespaceMapping[pvc.GetNamespace()] if !ok { // Use original namespace newNamespace = pvc.Namespace } operationID := "" additionalItems := []velero.ResourceIdentifier{} if boolptr.IsSetToFalse(input.Restore.Spec.RestorePVs) { logger.Info("Restore did not request for PVs to be restored from snapshot") pvc.Spec.VolumeName = "" pvc.Spec.DataSource = nil pvc.Spec.DataSourceRef = nil } else { backup := new(velerov1api.Backup) err := p.crClient.Get( context.TODO(), crclient.ObjectKey{ Namespace: input.Restore.Namespace, Name: input.Restore.Spec.BackupName, }, backup, ) if err != nil { logger.Error("Fail to get backup for restore.") return nil, fmt.Errorf("fail to get backup for restore: %s", err.Error()) } if boolptr.IsSetToTrue(backup.Spec.SnapshotMoveData) { logger.Info("Start DataMover restore.") // If PVC doesn't have a DataUploadNameLabel, which should be created // during backup, then CSI cannot handle the volume during to restore, // so return early to let Velero tries to fall back to Velero native snapshot. if _, ok := pvcFromBackup.Annotations[velerov1api.DataUploadNameAnnotation]; !ok { logger.Warnf("PVC doesn't have a DataUpload for data mover. Return.") return &velero.RestoreItemActionExecuteOutput{ UpdatedItem: input.Item, }, nil } operationID = label.GetValidName( string(velerov1api.AsyncOperationIDPrefixDataDownload) + string(input.Restore.UID) + "." + string(pvcFromBackup.UID)) dataDownload, err := restoreFromDataUploadResult( context.Background(), input.Restore, backup, &pvc, newNamespace, operationID, p.crClient) if err != nil { logger.Errorf("Fail to restore from DataUploadResult: %s", err.Error()) return nil, errors.WithStack(err) } logger.Infof("DataDownload %s/%s is created successfully.", dataDownload.Namespace, dataDownload.Name) } else { //CSI restore vsName, nameOK := pvcFromBackup.Annotations[velerov1api.VolumeSnapshotLabel] if !nameOK { logger.Info("Skipping PVCRestoreItemAction for PVC, PVC does not have a CSI VolumeSnapshot.") return &velero.RestoreItemActionExecuteOutput{ UpdatedItem: input.Item, }, nil } //To avoid confilcs, vs and vsc get a new uniq name based in restore UID // and vs name old name newVSName := util.GenerateSha256FromRestoreUIDAndVsName(string(input.Restore.UID), vsName) p.log.Debugf("Setting PVC source to VolumeSnapshot new name: %s", newVSName) resetPVCSourceToVolumeSnapshot(&pvc, newVSName) additionalItems = append(additionalItems, velero.ResourceIdentifier{ GroupResource: kuberesource.VolumeSnapshots, Name: vsName, Namespace: pvc.Namespace, }) } } pvcMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&pvc) if err != nil { return nil, errors.WithStack(err) } logger.Info("Returning from PVCRestoreItemAction for PVC") return &velero.RestoreItemActionExecuteOutput{ UpdatedItem: &unstructured.Unstructured{Object: pvcMap}, OperationID: operationID, AdditionalItems: additionalItems, }, nil } func resetPVCSourceToVolumeSnapshot(pvc *corev1api.PersistentVolumeClaim, vsName string) { // Restore operation for the PVC will use the VolumeSnapshot as the data source. // So clear out the volume name, which is a ref to the PV pvc.Spec.VolumeName = "" dataSource := &corev1api.TypedLocalObjectReference{ APIGroup: &snapshotv1api.SchemeGroupVersion.Group, Kind: "VolumeSnapshot", Name: vsName, } pvc.Spec.DataSource = dataSource pvc.Spec.DataSourceRef = nil } func (p *pvcRestoreItemAction) Name() string { return "PVCRestoreItemAction" } func (p *pvcRestoreItemAction) Progress( operationID string, restore *velerov1api.Restore, ) (velero.OperationProgress, error) { progress := velero.OperationProgress{} if operationID == "" { return progress, riav2.InvalidOperationIDError(operationID) } logger := p.log.WithFields(logrus.Fields{ "Action": "PVCRestoreItemAction", "OperationID": operationID, "Namespace": restore.Namespace, }) dataDownload, err := getDataDownload( context.Background(), restore.Namespace, operationID, p.crClient, ) if err != nil { logger.Errorf("fail to get DataDownload: %s", err.Error()) return progress, err } if dataDownload.Status.Phase == velerov2alpha1.DataDownloadPhaseNew || dataDownload.Status.Phase == "" { logger.Debugf("DataDownload is still not processed yet. Skip progress update.") return progress, nil } progress.Description = string(dataDownload.Status.Phase) progress.OperationUnits = "Bytes" progress.NCompleted = dataDownload.Status.Progress.BytesDone progress.NTotal = dataDownload.Status.Progress.TotalBytes if dataDownload.Status.StartTimestamp != nil { progress.Started = dataDownload.Status.StartTimestamp.Time } if dataDownload.Status.CompletionTimestamp != nil { progress.Updated = dataDownload.Status.CompletionTimestamp.Time } if dataDownload.Status.Phase == velerov2alpha1.DataDownloadPhaseCompleted { progress.Completed = true } else if dataDownload.Status.Phase == velerov2alpha1.DataDownloadPhaseCanceled { progress.Completed = true progress.Err = "DataDownload is canceled" } else if dataDownload.Status.Phase == velerov2alpha1.DataDownloadPhaseFailed { progress.Completed = true progress.Err = dataDownload.Status.Message } return progress, nil } func (p *pvcRestoreItemAction) Cancel( operationID string, restore *velerov1api.Restore) error { if operationID == "" { return riav2.InvalidOperationIDError(operationID) } logger := p.log.WithFields(logrus.Fields{ "Action": "PVCRestoreItemAction", "OperationID": operationID, "Namespace": restore.Namespace, }) dataDownload, err := getDataDownload( context.Background(), restore.Namespace, operationID, p.crClient, ) if err != nil { logger.Errorf("fail to get DataDownload: %s", err.Error()) return err } err = cancelDataDownload(context.Background(), p.crClient, dataDownload) if err != nil { logger.Errorf("fail to cancel DataDownload %s: %s", dataDownload.Name, err.Error()) } return err } func (p *pvcRestoreItemAction) AreAdditionalItemsReady( additionalItems []velero.ResourceIdentifier, restore *velerov1api.Restore, ) (bool, error) { return true, nil } func getDataUploadResult( ctx context.Context, restore *velerov1api.Restore, pvc *corev1api.PersistentVolumeClaim, crClient crclient.Client, ) (*velerov2alpha1.DataUploadResult, error) { selectorStr := fmt.Sprintf("%s=%s,%s=%s,%s=%s", velerov1api.PVCNamespaceNameLabel, label.GetValidName(pvc.Namespace+"."+pvc.Name), velerov1api.RestoreUIDLabel, label.GetValidName(string(restore.UID)), velerov1api.ResourceUsageLabel, label.GetValidName(string(velerov1api.VeleroResourceUsageDataUploadResult)), ) selector, _ := labels.Parse(selectorStr) cmList := new(corev1api.ConfigMapList) if err := crClient.List( ctx, cmList, &crclient.ListOptions{ LabelSelector: selector, Namespace: restore.Namespace, }); err != nil { return nil, errors.Wrapf(err, "error to get DataUpload result cm with labels %s", selectorStr) } if len(cmList.Items) == 0 { return nil, errors.Errorf( "no DataUpload result cm found with labels %s", selectorStr) } if len(cmList.Items) > 1 { return nil, errors.Errorf( "multiple DataUpload result cms found with labels %s", selectorStr) } jsonBytes, exist := cmList.Items[0].Data[string(restore.UID)] if !exist { return nil, errors.Errorf( "no DataUpload result found with restore key %s, restore %s", string(restore.UID), restore.Name) } result := velerov2alpha1.DataUploadResult{} if err := json.Unmarshal([]byte(jsonBytes), &result); err != nil { return nil, errors.Errorf( "error to unmarshal DataUploadResult, restore UID %s, restore name %s", string(restore.UID), restore.Name) } return &result, nil } func getDataDownload( ctx context.Context, namespace string, operationID string, crClient crclient.Client, ) (*velerov2alpha1.DataDownload, error) { dataDownloadList := new(velerov2alpha1.DataDownloadList) err := crClient.List(ctx, dataDownloadList, &crclient.ListOptions{ LabelSelector: labels.SelectorFromSet(map[string]string{ velerov1api.AsyncOperationIDLabel: operationID, }), Namespace: namespace, }) if err != nil { return nil, errors.Wrap(err, "fail to list DataDownload") } if len(dataDownloadList.Items) == 0 { return nil, errors.Errorf("didn't find DataDownload") } if len(dataDownloadList.Items) > 1 { return nil, errors.Errorf("find multiple DataDownloads") } return &dataDownloadList.Items[0], nil } func cancelDataDownload(ctx context.Context, crClient crclient.Client, dataDownload *velerov2alpha1.DataDownload) error { updatedDataDownload := dataDownload.DeepCopy() updatedDataDownload.Spec.Cancel = true return crClient.Patch(ctx, updatedDataDownload, crclient.MergeFrom(dataDownload)) } func newDataDownload( restore *velerov1api.Restore, backup *velerov1api.Backup, dataUploadResult *velerov2alpha1.DataUploadResult, pvc *corev1api.PersistentVolumeClaim, newNamespace, operationID string, ) *velerov2alpha1.DataDownload { dataDownload := &velerov2alpha1.DataDownload{ TypeMeta: metav1.TypeMeta{ APIVersion: velerov2alpha1.SchemeGroupVersion.String(), Kind: "DataDownload", }, ObjectMeta: metav1.ObjectMeta{ Namespace: restore.Namespace, GenerateName: restore.Name + "-", OwnerReferences: []metav1.OwnerReference{ { APIVersion: velerov1api.SchemeGroupVersion.String(), Kind: "Restore", Name: restore.Name, UID: restore.UID, Controller: boolptr.True(), }, }, Labels: map[string]string{ velerov1api.RestoreNameLabel: label.GetValidName(restore.Name), velerov1api.RestoreUIDLabel: string(restore.UID), velerov1api.AsyncOperationIDLabel: operationID, }, }, Spec: velerov2alpha1.DataDownloadSpec{ TargetVolume: velerov2alpha1.TargetVolumeSpec{ PVC: pvc.Name, Namespace: newNamespace, }, BackupStorageLocation: dataUploadResult.BackupStorageLocation, DataMover: dataUploadResult.DataMover, SnapshotID: dataUploadResult.SnapshotID, SnapshotSize: dataUploadResult.SnapshotSize, SourceNamespace: dataUploadResult.SourceNamespace, OperationTimeout: backup.Spec.CSISnapshotTimeout, NodeOS: dataUploadResult.NodeOS, }, } if restore.Spec.UploaderConfig != nil { dataDownload.Spec.DataMoverConfig = uploaderUtil.StoreRestoreConfig(restore.Spec.UploaderConfig) } return dataDownload } func restoreFromDataUploadResult( ctx context.Context, restore *velerov1api.Restore, backup *velerov1api.Backup, pvc *corev1api.PersistentVolumeClaim, newNamespace, operationID string, crClient crclient.Client, ) (*velerov2alpha1.DataDownload, error) { dataUploadResult, err := getDataUploadResult(ctx, restore, pvc, crClient) if err != nil { return nil, errors.Wrapf(err, "fail get DataUploadResult for restore: %s", restore.Name) } pvc.Spec.VolumeName = "" if pvc.Spec.Selector == nil { pvc.Spec.Selector = &metav1.LabelSelector{} } if pvc.Spec.Selector.MatchLabels == nil { pvc.Spec.Selector.MatchLabels = make(map[string]string) } pvc.Spec.Selector.MatchLabels[velerov1api.DynamicPVRestoreLabel] = label. GetValidName(fmt.Sprintf("%s.%s.%s", newNamespace, pvc.Name, utilrand.String(GenerateNameRandomLength))) dataDownload := newDataDownload( restore, backup, dataUploadResult, pvc, newNamespace, operationID, ) err = crClient.Create(ctx, dataDownload) if err != nil { return nil, errors.Wrapf(err, "fail to create DataDownload") } return dataDownload, nil } func (p *pvcRestoreItemAction) isResourceExist( pvc corev1api.PersistentVolumeClaim, restore velerov1api.Restore, ) bool { // get target namespace to restore into, if different from source namespace targetNamespace := pvc.Namespace if target, ok := restore.Spec.NamespaceMapping[pvc.Namespace]; ok { targetNamespace = target } tmpPVC := new(corev1api.PersistentVolumeClaim) if err := p.crClient.Get( context.Background(), crclient.ObjectKey{ Name: pvc.Name, Namespace: targetNamespace, }, tmpPVC, ); err == nil { return true } return false } func NewPvcRestoreItemAction(f client.Factory) plugincommon.HandlerInitializer { return func(logger logrus.FieldLogger) (any, error) { crClient, err := f.KubebuilderClient() if err != nil { return nil, err } return &pvcRestoreItemAction{ log: logger, crClient: crClient, }, nil } } ================================================ FILE: pkg/restore/actions/csi/pvc_action_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package csi import ( "fmt" "testing" "time" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/validation" crclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/pkg/apis/velero/shared" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" "github.com/vmware-tanzu/velero/pkg/builder" factorymocks "github.com/vmware-tanzu/velero/pkg/client/mocks" "github.com/vmware-tanzu/velero/pkg/label" "github.com/vmware-tanzu/velero/pkg/plugin/velero" velerotest "github.com/vmware-tanzu/velero/pkg/test" "github.com/vmware-tanzu/velero/pkg/util" "github.com/vmware-tanzu/velero/pkg/util/boolptr" ) func TestResetPVCSpec(t *testing.T) { fileMode := corev1api.PersistentVolumeFilesystem blockMode := corev1api.PersistentVolumeBlock testCases := []struct { name string pvc corev1api.PersistentVolumeClaim vsName string }{ { name: "should reset expected fields in pvc using file mode volumes", pvc: corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: "test-pvc", Namespace: "test-ns", }, Spec: corev1api.PersistentVolumeClaimSpec{ AccessModes: []corev1api.PersistentVolumeAccessMode{corev1api.ReadOnlyMany, corev1api.ReadWriteMany, corev1api.ReadWriteOnce}, Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "foo": "bar", "baz": "qux", }, }, Resources: corev1api.VolumeResourceRequirements{ Requests: corev1api.ResourceList{ corev1api.ResourceCPU: resource.Quantity{ Format: resource.DecimalExponent, }, }, }, VolumeName: "should-be-removed", VolumeMode: &fileMode, }, }, vsName: "test-vs", }, { name: "should reset expected fields in pvc using block mode volumes", pvc: corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: "test-pvc", Namespace: "test-ns", }, Spec: corev1api.PersistentVolumeClaimSpec{ AccessModes: []corev1api.PersistentVolumeAccessMode{corev1api.ReadOnlyMany, corev1api.ReadWriteMany, corev1api.ReadWriteOnce}, Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "foo": "bar", "baz": "qux", }, }, Resources: corev1api.VolumeResourceRequirements{ Requests: corev1api.ResourceList{ corev1api.ResourceCPU: resource.Quantity{ Format: resource.DecimalExponent, }, }, }, VolumeName: "should-be-removed", VolumeMode: &blockMode, }, }, vsName: "test-vs", }, { name: "should overwrite existing DataSource per reset parameters", pvc: corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: "test-pvc", Namespace: "test-ns", }, Spec: corev1api.PersistentVolumeClaimSpec{ AccessModes: []corev1api.PersistentVolumeAccessMode{corev1api.ReadOnlyMany, corev1api.ReadWriteMany, corev1api.ReadWriteOnce}, Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "foo": "bar", "baz": "qux", }, }, Resources: corev1api.VolumeResourceRequirements{ Requests: corev1api.ResourceList{ corev1api.ResourceCPU: resource.Quantity{ Format: resource.DecimalExponent, }, }, }, VolumeName: "should-be-removed", VolumeMode: &fileMode, DataSource: &corev1api.TypedLocalObjectReference{ Kind: "something-that-does-not-exist", Name: "not-found", }, DataSourceRef: &corev1api.TypedObjectReference{ Kind: "something-that-does-not-exist", Name: "not-found", }, }, }, vsName: "test-vs", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { before := tc.pvc.DeepCopy() resetPVCSourceToVolumeSnapshot(&tc.pvc, tc.vsName) assert.Equalf(t, tc.pvc.Name, before.Name, "unexpected change to Object.Name, Want: %s; Got %s", before.Name, tc.pvc.Name) assert.Equalf(t, tc.pvc.Namespace, before.Namespace, "unexpected change to Object.Namespace, Want: %s; Got %s", before.Namespace, tc.pvc.Namespace) assert.Equalf(t, tc.pvc.Spec.AccessModes, before.Spec.AccessModes, "unexpected Spec.AccessModes, Want: %v; Got: %v", before.Spec.AccessModes, tc.pvc.Spec.AccessModes) assert.Equalf(t, tc.pvc.Spec.Selector, before.Spec.Selector, "unexpected change to Spec.Selector, Want: %s; Got: %s", before.Spec.Selector.String(), tc.pvc.Spec.Selector.String()) assert.Equalf(t, tc.pvc.Spec.Resources, before.Spec.Resources, "unexpected change to Spec.Resources, Want: %s; Got: %s", before.Spec.Resources.String(), tc.pvc.Spec.Resources.String()) assert.Emptyf(t, tc.pvc.Spec.VolumeName, "expected change to Spec.VolumeName missing, Want: \"\"; Got: %s", tc.pvc.Spec.VolumeName) assert.Equalf(t, *tc.pvc.Spec.VolumeMode, *before.Spec.VolumeMode, "expected change to Spec.VolumeName missing, Want: \"\"; Got: %s", tc.pvc.Spec.VolumeName) assert.NotNil(t, tc.pvc.Spec.DataSource, "expected change to Spec.DataSource missing") assert.Equalf(t, "VolumeSnapshot", tc.pvc.Spec.DataSource.Kind, "expected change to Spec.DataSource.Kind missing, Want: VolumeSnapshot, Got: %s", tc.pvc.Spec.DataSource.Kind) assert.Equalf(t, tc.pvc.Spec.DataSource.Name, tc.vsName, "expected change to Spec.DataSource.Name missing, Want: %s, Got: %s", tc.vsName, tc.pvc.Spec.DataSource.Name) }) } } func TestProgress(t *testing.T) { currentTime := time.Now() tests := []struct { name string restore *velerov1api.Restore dataDownload *velerov2alpha1.DataDownload operationID string expectedErr string expectedProgress velero.OperationProgress }{ { name: "DataDownload cannot be found", restore: builder.ForRestore("velero", "test").Result(), operationID: "testing", expectedErr: "didn't find DataDownload", }, { name: "DataDownload is not in the expected namespace", restore: builder.ForRestore("velero", "test").Result(), dataDownload: &velerov2alpha1.DataDownload{ TypeMeta: metav1.TypeMeta{ Kind: "DataUpload", APIVersion: velerov2alpha1.SchemeGroupVersion.String(), }, ObjectMeta: metav1.ObjectMeta{ Namespace: "invalid-namespace", Name: "testing", Labels: map[string]string{ velerov1api.AsyncOperationIDLabel: "testing", }, }, }, operationID: "testing", expectedErr: "didn't find DataDownload", }, { name: "DataUpload is found", restore: builder.ForRestore("velero", "test").Result(), dataDownload: &velerov2alpha1.DataDownload{ TypeMeta: metav1.TypeMeta{ Kind: "DataUpload", APIVersion: velerov2alpha1.SchemeGroupVersion.String(), }, ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", Name: "testing", Labels: map[string]string{ velerov1api.AsyncOperationIDLabel: "testing", }, }, Status: velerov2alpha1.DataDownloadStatus{ Phase: velerov2alpha1.DataDownloadPhaseFailed, Progress: shared.DataMoveOperationProgress{ BytesDone: 1000, TotalBytes: 1000, }, StartTimestamp: &metav1.Time{Time: currentTime}, CompletionTimestamp: &metav1.Time{Time: currentTime}, Message: "Testing error", }, }, operationID: "testing", expectedProgress: velero.OperationProgress{ Completed: true, Err: "Testing error", NCompleted: 1000, NTotal: 1000, OperationUnits: "Bytes", Description: "Failed", Started: currentTime, Updated: currentTime, }, }, } for _, tc := range tests { t.Run(tc.name, func(*testing.T) { pvcRIA := pvcRestoreItemAction{ log: logrus.New(), crClient: velerotest.NewFakeControllerRuntimeClient(t), } if tc.dataDownload != nil { err := pvcRIA.crClient.Create(t.Context(), tc.dataDownload) require.NoError(t, err) } progress, err := pvcRIA.Progress(tc.operationID, tc.restore) if tc.expectedErr != "" { require.Equal(t, tc.expectedErr, err.Error()) return } require.NoError(t, err) require.True(t, cmp.Equal(tc.expectedProgress, progress, cmpopts.IgnoreFields(velero.OperationProgress{}, "Started", "Updated"))) }) } } func TestCancel(t *testing.T) { tests := []struct { name string restore *velerov1api.Restore dataDownload *velerov2alpha1.DataDownload operationID string expectedErr string expectedDataDownload velerov2alpha1.DataDownload }{ { name: "Cancel DataUpload", restore: builder.ForRestore("velero", "test").Result(), dataDownload: &velerov2alpha1.DataDownload{ TypeMeta: metav1.TypeMeta{ Kind: "DataDownload", APIVersion: velerov2alpha1.SchemeGroupVersion.String(), }, ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", Name: "testing", Labels: map[string]string{ velerov1api.AsyncOperationIDLabel: "testing", }, }, }, operationID: "testing", expectedErr: "", expectedDataDownload: velerov2alpha1.DataDownload{ TypeMeta: metav1.TypeMeta{ Kind: "DataDownload", APIVersion: velerov2alpha1.SchemeGroupVersion.String(), }, ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", Name: "testing", Labels: map[string]string{ velerov1api.AsyncOperationIDLabel: "testing", }, }, Spec: velerov2alpha1.DataDownloadSpec{ Cancel: true, }, }, }, { name: "Cannot find DataUpload", restore: builder.ForRestore("velero", "test").Result(), dataDownload: nil, operationID: "testing", expectedErr: "didn't find DataDownload", expectedDataDownload: velerov2alpha1.DataDownload{ TypeMeta: metav1.TypeMeta{ Kind: "DataDownload", APIVersion: velerov2alpha1.SchemeGroupVersion.String(), }, ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", Name: "testing", Labels: map[string]string{ velerov1api.AsyncOperationIDLabel: "testing", }, }, Spec: velerov2alpha1.DataDownloadSpec{ Cancel: true, }, }, }, } for _, tc := range tests { t.Run(tc.name, func(*testing.T) { pvcRIA := pvcRestoreItemAction{ log: logrus.New(), crClient: velerotest.NewFakeControllerRuntimeClient(t), } if tc.dataDownload != nil { err := pvcRIA.crClient.Create(t.Context(), tc.dataDownload) require.NoError(t, err) } err := pvcRIA.Cancel(tc.operationID, tc.restore) if tc.expectedErr != "" { require.Equal(t, tc.expectedErr, err.Error()) return } require.NoError(t, err) resultDataDownload := new(velerov2alpha1.DataDownload) err = pvcRIA.crClient.Get(t.Context(), crclient.ObjectKey{Namespace: tc.dataDownload.Namespace, Name: tc.dataDownload.Name}, resultDataDownload) require.NoError(t, err) require.True(t, cmp.Equal(tc.expectedDataDownload, *resultDataDownload, cmpopts.IgnoreFields(velerov2alpha1.DataDownload{}, "ResourceVersion", "Name"))) }) } } func TestExecute(t *testing.T) { vsName := util.GenerateSha256FromRestoreUIDAndVsName("restoreUID", "vsName") tests := []struct { name string backup *velerov1api.Backup restore *velerov1api.Restore pvc *corev1api.PersistentVolumeClaim vs *snapshotv1api.VolumeSnapshot dataUploadResult *corev1api.ConfigMap expectedErr string expectedDataDownload *velerov2alpha1.DataDownload expectedPVC *corev1api.PersistentVolumeClaim preCreatePVC bool }{ { name: "Don't restore PV", restore: builder.ForRestore("velero", "testRestore").Backup("testBackup").RestorePVs(false).Result(), pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").Result(), expectedPVC: builder.ForPersistentVolumeClaim("velero", "testPVC").VolumeName("").Result(), }, { name: "restore's backup cannot be found", restore: builder.ForRestore("velero", "testRestore").Backup("testBackup").Result(), pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").Result(), expectedErr: "fail to get backup for restore: backups.velero.io \"testBackup\" not found", }, { name: "Restore from VolumeSnapshot", backup: builder.ForBackup("velero", "testBackup").Result(), restore: builder.ForRestore("velero", "testRestore").ObjectMeta(builder.WithUID("restoreUID")).Backup("testBackup").Result(), pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotLabel, "vsName")). RequestResource(map[corev1api.ResourceName]resource.Quantity{corev1api.ResourceStorage: resource.MustParse("10Gi")}). DataSource(&corev1api.TypedLocalObjectReference{APIGroup: &snapshotv1api.SchemeGroupVersion.Group, Kind: "VolumeSnapshot", Name: "testVS"}). DataSourceRef(&corev1api.TypedObjectReference{APIGroup: &snapshotv1api.SchemeGroupVersion.Group, Kind: "VolumeSnapshot", Name: "testVS"}). Result(), vs: builder.ForVolumeSnapshot("velero", vsName).ObjectMeta( builder.WithAnnotations(velerov1api.VolumeSnapshotRestoreSize, "10Gi"), ).Result(), expectedPVC: builder.ForPersistentVolumeClaim("velero", "testPVC").ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotLabel, "vsName")).Result(), }, { name: "Restore from VolumeSnapshot without volume-snapshot-name annotation", backup: builder.ForBackup("velero", "testBackup").Result(), restore: builder.ForRestore("velero", "testRestore").Backup("testBackup").Result(), pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").ObjectMeta(builder.WithAnnotations(AnnSelectedNode, "node1")).Result(), vs: builder.ForVolumeSnapshot("velero", "testVS").ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotRestoreSize, "10Gi")).Result(), expectedPVC: builder.ForPersistentVolumeClaim("velero", "testPVC").ObjectMeta(builder.WithAnnotations(AnnSelectedNode, "node1")).Result(), }, { name: "DataUploadResult cannot be found", backup: builder.ForBackup("velero", "testBackup").SnapshotMoveData(true).Result(), restore: builder.ForRestore("velero", "testRestore").Backup("testBackup").Result(), pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotRestoreSize, "10Gi", velerov1api.DataUploadNameAnnotation, "velero/")).Result(), expectedPVC: builder.ForPersistentVolumeClaim("velero", "testPVC").Result(), expectedErr: "fail get DataUploadResult for restore: testRestore: no DataUpload result cm found with labels velero.io/pvc-namespace-name=velero.testPVC,velero.io/restore-uid=,velero.io/resource-usage=DataUpload", }, { name: "Restore from DataUploadResult", backup: builder.ForBackup("velero", "testBackup").SnapshotMoveData(true).Result(), restore: builder.ForRestore("velero", "testRestore").Backup("testBackup").ObjectMeta(builder.WithUID("uid")).Result(), pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotRestoreSize, "10Gi", velerov1api.DataUploadNameAnnotation, "velero/")).Result(), dataUploadResult: builder.ForConfigMap("velero", "testCM").Data("uid", "{}").ObjectMeta(builder.WithLabels(velerov1api.RestoreUIDLabel, "uid", velerov1api.PVCNamespaceNameLabel, "velero.testPVC", velerov1api.ResourceUsageLabel, label.GetValidName(string(velerov1api.VeleroResourceUsageDataUploadResult)))).Result(), expectedPVC: builder.ForPersistentVolumeClaim("velero", "testPVC").ObjectMeta(builder.WithAnnotations("velero.io/csi-volumesnapshot-restore-size", "10Gi", velerov1api.DataUploadNameAnnotation, "velero/")).Result(), expectedDataDownload: builder.ForDataDownload("velero", "name").TargetVolume(velerov2alpha1.TargetVolumeSpec{PVC: "testPVC", Namespace: "velero"}). ObjectMeta(builder.WithOwnerReference([]metav1.OwnerReference{{APIVersion: velerov1api.SchemeGroupVersion.String(), Kind: "Restore", Name: "testRestore", UID: "uid", Controller: boolptr.True()}}), builder.WithLabelsMap(map[string]string{velerov1api.AsyncOperationIDLabel: "dd-uid.", velerov1api.RestoreNameLabel: "testRestore", velerov1api.RestoreUIDLabel: "uid"}), builder.WithGenerateName("testRestore-")).Result(), }, { name: "Restore from DataUploadResult with long source PVC namespace and name", backup: builder.ForBackup("migre209d0da-49c7-45ba-8d5a-3e59fd591ec1", "testBackup").SnapshotMoveData(true).Result(), restore: builder.ForRestore("migre209d0da-49c7-45ba-8d5a-3e59fd591ec1", "testRestore").Backup("testBackup").ObjectMeta(builder.WithUID("uid")).Result(), pvc: builder.ForPersistentVolumeClaim("migre209d0da-49c7-45ba-8d5a-3e59fd591ec1", "kibishii-data-kibishii-deployment-0").ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotRestoreSize, "10Gi", velerov1api.DataUploadNameAnnotation, "velero/")).Result(), dataUploadResult: builder.ForConfigMap("migre209d0da-49c7-45ba-8d5a-3e59fd591ec1", "testCM").Data("uid", "{}").ObjectMeta(builder.WithLabels(velerov1api.RestoreUIDLabel, "uid", velerov1api.PVCNamespaceNameLabel, "migre209d0da-49c7-45ba-8d5a-3e59fd591ec1.kibishii-data-ki152333", velerov1api.ResourceUsageLabel, label.GetValidName(string(velerov1api.VeleroResourceUsageDataUploadResult)))).Result(), expectedPVC: builder.ForPersistentVolumeClaim("migre209d0da-49c7-45ba-8d5a-3e59fd591ec1", "kibishii-data-kibishii-deployment-0").ObjectMeta(builder.WithAnnotations("velero.io/csi-volumesnapshot-restore-size", "10Gi", velerov1api.DataUploadNameAnnotation, "velero/")).Result(), }, { name: "PVC had no DataUploadNameLabel annotation", backup: builder.ForBackup("migre209d0da-49c7-45ba-8d5a-3e59fd591ec1", "testBackup").SnapshotMoveData(true).Result(), restore: builder.ForRestore("migre209d0da-49c7-45ba-8d5a-3e59fd591ec1", "testRestore").Backup("testBackup").ObjectMeta(builder.WithUID("uid")).Result(), pvc: builder.ForPersistentVolumeClaim("migre209d0da-49c7-45ba-8d5a-3e59fd591ec1", "kibishii-data-kibishii-deployment-0").ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotRestoreSize, "10Gi")).Result(), }, { name: "Restore a PVC that already exists.", backup: builder.ForBackup("velero", "testBackup").SnapshotMoveData(true).Result(), restore: builder.ForRestore("velero", "testRestore").Backup("testBackup").ObjectMeta(builder.WithUID("uid")).Result(), pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotRestoreSize, "10Gi", velerov1api.DataUploadNameAnnotation, "velero/")).Result(), preCreatePVC: true, }, { name: "Restore a PVC that already exists in the mapping namespace", backup: builder.ForBackup("velero", "testBackup").SnapshotMoveData(true).Result(), restore: builder.ForRestore("velero", "testRestore").Backup("testBackup").NamespaceMappings("velero", "restore").ObjectMeta(builder.WithUID("uid")).Result(), pvc: builder.ForPersistentVolumeClaim("restore", "testPVC").ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotRestoreSize, "10Gi", velerov1api.DataUploadNameAnnotation, "velero/")).Result(), preCreatePVC: true, }, } for _, tc := range tests { t.Run(tc.name, func(*testing.T) { object := make([]runtime.Object, 0) if tc.backup != nil { object = append(object, tc.backup) } if tc.vs != nil { object = append(object, tc.vs) } input := new(velero.RestoreItemActionExecuteInput) if tc.pvc != nil { pvcMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.pvc) require.NoError(t, err) input.Item = &unstructured.Unstructured{Object: pvcMap} input.ItemFromBackup = &unstructured.Unstructured{Object: pvcMap} input.Restore = tc.restore } if tc.preCreatePVC { object = append(object, tc.pvc) } if tc.dataUploadResult != nil { object = append(object, tc.dataUploadResult) } pvcRIA := pvcRestoreItemAction{ log: logrus.New(), crClient: velerotest.NewFakeControllerRuntimeClient(t, object...), } output, err := pvcRIA.Execute(input) if tc.expectedErr != "" { require.Equal(t, tc.expectedErr, err.Error()) return } require.NoError(t, err) if tc.expectedPVC != nil { pvc := new(corev1api.PersistentVolumeClaim) err := runtime.DefaultUnstructuredConverter.FromUnstructured(output.UpdatedItem.UnstructuredContent(), pvc) require.NoError(t, err) require.Equal(t, tc.expectedPVC.GetObjectMeta(), pvc.GetObjectMeta()) if pvc.Spec.Selector != nil && pvc.Spec.Selector.MatchLabels != nil { // This is used for long name and namespace case. if len(tc.pvc.Namespace+"."+tc.pvc.Name) >= validation.DNS1035LabelMaxLength { require.Contains(t, pvc.Spec.Selector.MatchLabels[velerov1api.DynamicPVRestoreLabel], label.GetValidName(tc.pvc.Namespace + "." + tc.pvc.Name)[:56]) } else { require.Contains(t, pvc.Spec.Selector.MatchLabels[velerov1api.DynamicPVRestoreLabel], tc.pvc.Namespace+"."+tc.pvc.Name) } } } if tc.expectedDataDownload != nil { dataDownloadList := new(velerov2alpha1.DataDownloadList) err := pvcRIA.crClient.List(t.Context(), dataDownloadList, &crclient.ListOptions{ LabelSelector: labels.SelectorFromSet(tc.expectedDataDownload.Labels), }) require.NoError(t, err) require.True(t, cmp.Equal(tc.expectedDataDownload, &dataDownloadList.Items[0], cmpopts.IgnoreFields(velerov2alpha1.DataDownload{}, "ResourceVersion", "Name"))) } }) } } func TestPVCAppliesTo(t *testing.T) { p := pvcRestoreItemAction{ log: logrus.StandardLogger(), } selector, err := p.AppliesTo() require.NoError(t, err) require.Equal( t, velero.ResourceSelector{ IncludedResources: []string{"persistentvolumeclaims"}, }, selector, ) } func TestNewPvcRestoreItemAction(t *testing.T) { logger := logrus.StandardLogger() crClient := velerotest.NewFakeControllerRuntimeClient(t) f := &factorymocks.Factory{} f.On("KubebuilderClient").Return(nil, fmt.Errorf("")) plugin := NewPvcRestoreItemAction(f) _, err := plugin(logger) require.Error(t, err) f1 := &factorymocks.Factory{} f1.On("KubebuilderClient").Return(crClient, nil) plugin1 := NewPvcRestoreItemAction(f1) _, err1 := plugin1(logger) require.NoError(t, err1) } ================================================ FILE: pkg/restore/actions/csi/volumesnapshot_action.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package csi import ( "fmt" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" crclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/kuberesource" plugincommon "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" "github.com/vmware-tanzu/velero/pkg/plugin/velero" "github.com/vmware-tanzu/velero/pkg/util" "github.com/vmware-tanzu/velero/pkg/util/boolptr" ) // volumeSnapshotRestoreItemAction is a Velero restore item // action plugin for VolumeSnapshots type volumeSnapshotRestoreItemAction struct { log logrus.FieldLogger crClient crclient.Client } // AppliesTo returns information indicating that // VolumeSnapshotRestoreItemAction should be invoked while // restoring volumesnapshots.snapshot.storage.k8s.io resources. func (p *volumeSnapshotRestoreItemAction) AppliesTo() ( velero.ResourceSelector, error) { return velero.ResourceSelector{ IncludedResources: []string{"volumesnapshots.snapshot.storage.k8s.io"}, }, nil } func resetVolumeSnapshotSpecForRestore(vs *snapshotv1api.VolumeSnapshot, vscName *string) { // Spec of the backed-up object used the PVC as the source // of the volumeSnapshot. Restore operation will however, // restore the VolumeSnapshot from the VolumeSnapshotContent vs.Spec.Source.PersistentVolumeClaimName = nil vs.Spec.Source.VolumeSnapshotContentName = vscName } func resetVolumeSnapshotAnnotation(vs *snapshotv1api.VolumeSnapshot) { vs.ObjectMeta.Annotations[velerov1api.VSCDeletionPolicyAnnotation] = string(snapshotv1api.VolumeSnapshotContentRetain) } func (p *volumeSnapshotRestoreItemAction) Execute( input *velero.RestoreItemActionExecuteInput, ) (*velero.RestoreItemActionExecuteOutput, error) { p.log.Info("Starting VolumeSnapshotRestoreItemAction") if boolptr.IsSetToFalse(input.Restore.Spec.RestorePVs) { p.log.Infof("Restore %s/%s did not request for PVs to be restored.", input.Restore.Namespace, input.Restore.Name) return &velero.RestoreItemActionExecuteOutput{SkipRestore: true}, nil } var vs snapshotv1api.VolumeSnapshot if err := runtime.DefaultUnstructuredConverter.FromUnstructured( input.Item.UnstructuredContent(), &vs); err != nil { return &velero.RestoreItemActionExecuteOutput{}, errors.Wrapf(err, "failed to convert input.Item from unstructured") } var vsFromBackup snapshotv1api.VolumeSnapshot if err := runtime.DefaultUnstructuredConverter.FromUnstructured( input.ItemFromBackup.UnstructuredContent(), &vsFromBackup); err != nil { return &velero.RestoreItemActionExecuteOutput{}, errors.Wrapf(err, "failed to convert input.Item from unstructured") } generatedName := util.GenerateSha256FromRestoreUIDAndVsName(string(input.Restore.UID), vsFromBackup.Name) // Reset Spec to convert the VolumeSnapshot from using // the dynamic VolumeSnapshotContent to the static one. resetVolumeSnapshotSpecForRestore(&vs, &generatedName) // Also reset the VS name to avoid potential conflict caused by multiple restores of the same backup. // Both VS and VSC share the same generated name. vs.Name = generatedName // Reset VolumeSnapshot annotation. By now, only change // DeletionPolicy to Retain. resetVolumeSnapshotAnnotation(&vs) if vs.Spec.VolumeSnapshotClassName != nil { // Delete VolumeSnapshotClass from the VolumeSnapshot. // This is necessary to make the restore independent of the VolumeSnapshotClass. vs.Spec.VolumeSnapshotClassName = nil p.log.Debugf("Deleted VolumeSnapshotClassName from VolumeSnapshot %s/%s to make restore independent of VolumeSnapshotClass", vs.Namespace, vs.Name) } vsMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&vs) if err != nil { p.log.Errorf("Fail to convert VS %s to unstructured", vs.Namespace+"/"+vs.Name) return nil, errors.WithStack(err) } if vsFromBackup.Status == nil || vsFromBackup.Status.BoundVolumeSnapshotContentName == nil { p.log.Errorf("VS %s doesn't have bound VSC", vsFromBackup.Name) return nil, fmt.Errorf("VS %s doesn't have bound VSC", vsFromBackup.Name) } vsc := velero.ResourceIdentifier{ GroupResource: kuberesource.VolumeSnapshotContents, Name: *vsFromBackup.Status.BoundVolumeSnapshotContentName, } p.log.Infof(`Returning from VolumeSnapshotRestoreItemAction with VolumeSnapshotContent in additionalItems`) return &velero.RestoreItemActionExecuteOutput{ UpdatedItem: &unstructured.Unstructured{Object: vsMap}, AdditionalItems: []velero.ResourceIdentifier{vsc}, }, nil } func (p *volumeSnapshotRestoreItemAction) Name() string { return "VolumeSnapshotRestoreItemAction" } func (p *volumeSnapshotRestoreItemAction) Progress( operationID string, restore *velerov1api.Restore, ) (velero.OperationProgress, error) { return velero.OperationProgress{}, nil } func (p *volumeSnapshotRestoreItemAction) Cancel( operationID string, restore *velerov1api.Restore, ) error { // CSI Specification doesn't support canceling a snapshot creation. return nil } func (p *volumeSnapshotRestoreItemAction) AreAdditionalItemsReady( additionalItems []velero.ResourceIdentifier, restore *velerov1api.Restore, ) (bool, error) { return true, nil } func NewVolumeSnapshotRestoreItemAction( f client.Factory, ) plugincommon.HandlerInitializer { return func(logger logrus.FieldLogger) (any, error) { crClient, err := f.KubebuilderClient() if err != nil { return nil, err } return &volumeSnapshotRestoreItemAction{logger, crClient}, nil } } ================================================ FILE: pkg/restore/actions/csi/volumesnapshot_action_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package csi import ( "fmt" "testing" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" factorymocks "github.com/vmware-tanzu/velero/pkg/client/mocks" "github.com/vmware-tanzu/velero/pkg/plugin/velero" velerotest "github.com/vmware-tanzu/velero/pkg/test" "github.com/vmware-tanzu/velero/pkg/util" ) var ( testPVC = "test-pvc" testSnapClass = "snap-class" randText = "DEADFEED" ) func TestResetVolumeSnapshotSpecForRestore(t *testing.T) { testCases := []struct { name string vs snapshotv1api.VolumeSnapshot vscName string }{ { name: "should reset spec as expected", vs: snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: "test-vs", Namespace: "test-ns", }, Spec: snapshotv1api.VolumeSnapshotSpec{ Source: snapshotv1api.VolumeSnapshotSource{ PersistentVolumeClaimName: &testPVC, }, VolumeSnapshotClassName: &testSnapClass, }, }, vscName: "test-vsc", }, { name: "should reset spec and overwriting value for Source.VolumeSnapshotContentName", vs: snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: "test-vs", Namespace: "test-ns", }, Spec: snapshotv1api.VolumeSnapshotSpec{ Source: snapshotv1api.VolumeSnapshotSource{ VolumeSnapshotContentName: &randText, }, VolumeSnapshotClassName: &testSnapClass, }, }, vscName: "test-vsc", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { before := tc.vs.DeepCopy() resetVolumeSnapshotSpecForRestore(&tc.vs, &tc.vscName) assert.Equalf(t, tc.vs.Name, before.Name, "unexpected change to Object.Name, Want: %s; Got %s", before.Name, tc.vs.Name) assert.Equalf(t, tc.vs.Namespace, before.Namespace, "unexpected change to Object.Namespace, Want: %s; Got %s", before.Namespace, tc.vs.Namespace) assert.NotNil(t, tc.vs.Spec.Source) assert.Nil(t, tc.vs.Spec.Source.PersistentVolumeClaimName) assert.NotNil(t, tc.vs.Spec.Source.VolumeSnapshotContentName) assert.Equal(t, *tc.vs.Spec.Source.VolumeSnapshotContentName, tc.vscName) assert.Equalf(t, *tc.vs.Spec.VolumeSnapshotClassName, *before.Spec.VolumeSnapshotClassName, "unexpected value for Spec.VolumeSnapshotClassName, Want: %s, Got: %s", *tc.vs.Spec.VolumeSnapshotClassName, *before.Spec.VolumeSnapshotClassName) assert.Nil(t, tc.vs.Status) }) } } func TestVSExecute(t *testing.T) { newVscName := util.GenerateSha256FromRestoreUIDAndVsName("restoreUID", "vsName") tests := []struct { name string item runtime.Unstructured vs *snapshotv1api.VolumeSnapshot restore *velerov1api.Restore expectErr bool createVS bool expectedVS *snapshotv1api.VolumeSnapshot }{ { name: "Restore's RestorePVs is false", restore: builder.ForRestore("velero", "restore").RestorePVs(false).Result(), expectErr: false, }, { name: "VS doesn't have VSC in status", vs: builder.ForVolumeSnapshot("ns", "name").ObjectMeta(builder.WithAnnotations("1", "1")).Status().Result(), restore: builder.ForRestore("velero", "restore").NamespaceMappings("ns", "newNS").Result(), expectErr: true, }, { name: "Normal case, VSC should be created", vs: builder.ForVolumeSnapshot("ns", "vsName"). ObjectMeta( builder.WithAnnotationsMap( map[string]string{ velerov1api.VolumeSnapshotHandleAnnotation: "vsc", velerov1api.DriverNameAnnotation: "pd.csi.storage.gke.io", }, ), ). SourceVolumeSnapshotContentName(newVscName). VolumeSnapshotClass("vscClass"). Status(). BoundVolumeSnapshotContentName("vscName"). Result(), restore: builder.ForRestore("velero", "restore").ObjectMeta(builder.WithUID("restoreUID")).Result(), expectErr: false, expectedVS: builder.ForVolumeSnapshot("ns", "test").SourceVolumeSnapshotContentName(newVscName).Result(), }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { p := volumeSnapshotRestoreItemAction{ log: logrus.StandardLogger(), crClient: velerotest.NewFakeControllerRuntimeClient(t), } if test.vs != nil { vsMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(test.vs) require.NoError(t, err) test.item = &unstructured.Unstructured{Object: vsMap} if test.createVS == true { if newNS, ok := test.restore.Spec.NamespaceMapping[test.vs.Namespace]; ok { test.vs.SetNamespace(newNS) } require.NoError(t, p.crClient.Create(t.Context(), test.vs)) } } result, err := p.Execute( &velero.RestoreItemActionExecuteInput{ Item: test.item, ItemFromBackup: test.item, Restore: test.restore, }, ) if test.expectErr == false { require.NoError(t, err) } if test.expectedVS != nil { var vs snapshotv1api.VolumeSnapshot require.NoError(t, runtime.DefaultUnstructuredConverter.FromUnstructured( result.UpdatedItem.UnstructuredContent(), &vs)) require.Equal(t, test.expectedVS.Spec, vs.Spec) } }) } } func TestVSAppliesTo(t *testing.T) { p := volumeSnapshotRestoreItemAction{ log: logrus.StandardLogger(), } selector, err := p.AppliesTo() require.NoError(t, err) require.Equal( t, velero.ResourceSelector{ IncludedResources: []string{"volumesnapshots.snapshot.storage.k8s.io"}, }, selector, ) } func TestNewVolumeSnapshotRestoreItemAction(t *testing.T) { logger := logrus.StandardLogger() crClient := velerotest.NewFakeControllerRuntimeClient(t) f := &factorymocks.Factory{} f.On("KubebuilderClient").Return(nil, fmt.Errorf("")) plugin := NewVolumeSnapshotRestoreItemAction(f) _, err := plugin(logger) require.Error(t, err) f1 := &factorymocks.Factory{} f1.On("KubebuilderClient").Return(crClient, nil) plugin1 := NewVolumeSnapshotRestoreItemAction(f1) _, err1 := plugin1(logger) require.NoError(t, err1) } ================================================ FILE: pkg/restore/actions/csi/volumesnapshotclass_action.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package csi import ( snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/plugin/velero" "github.com/vmware-tanzu/velero/pkg/util/boolptr" "github.com/vmware-tanzu/velero/pkg/util/csi" ) // volumeSnapshotClassRestoreItemAction is a Velero restore // item action plugin for VolumeSnapshotClass type volumeSnapshotClassRestoreItemAction struct { log logrus.FieldLogger } // AppliesTo returns information indicating that VolumeSnapshotClassRestoreItemAction // should be invoked while restoring volumesnapshotclass.snapshot.storage.k8s.io resources. func (p *volumeSnapshotClassRestoreItemAction) AppliesTo() (velero.ResourceSelector, error) { return velero.ResourceSelector{ IncludedResources: []string{"volumesnapshotclasses.snapshot.storage.k8s.io"}, }, nil } // Execute restores VolumeSnapshotClass objects returning any // snapshotlister secret as additional items to restore func (p *volumeSnapshotClassRestoreItemAction) Execute( input *velero.RestoreItemActionExecuteInput, ) (*velero.RestoreItemActionExecuteOutput, error) { p.log.Info("Starting VolumeSnapshotClassRestoreItemAction") if boolptr.IsSetToFalse(input.Restore.Spec.RestorePVs) { p.log.Infof("Restore did not request for PVs to be restored %s/%s", input.Restore.Namespace, input.Restore.Name) return &velero.RestoreItemActionExecuteOutput{SkipRestore: true}, nil } var snapClass snapshotv1api.VolumeSnapshotClass if err := runtime.DefaultUnstructuredConverter.FromUnstructured( input.Item.UnstructuredContent(), &snapClass); err != nil { return &velero.RestoreItemActionExecuteOutput{}, errors.Wrapf(err, "failed to convert input.Item from unstructured") } additionalItems := []velero.ResourceIdentifier{} if csi.IsVolumeSnapshotClassHasListerSecret(&snapClass) { additionalItems = append(additionalItems, velero.ResourceIdentifier{ GroupResource: schema.GroupResource{Group: "", Resource: "secrets"}, Name: snapClass.Annotations[velerov1api.PrefixedListSecretNameAnnotation], Namespace: snapClass.Annotations[velerov1api.PrefixedListSecretNamespaceAnnotation], }) } p.log.Infof("Returning from VolumeSnapshotClassRestoreItemAction with %d additionalItems", len(additionalItems)) return &velero.RestoreItemActionExecuteOutput{ UpdatedItem: input.Item, AdditionalItems: additionalItems, }, nil } func (p *volumeSnapshotClassRestoreItemAction) Name() string { return "VolumeSnapshotClassRestoreItemAction" } func (p *volumeSnapshotClassRestoreItemAction) Progress( operationID string, restore *velerov1api.Restore, ) (velero.OperationProgress, error) { return velero.OperationProgress{}, nil } func (p *volumeSnapshotClassRestoreItemAction) Cancel( operationID string, restore *velerov1api.Restore, ) error { return nil } func (p *volumeSnapshotClassRestoreItemAction) AreAdditionalItemsReady( additionalItems []velero.ResourceIdentifier, restore *velerov1api.Restore, ) (bool, error) { return true, nil } func NewVolumeSnapshotClassRestoreItemAction( logger logrus.FieldLogger) (any, error) { return &volumeSnapshotClassRestoreItemAction{logger}, nil } ================================================ FILE: pkg/restore/actions/csi/volumesnapshotclass_action_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package csi import ( "testing" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/kuberesource" //"github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/plugin/velero" ) func TestVSClassExecute(t *testing.T) { tests := []struct { name string item runtime.Unstructured vsClass *snapshotv1api.VolumeSnapshotClass restore *velerov1api.Restore expectErr bool expectedItems []velero.ResourceIdentifier }{ { name: "Restore's RestorePVs is false", restore: builder.ForRestore("velero", "restore").RestorePVs(false).Result(), expectErr: false, }, { name: "No Secret in the VS Class, no return additional items", vsClass: builder.ForVolumeSnapshotClass("test").Result(), restore: builder.ForRestore("velero", "restore").Result(), expectErr: false, }, { name: "Normal case, additional items should return", vsClass: builder.ForVolumeSnapshotClass("test").ObjectMeta(builder.WithAnnotationsMap( map[string]string{ velerov1api.PrefixedListSecretNameAnnotation: "name", velerov1api.PrefixedListSecretNamespaceAnnotation: "namespace", }, )).Result(), restore: builder.ForRestore("velero", "restore").Result(), expectErr: false, expectedItems: []velero.ResourceIdentifier{ { GroupResource: kuberesource.Secrets, Namespace: "namespace", Name: "name", }, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { p, err := NewVolumeSnapshotClassRestoreItemAction(logrus.StandardLogger()) require.NoError(t, err) action := p.(*volumeSnapshotClassRestoreItemAction) if test.vsClass != nil { vsMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(test.vsClass) require.NoError(t, err) test.item = &unstructured.Unstructured{Object: vsMap} } output, err := action.Execute( &velero.RestoreItemActionExecuteInput{ Item: test.item, Restore: test.restore, }, ) if test.expectErr == false { require.NoError(t, err) } if len(test.expectedItems) > 0 { require.Equal(t, test.expectedItems, output.AdditionalItems) } }) } } func TestVSClassAppliesTo(t *testing.T) { p := volumeSnapshotClassRestoreItemAction{ log: logrus.StandardLogger(), } selector, err := p.AppliesTo() require.NoError(t, err) require.Equal( t, velero.ResourceSelector{ IncludedResources: []string{"volumesnapshotclasses.snapshot.storage.k8s.io"}, }, selector, ) } ================================================ FILE: pkg/restore/actions/csi/volumesnapshotcontent_action.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package csi import ( snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" crclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" plugincommon "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" "github.com/vmware-tanzu/velero/pkg/plugin/velero" "github.com/vmware-tanzu/velero/pkg/util" "github.com/vmware-tanzu/velero/pkg/util/boolptr" "github.com/vmware-tanzu/velero/pkg/util/csi" ) // volumeSnapshotContentRestoreItemAction is a restore item action // plugin for Velero type volumeSnapshotContentRestoreItemAction struct { log logrus.FieldLogger client crclient.Client } // AppliesTo returns information indicating VolumeSnapshotContentRestoreItemAction // action should be invoked while restoring // volumesnapshotcontent.snapshot.storage.k8s.io resources func (p *volumeSnapshotContentRestoreItemAction) AppliesTo() ( velero.ResourceSelector, error, ) { return velero.ResourceSelector{ IncludedResources: []string{"volumesnapshotcontents.snapshot.storage.k8s.io"}, }, nil } // Execute restores a VolumeSnapshotContent object without modification // returning the snapshot lister secret, if any, as additional items to restore. func (p *volumeSnapshotContentRestoreItemAction) Execute( input *velero.RestoreItemActionExecuteInput, ) (*velero.RestoreItemActionExecuteOutput, error) { if boolptr.IsSetToFalse(input.Restore.Spec.RestorePVs) { p.log.Infof("Restore did not request for PVs to be restored %s/%s", input.Restore.Namespace, input.Restore.Name) return &velero.RestoreItemActionExecuteOutput{SkipRestore: true}, nil } p.log.Info("Starting VolumeSnapshotContentRestoreItemAction") var vsc snapshotv1api.VolumeSnapshotContent if err := runtime.DefaultUnstructuredConverter.FromUnstructured( input.Item.UnstructuredContent(), &vsc); err != nil { return &velero.RestoreItemActionExecuteOutput{}, errors.Wrapf(err, "failed to convert input.Item from unstructured") } var vscFromBackup snapshotv1api.VolumeSnapshotContent if err := runtime.DefaultUnstructuredConverter.FromUnstructured( input.ItemFromBackup.UnstructuredContent(), &vscFromBackup); err != nil { return &velero.RestoreItemActionExecuteOutput{}, errors.Errorf(err.Error(), "failed to convert input.ItemFromBackup from unstructured") } // If cross-namespace restore is configured, change the namespace // for VolumeSnapshot object to be restored newNamespace, ok := input.Restore.Spec.NamespaceMapping[vsc.Spec.VolumeSnapshotRef.Namespace] if ok { // Update the referenced VS namespace to the mapping one. vsc.Spec.VolumeSnapshotRef.Namespace = newNamespace } // Reset VSC name to align with VS. vsc.Name = util.GenerateSha256FromRestoreUIDAndVsName( string(input.Restore.UID), vscFromBackup.Spec.VolumeSnapshotRef.Name) // Also reset the referenced VS name. vsc.Spec.VolumeSnapshotRef.Name = vsc.Name // Reset the ResourceVersion and UID of referenced VolumeSnapshot. vsc.Spec.VolumeSnapshotRef.ResourceVersion = "" vsc.Spec.VolumeSnapshotRef.UID = "" // Set the DeletionPolicy to Retain to avoid VS deletion will not trigger snapshot deletion vsc.Spec.DeletionPolicy = snapshotv1api.VolumeSnapshotContentRetain if vscFromBackup.Status != nil && vscFromBackup.Status.SnapshotHandle != nil { vsc.Spec.Source.VolumeHandle = nil vsc.Spec.Source.SnapshotHandle = vscFromBackup.Status.SnapshotHandle } else { p.log.Errorf("fail to get snapshot handle from VSC %s status", vsc.Name) return nil, errors.Errorf("fail to get snapshot handle from VSC %s status", vsc.Name) } if vsc.Spec.VolumeSnapshotClassName != nil { // Delete VolumeSnapshotClass from the VolumeSnapshotContent. // This is necessary to make the restore independent of the VolumeSnapshotClass. vsc.Spec.VolumeSnapshotClassName = nil p.log.Debugf("Deleted VolumeSnapshotClassName from VolumeSnapshotContent %s to make restore independent of VolumeSnapshotClass", vsc.Name) } additionalItems := []velero.ResourceIdentifier{} if csi.IsVolumeSnapshotContentHasDeleteSecret(&vsc) { additionalItems = append(additionalItems, velero.ResourceIdentifier{ GroupResource: schema.GroupResource{Group: "", Resource: "secrets"}, Name: vsc.Annotations[velerov1api.PrefixedSecretNameAnnotation], Namespace: vsc.Annotations[velerov1api.PrefixedSecretNamespaceAnnotation], }, ) } vscMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&vsc) if err != nil { return nil, errors.WithStack(err) } p.log.Infof("Returning from VolumeSnapshotContentRestoreItemAction with %d additionalItems", len(additionalItems)) return &velero.RestoreItemActionExecuteOutput{ UpdatedItem: &unstructured.Unstructured{Object: vscMap}, AdditionalItems: additionalItems, }, nil } func (p *volumeSnapshotContentRestoreItemAction) Name() string { return "VolumeSnapshotContentRestoreItemAction" } func (p *volumeSnapshotContentRestoreItemAction) Progress( operationID string, restore *velerov1api.Restore, ) (velero.OperationProgress, error) { return velero.OperationProgress{}, nil } func (p *volumeSnapshotContentRestoreItemAction) Cancel( operationID string, restore *velerov1api.Restore, ) error { return nil } func (p *volumeSnapshotContentRestoreItemAction) AreAdditionalItemsReady( additionalItems []velero.ResourceIdentifier, restore *velerov1api.Restore, ) (bool, error) { return true, nil } func NewVolumeSnapshotContentRestoreItemAction( f client.Factory, ) plugincommon.HandlerInitializer { return func(logger logrus.FieldLogger) (any, error) { crClient, err := f.KubebuilderClient() if err != nil { return nil, err } return &volumeSnapshotContentRestoreItemAction{logger, crClient}, nil } } ================================================ FILE: pkg/restore/actions/csi/volumesnapshotcontent_action_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package csi import ( "fmt" "testing" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" factorymocks "github.com/vmware-tanzu/velero/pkg/client/mocks" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/plugin/velero" velerotest "github.com/vmware-tanzu/velero/pkg/test" "github.com/vmware-tanzu/velero/pkg/util" ) func TestVSCExecute(t *testing.T) { snapshotHandleName := "testHandle" newVscName := util.GenerateSha256FromRestoreUIDAndVsName("restoreUID", "vsName") tests := []struct { name string item runtime.Unstructured vsc *snapshotv1api.VolumeSnapshotContent restore *velerov1api.Restore expectErr bool createVSC bool expectedItems []velero.ResourceIdentifier expectedVSC *snapshotv1api.VolumeSnapshotContent }{ { name: "Restore's RestorePVs is false", restore: builder.ForRestore("velero", "restore").RestorePVs(false).Result(), expectErr: false, }, { name: "Normal case, additional items should return ", vsc: builder.ForVolumeSnapshotContent("test"). ObjectMeta(builder.WithAnnotationsMap( map[string]string{ velerov1api.PrefixedSecretNameAnnotation: "name", velerov1api.PrefixedSecretNamespaceAnnotation: "namespace", }, )). VolumeSnapshotRef("velero", "vsName", "vsUID"). VolumeSnapshotClassName("vsClass"). Status(&snapshotv1api.VolumeSnapshotContentStatus{SnapshotHandle: &snapshotHandleName}). Result(), restore: builder.ForRestore("velero", "restore").ObjectMeta(builder.WithUID("restoreUID")). NamespaceMappings("velero", "restore").Result(), expectErr: false, expectedItems: []velero.ResourceIdentifier{ { GroupResource: kuberesource.Secrets, Namespace: "namespace", Name: "name", }, }, expectedVSC: builder.ForVolumeSnapshotContent(newVscName). ObjectMeta(builder.WithAnnotationsMap( map[string]string{ velerov1api.PrefixedSecretNameAnnotation: "name", velerov1api.PrefixedSecretNamespaceAnnotation: "namespace", }, )).VolumeSnapshotRef("restore", newVscName, ""). Source(snapshotv1api.VolumeSnapshotContentSource{SnapshotHandle: &snapshotHandleName}). DeletionPolicy(snapshotv1api.VolumeSnapshotContentRetain). Status(&snapshotv1api.VolumeSnapshotContentStatus{SnapshotHandle: &snapshotHandleName}). Result(), }, { name: "VSC exists in cluster, same as the normal case", vsc: builder.ForVolumeSnapshotContent("test").ObjectMeta(builder.WithAnnotationsMap( map[string]string{ velerov1api.PrefixedSecretNameAnnotation: "name", velerov1api.PrefixedSecretNamespaceAnnotation: "namespace", }, )).VolumeSnapshotRef("velero", "vsName", "vsUID"). Status(&snapshotv1api.VolumeSnapshotContentStatus{SnapshotHandle: &snapshotHandleName}).Result(), restore: builder.ForRestore("velero", "restore").ObjectMeta(builder.WithUID("restoreUID")). NamespaceMappings("velero", "restore").Result(), createVSC: true, expectErr: false, expectedVSC: builder.ForVolumeSnapshotContent(newVscName).ObjectMeta(builder.WithAnnotationsMap( map[string]string{ velerov1api.PrefixedSecretNameAnnotation: "name", velerov1api.PrefixedSecretNamespaceAnnotation: "namespace", }, )).VolumeSnapshotRef("restore", newVscName, ""). Source(snapshotv1api.VolumeSnapshotContentSource{SnapshotHandle: &snapshotHandleName}). DeletionPolicy(snapshotv1api.VolumeSnapshotContentRetain). Status(&snapshotv1api.VolumeSnapshotContentStatus{SnapshotHandle: &snapshotHandleName}).Result(), }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { action := volumeSnapshotContentRestoreItemAction{ log: logrus.StandardLogger(), client: velerotest.NewFakeControllerRuntimeClient(t), } if test.vsc != nil { vsMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(test.vsc) require.NoError(t, err) test.item = &unstructured.Unstructured{Object: vsMap} if test.createVSC { require.NoError(t, action.client.Create(t.Context(), test.vsc)) } } output, err := action.Execute( &velero.RestoreItemActionExecuteInput{ Item: test.item, ItemFromBackup: test.item, Restore: test.restore, }, ) if test.expectErr == false { require.NoError(t, err) } if test.expectedVSC != nil { vsc := new(snapshotv1api.VolumeSnapshotContent) require.NoError(t, runtime.DefaultUnstructuredConverter.FromUnstructured( output.UpdatedItem.UnstructuredContent(), vsc, ), ) require.Equal(t, test.expectedVSC, vsc) } if len(test.expectedItems) > 0 { require.Equal(t, test.expectedItems, output.AdditionalItems) } }) } } func TestVSCAppliesTo(t *testing.T) { p := volumeSnapshotContentRestoreItemAction{ log: logrus.StandardLogger(), } selector, err := p.AppliesTo() require.NoError(t, err) require.Equal( t, velero.ResourceSelector{ IncludedResources: []string{"volumesnapshotcontents.snapshot.storage.k8s.io"}, }, selector, ) } func TestNewVolumeSnapshotContentRestoreItemAction(t *testing.T) { logger := logrus.StandardLogger() crClient := velerotest.NewFakeControllerRuntimeClient(t) f := &factorymocks.Factory{} f.On("KubebuilderClient").Return(nil, fmt.Errorf("")) plugin := NewVolumeSnapshotContentRestoreItemAction(f) _, err := plugin(logger) require.Error(t, err) f1 := &factorymocks.Factory{} f1.On("KubebuilderClient").Return(crClient, nil) plugin1 := NewVolumeSnapshotContentRestoreItemAction(f1) _, err1 := plugin1(logger) require.NoError(t, err1) } ================================================ FILE: pkg/restore/actions/dataupload_retrieve_action.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actions import ( "context" "encoding/json" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" veleroclient "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/label" "github.com/vmware-tanzu/velero/pkg/plugin/velero" ) type DataUploadRetrieveAction struct { logger logrus.FieldLogger client client.Client } func NewDataUploadRetrieveAction(logger logrus.FieldLogger, client client.Client) *DataUploadRetrieveAction { return &DataUploadRetrieveAction{ logger: logger, client: client, } } func (d *DataUploadRetrieveAction) AppliesTo() (velero.ResourceSelector, error) { return velero.ResourceSelector{ IncludedResources: []string{"datauploads.velero.io"}, }, nil } func (d *DataUploadRetrieveAction) Execute(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { d.logger.Info("Executing DataUploadRetrieveAction") dataUpload := velerov2alpha1.DataUpload{} if err := runtime.DefaultUnstructuredConverter.FromUnstructured(input.ItemFromBackup.UnstructuredContent(), &dataUpload); err != nil { d.logger.Errorf("unable to convert unstructured item to DataUpload: %s", err.Error()) return nil, errors.Wrap(err, "unable to convert unstructured item to DataUpload.") } backup := &velerov1api.Backup{} err := d.client.Get(context.Background(), types.NamespacedName{ Namespace: input.Restore.Namespace, Name: input.Restore.Spec.BackupName, }, backup) if err != nil { d.logger.WithError(err).Errorf("Fail to get backup for restore %s.", input.Restore.Name) return nil, errors.Wrapf(err, "error to get backup for restore %s", input.Restore.Name) } dataUploadResult := velerov2alpha1.DataUploadResult{ BackupStorageLocation: backup.Spec.StorageLocation, DataMover: dataUpload.Spec.DataMover, SnapshotID: dataUpload.Status.SnapshotID, SnapshotSize: dataUpload.Status.Progress.TotalBytes, SourceNamespace: dataUpload.Spec.SourceNamespace, DataMoverResult: dataUpload.Status.DataMoverResult, NodeOS: dataUpload.Status.NodeOS, } jsonBytes, err := json.Marshal(dataUploadResult) if err != nil { d.logger.Errorf("fail to convert DataUploadResult to JSON: %s", err.Error()) return nil, errors.Wrap(err, "fail to convert DataUploadResult to JSON") } cm := corev1api.ConfigMap{ TypeMeta: metav1.TypeMeta{ Kind: "ConfigMap", APIVersion: corev1api.SchemeGroupVersion.String(), }, ObjectMeta: metav1.ObjectMeta{ GenerateName: dataUpload.Name + "-", Namespace: input.Restore.Namespace, Labels: map[string]string{ velerov1api.RestoreUIDLabel: label.GetValidName(string(input.Restore.UID)), velerov1api.PVCNamespaceNameLabel: label.GetValidName(dataUpload.Spec.SourceNamespace + "." + dataUpload.Spec.SourcePVC), velerov1api.ResourceUsageLabel: label.GetValidName(string(velerov1api.VeleroResourceUsageDataUploadResult)), }, }, Data: map[string]string{ string(input.Restore.UID): string(jsonBytes), }, } err = veleroclient.CreateRetryGenerateName(d.client, context.Background(), &cm) if err != nil { d.logger.Errorf("fail to create DataUploadResult ConfigMap %s/%s: %s", cm.Namespace, cm.Name, err.Error()) return nil, errors.Wrap(err, "fail to create DataUploadResult ConfigMap") } return &velero.RestoreItemActionExecuteOutput{ SkipRestore: true, }, nil } ================================================ FILE: pkg/restore/actions/dataupload_retrieve_action_test.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actions import ( "testing" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/label" "github.com/vmware-tanzu/velero/pkg/plugin/velero" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func TestDataUploadRetrieveActionExectue(t *testing.T) { scheme := runtime.NewScheme() velerov1.AddToScheme(scheme) corev1api.AddToScheme(scheme) tests := []struct { name string dataUpload *velerov2alpha1.DataUpload restore *velerov1.Restore expectedDataUploadResult *corev1api.ConfigMap expectedErr string runtimeScheme *runtime.Scheme veleroObjs []runtime.Object }{ { name: "error to find backup", dataUpload: builder.ForDataUpload("velero", "testDU").SourceNamespace("testNamespace").SourcePVC("testPVC").Result(), restore: builder.ForRestore("velero", "testRestore").ObjectMeta(builder.WithUID("testingUID")).Backup("testBackup").Result(), runtimeScheme: scheme, expectedErr: "error to get backup for restore testRestore: backups.velero.io \"testBackup\" not found", }, { name: "DataUploadRetrieve Action test", dataUpload: builder.ForDataUpload("velero", "testDU").SourceNamespace("testNamespace").SourcePVC("testPVC").SnapshotID("fake-id").TotalBytes(1000).Result(), restore: builder.ForRestore("velero", "testRestore").ObjectMeta(builder.WithUID("testingUID")).Backup("testBackup").Result(), runtimeScheme: scheme, veleroObjs: []runtime.Object{ builder.ForBackup("velero", "testBackup").StorageLocation("testLocation").Result(), }, expectedDataUploadResult: builder.ForConfigMap("velero", "").ObjectMeta(builder.WithGenerateName("testDU-"), builder.WithLabels(velerov1.PVCNamespaceNameLabel, "testNamespace.testPVC", velerov1.RestoreUIDLabel, "testingUID", velerov1.ResourceUsageLabel, string(velerov1.VeleroResourceUsageDataUploadResult))).Data("testingUID", `{"backupStorageLocation":"testLocation","snapshotID":"fake-id","sourceNamespace":"testNamespace","snapshotSize":1000}`).Result(), }, { name: "Long source namespace and PVC name should also work", dataUpload: builder.ForDataUpload("velero", "testDU").SourceNamespace("migre209d0da-49c7-45ba-8d5a-3e59fd591ec1").SourcePVC("kibishii-data-kibishii-deployment-0").Result(), restore: builder.ForRestore("velero", "testRestore").ObjectMeta(builder.WithUID("testingUID")).Backup("testBackup").Result(), runtimeScheme: scheme, veleroObjs: []runtime.Object{ builder.ForBackup("velero", "testBackup").StorageLocation("testLocation").Result(), }, expectedDataUploadResult: builder.ForConfigMap("velero", "").ObjectMeta(builder.WithGenerateName("testDU-"), builder.WithLabels(velerov1.PVCNamespaceNameLabel, "migre209d0da-49c7-45ba-8d5a-3e59fd591ec1.kibishii-data-ki152333", velerov1.RestoreUIDLabel, "testingUID", velerov1.ResourceUsageLabel, string(velerov1.VeleroResourceUsageDataUploadResult))).Data("testingUID", `{"backupStorageLocation":"testLocation","sourceNamespace":"migre209d0da-49c7-45ba-8d5a-3e59fd591ec1"}`).Result(), }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { logger := velerotest.NewLogger() fakeClientBuilder := fake.NewClientBuilder() if tc.runtimeScheme != nil { fakeClientBuilder = fakeClientBuilder.WithScheme(tc.runtimeScheme) } fakeClient := fakeClientBuilder.WithRuntimeObjects(tc.veleroObjs...).Build() var unstructuredDataUpload map[string]any if tc.dataUpload != nil { var err error unstructuredDataUpload, err = runtime.DefaultUnstructuredConverter.ToUnstructured(tc.dataUpload) require.NoError(t, err) } input := velero.RestoreItemActionExecuteInput{ Restore: tc.restore, ItemFromBackup: &unstructured.Unstructured{Object: unstructuredDataUpload}, } action := NewDataUploadRetrieveAction(logger, fakeClient) _, err := action.Execute(&input) if tc.expectedErr != "" { require.Equal(t, tc.expectedErr, err.Error()) } else { require.NoError(t, err) } if tc.expectedDataUploadResult != nil { var cmList corev1api.ConfigMapList err := fakeClient.List(t.Context(), &cmList, &client.ListOptions{ LabelSelector: labels.SelectorFromSet(map[string]string{ velerov1.RestoreUIDLabel: "testingUID", velerov1.PVCNamespaceNameLabel: label.GetValidName(tc.dataUpload.Spec.SourceNamespace + "." + tc.dataUpload.Spec.SourcePVC), }), }) require.NoError(t, err) require.Equal(t, tc.expectedDataUploadResult.Labels, cmList.Items[0].Labels) require.Equal(t, tc.expectedDataUploadResult.Data, cmList.Items[0].Data) } }) } } ================================================ FILE: pkg/restore/actions/init_restorehook_pod_action.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actions import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "github.com/vmware-tanzu/velero/internal/hook" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/plugin/velero" ) // InitRestoreHookPodAction is a RestoreItemAction plugin applicable to pods that runs // restore hooks to add init containers to pods prior to them being restored. type InitRestoreHookPodAction struct { logger logrus.FieldLogger } // NewInitRestoreHookPodAction returns a new InitRestoreHookPodAction. func NewInitRestoreHookPodAction(logger logrus.FieldLogger) *InitRestoreHookPodAction { return &InitRestoreHookPodAction{logger: logger} } // AppliesTo implements the RestoreItemAction plugin interface method. func (a *InitRestoreHookPodAction) AppliesTo() (velero.ResourceSelector, error) { return velero.ResourceSelector{ IncludedResources: []string{"pods"}, }, nil } // Execute implements the RestoreItemAction plugin interface method. func (a *InitRestoreHookPodAction) Execute(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { a.logger.Infof("Executing InitRestoreHookPodAction") // handle any init container restore hooks for the pod restoreHooks, err := hook.GetRestoreHooksFromSpec(&input.Restore.Spec.Hooks) nsMapping := input.Restore.Spec.NamespaceMapping if err != nil { return nil, errors.WithStack(err) } hookHandler := hook.InitContainerRestoreHookHandler{} postHooksItem, err := hookHandler.HandleRestoreHooks(a.logger, kuberesource.Pods, input.Item, restoreHooks, nsMapping) if err != nil { return nil, errors.WithStack(err) } a.logger.Infof("Returning from InitRestoreHookPodAction") return velero.NewRestoreItemActionExecuteOutput(&unstructured.Unstructured{Object: postHooksItem.UnstructuredContent()}), nil } ================================================ FILE: pkg/restore/actions/init_restorehook_pod_action_test.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actions import ( "testing" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/plugin/velero" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func TestInitContainerRestoreHookPodActionExecute(t *testing.T) { testCases := []struct { name string obj *corev1api.Pod expectedErr bool expectedRes *corev1api.Pod restore *velerov1api.Restore }{ { name: "should run restore hooks from pod annotation", restore: &velerov1api.Restore{}, obj: builder.ForPod("default", "app1"). ObjectMeta(builder.WithAnnotations( "init.hook.restore.velero.io/container-image", "nginx", "init.hook.restore.velero.io/container-name", "restore-init-container", "init.hook.restore.velero.io/command", `["a", "b", "c"]`, )). ServiceAccount("foo"). Volumes([]*corev1api.Volume{{Name: "foo"}}...). InitContainers([]*corev1api.Container{ builder.ForContainer("init-app-step1", "busy-box"). Command([]string{"init-step1"}).Result(), builder.ForContainer("init-app-step2", "busy-box"). Command([]string{"init-step2"}).Result(), builder.ForContainer("init-app-step3", "busy-box"). Command([]string{"init-step3"}).Result()}...).Result(), expectedRes: builder.ForPod("default", "app1"). ObjectMeta(builder.WithAnnotations( "init.hook.restore.velero.io/container-image", "nginx", "init.hook.restore.velero.io/container-name", "restore-init-container", "init.hook.restore.velero.io/command", `["a", "b", "c"]`, )). ServiceAccount("foo"). Volumes([]*corev1api.Volume{{Name: "foo"}}...). InitContainers([]*corev1api.Container{ builder.ForContainer("restore-init-container", "nginx"). Command([]string{"a", "b", "c"}).Result(), builder.ForContainer("init-app-step1", "busy-box"). Command([]string{"init-step1"}).Result(), builder.ForContainer("init-app-step2", "busy-box"). Command([]string{"init-step2"}).Result(), builder.ForContainer("init-app-step3", "busy-box"). Command([]string{"init-step3"}).Result()}...).Result(), }, { name: "should run restore hook from restore spec", restore: &velerov1api.Restore{ Spec: velerov1api.RestoreSpec{ Hooks: velerov1api.RestoreHooks{ Resources: []velerov1api.RestoreResourceHookSpec{ { Name: "h1", IncludedNamespaces: []string{"default"}, IncludedResources: []string{kuberesource.Pods.Resource}, PostHooks: []velerov1api.RestoreResourceHook{ { Init: &velerov1api.InitRestoreHook{ InitContainers: []runtime.RawExtension{ builder.ForContainer("restore-init1", "busy-box"). Command([]string{"foobarbaz"}).ResultRawExtension(), builder.ForContainer("restore-init2", "busy-box"). Command([]string{"foobarbaz"}).ResultRawExtension(), }, }, }, }, }, }, }, }, }, obj: builder.ForPod("default", "app1"). ServiceAccount("foo"). Volumes([]*corev1api.Volume{{Name: "foo"}}...).Result(), expectedRes: builder.ForPod("default", "app1"). ServiceAccount("foo"). Volumes([]*corev1api.Volume{{Name: "foo"}}...). InitContainers([]*corev1api.Container{ builder.ForContainer("restore-init1", "busy-box"). Command([]string{"foobarbaz"}).Result(), builder.ForContainer("restore-init2", "busy-box"). Command([]string{"foobarbaz"}).Result(), }...).Result(), }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { action := NewInitRestoreHookPodAction(velerotest.NewLogger()) unstructuredPod, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&tc.obj) require.NoError(t, err) res, err := action.Execute(&velero.RestoreItemActionExecuteInput{ Item: &unstructured.Unstructured{Object: unstructuredPod}, ItemFromBackup: &unstructured.Unstructured{Object: unstructuredPod}, Restore: tc.restore, }) if tc.expectedErr { assert.Error(t, err, "expected an error") return } require.NoError(t, err, "expected no error, got %v", err) var pod corev1api.Pod require.NoError(t, runtime.DefaultUnstructuredConverter.FromUnstructured(res.UpdatedItem.UnstructuredContent(), &pod)) assert.Equal(t, *tc.expectedRes, pod) }) } } ================================================ FILE: pkg/restore/actions/job_action.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actions import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" batchv1api "k8s.io/api/batch/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "github.com/vmware-tanzu/velero/pkg/plugin/velero" ) const ( legacyControllerUIDLabel = "controller-uid" // <=1.27 This still exists in 1.27 for backward compatibility, maybe remove in 1.28? controllerUIDLabel = "batch.kubernetes.io/controller-uid" // >=1.27 https://github.com/kubernetes/kubernetes/pull/114930#issuecomment-1384667494 ) type JobAction struct { logger logrus.FieldLogger } func NewJobAction(logger logrus.FieldLogger) *JobAction { return &JobAction{logger: logger} } func (a *JobAction) AppliesTo() (velero.ResourceSelector, error) { return velero.ResourceSelector{ IncludedResources: []string{"jobs"}, }, nil } func (a *JobAction) Execute(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { job := new(batchv1api.Job) if err := runtime.DefaultUnstructuredConverter.FromUnstructured(input.Item.UnstructuredContent(), job); err != nil { return nil, errors.WithStack(err) } if job.Spec.Selector != nil { delete(job.Spec.Selector.MatchLabels, controllerUIDLabel) delete(job.Spec.Selector.MatchLabels, legacyControllerUIDLabel) } delete(job.Spec.Template.ObjectMeta.Labels, controllerUIDLabel) delete(job.Spec.Template.ObjectMeta.Labels, legacyControllerUIDLabel) res, err := runtime.DefaultUnstructuredConverter.ToUnstructured(job) if err != nil { return nil, errors.WithStack(err) } return velero.NewRestoreItemActionExecuteOutput(&unstructured.Unstructured{Object: res}), nil } ================================================ FILE: pkg/restore/actions/job_action_test.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actions import ( "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" batchv1api "k8s.io/api/batch/v1" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "github.com/vmware-tanzu/velero/pkg/plugin/velero" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func TestJobActionExecute(t *testing.T) { tests := []struct { name string obj batchv1api.Job expectedErr bool expectedRes batchv1api.Job }{ { name: "missing spec.selector and/or spec.template should not error", obj: batchv1api.Job{ ObjectMeta: metav1.ObjectMeta{Name: "job-1"}, }, expectedRes: batchv1api.Job{ ObjectMeta: metav1.ObjectMeta{Name: "job-1"}, }, }, { name: "missing spec.selector.matchLabels should not error", obj: batchv1api.Job{ ObjectMeta: metav1.ObjectMeta{Name: "job-1"}, Spec: batchv1api.JobSpec{ Selector: new(metav1.LabelSelector), }, }, expectedRes: batchv1api.Job{ ObjectMeta: metav1.ObjectMeta{Name: "job-1"}, Spec: batchv1api.JobSpec{ Selector: new(metav1.LabelSelector), }, }, }, { name: "spec.selector.matchLabels[controller-uid] is removed", obj: batchv1api.Job{ ObjectMeta: metav1.ObjectMeta{Name: "job-1"}, Spec: batchv1api.JobSpec{ Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "controller-uid": "foo", "hello": "world", }, }, }, }, expectedRes: batchv1api.Job{ ObjectMeta: metav1.ObjectMeta{Name: "job-1"}, Spec: batchv1api.JobSpec{ Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "hello": "world", }, }, }, }, }, { name: "missing spec.template.metadata.labels should not error", obj: batchv1api.Job{ ObjectMeta: metav1.ObjectMeta{Name: "job-1"}, Spec: batchv1api.JobSpec{ Template: corev1api.PodTemplateSpec{}, }, }, expectedRes: batchv1api.Job{ ObjectMeta: metav1.ObjectMeta{Name: "job-1"}, Spec: batchv1api.JobSpec{ Template: corev1api.PodTemplateSpec{}, }, }, }, { name: "spec.template.metadata.labels[controller-uid] is removed", obj: batchv1api.Job{ ObjectMeta: metav1.ObjectMeta{Name: "job-1"}, Spec: batchv1api.JobSpec{ Template: corev1api.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ "controller-uid": "foo", "hello": "world", }, }, }, }, }, expectedRes: batchv1api.Job{ ObjectMeta: metav1.ObjectMeta{Name: "job-1"}, Spec: batchv1api.JobSpec{ Template: corev1api.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ "hello": "world", }, }, }, }, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { action := NewJobAction(velerotest.NewLogger()) unstructuredJob, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&test.obj) require.NoError(t, err) res, err := action.Execute(&velero.RestoreItemActionExecuteInput{ Item: &unstructured.Unstructured{Object: unstructuredJob}, ItemFromBackup: &unstructured.Unstructured{Object: unstructuredJob}, Restore: nil, }) if assert.Equal(t, test.expectedErr, err != nil) { var job batchv1api.Job require.NoError(t, runtime.DefaultUnstructuredConverter.FromUnstructured(res.UpdatedItem.UnstructuredContent(), &job)) assert.Equal(t, test.expectedRes, job) } }) } } ================================================ FILE: pkg/restore/actions/pod_action.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actions import ( "strings" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/plugin/velero" ) type PodAction struct { logger logrus.FieldLogger } func NewPodAction(logger logrus.FieldLogger) *PodAction { return &PodAction{logger: logger} } func (a *PodAction) AppliesTo() (velero.ResourceSelector, error) { return velero.ResourceSelector{ IncludedResources: []string{"pods"}, }, nil } func (a *PodAction) Execute(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { pod := new(corev1api.Pod) if err := runtime.DefaultUnstructuredConverter.FromUnstructured(input.Item.UnstructuredContent(), pod); err != nil { return nil, errors.WithStack(err) } pod.Spec.NodeName = "" pod.Spec.Priority = nil serviceAccountTokenPrefix := pod.Spec.ServiceAccountName + "-token-" var preservedVolumes []corev1api.Volume for _, vol := range pod.Spec.Volumes { if !strings.HasPrefix(vol.Name, serviceAccountTokenPrefix) { preservedVolumes = append(preservedVolumes, vol) } } pod.Spec.Volumes = preservedVolumes for i, container := range pod.Spec.Containers { var preservedVolumeMounts []corev1api.VolumeMount for _, mount := range container.VolumeMounts { if !strings.HasPrefix(mount.Name, serviceAccountTokenPrefix) { preservedVolumeMounts = append(preservedVolumeMounts, mount) } } pod.Spec.Containers[i].VolumeMounts = preservedVolumeMounts } for i, container := range pod.Spec.InitContainers { var preservedVolumeMounts []corev1api.VolumeMount for _, mount := range container.VolumeMounts { if !strings.HasPrefix(mount.Name, serviceAccountTokenPrefix) { preservedVolumeMounts = append(preservedVolumeMounts, mount) } } pod.Spec.InitContainers[i].VolumeMounts = preservedVolumeMounts } res, err := runtime.DefaultUnstructuredConverter.ToUnstructured(pod) if err != nil { return nil, errors.WithStack(err) } restoreExecuteOutput := velero.NewRestoreItemActionExecuteOutput(&unstructured.Unstructured{Object: res}) if pod.Spec.PriorityClassName != "" { a.logger.Infof("Adding priorityclass %s to AdditionalItems", pod.Spec.PriorityClassName) restoreExecuteOutput.AdditionalItems = []velero.ResourceIdentifier{ {GroupResource: kuberesource.PriorityClasses, Name: pod.Spec.PriorityClassName}} } return restoreExecuteOutput, nil } ================================================ FILE: pkg/restore/actions/pod_action_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actions import ( "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/plugin/velero" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func TestPodActionExecute(t *testing.T) { var priority int32 = 1 tests := []struct { name string obj corev1api.Pod expectedErr bool expectedRes corev1api.Pod additionalItems []velero.ResourceIdentifier }{ { name: "nodeName (only) should be deleted from spec", obj: corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "pod-1"}, Spec: corev1api.PodSpec{ NodeName: "foo", ServiceAccountName: "bar", }, }, expectedRes: corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "pod-1"}, Spec: corev1api.PodSpec{ ServiceAccountName: "bar", }, }, }, { name: "priority (only) should be deleted from spec", obj: corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "pod-1"}, Spec: corev1api.PodSpec{ Priority: &priority, ServiceAccountName: "bar", }, }, expectedRes: corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "pod-1"}, Spec: corev1api.PodSpec{ ServiceAccountName: "bar", }, }, }, { name: "volumes matching prefix -token- should be deleted", obj: corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "pod-1"}, Spec: corev1api.PodSpec{ ServiceAccountName: "foo", Volumes: []corev1api.Volume{ {Name: "foo"}, {Name: "foo-token-foo"}, }, }, }, expectedRes: corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "pod-1"}, Spec: corev1api.PodSpec{ ServiceAccountName: "foo", Volumes: []corev1api.Volume{ {Name: "foo"}, }, }, }, }, { name: "container volumeMounts matching prefix -token- should be deleted", obj: corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "pod-1"}, Spec: corev1api.PodSpec{ ServiceAccountName: "foo", Volumes: []corev1api.Volume{ {Name: "foo"}, {Name: "foo-token-foo"}, }, Containers: []corev1api.Container{ { VolumeMounts: []corev1api.VolumeMount{ {Name: "foo"}, {Name: "foo-token-foo"}, }, }, }, }, }, expectedRes: corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "pod-1"}, Spec: corev1api.PodSpec{ ServiceAccountName: "foo", Volumes: []corev1api.Volume{ {Name: "foo"}, }, Containers: []corev1api.Container{ { VolumeMounts: []corev1api.VolumeMount{ {Name: "foo"}, }, }, }, }, }, }, { name: "initContainer volumeMounts matching prefix -token- should be deleted", obj: corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "pod-1"}, Spec: corev1api.PodSpec{ ServiceAccountName: "foo", Volumes: []corev1api.Volume{ {Name: "foo"}, {Name: "foo-token-foo"}, }, InitContainers: []corev1api.Container{ { VolumeMounts: []corev1api.VolumeMount{ {Name: "foo"}, {Name: "foo-token-foo"}, }, }, }, }, }, expectedRes: corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "pod-1"}, Spec: corev1api.PodSpec{ ServiceAccountName: "foo", Volumes: []corev1api.Volume{ {Name: "foo"}, }, InitContainers: []corev1api.Container{ { VolumeMounts: []corev1api.VolumeMount{ {Name: "foo"}, }, }, }, }, }, }, { name: "containers and initContainers with no volume mounts should not error", obj: corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "pod-1"}, Spec: corev1api.PodSpec{ ServiceAccountName: "foo", Volumes: []corev1api.Volume{ {Name: "foo"}, {Name: "foo-token-foo"}, }, }, }, expectedRes: corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "pod-1"}, Spec: corev1api.PodSpec{ ServiceAccountName: "foo", Volumes: []corev1api.Volume{ {Name: "foo"}, }, }, }, }, { name: "test priority class", obj: corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "pod-1"}, Spec: corev1api.PodSpec{ ServiceAccountName: "foo", PriorityClassName: "testPriorityClass", Volumes: []corev1api.Volume{ {Name: "foo"}, {Name: "foo-token-foo"}, }, }, }, expectedRes: corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "pod-1"}, Spec: corev1api.PodSpec{ ServiceAccountName: "foo", PriorityClassName: "testPriorityClass", Volumes: []corev1api.Volume{ {Name: "foo"}, }, }, }, additionalItems: []velero.ResourceIdentifier{ {GroupResource: kuberesource.PriorityClasses, Name: "testPriorityClass", }, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { action := NewPodAction(velerotest.NewLogger()) unstructuredPod, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&test.obj) require.NoError(t, err) res, err := action.Execute(&velero.RestoreItemActionExecuteInput{ Item: &unstructured.Unstructured{Object: unstructuredPod}, ItemFromBackup: &unstructured.Unstructured{Object: unstructuredPod}, Restore: nil, }) if test.expectedErr { assert.Error(t, err, "expected an error") return } require.NoError(t, err, "expected no error, got %v", err) var pod corev1api.Pod require.NoError(t, runtime.DefaultUnstructuredConverter.FromUnstructured(res.UpdatedItem.UnstructuredContent(), &pod)) assert.Equal(t, test.expectedRes, pod) assert.Equal(t, test.additionalItems, res.AdditionalItems) }) } } ================================================ FILE: pkg/restore/actions/pod_volume_restore_action.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actions import ( "context" "fmt" "strings" "github.com/vmware-tanzu/velero/pkg/util/boolptr" "github.com/pkg/errors" "github.com/sirupsen/logrus" appsv1api "k8s.io/api/apps/v1" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" veleroimage "github.com/vmware-tanzu/velero/internal/velero" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/label" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" "github.com/vmware-tanzu/velero/pkg/plugin/velero" "github.com/vmware-tanzu/velero/pkg/podvolume" "github.com/vmware-tanzu/velero/pkg/restorehelper" "github.com/vmware-tanzu/velero/pkg/util/kube" veleroutil "github.com/vmware-tanzu/velero/pkg/util/velero" ) const ( defaultCPURequestLimit = "100m" defaultMemRequestLimit = "128Mi" defaultCommand = "/velero-restore-helper" restoreHelperUID = 1000 ) type PodVolumeRestoreAction struct { logger logrus.FieldLogger client corev1client.ConfigMapInterface crClient ctrlclient.Client veleroImage string } func NewPodVolumeRestoreAction(logger logrus.FieldLogger, client corev1client.ConfigMapInterface, crClient ctrlclient.Client, namespace string) (*PodVolumeRestoreAction, error) { deployment := &appsv1api.Deployment{} if err := crClient.Get(context.TODO(), types.NamespacedName{Name: "velero", Namespace: namespace}, deployment); err != nil { return nil, err } image := veleroutil.GetVeleroServerImage(deployment) return &PodVolumeRestoreAction{ logger: logger, client: client, crClient: crClient, veleroImage: image, }, nil } func (a *PodVolumeRestoreAction) AppliesTo() (velero.ResourceSelector, error) { return velero.ResourceSelector{ IncludedResources: []string{"pods"}, }, nil } func (a *PodVolumeRestoreAction) Execute(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { a.logger.Info("Executing PodVolumeRestoreAction") defer a.logger.Info("Done executing PodVolumeRestoreAction") var pod corev1api.Pod if err := runtime.DefaultUnstructuredConverter.FromUnstructured(input.Item.UnstructuredContent(), &pod); err != nil { return nil, errors.Wrap(err, "unable to convert pod from runtime.Unstructured") } // At the point when this function is called, the namespace mapping for the restore // has not yet been applied to `input.Item` so we can't perform a reverse-lookup in // the namespace mapping in the restore spec. Instead, use the pod from the backup // so that if the mapping is applied earlier, we still use the correct namespace. var podFromBackup corev1api.Pod if err := runtime.DefaultUnstructuredConverter.FromUnstructured(input.ItemFromBackup.UnstructuredContent(), &podFromBackup); err != nil { return nil, errors.Wrap(err, "unable to convert source pod from runtime.Unstructured") } log := a.logger.WithField("pod", kube.NamespaceAndName(&pod)) opts := &ctrlclient.ListOptions{ LabelSelector: label.NewSelectorForBackup(input.Restore.Spec.BackupName), } podVolumeBackupList := new(velerov1api.PodVolumeBackupList) if err := a.crClient.List(context.TODO(), podVolumeBackupList, opts); err != nil { return nil, errors.WithStack(err) } var podVolumeBackups []*velerov1api.PodVolumeBackup for i := range podVolumeBackupList.Items { podVolumeBackups = append(podVolumeBackups, &podVolumeBackupList.Items[i]) } // Remove all existing restore-wait init containers first to prevent duplicates // This ensures that even if the pod was previously restored with file system backup // but now backed up with native datamover or CSI, the unnecessary init container is removed var filteredInitContainers []corev1api.Container removedCount := 0 for _, initContainer := range pod.Spec.InitContainers { if initContainer.Name != restorehelper.WaitInitContainer && initContainer.Name != restorehelper.WaitInitContainerLegacy { filteredInitContainers = append(filteredInitContainers, initContainer) } else { removedCount++ } } pod.Spec.InitContainers = filteredInitContainers if removedCount > 0 { log.Infof("Removed %d existing restore-wait init container(s)", removedCount) } volumeSnapshots := podvolume.GetVolumeBackupsForPod(podVolumeBackups, &pod, podFromBackup.Namespace) if len(volumeSnapshots) == 0 { log.Debug("No pod volume backups found for pod") res, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&pod) if err != nil { return nil, errors.Wrap(err, "unable to convert pod to runtime.Unstructured") } return velero.NewRestoreItemActionExecuteOutput(&unstructured.Unstructured{Object: res}), nil } log.Info("Pod volume backups for pod found") log.Debugf("Getting plugin config") config, err := common.GetPluginConfig(common.PluginKindRestoreItemAction, "velero.io/pod-volume-restore", a.client) if err != nil { return nil, err } image := getImage(log, config, a.veleroImage) log.Infof("Using image %q", image) cpuRequest, memRequest := getResourceRequests(log, config) cpuLimit, memLimit := getResourceLimits(log, config) if cpuRequest == "" { cpuRequest = defaultCPURequestLimit } if cpuLimit == "" { cpuLimit = defaultCPURequestLimit } if memRequest == "" { memRequest = defaultMemRequestLimit } if memLimit == "" { memLimit = defaultMemRequestLimit } resourceReqs, err := kube.ParseCPUAndMemoryResources( cpuRequest, memRequest, cpuLimit, memLimit, ) if err != nil { log.Errorf("couldn't parse resource requirements: %s.", err) resourceReqs, _ = kube.ParseCPUAndMemoryResources( defaultCPURequestLimit, defaultMemRequestLimit, defaultCPURequestLimit, defaultMemRequestLimit, ) } runAsUser, runAsGroup, allowPrivilegeEscalation, secCtx := getSecurityContext(log, config) var securityContext corev1api.SecurityContext securityContextSet := false // Use securityContext settings from configmap if available if runAsUser != "" || runAsGroup != "" || allowPrivilegeEscalation != "" || secCtx != "" { securityContext, err = kube.ParseSecurityContext(runAsUser, runAsGroup, allowPrivilegeEscalation, secCtx) if err != nil { log.Errorf("Using default securityContext values, couldn't parse securityContext requirements: %s.", err) } else { securityContextSet = true } } // if securityContext configmap is unavailable but first container in pod has a SecurityContext set, then copy this security context if !securityContextSet && len(pod.Spec.Containers) != 0 && pod.Spec.Containers[0].SecurityContext != nil { securityContext = *pod.Spec.Containers[0].SecurityContext.DeepCopy() securityContextSet = true } if !securityContextSet { securityContext = defaultSecurityCtx() } initContainerBuilder := newRestoreInitContainerBuilder(image, string(input.Restore.UID)) initContainerBuilder.Resources(&resourceReqs) initContainerBuilder.SecurityContext(&securityContext) for volumeName := range volumeSnapshots { mount := &corev1api.VolumeMount{ Name: volumeName, MountPath: "/restores/" + volumeName, } initContainerBuilder.VolumeMounts(mount) } initContainerBuilder.Command(getCommand(log, config)) initContainer := *initContainerBuilder.Result() // Since we've already removed all restore-wait init containers above, // we can simply prepend the new init container pod.Spec.InitContainers = append([]corev1api.Container{initContainer}, pod.Spec.InitContainers...) res, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&pod) if err != nil { return nil, errors.Wrap(err, "unable to convert pod to runtime.Unstructured") } return velero.NewRestoreItemActionExecuteOutput(&unstructured.Unstructured{Object: res}), nil } func getCommand(log logrus.FieldLogger, config *corev1api.ConfigMap) []string { if config == nil { log.Debug("No config found for plugin") return []string{defaultCommand} } if config.Data["command"] == "" { log.Debugf("No custom command configured") return []string{defaultCommand} } log.Debugf("Using custom command %s", config.Data["command"]) return []string{config.Data["command"]} } func getImage(log logrus.FieldLogger, config *corev1api.ConfigMap, defaultImage string) string { if config == nil { log.Debug("No config found for plugin") return defaultImage } image := config.Data["image"] if image == "" { log.Debugf("No custom image configured") return defaultImage } log = log.WithField("image", image) parts := strings.Split(image, "/") if len(parts) == 1 { // Image supplied without registry part log.Infof("Plugin config contains image name without registry name. Using default init container image: %q", defaultImage) return defaultImage } if !(strings.Contains(parts[len(parts)-1], ":")) { tag := veleroimage.ImageTag() // tag-less image name: add default image tag for this version of Velero log.Infof("Plugin config contains image name without tag. Adding tag: %q", tag) return fmt.Sprintf("%s:%s", image, tag) } // tagged image name log.Debugf("Plugin config contains image name with tag") return image } // getResourceRequests extracts the CPU and memory requests from a ConfigMap. // The 0 values are valid if the keys are not present func getResourceRequests(log logrus.FieldLogger, config *corev1api.ConfigMap) (string, string) { if config == nil { log.Debug("No config found for plugin") return "", "" } return config.Data["cpuRequest"], config.Data["memRequest"] } // getResourceLimits extracts the CPU and memory limits from a ConfigMap. // The 0 values are valid if the keys are not present func getResourceLimits(log logrus.FieldLogger, config *corev1api.ConfigMap) (string, string) { if config == nil { log.Debug("No config found for plugin") return "", "" } return config.Data["cpuLimit"], config.Data["memLimit"] } // getSecurityContext extracts securityContext runAsUser, runAsGroup, allowPrivilegeEscalation, and securityContext from a ConfigMap. func getSecurityContext(log logrus.FieldLogger, config *corev1api.ConfigMap) (string, string, string, string) { if config == nil { log.Debug("No config found for plugin") return "", "", "", "" } return config.Data["secCtxRunAsUser"], config.Data["secCtxRunAsGroup"], config.Data["secCtxAllowPrivilegeEscalation"], config.Data["secCtx"] } func newRestoreInitContainerBuilder(image, restoreUID string) *builder.ContainerBuilder { return builder.ForContainer(restorehelper.WaitInitContainer, image). Args(restoreUID). Env([]*corev1api.EnvVar{ { Name: "POD_NAMESPACE", ValueFrom: &corev1api.EnvVarSource{ FieldRef: &corev1api.ObjectFieldSelector{ FieldPath: "metadata.namespace", }, }, }, { Name: "POD_NAME", ValueFrom: &corev1api.EnvVarSource{ FieldRef: &corev1api.ObjectFieldSelector{ FieldPath: "metadata.name", }, }, }, }...) } // defaultSecurityCtx returns a default security context for the init container, which has the level "restricted" per // Pod Security Standards. func defaultSecurityCtx() corev1api.SecurityContext { uid := int64(restoreHelperUID) return corev1api.SecurityContext{ AllowPrivilegeEscalation: boolptr.False(), Capabilities: &corev1api.Capabilities{ Drop: []corev1api.Capability{"ALL"}, }, SeccompProfile: &corev1api.SeccompProfile{ Type: corev1api.SeccompProfileTypeRuntimeDefault, }, RunAsUser: &uid, RunAsNonRoot: boolptr.True(), } } ================================================ FILE: pkg/restore/actions/pod_volume_restore_action_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actions import ( "sort" "testing" "github.com/vmware-tanzu/velero/pkg/util/boolptr" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" appsv1api "k8s.io/api/apps/v1" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes/fake" crfake "sigs.k8s.io/controller-runtime/pkg/client/fake" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/buildinfo" "github.com/vmware-tanzu/velero/pkg/plugin/velero" velerotest "github.com/vmware-tanzu/velero/pkg/test" "github.com/vmware-tanzu/velero/pkg/util/kube" "k8s.io/client-go/kubernetes/scheme" "github.com/vmware-tanzu/velero/pkg/restorehelper" ) func TestGetImage(t *testing.T) { configMapWithData := func(key, val string) *corev1api.ConfigMap { return &corev1api.ConfigMap{ Data: map[string]string{ key: val, }, } } defaultImage := "velero/velero:v1.0" tests := []struct { name string configMap *corev1api.ConfigMap buildInfoVersion string want string }{ { name: "nil config map returns default image", configMap: nil, want: defaultImage, }, { name: "config map without 'image' key returns default image", configMap: configMapWithData("non-matching-key", "val"), want: defaultImage, }, { name: "config map without '/' in image name returns default image", configMap: configMapWithData("image", "my-image"), want: defaultImage, }, { name: "config map with untagged image returns image with buildinfo.Version as tag", configMap: configMapWithData("image", "myregistry.io/my-image"), buildInfoVersion: "buildinfo-version", want: "myregistry.io/my-image:buildinfo-version", }, { name: "config map with untagged image and custom registry port with ':' returns image with buildinfo.Version as tag", configMap: configMapWithData("image", "myregistry.io:34567/my-image"), buildInfoVersion: "buildinfo-version", want: "myregistry.io:34567/my-image:buildinfo-version", }, { name: "config map with tagged image returns tagged image", configMap: configMapWithData("image", "myregistry.io/my-image:my-tag"), want: "myregistry.io/my-image:my-tag", }, { name: "config map with tagged image and custom registry port with ':' returns tagged image", configMap: configMapWithData("image", "myregistry.io:34567/my-image:my-tag"), want: "myregistry.io:34567/my-image:my-tag", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { if test.buildInfoVersion != "" { originalVersion := buildinfo.Version buildinfo.Version = test.buildInfoVersion defer func() { buildinfo.Version = originalVersion }() } assert.Equal(t, test.want, getImage(velerotest.NewLogger(), test.configMap, defaultImage)) }) } } // TestPodVolumeRestoreActionExecute tests the pod volume restore item action plugin's Execute method. func TestPodVolumeRestoreActionExecute(t *testing.T) { resourceReqs, _ := kube.ParseCPUAndMemoryResources( defaultCPURequestLimit, defaultMemRequestLimit, defaultCPURequestLimit, defaultMemRequestLimit, ) id := int64(1000) securityContext := corev1api.SecurityContext{ AllowPrivilegeEscalation: boolptr.False(), Capabilities: &corev1api.Capabilities{ Drop: []corev1api.Capability{"ALL"}, }, SeccompProfile: &corev1api.SeccompProfile{ Type: corev1api.SeccompProfileTypeRuntimeDefault, }, RunAsUser: &id, RunAsNonRoot: boolptr.True(), } customID := int64(44444) customSecurityContext := corev1api.SecurityContext{ AllowPrivilegeEscalation: boolptr.False(), Capabilities: &corev1api.Capabilities{ Drop: []corev1api.Capability{"ALL"}, }, SeccompProfile: &corev1api.SeccompProfile{ Type: corev1api.SeccompProfileTypeRuntimeDefault, }, RunAsUser: &customID, RunAsNonRoot: boolptr.True(), } var ( restoreName = "my-restore" backupName = "test-backup" veleroNs = "velero" ) defaultRestoreHelperImage := "velero/velero:v1.0" tests := []struct { name string pod *corev1api.Pod podFromBackup *corev1api.Pod podVolumeBackups []runtime.Object want *corev1api.Pod }{ { name: "Restoring pod with no other initContainers adds the restore initContainer when volumes need file system restores", pod: builder.ForPod("ns-1", "my-pod"). ObjectMeta(builder.WithAnnotations("snapshot.velero.io/myvol", "")). Volumes( builder.ForVolume("myvol").PersistentVolumeClaimSource("pvc-1").Result(), ). Result(), podVolumeBackups: []runtime.Object{ builder.ForPodVolumeBackup(veleroNs, "pvb-1"). PodName("my-pod"). PodNamespace("ns-1"). Volume("myvol"). ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, backupName)). SnapshotID("foo"). Result(), }, want: builder.ForPod("ns-1", "my-pod"). ObjectMeta( builder.WithAnnotations("snapshot.velero.io/myvol", "")). Volumes( builder.ForVolume("myvol").PersistentVolumeClaimSource("pvc-1").Result(), ). InitContainers( newRestoreInitContainerBuilder(defaultRestoreHelperImage, ""). Resources(&resourceReqs). SecurityContext(&securityContext). VolumeMounts(builder.ForVolumeMount("myvol", "/restores/myvol").Result()). Command([]string{"/velero-restore-helper"}).Result()).Result(), }, { name: "Restoring pod with other initContainers adds the restore initContainer as the first one when volumes need file system restores", pod: builder.ForPod("ns-1", "my-pod"). ObjectMeta( builder.WithAnnotations("snapshot.velero.io/myvol", "")). Volumes( builder.ForVolume("myvol").PersistentVolumeClaimSource("pvc-1").Result(), ). InitContainers(builder.ForContainer("first-container", "").Result()). Result(), podVolumeBackups: []runtime.Object{ builder.ForPodVolumeBackup(veleroNs, "pvb-1"). PodName("my-pod"). PodNamespace("ns-1"). Volume("myvol"). ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, backupName)). SnapshotID("foo"). Result(), }, want: builder.ForPod("ns-1", "my-pod"). ObjectMeta( builder.WithAnnotations("snapshot.velero.io/myvol", "")). Volumes( builder.ForVolume("myvol").PersistentVolumeClaimSource("pvc-1").Result(), ). InitContainers( newRestoreInitContainerBuilder(defaultRestoreHelperImage, ""). Resources(&resourceReqs). SecurityContext(&securityContext). VolumeMounts(builder.ForVolumeMount("myvol", "/restores/myvol").Result()). Command([]string{"/velero-restore-helper"}).Result(), builder.ForContainer("first-container", "").Result()). Result(), }, { name: "Restoring pod with other initContainers adds the restore initContainer as the first one using PVB to identify the volumes and not annotations", pod: builder.ForPod("ns-1", "my-pod"). Volumes( builder.ForVolume("vol-1").PersistentVolumeClaimSource("pvc-1").Result(), builder.ForVolume("vol-2").PersistentVolumeClaimSource("pvc-2").Result(), ). ObjectMeta( builder.WithAnnotations("snapshot.velero.io/not-used", "")). InitContainers(builder.ForContainer("first-container", "").Result()). Result(), podVolumeBackups: []runtime.Object{ builder.ForPodVolumeBackup(veleroNs, "pvb-1"). PodName("my-pod"). PodNamespace("ns-1"). Volume("vol-1"). ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, backupName)). SnapshotID("foo"). Result(), builder.ForPodVolumeBackup(veleroNs, "pvb-2"). PodName("my-pod"). PodNamespace("ns-1"). Volume("vol-2"). ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, backupName)). SnapshotID("foo"). Result(), }, want: builder.ForPod("ns-1", "my-pod"). Volumes( builder.ForVolume("vol-1").PersistentVolumeClaimSource("pvc-1").Result(), builder.ForVolume("vol-2").PersistentVolumeClaimSource("pvc-2").Result(), ). ObjectMeta( builder.WithAnnotations("snapshot.velero.io/not-used", "")). InitContainers( newRestoreInitContainerBuilder(defaultRestoreHelperImage, ""). Resources(&resourceReqs). SecurityContext(&securityContext). VolumeMounts(builder.ForVolumeMount("vol-1", "/restores/vol-1").Result(), builder.ForVolumeMount("vol-2", "/restores/vol-2").Result()). Command([]string{"/velero-restore-helper"}).Result(), builder.ForContainer("first-container", "").Result()). Result(), }, { name: "Restoring pod in another namespace adds the restore initContainer and uses the namespace of the backup pod for matching PVBs", pod: builder.ForPod("new-ns", "my-pod"). Volumes( builder.ForVolume("vol-1").PersistentVolumeClaimSource("pvc-1").Result(), builder.ForVolume("vol-2").PersistentVolumeClaimSource("pvc-2").Result(), ). Result(), podFromBackup: builder.ForPod("original-ns", "my-pod"). Volumes( builder.ForVolume("vol-1").PersistentVolumeClaimSource("pvc-1").Result(), builder.ForVolume("vol-2").PersistentVolumeClaimSource("pvc-2").Result(), ). Result(), podVolumeBackups: []runtime.Object{ builder.ForPodVolumeBackup(veleroNs, "pvb-1"). PodName("my-pod"). PodNamespace("original-ns"). Volume("vol-1"). ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, backupName)). SnapshotID("foo"). Result(), builder.ForPodVolumeBackup(veleroNs, "pvb-2"). PodName("my-pod"). PodNamespace("original-ns"). Volume("vol-2"). ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, backupName)). SnapshotID("foo"). Result(), }, want: builder.ForPod("new-ns", "my-pod"). Volumes( builder.ForVolume("vol-1").PersistentVolumeClaimSource("pvc-1").Result(), builder.ForVolume("vol-2").PersistentVolumeClaimSource("pvc-2").Result(), ). InitContainers( newRestoreInitContainerBuilder(defaultRestoreHelperImage, ""). Resources(&resourceReqs). SecurityContext(&securityContext). VolumeMounts(builder.ForVolumeMount("vol-1", "/restores/vol-1").Result(), builder.ForVolumeMount("vol-2", "/restores/vol-2").Result()). Command([]string{"/velero-restore-helper"}).Result()). Result(), }, { name: "Restoring pod with custom container SecurityContext uses this SecurityContext for the restore initContainer when volumes need file system restores", pod: builder.ForPod("ns-1", "my-pod"). ObjectMeta( builder.WithAnnotations("snapshot.velero.io/myvol", "")). Volumes( builder.ForVolume("myvol").PersistentVolumeClaimSource("pvc-1").Result(), ). Containers( builder.ForContainer("app-container", "app-image"). SecurityContext(&customSecurityContext).Result()). Result(), podVolumeBackups: []runtime.Object{ builder.ForPodVolumeBackup(veleroNs, "pvb-1"). PodName("my-pod"). PodNamespace("ns-1"). Volume("myvol"). ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, backupName)). SnapshotID("foo"). Result(), }, want: builder.ForPod("ns-1", "my-pod"). ObjectMeta( builder.WithAnnotations("snapshot.velero.io/myvol", "")). Volumes( builder.ForVolume("myvol").PersistentVolumeClaimSource("pvc-1").Result(), ). Containers( builder.ForContainer("app-container", "app-image"). SecurityContext(&customSecurityContext).Result()). InitContainers( newRestoreInitContainerBuilder(defaultRestoreHelperImage, ""). Resources(&resourceReqs). SecurityContext(&customSecurityContext). VolumeMounts(builder.ForVolumeMount("myvol", "/restores/myvol").Result()). Command([]string{"/velero-restore-helper"}).Result()).Result(), }, } veleroDeployment := &appsv1api.Deployment{ TypeMeta: metav1.TypeMeta{ APIVersion: appsv1api.SchemeGroupVersion.String(), Kind: "Deployment", }, ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", Name: "velero", }, Spec: appsv1api.DeploymentSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ Containers: []corev1api.Container{ { Image: "velero/velero:v1.0", }, }, }, }, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { clientset := fake.NewSimpleClientset() objects := []runtime.Object{veleroDeployment} objects = append(objects, tc.podVolumeBackups...) crClient := velerotest.NewFakeControllerRuntimeClient(t, objects...) unstructuredPod, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.pod) require.NoError(t, err) // Default to using the same pod for both Item and ItemFromBackup if podFromBackup not provided var unstructuredPodFromBackup map[string]any if tc.podFromBackup != nil { unstructuredPodFromBackup, err = runtime.DefaultUnstructuredConverter.ToUnstructured(tc.podFromBackup) require.NoError(t, err) } else { unstructuredPodFromBackup = unstructuredPod } input := &velero.RestoreItemActionExecuteInput{ Item: &unstructured.Unstructured{ Object: unstructuredPod, }, ItemFromBackup: &unstructured.Unstructured{ Object: unstructuredPodFromBackup, }, Restore: builder.ForRestore(veleroNs, restoreName). Backup(backupName). Phase(velerov1api.RestorePhaseInProgress). Result(), } a, err := NewPodVolumeRestoreAction( logrus.StandardLogger(), clientset.CoreV1().ConfigMaps(veleroNs), crClient, "velero", ) require.NoError(t, err) // method under test res, err := a.Execute(input) require.NoError(t, err) updatedPod := new(corev1api.Pod) require.NoError(t, runtime.DefaultUnstructuredConverter.FromUnstructured(res.UpdatedItem.UnstructuredContent(), updatedPod)) for _, container := range tc.want.Spec.InitContainers { sort.Slice(container.VolumeMounts, func(i, j int) bool { return container.VolumeMounts[i].Name < container.VolumeMounts[j].Name }) } for _, container := range updatedPod.Spec.InitContainers { sort.Slice(container.VolumeMounts, func(i, j int) bool { return container.VolumeMounts[i].Name < container.VolumeMounts[j].Name }) } assert.Equal(t, tc.want, updatedPod) }) } } func TestGetCommand(t *testing.T) { configMapWithData := func(key, val string) *corev1api.ConfigMap { return &corev1api.ConfigMap{ Data: map[string]string{ key: val, }, } } testCases := []struct { name string configMap *corev1api.ConfigMap expected []string }{ { name: "should get default command when config key is missing", configMap: configMapWithData("non-matching-key", "val"), expected: []string{defaultCommand}, }, { name: "should get default command when config key is empty", configMap: configMapWithData("command", ""), expected: []string{defaultCommand}, }, { name: "should get default command when config is nil", configMap: nil, expected: []string{defaultCommand}, }, { name: "should get command from config", configMap: configMapWithData("command", "foobarbz"), expected: []string{"foobarbz"}, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { actual := getCommand(velerotest.NewLogger(), tc.configMap) assert.Equal(t, tc.expected, actual) }) } } // This tests that restore-wait is added when file system restore volume exists, nothing added otherwise, // and removed if it exists but is not needed. // issue: 8870 func TestPodVolumeRestoreActionExecuteWithFileSystemShouldAddWaitInitContainer(t *testing.T) { tests := []struct { name string pod *corev1api.Pod podFromBackup *corev1api.Pod podVolumeBackups []*velerov1api.PodVolumeBackup restore *velerov1api.Restore expectedInitContainers int expectedError error }{ { name: "no pod volume backups results in no init container", pod: builder.ForPod("ns", "pod"). ObjectMeta(builder.WithUID("pod-uid")). Volumes( builder.ForVolume("volume-1").PersistentVolumeClaimSource("pvc-1").Result(), builder.ForVolume("volume-2").PersistentVolumeClaimSource("pvc-2").Result(), ). Result(), podFromBackup: builder.ForPod("ns", "pod"). ObjectMeta(builder.WithUID("pod-uid")). Volumes( builder.ForVolume("volume-1").PersistentVolumeClaimSource("pvc-1").Result(), builder.ForVolume("volume-2").PersistentVolumeClaimSource("pvc-2").Result(), ). Result(), podVolumeBackups: nil, restore: builder.ForRestore("velero", "restore-1").Backup("test-backup").Result(), expectedInitContainers: 0, expectedError: nil, }, { name: "pod volume backups that don't match pod's volumes results in no init container", pod: builder.ForPod("ns", "pod"). ObjectMeta(builder.WithUID("pod-uid")). Volumes( builder.ForVolume("volume-1").PersistentVolumeClaimSource("pvc-1").Result(), builder.ForVolume("volume-2").PersistentVolumeClaimSource("pvc-2").Result(), ). Result(), podFromBackup: builder.ForPod("ns", "pod"). ObjectMeta(builder.WithUID("pod-uid")). Volumes( builder.ForVolume("volume-1").PersistentVolumeClaimSource("pvc-1").Result(), builder.ForVolume("volume-2").PersistentVolumeClaimSource("pvc-2").Result(), ). Result(), podVolumeBackups: []*velerov1api.PodVolumeBackup{ builder.ForPodVolumeBackup("velero", "pvb-1"). ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "test-backup")). PodName("different-pod"). PodNamespace("ns"). Volume("volume-1"). SnapshotID("snapshot-1"). Result(), }, restore: builder.ForRestore("velero", "restore-1").Backup("test-backup").Result(), expectedInitContainers: 0, expectedError: nil, }, { name: "matching pod volume backup results in init container being added", pod: builder.ForPod("ns", "pod"). ObjectMeta(builder.WithUID("pod-uid")). Volumes( builder.ForVolume("volume-1").PersistentVolumeClaimSource("pvc-1").Result(), builder.ForVolume("volume-2").PersistentVolumeClaimSource("pvc-2").Result(), ). Result(), podFromBackup: builder.ForPod("ns", "pod"). ObjectMeta(builder.WithUID("pod-uid")). Volumes( builder.ForVolume("volume-1").PersistentVolumeClaimSource("pvc-1").Result(), builder.ForVolume("volume-2").PersistentVolumeClaimSource("pvc-2").Result(), ). Result(), podVolumeBackups: []*velerov1api.PodVolumeBackup{ builder.ForPodVolumeBackup("velero", "pvb-1"). ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "test-backup")). PodName("pod"). PodNamespace("ns"). Volume("volume-1"). SnapshotID("snapshot-1"). Result(), }, restore: builder.ForRestore("velero", "restore-1").Backup("test-backup").Result(), expectedInitContainers: 1, expectedError: nil, }, { name: "matching pod volume backup with matching pod name and namespace results in init container being added", pod: builder.ForPod("ns", "pod"). ObjectMeta(builder.WithUID("pod-uid")). Volumes( builder.ForVolume("volume-1").PersistentVolumeClaimSource("pvc-1").Result(), builder.ForVolume("volume-2").PersistentVolumeClaimSource("pvc-2").Result(), ). Result(), podFromBackup: builder.ForPod("ns", "pod"). ObjectMeta(builder.WithUID("pod-uid")). Volumes( builder.ForVolume("volume-1").PersistentVolumeClaimSource("pvc-1").Result(), builder.ForVolume("volume-2").PersistentVolumeClaimSource("pvc-2").Result(), ). Result(), podVolumeBackups: []*velerov1api.PodVolumeBackup{ builder.ForPodVolumeBackup("velero", "pvb-1"). ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "test-backup")). PodName("pod"). PodNamespace("ns"). Volume("volume-1"). SnapshotID("snapshot-1"). Result(), }, restore: builder.ForRestore("velero", "restore-1").Backup("test-backup").Result(), expectedInitContainers: 1, expectedError: nil, }, { name: "existing init container is removed when no file system restore is needed", pod: builder.ForPod("ns", "pod"). ObjectMeta(builder.WithUID("pod-uid")). Volumes( builder.ForVolume("volume-1").PersistentVolumeClaimSource("pvc-1").Result(), builder.ForVolume("volume-2").PersistentVolumeClaimSource("pvc-2").Result(), ). InitContainers( builder.ForContainer(restorehelper.WaitInitContainer, "velero/velero:latest"). Command([]string{"/velero-restore-helper"}). Args("restore-1"). Result(), builder.ForContainer("another-init", "another-image").Result(), ). Result(), podFromBackup: builder.ForPod("ns", "pod"). ObjectMeta(builder.WithUID("pod-uid")). Volumes( builder.ForVolume("volume-1").PersistentVolumeClaimSource("pvc-1").Result(), builder.ForVolume("volume-2").PersistentVolumeClaimSource("pvc-2").Result(), ). Result(), podVolumeBackups: []*velerov1api.PodVolumeBackup{ // This PVB doesn't match the pod's name, so needsFileSystemRestore will be false builder.ForPodVolumeBackup("velero", "pvb-1"). ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "test-backup")). PodName("different-pod"). PodNamespace("ns"). Volume("volume-1"). SnapshotID("snapshot-1"). Result(), }, restore: builder.ForRestore("velero", "restore-1").Backup("test-backup").Result(), expectedInitContainers: 1, // Only the "another-init" container should remain expectedError: nil, }, { name: "existing legacy init container is removed when no file system restore is needed", pod: builder.ForPod("ns", "pod"). ObjectMeta(builder.WithUID("pod-uid")). Volumes( builder.ForVolume("volume-1").PersistentVolumeClaimSource("pvc-1").Result(), builder.ForVolume("volume-2").PersistentVolumeClaimSource("pvc-2").Result(), ). InitContainers( builder.ForContainer(restorehelper.WaitInitContainerLegacy, "velero/velero:latest"). Command([]string{"/velero-restore-helper"}). Args("restore-1"). Result(), builder.ForContainer("another-init", "another-image").Result(), ). Result(), podFromBackup: builder.ForPod("ns", "pod"). ObjectMeta(builder.WithUID("pod-uid")). Volumes( builder.ForVolume("volume-1").PersistentVolumeClaimSource("pvc-1").Result(), builder.ForVolume("volume-2").PersistentVolumeClaimSource("pvc-2").Result(), ). Result(), podVolumeBackups: []*velerov1api.PodVolumeBackup{ // This PVB doesn't match the pod's name, so needsFileSystemRestore will be false builder.ForPodVolumeBackup("velero", "pvb-1"). ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "test-backup")). PodName("different-pod"). PodNamespace("ns"). Volume("volume-1"). SnapshotID("snapshot-1"). Result(), }, restore: builder.ForRestore("velero", "restore-1").Backup("test-backup").Result(), expectedInitContainers: 1, // Only the "another-init" container should remain expectedError: nil, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { // Setup var ( client = crfake.NewClientBuilder().Build() crClient = client ) // Register the PodVolumeBackup type with the scheme require.NoError(t, velerov1api.AddToScheme(scheme.Scheme)) // Create the PodVolumeBackups in the fake client for _, pvb := range tc.podVolumeBackups { require.NoError(t, crClient.Create(t.Context(), pvb)) } // Create a fake clientset clientset := fake.NewSimpleClientset() // Create the action action := &PodVolumeRestoreAction{ logger: logrus.StandardLogger(), client: clientset.CoreV1().ConfigMaps("velero"), crClient: crClient, veleroImage: "velero/velero:latest", } // Convert the pod to unstructured podMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.pod) require.NoError(t, err) podFromBackupMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.podFromBackup) require.NoError(t, err) // Create the input input := &velero.RestoreItemActionExecuteInput{ Item: &unstructured.Unstructured{Object: podMap}, ItemFromBackup: &unstructured.Unstructured{Object: podFromBackupMap}, Restore: tc.restore, } // Execute the action output, err := action.Execute(input) // Verify the results if tc.expectedError != nil { assert.Equal(t, tc.expectedError, err) } else { require.NoError(t, err) // Convert the output back to a pod outputPod := new(corev1api.Pod) err = runtime.DefaultUnstructuredConverter.FromUnstructured(output.UpdatedItem.UnstructuredContent(), outputPod) require.NoError(t, err) // Check if the init container was added or removed as expected assert.Len(t, outputPod.Spec.InitContainers, tc.expectedInitContainers, "Unexpected number of init containers") } }) } } ================================================ FILE: pkg/restore/actions/pvc_action.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actions import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/plugin/velero" "github.com/vmware-tanzu/velero/pkg/util" ) const ( AnnBindCompleted = "pv.kubernetes.io/bind-completed" AnnBoundByController = "pv.kubernetes.io/bound-by-controller" AnnStorageProvisioner = "volume.kubernetes.io/storage-provisioner" AnnBetaStorageProvisioner = "volume.beta.kubernetes.io/storage-provisioner" AnnSelectedNode = "volume.kubernetes.io/selected-node" ) // PVCAction updates/reset PVC's node selector // if a mapping is found in the plugin's config map. type PVCAction struct { logger logrus.FieldLogger configMapClient corev1client.ConfigMapInterface nodeClient corev1client.NodeInterface } // NewPVCAction is the constructor for PVCAction. func NewPVCAction( logger logrus.FieldLogger, configMapClient corev1client.ConfigMapInterface, nodeClient corev1client.NodeInterface, ) *PVCAction { return &PVCAction{ logger: logger, configMapClient: configMapClient, nodeClient: nodeClient, } } // AppliesTo returns the resources that PVCAction should be run for func (p *PVCAction) AppliesTo() (velero.ResourceSelector, error) { return velero.ResourceSelector{ IncludedResources: []string{"persistentvolumeclaims"}, }, nil } // PVC actions for restore: // 1. updates the pvc's selected-node annotation: // a) if node mapping found in the config map for the plugin // b) if node mentioned in annotation doesn't exist // 2. removes some additional annotations // 3. returns bound PV as an additional item func (p *PVCAction) Execute(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { p.logger.Info("Executing PVCAction") defer p.logger.Info("Done executing PVCAction") var pvc, pvcFromBackup corev1api.PersistentVolumeClaim if err := runtime.DefaultUnstructuredConverter.FromUnstructured( input.Item.UnstructuredContent(), &pvc); err != nil { return nil, errors.WithStack(err) } if err := runtime.DefaultUnstructuredConverter.FromUnstructured( input.ItemFromBackup.UnstructuredContent(), &pvcFromBackup); err != nil { return nil, errors.WithStack(err) } log := p.logger.WithFields(map[string]any{ "kind": pvc.Kind, "namespace": pvc.Namespace, "name": pvc.Name, }) // Remove PVC annotations removePVCAnnotations( &pvc, []string{ AnnBindCompleted, AnnBoundByController, AnnStorageProvisioner, AnnBetaStorageProvisioner, AnnSelectedNode, velerov1api.VolumeSnapshotLabel, velerov1api.DataUploadNameAnnotation, }, ) pvcMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&pvc) if err != nil { return nil, errors.WithStack(err) } output := &velero.RestoreItemActionExecuteOutput{ UpdatedItem: &unstructured.Unstructured{Object: pvcMap}, } // Add PV as additional item if bound // use pvcFromBackup because we need to look at status fields, which have been removed from pvc if pvcFromBackup.Status.Phase != corev1api.ClaimBound || pvcFromBackup.Spec.VolumeName == "" { log.Info("PVC is not bound or its volume name is empty") } else { log.Infof("Adding PV %s as an additional item to restore", pvcFromBackup.Spec.VolumeName) output.AdditionalItems = []velero.ResourceIdentifier{ { GroupResource: kuberesource.PersistentVolumes, Name: pvcFromBackup.Spec.VolumeName, }, } } return output, nil } func removePVCAnnotations(pvc *corev1api.PersistentVolumeClaim, remove []string) { for k := range pvc.Annotations { if util.Contains(remove, k) { delete(pvc.Annotations, k) } } } ================================================ FILE: pkg/restore/actions/pvc_action_test.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actions import ( "fmt" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes/fake" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/plugin/velero" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) // TestPVCActionExecute runs the PVCAction's Execute // method and validates that the item's PVC is modified (or not) as expected. // Validation is done by comparing the result of the Execute method to the test case's // desired result. func TestPVCActionExecute(t *testing.T) { tests := []struct { name string pvc *corev1api.PersistentVolumeClaim want *corev1api.PersistentVolumeClaim wantErr error }{ { name: "a persistent volume claim with no annotation", pvc: builder.ForPersistentVolumeClaim("source-ns", "pvc-1").Result(), want: builder.ForPersistentVolumeClaim("source-ns", "pvc-1").Result(), }, { name: "a persistent volume claim with selected-node annotation", pvc: builder.ForPersistentVolumeClaim("source-ns", "pvc-1"). ObjectMeta( builder.WithAnnotations("volume.kubernetes.io/selected-node", "source-node"), ).Result(), want: builder.ForPersistentVolumeClaim("source-ns", "pvc-1").ObjectMeta(builder.WithAnnotationsMap(map[string]string{})).Result(), }, { name: "a persistent volume claim with other annotation", pvc: builder.ForPersistentVolumeClaim("source-ns", "pvc-1"). ObjectMeta( builder.WithAnnotations("other-anno-1", "other-value-1", "other-anno-2", "other-value-2"), ).Result(), want: builder.ForPersistentVolumeClaim("source-ns", "pvc-1").ObjectMeta( builder.WithAnnotations("other-anno-1", "other-value-1", "other-anno-2", "other-value-2"), ).Result(), }, { name: "a persistent volume claim with other annotation and selected-node annotation", pvc: builder.ForPersistentVolumeClaim("source-ns", "pvc-1"). ObjectMeta( builder.WithAnnotations("other-anno", "other-value", "volume.kubernetes.io/selected-node", "source-node"), ).Result(), want: builder.ForPersistentVolumeClaim("source-ns", "pvc-1").ObjectMeta( builder.WithAnnotations("other-anno", "other-value"), ).Result(), }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { clientset := fake.NewSimpleClientset() a := NewPVCAction( velerotest.NewLogger(), clientset.CoreV1().ConfigMaps("velero"), clientset.CoreV1().Nodes(), ) // set up test data unstructuredMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.pvc) require.NoError(t, err) input := &velero.RestoreItemActionExecuteInput{ Item: &unstructured.Unstructured{ Object: unstructuredMap, }, ItemFromBackup: &unstructured.Unstructured{ Object: unstructuredMap, }, } // execute method under test res, err := a.Execute(input) // validate for both error and non-error cases switch { case tc.wantErr != nil: require.EqualError(t, err, tc.wantErr.Error()) default: fmt.Printf("got +%v\n", res.UpdatedItem) require.NoError(t, err) wantUnstructured, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.want) fmt.Printf("expected +%v\n", wantUnstructured) require.NoError(t, err) assert.Equal(t, &unstructured.Unstructured{Object: wantUnstructured}, res.UpdatedItem) } }) } } func TestAddPVFromPVCActionExecute(t *testing.T) { tests := []struct { name string itemFromBackup *corev1api.PersistentVolumeClaim want []velero.ResourceIdentifier }{ { name: "bound PVC with volume name returns associated PV", itemFromBackup: &corev1api.PersistentVolumeClaim{ Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "bound-pv", }, Status: corev1api.PersistentVolumeClaimStatus{ Phase: corev1api.ClaimBound, }, }, want: []velero.ResourceIdentifier{ { GroupResource: kuberesource.PersistentVolumes, Name: "bound-pv", }, }, }, { name: "unbound PVC with volume name does not return any additional items", itemFromBackup: &corev1api.PersistentVolumeClaim{ Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "pending-pv", }, Status: corev1api.PersistentVolumeClaimStatus{ Phase: corev1api.ClaimPending, }, }, want: nil, }, { name: "bound PVC without volume name does not return any additional items", itemFromBackup: &corev1api.PersistentVolumeClaim{ Spec: corev1api.PersistentVolumeClaimSpec{}, Status: corev1api.PersistentVolumeClaimStatus{ Phase: corev1api.ClaimBound, }, }, want: nil, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { itemFromBackupData, err := runtime.DefaultUnstructuredConverter.ToUnstructured(test.itemFromBackup) require.NoError(t, err) itemData, err := runtime.DefaultUnstructuredConverter.ToUnstructured(test.itemFromBackup) require.NoError(t, err) // item should have no status delete(itemData, "status") clientset := fake.NewSimpleClientset() action := NewPVCAction( velerotest.NewLogger(), clientset.CoreV1().ConfigMaps("velero"), clientset.CoreV1().Nodes(), ) input := &velero.RestoreItemActionExecuteInput{ Item: &unstructured.Unstructured{Object: itemData}, ItemFromBackup: &unstructured.Unstructured{Object: itemFromBackupData}, } res, err := action.Execute(input) require.NoError(t, err) assert.Equal(t, test.want, res.AdditionalItems) }) } } func TestRemovePVCAnnotations(t *testing.T) { testCases := []struct { name string pvc corev1api.PersistentVolumeClaim removeAnnotations []string expectedAnnotations map[string]string }{ { name: "should preserve all existing annotations", pvc: corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ "ann1": "ann1-val", "ann2": "ann2-val", "ann3": "ann3-val", "ann4": "ann4-val", }, }, }, removeAnnotations: []string{}, expectedAnnotations: map[string]string{ "ann1": "ann1-val", "ann2": "ann2-val", "ann3": "ann3-val", "ann4": "ann4-val", }, }, { name: "should remove all existing annotations", pvc: corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ "ann1": "ann1-val", "ann2": "ann2-val", "ann3": "ann3-val", "ann4": "ann4-val", }, }, }, removeAnnotations: []string{"ann1", "ann2", "ann3", "ann4"}, expectedAnnotations: map[string]string{}, }, { name: "should preserve some existing annotations", pvc: corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ "ann1": "ann1-val", "ann2": "ann2-val", "ann3": "ann3-val", "ann4": "ann4-val", "ann5": "ann5-val", "ann6": "ann6-val", "ann7": "ann7-val", "ann8": "ann8-val", }, }, }, removeAnnotations: []string{"ann1", "ann2", "ann3", "ann4"}, expectedAnnotations: map[string]string{ "ann5": "ann5-val", "ann6": "ann6-val", "ann7": "ann7-val", "ann8": "ann8-val", }, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { removePVCAnnotations(&tc.pvc, tc.removeAnnotations) assert.Equal(t, tc.expectedAnnotations, tc.pvc.Annotations) }) } } ================================================ FILE: pkg/restore/actions/rolebinding_action.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actions import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "github.com/vmware-tanzu/velero/pkg/plugin/velero" ) // RoleBindingAction handle namespace remappings for role bindings type RoleBindingAction struct { logger logrus.FieldLogger } func NewRoleBindingAction(logger logrus.FieldLogger) *RoleBindingAction { return &RoleBindingAction{logger: logger} } func (a *RoleBindingAction) AppliesTo() (velero.ResourceSelector, error) { return velero.ResourceSelector{ IncludedResources: []string{"rolebindings"}, }, nil } func (a *RoleBindingAction) Execute(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { namespaceMapping := input.Restore.Spec.NamespaceMapping if len(namespaceMapping) == 0 { return velero.NewRestoreItemActionExecuteOutput(&unstructured.Unstructured{Object: input.Item.UnstructuredContent()}), nil } roleBinding := new(rbacv1.RoleBinding) if err := runtime.DefaultUnstructuredConverter.FromUnstructured(input.Item.UnstructuredContent(), roleBinding); err != nil { return nil, errors.WithStack(err) } for i, subject := range roleBinding.Subjects { if newNamespace, ok := namespaceMapping[subject.Namespace]; ok { roleBinding.Subjects[i].Namespace = newNamespace } } res, err := runtime.DefaultUnstructuredConverter.ToUnstructured(roleBinding) if err != nil { return nil, errors.WithStack(err) } return velero.NewRestoreItemActionExecuteOutput(&unstructured.Unstructured{Object: res}), nil } ================================================ FILE: pkg/restore/actions/rolebinding_action_test.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actions import ( "sort" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/plugin/velero" "github.com/vmware-tanzu/velero/pkg/test" ) func TestRoleBindingActionAppliesTo(t *testing.T) { action := NewRoleBindingAction(test.NewLogger()) actual, err := action.AppliesTo() require.NoError(t, err) assert.Equal(t, velero.ResourceSelector{IncludedResources: []string{"rolebindings"}}, actual) } func TestRoleBindingActionExecute(t *testing.T) { tests := []struct { name string namespaces []string namespaceMapping map[string]string expected []string }{ { name: "namespace mapping disabled", namespaces: []string{"foo"}, namespaceMapping: map[string]string{}, expected: []string{"foo"}, }, { name: "namespace mapping enabled", namespaces: []string{"foo"}, namespaceMapping: map[string]string{"foo": "bar", "fizz": "buzz"}, expected: []string{"bar"}, }, { name: "namespace mapping enabled, not included namespace remains unchanged", namespaces: []string{"foo", "xyz"}, namespaceMapping: map[string]string{"foo": "bar", "fizz": "buzz"}, expected: []string{"bar", "xyz"}, }, { name: "namespace mapping enabled, not included namespace remains unchanged", namespaces: []string{"foo", "xyz"}, namespaceMapping: map[string]string{"a": "b", "c": "d"}, expected: []string{"foo", "xyz"}, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { subjects := []rbacv1.Subject{} for _, ns := range tc.namespaces { subjects = append(subjects, rbacv1.Subject{ Namespace: ns, }) } roleBinding := rbacv1.RoleBinding{ Subjects: subjects, } roleBindingUnstructured, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&roleBinding) require.NoError(t, err) action := NewRoleBindingAction(test.NewLogger()) res, err := action.Execute(&velero.RestoreItemActionExecuteInput{ Item: &unstructured.Unstructured{Object: roleBindingUnstructured}, ItemFromBackup: &unstructured.Unstructured{Object: roleBindingUnstructured}, Restore: &api.Restore{ Spec: api.RestoreSpec{ NamespaceMapping: tc.namespaceMapping, }, }, }) require.NoError(t, err) var resRoleBinding *rbacv1.RoleBinding err = runtime.DefaultUnstructuredConverter.FromUnstructured(res.UpdatedItem.UnstructuredContent(), &resRoleBinding) require.NoError(t, err) actual := []string{} for _, subject := range resRoleBinding.Subjects { actual = append(actual, subject.Namespace) } sort.Strings(tc.expected) sort.Strings(actual) assert.Equal(t, tc.expected, actual) }) } } ================================================ FILE: pkg/restore/actions/secret_action.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actions import ( "context" "fmt" "strings" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/pkg/plugin/velero" "github.com/vmware-tanzu/velero/pkg/util/kube" ) // SecretAction is a restore item action for secrets type SecretAction struct { logger logrus.FieldLogger client client.Client } // NewSecretAction creates a new SecretAction instance func NewSecretAction(logger logrus.FieldLogger, client client.Client) *SecretAction { return &SecretAction{ logger: logger, client: client, } } // AppliesTo indicates which resources this action applies func (s *SecretAction) AppliesTo() (velero.ResourceSelector, error) { return velero.ResourceSelector{ IncludedResources: []string{"secrets"}, }, nil } // Execute the action func (s *SecretAction) Execute(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { s.logger.Info("Executing SecretAction") defer s.logger.Info("Done executing SecretAction") var secret corev1api.Secret if err := runtime.DefaultUnstructuredConverter.FromUnstructured(input.Item.UnstructuredContent(), &secret); err != nil { return nil, errors.Wrap(err, "unable to convert secret from runtime.Unstructured") } log := s.logger.WithField("secret", kube.NamespaceAndName(&secret)) if secret.Type != corev1api.SecretTypeServiceAccountToken { log.Debug("No match found - including this secret") return &velero.RestoreItemActionExecuteOutput{ UpdatedItem: input.Item, }, nil } // The auto created service account token secret will be created by kube controller automatically again(before Kubernetes v1.22), no need to restore. // This will cause the patch operation of managedFields failed if we restore it as the secret is removed immediately // after restoration and the patch operation reports not found error. list := &corev1api.ServiceAccountList{} if err := s.client.List(context.Background(), list, &client.ListOptions{Namespace: secret.Namespace}); err != nil { return nil, errors.Wrap(err, "unable to list the service accounts") } for _, sa := range list.Items { if strings.HasPrefix(secret.Name, fmt.Sprintf("%s-token-", sa.Name)) { log.Debug("auto created service account token secret found - excluding this secret") return &velero.RestoreItemActionExecuteOutput{ UpdatedItem: input.Item, SkipRestore: true, }, nil } } log.Debug("service account token secret(not auto created) found - remove some fields from this secret") // If the annotation and data are not removed, the secret cannot be restored successfully. // The kube controller will fill the annotation and data with new value automatically: // https://kubernetes.io/docs/concepts/configuration/secret/#service-account-token-secrets delete(secret.Annotations, "kubernetes.io/service-account.uid") delete(secret.Data, "token") delete(secret.Data, "ca.crt") res, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&secret) if err != nil { return nil, errors.Wrap(err, "unable to convert secret to runtime.Unstructured") } return &velero.RestoreItemActionExecuteOutput{ UpdatedItem: &unstructured.Unstructured{Object: res}, }, nil } ================================================ FILE: pkg/restore/actions/secret_action_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actions import ( "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "github.com/vmware-tanzu/velero/pkg/plugin/velero" "github.com/vmware-tanzu/velero/pkg/test" ) func TestSecretActionAppliesTo(t *testing.T) { action := NewSecretAction(test.NewLogger(), nil) actual, err := action.AppliesTo() require.NoError(t, err) assert.Equal(t, velero.ResourceSelector{IncludedResources: []string{"secrets"}}, actual) } func TestSecretActionExecute(t *testing.T) { tests := []struct { name string input *corev1api.Secret serviceAccount *corev1api.ServiceAccount skipped bool output *corev1api.Secret }{ { name: "not service account token secret", input: &corev1api.Secret{ ObjectMeta: metav1.ObjectMeta{ Namespace: "foo", Name: "default-token-sfafa", }, Type: corev1api.SecretTypeOpaque, }, skipped: false, output: &corev1api.Secret{ ObjectMeta: metav1.ObjectMeta{ Namespace: "foo", Name: "default-token-sfafa", }, Type: corev1api.SecretTypeOpaque, }, }, { name: "auto created service account token", input: &corev1api.Secret{ ObjectMeta: metav1.ObjectMeta{ Namespace: "foo", Name: "default-token-sfafa", }, Type: corev1api.SecretTypeServiceAccountToken, }, serviceAccount: &corev1api.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ Namespace: "foo", Name: "default", }, }, skipped: true, }, { name: "not auto created service account token", input: &corev1api.Secret{ ObjectMeta: metav1.ObjectMeta{ Namespace: "foo", Name: "my-token", Annotations: map[string]string{ "kubernetes.io/service-account.uid": "uid", "key": "value", }, }, Type: corev1api.SecretTypeServiceAccountToken, Data: map[string][]byte{ "token": []byte("token"), "ca.crt": []byte("ca"), "key": []byte("value"), }, }, skipped: false, output: &corev1api.Secret{ ObjectMeta: metav1.ObjectMeta{ Namespace: "foo", Name: "my-token", Annotations: map[string]string{ "key": "value", }, }, Type: corev1api.SecretTypeServiceAccountToken, Data: map[string][]byte{ "key": []byte("value"), }, }, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { secretUnstructured, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.input) require.NoError(t, err) var serviceAccounts []client.Object if tc.serviceAccount != nil { serviceAccounts = append(serviceAccounts, tc.serviceAccount) } client := fake.NewClientBuilder().WithObjects(serviceAccounts...).Build() action := NewSecretAction(test.NewLogger(), client) res, err := action.Execute(&velero.RestoreItemActionExecuteInput{ Item: &unstructured.Unstructured{Object: secretUnstructured}, }) require.NoError(t, err) assert.Equal(t, tc.skipped, res.SkipRestore) if !tc.skipped { r, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.output) require.NoError(t, err) assert.EqualValues(t, &unstructured.Unstructured{Object: r}, res.UpdatedItem) } }) } } ================================================ FILE: pkg/restore/actions/service_account_action.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actions import ( "strings" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "github.com/vmware-tanzu/velero/pkg/plugin/velero" "github.com/vmware-tanzu/velero/pkg/util/kube" ) type ServiceAccountAction struct { logger logrus.FieldLogger } func NewServiceAccountAction(logger logrus.FieldLogger) *ServiceAccountAction { return &ServiceAccountAction{logger: logger} } func (a *ServiceAccountAction) AppliesTo() (velero.ResourceSelector, error) { return velero.ResourceSelector{ IncludedResources: []string{"serviceaccounts"}, }, nil } func (a *ServiceAccountAction) Execute(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { a.logger.Info("Executing ServiceAccountAction") defer a.logger.Info("Done executing ServiceAccountAction") var serviceAccount corev1api.ServiceAccount if err := runtime.DefaultUnstructuredConverter.FromUnstructured(input.Item.UnstructuredContent(), &serviceAccount); err != nil { return nil, errors.Wrap(err, "unable to convert serviceaccount from runtime.Unstructured") } log := a.logger.WithField("serviceaccount", kube.NamespaceAndName(&serviceAccount)) log.Debug("Checking secrets") check := serviceAccount.Name + "-token-" for i := len(serviceAccount.Secrets) - 1; i >= 0; i-- { secret := &serviceAccount.Secrets[i] log.Debugf("Checking if secret %s matches %s", secret.Name, check) if strings.HasPrefix(secret.Name, check) { // Copy all secrets *except* -token- log.Debug("Match found - excluding this secret") serviceAccount.Secrets = append(serviceAccount.Secrets[:i], serviceAccount.Secrets[i+1:]...) break } else { log.Debug("No match found - including this secret") } } res, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&serviceAccount) if err != nil { return nil, errors.Wrap(err, "unable to convert serviceaccount to runtime.Unstructured") } return velero.NewRestoreItemActionExecuteOutput(&unstructured.Unstructured{Object: res}), nil } ================================================ FILE: pkg/restore/actions/service_account_action_test.go ================================================ /* Copyright 2018, 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actions import ( "sort" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "github.com/vmware-tanzu/velero/pkg/plugin/velero" "github.com/vmware-tanzu/velero/pkg/test" ) func TestServiceAccountActionAppliesTo(t *testing.T) { action := NewServiceAccountAction(test.NewLogger()) actual, err := action.AppliesTo() require.NoError(t, err) assert.Equal(t, velero.ResourceSelector{IncludedResources: []string{"serviceaccounts"}}, actual) } func TestServiceAccountActionExecute(t *testing.T) { tests := []struct { name string secrets []string expected []string }{ { name: "no secrets", secrets: []string{}, expected: []string{}, }, { name: "no match", secrets: []string{"a", "bar-TOKN-nomatch", "baz"}, expected: []string{"a", "bar-TOKN-nomatch", "baz"}, }, { name: "match - first", secrets: []string{"bar-token-a1b2c", "a", "baz"}, expected: []string{"a", "baz"}, }, { name: "match - middle", secrets: []string{"a", "bar-token-a1b2c", "baz"}, expected: []string{"a", "baz"}, }, { name: "match - end", secrets: []string{"a", "baz", "bar-token-a1b2c"}, expected: []string{"a", "baz"}, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { sa := corev1api.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ Namespace: "foo", Name: "bar", }, } for _, secret := range tc.secrets { sa.Secrets = append(sa.Secrets, corev1api.ObjectReference{ Name: secret, }) } saUnstructured, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&sa) require.NoError(t, err) action := NewServiceAccountAction(test.NewLogger()) res, err := action.Execute(&velero.RestoreItemActionExecuteInput{ Item: &unstructured.Unstructured{Object: saUnstructured}, ItemFromBackup: &unstructured.Unstructured{Object: saUnstructured}, Restore: nil, }) require.NoError(t, err) var resSA *corev1api.ServiceAccount err = runtime.DefaultUnstructuredConverter.FromUnstructured(res.UpdatedItem.UnstructuredContent(), &resSA) require.NoError(t, err) actual := []string{} for _, secret := range resSA.Secrets { actual = append(actual, secret.Name) } sort.Strings(tc.expected) sort.Strings(actual) assert.Equal(t, tc.expected, actual) }) } } ================================================ FILE: pkg/restore/actions/service_action.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actions import ( "encoding/json" "fmt" "strconv" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" "github.com/vmware-tanzu/velero/pkg/plugin/velero" "github.com/vmware-tanzu/velero/pkg/util/boolptr" ) const annotationLastAppliedConfig = "kubectl.kubernetes.io/last-applied-configuration" type ServiceAction struct { log logrus.FieldLogger } func NewServiceAction(logger logrus.FieldLogger) *ServiceAction { return &ServiceAction{log: logger} } func (a *ServiceAction) AppliesTo() (velero.ResourceSelector, error) { return velero.ResourceSelector{ IncludedResources: []string{"services"}, }, nil } func (a *ServiceAction) Execute(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { service := new(corev1api.Service) if err := runtime.DefaultUnstructuredConverter.FromUnstructured(input.Item.UnstructuredContent(), service); err != nil { return nil, errors.WithStack(err) } if service.Spec.ClusterIP != "None" { service.Spec.ClusterIP = "" service.Spec.ClusterIPs = nil } /* Do not delete NodePorts if restore triggered with "--preserve-nodeports" flag */ if boolptr.IsSetToTrue(input.Restore.Spec.PreserveNodePorts) { a.log.Info("Restoring Services with original NodePort(s)") } else { if err := deleteNodePorts(service); err != nil { return nil, err } if err := deleteHealthCheckNodePort(service); err != nil { return nil, err } } res, err := runtime.DefaultUnstructuredConverter.ToUnstructured(service) if err != nil { return nil, errors.WithStack(err) } return velero.NewRestoreItemActionExecuteOutput(&unstructured.Unstructured{Object: res}), nil } func deleteHealthCheckNodePort(service *corev1api.Service) error { // Check service type and external traffic policy setting, // if the setting is not applicable for HealthCheckNodePort, return early. if service.Spec.ExternalTrafficPolicy != corev1api.ServiceExternalTrafficPolicyTypeLocal || service.Spec.Type != corev1api.ServiceTypeLoadBalancer { return nil } // HealthCheckNodePort is already 0, return. if service.Spec.HealthCheckNodePort == 0 { return nil } // Search HealthCheckNodePort from server's last-applied-configuration // annotation(HealthCheckNodePort is specified by `kubectl apply` command) lastAppliedConfig, ok := service.Annotations[annotationLastAppliedConfig] if ok { appliedServiceUnstructured := new(map[string]any) if err := json.Unmarshal([]byte(lastAppliedConfig), appliedServiceUnstructured); err != nil { return errors.WithStack(err) } healthCheckNodePort, exist, err := unstructured.NestedFloat64(*appliedServiceUnstructured, "spec", "healthCheckNodePort") if err != nil { return errors.WithStack(err) } // Found healthCheckNodePort in lastAppliedConfig annotation, // and the value is not 0. No need to delete, return. if exist && healthCheckNodePort != 0 { return nil } } // Search HealthCheckNodePort from ManagedFields(HealthCheckNodePort // is specified by `kubectl apply --server-side` command). for _, entry := range service.GetManagedFields() { if entry.FieldsV1 == nil { continue } fields := new(map[string]any) if err := json.Unmarshal(entry.FieldsV1.Raw, fields); err != nil { return errors.WithStack(err) } _, exist, err := unstructured.NestedMap(*fields, "f:spec", "f:healthCheckNodePort") if err != nil { return errors.WithStack(err) } if !exist { continue } // Because the format in ManagedFields is `f:healthCheckNodePort: {}`, // cannot get the value, check whether exists is enough. // Found healthCheckNodePort in ManagedFields. // No need to delete. Return. return nil } // Cannot find HealthCheckNodePort from Annotation and // ManagedFields, which means it's auto-generated. Delete it. service.Spec.HealthCheckNodePort = 0 return nil } func deleteNodePorts(service *corev1api.Service) error { if service.Spec.Type == corev1api.ServiceTypeExternalName { return nil } // find any NodePorts whose values were explicitly specified according // to the last-applied-config annotation. We'll retain these values, and // clear out any other (presumably auto-assigned) NodePort values. lastAppliedConfig, ok := service.Annotations[annotationLastAppliedConfig] if ok { explicitNodePorts := sets.NewString() unnamedPortInts := sets.NewInt() appliedServiceUnstructured := new(map[string]any) if err := json.Unmarshal([]byte(lastAppliedConfig), appliedServiceUnstructured); err != nil { return errors.WithStack(err) } ports, bool, err := unstructured.NestedSlice(*appliedServiceUnstructured, "spec", "ports") if err != nil { return errors.WithStack(err) } if bool { for _, port := range ports { p, ok := port.(map[string]any) if !ok { continue } nodePort, nodePortBool, err := unstructured.NestedFieldNoCopy(p, "nodePort") if err != nil { return errors.WithStack(err) } if nodePortBool { nodePortInt := 0 switch nodePort := nodePort.(type) { case int32: nodePortInt = int(nodePort) case float64: nodePortInt = int(nodePort) case string: nodePortInt, err = strconv.Atoi(nodePort) if err != nil { return errors.WithStack(err) } } if nodePortInt > 0 { portName, ok := p["name"] if !ok { // unnamed port unnamedPortInts.Insert(nodePortInt) } else { explicitNodePorts.Insert(portName.(string)) } } } } } for i, port := range service.Spec.Ports { if port.Name != "" { if !explicitNodePorts.Has(port.Name) { service.Spec.Ports[i].NodePort = 0 } } else { if !unnamedPortInts.Has(int(port.NodePort)) { service.Spec.Ports[i].NodePort = 0 } } } return nil } explicitNodePorts := sets.NewString() for _, entry := range service.GetManagedFields() { if entry.FieldsV1 == nil { continue } fields := new(map[string]any) if err := json.Unmarshal(entry.FieldsV1.Raw, fields); err != nil { return errors.WithStack(err) } ports, exist, err := unstructured.NestedMap(*fields, "f:spec", "f:ports") if err != nil { return errors.WithStack(err) } if !exist { continue } for key, port := range ports { p, ok := port.(map[string]any) if !ok { continue } if _, exist := p["f:nodePort"]; exist { explicitNodePorts.Insert(key) } } } for i, port := range service.Spec.Ports { k := portKey(port) if !explicitNodePorts.Has(k) { service.Spec.Ports[i].NodePort = 0 } } return nil } func portKey(port corev1api.ServicePort) string { return fmt.Sprintf(`k:{"port":%d,"protocol":"%s"}`, port.Port, port.Protocol) } ================================================ FILE: pkg/restore/actions/service_action_test.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actions import ( "encoding/json" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/plugin/velero" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func svcJSON(ports ...corev1api.ServicePort) string { svc := corev1api.Service{ Spec: corev1api.ServiceSpec{ HealthCheckNodePort: 8080, Ports: ports, }, } data, err := json.Marshal(svc) if err != nil { panic(err) } return string(data) } func svcJSONFromUnstructured(ports ...map[string]any) string { svc := map[string]any{ "spec": map[string]any{ "ports": ports, }, } data, err := json.Marshal(svc) if err != nil { panic(err) } return string(data) } func TestServiceActionExecute(t *testing.T) { tests := []struct { name string obj corev1api.Service restore *api.Restore expectedErr bool expectedRes corev1api.Service }{ { name: "clusterIP/clusterIPs should be deleted from spec", obj: corev1api.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "svc-1", }, Spec: corev1api.ServiceSpec{ ClusterIP: "should-be-removed", ClusterIPs: []string{"should-be-removed"}, LoadBalancerIP: "should-be-kept", }, }, restore: builder.ForRestore(api.DefaultNamespace, "").Result(), expectedErr: false, expectedRes: corev1api.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "svc-1", }, Spec: corev1api.ServiceSpec{ LoadBalancerIP: "should-be-kept", }, }, }, { name: "headless clusterIP should not be deleted from spec", obj: corev1api.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "svc-1", }, Spec: corev1api.ServiceSpec{ ClusterIP: "None", }, }, restore: builder.ForRestore(api.DefaultNamespace, "").Result(), expectedRes: corev1api.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "svc-1", }, Spec: corev1api.ServiceSpec{ ClusterIP: "None", }, }, }, { name: "nodePort (only) should be deleted from all spec.ports", obj: corev1api.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "svc-1", }, Spec: corev1api.ServiceSpec{ Ports: []corev1api.ServicePort{ { Port: 32000, NodePort: 32000, }, { Port: 32001, NodePort: 32001, }, }, }, }, restore: builder.ForRestore(api.DefaultNamespace, "").Result(), expectedRes: corev1api.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "svc-1", }, Spec: corev1api.ServiceSpec{ Ports: []corev1api.ServicePort{ { Port: 32000, }, { Port: 32001, }, }, }, }, }, { name: "unnamed nodePort should be deleted when missing in annotation", obj: corev1api.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "svc-1", Annotations: map[string]string{ annotationLastAppliedConfig: svcJSON(), }, }, Spec: corev1api.ServiceSpec{ Ports: []corev1api.ServicePort{ { NodePort: 8080, }, }, }, }, restore: builder.ForRestore(api.DefaultNamespace, "").Result(), expectedRes: corev1api.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "svc-1", Annotations: map[string]string{ annotationLastAppliedConfig: svcJSON(), }, }, Spec: corev1api.ServiceSpec{ Ports: []corev1api.ServicePort{ {}, }, }, }, }, { name: "unnamed nodePort should be preserved when specified in annotation", obj: corev1api.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "svc-1", Annotations: map[string]string{ annotationLastAppliedConfig: svcJSON(corev1api.ServicePort{NodePort: 8080}), }, }, Spec: corev1api.ServiceSpec{ Ports: []corev1api.ServicePort{ { NodePort: 8080, }, { NodePort: 9090, }, }, }, }, restore: builder.ForRestore(api.DefaultNamespace, "").Result(), expectedRes: corev1api.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "svc-1", Annotations: map[string]string{ annotationLastAppliedConfig: svcJSON(corev1api.ServicePort{NodePort: 8080}), }, }, Spec: corev1api.ServiceSpec{ Ports: []corev1api.ServicePort{ { NodePort: 8080, }, {}, }, }, }, }, { name: "unnamed nodePort should be deleted when named nodePort specified in annotation", obj: corev1api.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "svc-1", Annotations: map[string]string{ annotationLastAppliedConfig: svcJSON(corev1api.ServicePort{Name: "http", NodePort: 8080}), }, }, Spec: corev1api.ServiceSpec{ Ports: []corev1api.ServicePort{ { NodePort: 8080, }, }, }, }, restore: builder.ForRestore(api.DefaultNamespace, "").Result(), expectedRes: corev1api.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "svc-1", Annotations: map[string]string{ annotationLastAppliedConfig: svcJSON(corev1api.ServicePort{Name: "http", NodePort: 8080}), }, }, Spec: corev1api.ServiceSpec{ Ports: []corev1api.ServicePort{ {}, }, }, }, }, { name: "unnamed nodePort should be deleted when named a string nodePort specified in annotation", obj: corev1api.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "svc-1", Annotations: map[string]string{ annotationLastAppliedConfig: svcJSONFromUnstructured(map[string]any{"name": "http", "nodePort": "8080"}), }, }, Spec: corev1api.ServiceSpec{ Ports: []corev1api.ServicePort{ { NodePort: 8080, }, }, }, }, restore: builder.ForRestore(api.DefaultNamespace, "").Result(), expectedRes: corev1api.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "svc-1", Annotations: map[string]string{ annotationLastAppliedConfig: svcJSONFromUnstructured(map[string]any{"name": "http", "nodePort": "8080"}), }, }, Spec: corev1api.ServiceSpec{ Ports: []corev1api.ServicePort{ {}, }, }, }, }, { name: "named nodePort should be preserved when specified in annotation", obj: corev1api.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "svc-1", Annotations: map[string]string{ annotationLastAppliedConfig: svcJSON(corev1api.ServicePort{Name: "http", NodePort: 8080}), }, }, Spec: corev1api.ServiceSpec{ Ports: []corev1api.ServicePort{ { Name: "http", NodePort: 8080, }, { Name: "admin", NodePort: 9090, }, }, }, }, restore: builder.ForRestore(api.DefaultNamespace, "").Result(), expectedRes: corev1api.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "svc-1", Annotations: map[string]string{ annotationLastAppliedConfig: svcJSON(corev1api.ServicePort{Name: "http", NodePort: 8080}), }, }, Spec: corev1api.ServiceSpec{ Ports: []corev1api.ServicePort{ { Name: "http", NodePort: 8080, }, { Name: "admin", }, }, }, }, }, { name: "If PreserveNodePorts is True in restore spec then nodePort always preserved.", obj: corev1api.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "svc-1", }, Spec: corev1api.ServiceSpec{ Ports: []corev1api.ServicePort{ { Name: "http", Port: 80, NodePort: 8080, }, { Name: "hepsiburada", NodePort: 9025, }, }, }, }, restore: builder.ForRestore(api.DefaultNamespace, "").PreserveNodePorts(true).Result(), expectedRes: corev1api.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "svc-1", }, Spec: corev1api.ServiceSpec{ Ports: []corev1api.ServicePort{ { Name: "http", Port: 80, NodePort: 8080, }, { Name: "hepsiburada", NodePort: 9025, }, }, }, }, }, { name: "nodePort should be delete when not specified in managedFields", obj: corev1api.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "svc-1", ManagedFields: []metav1.ManagedFieldsEntry{ { FieldsV1: &metav1.FieldsV1{ Raw: []byte(`{"f:spec":{"f:ports":{"k:{\"port\":443,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{}},"k:{\"port\":80,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{}}},"f:selector":{},"f:type":{}}}`), }, }, }, }, Spec: corev1api.ServiceSpec{ Ports: []corev1api.ServicePort{ { Name: "http", Port: 80, Protocol: "TCP", }, { Name: "https", Port: 443, Protocol: "TCP", }, }, }, }, restore: builder.ForRestore(api.DefaultNamespace, "").Result(), expectedRes: corev1api.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "svc-1", ManagedFields: []metav1.ManagedFieldsEntry{ { FieldsV1: &metav1.FieldsV1{ Raw: []byte(`{"f:spec":{"f:ports":{"k:{\"port\":443,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{}},"k:{\"port\":80,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{}}},"f:selector":{},"f:type":{}}}`), }, }, }, }, Spec: corev1api.ServiceSpec{ Ports: []corev1api.ServicePort{ { Name: "http", Port: 80, NodePort: 0, Protocol: "TCP", }, { Name: "https", Port: 443, NodePort: 0, Protocol: "TCP", }, }, }, }, }, { name: "nodePort should be preserved when specified in managedFields", obj: corev1api.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "svc-1", ManagedFields: []metav1.ManagedFieldsEntry{ { FieldsV1: &metav1.FieldsV1{ Raw: []byte(`{"f:spec":{"f:ports":{"k:{\"port\":443,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:nodePort":{},"f:port":{}},"k:{\"port\":80,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:nodePort":{},"f:port":{}}},"f:selector":{},"f:type":{}}}`), }, }, }, }, Spec: corev1api.ServiceSpec{ Ports: []corev1api.ServicePort{ { Name: "http", Port: 80, NodePort: 30000, Protocol: "TCP", }, { Name: "https", Port: 443, NodePort: 30002, Protocol: "TCP", }, }, }, }, restore: builder.ForRestore(api.DefaultNamespace, "").Result(), expectedRes: corev1api.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "svc-1", ManagedFields: []metav1.ManagedFieldsEntry{ { FieldsV1: &metav1.FieldsV1{ Raw: []byte(`{"f:spec":{"f:ports":{"k:{\"port\":443,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:nodePort":{},"f:port":{}},"k:{\"port\":80,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:nodePort":{},"f:port":{}}},"f:selector":{},"f:type":{}}}`), }, }, }, }, Spec: corev1api.ServiceSpec{ Ports: []corev1api.ServicePort{ { Name: "http", Port: 80, NodePort: 30000, Protocol: "TCP", }, { Name: "https", Port: 443, NodePort: 30002, Protocol: "TCP", }, }, }, }, }, { name: "If PreserveNodePorts is True in restore spec then HealthCheckNodePort always preserved.", obj: corev1api.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "svc-1", }, Spec: corev1api.ServiceSpec{ HealthCheckNodePort: 8080, ExternalTrafficPolicy: corev1api.ServiceExternalTrafficPolicyTypeLocal, Type: corev1api.ServiceTypeLoadBalancer, Ports: []corev1api.ServicePort{ { Name: "http", Port: 80, NodePort: 8080, }, { Name: "hepsiburada", NodePort: 9025, }, }, }, }, restore: builder.ForRestore(api.DefaultNamespace, "").PreserveNodePorts(true).Result(), expectedRes: corev1api.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "svc-1", }, Spec: corev1api.ServiceSpec{ HealthCheckNodePort: 8080, ExternalTrafficPolicy: corev1api.ServiceExternalTrafficPolicyTypeLocal, Type: corev1api.ServiceTypeLoadBalancer, Ports: []corev1api.ServicePort{ { Name: "http", Port: 80, NodePort: 8080, }, { Name: "hepsiburada", NodePort: 9025, }, }, }, }, }, { name: "If PreserveNodePorts is False in restore spec then HealthCheckNodePort should be cleaned.", obj: corev1api.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "svc-1", }, Spec: corev1api.ServiceSpec{ HealthCheckNodePort: 8080, ExternalTrafficPolicy: corev1api.ServiceExternalTrafficPolicyTypeLocal, Type: corev1api.ServiceTypeLoadBalancer, }, }, restore: builder.ForRestore(api.DefaultNamespace, "").PreserveNodePorts(false).Result(), expectedRes: corev1api.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "svc-1", }, Spec: corev1api.ServiceSpec{ HealthCheckNodePort: 0, ExternalTrafficPolicy: corev1api.ServiceExternalTrafficPolicyTypeLocal, Type: corev1api.ServiceTypeLoadBalancer, }, }, }, { name: "If PreserveNodePorts is false in restore spec, but service is not expected, then HealthCheckNodePort should be kept.", obj: corev1api.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "svc-1", }, Spec: corev1api.ServiceSpec{ HealthCheckNodePort: 8080, ExternalTrafficPolicy: corev1api.ServiceExternalTrafficPolicyTypeCluster, Type: corev1api.ServiceTypeLoadBalancer, }, }, restore: builder.ForRestore(api.DefaultNamespace, "").PreserveNodePorts(false).Result(), expectedRes: corev1api.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "svc-1", }, Spec: corev1api.ServiceSpec{ HealthCheckNodePort: 8080, ExternalTrafficPolicy: corev1api.ServiceExternalTrafficPolicyTypeCluster, Type: corev1api.ServiceTypeLoadBalancer, }, }, }, { name: "If PreserveNodePorts is false in restore spec, but HealthCheckNodePort can be found in Annotation, then it should be kept.", obj: corev1api.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "svc-1", Annotations: map[string]string{annotationLastAppliedConfig: svcJSON()}, }, Spec: corev1api.ServiceSpec{ HealthCheckNodePort: 8080, ExternalTrafficPolicy: corev1api.ServiceExternalTrafficPolicyTypeLocal, Type: corev1api.ServiceTypeLoadBalancer, }, }, restore: builder.ForRestore(api.DefaultNamespace, "").PreserveNodePorts(false).Result(), expectedRes: corev1api.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "svc-1", Annotations: map[string]string{annotationLastAppliedConfig: svcJSON()}, }, Spec: corev1api.ServiceSpec{ HealthCheckNodePort: 8080, ExternalTrafficPolicy: corev1api.ServiceExternalTrafficPolicyTypeLocal, Type: corev1api.ServiceTypeLoadBalancer, }, }, }, { name: "If PreserveNodePorts is false in restore spec, but HealthCheckNodePort can be found in ManagedFields, then it should be kept.", obj: corev1api.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "svc-1", ManagedFields: []metav1.ManagedFieldsEntry{ { FieldsV1: &metav1.FieldsV1{ Raw: []byte(`{"f:spec":{"f:healthCheckNodePort":{}}}`), }, }, }, }, Spec: corev1api.ServiceSpec{ HealthCheckNodePort: 8080, ExternalTrafficPolicy: corev1api.ServiceExternalTrafficPolicyTypeLocal, Type: corev1api.ServiceTypeLoadBalancer, }, }, restore: builder.ForRestore(api.DefaultNamespace, "").PreserveNodePorts(false).Result(), expectedRes: corev1api.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "svc-1", ManagedFields: []metav1.ManagedFieldsEntry{ { FieldsV1: &metav1.FieldsV1{ Raw: []byte(`{"f:spec":{"f:healthCheckNodePort":{}}}`), }, }, }, }, Spec: corev1api.ServiceSpec{ HealthCheckNodePort: 8080, ExternalTrafficPolicy: corev1api.ServiceExternalTrafficPolicyTypeLocal, Type: corev1api.ServiceTypeLoadBalancer, }, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { action := NewServiceAction(velerotest.NewLogger()) unstructuredSvc, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&test.obj) require.NoError(t, err) res, err := action.Execute(&velero.RestoreItemActionExecuteInput{ Item: &unstructured.Unstructured{Object: unstructuredSvc}, ItemFromBackup: &unstructured.Unstructured{Object: unstructuredSvc}, Restore: test.restore, }) if assert.Equal(t, test.expectedErr, err != nil) && !test.expectedErr { var svc corev1api.Service require.NoError(t, runtime.DefaultUnstructuredConverter.FromUnstructured(res.UpdatedItem.UnstructuredContent(), &svc)) assert.Equal(t, test.expectedRes, svc) } }) } } ================================================ FILE: pkg/restore/merge_service_account.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package restore import ( "encoding/json" jsonpatch "github.com/evanphx/json-patch/v5" "github.com/pkg/errors" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" ) // mergeServiceAccount takes a backed up serviceaccount and merges attributes into the current in-cluster service account. // Labels and Annotations on the backed up version but not on the in-cluster version will be merged. If a key is specified in both, the in-cluster version is retained. func mergeServiceAccounts(fromCluster, fromBackup *unstructured.Unstructured) (*unstructured.Unstructured, error) { desired := new(corev1api.ServiceAccount) if err := runtime.DefaultUnstructuredConverter.FromUnstructured(fromCluster.UnstructuredContent(), desired); err != nil { return nil, errors.Wrap(err, "unable to convert from-cluster service account from unstructured to serviceaccount") } backupSA := new(corev1api.ServiceAccount) if err := runtime.DefaultUnstructuredConverter.FromUnstructured(fromBackup.UnstructuredContent(), backupSA); err != nil { return nil, errors.Wrap(err, "unable to convert from backed up service account unstructured to serviceaccount") } desired.Secrets = mergeObjectReferenceSlices(desired.Secrets, backupSA.Secrets) desired.ImagePullSecrets = mergeLocalObjectReferenceSlices(desired.ImagePullSecrets, backupSA.ImagePullSecrets) desired.Labels = mergeMaps(desired.Labels, backupSA.Labels) desired.Annotations = mergeMaps(desired.Annotations, backupSA.Annotations) desiredUnstructured, err := runtime.DefaultUnstructuredConverter.ToUnstructured(desired) if err != nil { return nil, errors.Wrap(err, "unable to convert desired service account to unstructured") } // The DefaultUnstructuredConverter.ToUnstructured function will populate the creation timestamp with the nil value // However, we remove this on both the backup and cluster objects before comparison, and we don't want it in any patches. delete(desiredUnstructured["metadata"].(map[string]any), "creationTimestamp") return &unstructured.Unstructured{Object: desiredUnstructured}, nil } func mergeObjectReferenceSlices(first, second []corev1api.ObjectReference) []corev1api.ObjectReference { for _, s := range second { var exists bool for _, f := range first { if s.Name == f.Name { exists = true break } } if !exists { first = append(first, s) } } return first } func mergeLocalObjectReferenceSlices(first, second []corev1api.LocalObjectReference) []corev1api.LocalObjectReference { for _, s := range second { var exists bool for _, f := range first { if s.Name == f.Name { exists = true break } } if !exists { first = append(first, s) } } return first } // mergeMaps takes two map[string]string and merges missing keys from the second into the first. // If a key already exists, its value is not overwritten. func mergeMaps(first, second map[string]string) map[string]string { // If the first map passed in is empty, just use all of the second map's data if first == nil { first = map[string]string{} } for k, v := range second { _, ok := first[k] if !ok { first[k] = v } } return first } // generatePatch will calculate a JSON merge patch for an object's desired state. // If the passed in objects are already equal, nil is returned. func generatePatch(fromCluster, desired *unstructured.Unstructured) ([]byte, error) { // If the objects are already equal, there's no need to generate a patch. if equality.Semantic.DeepEqual(fromCluster, desired) { return nil, nil } desiredBytes, err := json.Marshal(desired.Object) if err != nil { return nil, errors.Wrap(err, "unable to marshal desired object") } fromClusterBytes, err := json.Marshal(fromCluster.Object) if err != nil { return nil, errors.Wrap(err, "unable to marshal in-cluster object") } patchBytes, err := jsonpatch.CreateMergePatch(fromClusterBytes, desiredBytes) if err != nil { return nil, errors.Wrap(err, "unable to create merge patch") } return patchBytes, nil } ================================================ FILE: pkg/restore/merge_service_account_test.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package restore import ( "strings" "testing" "unicode" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) var mergedServiceAccountsBenchmarkResult *unstructured.Unstructured func BenchmarkMergeServiceAccountBasic(b *testing.B) { tests := []struct { name string fromCluster *unstructured.Unstructured fromBackup *unstructured.Unstructured }{ { name: "only default tokens present", fromCluster: velerotest.UnstructuredOrDie( `{ "apiVersion": "v1", "kind": "ServiceAccount", "metadata": { "namespace": "ns1", "name": "default" }, "secrets": [ { "name": "default-token-abcde" } ] }`, ), fromBackup: velerotest.UnstructuredOrDie( `{ "kind": "ServiceAccount", "apiVersion": "v1", "metadata": { "namespace": "ns1", "name": "default" }, "secrets": [ { "name": "default-token-xzy12" } ] }`, ), }, { name: "service accounts with multiple secrets", fromCluster: velerotest.UnstructuredOrDie( `{ "apiVersion": "v1", "kind": "ServiceAccount", "metadata": { "namespace": "ns1", "name": "default" }, "secrets": [ { "name": "default-token-abcde" }, { "name": "my-secret" }, { "name": "sekrit" } ] }`, ), fromBackup: velerotest.UnstructuredOrDie( `{ "kind": "ServiceAccount", "apiVersion": "v1", "metadata": { "namespace": "ns1", "name": "default" }, "secrets": [ { "name": "default-token-xzy12" }, { "name": "my-old-secret" }, { "name": "secrete"} ] }`, ), }, { name: "service accounts with labels and annotations", fromCluster: velerotest.UnstructuredOrDie( `{ "apiVersion": "v1", "kind": "ServiceAccount", "metadata": { "namespace": "ns1", "name": "default", "labels": { "l1": "v1", "l2": "v2", "l3": "v3" }, "annotations": { "a1": "v1", "a2": "v2", "a3": "v3", "a4": "v4" } }, "secrets": [ { "name": "default-token-abcde" } ] }`, ), fromBackup: velerotest.UnstructuredOrDie( `{ "kind": "ServiceAccount", "apiVersion": "v1", "metadata": { "namespace": "ns1", "name": "default", "labels": { "l1": "v1", "l2": "v2", "l3": "v3", "l4": "v4", "l5": "v5" }, "annotations": { "a1": "v1", "a2": "v2", "a3": "v3", "a4": "v4", "a5": "v5", "a6": "v6" } }, "secrets": [ { "name": "default-token-xzy12" } ] }`, ), }, } var desired *unstructured.Unstructured for _, test := range tests { b.Run(test.name, func(b *testing.B) { for n := 0; n < b.N; n++ { desired, _ = mergeServiceAccounts(test.fromCluster, test.fromBackup) } mergedServiceAccountsBenchmarkResult = desired }) } } func TestMergeLocalObjectReferenceSlices(t *testing.T) { tests := []struct { name string first []corev1api.LocalObjectReference second []corev1api.LocalObjectReference expected []corev1api.LocalObjectReference }{ { name: "two slices without overlapping elements", first: []corev1api.LocalObjectReference{ {Name: "lor1"}, {Name: "lor2"}, }, second: []corev1api.LocalObjectReference{ {Name: "lor3"}, {Name: "lor4"}, }, expected: []corev1api.LocalObjectReference{ {Name: "lor1"}, {Name: "lor2"}, {Name: "lor3"}, {Name: "lor4"}, }, }, { name: "two slices with an overlapping element", first: []corev1api.LocalObjectReference{ {Name: "lor1"}, {Name: "lor2"}, }, second: []corev1api.LocalObjectReference{ {Name: "lor3"}, {Name: "lor2"}, }, expected: []corev1api.LocalObjectReference{ {Name: "lor1"}, {Name: "lor2"}, {Name: "lor3"}, }, }, { name: "merging always adds elements to the end", first: []corev1api.LocalObjectReference{ {Name: "lor3"}, {Name: "lor4"}, }, second: []corev1api.LocalObjectReference{ {Name: "lor1"}, {Name: "lor2"}, }, expected: []corev1api.LocalObjectReference{ {Name: "lor3"}, {Name: "lor4"}, {Name: "lor1"}, {Name: "lor2"}, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { result := mergeLocalObjectReferenceSlices(test.first, test.second) assert.Equal(t, test.expected, result) }) } } func TestMergeObjectReferenceSlices(t *testing.T) { tests := []struct { name string first []corev1api.ObjectReference second []corev1api.ObjectReference expected []corev1api.ObjectReference }{ { name: "two slices without overlapping elements", first: []corev1api.ObjectReference{ {Name: "or1"}, {Name: "or2"}, }, second: []corev1api.ObjectReference{ {Name: "or3"}, {Name: "or4"}, }, expected: []corev1api.ObjectReference{ {Name: "or1"}, {Name: "or2"}, {Name: "or3"}, {Name: "or4"}, }, }, { name: "two slices with an overlapping element", first: []corev1api.ObjectReference{ {Name: "or1"}, {Name: "or2"}, }, second: []corev1api.ObjectReference{ {Name: "or3"}, {Name: "or2"}, }, expected: []corev1api.ObjectReference{ {Name: "or1"}, {Name: "or2"}, {Name: "or3"}, }, }, { name: "merging always adds elements to the end", first: []corev1api.ObjectReference{ {Name: "or3"}, {Name: "or4"}, }, second: []corev1api.ObjectReference{ {Name: "or1"}, {Name: "or2"}, }, expected: []corev1api.ObjectReference{ {Name: "or3"}, {Name: "or4"}, {Name: "or1"}, {Name: "or2"}, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { result := mergeObjectReferenceSlices(test.first, test.second) assert.Equal(t, test.expected, result) }) } } // stripWhitespace removes any Unicode whitespace from a string. // Useful for cleaning up formatting on expected JSON strings before comparison func stripWhitespace(s string) string { return strings.Map(func(r rune) rune { if unicode.IsSpace(r) { return -1 } return r }, s) } func TestMergeMaps(t *testing.T) { var testCases = []struct { name string source map[string]string destination map[string]string expected map[string]string }{ { name: "nil destination should result in source being copied", destination: nil, source: map[string]string{ "k1": "v1", }, expected: map[string]string{ "k1": "v1", }, }, { name: "keys missing from destination should be copied from source", destination: map[string]string{ "k2": "v2", }, source: map[string]string{ "k1": "v1", }, expected: map[string]string{ "k1": "v1", "k2": "v2", }, }, { name: "matching key should not have value copied from source", destination: map[string]string{ "k1": "v1", }, source: map[string]string{ "k1": "v2", }, expected: map[string]string{ "k1": "v1", }, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { result := mergeMaps(tc.destination, tc.source) assert.Equal(t, tc.expected, result) }) } } func TestGeneratePatch(t *testing.T) { tests := []struct { name string fromCluster *unstructured.Unstructured desired *unstructured.Unstructured expectedString string expectedErr bool }{ { name: "objects are equal, no patch needed", fromCluster: velerotest.UnstructuredOrDie( `{ "apiVersion": "v1", "kind": "ServiceAccount", "metadata": { "namespace": "ns1", "name": "default" }, "secrets": [ { "name": "default-token-abcde" } ] }`, ), desired: velerotest.UnstructuredOrDie( `{ "apiVersion": "v1", "kind": "ServiceAccount", "metadata": { "namespace": "ns1", "name": "default" }, "secrets": [ { "name": "default-token-abcde" } ] }`, ), expectedString: "", expectedErr: false, }, { name: "patch is required when labels are present", fromCluster: velerotest.UnstructuredOrDie( `{ "apiVersion": "v1", "kind": "ServiceAccount", "metadata": { "namespace": "ns1", "name": "default" }, "secrets": [ { "name": "default-token-abcde" } ] }`, ), desired: velerotest.UnstructuredOrDie( `{ "apiVersion": "v1", "kind": "ServiceAccount", "metadata": { "namespace": "ns1", "name": "default", "labels": { "label1": "value1", "label2": "value2" } }, "secrets": [ { "name": "default-token-abcde" } ] }`, ), expectedString: stripWhitespace( `{ "metadata": { "labels": { "label1":"value1", "label2":"value2" } } }`, ), expectedErr: false, }, { name: "patch is required when annotations are present", fromCluster: velerotest.UnstructuredOrDie( `{ "apiVersion": "v1", "kind": "ServiceAccount", "metadata": { "namespace": "ns1", "name": "default" }, "secrets": [ { "name": "default-token-abcde" } ] }`, ), desired: velerotest.UnstructuredOrDie( `{ "apiVersion": "v1", "kind": "ServiceAccount", "metadata": { "namespace": "ns1", "name": "default", "annotations" :{ "a1": "v1", "a2": "v2" } }, "secrets": [ { "name": "default-token-abcde" } ] }`, ), expectedString: stripWhitespace( `{ "metadata": { "annotations": { "a1":"v1", "a2":"v2" } } }`, ), expectedErr: false, }, { name: "patch is required many secrets are present", fromCluster: velerotest.UnstructuredOrDie( `{ "apiVersion": "v1", "kind": "ServiceAccount", "metadata": { "namespace": "ns1", "name": "default" }, "secrets": [ { "name": "default-token-abcde" } ] }`, ), desired: velerotest.UnstructuredOrDie( `{ "apiVersion": "v1", "kind": "ServiceAccount", "metadata": { "namespace": "ns1", "name": "default" }, "secrets": [ { "name": "default-token-abcde" }, { "name": "sekrit" }, { "name": "secrete" } ] }`, ), expectedString: stripWhitespace( `{ "secrets": [ {"name": "default-token-abcde"}, {"name": "sekrit"}, {"name": "secrete"} ] }`, ), expectedErr: false, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { result, err := generatePatch(test.fromCluster, test.desired) if assert.Equal(t, test.expectedErr, err != nil) { assert.Equal(t, test.expectedString, string(result)) } }) } } func TestMergeServiceAccountBasic(t *testing.T) { tests := []struct { name string fromCluster *unstructured.Unstructured fromBackup *unstructured.Unstructured expectedRes *unstructured.Unstructured expectedErr bool }{ { name: "only default token", fromCluster: velerotest.UnstructuredOrDie( `{ "apiVersion": "v1", "kind": "ServiceAccount", "metadata": { "namespace": "ns1", "name": "default" }, "secrets": [ { "name": "default-token-abcde" } ] }`, ), // fromBackup doesn't have the default token because it is expected to already have been removed // by the service account action fromBackup: velerotest.UnstructuredOrDie( `{ "kind": "ServiceAccount", "apiVersion": "v1", "metadata": { "namespace": "ns1", "name": "default" }, "secrets": [] }`, ), expectedRes: velerotest.UnstructuredOrDie( `{ "apiVersion": "v1", "kind": "ServiceAccount", "metadata": { "namespace": "ns1", "name": "default" }, "secrets": [ { "name": "default-token-abcde" } ] }`, ), }, { name: "service accounts with multiple secrets", fromCluster: velerotest.UnstructuredOrDie( `{ "apiVersion": "v1", "kind": "ServiceAccount", "metadata": { "namespace": "ns1", "name": "default" }, "secrets": [ { "name": "default-token-abcde" }, { "name": "my-secret" }, { "name": "sekrit" } ] }`, ), // fromBackup doesn't have the default token because it is expected to already have been removed // by the service account action fromBackup: velerotest.UnstructuredOrDie( `{ "kind": "ServiceAccount", "apiVersion": "v1", "metadata": { "namespace": "ns1", "name": "default" }, "secrets": [ { "name": "my-old-secret" }, { "name": "secrete"} ] }`, ), expectedRes: velerotest.UnstructuredOrDie( `{ "apiVersion": "v1", "kind": "ServiceAccount", "metadata": { "namespace": "ns1", "name": "default" }, "secrets": [ { "name": "default-token-abcde" }, { "name": "my-secret" }, { "name": "sekrit" }, { "name": "my-old-secret" }, { "name": "secrete"} ] }`, ), }, { name: "service accounts with labels and annotations", fromCluster: velerotest.UnstructuredOrDie( `{ "apiVersion": "v1", "kind": "ServiceAccount", "metadata": { "namespace": "ns1", "name": "default", "labels": { "l1": "v1", "l2": "v2", "l3": "v3" }, "annotations": { "a1": "v1", "a2": "v2", "a3": "v3", "a4": "v4" } }, "secrets": [ { "name": "default-token-abcde" } ] }`, ), // fromBackup doesn't have the default token because it is expected to already have been removed // by the service account action fromBackup: velerotest.UnstructuredOrDie( `{ "kind": "ServiceAccount", "apiVersion": "v1", "metadata": { "namespace": "ns1", "name": "default", "labels": { "l1": "v1", "l2": "v2", "l3": "v3", "l4": "v4", "l5": "v5" }, "annotations": { "a1": "v1", "a2": "v2", "a3": "v3", "a4": "v4", "a5": "v5", "a6": "v6" } }, "secrets": [] }`, ), expectedRes: velerotest.UnstructuredOrDie( `{ "kind": "ServiceAccount", "apiVersion": "v1", "metadata": { "namespace": "ns1", "name": "default", "labels": { "l1": "v1", "l2": "v2", "l3": "v3", "l4": "v4", "l5": "v5" }, "annotations": { "a1": "v1", "a2": "v2", "a3": "v3", "a4": "v4", "a5": "v5", "a6": "v6" } }, "secrets": [ { "name": "default-token-abcde" } ] }`, ), }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { result, err := mergeServiceAccounts(test.fromCluster, test.fromBackup) require.NoError(t, err) assert.Equal(t, test.expectedRes, result) }) } } ================================================ FILE: pkg/restore/prioritize_group_version.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package restore import ( "context" "sort" "strings" "github.com/pkg/errors" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/version" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/archive" "github.com/vmware-tanzu/velero/pkg/client" ) // ChosenGroupVersion is the API Group version that was selected to restore // from potentially multiple backed up version enabled by the feature flag // APIGroupVersionsFeatureFlag type ChosenGroupVersion struct { Group string Version string Dir string } // chooseAPIVersionsToRestore will choose a version to restore based on a user- // provided config map prioritization or our version prioritization. func (ctx *restoreContext) chooseAPIVersionsToRestore() error { sourceGVs, targetGVs, userGVs, err := ctx.gatherSourceTargetUserGroupVersions() if err != nil { return err } OUTER: for rg, sg := range sourceGVs { // Default to the source preferred version if no other common version // can be found. cgv := ChosenGroupVersion{ Group: sg.Name, Version: sg.PreferredVersion.Version, Dir: sg.PreferredVersion.Version + velerov1api.PreferredVersionDir, } tg := findAPIGroup(targetGVs, sg.Name) if len(tg.Versions) == 0 { ctx.chosenGrpVersToRestore[rg] = cgv ctx.log.Debugf("Chose %s/%s API group version to restore", cgv.Group, cgv.Version) continue } // Priority 0: User Priority Version if userGVs != nil { uv := findSupportedUserVersion(userGVs[rg].Versions, tg.Versions, sg.Versions) if uv != "" { cgv.Version = uv cgv.Dir = uv if uv == sg.PreferredVersion.Version { cgv.Dir += velerov1api.PreferredVersionDir } ctx.chosenGrpVersToRestore[rg] = cgv ctx.log.Debugf("APIGroupVersionsFeatureFlag Priority 0: User defined API group version %s chosen for %s", uv, rg) continue } ctx.log.Infof("Cannot find user defined version in both the cluster and backup cluster. Ignoring version %s for %s", uv, rg) } // Priority 1: Target Cluster Preferred Version if versionsContain(sg.Versions, tg.PreferredVersion.Version) { cgv.Version = tg.PreferredVersion.Version cgv.Dir = tg.PreferredVersion.Version if tg.PreferredVersion.Version == sg.PreferredVersion.Version { cgv.Dir += velerov1api.PreferredVersionDir } ctx.chosenGrpVersToRestore[rg] = cgv ctx.log.Debugf( "APIGroupVersionsFeatureFlag Priority 1: Cluster preferred API group version %s found in backup for %s", tg.PreferredVersion.Version, rg, ) continue } ctx.log.Infof("Cannot find cluster preferred API group version in backup. Ignoring version %s for %s", tg.PreferredVersion.Version, rg) // Priority 2: Source Cluster Preferred Version if versionsContain(tg.Versions, sg.PreferredVersion.Version) { cgv.Version = sg.PreferredVersion.Version cgv.Dir = cgv.Version + velerov1api.PreferredVersionDir ctx.chosenGrpVersToRestore[rg] = cgv ctx.log.Debugf( "APIGroupVersionsFeatureFlag Priority 2: Cluster preferred API group version not found in backup. Using backup preferred version %s for %s", sg.PreferredVersion.Version, rg, ) continue } ctx.log.Infof("Cannot find backup preferred API group version in cluster. Ignoring version %s for %s", sg.PreferredVersion.Version, rg) // Priority 3: The Common Supported Version with the Highest Kubernetes Version Priority for _, tv := range tg.Versions[1:] { if versionsContain(sg.Versions[1:], tv.Version) { cgv.Version = tv.Version cgv.Dir = tv.Version ctx.chosenGrpVersToRestore[rg] = cgv ctx.log.Debugf( "APIGroupVersionsFeatureFlag Priority 3: Common supported but not preferred API group version %s chosen for %s", tv.Version, rg, ) continue OUTER } } ctx.log.Infof("Cannot find non-preferred a common supported API group version. Using %s (default behavior without feature flag) for %s", sg.PreferredVersion.Version, rg) // Use default group version. ctx.chosenGrpVersToRestore[rg] = cgv ctx.log.Debugf( "APIGroupVersionsFeatureFlag: Unable to find supported priority API group version. Using backup preferred version %s for %s (default behavior without feature flag).", tg.PreferredVersion.Version, rg, ) } return nil } // gatherSourceTargetUserGroupVersions collects the source, target, and user priority versions. func (ctx *restoreContext) gatherSourceTargetUserGroupVersions() ( map[string]metav1.APIGroup, []metav1.APIGroup, map[string]metav1.APIGroup, error, ) { sourceRGVersions, err := archive.NewParser(ctx.log, ctx.fileSystem).ParseGroupVersions(ctx.restoreDir) if err != nil { return nil, nil, nil, errors.Wrap(err, "parsing versions from directory names") } // Sort the versions in the APIGroups in sourceRGVersions map values. for _, src := range sourceRGVersions { k8sPrioritySort(src.Versions) } targetGroupVersions := ctx.discoveryHelper.APIGroups() // Sort the versions in the APIGroups slice in targetGroupVersions. for _, target := range targetGroupVersions { k8sPrioritySort(target.Versions) } // Get the user-provided enableapigroupversion config map. cm, err := userPriorityConfigMap() if err != nil { return nil, nil, nil, errors.Wrap(err, "retrieving enableapigroupversion config map") } // Read user-defined version priorities from config map. userRGVPriorities := userResourceGroupVersionPriorities(ctx, cm) return sourceRGVersions, targetGroupVersions, userRGVPriorities, nil } // k8sPrioritySort sorts slices using Kubernetes' version prioritization. func k8sPrioritySort(gvs []metav1.GroupVersionForDiscovery) { sort.SliceStable(gvs, func(i, j int) bool { return version.CompareKubeAwareVersionStrings(gvs[i].Version, gvs[j].Version) > 0 }) } // userResourceGroupVersionPriorities retrieves a user-provided config map and // extracts the user priority versions for each resource. func userResourceGroupVersionPriorities(ctx *restoreContext, cm *corev1api.ConfigMap) map[string]metav1.APIGroup { if cm == nil { ctx.log.Debugf("No enableapigroupversion config map found in velero namespace. Using pre-defined priorities.") return nil } priorities := parseUserPriorities(ctx, cm.Data["restoreResourcesVersionPriority"]) if len(priorities) == 0 { ctx.log.Debugf("No valid user version priorities found in enableapigroupversion config map. Using pre-defined priorities.") return nil } return priorities } func userPriorityConfigMap() (*corev1api.ConfigMap, error) { cfg, err := client.LoadConfig() if err != nil { return nil, errors.Wrap(err, "reading client config file") } fc := client.NewFactory("APIGroupVersionsRestore", cfg) kc, err := fc.KubeClient() if err != nil { return nil, errors.Wrap(err, "getting Kube client") } cm, err := kc.CoreV1().ConfigMaps(fc.Namespace()).Get( context.Background(), "enableapigroupversions", metav1.GetOptions{}, ) if err != nil { if apierrors.IsNotFound(err) { return nil, nil } return nil, errors.Wrap(err, "getting enableapigroupversions config map from velero namespace") } return cm, nil } func parseUserPriorities(ctx *restoreContext, prioritiesData string) map[string]metav1.APIGroup { userPriorities := make(map[string]metav1.APIGroup) // The user priorities will be in a string of the form // rockbands.music.example.io=v2beta1,v2beta2\n // orchestras.music.example.io=v2,v3alpha1\n // subscriptions.operators.coreos.com=v2,v1 lines := strings.Split(prioritiesData, "\n") lines = formatUserPriorities(lines) for _, line := range lines { err := validateUserPriority(line) if err == nil { rgvs := strings.SplitN(line, "=", 2) rg := rgvs[0] // rockbands.music.example.io versions := rgvs[1] // v2beta1,v2beta2 vers := strings.Split(versions, ",") userPriorities[rg] = metav1.APIGroup{ Versions: versionsToGroupVersionForDiscovery(vers), } } else { ctx.log.Debugf("Unable to validate user priority versions %q due to %v", line, err) } } return userPriorities } // formatUserPriorities removes extra white spaces that cause validation to fail. func formatUserPriorities(lines []string) []string { trimmed := []string{} for _, line := range lines { temp := strings.ReplaceAll(line, " ", "") if len(temp) > 0 { trimmed = append(trimmed, temp) } } return trimmed } func validateUserPriority(line string) error { if strings.Count(line, "=") != 1 { return errors.New("line must have one and only one equal sign") } pair := strings.Split(line, "=") if len(pair[0]) < 1 || len(pair[1]) < 1 { return errors.New("line must contain at least one character before and after equal sign") } // Line must not contain any spaces if strings.Count(line, " ") > 0 { return errors.New("line must not contain any spaces") } return nil } // versionsToGroupVersionForDiscovery converts version strings into a Kubernetes format // for group versions. func versionsToGroupVersionForDiscovery(vs []string) []metav1.GroupVersionForDiscovery { gvs := make([]metav1.GroupVersionForDiscovery, len(vs)) for i, v := range vs { gvs[i] = metav1.GroupVersionForDiscovery{ Version: v, } } return gvs } // findAPIGroup looks for an API Group by a group name. func findAPIGroup(groups []metav1.APIGroup, name string) metav1.APIGroup { for _, g := range groups { if g.Name == name { return g } } return metav1.APIGroup{} } // findSupportedUserVersion finds the first user priority version that both source // and target support. func findSupportedUserVersion(userGVs, targetGVs, sourceGVs []metav1.GroupVersionForDiscovery) string { for _, ug := range userGVs { if versionsContain(targetGVs, ug.Version) && versionsContain(sourceGVs, ug.Version) { return ug.Version } } return "" } // versionsContain will check if a version can be found in a slice of versions. func versionsContain(list []metav1.GroupVersionForDiscovery, version string) bool { for _, v := range list { if v.Version == version { return true } } return false } ================================================ FILE: pkg/restore/prioritize_group_version_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package restore import ( "testing" "github.com/stretchr/testify/assert" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/test" ) func TestK8sPrioritySort(t *testing.T) { tests := []struct { name string orig []metav1.GroupVersionForDiscovery want []metav1.GroupVersionForDiscovery }{ { name: "sorts Kubernetes API group versions per k8s priority", orig: []metav1.GroupVersionForDiscovery{ {Version: "v2"}, {Version: "v11alpha2"}, {Version: "foo10"}, {Version: "v10"}, {Version: "v12alpha1"}, {Version: "v3beta1"}, {Version: "foo1"}, {Version: "v1"}, {Version: "v10beta3"}, {Version: "v11beta2"}, }, want: []metav1.GroupVersionForDiscovery{ {Version: "v10"}, {Version: "v2"}, {Version: "v1"}, {Version: "v11beta2"}, {Version: "v10beta3"}, {Version: "v3beta1"}, {Version: "v12alpha1"}, {Version: "v11alpha2"}, {Version: "foo1"}, {Version: "foo10"}, }, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { k8sPrioritySort(tc.orig) assert.Equal(t, tc.want, tc.orig) }) } } func TestUserResourceGroupVersionPriorities(t *testing.T) { tests := []struct { name string cm *corev1api.ConfigMap want map[string]metav1.APIGroup wantErrMsg string }{ { name: "retrieve version priority data from config map", cm: builder. ForConfigMap("velero", "enableapigroupversions"). Data( "restoreResourcesVersionPriority", `rockbands.music.example.io=v2beta1,v2beta2 orchestras.music.example.io=v2,v3alpha1 subscriptions.operators.coreos.com=v2,v1`, ). Result(), want: map[string]metav1.APIGroup{ "rockbands.music.example.io": {Versions: []metav1.GroupVersionForDiscovery{ {Version: "v2beta1"}, {Version: "v2beta2"}, }}, "orchestras.music.example.io": {Versions: []metav1.GroupVersionForDiscovery{ {Version: "v2"}, {Version: "v3alpha1"}, }}, "subscriptions.operators.coreos.com": {Versions: []metav1.GroupVersionForDiscovery{ {Version: "v2"}, {Version: "v1"}, }}, }, }, { name: "incorrect data format returns an error", cm: builder. ForConfigMap("velero", "enableapigroupversions"). Data( "restoreResourcesVersionPriority", `rockbands.music.example.io=v2beta1,v2beta2\n orchestras.music.example.io=v2,v3alpha1`, ). Result(), want: nil, wantErrMsg: "parsing user priorities: validating user priority: line must have one and only one equal sign", }, { name: "spaces and empty lines are removed before storing user version priorities", cm: builder. ForConfigMap("velero", "enableapigroupversions"). Data( "restoreResourcesVersionPriority", ` pods=v2,v1beta2 horizontalpodautoscalers.autoscaling = v2beta2 jobs.batch=v3 `, ). Result(), want: map[string]metav1.APIGroup{ "pods": {Versions: []metav1.GroupVersionForDiscovery{ {Version: "v2"}, {Version: "v1beta2"}, }}, "horizontalpodautoscalers.autoscaling": {Versions: []metav1.GroupVersionForDiscovery{ {Version: "v2beta2"}, }}, "jobs.batch": {Versions: []metav1.GroupVersionForDiscovery{ {Version: "v3"}, }}, }, }, } fakeCtx := &restoreContext{ log: test.NewLogger(), } for _, tc := range tests { t.Log(tc.name) priorities := userResourceGroupVersionPriorities(fakeCtx, tc.cm) assert.Equal(t, tc.want, priorities) } } func TestFindAPIGroup(t *testing.T) { tests := []struct { name string targetGrps []metav1.APIGroup grpName string want metav1.APIGroup }{ { name: "return the API Group in target list matching group string", targetGrps: []metav1.APIGroup{ { Name: "rbac.authorization.k8s.io", Versions: []metav1.GroupVersionForDiscovery{ {Version: "v2"}, }, PreferredVersion: metav1.GroupVersionForDiscovery{Version: "v2"}, }, { Name: "", Versions: []metav1.GroupVersionForDiscovery{ {Version: "v1"}, }, PreferredVersion: metav1.GroupVersionForDiscovery{Version: "v1"}, }, { Name: "velero.io", Versions: []metav1.GroupVersionForDiscovery{ {Version: "v2beta1"}, {Version: "v2beta2"}, {Version: "v2"}, }, PreferredVersion: metav1.GroupVersionForDiscovery{Version: "v2"}, }, }, grpName: "velero.io", want: metav1.APIGroup{ Name: "velero.io", Versions: []metav1.GroupVersionForDiscovery{ {Version: "v2beta1"}, {Version: "v2beta2"}, {Version: "v2"}, }, PreferredVersion: metav1.GroupVersionForDiscovery{Version: "v2"}, }, }, { name: "return empty API Group if no match in target list", targetGrps: []metav1.APIGroup{ { Name: "rbac.authorization.k8s.io", Versions: []metav1.GroupVersionForDiscovery{ {Version: "v2"}, }, PreferredVersion: metav1.GroupVersionForDiscovery{Version: "v2"}, }, { Name: "", Versions: []metav1.GroupVersionForDiscovery{ {Version: "v1"}, }, PreferredVersion: metav1.GroupVersionForDiscovery{Version: "v1"}, }, { Name: "velero.io", Versions: []metav1.GroupVersionForDiscovery{ {Version: "v2beta1"}, {Version: "v2beta2"}, {Version: "v2"}, }, PreferredVersion: metav1.GroupVersionForDiscovery{Version: "v2"}, }, }, grpName: "autoscaling", want: metav1.APIGroup{}, }, } for _, tc := range tests { grp := findAPIGroup(tc.targetGrps, tc.grpName) assert.Equal(t, tc.want, grp) } } func TestFindSupportedUserVersion(t *testing.T) { tests := []struct { name string userGVs []metav1.GroupVersionForDiscovery targetGVs []metav1.GroupVersionForDiscovery sourceGVs []metav1.GroupVersionForDiscovery want string }{ { name: "return the single user group version that has a match in both source and target clusters", userGVs: []metav1.GroupVersionForDiscovery{ {Version: "foo"}, {Version: "v10alpha2"}, {Version: "v3"}, }, targetGVs: []metav1.GroupVersionForDiscovery{ {Version: "v9"}, {Version: "v10beta1"}, {Version: "v10alpha2"}, {Version: "v10alpha3"}, }, sourceGVs: []metav1.GroupVersionForDiscovery{ {Version: "v10alpha2"}, {Version: "v9beta1"}, }, want: "v10alpha2", }, { name: "return the first user group version that has a match in both source and target clusters", userGVs: []metav1.GroupVersionForDiscovery{ {Version: "v2beta1"}, {Version: "v2beta2"}, }, targetGVs: []metav1.GroupVersionForDiscovery{ {Version: "v2beta2"}, {Version: "v2beta1"}, }, sourceGVs: []metav1.GroupVersionForDiscovery{ {Version: "v1"}, {Version: "v2beta2"}, {Version: "v2beta1"}, }, want: "v2beta1", }, { name: "return empty string if there's only matches in the source cluster, but not target", userGVs: []metav1.GroupVersionForDiscovery{ {Version: "v1"}, }, targetGVs: []metav1.GroupVersionForDiscovery{ {Version: "v2"}, }, sourceGVs: []metav1.GroupVersionForDiscovery{ {Version: "v1"}, }, want: "", }, { name: "return empty string if there's only matches in the target cluster, but not source", userGVs: []metav1.GroupVersionForDiscovery{ {Version: "v3"}, {Version: "v1"}, }, targetGVs: []metav1.GroupVersionForDiscovery{ {Version: "v3"}, {Version: "v3beta2"}, }, sourceGVs: []metav1.GroupVersionForDiscovery{ {Version: "v2"}, {Version: "v2beta1"}, }, want: "", }, { name: "return empty string if there is no match with either target and source clusters", userGVs: []metav1.GroupVersionForDiscovery{ {Version: "v2beta2"}, {Version: "v2beta1"}, {Version: "v2beta3"}, }, targetGVs: []metav1.GroupVersionForDiscovery{ {Version: "v2"}, {Version: "v1"}, {Version: "v2alpha1"}, }, sourceGVs: []metav1.GroupVersionForDiscovery{ {Version: "v1"}, {Version: "v2alpha1"}, }, want: "", }, } for _, tc := range tests { uv := findSupportedUserVersion(tc.userGVs, tc.targetGVs, tc.sourceGVs) assert.Equal(t, tc.want, uv) } } func TestVersionsContain(t *testing.T) { tests := []struct { name string GVs []metav1.GroupVersionForDiscovery ver string want bool }{ { name: "version is not in list", GVs: []metav1.GroupVersionForDiscovery{ {Version: "v1"}, {Version: "v2alpha1"}, {Version: "v2beta1"}, }, ver: "v2", want: false, }, { name: "version is in list", GVs: []metav1.GroupVersionForDiscovery{ {Version: "v2"}, {Version: "v2alpha1"}, {Version: "v2beta1"}, }, ver: "v2", want: true, }, } for _, tc := range tests { assert.Equal(t, tc.want, versionsContain(tc.GVs, tc.ver)) } } ================================================ FILE: pkg/restore/pv_restorer.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package restore import ( "context" "github.com/pkg/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/internal/credentials" "github.com/vmware-tanzu/velero/internal/volume" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/util/boolptr" ) type PVRestorer interface { executePVAction(obj *unstructured.Unstructured) (*unstructured.Unstructured, error) } type pvRestorer struct { logger logrus.FieldLogger backup *api.Backup restorePVs *bool volumeSnapshots []*volume.Snapshot volumeSnapshotterGetter VolumeSnapshotterGetter kbclient client.Client credentialFileStore credentials.FileStore volInfoTracker *volume.RestoreVolumeInfoTracker } func (r *pvRestorer) executePVAction(obj *unstructured.Unstructured) (*unstructured.Unstructured, error) { pvName := obj.GetName() if pvName == "" { return nil, errors.New("PersistentVolume is missing its name") } if boolptr.IsSetToFalse(r.restorePVs) { // The restore has pv restores disabled, so we can return early return obj, nil } log := r.logger.WithFields(logrus.Fields{"persistentVolume": pvName}) snapshotInfo, err := getSnapshotInfo(pvName, r.backup, r.volumeSnapshots, r.kbclient, r.credentialFileStore) if err != nil { return nil, err } if snapshotInfo == nil { log.Infof("No snapshot found for persistent volume") return obj, nil } volumeSnapshotter, err := r.volumeSnapshotterGetter.GetVolumeSnapshotter(snapshotInfo.location.Spec.Provider) if err != nil { return nil, errors.WithStack(err) } if err := volumeSnapshotter.Init(snapshotInfo.location.Spec.Config); err != nil { return nil, errors.WithStack(err) } volumeID, err := volumeSnapshotter.CreateVolumeFromSnapshot(snapshotInfo.providerSnapshotID, snapshotInfo.volumeType, snapshotInfo.volumeAZ, snapshotInfo.volumeIOPS) if err != nil { return nil, errors.WithStack(err) } log.WithField("providerSnapshotID", snapshotInfo.providerSnapshotID).Info("successfully restored persistent volume from snapshot") updated1, err := volumeSnapshotter.SetVolumeID(obj, volumeID) if err != nil { return nil, errors.WithStack(err) } updated2, ok := updated1.(*unstructured.Unstructured) if !ok { return nil, errors.Errorf("unexpected type %T", updated1) } var iops int64 if snapshotInfo.volumeIOPS != nil { iops = *snapshotInfo.volumeIOPS } r.volInfoTracker.TrackNativeSnapshot(updated2.GetName(), snapshotInfo.providerSnapshotID, snapshotInfo.volumeType, snapshotInfo.volumeAZ, iops) return updated2, nil } type snapshotInfo struct { providerSnapshotID string volumeType string volumeAZ string volumeIOPS *int64 location *api.VolumeSnapshotLocation } func getSnapshotInfo(pvName string, backup *api.Backup, volumeSnapshots []*volume.Snapshot, client client.Client, credentialStore credentials.FileStore) (*snapshotInfo, error) { var pvSnapshot *volume.Snapshot for _, snapshot := range volumeSnapshots { if snapshot.Spec.PersistentVolumeName == pvName { pvSnapshot = snapshot break } } if pvSnapshot == nil { return nil, nil } snapshotLocation := &api.VolumeSnapshotLocation{} err := client.Get( context.Background(), types.NamespacedName{Namespace: backup.Namespace, Name: pvSnapshot.Spec.Location}, snapshotLocation, ) if err != nil { return nil, errors.WithStack(err) } // add credential to config err = volume.UpdateVolumeSnapshotLocationWithCredentialConfig(snapshotLocation, credentialStore) if err != nil { return nil, errors.WithStack(err) } return &snapshotInfo{ providerSnapshotID: pvSnapshot.Status.ProviderSnapshotID, volumeType: pvSnapshot.Spec.VolumeType, volumeAZ: pvSnapshot.Spec.VolumeAZ, volumeIOPS: pvSnapshot.Spec.VolumeIOPS, location: snapshotLocation, }, nil } ================================================ FILE: pkg/restore/pv_restorer_test.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package restore import ( "testing" "github.com/sirupsen/logrus" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "github.com/vmware-tanzu/velero/internal/volume" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" providermocks "github.com/vmware-tanzu/velero/pkg/plugin/velero/mocks/volumesnapshotter/v1" vsv1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/volumesnapshotter/v1" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func defaultBackup() *builder.BackupBuilder { return builder.ForBackup(api.DefaultNamespace, "backup-1") } func TestExecutePVAction_NoSnapshotRestores(t *testing.T) { fakeClient := velerotest.NewFakeControllerRuntimeClient(t) tests := []struct { name string obj *unstructured.Unstructured restore *api.Restore backup *api.Backup volumeSnapshots []*volume.Snapshot locations []*api.VolumeSnapshotLocation expectedErr bool expectedRes *unstructured.Unstructured }{ { name: "no name should error", obj: newTestUnstructured().WithMetadata().Unstructured, restore: builder.ForRestore(api.DefaultNamespace, "").Result(), expectedErr: true, }, { name: "ensure spec.storageClassName is retained", obj: newTestUnstructured().WithName("pv-1").WithAnnotations("a", "b").WithSpec("storageClassName", "someOtherField").Unstructured, restore: builder.ForRestore(api.DefaultNamespace, "").RestorePVs(false).Result(), backup: defaultBackup().Phase(api.BackupPhaseInProgress).Result(), expectedRes: newTestUnstructured().WithAnnotations("a", "b").WithName("pv-1").WithSpec("storageClassName", "someOtherField").Unstructured, }, { name: "if backup.spec.snapshotVolumes is false, ignore restore.spec.restorePVs and return early", obj: newTestUnstructured().WithName("pv-1").WithAnnotations("a", "b").WithSpec("claimRef", "storageClassName", "someOtherField").Unstructured, restore: builder.ForRestore(api.DefaultNamespace, "").RestorePVs(true).Result(), backup: defaultBackup().Phase(api.BackupPhaseInProgress).SnapshotVolumes(false).Result(), expectedRes: newTestUnstructured().WithName("pv-1").WithAnnotations("a", "b").WithSpec("claimRef", "storageClassName", "someOtherField").Unstructured, }, { name: "restore.spec.restorePVs=false, return early", obj: newTestUnstructured().WithName("pv-1").WithSpec().Unstructured, restore: builder.ForRestore(api.DefaultNamespace, "").RestorePVs(false).Result(), backup: defaultBackup().Phase(api.BackupPhaseInProgress).Result(), volumeSnapshots: []*volume.Snapshot{ newSnapshot("pv-1", "loc-1", "gp", "az-1", "snap-1", 1000), }, locations: []*api.VolumeSnapshotLocation{ builder.ForVolumeSnapshotLocation(api.DefaultNamespace, "loc-1").Result(), }, expectedErr: false, expectedRes: newTestUnstructured().WithName("pv-1").WithSpec().Unstructured, }, { name: "volumeSnapshots is empty: return early", obj: newTestUnstructured().WithName("pv-1").WithSpec().Unstructured, restore: builder.ForRestore(api.DefaultNamespace, "").RestorePVs(true).Result(), backup: defaultBackup().Result(), locations: []*api.VolumeSnapshotLocation{ builder.ForVolumeSnapshotLocation(api.DefaultNamespace, "loc-1").Result(), builder.ForVolumeSnapshotLocation(api.DefaultNamespace, "loc-2").Result(), }, volumeSnapshots: []*volume.Snapshot{}, expectedRes: newTestUnstructured().WithName("pv-1").WithSpec().Unstructured, }, { name: "volumeSnapshots doesn't have a snapshot for PV: return early", obj: newTestUnstructured().WithName("pv-1").WithSpec().Unstructured, restore: builder.ForRestore(api.DefaultNamespace, "").RestorePVs(true).Result(), backup: defaultBackup().Result(), locations: []*api.VolumeSnapshotLocation{ builder.ForVolumeSnapshotLocation(api.DefaultNamespace, "loc-1").Result(), builder.ForVolumeSnapshotLocation(api.DefaultNamespace, "loc-2").Result(), }, volumeSnapshots: []*volume.Snapshot{ newSnapshot("non-matching-pv-1", "loc-1", "type-1", "az-1", "snap-1", 1), newSnapshot("non-matching-pv-2", "loc-2", "type-2", "az-2", "snap-2", 2), }, expectedRes: newTestUnstructured().WithName("pv-1").WithSpec().Unstructured, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { r := &pvRestorer{ logger: velerotest.NewLogger(), restorePVs: tc.restore.Spec.RestorePVs, kbclient: velerotest.NewFakeControllerRuntimeClient(t), volInfoTracker: volume.NewRestoreVolInfoTracker(tc.restore, logrus.New(), fakeClient), } if tc.backup != nil { r.backup = tc.backup } for _, loc := range tc.locations { require.NoError(t, r.kbclient.Create(t.Context(), loc)) } res, err := r.executePVAction(tc.obj) switch tc.expectedErr { case true: assert.Nil(t, res) require.Error(t, err) case false: assert.Equal(t, tc.expectedRes, res) assert.NoError(t, err) } }) } } func TestExecutePVAction_SnapshotRestores(t *testing.T) { tests := []struct { name string obj *unstructured.Unstructured restore *api.Restore backup *api.Backup volumeSnapshots []*volume.Snapshot locations []*api.VolumeSnapshotLocation expectedProvider string expectedSnapshotID string expectedVolumeType string expectedVolumeAZ string expectedVolumeIOPS *int64 expectedSnapshot *volume.Snapshot }{ { name: "backup with a matching volume.Snapshot for PV executes restore", obj: newTestUnstructured().WithName("pv-1").WithSpec().Unstructured, restore: builder.ForRestore(api.DefaultNamespace, "").RestorePVs(true).Result(), backup: defaultBackup().Result(), locations: []*api.VolumeSnapshotLocation{ builder.ForVolumeSnapshotLocation(api.DefaultNamespace, "loc-1").Provider("provider-1").Result(), builder.ForVolumeSnapshotLocation(api.DefaultNamespace, "loc-2").Provider("provider-2").Result(), }, volumeSnapshots: []*volume.Snapshot{ newSnapshot("pv-1", "loc-1", "type-1", "az-1", "snap-1", 1), newSnapshot("pv-2", "loc-2", "type-2", "az-2", "snap-2", 2), }, expectedProvider: "provider-1", expectedSnapshotID: "snap-1", expectedVolumeType: "type-1", expectedVolumeAZ: "az-1", expectedVolumeIOPS: int64Ptr(1), }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { var ( logger = velerotest.NewLogger() volumeSnapshotter = new(providermocks.VolumeSnapshotter) volumeSnapshotterGetter = providerToVolumeSnapshotterMap(map[string]vsv1.VolumeSnapshotter{ tc.expectedProvider: volumeSnapshotter, }) fakeClient = velerotest.NewFakeControllerRuntimeClientBuilder(t).Build() ) for _, loc := range tc.locations { require.NoError(t, fakeClient.Create(t.Context(), loc)) } r := &pvRestorer{ logger: logger, backup: tc.backup, volumeSnapshots: tc.volumeSnapshots, kbclient: fakeClient, volumeSnapshotterGetter: volumeSnapshotterGetter, volInfoTracker: volume.NewRestoreVolInfoTracker(tc.restore, logger, fakeClient), } volumeSnapshotter.On("Init", mock.Anything).Return(nil) volumeSnapshotter.On("CreateVolumeFromSnapshot", tc.expectedSnapshotID, tc.expectedVolumeType, tc.expectedVolumeAZ, tc.expectedVolumeIOPS).Return("volume-1", nil) volumeSnapshotter.On("SetVolumeID", tc.obj, "volume-1").Return(tc.obj, nil) _, err := r.executePVAction(tc.obj) require.NoError(t, err) volumeSnapshotter.AssertExpectations(t) }) } } type providerToVolumeSnapshotterMap map[string]vsv1.VolumeSnapshotter func (g providerToVolumeSnapshotterMap) GetVolumeSnapshotter(provider string) (vsv1.VolumeSnapshotter, error) { bs, ok := g[provider] if !ok { return nil, errors.New("volume snapshotter not found for provider") } return bs, nil } func newSnapshot(pvName, location, volumeType, volumeAZ, snapshotID string, volumeIOPS int64) *volume.Snapshot { return &volume.Snapshot{ Spec: volume.SnapshotSpec{ PersistentVolumeName: pvName, Location: location, VolumeType: volumeType, VolumeAZ: volumeAZ, VolumeIOPS: &volumeIOPS, }, Status: volume.SnapshotStatus{ ProviderSnapshotID: snapshotID, }, } } func int64Ptr(val int) *int64 { r := int64(val) return &r } type testUnstructured struct { *unstructured.Unstructured } func newTestUnstructured() *testUnstructured { obj := &testUnstructured{ Unstructured: &unstructured.Unstructured{ Object: make(map[string]any), }, } return obj } func (obj *testUnstructured) WithMetadata(fields ...string) *testUnstructured { return obj.withMap("metadata", fields...) } func (obj *testUnstructured) WithSpec(fields ...string) *testUnstructured { if _, found := obj.Object["spec"]; found { panic("spec already set - you probably didn't mean to do this twice!") } return obj.withMap("spec", fields...) } func (obj *testUnstructured) WithStatus(fields ...string) *testUnstructured { return obj.withMap("status", fields...) } func (obj *testUnstructured) WithMetadataField(field string, value any) *testUnstructured { return obj.withMapEntry("metadata", field, value) } func (obj *testUnstructured) WithSpecField(field string, value any) *testUnstructured { return obj.withMapEntry("spec", field, value) } func (obj *testUnstructured) WithStatusField(field string, value any) *testUnstructured { return obj.withMapEntry("status", field, value) } func (obj *testUnstructured) WithAnnotations(fields ...string) *testUnstructured { vals := map[string]string{} for _, field := range fields { vals[field] = "foo" } return obj.WithAnnotationValues(vals) } func (obj *testUnstructured) WithAnnotationValues(fieldVals map[string]string) *testUnstructured { annotations := make(map[string]any) for field, val := range fieldVals { annotations[field] = val } obj = obj.WithMetadataField("annotations", annotations) return obj } func (obj *testUnstructured) WithName(name string) *testUnstructured { return obj.WithMetadataField("name", name) } func (obj *testUnstructured) withMap(name string, fields ...string) *testUnstructured { m := make(map[string]any) obj.Object[name] = m for _, field := range fields { m[field] = "foo" } return obj } func (obj *testUnstructured) withMapEntry(mapName, field string, value any) *testUnstructured { var m map[string]any if res, ok := obj.Unstructured.Object[mapName]; !ok { m = make(map[string]any) obj.Unstructured.Object[mapName] = m } else { m = res.(map[string]any) } m[field] = value return obj } ================================================ FILE: pkg/restore/request.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package restore import ( "fmt" "io" "sort" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/runtime" "github.com/vmware-tanzu/velero/internal/resourcemodifiers" "github.com/vmware-tanzu/velero/internal/volume" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/itemoperation" "github.com/vmware-tanzu/velero/pkg/util/kube" ) const ( ItemRestoreResultCreated = "created" ItemRestoreResultUpdated = "updated" ItemRestoreResultFailed = "failed" ItemRestoreResultSkipped = "skipped" ) type itemKey struct { resource string namespace string name string } func resourceKey(obj runtime.Object) string { gvk := obj.GetObjectKind().GroupVersionKind() return fmt.Sprintf("%s/%s", gvk.GroupVersion().String(), gvk.Kind) } type Request struct { *velerov1api.Restore Log logrus.FieldLogger Backup *velerov1api.Backup PodVolumeBackups []*velerov1api.PodVolumeBackup VolumeSnapshots []*volume.Snapshot BackupReader io.Reader RestoredItems map[itemKey]restoredItemStatus itemOperationsList *[]*itemoperation.RestoreOperation ResourceModifiers *resourcemodifiers.ResourceModifiers DisableInformerCache bool CSIVolumeSnapshots []*snapshotv1api.VolumeSnapshot BackupVolumeInfoMap map[string]volume.BackupVolumeInfo RestoreVolumeInfoTracker *volume.RestoreVolumeInfoTracker ResourceDeletionStatusTracker kube.ResourceDeletionStatusTracker } type restoredItemStatus struct { action string itemExists bool createdName string // Actual name assigned by K8s for GenerateName resources } // GetItemOperationsList returns ItemOperationsList, initializing it if necessary func (r *Request) GetItemOperationsList() *[]*itemoperation.RestoreOperation { if r.itemOperationsList == nil { list := []*itemoperation.RestoreOperation{} r.itemOperationsList = &list } return r.itemOperationsList } // RestoredResourceList returns the list of restored resources grouped by the API // Version and Kind func (r *Request) RestoredResourceList() map[string][]string { resources := map[string][]string{} for i, item := range r.RestoredItems { // Use createdName if available (GenerateName case), otherwise itemKey.name name := i.name if item.createdName != "" { name = item.createdName } entry := name if i.namespace != "" { entry = fmt.Sprintf("%s/%s", i.namespace, name) } entry = fmt.Sprintf("%s(%s)", entry, item.action) resources[i.resource] = append(resources[i.resource], entry) } // sort namespace/name entries for each GVK for _, v := range resources { sort.Strings(v) } return resources } ================================================ FILE: pkg/restore/request_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package restore import ( "testing" "github.com/stretchr/testify/assert" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func TestResourceKey(t *testing.T) { namespace := &corev1api.Namespace{ TypeMeta: metav1.TypeMeta{ APIVersion: "v1", Kind: "Namespace", }, } assert.Equal(t, "v1/Namespace", resourceKey(namespace)) cr := &corev1api.Namespace{ TypeMeta: metav1.TypeMeta{ APIVersion: "customized/v1", Kind: "Cron", }, } assert.Equal(t, "customized/v1/Cron", resourceKey(cr)) } func TestRestoredResourceList(t *testing.T) { request := &Request{ RestoredItems: map[itemKey]restoredItemStatus{ { resource: "v1/Namespace", namespace: "", name: "default", }: {action: "created"}, { resource: "v1/ConfigMap", namespace: "default", name: "cm", }: {action: "skipped"}, }, } expected := map[string][]string{ "v1/ConfigMap": {"default/cm(skipped)"}, "v1/Namespace": {"default(created)"}, } assert.Equal(t, expected, request.RestoredResourceList()) } ================================================ FILE: pkg/restore/restore.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package restore import ( go_context "context" "encoding/json" "fmt" "io" "os" "os/signal" "path/filepath" "reflect" "slices" "sort" "strings" "sync" "time" "github.com/google/uuid" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" kubeerrs "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/dynamic/dynamicinformer" corev1 "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/retry" crclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/internal/credentials" "github.com/vmware-tanzu/velero/internal/hook" "github.com/vmware-tanzu/velero/internal/resourcemodifiers" "github.com/vmware-tanzu/velero/internal/volume" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/archive" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/discovery" "github.com/vmware-tanzu/velero/pkg/features" "github.com/vmware-tanzu/velero/pkg/itemoperation" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/label" "github.com/vmware-tanzu/velero/pkg/plugin/framework" "github.com/vmware-tanzu/velero/pkg/plugin/velero" riav2 "github.com/vmware-tanzu/velero/pkg/plugin/velero/restoreitemaction/v2" vsv1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/volumesnapshotter/v1" "github.com/vmware-tanzu/velero/pkg/podexec" "github.com/vmware-tanzu/velero/pkg/podvolume" "github.com/vmware-tanzu/velero/pkg/podvolume/configs" "github.com/vmware-tanzu/velero/pkg/types" "github.com/vmware-tanzu/velero/pkg/util/boolptr" "github.com/vmware-tanzu/velero/pkg/util/collections" csiutil "github.com/vmware-tanzu/velero/pkg/util/csi" "github.com/vmware-tanzu/velero/pkg/util/filesystem" "github.com/vmware-tanzu/velero/pkg/util/kube" "github.com/vmware-tanzu/velero/pkg/util/results" "github.com/vmware-tanzu/velero/pkg/util/wildcard" ) const ObjectStatusRestoreAnnotationKey = "velero.io/restore-status" var resourceMustHave = []string{ "datauploads.velero.io", "volumesnapshotcontents.snapshot.storage.k8s.io", } type VolumeSnapshotterGetter interface { GetVolumeSnapshotter(name string) (vsv1.VolumeSnapshotter, error) } // Restorer knows how to restore a backup. type Restorer interface { // Restore restores the backup data from backupReader, returning warnings and errors. Restore(req *Request, actions []riav2.RestoreItemAction, volumeSnapshotterGetter VolumeSnapshotterGetter, ) (results.Result, results.Result) RestoreWithResolvers( req *Request, restoreItemActionResolver framework.RestoreItemActionResolverV2, volumeSnapshotterGetter VolumeSnapshotterGetter, ) (results.Result, results.Result) } // kubernetesRestorer implements Restorer for restoring into a Kubernetes cluster. type kubernetesRestorer struct { discoveryHelper discovery.Helper dynamicFactory client.DynamicFactory namespaceClient corev1.NamespaceInterface podVolumeRestorerFactory podvolume.RestorerFactory podVolumeTimeout time.Duration podVolumeContext go_context.Context resourceTerminatingTimeout time.Duration resourceTimeout time.Duration resourcePriorities types.Priorities fileSystem filesystem.Interface pvRenamer func(string) (string, error) logger logrus.FieldLogger podCommandExecutor podexec.PodCommandExecutor podGetter cache.Getter credentialFileStore credentials.FileStore kbClient crclient.Client multiHookTracker *hook.MultiHookTracker resourceDeletionStatusTracker kube.ResourceDeletionStatusTracker } // NewKubernetesRestorer creates a new kubernetesRestorer. func NewKubernetesRestorer( discoveryHelper discovery.Helper, dynamicFactory client.DynamicFactory, resourcePriorities types.Priorities, namespaceClient corev1.NamespaceInterface, podVolumeRestorerFactory podvolume.RestorerFactory, podVolumeTimeout time.Duration, resourceTerminatingTimeout time.Duration, resourceTimeout time.Duration, logger logrus.FieldLogger, podCommandExecutor podexec.PodCommandExecutor, podGetter cache.Getter, credentialStore credentials.FileStore, kbClient crclient.Client, multiHookTracker *hook.MultiHookTracker, ) (Restorer, error) { return &kubernetesRestorer{ discoveryHelper: discoveryHelper, dynamicFactory: dynamicFactory, namespaceClient: namespaceClient, podVolumeRestorerFactory: podVolumeRestorerFactory, podVolumeTimeout: podVolumeTimeout, resourceTerminatingTimeout: resourceTerminatingTimeout, resourceTimeout: resourceTimeout, resourcePriorities: resourcePriorities, logger: logger, pvRenamer: func(string) (string, error) { veleroCloneUUID, err := uuid.NewRandom() if err != nil { return "", errors.WithStack(err) } veleroCloneName := "velero-clone-" + veleroCloneUUID.String() return veleroCloneName, nil }, fileSystem: filesystem.NewFileSystem(), podCommandExecutor: podCommandExecutor, podGetter: podGetter, credentialFileStore: credentialStore, kbClient: kbClient, multiHookTracker: multiHookTracker, }, nil } // Restore executes a restore into the target Kubernetes cluster according to the restore spec // and using data from the provided backup/backup reader. Returns a warnings and errors RestoreResult, // respectively, summarizing info about the restore. func (kr *kubernetesRestorer) Restore( req *Request, actions []riav2.RestoreItemAction, volumeSnapshotterGetter VolumeSnapshotterGetter, ) (results.Result, results.Result) { resolver := framework.NewRestoreItemActionResolverV2(actions) return kr.RestoreWithResolvers(req, resolver, volumeSnapshotterGetter) } func (kr *kubernetesRestorer) RestoreWithResolvers( req *Request, restoreItemActionResolver framework.RestoreItemActionResolverV2, volumeSnapshotterGetter VolumeSnapshotterGetter, ) (results.Result, results.Result) { // metav1.LabelSelectorAsSelector converts a nil LabelSelector to a // Nothing Selector, i.e. a selector that matches nothing. We want // a selector that matches everything. This can be accomplished by // passing a non-nil empty LabelSelector. ls := req.Restore.Spec.LabelSelector if ls == nil { ls = &metav1.LabelSelector{} } var OrSelectors []labels.Selector if req.Restore.Spec.OrLabelSelectors != nil { for _, s := range req.Restore.Spec.OrLabelSelectors { labelAsSelector, err := metav1.LabelSelectorAsSelector(s) if err != nil { return results.Result{}, results.Result{Velero: []string{err.Error()}} } OrSelectors = append(OrSelectors, labelAsSelector) } } selector, err := metav1.LabelSelectorAsSelector(ls) if err != nil { return results.Result{}, results.Result{Velero: []string{err.Error()}} } // Get resource includes-excludes. resourceIncludesExcludes := collections.GetResourceIncludesExcludes( kr.discoveryHelper, req.Restore.Spec.IncludedResources, req.Restore.Spec.ExcludedResources, ) // Get resource status includes-excludes. Defaults to excluding all resources var restoreStatusIncludesExcludes *collections.IncludesExcludes if req.Restore.Spec.RestoreStatus != nil { restoreStatusIncludesExcludes = collections.GetResourceIncludesExcludes( kr.discoveryHelper, req.Restore.Spec.RestoreStatus.IncludedResources, req.Restore.Spec.RestoreStatus.ExcludedResources, ) } // Get namespace includes-excludes. namespaceIncludesExcludes := collections.NewIncludesExcludes(). Includes(req.Restore.Spec.IncludedNamespaces...). Excludes(req.Restore.Spec.ExcludedNamespaces...) resolvedActions, err := restoreItemActionResolver.ResolveActions(kr.discoveryHelper, kr.logger) if err != nil { return results.Result{}, results.Result{Velero: []string{err.Error()}} } podVolumeTimeout := kr.podVolumeTimeout if val := req.Restore.Annotations[velerov1api.PodVolumeOperationTimeoutAnnotation]; val != "" { parsed, err := time.ParseDuration(val) if err != nil { req.Log.WithError(errors.WithStack(err)).Errorf( "Unable to parse pod volume timeout annotation %s, using server value.", val, ) } else { podVolumeTimeout = parsed } } var podVolumeCancelFunc go_context.CancelFunc kr.podVolumeContext, podVolumeCancelFunc = go_context.WithTimeout(go_context.Background(), podVolumeTimeout) defer podVolumeCancelFunc() var podVolumeRestorer podvolume.Restorer if kr.podVolumeRestorerFactory != nil { podVolumeRestorer, err = kr.podVolumeRestorerFactory.NewRestorer(kr.podVolumeContext, req.Restore) if err != nil { return results.Result{}, results.Result{Velero: []string{err.Error()}} } } waitExecHookHandler := &hook.DefaultWaitExecHookHandler{ PodCommandExecutor: kr.podCommandExecutor, ListWatchFactory: &hook.DefaultListWatchFactory{ PodsGetter: kr.podGetter, }, } hooksWaitExecutor, err := newHooksWaitExecutor(req.Restore, waitExecHookHandler) if err != nil { return results.Result{}, results.Result{Velero: []string{err.Error()}} } pvRestorer := &pvRestorer{ logger: req.Log, backup: req.Backup, restorePVs: req.Restore.Spec.RestorePVs, volumeSnapshots: req.VolumeSnapshots, volumeSnapshotterGetter: volumeSnapshotterGetter, kbclient: kr.kbClient, credentialFileStore: kr.credentialFileStore, volInfoTracker: req.RestoreVolumeInfoTracker, } req.RestoredItems = make(map[itemKey]restoredItemStatus) restoreCtx := &restoreContext{ backup: req.Backup, backupReader: req.BackupReader, restore: req.Restore, resourceIncludesExcludes: resourceIncludesExcludes, resourceStatusIncludesExcludes: restoreStatusIncludesExcludes, namespaceIncludesExcludes: namespaceIncludesExcludes, resourceMustHave: sets.New[string](resourceMustHave...), chosenGrpVersToRestore: make(map[string]ChosenGroupVersion), selector: selector, OrSelectors: OrSelectors, log: req.Log, dynamicFactory: kr.dynamicFactory, fileSystem: kr.fileSystem, namespaceClient: kr.namespaceClient, restoreItemActions: resolvedActions, volumeSnapshotterGetter: volumeSnapshotterGetter, podVolumeRestorer: podVolumeRestorer, podVolumeErrs: make(chan error), pvsToProvision: sets.New[string](), pvRestorer: pvRestorer, volumeSnapshots: req.VolumeSnapshots, csiVolumeSnapshots: req.CSIVolumeSnapshots, podVolumeBackups: req.PodVolumeBackups, resourceTerminatingTimeout: kr.resourceTerminatingTimeout, resourceTimeout: kr.resourceTimeout, resourceClients: make(map[resourceClientKey]client.Dynamic), restoredItems: req.RestoredItems, renamedPVs: make(map[string]string), pvRenamer: kr.pvRenamer, discoveryHelper: kr.discoveryHelper, resourcePriorities: kr.resourcePriorities, kbClient: kr.kbClient, itemOperationsList: req.GetItemOperationsList(), resourceModifiers: req.ResourceModifiers, disableInformerCache: req.DisableInformerCache, multiHookTracker: kr.multiHookTracker, backupVolumeInfoMap: req.BackupVolumeInfoMap, restoreVolumeInfoTracker: req.RestoreVolumeInfoTracker, hooksWaitExecutor: hooksWaitExecutor, resourceDeletionStatusTracker: req.ResourceDeletionStatusTracker, } return restoreCtx.execute() } type restoreContext struct { backup *velerov1api.Backup backupReader io.Reader restore *velerov1api.Restore restoreDir string resourceIncludesExcludes *collections.IncludesExcludes resourceStatusIncludesExcludes *collections.IncludesExcludes namespaceIncludesExcludes *collections.IncludesExcludes resourceMustHave sets.Set[string] chosenGrpVersToRestore map[string]ChosenGroupVersion selector labels.Selector OrSelectors []labels.Selector log logrus.FieldLogger dynamicFactory client.DynamicFactory fileSystem filesystem.Interface namespaceClient corev1.NamespaceInterface restoreItemActions []framework.RestoreItemResolvedActionV2 volumeSnapshotterGetter VolumeSnapshotterGetter podVolumeRestorer podvolume.Restorer podVolumeWaitGroup sync.WaitGroup podVolumeErrs chan error pvsToProvision sets.Set[string] pvRestorer PVRestorer volumeSnapshots []*volume.Snapshot csiVolumeSnapshots []*snapshotv1api.VolumeSnapshot podVolumeBackups []*velerov1api.PodVolumeBackup resourceTerminatingTimeout time.Duration resourceTimeout time.Duration resourceClients map[resourceClientKey]client.Dynamic dynamicInformerFactory *informerFactoryWithContext restoredItems map[itemKey]restoredItemStatus renamedPVs map[string]string pvRenamer func(string) (string, error) discoveryHelper discovery.Helper resourcePriorities types.Priorities kbClient crclient.Client itemOperationsList *[]*itemoperation.RestoreOperation resourceModifiers *resourcemodifiers.ResourceModifiers disableInformerCache bool multiHookTracker *hook.MultiHookTracker backupVolumeInfoMap map[string]volume.BackupVolumeInfo restoreVolumeInfoTracker *volume.RestoreVolumeInfoTracker hooksWaitExecutor *hooksWaitExecutor resourceDeletionStatusTracker kube.ResourceDeletionStatusTracker } type resourceClientKey struct { resource schema.GroupVersionResource namespace string } type informerFactoryWithContext struct { factory dynamicinformer.DynamicSharedInformerFactory context go_context.Context cancel go_context.CancelFunc } // getOrderedResources returns an ordered list of resource identifiers to restore, // based on the provided resource priorities and backup contents. The returned list // begins with all of the high prioritized resources (in order), ends with all of // the low prioritized resources(in order), and an alphabetized list of resources // in the backup(pick out the prioritized resources) is put in the middle. func getOrderedResources(resourcePriorities types.Priorities, backupResources map[string]*archive.ResourceItems) []string { priorities := map[string]struct{}{} for _, priority := range resourcePriorities.HighPriorities { priorities[priority] = struct{}{} } for _, priority := range resourcePriorities.LowPriorities { priorities[priority] = struct{}{} } // pick the prioritized resources out var orderedBackupResources []string for resource := range backupResources { if _, exist := priorities[resource]; exist { continue } orderedBackupResources = append(orderedBackupResources, resource) } // alphabetize resources in the backup sort.Strings(orderedBackupResources) list := append(resourcePriorities.HighPriorities, orderedBackupResources...) return append(list, resourcePriorities.LowPriorities...) } type progressUpdate struct { totalItems, itemsRestored int } func (ctx *restoreContext) execute() (results.Result, results.Result) { warnings, errs := results.Result{}, results.Result{} ctx.log.Infof("Starting restore of backup %s", kube.NamespaceAndName(ctx.backup)) dir, err := archive.NewExtractor(ctx.log, ctx.fileSystem).UnzipAndExtractBackup(ctx.backupReader) if err != nil { ctx.log.Infof("error unzipping and extracting: %v", err) errs.AddVeleroError(err) return warnings, errs } defer func() { if err := ctx.fileSystem.RemoveAll(dir); err != nil { ctx.log.Errorf("error removing temporary directory %s: %s", dir, err.Error()) } }() // Need to stop all informers if enabled if !ctx.disableInformerCache { context, cancel := signal.NotifyContext(go_context.Background(), os.Interrupt) ctx.dynamicInformerFactory = &informerFactoryWithContext{ factory: ctx.dynamicFactory.DynamicSharedInformerFactory(), context: context, cancel: cancel, } defer func() { // Call the cancel func to close the channel for each started informer ctx.dynamicInformerFactory.cancel() // After upgrading to client-go 0.27 or newer, also call Shutdown for each informer factory }() } // Need to set this for additionalItems to be restored. ctx.restoreDir = dir backupResources, err := archive.NewParser(ctx.log, ctx.fileSystem).Parse(ctx.restoreDir) // If ErrNotExist occurs, it implies that the backup to be restored includes zero items. // Need to add a warning about it and jump out of the function. if errors.Cause(err) == archive.ErrNotExist { warnings.AddVeleroError(errors.Wrap(err, "zero items to be restored")) return warnings, errs } if err != nil { errs.AddVeleroError(errors.Wrap(err, "error parsing backup contents")) return warnings, errs } // Expand wildcard patterns in namespace includes/excludes if needed if err := ctx.expandNamespaceWildcards(backupResources); err != nil { errs.AddVeleroError(err) return warnings, errs } // TODO: Remove outer feature flag check to make this feature a default in Velero. if features.IsEnabled(velerov1api.APIGroupVersionsFeatureFlag) { if ctx.backup.Status.FormatVersion >= "1.1.0" { if err := ctx.chooseAPIVersionsToRestore(); err != nil { errs.AddVeleroError(errors.Wrap(err, "choosing API version to restore")) return warnings, errs } } } update := make(chan progressUpdate) quit := make(chan struct{}) go func() { ticker := time.NewTicker(1 * time.Second) var lastUpdate *progressUpdate for { select { case <-quit: ticker.Stop() return case val := <-update: lastUpdate = &val case <-ticker.C: if lastUpdate != nil { updated := ctx.restore.DeepCopy() if updated.Status.Progress == nil { updated.Status.Progress = &velerov1api.RestoreProgress{} } updated.Status.Progress.TotalItems = lastUpdate.totalItems updated.Status.Progress.ItemsRestored = lastUpdate.itemsRestored err = kube.PatchResource(ctx.restore, updated, ctx.kbClient) if err != nil { ctx.log.WithError(errors.WithStack((err))). Warn("Got error trying to update restore's status.progress") } lastUpdate = nil } } } }() // totalItems: previously discovered items, i: iteration counter. totalItems, processedItems, existingNamespaces := 0, 0, sets.New[string]() // First restore CRDs. This is needed so that they are available in the cluster // when getOrderedResourceCollection is called again on the whole backup and // needs to validate all resources listed. crdResourceCollection, processedResources, w, e := ctx.getOrderedResourceCollection( backupResources, make([]restoreableResource, 0), sets.New[string](), types.Priorities{HighPriorities: []string{"customresourcedefinitions"}}, false, ) warnings.Merge(&w) errs.Merge(&e) for _, selectedResource := range crdResourceCollection { totalItems += selectedResource.totalItems } for _, selectedResource := range crdResourceCollection { var w, e results.Result // Restore this resource, the update channel is set to nil, to avoid misleading value of "totalItems" // more details see #5990 processedItems, w, e = ctx.processSelectedResource( selectedResource, totalItems, processedItems, existingNamespaces, nil, ) warnings.Merge(&w) errs.Merge(&e) } var createdOrUpdatedCRDs bool for _, restoredItem := range ctx.restoredItems { if restoredItem.action == ItemRestoreResultCreated || restoredItem.action == ItemRestoreResultUpdated { createdOrUpdatedCRDs = true break } } // If we just restored custom resource definitions (CRDs), refresh // discovery because the restored CRDs may have created or updated new APIs that // didn't previously exist in the cluster, and we want to be able to // resolve & restore instances of them in subsequent loop iterations. if createdOrUpdatedCRDs { if err := ctx.discoveryHelper.Refresh(); err != nil { warnings.Add("", errors.Wrap(err, "refresh discovery after restoring CRDs")) } } // Restore everything else selectedResourceCollection, _, w, e := ctx.getOrderedResourceCollection( backupResources, crdResourceCollection, processedResources, ctx.resourcePriorities, true, ) warnings.Merge(&w) errs.Merge(&e) // initialize informer caches for selected resources if enabled if !ctx.disableInformerCache { for _, informerResource := range selectedResourceCollection { if informerResource.totalItems == 0 { continue } version := "" for _, items := range informerResource.selectedItemsByNamespace { if len(items) == 0 { continue } version = items[0].version break } gvr := schema.ParseGroupResource(informerResource.resource).WithVersion(version) _, _, err := ctx.discoveryHelper.ResourceFor(gvr) if err != nil { ctx.log.Infof("failed to create informer for %s: %v", gvr, err) continue } ctx.dynamicInformerFactory.factory.ForResource(gvr) } ctx.dynamicInformerFactory.factory.Start(ctx.dynamicInformerFactory.context.Done()) ctx.log.Info("waiting informer cache sync ...") ctx.dynamicInformerFactory.factory.WaitForCacheSync(ctx.dynamicInformerFactory.context.Done()) } // reset processedItems and totalItems before processing full resource list processedItems = 0 totalItems = 0 for _, selectedResource := range selectedResourceCollection { totalItems += selectedResource.totalItems } for _, selectedResource := range selectedResourceCollection { var w, e results.Result // Restore this resource processedItems, w, e = ctx.processSelectedResource( selectedResource, totalItems, processedItems, existingNamespaces, update, ) warnings.Merge(&w) errs.Merge(&e) } // Close the progress update channel. quit <- struct{}{} // Clean the DataUploadResult ConfigMaps defer func() { opts := []crclient.DeleteAllOfOption{ crclient.InNamespace(ctx.restore.Namespace), crclient.MatchingLabels{ velerov1api.RestoreUIDLabel: string(ctx.restore.UID), velerov1api.ResourceUsageLabel: string(velerov1api.VeleroResourceUsageDataUploadResult), }, } err := ctx.kbClient.DeleteAllOf(go_context.Background(), &corev1api.ConfigMap{}, opts...) if err != nil { ctx.log.Errorf("Fail to batch delete DataUploadResult ConfigMaps for restore %s: %s", ctx.restore.Name, err.Error()) } }() // Do a final progress update as stopping the ticker might have left last few // updates from taking place. updated := ctx.restore.DeepCopy() if updated.Status.Progress == nil { updated.Status.Progress = &velerov1api.RestoreProgress{} } updated.Status.Progress.TotalItems = len(ctx.restoredItems) updated.Status.Progress.ItemsRestored = len(ctx.restoredItems) // patch the restore err = kube.PatchResource(ctx.restore, updated, ctx.kbClient) if err != nil { ctx.log.WithError(errors.WithStack((err))).Warn("Updating restore status") } // Wait for all of the pod volume restore goroutines to be done, which is // only possible once all of their errors have been received by the loop // below, then close the podVolumeErrs channel so the loop terminates. go func() { ctx.log.Info("Waiting for all pod volume restores to complete") // TODO timeout? ctx.podVolumeWaitGroup.Wait() close(ctx.podVolumeErrs) }() // This loop will only terminate when the ctx.podVolumeErrs channel is closed // in the above goroutine, *after* all errors from the goroutines have been // received by this loop. for err := range ctx.podVolumeErrs { // TODO: not ideal to be adding these to Velero-level errors // rather than a specific namespace, but don't have a way // to track the namespace right now. errs.Velero = append(errs.Velero, err.Error()) } ctx.log.Info("Done waiting for all pod volume restores to complete") return warnings, errs } // Process and restore one restoreableResource from the backup and update restore progress // metadata. At this point, the resource has already been validated and counted for inclusion // in the expected total restore count. func (ctx *restoreContext) processSelectedResource( selectedResource restoreableResource, totalItems int, processedItems int, existingNamespaces sets.Set[string], update chan progressUpdate, ) (int, results.Result, results.Result) { warnings, errs := results.Result{}, results.Result{} groupResource := schema.ParseGroupResource(selectedResource.resource) for namespace, selectedItems := range selectedResource.selectedItemsByNamespace { for _, selectedItem := range selectedItems { targetNS := selectedItem.targetNamespace if groupResource == kuberesource.Namespaces { // namespace is a cluster-scoped resource and doesn't have "targetNamespace" attribute in the restoreableItem instance namespace = selectedItem.name if n, ok := ctx.restore.Spec.NamespaceMapping[namespace]; ok { targetNS = n } else { targetNS = namespace } } // If we don't know whether this namespace exists yet, attempt to create // it in order to ensure it exists. Try to get it from the backup tarball // (in order to get any backed-up metadata), but if we don't find it there, // create a blank one. if namespace != "" && !existingNamespaces.Has(targetNS) { logger := ctx.log.WithField("namespace", namespace) ns := getNamespace( logger, archive.GetItemFilePath(ctx.restoreDir, "namespaces", "", namespace), targetNS, ) _, nsCreated, err := kube.EnsureNamespaceExistsAndIsReady( ns, ctx.namespaceClient, ctx.resourceTerminatingTimeout, ctx.resourceDeletionStatusTracker, ) if err != nil { errs.AddVeleroError(err) continue } // Add the newly created namespace to the list of restored items. if nsCreated { itemKey := itemKey{ resource: resourceKey(ns), namespace: ns.Namespace, name: ns.Name, } ctx.restoredItems[itemKey] = restoredItemStatus{action: ItemRestoreResultCreated, itemExists: true, createdName: ns.Name} } // Keep track of namespaces that we know exist so we don't // have to try to create them multiple times. existingNamespaces.Insert(targetNS) } // For namespaces resources we don't need to following steps if groupResource == kuberesource.Namespaces { continue } obj, err := archive.Unmarshal(ctx.fileSystem, selectedItem.path) if err != nil { errs.Add( selectedItem.targetNamespace, fmt.Errorf( "error decoding %q: %v", strings.Replace(selectedItem.path, ctx.restoreDir+"/", "", -1), err, ), ) continue } w, e, _ := ctx.restoreItem(obj, groupResource, targetNS) warnings.Merge(&w) errs.Merge(&e) processedItems++ // totalItems keeps the count of items previously known. There // may be additional items restored by plugins. We want to include // the additional items by looking at restoredItems at the same // time, we don't want previously known items counted twice as // they are present in both restoredItems and totalItems. actualTotalItems := len(ctx.restoredItems) + (totalItems - processedItems) if update != nil { update <- progressUpdate{ totalItems: actualTotalItems, itemsRestored: len(ctx.restoredItems), } } ctx.log.WithFields(map[string]any{ "progress": "", "resource": groupResource.String(), "namespace": selectedItem.targetNamespace, "name": selectedItem.name, }).Infof("Restored %d items out of an estimated total of %d (estimate will change throughout the restore)", len(ctx.restoredItems), actualTotalItems) } } return processedItems, warnings, errs } // getNamespace returns a namespace API object that we should attempt to // create before restoring anything into it. It will come from the backup // tarball if it exists, else will be a new one. If from the tarball, it // will retain its labels, annotations, and spec. func getNamespace(logger logrus.FieldLogger, path, remappedName string) *corev1api.Namespace { var nsBytes []byte var err error if nsBytes, err = os.ReadFile(path); err != nil { return &corev1api.Namespace{ TypeMeta: metav1.TypeMeta{ Kind: "Namespace", APIVersion: "v1", }, ObjectMeta: metav1.ObjectMeta{ Name: remappedName, }, } } var backupNS corev1api.Namespace if err := json.Unmarshal(nsBytes, &backupNS); err != nil { logger.Warnf("Error unmarshaling namespace from backup, creating new one.") return &corev1api.Namespace{ TypeMeta: metav1.TypeMeta{ Kind: "Namespace", APIVersion: "v1", }, ObjectMeta: metav1.ObjectMeta{ Name: remappedName, }, } } return &corev1api.Namespace{ TypeMeta: metav1.TypeMeta{ Kind: backupNS.Kind, APIVersion: backupNS.APIVersion, }, ObjectMeta: metav1.ObjectMeta{ Name: remappedName, Labels: backupNS.Labels, Annotations: backupNS.Annotations, }, Spec: backupNS.Spec, } } func (ctx *restoreContext) getApplicableActions(groupResource schema.GroupResource, namespace string) []framework.RestoreItemResolvedActionV2 { var actions []framework.RestoreItemResolvedActionV2 for _, action := range ctx.restoreItemActions { if action.ShouldUse(groupResource, namespace, nil, ctx.log) { actions = append(actions, action) } } return actions } func (ctx *restoreContext) shouldRestore(name string, pvClient client.Dynamic) (bool, error) { pvLogger := ctx.log.WithField("pvName", name) var shouldRestore bool err := wait.PollUntilContextTimeout(go_context.Background(), time.Second, ctx.resourceTerminatingTimeout, true, func(go_context.Context) (bool, error) { unstructuredPV, err := pvClient.Get(name, metav1.GetOptions{}) if apierrors.IsNotFound(err) { pvLogger.Debug("PV not found, safe to restore") // PV not found, can safely exit loop and proceed with restore. shouldRestore = true return true, nil } if err != nil { return false, errors.Wrapf(err, "could not retrieve in-cluster copy of PV %s", name) } clusterPV := new(corev1api.PersistentVolume) if err := runtime.DefaultUnstructuredConverter.FromUnstructured(unstructuredPV.Object, clusterPV); err != nil { return false, errors.Wrap(err, "error converting PV from unstructured") } if clusterPV.Status.Phase == corev1api.VolumeReleased || clusterPV.DeletionTimestamp != nil { // PV was found and marked for deletion, or it was released; wait for it to go away. pvLogger.Debugf("PV found, but marked for deletion, waiting") return false, nil } // Check for the namespace and PVC to see if anything that's referencing the PV is deleting. // If either the namespace or PVC is in a deleting/terminating state, wait for them to finish before // trying to restore the PV // Not doing so may result in the underlying PV disappearing but not restoring due to timing issues, // then the PVC getting restored and showing as lost. if clusterPV.Spec.ClaimRef == nil { pvLogger.Debugf("PV is not marked for deletion and is not claimed by a PVC") return true, nil } namespace := clusterPV.Spec.ClaimRef.Namespace pvcName := clusterPV.Spec.ClaimRef.Name // Have to create the PVC client here because we don't know what namespace we're using til we get to this point. // Using a dynamic client since it's easier to mock for testing pvcResource := metav1.APIResource{Name: "persistentvolumeclaims", Namespaced: true} pvcClient, err := ctx.dynamicFactory.ClientForGroupVersionResource(schema.GroupVersion{Group: "", Version: "v1"}, pvcResource, namespace) if err != nil { return false, errors.Wrapf(err, "error getting pvc client") } pvc, err := pvcClient.Get(pvcName, metav1.GetOptions{}) if apierrors.IsNotFound(err) { pvLogger.Debugf("PVC %s for PV not found, waiting", pvcName) // PVC wasn't found, but the PV still exists, so continue to wait. return false, nil } if err != nil { return false, errors.Wrapf(err, "error getting claim %s for persistent volume", pvcName) } if pvc != nil && pvc.GetDeletionTimestamp() != nil { pvLogger.Debugf("PVC for PV marked for deletion, waiting") // PVC is still deleting, continue to wait. return false, nil } // Check the namespace associated with the claimRef to see if it's // deleting/terminating before proceeding. ns, err := ctx.namespaceClient.Get(go_context.TODO(), namespace, metav1.GetOptions{}) if apierrors.IsNotFound(err) { pvLogger.Debugf("namespace %s for PV not found, waiting", namespace) // Namespace not found but the PV still exists, so continue to wait. return false, nil } if err != nil { return false, errors.Wrapf(err, "error getting namespace %s associated with PV %s", namespace, name) } if ns != nil && (ns.GetDeletionTimestamp() != nil || ns.Status.Phase == corev1api.NamespaceTerminating) { pvLogger.Debugf("namespace %s associated with PV is deleting, waiting", namespace) // Namespace is in the process of deleting, keep looping. return false, nil } // None of the PV, PVC, or NS are marked for deletion, break the loop. pvLogger.Debug("PV, associated PVC and namespace are not marked for deletion") return true, nil }) if wait.Interrupted(err) { pvLogger.Warn("timeout reached waiting for persistent volume to delete") } return shouldRestore, err } // crdAvailable waits for a CRD to be available for use before letting the // restore continue. func (ctx *restoreContext) crdAvailable(name string, crdClient client.Dynamic) (bool, error) { crdLogger := ctx.log.WithField("crdName", name) var available bool err := wait.PollUntilContextTimeout(go_context.Background(), time.Second, ctx.resourceTimeout, true, func(ctx go_context.Context) (bool, error) { unstructuredCRD, err := crdClient.Get(name, metav1.GetOptions{}) if err != nil { return true, err } available, err = kube.IsCRDReady(unstructuredCRD) if err != nil { return true, err } if !available { crdLogger.Debug("CRD not yet ready for use") } // If the CRD is not available, keep polling (false, nil). // If the CRD is available, break the poll and return to caller (true, nil). return available, nil }) if wait.Interrupted(err) { crdLogger.Debug("timeout reached waiting for custom resource definition to be ready") } return available, err } // itemsAvailable waits for the passed-in additional items to be available for use before letting the restore continue. func (ctx *restoreContext) itemsAvailable(action framework.RestoreItemResolvedActionV2, restoreItemOut *velero.RestoreItemActionExecuteOutput) (bool, error) { // if RestoreItemAction doesn't define set WaitForAdditionalItems, then return true if !restoreItemOut.WaitForAdditionalItems { return true, nil } var available bool timeout := ctx.resourceTimeout if restoreItemOut.AdditionalItemsReadyTimeout != 0 { timeout = restoreItemOut.AdditionalItemsReadyTimeout } err := wait.PollUntilContextTimeout(go_context.Background(), time.Second, timeout, true, func(go_context.Context) (bool, error) { var err error available, err = action.AreAdditionalItemsReady(restoreItemOut.AdditionalItems, ctx.restore) if err != nil { return true, err } if !available { ctx.log.Debug("AdditionalItems not yet ready for use") } // If the AdditionalItems are not available, keep polling (false, nil) // If the AdditionalItems are available, break the poll and return back to caller (true, nil) return available, nil }) if wait.Interrupted(err) { ctx.log.Debug("timeout reached waiting for AdditionalItems to be ready") } return available, err } func getResourceClientKey(groupResource schema.GroupResource, version, namespace string) resourceClientKey { return resourceClientKey{ resource: groupResource.WithVersion(version), namespace: namespace, } } func (ctx *restoreContext) getResourceClient(groupResource schema.GroupResource, obj *unstructured.Unstructured, namespace string) (client.Dynamic, error) { key := getResourceClientKey(groupResource, obj.GroupVersionKind().Version, namespace) if client, ok := ctx.resourceClients[key]; ok { return client, nil } // Initialize client for this resource. We need metadata from an object to // do this. ctx.log.Infof("Getting client for %v", obj.GroupVersionKind()) resource := metav1.APIResource{ Namespaced: len(namespace) > 0, Name: groupResource.Resource, } client, err := ctx.dynamicFactory.ClientForGroupVersionResource(obj.GroupVersionKind().GroupVersion(), resource, namespace) if err != nil { return nil, err } ctx.resourceClients[key] = client return client, nil } func (ctx *restoreContext) getResourceLister(groupResource schema.GroupResource, obj *unstructured.Unstructured, namespace string) (cache.GenericNamespaceLister, error) { _, _, err := ctx.discoveryHelper.KindFor(obj.GroupVersionKind()) if err != nil { return nil, err } informer := ctx.dynamicInformerFactory.factory.ForResource(groupResource.WithVersion(obj.GroupVersionKind().Version)) // if the restore contains CRDs or the RIA returns new resources, need to make sure the corresponding informers are synced if !informer.Informer().HasSynced() { ctx.dynamicInformerFactory.factory.Start(ctx.dynamicInformerFactory.context.Done()) ctx.log.Infof("waiting informer cache sync for %s, %s/%s ...", groupResource, namespace, obj.GetName()) ctx.dynamicInformerFactory.factory.WaitForCacheSync(ctx.dynamicInformerFactory.context.Done()) } if namespace == "" { return informer.Lister(), nil } return informer.Lister().ByNamespace(namespace), nil } func getResourceID(groupResource schema.GroupResource, namespace, name string) string { if namespace == "" { return fmt.Sprintf("%s/%s", groupResource.String(), name) } return fmt.Sprintf("%s/%s/%s", groupResource.String(), namespace, name) } func (ctx *restoreContext) getResource(groupResource schema.GroupResource, obj *unstructured.Unstructured, namespace string) (*unstructured.Unstructured, error) { lister, err := ctx.getResourceLister(groupResource, obj, namespace) if err != nil { return nil, errors.Wrapf(err, "Error getting lister for %s", getResourceID(groupResource, namespace, obj.GetName())) } clusterObj, err := lister.Get(obj.GetName()) if err != nil { return nil, errors.Wrapf(err, "error getting resource from lister for %s, %s/%s", groupResource, namespace, obj.GetName()) } u, ok := clusterObj.(*unstructured.Unstructured) if !ok { ctx.log.WithError(errors.WithStack(fmt.Errorf("expected *unstructured.Unstructured but got %T", u))).Error("unable to understand entry returned from client") return nil, fmt.Errorf("expected *unstructured.Unstructured but got %T", u) } ctx.log.Debugf("get %s, %s/%s from informer cache", groupResource, namespace, obj.GetName()) return u, nil } func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupResource schema.GroupResource, namespace string) (results.Result, results.Result, bool) { warnings, errs := results.Result{}, results.Result{} // itemExists bool is used to determine whether to include this item in the "wait for additional items" list itemExists := false resourceID := getResourceID(groupResource, namespace, obj.GetName()) resourceKind := obj.GetKind() backupResourceName := obj.GetName() restoreLogger := ctx.log.WithFields(logrus.Fields{ "namespace": obj.GetNamespace(), "original name": backupResourceName, "groupResource": groupResource.String(), }) // Check if group/resource should be restored. We need to do this here since // this method may be getting called for an additional item which is a group/resource // that's excluded. if !ctx.resourceIncludesExcludes.ShouldInclude(groupResource.String()) && !ctx.resourceMustHave.Has(groupResource.String()) { restoreLogger.Info("Not restoring item because resource is excluded") return warnings, errs, itemExists } // Check if namespace/cluster-scoped resource should be restored. We need // to do this here since this method may be getting called for an additional // item which is in a namespace that's excluded, or which is cluster-scoped // and should be excluded. Note that we're checking the object's namespace ( // via obj.GetNamespace()) instead of the namespace parameter, because we want // to check the *original* namespace, not the remapped one if it's been remapped. if namespace != "" { if !ctx.namespaceIncludesExcludes.ShouldInclude(obj.GetNamespace()) && !ctx.resourceMustHave.Has(groupResource.String()) { restoreLogger.Info("Not restoring item because namespace is excluded") return warnings, errs, itemExists } // If the namespace scoped resource should be restored, ensure that the // namespace into which the resource is being restored into exists. // This is the *remapped* namespace that we are ensuring exists. nsToEnsure := getNamespace(restoreLogger, archive.GetItemFilePath(ctx.restoreDir, "namespaces", "", obj.GetNamespace()), namespace) _, nsCreated, err := kube.EnsureNamespaceExistsAndIsReady(nsToEnsure, ctx.namespaceClient, ctx.resourceTerminatingTimeout, ctx.resourceDeletionStatusTracker) if err != nil { errs.AddVeleroError(err) return warnings, errs, itemExists } // Add the newly created namespace to the list of restored items. if nsCreated { itemKey := itemKey{ resource: resourceKey(nsToEnsure), namespace: nsToEnsure.Namespace, name: nsToEnsure.Name, } ctx.restoredItems[itemKey] = restoredItemStatus{action: ItemRestoreResultCreated, itemExists: true, createdName: nsToEnsure.Name} } } else { if boolptr.IsSetToFalse(ctx.restore.Spec.IncludeClusterResources) { restoreLogger.Info("Not restoring item because it's cluster-scoped") return warnings, errs, itemExists } } // Make a copy of object retrieved from backup to make it available unchanged //inside restore actions. itemFromBackup := obj.DeepCopy() complete, err := isCompleted(obj, groupResource) if err != nil { errs.Add(namespace, fmt.Errorf("error checking completion of %q: %v", resourceID, err)) return warnings, errs, itemExists } if complete { restoreLogger.Infof("%s is complete - skipping", kube.NamespaceAndName(obj)) return warnings, errs, itemExists } // Check if we've already restored this itemKey. itemKey := itemKey{ resource: resourceKey(obj), namespace: namespace, name: backupResourceName, } if prevRestoredItemStatus, exists := ctx.restoredItems[itemKey]; exists { restoreLogger.Infof("Skipping %s because it's already been restored.", resourceID) itemExists = prevRestoredItemStatus.itemExists return warnings, errs, itemExists } ctx.restoredItems[itemKey] = restoredItemStatus{itemExists: itemExists} defer func() { itemStatus := ctx.restoredItems[itemKey] // the action field is set explicitly if len(itemStatus.action) > 0 { return } // no action specified, and no warnings and errors if errs.IsEmpty() && warnings.IsEmpty() { itemStatus.action = ItemRestoreResultSkipped ctx.restoredItems[itemKey] = itemStatus return } // others are all failed itemStatus.action = ItemRestoreResultFailed ctx.restoredItems[itemKey] = itemStatus }() // TODO: move to restore item action if/when we add a ShouldRestore() method // to the interface. if groupResource == kuberesource.Pods && obj.GetAnnotations()[corev1api.MirrorPodAnnotationKey] != "" { restoreLogger.Infof("Not restoring pod because it's a mirror pod") return warnings, errs, itemExists } if groupResource == kuberesource.PersistentVolumes { resourceClient, err := ctx.getResourceClient(groupResource, obj, namespace) if err != nil { errs.AddVeleroError(fmt.Errorf("error getting resource client for namespace %q, resource %q: %v", namespace, &groupResource, err)) return warnings, errs, itemExists } if volumeInfo, ok := ctx.backupVolumeInfoMap[obj.GetName()]; ok { restoreLogger.Infof("Find BackupVolumeInfo for PV %s.", obj.GetName()) switch volumeInfo.BackupMethod { case volume.NativeSnapshot: obj, err = ctx.handlePVHasNativeSnapshot(obj, resourceClient) if err != nil { errs.Add(namespace, err) return warnings, errs, itemExists } case volume.PodVolumeBackup: restoreLogger.Infof("Dynamically re-provisioning persistent volume because it has a pod volume backup to be restored.") ctx.pvsToProvision.Insert(backupResourceName) // Return early because we don't want to restore the PV itself, we // want to dynamically re-provision it. return warnings, errs, itemExists case volume.CSISnapshot: restoreLogger.Infof("Dynamically re-provisioning persistent volume because it has a CSI VolumeSnapshot or a related snapshot DataUpload.") ctx.pvsToProvision.Insert(backupResourceName) // Return early because we don't want to restore the PV itself, we // want to dynamically re-provision it. return warnings, errs, itemExists // When the PV data is skipped from backup, it's BackupVolumeInfo BackupMethod // is not set, and it will fall into the default case. default: if hasDeleteReclaimPolicy(obj.Object) { restoreLogger.Infof("Dynamically re-provisioning persistent volume because it doesn't have a snapshot and its reclaim policy is Delete.") ctx.pvsToProvision.Insert(backupResourceName) // Return early because we don't want to restore the PV itself, we // want to dynamically re-provision it. return warnings, errs, itemExists } else { obj, err = ctx.handleSkippedPVHasRetainPolicy(obj, restoreLogger) if err != nil { errs.Add(namespace, err) return warnings, errs, itemExists } } } } else { // TODO: BackupVolumeInfo is adopted and old logic is deprecated in v1.13. // Remove the old logic in v1.15. restoreLogger.Infof("Cannot find BackupVolumeInfo for PV %s.", obj.GetName()) switch { case hasSnapshot(backupResourceName, ctx.volumeSnapshots): obj, err = ctx.handlePVHasNativeSnapshot(obj, resourceClient) if err != nil { errs.Add(namespace, err) return warnings, errs, itemExists } case hasPodVolumeBackup(obj, ctx): restoreLogger.Infof("Dynamically re-provisioning persistent volume because it has a pod volume backup to be restored.") ctx.pvsToProvision.Insert(backupResourceName) // Return early because we don't want to restore the PV itself, we // want to dynamically re-provision it. return warnings, errs, itemExists case hasCSIVolumeSnapshot(ctx, obj): fallthrough case hasSnapshotDataUpload(ctx, obj): restoreLogger.Infof("Dynamically re-provisioning persistent volume because it has a CSI VolumeSnapshot or a related snapshot DataUpload.") ctx.pvsToProvision.Insert(backupResourceName) // Return early because we don't want to restore the PV itself, we // want to dynamically re-provision it. return warnings, errs, itemExists case hasDeleteReclaimPolicy(obj.Object): restoreLogger.Infof("Dynamically re-provisioning persistent volume because it doesn't have a snapshot and its reclaim policy is Delete.") ctx.pvsToProvision.Insert(backupResourceName) // Return early because we don't want to restore the PV itself, we // want to dynamically re-provision it. return warnings, errs, itemExists default: obj, err = ctx.handleSkippedPVHasRetainPolicy(obj, restoreLogger) if err != nil { errs.Add(namespace, err) return warnings, errs, itemExists } } } } objStatus, statusFieldExists, statusFieldErr := unstructured.NestedFieldCopy(obj.Object, "status") // Clear out non-core metadata fields and status. if obj, err = resetMetadataAndStatus(obj); err != nil { errs.Add(namespace, err) return warnings, errs, itemExists } restoreLogger.Infof("restore status includes excludes: %+v", ctx.resourceStatusIncludesExcludes) for _, action := range ctx.getApplicableActions(groupResource, namespace) { if !action.Selector.Matches(labels.Set(obj.GetLabels())) { continue } // If the EnableCSI feature is not enabled, but the executing action is from CSI plugin, skip the action. if csiutil.ShouldSkipAction(action.Name()) { restoreLogger.Infof("Skip action %s for resource %s:%s/%s, because the CSI feature is not enabled. Feature setting is %s.", action.Name(), groupResource.String(), obj.GetNamespace(), obj.GetName(), features.Serialize()) continue } restoreLogger.Infof("Executing item action for %v", &groupResource) executeOutput, err := action.RestoreItemAction.Execute(&velero.RestoreItemActionExecuteInput{ Item: obj, ItemFromBackup: itemFromBackup, Restore: ctx.restore, }) if err != nil { errs.Add(namespace, fmt.Errorf("error preparing %s: %v", resourceID, err)) return warnings, errs, itemExists } // If async plugin started async operation, add it to the ItemOperations list if executeOutput.OperationID != "" { resourceIdentifier := velero.ResourceIdentifier{ GroupResource: groupResource, Namespace: namespace, Name: backupResourceName, } now := metav1.Now() newOperation := itemoperation.RestoreOperation{ Spec: itemoperation.RestoreOperationSpec{ RestoreName: ctx.restore.Name, RestoreUID: string(ctx.restore.UID), RestoreItemAction: action.RestoreItemAction.Name(), ResourceIdentifier: resourceIdentifier, OperationID: executeOutput.OperationID, }, Status: itemoperation.OperationStatus{ Phase: itemoperation.OperationPhaseNew, Created: &now, }, } itemOperList := ctx.itemOperationsList *itemOperList = append(*itemOperList, &newOperation) } if executeOutput.SkipRestore { restoreLogger.Infof("Skipping restore because a registered plugin discarded it") return warnings, errs, itemExists } unstructuredObj, ok := executeOutput.UpdatedItem.(*unstructured.Unstructured) if !ok { errs.Add(namespace, fmt.Errorf("%s: unexpected type %T", resourceID, executeOutput.UpdatedItem)) return warnings, errs, itemExists } obj = unstructuredObj var filteredAdditionalItems []velero.ResourceIdentifier for _, additionalItem := range executeOutput.AdditionalItems { itemPath := archive.GetItemFilePath(ctx.restoreDir, additionalItem.GroupResource.String(), additionalItem.Namespace, additionalItem.Name) if _, err := ctx.fileSystem.Stat(itemPath); err != nil { restoreLogger.WithError(err).WithFields(logrus.Fields{ "additionalResource": additionalItem.GroupResource.String(), "additionalResourceNamespace": additionalItem.Namespace, "additionalResourceName": additionalItem.Name, }).Warn("unable to restore additional item") warnings.Add(additionalItem.Namespace, err) continue } additionalResourceID := getResourceID(additionalItem.GroupResource, additionalItem.Namespace, additionalItem.Name) additionalObj, err := archive.Unmarshal(ctx.fileSystem, itemPath) if err != nil { errs.Add(namespace, errors.Wrapf(err, "error restoring additional item %s", additionalResourceID)) } additionalItemNamespace := additionalItem.Namespace if additionalItemNamespace != "" { if remapped, ok := ctx.restore.Spec.NamespaceMapping[additionalItemNamespace]; ok { additionalItemNamespace = remapped } } w, e, additionalItemExists := ctx.restoreItem(additionalObj, additionalItem.GroupResource, additionalItemNamespace) if additionalItemExists { filteredAdditionalItems = append(filteredAdditionalItems, additionalItem) } warnings.Merge(&w) errs.Merge(&e) } executeOutput.AdditionalItems = filteredAdditionalItems available, err := ctx.itemsAvailable(action, executeOutput) if err != nil { errs.Add(namespace, errors.Wrapf(err, "error verifying additional items are ready to use")) } else if !available { errs.Add(namespace, fmt.Errorf("additional items for %s are not ready to use", resourceID)) } } // This comes after running item actions because we have built-in actions that restore // a PVC's associated PV (if applicable). As part of the PV being restored, the 'pvsToProvision' // set may be inserted into, and this needs to happen *before* running the following block of logic. // // The side effect of this is that it's impossible for a user to write a restore item action that // adjusts this behavior (i.e. of resetting the PVC for dynamic provisioning if it claims a PV with // a reclaim policy of Delete and no snapshot). If/when that becomes an issue for users, we can // revisit. This would be easier with a multi-pass restore process. if groupResource == kuberesource.PersistentVolumeClaims { pvc := new(corev1api.PersistentVolumeClaim) if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), pvc); err != nil { errs.Add(namespace, err) return warnings, errs, itemExists } if pvc.Spec.VolumeName != "" { // This used to only happen with PVB volumes, but now always remove this binding metadata obj = resetVolumeBindingInfo(obj) // This is the case for PVB volumes, where we need to actually have an empty volume created instead of restoring one. // The assumption is that any PV in pvsToProvision doesn't have an associated snapshot. if ctx.pvsToProvision.Has(pvc.Spec.VolumeName) { restoreLogger.Infof("Resetting PersistentVolumeClaim for dynamic provisioning") unstructured.RemoveNestedField(obj.Object, "spec", "volumeName") } } if newName, ok := ctx.renamedPVs[pvc.Spec.VolumeName]; ok { restoreLogger.Infof("Updating persistent volume claim %s/%s to reference renamed persistent volume (%s -> %s)", namespace, obj.GetName(), pvc.Spec.VolumeName, newName) if err := unstructured.SetNestedField(obj.Object, newName, "spec", "volumeName"); err != nil { errs.Add(namespace, err) return warnings, errs, itemExists } } } if ctx.resourceModifiers != nil { if errList := ctx.resourceModifiers.ApplyResourceModifierRules(obj, groupResource.String(), ctx.kbClient.Scheme(), restoreLogger); errList != nil { for _, err := range errList { errs.Add(namespace, err) } } } // Necessary because we may have remapped the namespace if the namespace is // blank, don't create the key. originalNamespace := obj.GetNamespace() if namespace != "" { obj.SetNamespace(namespace) } // Label the resource with the restore's name and the restored backup's name // for easy identification of all cluster resources created by this restore // and which backup they came from. addRestoreLabels(obj, ctx.restore.Name, ctx.restore.Spec.BackupName) // The object apiVersion might get modified by a RestorePlugin so we need to // get a new client to reflect updated resource path. newGR := schema.GroupResource{Group: obj.GroupVersionKind().Group, Resource: groupResource.Resource} // obj kind might change within a special RIA which is used to convert objects, // like from Openshift DeploymentConfig to native Deployment. // we should re-get the newGR.Resource again in such a case if obj.GetKind() != resourceKind { restoreLogger.Infof("Resource kind changed from %s to %s", resourceKind, obj.GetKind()) gvr, _, err := ctx.discoveryHelper.KindFor(obj.GroupVersionKind()) if err != nil { errs.Add(namespace, fmt.Errorf("error getting GVR for %s: %v", obj.GroupVersionKind(), err)) return warnings, errs, itemExists } newGR.Resource = gvr.Resource } if !reflect.DeepEqual(newGR, groupResource) { restoreLogger.Infof("Resource to be restored changed from %v to %v", groupResource, newGR) } resourceClient, err := ctx.getResourceClient(newGR, obj, obj.GetNamespace()) if err != nil { warnings.Add(namespace, fmt.Errorf("error getting updated resource client for namespace %q, resource %q: %v", namespace, &newGR, err)) return warnings, errs, itemExists } restoreLogger.Infof("Attempting to restore %s: %s.", obj.GroupVersionKind().Kind, obj.GetName()) // check if we want to treat the error as a warning, in some cases the creation call might not get executed due to object API validations // and Velero might not get the already exists error type but in reality the object already exists var fromCluster, createdObj *unstructured.Unstructured var restoreErr error // only attempt Get before Create if using informer cache, otherwise this will slow down restore into // new namespace if !ctx.disableInformerCache { restoreLogger.Debugf("Checking for existence %s", obj.GetName()) fromCluster, err = ctx.getResource(newGR, obj, namespace) } if err != nil || fromCluster == nil { // couldn't find the resource, attempt to create restoreLogger.Debugf("Creating %s", obj.GetName()) createdObj, restoreErr = resourceClient.Create(obj) if restoreErr == nil { itemExists = true ctx.restoredItems[itemKey] = restoredItemStatus{ action: ItemRestoreResultCreated, itemExists: itemExists, createdName: createdObj.GetName(), } } } isAlreadyExistsError, err := isAlreadyExistsError(ctx, obj, restoreErr, resourceClient) if err != nil { errs.Add(namespace, err) return warnings, errs, itemExists } if restoreErr != nil { // check for the existence of the object that failed creation due to alreadyExist in cluster, if no error then it implies that object exists. // and if err then itemExists remains false as we were not able to confirm the existence of the object via Get call or creation call. // We return the get error as a warning to notify the user that the object could exist in cluster and we were not able to confirm it. if !ctx.disableInformerCache { fromCluster, err = ctx.getResource(newGR, obj, namespace) } else { fromCluster, err = resourceClient.Get(obj.GetName(), metav1.GetOptions{}) } if err != nil && isAlreadyExistsError { restoreLogger.Warnf("Unable to retrieve in-cluster version of %s: %s, object won't be restored by velero or have restore labels, and existing resource policy is not applied", kube.NamespaceAndName(obj), err.Error()) warnings.Add(namespace, err) return warnings, errs, itemExists } } if fromCluster != nil { itemExists = true itemStatus := ctx.restoredItems[itemKey] itemStatus.itemExists = itemExists ctx.restoredItems[itemKey] = itemStatus // Remove insubstantial metadata. fromCluster, err = resetMetadataAndStatus(fromCluster) if err != nil { restoreLogger.Infof("Error trying to reset metadata for %s: %s", kube.NamespaceAndName(obj), err.Error()) warnings.Add(namespace, err) return warnings, errs, itemExists } // We know the object from the cluster won't have the backup/restore name // labels, so copy them from the object we attempted to restore. labels := obj.GetLabels() addRestoreLabels(fromCluster, labels[velerov1api.RestoreNameLabel], labels[velerov1api.BackupNameLabel]) fromClusterWithLabels := fromCluster.DeepCopy() // saving the in-cluster object so that we can create label patch if overall patch fails if !equality.Semantic.DeepEqual(fromCluster, obj) { switch newGR { case kuberesource.ServiceAccounts: desired, err := mergeServiceAccounts(fromCluster, obj) if err != nil { restoreLogger.Infof("error merging secrets for ServiceAccount %s: %s", kube.NamespaceAndName(obj), err.Error()) warnings.Add(namespace, err) return warnings, errs, itemExists } patchBytes, err := generatePatch(fromCluster, desired) if err != nil { restoreLogger.Infof("error generating patch for ServiceAccount %s: %s", kube.NamespaceAndName(obj), err.Error()) warnings.Add(namespace, err) return warnings, errs, itemExists } if patchBytes == nil { // In-cluster and desired state are the same, so move on to // the next item. return warnings, errs, itemExists } _, err = resourceClient.Patch(obj.GetName(), patchBytes) if err != nil { warnings.Add(namespace, err) // check if there is existingResourcePolicy and if it is set to update policy if len(ctx.restore.Spec.ExistingResourcePolicy) > 0 && ctx.restore.Spec.ExistingResourcePolicy == velerov1api.PolicyTypeUpdate { // remove restore labels so that we apply the latest backup/restore names on the object via patch removeRestoreLabels(fromCluster) //try patching just the backup/restore labels warningsFromUpdate, errsFromUpdate := ctx.updateBackupRestoreLabels(fromCluster, fromClusterWithLabels, namespace, resourceClient) warnings.Merge(&warningsFromUpdate) errs.Merge(&errsFromUpdate) } } else { itemStatus.action = ItemRestoreResultUpdated ctx.restoredItems[itemKey] = itemStatus restoreLogger.Infof("ServiceAccount %s successfully updated", kube.NamespaceAndName(obj)) } default: // check for the presence of existingResourcePolicy if len(ctx.restore.Spec.ExistingResourcePolicy) > 0 { resourcePolicy := ctx.restore.Spec.ExistingResourcePolicy restoreLogger.Infof("restore API has resource policy defined %s, executing restore workflow accordingly for changed resource %s %s", resourcePolicy, fromCluster.GroupVersionKind().Kind, kube.NamespaceAndName(fromCluster)) // existingResourcePolicy is set as none, add warning if resourcePolicy == velerov1api.PolicyTypeNone { e := errors.Errorf("could not restore, %s %q already exists. Warning: the in-cluster version is different than the backed-up version", obj.GetKind(), obj.GetName()) warnings.Add(namespace, e) // existingResourcePolicy is set as update, attempt patch on the resource and add warning if it fails } else if resourcePolicy == velerov1api.PolicyTypeUpdate { // processing update as existingResourcePolicy warningsFromUpdateRP, errsFromUpdateRP := ctx.processUpdateResourcePolicy(fromCluster, fromClusterWithLabels, obj, namespace, resourceClient) if warningsFromUpdateRP.IsEmpty() && errsFromUpdateRP.IsEmpty() { itemStatus.action = ItemRestoreResultUpdated ctx.restoredItems[itemKey] = itemStatus } warnings.Merge(&warningsFromUpdateRP) errs.Merge(&errsFromUpdateRP) } } else { // Preserved Velero behavior when existingResourcePolicy is not specified by the user e := errors.Errorf("could not restore, %s:%s already exists. Warning: the in-cluster version is different than the backed-up version", obj.GetKind(), obj.GetName()) warnings.Add(namespace, e) } } return warnings, errs, itemExists } //update backup/restore labels on the unchanged resources if existingResourcePolicy is set as update if ctx.restore.Spec.ExistingResourcePolicy == velerov1api.PolicyTypeUpdate { resourcePolicy := ctx.restore.Spec.ExistingResourcePolicy restoreLogger.Infof("restore API has resource policy defined %s, executing restore workflow accordingly for unchanged resource %s %s ", resourcePolicy, obj.GroupVersionKind().Kind, kube.NamespaceAndName(fromCluster)) // remove restore labels so that we apply the latest backup/restore names on the object via patch removeRestoreLabels(fromCluster) // try updating the backup/restore labels for the in-cluster object warningsFromUpdate, errsFromUpdate := ctx.updateBackupRestoreLabels(fromCluster, obj, namespace, resourceClient) warnings.Merge(&warningsFromUpdate) errs.Merge(&errsFromUpdate) } restoreLogger.Infof("Restore of %s skipped: it already exists in the cluster and is the same as the backed up version", obj.GetName()) return warnings, errs, itemExists } // Error was something other than an AlreadyExists. if restoreErr != nil { restoreLogger.Errorf("error restoring %s: %s", obj.GetName(), restoreErr.Error()) errs.Add(namespace, fmt.Errorf("error restoring %s: %v", resourceID, restoreErr)) return warnings, errs, itemExists } // determine whether to restore status according to original GR shouldRestoreStatus := determineRestoreStatus(obj, ctx.resourceStatusIncludesExcludes, groupResource.String(), restoreLogger) if shouldRestoreStatus && statusFieldErr != nil { err := fmt.Errorf("could not get status to be restored %s: %v", kube.NamespaceAndName(obj), statusFieldErr) restoreLogger.Error(err.Error()) errs.Add(namespace, err) return warnings, errs, itemExists } // Proceed with status restoration if decided if statusFieldExists && shouldRestoreStatus { if err := unstructured.SetNestedField(obj.Object, objStatus, "status"); err != nil { restoreLogger.Errorf("could not set status field %s: %s", kube.NamespaceAndName(obj), err.Error()) errs.Add(namespace, err) return warnings, errs, itemExists } resourceVersion := createdObj.GetResourceVersion() if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { obj.SetResourceVersion(resourceVersion) updated, err := resourceClient.UpdateStatus(obj, metav1.UpdateOptions{}) if err != nil { if apierrors.IsConflict(err) { res, err := resourceClient.Get(obj.GetName(), metav1.GetOptions{}) if err == nil { resourceVersion = res.GetResourceVersion() } } return err } createdObj = updated return nil }); err != nil { restoreLogger.Infof("status field update failed %s: %s", kube.NamespaceAndName(obj), err.Error()) warnings.Add(namespace, err) } } // restore the managedFields withoutManagedFields := createdObj.DeepCopy() createdObj.SetManagedFields(obj.GetManagedFields()) patchBytes, err := generatePatch(withoutManagedFields, createdObj) if err != nil { restoreLogger.Errorf("error generating patch for managed fields %s: %s", kube.NamespaceAndName(createdObj), err.Error()) errs.Add(namespace, err) return warnings, errs, itemExists } if patchBytes != nil { if _, err = resourceClient.Patch(createdObj.GetName(), patchBytes); err != nil { if !apierrors.IsNotFound(err) { restoreLogger.Errorf("error patch for managed fields %s: %s", kube.NamespaceAndName(createdObj), err.Error()) errs.Add(namespace, err) return warnings, errs, itemExists } restoreLogger.Warnf("item not found when patching managed fields %s: %s", kube.NamespaceAndName(createdObj), err.Error()) warnings.Add(namespace, err) } else { restoreLogger.Infof("the managed fields for %s is patched", kube.NamespaceAndName(createdObj)) } } if newGR == kuberesource.Pods { pod := new(corev1api.Pod) if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), pod); err != nil { errs.Add(namespace, err) return warnings, errs, itemExists } // Do not create podvolumerestore when current restore excludes pv/pvc if ctx.resourceIncludesExcludes.ShouldInclude(kuberesource.PersistentVolumeClaims.String()) && ctx.resourceIncludesExcludes.ShouldInclude(kuberesource.PersistentVolumes.String()) && len(podvolume.GetVolumeBackupsForPod(ctx.podVolumeBackups, pod, originalNamespace)) > 0 { restorePodVolumeBackups(ctx, createdObj, originalNamespace) } } // Asynchronously executes restore exec hooks if any // Velero will wait for all the asynchronous hook operations to finish in finalizing phase, using hook tracker to track the execution progress. if newGR == kuberesource.Pods { pod := new(corev1api.Pod) if err := runtime.DefaultUnstructuredConverter.FromUnstructured(createdObj.UnstructuredContent(), &pod); err != nil { restoreLogger.Errorf("error converting pod %s: %s", kube.NamespaceAndName(obj), err.Error()) errs.Add(namespace, err) return warnings, errs, itemExists } execHooksByContainer, err := ctx.hooksWaitExecutor.groupHooks(ctx.restore.Name, pod, ctx.multiHookTracker) if err != nil { restoreLogger.Errorf("error grouping hooks from pod %s: %s", kube.NamespaceAndName(obj), err.Error()) errs.Add(namespace, err) return warnings, errs, itemExists } ctx.hooksWaitExecutor.exec(execHooksByContainer, pod, ctx.multiHookTracker, ctx.restore.Name) } // Wait for a CRD to be available for instantiating resources // before continuing. if newGR == kuberesource.CustomResourceDefinitions { available, err := ctx.crdAvailable(obj.GetName(), resourceClient) if err != nil { errs.Add(namespace, errors.Wrapf(err, "error verifying the CRD %s is ready to use", obj.GetName())) } else if !available { errs.Add(namespace, fmt.Errorf("the CRD %s is not available to use for custom resources", obj.GetName())) } } return warnings, errs, itemExists } func isAlreadyExistsError(ctx *restoreContext, obj *unstructured.Unstructured, err error, client client.Dynamic) (bool, error) { if err == nil { return false, nil } if apierrors.IsAlreadyExists(err) { return true, nil } // The "invalid value error" or "internal error" rather than "already exists" error returns when restoring nodePort service in the following two cases: // 1. For NodePort service, the service has nodePort preservation while the same nodePort service already exists. - Get invalid value error // 2. For LoadBalancer service, the "healthCheckNodePort" already exists. - Get internal error // If this is the case, the function returns true to avoid reporting error. // Refer to https://github.com/vmware-tanzu/velero/issues/2308 for more details if obj.GetKind() != "Service" { return false, nil } statusErr, ok := err.(*apierrors.StatusError) if !ok || statusErr.Status().Details == nil || len(statusErr.Status().Details.Causes) == 0 { return false, nil } // make sure all the causes are "port allocated" error for _, cause := range statusErr.Status().Details.Causes { if !strings.Contains(cause.Message, "provided port is already allocated") { return false, nil } } // the "already allocated" error may be caused by other services, check whether the expected service exists or not if _, err = client.Get(obj.GetName(), metav1.GetOptions{}); err != nil { if apierrors.IsNotFound(err) { ctx.log.Debugf("Service %s not found", kube.NamespaceAndName(obj)) return false, nil } return false, errors.Wrapf(err, "Unable to get the service %s while checking the NodePort is already allocated error", kube.NamespaceAndName(obj)) } ctx.log.Infof("Service %s exists, ignore the provided port is already allocated error", kube.NamespaceAndName(obj)) return true, nil } // shouldRenamePV returns a boolean indicating whether a persistent volume should // be given a new name before being restored, or an error if this cannot be determined. // A persistent volume will be given a new name if and only if (a) a PV with the // original name already exists in-cluster, and (b) in the backup, the PV is claimed // by a PVC in a namespace that's being remapped during the restore. func shouldRenamePV(ctx *restoreContext, obj *unstructured.Unstructured, client client.Dynamic) (bool, error) { if len(ctx.restore.Spec.NamespaceMapping) == 0 { ctx.log.Debugf("Persistent volume does not need to be renamed because restore is not remapping any namespaces") return false, nil } pv := new(corev1api.PersistentVolume) if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, pv); err != nil { return false, errors.Wrapf(err, "error converting persistent volume to structured") } if pv.Spec.ClaimRef == nil { ctx.log.Debugf("Persistent volume does not need to be renamed because it's not claimed") return false, nil } if _, ok := ctx.restore.Spec.NamespaceMapping[pv.Spec.ClaimRef.Namespace]; !ok { ctx.log.Debugf("Persistent volume does not need to be renamed because it's not claimed by a PVC in a namespace that's being remapped") return false, nil } _, err := client.Get(pv.Name, metav1.GetOptions{}) switch { case apierrors.IsNotFound(err): ctx.log.Debugf("Persistent volume does not need to be renamed because it does not exist in the cluster") return false, nil case err != nil: return false, errors.Wrapf(err, "error checking if persistent volume exists in the cluster") } // No error returned: the PV was found in-cluster, so we need to rename it. return true, nil } // remapClaimRefNS remaps a PersistentVolume's claimRef.Namespace based on a // restore's NamespaceMappings, if necessary. Returns true if the namespace was // remapped, false if it was not required. func remapClaimRefNS(ctx *restoreContext, obj *unstructured.Unstructured) (bool, error) { //nolint:unparam // ignore the result 0 (bool) is never used warning. if len(ctx.restore.Spec.NamespaceMapping) == 0 { ctx.log.Debug("Persistent volume does not need to have the claimRef.namespace remapped because restore is not remapping any namespaces") return false, nil } // Conversion to the real type here is more readable than all the error checking // involved with reading each field individually. pv := new(corev1api.PersistentVolume) if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, pv); err != nil { return false, errors.Wrapf(err, "error converting persistent volume to structured") } if pv.Spec.ClaimRef == nil { ctx.log.Debugf("Persistent volume does not need to have the claimRef.namespace remapped because it's not claimed") return false, nil } targetNS, ok := ctx.restore.Spec.NamespaceMapping[pv.Spec.ClaimRef.Namespace] if !ok { ctx.log.Debugf("Persistent volume does not need to have the claimRef.namespace remapped because it's not claimed by a PVC in a namespace that's being remapped") return false, nil } err := unstructured.SetNestedField(obj.Object, targetNS, "spec", "claimRef", "namespace") if err != nil { return false, err } ctx.log.Debug("Persistent volume's namespace was updated") return true, nil } // restorePodVolumeBackups restores the PodVolumeBackups for the given restored pod func restorePodVolumeBackups(ctx *restoreContext, createdObj *unstructured.Unstructured, originalNamespace string) { if ctx.podVolumeRestorer == nil { ctx.log.Warn("No pod volume restorer, not restoring pod's volumes") } else { ctx.podVolumeWaitGroup.Add(1) go func() { // Done() will only be called after all errors have been successfully // sent on the ctx.podVolumeErrs channel defer ctx.podVolumeWaitGroup.Done() pod := new(corev1api.Pod) if err := runtime.DefaultUnstructuredConverter.FromUnstructured(createdObj.UnstructuredContent(), &pod); err != nil { ctx.log.WithError(err).Error("error converting unstructured pod") ctx.podVolumeErrs <- err return } data := podvolume.RestoreData{ Restore: ctx.restore, Pod: pod, PodVolumeBackups: ctx.podVolumeBackups, SourceNamespace: originalNamespace, BackupLocation: ctx.backup.Spec.StorageLocation, } if errs := ctx.podVolumeRestorer.RestorePodVolumes(data, ctx.restoreVolumeInfoTracker); errs != nil { ctx.log.WithError(kubeerrs.NewAggregate(errs)).Error("unable to successfully complete pod volume restores of pod's volumes") for _, err := range errs { ctx.podVolumeErrs <- err } } }() } } // hooksWaitExecutor is used to collect necessary fields that are required to asynchronously execute restore exec hooks // note that fields are shared across different pods within a specific restore // and separate hooksWaitExecutors instance will be created for different restores without interfering with each other. type hooksWaitExecutor struct { log logrus.FieldLogger hooksContext go_context.Context hooksCancelFunc go_context.CancelFunc resourceRestoreHooks []hook.ResourceRestoreHook waitExecHookHandler hook.WaitExecHookHandler } func newHooksWaitExecutor(restore *velerov1api.Restore, waitExecHookHandler hook.WaitExecHookHandler) (*hooksWaitExecutor, error) { resourceRestoreHooks, err := hook.GetRestoreHooksFromSpec(&restore.Spec.Hooks) if err != nil { return nil, err } hooksCtx, hooksCancelFunc := go_context.WithCancel(go_context.Background()) hwe := &hooksWaitExecutor{ log: logrus.WithField("restore", restore.Name), hooksContext: hooksCtx, hooksCancelFunc: hooksCancelFunc, resourceRestoreHooks: resourceRestoreHooks, waitExecHookHandler: waitExecHookHandler, } return hwe, nil } // groupHooks returns a list of hooks to be executed in a pod grouped bycontainer name. func (hwe *hooksWaitExecutor) groupHooks(restoreName string, pod *corev1api.Pod, multiHookTracker *hook.MultiHookTracker) (map[string][]hook.PodExecRestoreHook, error) { execHooksByContainer, err := hook.GroupRestoreExecHooks(restoreName, hwe.resourceRestoreHooks, pod, hwe.log, multiHookTracker) return execHooksByContainer, err } // exec asynchronously executes hooks in a restored pod's containers when they become ready. // Goroutine within this function will continue running until the hook executions are complete. // Velero will wait for goroutine to finish in finalizing phase, using hook tracker to track the progress. // To optimize memory usage, ensure that the variables used in this function are kept to a minimum to prevent unnecessary retention in memory. func (hwe *hooksWaitExecutor) exec(execHooksByContainer map[string][]hook.PodExecRestoreHook, pod *corev1api.Pod, multiHookTracker *hook.MultiHookTracker, restoreName string) { go func() { if errs := hwe.waitExecHookHandler.HandleHooks(hwe.hooksContext, hwe.log, pod, execHooksByContainer, multiHookTracker, restoreName); len(errs) > 0 { hwe.log.WithError(kubeerrs.NewAggregate(errs)).Error("unable to successfully execute post-restore hooks") hwe.hooksCancelFunc() } }() } func hasSnapshot(pvName string, snapshots []*volume.Snapshot) bool { for _, snapshot := range snapshots { if snapshot.Spec.PersistentVolumeName == pvName { return true } } return false } func hasCSIVolumeSnapshot(ctx *restoreContext, unstructuredPV *unstructured.Unstructured) bool { pv := new(corev1api.PersistentVolume) if err := runtime.DefaultUnstructuredConverter.FromUnstructured(unstructuredPV.Object, pv); err != nil { ctx.log.WithError(err).Warnf("Unable to convert PV from unstructured to structured") return false } // ignoring static PV cases where there is no claimRef if pv.Spec.ClaimRef == nil { return false } for _, vs := range ctx.csiVolumeSnapshots { // In some error cases, the VSs' source PVC could be nil. Skip them. if vs.Spec.Source.PersistentVolumeClaimName == nil { continue } if pv.Spec.ClaimRef.Name == *vs.Spec.Source.PersistentVolumeClaimName && pv.Spec.ClaimRef.Namespace == vs.Namespace { return true } } return false } func hasSnapshotDataUpload(ctx *restoreContext, unstructuredPV *unstructured.Unstructured) bool { pv := new(corev1api.PersistentVolume) if err := runtime.DefaultUnstructuredConverter.FromUnstructured(unstructuredPV.Object, pv); err != nil { ctx.log.WithError(err).Warnf("Unable to convert PV from unstructured to structured") return false } if pv.Spec.ClaimRef == nil { return false } dataUploadResultList := new(corev1api.ConfigMapList) err := ctx.kbClient.List(go_context.TODO(), dataUploadResultList, &crclient.ListOptions{ LabelSelector: labels.SelectorFromSet(map[string]string{ velerov1api.RestoreUIDLabel: label.GetValidName(string(ctx.restore.GetUID())), velerov1api.PVCNamespaceNameLabel: label.GetValidName(pv.Spec.ClaimRef.Namespace + "." + pv.Spec.ClaimRef.Name), velerov1api.ResourceUsageLabel: label.GetValidName(string(velerov1api.VeleroResourceUsageDataUploadResult)), }), }) if err != nil { ctx.log.WithError(err).Warnf("Fail to list DataUpload result CM.") return false } if len(dataUploadResultList.Items) != 1 { ctx.log.WithError(fmt.Errorf("dataupload result number is not expected")). Warnf("Got %d DataUpload result. Expect one.", len(dataUploadResultList.Items)) return false } return true } func hasPodVolumeBackup(unstructuredPV *unstructured.Unstructured, ctx *restoreContext) bool { if len(ctx.podVolumeBackups) == 0 { return false } pv := new(corev1api.PersistentVolume) if err := runtime.DefaultUnstructuredConverter.FromUnstructured(unstructuredPV.Object, pv); err != nil { ctx.log.WithError(err).Warnf("Unable to convert PV from unstructured to structured") return false } if pv.Spec.ClaimRef == nil { return false } var found bool for _, pvb := range ctx.podVolumeBackups { if pvb.Spec.Pod.Namespace == pv.Spec.ClaimRef.Namespace && pvb.GetAnnotations()[configs.PVCNameAnnotation] == pv.Spec.ClaimRef.Name { found = true break } } return found } func hasDeleteReclaimPolicy(obj map[string]any) bool { policy, _, _ := unstructured.NestedString(obj, "spec", "persistentVolumeReclaimPolicy") return policy == string(corev1api.PersistentVolumeReclaimDelete) } // resetVolumeBindingInfo clears any necessary metadata out of a PersistentVolume // or PersistentVolumeClaim that would make it ineligible to be re-bound by Velero. func resetVolumeBindingInfo(obj *unstructured.Unstructured) *unstructured.Unstructured { // Clean out ClaimRef UID and resourceVersion, since this information is // highly unique. unstructured.RemoveNestedField(obj.Object, "spec", "claimRef", "uid") unstructured.RemoveNestedField(obj.Object, "spec", "claimRef", "resourceVersion") // Clear out any annotations used by the Kubernetes PV controllers to track // bindings. annotations := obj.GetAnnotations() // Upon restore, this new PV will look like a statically provisioned, manually- // bound volume rather than one bound by the controller, so remove the annotation // that signals that a controller bound it. delete(annotations, kube.KubeAnnBindCompleted) // Remove the annotation that signals that the PV is already bound; we want // the PV(C) controller to take the two objects and bind them again. delete(annotations, kube.KubeAnnBoundByController) // GetAnnotations returns a copy, so we have to set them again. obj.SetAnnotations(annotations) return obj } func resetMetadata(obj *unstructured.Unstructured) (*unstructured.Unstructured, error) { res, ok := obj.Object["metadata"] if !ok { return nil, errors.New("metadata not found") } metadata, ok := res.(map[string]any) if !ok { return nil, errors.Errorf("metadata was of type %T, expected map[string]any", res) } for k := range metadata { switch k { case "generateName", "selfLink", "uid", "resourceVersion", "generation", "creationTimestamp", "deletionTimestamp", "deletionGracePeriodSeconds", "ownerReferences": delete(metadata, k) } } return obj, nil } func resetStatus(obj *unstructured.Unstructured) { unstructured.RemoveNestedField(obj.UnstructuredContent(), "status") } func resetMetadataAndStatus(obj *unstructured.Unstructured) (*unstructured.Unstructured, error) { _, err := resetMetadata(obj) if err != nil { return nil, err } resetStatus(obj) return obj, nil } // addRestoreLabels labels the provided object with the restore name and the // restored backup's name. func addRestoreLabels(obj metav1.Object, restoreName, backupName string) { labels := obj.GetLabels() if labels == nil { labels = make(map[string]string) } labels[velerov1api.BackupNameLabel] = label.GetValidName(backupName) labels[velerov1api.RestoreNameLabel] = label.GetValidName(restoreName) obj.SetLabels(labels) } // isCompleted returns whether or not an object is considered completed. Used to // identify whether or not an object should be restored. Only Jobs or Pods are // considered. func isCompleted(obj *unstructured.Unstructured, groupResource schema.GroupResource) (bool, error) { switch groupResource { case kuberesource.Pods: phase, _, err := unstructured.NestedString(obj.UnstructuredContent(), "status", "phase") if err != nil { return false, errors.WithStack(err) } if phase == string(corev1api.PodFailed) || phase == string(corev1api.PodSucceeded) { return true, nil } case kuberesource.Jobs: ct, found, err := unstructured.NestedString(obj.UnstructuredContent(), "status", "completionTime") if err != nil { return false, errors.WithStack(err) } if found && ct != "" { return true, nil } } // Assume any other resource isn't complete and can be restored. return false, nil } // restoreableResource represents map of individual items of each resource // identifier grouped by their original namespaces. type restoreableResource struct { resource string selectedItemsByNamespace map[string][]restoreableItem totalItems int } // restoreableItem represents an item by its target namespace contains enough // information required to restore the item. type restoreableItem struct { path string targetNamespace string name string version string // used for initializing informer cache } // getOrderedResourceCollection iterates over list of ordered resource // identifiers, applies resource include/exclude criteria, and Kubernetes // selectors to make a list of resources to be actually restored preserving the // original order. func (ctx *restoreContext) getOrderedResourceCollection( backupResources map[string]*archive.ResourceItems, restoreResourceCollection []restoreableResource, processedResources sets.Set[string], resourcePriorities types.Priorities, includeAllResources bool, ) ([]restoreableResource, sets.Set[string], results.Result, results.Result) { var warnings, errs results.Result // Iterate through an ordered list of resources to restore, checking each // one to see if it should be restored. Note that resources *may* be in this // list twice, i.e. once due to being a prioritized resource, and once due // to being in the backup tarball. We can't de-dupe this upfront, because // it's possible that items in the prioritized resources list may not be // fully resolved group-resource strings (e.g. may be specified as "po" // instead of "pods"), and we don't want to fully resolve them via discovery // until we reach them in the loop, because it is possible that the // resource/API itself is being restored via a custom resource definition, // meaning it's not available via discovery prior to beginning the restore. // // Since we keep track of the fully-resolved group-resources that we *have* // restored, we won't try to restore a resource twice even if it's in the // ordered list twice. var resourceList []string if includeAllResources { resourceList = getOrderedResources(resourcePriorities, backupResources) } else { resourceList = resourcePriorities.HighPriorities } for _, resource := range resourceList { groupResource := schema.ParseGroupResource(resource) // try to resolve the resource via discovery to a complete group/version/resource gvr, _, err := ctx.discoveryHelper.ResourceFor(groupResource.WithVersion("")) if err != nil { // don't skip if we can't resolve the resource via discovery, log it // the gv of this resource may be changed in a RIA, we can try to get it after that ctx.log.WithField("resource", resource).Infof("resource cannot be resolved via discovery") } else { groupResource = gvr.GroupResource() } // Check if we've already restored this resource (this would happen if // the resource we're currently looking at was already restored because // it was a prioritized resource, and now we're looking at it as part of // the backup contents). if processedResources.Has(groupResource.String()) { ctx.log.WithField("resource", groupResource.String()).Debugf("Skipping restore of resource because it's already been processed") continue } // Check if the resource should be restored according to the resource // includes/excludes. if !ctx.resourceIncludesExcludes.ShouldInclude(groupResource.String()) && !ctx.resourceMustHave.Has(groupResource.String()) { ctx.log.WithField("resource", groupResource.String()).Infof("Skipping restore of resource because the restore spec excludes it") continue } // Check if the resource is present in the backup resourceList := backupResources[groupResource.String()] if resourceList == nil { ctx.log.WithField("resource", groupResource.String()).Debugf("Skipping restore of resource because it's not present in the backup tarball") continue } // Iterate through each namespace that contains instances of the // resource and append to the list of to-be restored resources. for namespace, items := range resourceList.ItemsByNamespace { if namespace != "" && !ctx.namespaceIncludesExcludes.ShouldInclude(namespace) && !ctx.resourceMustHave.Has(groupResource.String()) { ctx.log.Infof("Skipping namespace %s", namespace) continue } if namespace == "" && boolptr.IsSetToFalse(ctx.restore.Spec.IncludeClusterResources) { ctx.log.Infof("Skipping resource %s because it's cluster-scoped", resource) continue } if namespace == "" && !boolptr.IsSetToTrue(ctx.restore.Spec.IncludeClusterResources) && !ctx.namespaceIncludesExcludes.IncludeEverything() { ctx.log.Infof("Skipping resource %s because it's cluster-scoped and only specific namespaces are included in the restore", resource) continue } res, w, e := ctx.getSelectedRestoreableItems(groupResource.String(), namespace, items) warnings.Merge(&w) errs.Merge(&e) restoreResourceCollection = append(restoreResourceCollection, res) } // record that we've restored the resource processedResources.Insert(groupResource.String()) } return restoreResourceCollection, processedResources, warnings, errs } // getSelectedRestoreableItems applies Kubernetes selectors on individual items // of each resource type to create a list of items which will be actually // restored. func (ctx *restoreContext) getSelectedRestoreableItems(resource string, originalNamespace string, items []string) (restoreableResource, results.Result, results.Result) { //nolint:unparam // Ignore the warnings is always nil warning. warnings, errs := results.Result{}, results.Result{} restorable := restoreableResource{ resource: resource, } if restorable.selectedItemsByNamespace == nil { restorable.selectedItemsByNamespace = make(map[string][]restoreableItem) } targetNamespace := originalNamespace if target, ok := ctx.restore.Spec.NamespaceMapping[originalNamespace]; ok { targetNamespace = target } if targetNamespace != "" { ctx.log.Infof("Resource '%s' will be restored into namespace '%s'", resource, targetNamespace) } else { ctx.log.Infof("Resource '%s' will be restored at cluster scope", resource) } resourceForPath := resource // If the APIGroupVersionsFeatureFlag is enabled, the item path will be // updated to include the API group version that was chosen for restore. For // example, for "horizontalpodautoscalers.autoscaling", if v2beta1 is chosen // to be restored, then "horizontalpodautoscalers.autoscaling/v2beta1" will // be part of item path. Different versions would only have been stored // if the APIGroupVersionsFeatureFlag was enabled during backup. The // chosenGrpVersToRestore map would only be populated if // APIGroupVersionsFeatureFlag was enabled for restore and the minimum // required backup format version has been met. cgv, ok := ctx.chosenGrpVersToRestore[resource] if ok { resourceForPath = filepath.Join(resource, cgv.Dir) } for _, item := range items { itemPath := archive.GetItemFilePath(ctx.restoreDir, resourceForPath, originalNamespace, item) obj, err := archive.Unmarshal(ctx.fileSystem, itemPath) if err != nil { errs.Add( targetNamespace, fmt.Errorf( "error decoding %q: %v", strings.Replace(itemPath, ctx.restoreDir+"/", "", -1), err, ), ) continue } if !ctx.resourceMustHave.Has(resource) { if !ctx.selector.Matches(labels.Set(obj.GetLabels())) { continue } // Processing OrLabelSelectors when specified in the restore request. LabelSelectors as well as OrLabelSelectors // cannot co-exist, only one of them can be specified var skipItem = false var skip = 0 ctx.log.Debugf("orSelectors specified: %s for item: %s", ctx.OrSelectors, item) for _, s := range ctx.OrSelectors { if !s.Matches(labels.Set(obj.GetLabels())) { skip++ } if len(ctx.OrSelectors) == skip && skip > 0 { ctx.log.Infof("setting skip flag to true for item: %s", item) skipItem = true } } if skipItem { ctx.log.Infof("restore orSelector labels did not match, skipping restore of item: %s", skipItem, item) continue } } selectedItem := restoreableItem{ path: itemPath, name: item, targetNamespace: targetNamespace, version: obj.GroupVersionKind().Version, } restorable.selectedItemsByNamespace[originalNamespace] = append(restorable.selectedItemsByNamespace[originalNamespace], selectedItem) restorable.totalItems++ } return restorable, warnings, errs } // extractNamespacesFromBackup extracts all available namespaces from backup resources func extractNamespacesFromBackup(backupResources map[string]*archive.ResourceItems) []string { namespaceSet := make(map[string]struct{}) for _, resource := range backupResources { for namespace := range resource.ItemsByNamespace { if namespace != "" { // Skip cluster-scoped resources (empty namespace) namespaceSet[namespace] = struct{}{} } } } namespaces := make([]string, 0, len(namespaceSet)) for ns := range namespaceSet { namespaces = append(namespaces, ns) } return namespaces } // expandNamespaceWildcards expands wildcard patterns in namespace includes/excludes // and updates the restore context with the expanded patterns and status func (ctx *restoreContext) expandNamespaceWildcards(backupResources map[string]*archive.ResourceItems) error { if !wildcard.ShouldExpandWildcards(ctx.restore.Spec.IncludedNamespaces, ctx.restore.Spec.ExcludedNamespaces) { return nil } // If `*` is mentioned in restore excludes, something is wrong if slices.Contains(ctx.restore.Spec.ExcludedNamespaces, "*") { return errors.New("wildcard '*' is not allowed in restore excludes") } availableNamespaces := extractNamespacesFromBackup(backupResources) expandedIncludes, expandedExcludes, err := wildcard.ExpandWildcards( availableNamespaces, ctx.restore.Spec.IncludedNamespaces, ctx.restore.Spec.ExcludedNamespaces, ) if err != nil { return errors.Wrap(err, "error expanding wildcard patterns in namespace includes/excludes") } // Update namespace includes/excludes with expanded patterns ctx.namespaceIncludesExcludes = collections.NewIncludesExcludes(). Includes(expandedIncludes...). Excludes(expandedExcludes...) selectedNamespaces := wildcard.GetWildcardResult(expandedIncludes, expandedExcludes) ctx.log.Infof("Expanded namespace wildcards - includes: %v, excludes: %v, final: %v", expandedIncludes, expandedExcludes, selectedNamespaces) return nil } // removeRestoreLabels removes the restore name and the // restored backup's name. func removeRestoreLabels(obj metav1.Object) { labels := obj.GetLabels() if labels == nil { labels = make(map[string]string) } labels[velerov1api.BackupNameLabel] = "" labels[velerov1api.RestoreNameLabel] = "" obj.SetLabels(labels) } // updates the backup/restore labels func (ctx *restoreContext) updateBackupRestoreLabels(fromCluster, fromClusterWithLabels *unstructured.Unstructured, namespace string, resourceClient client.Dynamic) (warnings, errs results.Result) { //nolint:unparam // Ignore the warnings is nil warning. patchBytes, err := generatePatch(fromCluster, fromClusterWithLabels) if err != nil { ctx.log.Errorf("error generating patch for %s %s: %v", fromCluster.GroupVersionKind().Kind, kube.NamespaceAndName(fromCluster), err) errs.Add(namespace, err) return warnings, errs } if patchBytes == nil { // In-cluster and desired state are the same, so move on to // the next items ctx.log.Errorf("skipped updating backup/restore labels for %s %s: in-cluster and desired state are the same along-with the labels", fromCluster.GroupVersionKind().Kind, kube.NamespaceAndName(fromCluster)) return warnings, errs } // try patching the in-cluster resource (with only latest backup/restore labels) _, err = resourceClient.Patch(fromCluster.GetName(), patchBytes) if err != nil { ctx.log.Errorf("backup/restore label patch attempt failed for %s %s: %v", fromCluster.GroupVersionKind(), kube.NamespaceAndName(fromCluster), err) errs.Add(namespace, err) } else { ctx.log.Infof("backup/restore labels successfully updated for %s %s", fromCluster.GroupVersionKind().Kind, kube.NamespaceAndName(fromCluster)) } return warnings, errs } // function to process existingResourcePolicy as update, tries to patch the diff between in-cluster and restore obj first // if the patch fails then tries to update the backup/restore labels for the in-cluster version func (ctx *restoreContext) processUpdateResourcePolicy(fromCluster, fromClusterWithLabels, obj *unstructured.Unstructured, namespace string, resourceClient client.Dynamic) (warnings, errs results.Result) { ctx.log.Infof("restore API has existingResourcePolicy defined as update , executing restore workflow accordingly for changed resource %s %s ", obj.GroupVersionKind().Kind, kube.NamespaceAndName(fromCluster)) ctx.log.Infof("attempting patch on %s %q", fromCluster.GetKind(), fromCluster.GetName()) // remove restore labels so that we apply the latest backup/restore names on the object via patch removeRestoreLabels(fromCluster) patchBytes, err := generatePatch(fromCluster, obj) if err != nil { ctx.log.Errorf("error generating patch for %s %s: %v", obj.GroupVersionKind().Kind, kube.NamespaceAndName(obj), err) errs.Add(namespace, err) return warnings, errs } if patchBytes == nil { // In-cluster and desired state are the same, so move on to // the next items ctx.log.Errorf("skipped updating %s %s: in-cluster and desired state are the same", fromCluster.GroupVersionKind().Kind, kube.NamespaceAndName(fromCluster)) return warnings, errs } // try patching the in-cluster resource (resource diff plus latest backup/restore labels) _, err = resourceClient.Patch(obj.GetName(), patchBytes) if err != nil { ctx.log.Warnf("patch attempt failed for %s %s: %v", fromCluster.GroupVersionKind(), kube.NamespaceAndName(fromCluster), err) warnings.Add(namespace, err) // try just patching the labels warningsFromUpdate, errsFromUpdate := ctx.updateBackupRestoreLabels(fromCluster, fromClusterWithLabels, namespace, resourceClient) warnings.Merge(&warningsFromUpdate) errs.Merge(&errsFromUpdate) } else { ctx.log.Infof("%s %s successfully updated", obj.GroupVersionKind().Kind, kube.NamespaceAndName(obj)) } return warnings, errs } func (ctx *restoreContext) handlePVHasNativeSnapshot(obj *unstructured.Unstructured, resourceClient client.Dynamic) (*unstructured.Unstructured, error) { retObj := obj.DeepCopy() oldName := obj.GetName() shouldRenamePV, err := shouldRenamePV(ctx, retObj, resourceClient) if err != nil { return nil, err } // Check to see if the claimRef.namespace field needs to be remapped, // and do so if necessary. _, err = remapClaimRefNS(ctx, retObj) if err != nil { return nil, err } var shouldRestoreSnapshot bool if !shouldRenamePV { // Check if the PV exists in the cluster before attempting to create // a volume from the snapshot, in order to avoid orphaned volumes (GH #609) shouldRestoreSnapshot, err = ctx.shouldRestore(oldName, resourceClient) if err != nil { return nil, errors.Wrapf(err, "error waiting on in-cluster persistentvolume %s", oldName) } } else { // If we're renaming the PV, we're going to give it a new random name, // so we can assume it doesn't already exist in the cluster and therefore // we should proceed with restoring from snapshot. shouldRestoreSnapshot = true } if shouldRestoreSnapshot { // Reset the PV's binding status so that Kubernetes can properly // associate it with the restored PVC. retObj = resetVolumeBindingInfo(retObj) // Even if we're renaming the PV, obj still has the old name here, because the pvRestorer // uses the original name to look up metadata about the snapshot. ctx.log.Infof("Restoring persistent volume from snapshot.") retObj, err = ctx.pvRestorer.executePVAction(retObj) if err != nil { return nil, fmt.Errorf("error executing PVAction for %s: %v", getResourceID(kuberesource.PersistentVolumes, "", oldName), err) } // VolumeSnapshotter has modified the PV name, we should rename the PV. if oldName != retObj.GetName() { shouldRenamePV = true } } if shouldRenamePV { var pvName string if oldName == retObj.GetName() { // pvRestorer hasn't modified the PV name, we need to rename the PV. pvName, err = ctx.pvRenamer(oldName) if err != nil { return nil, errors.Wrapf(err, "error renaming PV") } } else { // VolumeSnapshotter could have modified the PV name through // function `SetVolumeID`, pvName = retObj.GetName() } ctx.renamedPVs[oldName] = pvName retObj.SetName(pvName) ctx.restoreVolumeInfoTracker.RenamePVForNativeSnapshot(oldName, pvName) // Add the original PV name as an annotation. annotations := retObj.GetAnnotations() if annotations == nil { annotations = map[string]string{} } annotations["velero.io/original-pv-name"] = oldName retObj.SetAnnotations(annotations) } return retObj, nil } func (ctx *restoreContext) handleSkippedPVHasRetainPolicy( obj *unstructured.Unstructured, logger logrus.FieldLogger, ) (*unstructured.Unstructured, error) { logger.Infof("Restoring persistent volume as-is because it doesn't have a snapshot and its reclaim policy is not Delete.") // Check to see if the claimRef.namespace field needs to be remapped, and do so if necessary. if _, err := remapClaimRefNS(ctx, obj); err != nil { return nil, err } obj = resetVolumeBindingInfo(obj) return obj, nil } func determineRestoreStatus( obj *unstructured.Unstructured, resourceIncludesExcludes *collections.IncludesExcludes, groupResource string, log logrus.FieldLogger, ) bool { var shouldRestoreStatus bool // Determine restore spec behavior if resourceIncludesExcludes != nil { shouldRestoreStatus = resourceIncludesExcludes.ShouldInclude(groupResource) } // Retrieve annotations annotations := obj.GetAnnotations() if annotations == nil { log.Warnf("No annotations found for %s, using restore spec setting: %v", kube.NamespaceAndName(obj), shouldRestoreStatus) return shouldRestoreStatus } // Check for object-level annotation objectAnnotation, annotationExists := annotations[ObjectStatusRestoreAnnotationKey] if !annotationExists { log.Debugf("No restore status-specific annotation found for %s, using restore spec setting: %v", kube.NamespaceAndName(obj), shouldRestoreStatus) return shouldRestoreStatus } normalizedValue := strings.ToLower(strings.TrimSpace(objectAnnotation)) switch normalizedValue { case "true": shouldRestoreStatus = true case "false": shouldRestoreStatus = false default: log.Warnf("Invalid annotation value '%s' on %s, using restore spec setting: %v", objectAnnotation, kube.NamespaceAndName(obj), shouldRestoreStatus) } log.Infof("Final status restore decision for %s: %v (annotation: %v, restore spec: %v)", kube.NamespaceAndName(obj), shouldRestoreStatus, annotationExists, shouldRestoreStatus) return shouldRestoreStatus } ================================================ FILE: pkg/restore/restore_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package restore import ( "context" "encoding/json" "fmt" "io" "sort" "testing" "time" "github.com/vmware-tanzu/velero/pkg/util/boolptr" "github.com/vmware-tanzu/velero/pkg/util/collections" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/dynamic" kubetesting "k8s.io/client-go/testing" "github.com/vmware-tanzu/velero/internal/volume" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/archive" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/discovery" "github.com/vmware-tanzu/velero/pkg/features" "github.com/vmware-tanzu/velero/pkg/itemoperation" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/plugin/velero" riav2 "github.com/vmware-tanzu/velero/pkg/plugin/velero/restoreitemaction/v2" vsv1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/volumesnapshotter/v1" "github.com/vmware-tanzu/velero/pkg/podvolume" uploadermocks "github.com/vmware-tanzu/velero/pkg/podvolume/mocks" "github.com/vmware-tanzu/velero/pkg/test" "github.com/vmware-tanzu/velero/pkg/types" "github.com/vmware-tanzu/velero/pkg/util/kube" kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube" . "github.com/vmware-tanzu/velero/pkg/util/results" ) func TestRestorePVWithVolumeInfo(t *testing.T) { tests := []struct { name string restore *velerov1api.Restore backup *velerov1api.Backup apiResources []*test.APIResource tarball io.Reader want map[*test.APIResource][]string volumeInfoMap map[string]volume.BackupVolumeInfo }{ { name: "Restore PV with native snapshot", restore: defaultRestore().Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").ReclaimPolicy(corev1api.PersistentVolumeReclaimRetain).Result(), ).Done(), apiResources: []*test.APIResource{ test.PVs(), }, volumeInfoMap: map[string]volume.BackupVolumeInfo{ "pv-1": { BackupMethod: volume.NativeSnapshot, PVName: "pv-1", NativeSnapshotInfo: &volume.NativeSnapshotInfo{ SnapshotHandle: "testSnapshotHandle", }, }, }, want: map[*test.APIResource][]string{ test.PVs(): {"/pv-1"}, }, }, { name: "Restore PV with PVB", restore: defaultRestore().Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").ReclaimPolicy(corev1api.PersistentVolumeReclaimRetain).Result(), ).Done(), apiResources: []*test.APIResource{ test.PVs(), }, volumeInfoMap: map[string]volume.BackupVolumeInfo{ "pv-1": { BackupMethod: volume.PodVolumeBackup, PVName: "pv-1", PVBInfo: &volume.PodVolumeInfo{ SnapshotHandle: "testSnapshotHandle", Size: 100, NodeName: "testNode", }, }, }, want: map[*test.APIResource][]string{ test.PVs(): {}, }, }, { name: "Restore PV with CSI VolumeSnapshot", restore: defaultRestore().Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").ReclaimPolicy(corev1api.PersistentVolumeReclaimRetain).Result(), ).Done(), apiResources: []*test.APIResource{ test.PVs(), }, volumeInfoMap: map[string]volume.BackupVolumeInfo{ "pv-1": { BackupMethod: volume.CSISnapshot, SnapshotDataMoved: false, PVName: "pv-1", CSISnapshotInfo: &volume.CSISnapshotInfo{ Driver: "pd.csi.storage.gke.io", }, }, }, want: map[*test.APIResource][]string{ test.PVs(): {}, }, }, { name: "Restore PV with DataUpload", restore: defaultRestore().Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").ReclaimPolicy(corev1api.PersistentVolumeReclaimRetain).Result(), ).Done(), apiResources: []*test.APIResource{ test.PVs(), }, volumeInfoMap: map[string]volume.BackupVolumeInfo{ "pv-1": { BackupMethod: volume.CSISnapshot, SnapshotDataMoved: true, PVName: "pv-1", CSISnapshotInfo: &volume.CSISnapshotInfo{ Driver: "pd.csi.storage.gke.io", }, SnapshotDataMovementInfo: &volume.SnapshotDataMovementInfo{ DataMover: "velero", }, }, }, want: map[*test.APIResource][]string{ test.PVs(): {}, }, }, { name: "Restore PV with ClaimPolicy as Delete", restore: defaultRestore().Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").ReclaimPolicy(corev1api.PersistentVolumeReclaimDelete).Result(), ).Done(), apiResources: []*test.APIResource{ test.PVs(), }, volumeInfoMap: map[string]volume.BackupVolumeInfo{ "pv-1": { PVName: "pv-1", Skipped: true, }, }, want: map[*test.APIResource][]string{ test.PVs(): {}, }, }, { name: "Restore PV with ClaimPolicy as Retain", restore: defaultRestore().Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").ReclaimPolicy(corev1api.PersistentVolumeReclaimRetain).Result(), ).Done(), apiResources: []*test.APIResource{ test.PVs(), }, volumeInfoMap: map[string]volume.BackupVolumeInfo{ "pv-1": { PVName: "pv-1", Skipped: true, }, }, want: map[*test.APIResource][]string{ test.PVs(): {"/pv-1"}, }, }, } features.Enable("EnableCSI") for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { h := newHarness(t) for _, r := range tc.apiResources { h.DiscoveryClient.WithAPIResource(r) } require.NoError(t, h.restorer.discoveryHelper.Refresh()) data := &Request{ Log: h.log, Restore: tc.restore, Backup: tc.backup, PodVolumeBackups: nil, VolumeSnapshots: nil, BackupReader: tc.tarball, BackupVolumeInfoMap: tc.volumeInfoMap, } warnings, errs := h.restorer.Restore( data, nil, // restoreItemActions nil, // volume snapshotter getter ) assertEmptyResults(t, warnings, errs) assertAPIContents(t, h, tc.want) }) } } // TestRestoreResourceFiltering runs restores with different combinations // of resource filters (included/excluded resources, included/excluded // namespaces, label selectors, "include cluster resources" flag), and // verifies that the set of items created in the API are correct. // Validation is done by looking at the namespaces/names of the items in // the API; contents are not checked. func TestRestoreResourceFiltering(t *testing.T) { tests := []struct { name string restore *velerov1api.Restore backup *velerov1api.Backup apiResources []*test.APIResource tarball io.Reader want map[*test.APIResource][]string }{ { name: "no filters restores everything", restore: defaultRestore().Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("pods", builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result(), ). AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result(), ). Done(), apiResources: []*test.APIResource{ test.Pods(), test.PVs(), }, want: map[*test.APIResource][]string{ test.Pods(): {"ns-1/pod-1", "ns-2/pod-2"}, test.PVs(): {"/pv-1", "/pv-2"}, }, }, { name: "included resources filter only restores resources of those types", restore: defaultRestore().IncludedResources("pods").Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("pods", builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result(), ). AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result(), ). Done(), apiResources: []*test.APIResource{ test.Pods(), test.PVs(), }, want: map[*test.APIResource][]string{ test.Pods(): {"ns-1/pod-1", "ns-2/pod-2"}, }, }, { name: "excluded resources filter only restores resources not of those types", restore: defaultRestore().ExcludedResources("pvs").Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("pods", builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result(), ). AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result(), ). Done(), apiResources: []*test.APIResource{ test.Pods(), test.PVs(), }, want: map[*test.APIResource][]string{ test.Pods(): {"ns-1/pod-1", "ns-2/pod-2"}, }, }, { name: "included namespaces filter only restores resources in those namespaces", restore: defaultRestore().IncludedNamespaces("ns-1").Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("pods", builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result(), ). AddItems("deployments.apps", builder.ForDeployment("ns-1", "deploy-1").Result(), builder.ForDeployment("ns-2", "deploy-2").Result(), ). AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result(), ). Done(), apiResources: []*test.APIResource{ test.Pods(), test.Deployments(), test.PVs(), }, want: map[*test.APIResource][]string{ test.Pods(): {"ns-1/pod-1"}, test.Deployments(): {"ns-1/deploy-1"}, }, }, { name: "excluded namespaces filter only restores resources not in those namespaces", restore: defaultRestore().ExcludedNamespaces("ns-2").Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("pods", builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result(), ). AddItems("deployments.apps", builder.ForDeployment("ns-1", "deploy-1").Result(), builder.ForDeployment("ns-2", "deploy-2").Result(), ). AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result(), ). Done(), apiResources: []*test.APIResource{ test.Pods(), test.Deployments(), test.PVs(), }, want: map[*test.APIResource][]string{ test.Pods(): {"ns-1/pod-1"}, test.Deployments(): {"ns-1/deploy-1"}, }, }, { name: "IncludeClusterResources=false only restores namespaced resources", restore: defaultRestore().IncludeClusterResources(false).Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("pods", builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result(), ). AddItems("deployments.apps", builder.ForDeployment("ns-1", "deploy-1").Result(), builder.ForDeployment("ns-2", "deploy-2").Result(), ). AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result(), ). Done(), apiResources: []*test.APIResource{ test.Pods(), test.Deployments(), test.PVs(), }, want: map[*test.APIResource][]string{ test.Pods(): {"ns-1/pod-1", "ns-2/pod-2"}, test.Deployments(): {"ns-1/deploy-1", "ns-2/deploy-2"}, }, }, { name: "label selector only restores matching resources", restore: defaultRestore().LabelSelector(&metav1.LabelSelector{MatchLabels: map[string]string{"a": "b"}}).Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("pods", builder.ForPod("ns-1", "pod-1").ObjectMeta(builder.WithLabels("a", "b")).Result(), builder.ForPod("ns-2", "pod-2").Result(), ). AddItems("deployments.apps", builder.ForDeployment("ns-1", "deploy-1").Result(), builder.ForDeployment("ns-2", "deploy-2").ObjectMeta(builder.WithLabels("a", "b")).Result(), ). AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").ObjectMeta(builder.WithLabels("a", "b")).Result(), builder.ForPersistentVolume("pv-2").ObjectMeta(builder.WithLabels("a", "c")).Result(), ). Done(), apiResources: []*test.APIResource{ test.Pods(), test.Deployments(), test.PVs(), }, want: map[*test.APIResource][]string{ test.Pods(): {"ns-1/pod-1"}, test.Deployments(): {"ns-2/deploy-2"}, test.PVs(): {"/pv-1"}, }, }, { name: "OrLabelSelectors only restores matching resources", restore: defaultRestore().OrLabelSelector([]*metav1.LabelSelector{{MatchLabels: map[string]string{"a1": "b1"}}, {MatchLabels: map[string]string{"a2": "b2"}}, {MatchLabels: map[string]string{"a3": "b3"}}, {MatchLabels: map[string]string{"a4": "b4"}}}).Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("pods", builder.ForPod("ns-1", "pod-1").ObjectMeta(builder.WithLabels("a1", "b1")).Result(), builder.ForPod("ns-2", "pod-2").Result(), ). AddItems("deployments.apps", builder.ForDeployment("ns-1", "deploy-1").Result(), builder.ForDeployment("ns-2", "deploy-2").ObjectMeta(builder.WithLabels("a3", "b3")).Result(), ). AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").ObjectMeta(builder.WithLabels("a5", "b5")).Result(), builder.ForPersistentVolume("pv-2").ObjectMeta(builder.WithLabels("a4", "b4")).Result(), ). Done(), apiResources: []*test.APIResource{ test.Pods(), test.Deployments(), test.PVs(), }, want: map[*test.APIResource][]string{ test.Pods(): {"ns-1/pod-1"}, test.Deployments(): {"ns-2/deploy-2"}, test.PVs(): {"/pv-2"}, }, }, { name: "should include cluster-scoped resources if restoring subset of namespaces and IncludeClusterResources=true", restore: defaultRestore().IncludedNamespaces("ns-1").IncludeClusterResources(true).Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("pods", builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result(), ). AddItems("deployments.apps", builder.ForDeployment("ns-1", "deploy-1").Result(), builder.ForDeployment("ns-2", "deploy-2").Result(), ). AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result(), ). Done(), apiResources: []*test.APIResource{ test.Pods(), test.Deployments(), test.PVs(), }, want: map[*test.APIResource][]string{ test.Pods(): {"ns-1/pod-1"}, test.Deployments(): {"ns-1/deploy-1"}, test.PVs(): {"/pv-1", "/pv-2"}, }, }, { name: "should not include cluster-scoped resources if restoring subset of namespaces and IncludeClusterResources=false", restore: defaultRestore().IncludedNamespaces("ns-1").IncludeClusterResources(false).Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("pods", builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result(), ). AddItems("deployments.apps", builder.ForDeployment("ns-1", "deploy-1").Result(), builder.ForDeployment("ns-2", "deploy-2").Result(), ). AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result(), ). Done(), apiResources: []*test.APIResource{ test.Pods(), test.Deployments(), test.PVs(), }, want: map[*test.APIResource][]string{ test.Pods(): {"ns-1/pod-1"}, test.Deployments(): {"ns-1/deploy-1"}, test.PVs(): {}, }, }, { name: "should not include cluster-scoped resources if restoring subset of namespaces and IncludeClusterResources=nil", restore: defaultRestore().IncludedNamespaces("ns-1").Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("pods", builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result(), ). AddItems("deployments.apps", builder.ForDeployment("ns-1", "deploy-1").Result(), builder.ForDeployment("ns-2", "deploy-2").Result(), ). AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result(), ). Done(), apiResources: []*test.APIResource{ test.Pods(), test.Deployments(), test.PVs(), }, want: map[*test.APIResource][]string{ test.Pods(): {"ns-1/pod-1"}, test.Deployments(): {"ns-1/deploy-1"}, test.PVs(): {}, }, }, { name: "should include cluster-scoped resources if restoring all namespaces and IncludeClusterResources=true", restore: defaultRestore().IncludeClusterResources(true).Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("pods", builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result(), ). AddItems("deployments.apps", builder.ForDeployment("ns-1", "deploy-1").Result(), builder.ForDeployment("ns-2", "deploy-2").Result(), ). AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result(), ). Done(), apiResources: []*test.APIResource{ test.Pods(), test.Deployments(), test.PVs(), }, want: map[*test.APIResource][]string{ test.Pods(): {"ns-1/pod-1", "ns-2/pod-2"}, test.Deployments(): {"ns-1/deploy-1", "ns-2/deploy-2"}, test.PVs(): {"/pv-1", "/pv-2"}, }, }, { name: "should not include cluster-scoped resources if restoring all namespaces and IncludeClusterResources=false", restore: defaultRestore().IncludeClusterResources(false).Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("pods", builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result(), ). AddItems("deployments.apps", builder.ForDeployment("ns-1", "deploy-1").Result(), builder.ForDeployment("ns-2", "deploy-2").Result(), ). AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result(), ). Done(), apiResources: []*test.APIResource{ test.Pods(), test.Deployments(), test.PVs(), }, want: map[*test.APIResource][]string{ test.Pods(): {"ns-1/pod-1", "ns-2/pod-2"}, test.Deployments(): {"ns-1/deploy-1", "ns-2/deploy-2"}, }, }, { name: "when a wildcard and a specific resource are included, the wildcard takes precedence", restore: defaultRestore().IncludedResources("*", "pods").Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("pods", builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result(), ). AddItems("deployments.apps", builder.ForDeployment("ns-1", "deploy-1").Result(), builder.ForDeployment("ns-2", "deploy-2").Result(), ). AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result(), ). Done(), apiResources: []*test.APIResource{ test.Pods(), test.Deployments(), test.PVs(), }, want: map[*test.APIResource][]string{ test.Pods(): {"ns-1/pod-1", "ns-2/pod-2"}, test.Deployments(): {"ns-1/deploy-1", "ns-2/deploy-2"}, test.PVs(): {"/pv-1", "/pv-2"}, }, }, { name: "wildcard excludes are ignored", restore: defaultRestore().ExcludedResources("*").Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("pods", builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result(), ). AddItems("deployments.apps", builder.ForDeployment("ns-1", "deploy-1").Result(), builder.ForDeployment("ns-2", "deploy-2").Result(), ). AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result(), ). Done(), apiResources: []*test.APIResource{ test.Pods(), test.Deployments(), test.PVs(), }, want: map[*test.APIResource][]string{ test.Pods(): {"ns-1/pod-1", "ns-2/pod-2"}, test.Deployments(): {"ns-1/deploy-1", "ns-2/deploy-2"}, test.PVs(): {"/pv-1", "/pv-2"}, }, }, { name: "unresolvable included resources are ignored", restore: defaultRestore().IncludedResources("pods", "unresolvable").Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("pods", builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result(), ). AddItems("deployments.apps", builder.ForDeployment("ns-1", "deploy-1").Result(), builder.ForDeployment("ns-2", "deploy-2").Result(), ). AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result(), ). Done(), apiResources: []*test.APIResource{ test.Pods(), test.Deployments(), test.PVs(), }, want: map[*test.APIResource][]string{ test.Pods(): {"ns-1/pod-1", "ns-2/pod-2"}, }, }, { name: "unresolvable excluded resources are ignored", restore: defaultRestore().ExcludedResources("deployments", "unresolvable").Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("pods", builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result(), ). AddItems("deployments.apps", builder.ForDeployment("ns-1", "deploy-1").Result(), builder.ForDeployment("ns-2", "deploy-2").Result(), ). AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result(), ). Done(), apiResources: []*test.APIResource{ test.Pods(), test.Deployments(), test.PVs(), }, want: map[*test.APIResource][]string{ test.Pods(): {"ns-1/pod-1", "ns-2/pod-2"}, test.PVs(): {"/pv-1", "/pv-2"}, }, }, { name: "mirror pods are not restored", restore: defaultRestore().Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t).AddItems("pods", builder.ForPod("ns-1", "pod-1").ObjectMeta(builder.WithAnnotations(corev1api.MirrorPodAnnotationKey, "foo")).Result()).Done(), apiResources: []*test.APIResource{test.Pods()}, want: map[*test.APIResource][]string{test.Pods(): {}}, }, { name: "service accounts are restored", restore: defaultRestore().Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t).AddItems("serviceaccounts", builder.ForServiceAccount("ns-1", "sa-1").Result()).Done(), apiResources: []*test.APIResource{test.ServiceAccounts()}, want: map[*test.APIResource][]string{test.ServiceAccounts(): {"ns-1/sa-1"}}, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { h := newHarness(t) for _, r := range tc.apiResources { h.DiscoveryClient.WithAPIResource(r) } require.NoError(t, h.restorer.discoveryHelper.Refresh()) data := &Request{ Log: h.log, Restore: tc.restore, Backup: tc.backup, PodVolumeBackups: nil, VolumeSnapshots: nil, BackupReader: tc.tarball, } warnings, errs := h.restorer.Restore( data, nil, // restoreItemActions nil, // volume snapshotter getter ) assertEmptyResults(t, warnings, errs) assertAPIContents(t, h, tc.want) }) } } // TestRestoreNamespaceMapping runs restores with namespace mappings specified, // and verifies that the set of items created in the API are in the correct // namespaces. Validation is done by looking at the namespaces/names of the items // in the API; contents are not checked. func TestRestoreNamespaceMapping(t *testing.T) { tests := []struct { name string restore *velerov1api.Restore backup *velerov1api.Backup apiResources []*test.APIResource tarball io.Reader want map[*test.APIResource][]string }{ { name: "namespace mappings are applied", restore: defaultRestore().NamespaceMappings("ns-1", "mapped-ns-1", "ns-2", "mapped-ns-2").Result(), backup: defaultBackup().Result(), apiResources: []*test.APIResource{ test.Pods(), }, tarball: test.NewTarWriter(t). AddItems("pods", builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result(), builder.ForPod("ns-3", "pod-3").Result(), ). Done(), want: map[*test.APIResource][]string{ test.Pods(): {"mapped-ns-1/pod-1", "mapped-ns-2/pod-2", "ns-3/pod-3"}, }, }, { name: "namespace mappings are applied when IncludedNamespaces are specified", restore: defaultRestore().IncludedNamespaces("ns-1", "ns-2").NamespaceMappings("ns-1", "mapped-ns-1", "ns-2", "mapped-ns-2").Result(), backup: defaultBackup().Result(), apiResources: []*test.APIResource{ test.Pods(), }, tarball: test.NewTarWriter(t). AddItems("pods", builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result(), builder.ForPod("ns-3", "pod-3").Result(), ). Done(), want: map[*test.APIResource][]string{ test.Pods(): {"mapped-ns-1/pod-1", "mapped-ns-2/pod-2"}, }, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { h := newHarness(t) for _, r := range tc.apiResources { h.DiscoveryClient.WithAPIResource(r) } require.NoError(t, h.restorer.discoveryHelper.Refresh()) data := &Request{ Log: h.log, Restore: tc.restore, Backup: tc.backup, PodVolumeBackups: nil, VolumeSnapshots: nil, BackupReader: tc.tarball, } warnings, errs := h.restorer.Restore( data, nil, // restoreItemActions nil, // volume snapshotter getter ) assertEmptyResults(t, warnings, errs) assertAPIContents(t, h, tc.want) }) } } // TestRestoreResourcePriorities runs restores with resource priorities specified, // and verifies that the set of items created in the API are created in the expected // order. Validation is done by adding a Reactor to the fake dynamic client that records // resource identifiers as they're created, and comparing that to the expected order. func TestRestoreResourcePriorities(t *testing.T) { tests := []struct { name string restore *velerov1api.Restore backup *velerov1api.Backup apiResources []*test.APIResource tarball io.Reader resourcePriorities types.Priorities }{ { name: "resources are restored according to the specified resource priorities", restore: defaultRestore().Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("pods", builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result(), ). AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result(), ). AddItems("deployments.apps", builder.ForDeployment("ns-1", "deploy-1").Result(), builder.ForDeployment("ns-2", "deploy-2").Result(), ). AddItems("serviceaccounts", builder.ForServiceAccount("ns-1", "sa-1").Result(), builder.ForServiceAccount("ns-2", "sa-2").Result(), ). AddItems("persistentvolumeclaims", builder.ForPersistentVolumeClaim("ns-1", "pvc-1").Result(), builder.ForPersistentVolumeClaim("ns-2", "pvc-2").Result(), ). Done(), apiResources: []*test.APIResource{ test.Pods(), test.PVs(), test.Deployments(), test.ServiceAccounts(), }, resourcePriorities: types.Priorities{ HighPriorities: []string{"persistentvolumes", "persistentvolumeclaims", "serviceaccounts"}, LowPriorities: []string{"deployments.apps"}, }, }, } for _, tc := range tests { h := newHarness(t) h.restorer.resourcePriorities = tc.resourcePriorities recorder := &createRecorder{t: t} h.DynamicClient.PrependReactor("create", "*", recorder.reactor()) for _, r := range tc.apiResources { h.DiscoveryClient.WithAPIResource(r) } require.NoError(t, h.restorer.discoveryHelper.Refresh()) data := &Request{ Log: h.log, Restore: tc.restore, Backup: tc.backup, PodVolumeBackups: nil, VolumeSnapshots: nil, BackupReader: tc.tarball, } warnings, errs := h.restorer.Restore( data, nil, // restoreItemActions nil, // volume snapshotter getter ) assertEmptyResults(t, warnings, errs) assertResourceCreationOrder(t, []string{"persistentvolumes", "persistentvolumeclaims", "serviceaccounts", "pods", "deployments.apps"}, recorder.resources) } } // TestInvalidTarballContents runs restores for tarballs that are invalid in some way, and // verifies that the set of items created in the API and the errors returned are correct. // Validation is done by looking at the namespaces/names of the items in the API and the // Result objects returned from the restorer. func TestInvalidTarballContents(t *testing.T) { tests := []struct { name string restore *velerov1api.Restore backup *velerov1api.Backup apiResources []*test.APIResource tarball io.Reader want map[*test.APIResource][]string wantErrs Result wantWarnings Result }{ { name: "empty tarball returns a warning", restore: defaultRestore().Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). Done(), wantWarnings: Result{ Velero: []string{archive.ErrNotExist.Error()}, }, }, { name: "invalid JSON is reported as an error and restore continues", restore: defaultRestore().Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). Add("resources/pods/namespaces/ns-1/pod-1.json", []byte("invalid JSON")). AddItems("pods", builder.ForPod("ns-1", "pod-2").Result(), ). Done(), apiResources: []*test.APIResource{ test.Pods(), }, want: map[*test.APIResource][]string{ test.Pods(): {"ns-1/pod-2"}, }, wantErrs: Result{ Namespaces: map[string][]string{ "ns-1": {"error decoding"}, }, }, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { h := newHarness(t) for _, r := range tc.apiResources { h.DiscoveryClient.WithAPIResource(r) } require.NoError(t, h.restorer.discoveryHelper.Refresh()) data := &Request{ Log: h.log, Restore: tc.restore, Backup: tc.backup, PodVolumeBackups: nil, VolumeSnapshots: nil, BackupReader: tc.tarball, } warnings, errs := h.restorer.Restore( data, nil, // restoreItemActions nil, // volume snapshotter getter ) assertWantErrsOrWarnings(t, tc.wantWarnings, warnings) assertWantErrsOrWarnings(t, tc.wantErrs, errs) assertAPIContents(t, h, tc.want) }) } } func assertWantErrsOrWarnings(t *testing.T, wantRes Result, res Result) { t.Helper() if wantRes.Velero != nil { assert.Len(t, res.Velero, len(wantRes.Velero)) for i := range res.Velero { assert.Contains(t, res.Velero[i], wantRes.Velero[i]) } } if wantRes.Namespaces != nil { assert.Len(t, res.Namespaces, len(wantRes.Namespaces)) for ns := range res.Namespaces { assert.Len(t, res.Namespaces[ns], len(wantRes.Namespaces[ns])) for i := range res.Namespaces[ns] { assert.Contains(t, res.Namespaces[ns][i], wantRes.Namespaces[ns][i]) } } } if wantRes.Cluster != nil { assert.Equal(t, wantRes.Cluster, res.Cluster) } } // TestRestoreItems runs restores of specific items and validates that they are created // with the expected metadata/spec/status in the API. func TestRestoreItems(t *testing.T) { tests := []struct { name string restore *velerov1api.Restore backup *velerov1api.Backup apiResources []*test.APIResource tarball io.Reader want []*test.APIResource expectedRestoreItems map[itemKey]restoredItemStatus disableInformer bool }{ { name: "metadata uid/resourceVersion/etc. gets removed", restore: defaultRestore().Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("pods", builder.ForPod("ns-1", "pod-1"). ObjectMeta( builder.WithLabels("key-1", "val-1"), builder.WithAnnotations("key-1", "val-1"), builder.WithFinalizers("finalizer-1"), builder.WithUID("uid"), ). Result(), ). Done(), apiResources: []*test.APIResource{ test.Pods(), }, want: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1"). ObjectMeta( builder.WithLabels("key-1", "val-1", "velero.io/backup-name", "backup-1", "velero.io/restore-name", "restore-1"), builder.WithAnnotations("key-1", "val-1"), builder.WithFinalizers("finalizer-1"), ). Result(), ), }, expectedRestoreItems: map[itemKey]restoredItemStatus{ {resource: "v1/Namespace", namespace: "", name: "ns-1"}: {action: "created", itemExists: true, createdName: "ns-1"}, {resource: "v1/Pod", namespace: "ns-1", name: "pod-1"}: {action: "created", itemExists: true, createdName: "pod-1"}, }, }, { name: "status gets removed", restore: defaultRestore().Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("pods", &corev1api.Pod{ TypeMeta: metav1.TypeMeta{ APIVersion: "v1", Kind: "Pod", }, ObjectMeta: metav1.ObjectMeta{ Namespace: "ns-1", Name: "pod-1", }, Status: corev1api.PodStatus{ Message: "a non-empty status", }, }, ). Done(), apiResources: []*test.APIResource{ test.Pods(), }, want: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").ObjectMeta(builder.WithLabels("velero.io/backup-name", "backup-1", "velero.io/restore-name", "restore-1")).Result(), ), }, }, { name: "object gets labeled with full backup and restore names when they're both shorter than 63 characters", restore: defaultRestore().Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("pods", builder.ForPod("ns-1", "pod-1").Result()). Done(), apiResources: []*test.APIResource{ test.Pods(), }, want: []*test.APIResource{ test.Pods(builder.ForPod("ns-1", "pod-1").ObjectMeta(builder.WithLabels("velero.io/backup-name", "backup-1", "velero.io/restore-name", "restore-1")).Result()), }, }, { name: "object gets labeled with full backup and restore names when they're both equal to 63 characters", restore: builder.ForRestore(velerov1api.DefaultNamespace, "the-really-long-kube-service-name-that-is-exactly-63-characters"). Backup("the-really-long-kube-service-name-that-is-exactly-63-characters"). Result(), backup: builder.ForBackup(velerov1api.DefaultNamespace, "the-really-long-kube-service-name-that-is-exactly-63-characters").Result(), tarball: test.NewTarWriter(t). AddItems("pods", builder.ForPod("ns-1", "pod-1").Result()). Done(), apiResources: []*test.APIResource{ test.Pods(), }, want: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1"). ObjectMeta( builder.WithLabels( "velero.io/backup-name", "the-really-long-kube-service-name-that-is-exactly-63-characters", "velero.io/restore-name", "the-really-long-kube-service-name-that-is-exactly-63-characters", ), ).Result(), ), }, }, { name: "object gets labeled with shortened backup and restore names when they're both longer than 63 characters", restore: builder.ForRestore(velerov1api.DefaultNamespace, "the-really-long-kube-service-name-that-is-much-greater-than-63-characters"). Backup("the-really-long-kube-service-name-that-is-much-greater-than-63-characters"). Result(), backup: builder.ForBackup(velerov1api.DefaultNamespace, "the-really-long-kube-service-name-that-is-much-greater-than-63-characters").Result(), tarball: test.NewTarWriter(t). AddItems("pods", builder.ForPod("ns-1", "pod-1").Result()). Done(), apiResources: []*test.APIResource{ test.Pods(), }, want: []*test.APIResource{ test.Pods(builder.ForPod("ns-1", "pod-1"). ObjectMeta( builder.WithLabels( "velero.io/backup-name", "the-really-long-kube-service-name-that-is-much-greater-th8a11b3", "velero.io/restore-name", "the-really-long-kube-service-name-that-is-much-greater-th8a11b3", ), ). Result(), ), }, }, { name: "no error when service account already exists in cluster and is identical to the backed up one", restore: defaultRestore().Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("serviceaccounts", builder.ForServiceAccount("ns-1", "sa-1").Result()). Done(), apiResources: []*test.APIResource{ test.ServiceAccounts(builder.ForServiceAccount("ns-1", "sa-1").Result()), }, want: []*test.APIResource{ test.ServiceAccounts(builder.ForServiceAccount("ns-1", "sa-1").Result()), }, expectedRestoreItems: map[itemKey]restoredItemStatus{ {resource: "v1/Namespace", namespace: "", name: "ns-1"}: {action: "created", itemExists: true, createdName: "ns-1"}, {resource: "v1/ServiceAccount", namespace: "ns-1", name: "sa-1"}: {action: "skipped", itemExists: true}, }, }, { name: "update secret data and labels when secret exists in cluster and is not identical to the backed up one, existing resource policy is update", restore: defaultRestore().ExistingResourcePolicy("update").Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("secrets", builder.ForSecret("ns-1", "sa-1").Data(map[string][]byte{"key-1": []byte("value-1")}).Result()). Done(), apiResources: []*test.APIResource{ test.Secrets(builder.ForSecret("ns-1", "sa-1").Data(map[string][]byte{"foo": []byte("bar")}).Result()), }, disableInformer: true, want: []*test.APIResource{ test.Secrets(builder.ForSecret("ns-1", "sa-1").ObjectMeta(builder.WithLabels("velero.io/backup-name", "backup-1", "velero.io/restore-name", "restore-1")).Data(map[string][]byte{"key-1": []byte("value-1")}).Result()), }, expectedRestoreItems: map[itemKey]restoredItemStatus{ {resource: "v1/Namespace", namespace: "", name: "ns-1"}: {action: "created", itemExists: true, createdName: "ns-1"}, {resource: "v1/Secret", namespace: "ns-1", name: "sa-1"}: {action: "updated", itemExists: true}, }, }, { name: "update secret data and labels when secret exists in cluster and is not identical to the backed up one, existing resource policy is update, using informer cache", restore: defaultRestore().ExistingResourcePolicy("update").Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("secrets", builder.ForSecret("ns-1", "sa-1").Data(map[string][]byte{"key-1": []byte("value-1")}).Result()). Done(), apiResources: []*test.APIResource{ test.Secrets(builder.ForSecret("ns-1", "sa-1").Data(map[string][]byte{"foo": []byte("bar")}).Result()), }, disableInformer: false, want: []*test.APIResource{ test.Secrets(builder.ForSecret("ns-1", "sa-1").ObjectMeta(builder.WithLabels("velero.io/backup-name", "backup-1", "velero.io/restore-name", "restore-1")).Data(map[string][]byte{"key-1": []byte("value-1")}).Result()), }, expectedRestoreItems: map[itemKey]restoredItemStatus{ {resource: "v1/Namespace", namespace: "", name: "ns-1"}: {action: "created", itemExists: true, createdName: "ns-1"}, {resource: "v1/Secret", namespace: "ns-1", name: "sa-1"}: {action: "updated", itemExists: true}, }, }, { name: "update service account labels when service account exists in cluster and is identical to the backed up one, existing resource policy is update", restore: defaultRestore().ExistingResourcePolicy("update").Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("serviceaccounts", builder.ForServiceAccount("ns-1", "sa-1").Result()). Done(), apiResources: []*test.APIResource{ test.ServiceAccounts(builder.ForServiceAccount("ns-1", "sa-1").ObjectMeta(builder.WithLabels("velero.io/backup-name", "foo", "velero.io/restore-name", "bar")).Result()), }, want: []*test.APIResource{ test.ServiceAccounts(builder.ForServiceAccount("ns-1", "sa-1").ObjectMeta(builder.WithLabels("velero.io/backup-name", "backup-1", "velero.io/restore-name", "restore-1")).Result()), }, }, { name: "update pod labels when pod exists in cluster and is identical to the backed up one, existing resource policy is update", restore: defaultRestore().ExistingResourcePolicy("update").Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("pods", builder.ForPod("ns-1", "sa-1").Result()). Done(), apiResources: []*test.APIResource{ test.Pods(builder.ForPod("ns-1", "sa-1").ObjectMeta(builder.WithLabels("velero.io/backup-name", "foo", "velero.io/restore-name", "bar")).Result()), }, want: []*test.APIResource{ test.Pods(builder.ForPod("ns-1", "sa-1").ObjectMeta(builder.WithLabels("velero.io/backup-name", "backup-1", "velero.io/restore-name", "restore-1")).Result()), }, }, { name: "do not update pod labels when pod exists in cluster and is identical to the backed up one, existing resource policy is none", restore: defaultRestore().ExistingResourcePolicy("none").Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("pods", builder.ForPod("ns-1", "sa-1").Result()). Done(), apiResources: []*test.APIResource{ test.Pods(builder.ForPod("ns-1", "sa-1").ObjectMeta(builder.WithLabels("velero.io/backup-name", "foo", "velero.io/restore-name", "bar")).Result()), }, want: []*test.APIResource{ test.Pods(builder.ForPod("ns-1", "sa-1").ObjectMeta(builder.WithLabels("velero.io/backup-name", "foo", "velero.io/restore-name", "bar")).Result()), }, }, { name: "do not update pod labels when pod exists in cluster and is identical to the backed up one, existing resource policy is not specified, velero behavior is preserved", restore: defaultRestore().Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("pods", builder.ForPod("ns-1", "sa-1").Result()). Done(), apiResources: []*test.APIResource{ test.Pods(builder.ForPod("ns-1", "sa-1").ObjectMeta(builder.WithLabels("velero.io/backup-name", "foo", "velero.io/restore-name", "bar")).Result()), }, want: []*test.APIResource{ test.Pods(builder.ForPod("ns-1", "sa-1").ObjectMeta(builder.WithLabels("velero.io/backup-name", "foo", "velero.io/restore-name", "bar")).Result()), }, }, { name: "service account secrets and image pull secrets are restored when service account already exists in cluster", restore: defaultRestore().Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("serviceaccounts", &corev1api.ServiceAccount{ TypeMeta: metav1.TypeMeta{ APIVersion: "v1", Kind: "ServiceAccount", }, ObjectMeta: metav1.ObjectMeta{ Namespace: "ns-1", Name: "sa-1", }, Secrets: []corev1api.ObjectReference{{Name: "secret-1"}}, ImagePullSecrets: []corev1api.LocalObjectReference{{Name: "pull-secret-1"}}, }). Done(), apiResources: []*test.APIResource{ test.ServiceAccounts(builder.ForServiceAccount("ns-1", "sa-1").Result()), }, want: []*test.APIResource{ test.ServiceAccounts(&corev1api.ServiceAccount{ TypeMeta: metav1.TypeMeta{ APIVersion: "v1", Kind: "ServiceAccount", }, ObjectMeta: metav1.ObjectMeta{ Namespace: "ns-1", Name: "sa-1", }, Secrets: []corev1api.ObjectReference{{Name: "secret-1"}}, ImagePullSecrets: []corev1api.LocalObjectReference{{Name: "pull-secret-1"}}, }), }, }, { name: "metadata managedFields gets restored", restore: defaultRestore().Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("pods", builder.ForPod("ns-1", "pod-1"). ObjectMeta( builder.WithManagedFields([]metav1.ManagedFieldsEntry{ { Manager: "kubectl", Operation: "Apply", APIVersion: "v1", FieldsType: "FieldsV1", FieldsV1: &metav1.FieldsV1{ Raw: []byte(`{"f:data": {"f:key":{}}}`), }, }, }), ). Result(), ). Done(), apiResources: []*test.APIResource{ test.Pods(), }, want: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1"). ObjectMeta( builder.WithLabels("velero.io/backup-name", "backup-1", "velero.io/restore-name", "restore-1"), builder.WithManagedFields([]metav1.ManagedFieldsEntry{ { Manager: "kubectl", Operation: "Apply", APIVersion: "v1", FieldsType: "FieldsV1", FieldsV1: &metav1.FieldsV1{ Raw: []byte(`{"f:data": {"f:key":{}}}`), }, }, }), ). Result(), ), }, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { h := newHarness(t) for _, r := range tc.apiResources { h.AddItems(t, r) } data := &Request{ Log: h.log, Restore: tc.restore, Backup: tc.backup, PodVolumeBackups: nil, VolumeSnapshots: nil, BackupReader: tc.tarball, RestoredItems: map[itemKey]restoredItemStatus{}, DisableInformerCache: tc.disableInformer, } warnings, errs := h.restorer.Restore( data, nil, // restoreItemActions nil, // volume snapshotter getter ) assertEmptyResults(t, warnings, errs) assertRestoredItems(t, h, tc.want) if len(tc.expectedRestoreItems) > 0 { assert.Equal(t, tc.expectedRestoreItems, data.RestoredItems) } }) } } // recordResourcesAction is a restore item action that can be configured // to run for specific resources/namespaces and simply records the items // that it is executed for. type recordResourcesAction struct { selector velero.ResourceSelector ids []string additionalItems []velero.ResourceIdentifier operationID string waitForAdditionalItems bool additionalItemsReadyTimeout time.Duration } func (a *recordResourcesAction) Name() string { return "" } func (a *recordResourcesAction) AppliesTo() (velero.ResourceSelector, error) { return a.selector, nil } func (a *recordResourcesAction) Execute(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { metadata, err := meta.Accessor(input.Item) if err != nil { return &velero.RestoreItemActionExecuteOutput{ UpdatedItem: input.Item, AdditionalItems: a.additionalItems, OperationID: a.operationID, WaitForAdditionalItems: a.waitForAdditionalItems, AdditionalItemsReadyTimeout: a.additionalItemsReadyTimeout, }, err } a.ids = append(a.ids, kubeutil.NamespaceAndName(metadata)) return &velero.RestoreItemActionExecuteOutput{ UpdatedItem: input.Item, AdditionalItems: a.additionalItems, OperationID: a.operationID, WaitForAdditionalItems: a.waitForAdditionalItems, AdditionalItemsReadyTimeout: a.additionalItemsReadyTimeout, }, nil } func (a *recordResourcesAction) Progress(operationID string, restore *velerov1api.Restore) (velero.OperationProgress, error) { return velero.OperationProgress{}, nil } func (a *recordResourcesAction) Cancel(operationID string, restore *velerov1api.Restore) error { return nil } func (a *recordResourcesAction) AreAdditionalItemsReady(additionalItems []velero.ResourceIdentifier, restore *velerov1api.Restore) (bool, error) { return true, nil } func (a *recordResourcesAction) ForResource(resource string) *recordResourcesAction { a.selector.IncludedResources = append(a.selector.IncludedResources, resource) return a } func (a *recordResourcesAction) ForNamespace(namespace string) *recordResourcesAction { a.selector.IncludedNamespaces = append(a.selector.IncludedNamespaces, namespace) return a } func (a *recordResourcesAction) ForLabelSelector(selector string) *recordResourcesAction { a.selector.LabelSelector = selector return a } func (a *recordResourcesAction) WithAdditionalItems(items []velero.ResourceIdentifier) *recordResourcesAction { a.additionalItems = items return a } // TestRestoreActionsRunForCorrectItems runs restores with restore item actions, and // verifies that each restore item action is run for the correct set of resources based on its // AppliesTo() resource selector. Verification is done by using the recordResourcesAction struct, // which records which resources it's executed for. func TestRestoreActionsRunForCorrectItems(t *testing.T) { tests := []struct { name string restore *velerov1api.Restore backup *velerov1api.Backup apiResources []*test.APIResource tarball io.Reader actions map[*recordResourcesAction][]string }{ { name: "single action with no selector runs for all items", restore: defaultRestore().Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("pods", builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result()). AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result()). Done(), apiResources: []*test.APIResource{test.Pods(), test.PVs()}, actions: map[*recordResourcesAction][]string{ new(recordResourcesAction): {"ns-1/pod-1", "ns-2/pod-2", "pv-1", "pv-2"}, }, }, { name: "single action with a resource selector for namespaced resources runs only for matching resources", restore: defaultRestore().Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("pods", builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result()). AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result()). Done(), apiResources: []*test.APIResource{test.Pods(), test.PVs()}, actions: map[*recordResourcesAction][]string{ new(recordResourcesAction).ForResource("pods"): {"ns-1/pod-1", "ns-2/pod-2"}, }, }, { name: "single action with a resource selector for cluster-scoped resources runs only for matching resources", restore: defaultRestore().Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("pods", builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result()). AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result()). Done(), apiResources: []*test.APIResource{test.Pods(), test.PVs()}, actions: map[*recordResourcesAction][]string{ new(recordResourcesAction).ForResource("persistentvolumes"): {"pv-1", "pv-2"}, }, }, { name: "single action with a namespace selector runs only for resources in that namespace", restore: defaultRestore().Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("pods", builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result()). AddItems("persistentvolumeclaims", builder.ForPersistentVolumeClaim("ns-1", "pvc-1").Result(), builder.ForPersistentVolumeClaim("ns-2", "pvc-2").Result()). AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result()). Done(), apiResources: []*test.APIResource{test.Pods(), test.PVCs(), test.PVs()}, actions: map[*recordResourcesAction][]string{ new(recordResourcesAction).ForNamespace("ns-1"): {"ns-1/pod-1", "ns-1/pvc-1"}, }, }, { name: "single action with a resource and namespace selector runs only for matching resources in that namespace", restore: defaultRestore().Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("pods", builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result()). AddItems("persistentvolumeclaims", builder.ForPersistentVolumeClaim("ns-1", "pvc-1").Result(), builder.ForPersistentVolumeClaim("ns-2", "pvc-2").Result()). AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result()). Done(), apiResources: []*test.APIResource{test.Pods(), test.PVCs(), test.PVs()}, actions: map[*recordResourcesAction][]string{ new(recordResourcesAction).ForNamespace("ns-1").ForResource("pods"): {"ns-1/pod-1"}, }, }, { name: "single action with a resource and label selector runs only for resources matching that label", restore: defaultRestore().Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("pods", builder.ForPod("ns-1", "pod-1").ObjectMeta(builder.WithLabels("restore-resource", "true")).Result(), builder.ForPod("ns-1", "pod-2").ObjectMeta(builder.WithLabels("do-not-restore-resource", "true")).Result(), builder.ForPod("ns-2", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").ObjectMeta(builder.WithLabels("restore-resource")).Result(), ).Done(), apiResources: []*test.APIResource{test.Pods()}, actions: map[*recordResourcesAction][]string{ new(recordResourcesAction).ForResource("pods").ForLabelSelector("restore-resource"): {"ns-1/pod-1", "ns-2/pod-2"}, }, }, { name: "multiple actions, each with a different resource selector using short name, run for matching resources", restore: defaultRestore().Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("pods", builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result()). AddItems("persistentvolumeclaims", builder.ForPersistentVolumeClaim("ns-1", "pvc-1").Result(), builder.ForPersistentVolumeClaim("ns-2", "pvc-2").Result()). AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").Result(), builder.ForPersistentVolume("pv-2").Result()). Done(), apiResources: []*test.APIResource{test.Pods(), test.PVCs(), test.PVs()}, actions: map[*recordResourcesAction][]string{ new(recordResourcesAction).ForResource("po"): {"ns-1/pod-1", "ns-2/pod-2"}, new(recordResourcesAction).ForResource("pv"): {"pv-1", "pv-2"}, }, }, { name: "actions with selectors that don't match anything don't run for any resources", restore: defaultRestore().Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("pods", builder.ForPod("ns-1", "pod-1").Result()). AddItems("persistentvolumeclaims", builder.ForPersistentVolumeClaim("ns-2", "pvc-2").Result()). Done(), apiResources: []*test.APIResource{test.Pods(), test.PVCs(), test.PVs()}, actions: map[*recordResourcesAction][]string{ new(recordResourcesAction).ForNamespace("ns-1").ForResource("persistentvolumeclaims"): nil, new(recordResourcesAction).ForNamespace("ns-2").ForResource("pods"): nil, }, }, { name: "actions run for datauploads resource", restore: defaultRestore().Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("datauploads.velero.io", builder.ForDataUpload("velero", "du").Result()). Done(), apiResources: []*test.APIResource{test.DataUploads()}, actions: map[*recordResourcesAction][]string{ new(recordResourcesAction).ForNamespace("velero").ForResource("datauploads.velero.io"): {"velero/du"}, }, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { h := newHarness(t) for _, r := range tc.apiResources { h.AddItems(t, r) } actions := []riav2.RestoreItemAction{} for action := range tc.actions { actions = append(actions, action) } data := &Request{ Log: h.log, Restore: tc.restore, Backup: tc.backup, PodVolumeBackups: nil, VolumeSnapshots: nil, BackupReader: tc.tarball, } warnings, errs := h.restorer.Restore( data, actions, nil, // volume snapshotter getter ) assertEmptyResults(t, warnings, errs) for action, want := range tc.actions { sort.Strings(want) sort.Strings(action.ids) assert.Equal(t, want, action.ids) } }) } } // pluggableAction is a restore item action that can be plugged with an Execute // function body at runtime. type pluggableAction struct { selector velero.ResourceSelector executeFunc func(*velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) progressFunc func(string, *velerov1api.Restore) (velero.OperationProgress, error) } func (a *pluggableAction) Execute(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { if a.executeFunc == nil { return &velero.RestoreItemActionExecuteOutput{ UpdatedItem: input.Item, }, nil } return a.executeFunc(input) } func (a *pluggableAction) Name() string { return "" } func (a *pluggableAction) AppliesTo() (velero.ResourceSelector, error) { return a.selector, nil } func (a *pluggableAction) Progress(operationID string, restore *velerov1api.Restore) (velero.OperationProgress, error) { return velero.OperationProgress{}, nil } func (a *pluggableAction) Cancel(operationID string, restore *velerov1api.Restore) error { return nil } func (a *pluggableAction) addSelector(selector velero.ResourceSelector) *pluggableAction { a.selector = selector return a } func (a *pluggableAction) AreAdditionalItemsReady(additionalItems []velero.ResourceIdentifier, restore *velerov1api.Restore) (bool, error) { return true, nil } // TestRestoreActionModifications runs restores with restore item actions that modify resources, and // verifies that the modified item is correctly created in the API. Verification is done by looking // at the full object in the API. func TestRestoreActionModifications(t *testing.T) { // modifyingActionGetter is a helper function that returns a *pluggableAction, whose Execute(...) // method modifies the item being passed in by calling the 'modify' function on it. modifyingActionGetter := func(modify func(*unstructured.Unstructured)) *pluggableAction { return &pluggableAction{ executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { obj, ok := input.Item.(*unstructured.Unstructured) if !ok { return nil, errors.Errorf("unexpected type %T", input.Item) } res := obj.DeepCopy() modify(res) return &velero.RestoreItemActionExecuteOutput{ UpdatedItem: res, }, nil }, } } tests := []struct { name string restore *velerov1api.Restore backup *velerov1api.Backup apiResources []*test.APIResource tarball io.Reader actions []riav2.RestoreItemAction want []*test.APIResource }{ { name: "action that adds a label to item gets restored", restore: defaultRestore().Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t).AddItems("pods", builder.ForPod("ns-1", "pod-1").Result()).Done(), apiResources: []*test.APIResource{test.Pods()}, actions: []riav2.RestoreItemAction{ modifyingActionGetter(func(item *unstructured.Unstructured) { item.SetLabels(map[string]string{"updated": "true"}) }), }, want: []*test.APIResource{ test.Pods( builder.ForPod("ns-1", "pod-1").ObjectMeta(builder.WithLabels("updated", "true")).Result(), ), }, }, { name: "action that removes a label to item gets restored", restore: defaultRestore().Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t).AddItems("pods", builder.ForPod("ns-1", "pod-1").ObjectMeta(builder.WithLabels("should-be-removed", "true")).Result()).Done(), apiResources: []*test.APIResource{test.Pods()}, actions: []riav2.RestoreItemAction{ modifyingActionGetter(func(item *unstructured.Unstructured) { item.SetLabels(nil) }), }, want: []*test.APIResource{ test.Pods(builder.ForPod("ns-1", "pod-1").Result()), }, }, { name: "action with non-matching label selector doesn't prevent restore", restore: defaultRestore().Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t).AddItems("pods", builder.ForPod("ns-1", "pod-1").Result()).Done(), apiResources: []*test.APIResource{test.Pods()}, actions: []riav2.RestoreItemAction{ modifyingActionGetter(func(item *unstructured.Unstructured) { item.SetLabels(map[string]string{"updated": "true"}) }).addSelector(velero.ResourceSelector{ IncludedResources: []string{ "Pod", }, LabelSelector: "nonmatching=label", }), }, want: []*test.APIResource{ test.Pods(builder.ForPod("ns-1", "pod-1").Result()), }, }, // TODO action that modifies namespace/name - what's the expected behavior? } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { h := newHarness(t) for _, r := range tc.apiResources { h.AddItems(t, r) } // every restored item should have the restore and backup name labels, set // them here so we don't have to do it in every test case definition above. for _, resource := range tc.want { for _, item := range resource.Items { labels := item.GetLabels() if labels == nil { labels = make(map[string]string) } labels["velero.io/restore-name"] = tc.restore.Name labels["velero.io/backup-name"] = tc.restore.Spec.BackupName item.SetLabels(labels) } } data := &Request{ Log: h.log, Restore: tc.restore, Backup: tc.backup, PodVolumeBackups: nil, VolumeSnapshots: nil, BackupReader: tc.tarball, } warnings, errs := h.restorer.Restore( data, tc.actions, nil, // volume snapshotter getter ) assertEmptyResults(t, warnings, errs) assertRestoredItems(t, h, tc.want) }) } } // TestRestoreWithAsyncOperations runs restores which return operationIDs and // verifies that the itemoperations are tracked as appropriate. Verification is done by // looking at the restore request's itemOperationsList field. func TestRestoreWithAsyncOperations(t *testing.T) { // completedOperationAction is a *pluggableAction, whose Execute(...) // method returns an operationID which will always be done when calling Progress. completedOperationAction := &pluggableAction{ executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { obj, ok := input.Item.(*unstructured.Unstructured) if !ok { return nil, errors.Errorf("unexpected type %T", input.Item) } return &velero.RestoreItemActionExecuteOutput{ UpdatedItem: obj, OperationID: obj.GetName() + "-1", }, nil }, progressFunc: func(operationID string, restore *velerov1api.Restore) (velero.OperationProgress, error) { return velero.OperationProgress{ Completed: true, Description: "Done!", }, nil }, } // incompleteOperationAction is a *pluggableAction, whose Execute(...) // method returns an operationID which will never be done when calling Progress. incompleteOperationAction := &pluggableAction{ executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { obj, ok := input.Item.(*unstructured.Unstructured) if !ok { return nil, errors.Errorf("unexpected type %T", input.Item) } return &velero.RestoreItemActionExecuteOutput{ UpdatedItem: obj, OperationID: obj.GetName() + "-1", }, nil }, progressFunc: func(operationID string, restore *velerov1api.Restore) (velero.OperationProgress, error) { return velero.OperationProgress{ Completed: false, Description: "Working...", }, nil }, } // noOperationAction is a *pluggableAction, whose Execute(...) // method does not return an operationID. noOperationAction := &pluggableAction{ executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { obj, ok := input.Item.(*unstructured.Unstructured) if !ok { return nil, errors.Errorf("unexpected type %T", input.Item) } return &velero.RestoreItemActionExecuteOutput{ UpdatedItem: obj, }, nil }, } tests := []struct { name string restore *velerov1api.Restore backup *velerov1api.Backup apiResources []*test.APIResource tarball io.Reader actions []riav2.RestoreItemAction want []*itemoperation.RestoreOperation }{ { name: "action that starts a short-running process records operation", restore: defaultRestore().Result(), backup: defaultBackup().Result(), apiResources: []*test.APIResource{test.Pods()}, tarball: test.NewTarWriter(t).AddItems("pods", builder.ForPod("ns-1", "pod-1").Result()).Done(), actions: []riav2.RestoreItemAction{ completedOperationAction, }, want: []*itemoperation.RestoreOperation{ { Spec: itemoperation.RestoreOperationSpec{ RestoreName: "restore-1", ResourceIdentifier: velero.ResourceIdentifier{ GroupResource: kuberesource.Pods, Namespace: "ns-1", Name: "pod-1"}, OperationID: "pod-1-1", }, Status: itemoperation.OperationStatus{ Phase: "New", }, }, }, }, { name: "action that starts a long-running process records operation", restore: defaultRestore().Result(), backup: defaultBackup().Result(), apiResources: []*test.APIResource{test.Pods()}, tarball: test.NewTarWriter(t).AddItems("pods", builder.ForPod("ns-1", "pod-2").Result()).Done(), actions: []riav2.RestoreItemAction{ incompleteOperationAction, }, want: []*itemoperation.RestoreOperation{ { Spec: itemoperation.RestoreOperationSpec{ RestoreName: "restore-1", ResourceIdentifier: velero.ResourceIdentifier{ GroupResource: kuberesource.Pods, Namespace: "ns-1", Name: "pod-2"}, OperationID: "pod-2-1", }, Status: itemoperation.OperationStatus{ Phase: "New", }, }, }, }, { name: "action that has no operation doesn't record one", restore: defaultRestore().Result(), backup: defaultBackup().Result(), apiResources: []*test.APIResource{test.Pods()}, tarball: test.NewTarWriter(t).AddItems("pods", builder.ForPod("ns-1", "pod-3").Result()).Done(), actions: []riav2.RestoreItemAction{ noOperationAction, }, want: []*itemoperation.RestoreOperation{}, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { h := newHarness(t) for _, r := range tc.apiResources { h.AddItems(t, r) } data := &Request{ Log: h.log, Restore: tc.restore, Backup: tc.backup, PodVolumeBackups: nil, VolumeSnapshots: nil, BackupReader: tc.tarball, } warnings, errs := h.restorer.Restore( data, tc.actions, nil, // volume snapshotter getter ) assertEmptyResults(t, warnings, errs) resultOper := *data.GetItemOperationsList() // set want Created times so it won't fail the assert.Equal test for i, wantOper := range tc.want { wantOper.Status.Created = resultOper[i].Status.Created } assert.Equal(t, tc.want, *data.GetItemOperationsList()) }) } } // TestRestoreActionAdditionalItems runs restores with restore item actions that return additional items // to be restored, and verifies that the correct set of items is created in the API. Verification is // done by looking at the namespaces/names of the items in the API; contents are not checked. func TestRestoreActionAdditionalItems(t *testing.T) { tests := []struct { name string restore *velerov1api.Restore backup *velerov1api.Backup tarball io.Reader apiResources []*test.APIResource actions []riav2.RestoreItemAction want map[*test.APIResource][]string }{ { name: "additional items that are already being restored are not restored twice", restore: defaultRestore().Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t).AddItems("pods", builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result()).Done(), apiResources: []*test.APIResource{test.Pods()}, actions: []riav2.RestoreItemAction{ &pluggableAction{ selector: velero.ResourceSelector{IncludedNamespaces: []string{"ns-1"}}, executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { return &velero.RestoreItemActionExecuteOutput{ UpdatedItem: input.Item, AdditionalItems: []velero.ResourceIdentifier{ {GroupResource: kuberesource.Pods, Namespace: "ns-2", Name: "pod-2"}, }, }, nil }, }, }, want: map[*test.APIResource][]string{ test.Pods(): {"ns-1/pod-1", "ns-2/pod-2"}, }, }, { name: "when using a restore namespace filter, additional items that are in a non-included namespace are not restored", restore: defaultRestore().IncludedNamespaces("ns-1").Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t).AddItems("pods", builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result()).Done(), apiResources: []*test.APIResource{test.Pods()}, actions: []riav2.RestoreItemAction{ &pluggableAction{ executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { return &velero.RestoreItemActionExecuteOutput{ UpdatedItem: input.Item, AdditionalItems: []velero.ResourceIdentifier{ {GroupResource: kuberesource.Pods, Namespace: "ns-2", Name: "pod-2"}, }, }, nil }, }, }, want: map[*test.APIResource][]string{ test.Pods(): {"ns-1/pod-1"}, }, }, { name: "when using a restore namespace filter, additional items that are cluster-scoped are restored when IncludeClusterResources=nil", restore: defaultRestore().IncludedNamespaces("ns-1").Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("pods", builder.ForPod("ns-1", "pod-1").Result()). AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").Result()). Done(), apiResources: []*test.APIResource{test.Pods(), test.PVs()}, actions: []riav2.RestoreItemAction{ &pluggableAction{ executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { return &velero.RestoreItemActionExecuteOutput{ UpdatedItem: input.Item, AdditionalItems: []velero.ResourceIdentifier{ {GroupResource: kuberesource.PersistentVolumes, Name: "pv-1"}, }, }, nil }, }, }, want: map[*test.APIResource][]string{ test.Pods(): {"ns-1/pod-1"}, test.PVs(): {"/pv-1"}, }, }, { name: "additional items that are cluster-scoped are not restored when IncludeClusterResources=false", restore: defaultRestore().IncludeClusterResources(false).Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("pods", builder.ForPod("ns-1", "pod-1").Result()). AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").Result()). Done(), apiResources: []*test.APIResource{test.Pods(), test.PVs()}, actions: []riav2.RestoreItemAction{ &pluggableAction{ executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { return &velero.RestoreItemActionExecuteOutput{ UpdatedItem: input.Item, AdditionalItems: []velero.ResourceIdentifier{ {GroupResource: kuberesource.PersistentVolumes, Name: "pv-1"}, }, }, nil }, }, }, want: map[*test.APIResource][]string{ test.Pods(): {"ns-1/pod-1"}, test.PVs(): nil, }, }, { name: "when using a restore resource filter, additional items that are non-included resources are not restored", restore: defaultRestore().IncludedResources("pods").Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("pods", builder.ForPod("ns-1", "pod-1").Result()). AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").Result()). Done(), apiResources: []*test.APIResource{test.Pods(), test.PVs()}, actions: []riav2.RestoreItemAction{ &pluggableAction{ executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { return &velero.RestoreItemActionExecuteOutput{ UpdatedItem: input.Item, AdditionalItems: []velero.ResourceIdentifier{ {GroupResource: kuberesource.PersistentVolumes, Name: "pv-1"}, }, }, nil }, }, }, want: map[*test.APIResource][]string{ test.Pods(): {"ns-1/pod-1"}, test.PVs(): nil, }, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { h := newHarness(t) for _, r := range tc.apiResources { h.AddItems(t, r) } data := &Request{ Log: h.log, Restore: tc.restore, Backup: tc.backup, PodVolumeBackups: nil, VolumeSnapshots: nil, BackupReader: tc.tarball, } warnings, errs := h.restorer.Restore( data, tc.actions, nil, // volume snapshotter getter ) assertEmptyResults(t, warnings, errs) assertAPIContents(t, h, tc.want) }) } } // TestShouldRestore runs the ShouldRestore function for various permutations of // existing/nonexisting/being-deleted PVs, PVCs, and namespaces, and verifies the // result/error matches expectations. func TestShouldRestore(t *testing.T) { tests := []struct { name string pvName string apiResources []*test.APIResource namespaces []*corev1api.Namespace want bool wantErr error }{ { name: "when PV is not found, result is true", pvName: "pv-1", want: true, }, { name: "when PV is found and has Phase=Released, result is false", pvName: "pv-1", apiResources: []*test.APIResource{ test.PVs(&corev1api.PersistentVolume{ TypeMeta: metav1.TypeMeta{ APIVersion: "v1", Kind: "PersistentVolume", }, ObjectMeta: metav1.ObjectMeta{ Name: "pv-1", }, Status: corev1api.PersistentVolumeStatus{ Phase: corev1api.VolumeReleased, }, }), }, want: false, }, { name: "when PV is found and has associated PVC and namespace that aren't deleting, result is false", pvName: "pv-1", apiResources: []*test.APIResource{ test.PVs( builder.ForPersistentVolume("pv-1").ClaimRef("ns-1", "pvc-1").Result(), ), test.PVCs(builder.ForPersistentVolumeClaim("ns-1", "pvc-1").Result()), }, namespaces: []*corev1api.Namespace{builder.ForNamespace("ns-1").Result()}, want: false, }, { name: "when PV is found and has associated PVC that is deleting, result is false + timeout error", pvName: "pv-1", apiResources: []*test.APIResource{ test.PVs( builder.ForPersistentVolume("pv-1").ClaimRef("ns-1", "pvc-1").Result(), ), test.PVCs( builder.ForPersistentVolumeClaim("ns-1", "pvc-1").ObjectMeta(builder.WithDeletionTimestamp(time.Now())).Result(), ), }, want: false, wantErr: errors.New("context deadline exceeded"), }, { name: "when PV is found, has associated PVC that's not deleting, has associated NS that is terminating, result is false + timeout error", pvName: "pv-1", apiResources: []*test.APIResource{ test.PVs( builder.ForPersistentVolume("pv-1").ClaimRef("ns-1", "pvc-1").Result(), ), test.PVCs(builder.ForPersistentVolumeClaim("ns-1", "pvc-1").Result()), }, namespaces: []*corev1api.Namespace{ builder.ForNamespace("ns-1").Phase(corev1api.NamespaceTerminating).Result(), }, want: false, wantErr: errors.New("context deadline exceeded"), }, { name: "when PV is found, has associated PVC that's not deleting, has associated NS that has deletion timestamp, result is false + timeout error", pvName: "pv-1", apiResources: []*test.APIResource{ test.PVs( builder.ForPersistentVolume("pv-1").ClaimRef("ns-1", "pvc-1").Result(), ), test.PVCs(builder.ForPersistentVolumeClaim("ns-1", "pvc-1").Result()), }, namespaces: []*corev1api.Namespace{ builder.ForNamespace("ns-1").ObjectMeta(builder.WithDeletionTimestamp(time.Now())).Result(), }, want: false, wantErr: errors.New("context deadline exceeded"), }, { name: "when PV is found, associated PVC is not found, result is false + timeout error", pvName: "pv-1", apiResources: []*test.APIResource{ test.PVs( builder.ForPersistentVolume("pv-1").ClaimRef("ns-1", "pvc-1").Result(), ), }, want: false, wantErr: errors.New("context deadline exceeded"), }, { name: "when PV is found, has associated PVC, associated namespace not found, result is false + timeout error", pvName: "pv-1", apiResources: []*test.APIResource{ test.PVs( builder.ForPersistentVolume("pv-1").ClaimRef("ns-1", "pvc-1").Result(), ), test.PVCs(builder.ForPersistentVolumeClaim("ns-1", "pvc-1").Result()), }, want: false, wantErr: errors.New("context deadline exceeded"), }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { h := newHarness(t) ctx := &restoreContext{ log: h.log, dynamicFactory: client.NewDynamicFactory(h.DynamicClient), namespaceClient: h.KubeClient.CoreV1().Namespaces(), resourceTerminatingTimeout: time.Millisecond, resourceDeletionStatusTracker: kube.NewResourceDeletionStatusTracker(), } for _, resource := range tc.apiResources { h.AddItems(t, resource) } for _, ns := range tc.namespaces { _, err := ctx.namespaceClient.Create(t.Context(), ns, metav1.CreateOptions{}) require.NoError(t, err) } pvClient, err := ctx.dynamicFactory.ClientForGroupVersionResource( schema.GroupVersion{Group: "", Version: "v1"}, metav1.APIResource{Name: "persistentvolumes"}, "", ) require.NoError(t, err) res, err := ctx.shouldRestore(tc.pvName, pvClient) assert.Equal(t, tc.want, res) if tc.wantErr != nil { if assert.Error(t, err, "expected a non-nil error") { assert.EqualError(t, err, tc.wantErr.Error()) } } else { assert.NoError(t, err) } }) } } func assertRestoredItems(t *testing.T, h *harness, want []*test.APIResource) { t.Helper() for _, resource := range want { resourceClient := h.DynamicClient.Resource(resource.GVR()) for _, item := range resource.Items { var client dynamic.ResourceInterface if item.GetNamespace() != "" { client = resourceClient.Namespace(item.GetNamespace()) } else { client = resourceClient } res, err := client.Get(t.Context(), item.GetName(), metav1.GetOptions{}) if !assert.NoError(t, err) { //nolint:testifylint // require is inappropriate continue } itemJSON, err := json.Marshal(item) if !assert.NoError(t, err) { //nolint:testifylint // require is inappropriate continue } t.Logf("%v", string(itemJSON)) u := make(map[string]any) if !assert.NoError(t, json.Unmarshal(itemJSON, &u)) { //nolint:testifylint // require is inappropriate continue } want := &unstructured.Unstructured{Object: u} // These fields get non-nil zero values in the unstructured objects if they're // empty in the structured objects. Remove them to make comparison easier. unstructured.RemoveNestedField(want.Object, "metadata", "creationTimestamp") unstructured.RemoveNestedField(want.Object, "status") unstructured.RemoveNestedField(res.Object, "status") assert.Equal(t, want, res) } } } // volumeSnapshotterGetter is a simple implementation of the VolumeSnapshotterGetter // interface that returns vsv1.VolumeSnapshotters from a map if they exist. type volumeSnapshotterGetter map[string]vsv1.VolumeSnapshotter func (vsg volumeSnapshotterGetter) GetVolumeSnapshotter(name string) (vsv1.VolumeSnapshotter, error) { snapshotter, ok := vsg[name] if !ok { return nil, errors.New("volume snapshotter not found") } return snapshotter, nil } // volumeSnapshotter is a test fake for the vsv1.VolumeSnapshotter interface type volumeSnapshotter struct { // a map from snapshotID to volumeID snapshotVolumes map[string]string // a map from volumeID to new pv name pvName map[string]string } // Init is a no-op. func (vs *volumeSnapshotter) Init(config map[string]string) error { return nil } // CreateVolumeFromSnapshot looks up the specified snapshotID in the snapshotVolumes // map and returns the corresponding volumeID if it exists, or an error otherwise. func (vs *volumeSnapshotter) CreateVolumeFromSnapshot(snapshotID, volumeType, volumeAZ string, iops *int64) (volumeID string, err error) { volumeID, ok := vs.snapshotVolumes[snapshotID] if !ok { return "", errors.New("snapshot not found") } return volumeID, nil } // SetVolumeID sets the persistent volume's spec.awsElasticBlockStore.volumeID field // with the provided volumeID. func (vs *volumeSnapshotter) SetVolumeID(pv runtime.Unstructured, volumeID string) (runtime.Unstructured, error) { unstructured.SetNestedField(pv.UnstructuredContent(), volumeID, "spec", "awsElasticBlockStore", "volumeID") newPVName, ok := vs.pvName[volumeID] if !ok { return pv, nil } unstructured.SetNestedField(pv.UnstructuredContent(), newPVName, "metadata", "name") return pv, nil } // GetVolumeID panics because it's not expected to be used for restores. func (*volumeSnapshotter) GetVolumeID(pv runtime.Unstructured) (string, error) { panic("GetVolumeID should not be used for restores") } // CreateSnapshot panics because it's not expected to be used for restores. func (*volumeSnapshotter) CreateSnapshot(volumeID, volumeAZ string, tags map[string]string) (snapshotID string, err error) { panic("CreateSnapshot should not be used for restores") } // GetVolumeInfo panics because it's not expected to be used for restores. func (*volumeSnapshotter) GetVolumeInfo(volumeID, volumeAZ string) (string, *int64, error) { panic("GetVolumeInfo should not be used for restores") } // DeleteSnapshot panics because it's not expected to be used for restores. func (*volumeSnapshotter) DeleteSnapshot(snapshotID string) error { panic("DeleteSnapshot should not be used for backups") } // TestRestorePersistentVolumes runs restores for persistent volumes and verifies that // they are restored as expected, including restoring volumes from snapshots when expected. // Verification is done by looking at the contents of the API and the metadata/spec/status of // the items in the API. func TestRestorePersistentVolumes(t *testing.T) { testPVCName := "testPVC" tests := []struct { name string restore *velerov1api.Restore backup *velerov1api.Backup tarball io.Reader apiResources []*test.APIResource volumeSnapshots []*volume.Snapshot volumeSnapshotLocations []*velerov1api.VolumeSnapshotLocation volumeSnapshotterGetter volumeSnapshotterGetter csiVolumeSnapshots []*snapshotv1api.VolumeSnapshot dataUploadResult *corev1api.ConfigMap want []*test.APIResource wantError bool wantWarning bool csiFeatureVerifierErr string }{ { name: "when a PV with a reclaim policy of delete has no snapshot and does not exist in-cluster, it does not get restored, and its PVC gets reset for dynamic provisioning", restore: defaultRestore().Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").ReclaimPolicy(corev1api.PersistentVolumeReclaimDelete).ClaimRef("ns-1", "pvc-1").Result(), ). AddItems("persistentvolumeclaims", builder.ForPersistentVolumeClaim("ns-1", "pvc-1"). VolumeName("pv-1"). ObjectMeta( builder.WithAnnotations("pv.kubernetes.io/bind-completed", "true", "pv.kubernetes.io/bound-by-controller", "true", "foo", "bar"), ). Result(), ). Done(), apiResources: []*test.APIResource{ test.PVs(), test.PVCs(), }, want: []*test.APIResource{ test.PVs(), test.PVCs( builder.ForPersistentVolumeClaim("ns-1", "pvc-1"). ObjectMeta( builder.WithAnnotations("foo", "bar"), builder.WithLabels("velero.io/backup-name", "backup-1", "velero.io/restore-name", "restore-1"), ). Result(), ), }, }, { name: "when a PV with a reclaim policy of retain has no snapshot and does not exist in-cluster, it gets restored, with its claim ref", restore: defaultRestore().Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").ReclaimPolicy(corev1api.PersistentVolumeReclaimRetain).ClaimRef("ns-1", "pvc-1").Result(), ). Done(), apiResources: []*test.APIResource{ test.PVs(), test.PVCs(), }, want: []*test.APIResource{ test.PVs( builder.ForPersistentVolume("pv-1"). ReclaimPolicy(corev1api.PersistentVolumeReclaimRetain). ObjectMeta( builder.WithLabels("velero.io/backup-name", "backup-1", "velero.io/restore-name", "restore-1"), ). ClaimRef("ns-1", "pvc-1"). Result(), ), }, }, { name: "when a PV with a reclaim policy of delete has a snapshot and does not exist in-cluster, the snapshot and PV are restored", restore: defaultRestore().Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").ReclaimPolicy(corev1api.PersistentVolumeReclaimDelete).AWSEBSVolumeID("old-volume").Result(), ). Done(), apiResources: []*test.APIResource{ test.PVs(), test.PVCs(), }, volumeSnapshots: []*volume.Snapshot{ { Spec: volume.SnapshotSpec{ BackupName: "backup-1", Location: "default", PersistentVolumeName: "pv-1", }, Status: volume.SnapshotStatus{ Phase: volume.SnapshotPhaseCompleted, ProviderSnapshotID: "snapshot-1", }, }, }, volumeSnapshotLocations: []*velerov1api.VolumeSnapshotLocation{ builder.ForVolumeSnapshotLocation(velerov1api.DefaultNamespace, "default").Provider("provider-1").Result(), }, volumeSnapshotterGetter: map[string]vsv1.VolumeSnapshotter{ "provider-1": &volumeSnapshotter{ snapshotVolumes: map[string]string{"snapshot-1": "new-volume"}, }, }, want: []*test.APIResource{ test.PVs( builder.ForPersistentVolume("pv-1"). ReclaimPolicy(corev1api.PersistentVolumeReclaimDelete). AWSEBSVolumeID("new-volume"). ObjectMeta( builder.WithLabels("velero.io/backup-name", "backup-1", "velero.io/restore-name", "restore-1"), ). Result(), ), }, }, { name: "when a PV with a reclaim policy of retain has a snapshot and does not exist in-cluster, the snapshot and PV are restored", restore: defaultRestore().Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1"). ReclaimPolicy(corev1api.PersistentVolumeReclaimRetain). AWSEBSVolumeID("old-volume"). Result(), ). Done(), apiResources: []*test.APIResource{ test.PVs(), test.PVCs(), }, volumeSnapshots: []*volume.Snapshot{ { Spec: volume.SnapshotSpec{ BackupName: "backup-1", Location: "default", PersistentVolumeName: "pv-1", }, Status: volume.SnapshotStatus{ Phase: volume.SnapshotPhaseCompleted, ProviderSnapshotID: "snapshot-1", }, }, }, volumeSnapshotLocations: []*velerov1api.VolumeSnapshotLocation{ builder.ForVolumeSnapshotLocation(velerov1api.DefaultNamespace, "default").Provider("provider-1").Result(), }, volumeSnapshotterGetter: map[string]vsv1.VolumeSnapshotter{ "provider-1": &volumeSnapshotter{ snapshotVolumes: map[string]string{"snapshot-1": "new-volume"}, }, }, want: []*test.APIResource{ test.PVs( builder.ForPersistentVolume("pv-1"). ReclaimPolicy(corev1api.PersistentVolumeReclaimRetain). AWSEBSVolumeID("new-volume"). ObjectMeta( builder.WithLabels("velero.io/backup-name", "backup-1", "velero.io/restore-name", "restore-1"), ). Result(), ), }, }, { name: "when a PV with a reclaim policy of delete has a snapshot and exists in-cluster, neither the snapshot nor the PV are restored", restore: defaultRestore().Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1"). ReclaimPolicy(corev1api.PersistentVolumeReclaimDelete). AWSEBSVolumeID("old-volume"). Result(), ). Done(), apiResources: []*test.APIResource{ test.PVs( builder.ForPersistentVolume("pv-1"). ReclaimPolicy(corev1api.PersistentVolumeReclaimDelete). AWSEBSVolumeID("old-volume"). Result(), ), test.PVCs(), }, volumeSnapshots: []*volume.Snapshot{ { Spec: volume.SnapshotSpec{ BackupName: "backup-1", Location: "default", PersistentVolumeName: "pv-1", }, Status: volume.SnapshotStatus{ Phase: volume.SnapshotPhaseCompleted, ProviderSnapshotID: "snapshot-1", }, }, }, volumeSnapshotLocations: []*velerov1api.VolumeSnapshotLocation{ builder.ForVolumeSnapshotLocation(velerov1api.DefaultNamespace, "default").Provider("provider-1").Result(), }, volumeSnapshotterGetter: map[string]vsv1.VolumeSnapshotter{ // the volume snapshotter fake is not configured with any snapshotID -> volumeID // mappings as a way to verify that the snapshot is not restored, since if it were // restored, we'd get an error of "snapshot not found". "provider-1": &volumeSnapshotter{}, }, want: []*test.APIResource{ test.PVs( builder.ForPersistentVolume("pv-1"). ReclaimPolicy(corev1api.PersistentVolumeReclaimDelete). AWSEBSVolumeID("old-volume"). Result(), ), }, }, { name: "when a PV with a reclaim policy of retain has a snapshot and exists in-cluster, neither the snapshot nor the PV are restored", restore: defaultRestore().Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1"). ReclaimPolicy(corev1api.PersistentVolumeReclaimRetain). AWSEBSVolumeID("old-volume"). Result(), ). Done(), apiResources: []*test.APIResource{ test.PVs( builder.ForPersistentVolume("pv-1"). ReclaimPolicy(corev1api.PersistentVolumeReclaimRetain). AWSEBSVolumeID("old-volume"). Result(), ), test.PVCs(), }, volumeSnapshots: []*volume.Snapshot{ { Spec: volume.SnapshotSpec{ BackupName: "backup-1", Location: "default", PersistentVolumeName: "pv-1", }, Status: volume.SnapshotStatus{ Phase: volume.SnapshotPhaseCompleted, ProviderSnapshotID: "snapshot-1", }, }, }, volumeSnapshotLocations: []*velerov1api.VolumeSnapshotLocation{ builder.ForVolumeSnapshotLocation(velerov1api.DefaultNamespace, "default").Provider("provider-1").Result(), }, volumeSnapshotterGetter: map[string]vsv1.VolumeSnapshotter{ // the volume snapshotter fake is not configured with any snapshotID -> volumeID // mappings as a way to verify that the snapshot is not restored, since if it were // restored, we'd get an error of "snapshot not found". "provider-1": &volumeSnapshotter{}, }, want: []*test.APIResource{ test.PVs( builder.ForPersistentVolume("pv-1"). ReclaimPolicy(corev1api.PersistentVolumeReclaimRetain). AWSEBSVolumeID("old-volume"). Result(), ), }, }, { name: "when a PV with a snapshot is used by a PVC in a namespace that's being remapped, and the original PV exists in-cluster, the PV is renamed", restore: defaultRestore().NamespaceMappings("source-ns", "target-ns").Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems( "persistentvolumes", builder.ForPersistentVolume("source-pv").AWSEBSVolumeID("source-volume").ClaimRef("source-ns", "pvc-1").Result(), ). AddItems( "persistentvolumeclaims", builder.ForPersistentVolumeClaim("source-ns", "pvc-1").VolumeName("source-pv").Result(), ). Done(), apiResources: []*test.APIResource{ test.PVs( builder.ForPersistentVolume("source-pv").AWSEBSVolumeID("source-volume").ClaimRef("source-ns", "pvc-1").Result(), ), test.PVCs(), }, volumeSnapshots: []*volume.Snapshot{ { Spec: volume.SnapshotSpec{ BackupName: "backup-1", Location: "default", PersistentVolumeName: "source-pv", }, Status: volume.SnapshotStatus{ Phase: volume.SnapshotPhaseCompleted, ProviderSnapshotID: "snapshot-1", }, }, }, volumeSnapshotLocations: []*velerov1api.VolumeSnapshotLocation{ builder.ForVolumeSnapshotLocation(velerov1api.DefaultNamespace, "default").Provider("provider-1").Result(), }, volumeSnapshotterGetter: map[string]vsv1.VolumeSnapshotter{ "provider-1": &volumeSnapshotter{ snapshotVolumes: map[string]string{"snapshot-1": "new-volume"}, }, }, want: []*test.APIResource{ test.PVs( builder.ForPersistentVolume("source-pv").AWSEBSVolumeID("source-volume").ClaimRef("source-ns", "pvc-1").Result(), builder.ForPersistentVolume("renamed-source-pv"). ObjectMeta( builder.WithAnnotations("velero.io/original-pv-name", "source-pv"), builder.WithLabels("velero.io/backup-name", "backup-1", "velero.io/restore-name", "restore-1"), // the namespace for this PV's claimRef should be the one that the PVC was remapped into. ).ClaimRef("target-ns", "pvc-1"). AWSEBSVolumeID("new-volume"). Result(), ), test.PVCs( builder.ForPersistentVolumeClaim("target-ns", "pvc-1"). ObjectMeta( builder.WithLabels("velero.io/backup-name", "backup-1", "velero.io/restore-name", "restore-1"), ). VolumeName("renamed-source-pv"). Result(), ), }, }, { name: "when a PV with a snapshot is used by a PVC in a namespace that's being remapped, and the original PV does not exist in-cluster, the PV is not renamed", restore: defaultRestore().NamespaceMappings("source-ns", "target-ns").Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems( "persistentvolumes", builder.ForPersistentVolume("source-pv").AWSEBSVolumeID("source-volume").ClaimRef("source-ns", "pvc-1").Result(), ). AddItems( "persistentvolumeclaims", builder.ForPersistentVolumeClaim("source-ns", "pvc-1").VolumeName("source-pv").Result(), ). Done(), apiResources: []*test.APIResource{ test.PVs(), test.PVCs(), }, volumeSnapshots: []*volume.Snapshot{ { Spec: volume.SnapshotSpec{ BackupName: "backup-1", Location: "default", PersistentVolumeName: "source-pv", }, Status: volume.SnapshotStatus{ Phase: volume.SnapshotPhaseCompleted, ProviderSnapshotID: "snapshot-1", }, }, }, volumeSnapshotLocations: []*velerov1api.VolumeSnapshotLocation{ builder.ForVolumeSnapshotLocation(velerov1api.DefaultNamespace, "default").Provider("provider-1").Result(), }, volumeSnapshotterGetter: map[string]vsv1.VolumeSnapshotter{ "provider-1": &volumeSnapshotter{ snapshotVolumes: map[string]string{"snapshot-1": "new-volume"}, }, }, want: []*test.APIResource{ test.PVs( builder.ForPersistentVolume("source-pv"). ObjectMeta( builder.WithLabels("velero.io/backup-name", "backup-1", "velero.io/restore-name", "restore-1"), ). ClaimRef("target-ns", "pvc-1"). AWSEBSVolumeID("new-volume"). Result(), ), test.PVCs( builder.ForPersistentVolumeClaim("target-ns", "pvc-1"). ObjectMeta( builder.WithLabels("velero.io/backup-name", "backup-1", "velero.io/restore-name", "restore-1"), ). VolumeName("source-pv"). Result(), ), }, }, { name: "when a PV without a snapshot is used by a PVC in a namespace that's being remapped, and the original PV exists in-cluster, the PV is not replaced and there is a restore warning", restore: defaultRestore().NamespaceMappings("source-ns", "target-ns").Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems( "persistentvolumes", builder.ForPersistentVolume("source-pv"). //ReclaimPolicy(corev1api.PersistentVolumeReclaimRetain). AWSEBSVolumeID("source-volume"). ClaimRef("source-ns", "pvc-1"). Result(), ). AddItems( "persistentvolumeclaims", builder.ForPersistentVolumeClaim("source-ns", "pvc-1").VolumeName("source-pv").Result(), ). Done(), apiResources: []*test.APIResource{ test.PVs( builder.ForPersistentVolume("source-pv"). //ReclaimPolicy(corev1api.PersistentVolumeReclaimRetain). AWSEBSVolumeID("source-volume"). ClaimRef("source-ns", "pvc-1"). Result(), ), test.PVCs(), }, want: []*test.APIResource{ test.PVs( builder.ForPersistentVolume("source-pv"). AWSEBSVolumeID("source-volume"). ClaimRef("source-ns", "pvc-1"). Result(), ), test.PVCs( builder.ForPersistentVolumeClaim("target-ns", "pvc-1"). ObjectMeta( builder.WithLabels("velero.io/backup-name", "backup-1", "velero.io/restore-name", "restore-1"), ). VolumeName("source-pv"). Result(), ), }, wantWarning: true, }, { name: "when a PV without a snapshot is used by a PVC in a namespace that's being remapped, and the original PV does not exist in-cluster, the PV is not renamed", restore: defaultRestore().NamespaceMappings("source-ns", "target-ns").Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems( "persistentvolumes", builder.ForPersistentVolume("source-pv"). AWSEBSVolumeID("source-volume"). ClaimRef("source-ns", "pvc-1"). Result(), ). AddItems( "persistentvolumeclaims", builder.ForPersistentVolumeClaim("source-ns", "pvc-1").VolumeName("source-pv").Result(), ). Done(), apiResources: []*test.APIResource{ test.PVs(), test.PVCs(), }, want: []*test.APIResource{ test.PVs( builder.ForPersistentVolume("source-pv"). //ReclaimPolicy(corev1api.PersistentVolumeReclaimRetain). ObjectMeta( builder.WithLabels("velero.io/backup-name", "backup-1", "velero.io/restore-name", "restore-1"), ). // the namespace for this PV's claimRef should be the one that the PVC was remapped into. ClaimRef("target-ns", "pvc-1"). AWSEBSVolumeID("source-volume"). Result(), ), test.PVCs( builder.ForPersistentVolumeClaim("target-ns", "pvc-1"). ObjectMeta( builder.WithLabels("velero.io/backup-name", "backup-1", "velero.io/restore-name", "restore-1"), ). VolumeName("source-pv"). Result(), ), }, }, { name: "when a PV is renamed and the original PV does not exist in-cluster, the PV should be renamed", restore: defaultRestore().NamespaceMappings("source-ns", "target-ns").Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems( "persistentvolumes", builder.ForPersistentVolume("source-pv").AWSEBSVolumeID("source-volume").ClaimRef("source-ns", "pvc-1").Result(), ). AddItems( "persistentvolumeclaims", builder.ForPersistentVolumeClaim("source-ns", "pvc-1").VolumeName("source-pv").Result(), ). Done(), apiResources: []*test.APIResource{ test.PVs(), test.PVCs(), }, volumeSnapshots: []*volume.Snapshot{ { Spec: volume.SnapshotSpec{ BackupName: "backup-1", Location: "default", PersistentVolumeName: "source-pv", }, Status: volume.SnapshotStatus{ Phase: volume.SnapshotPhaseCompleted, ProviderSnapshotID: "snapshot-1", }, }, }, volumeSnapshotLocations: []*velerov1api.VolumeSnapshotLocation{ builder.ForVolumeSnapshotLocation(velerov1api.DefaultNamespace, "default").Provider("provider-1").Result(), }, volumeSnapshotterGetter: map[string]vsv1.VolumeSnapshotter{ "provider-1": &volumeSnapshotter{ snapshotVolumes: map[string]string{"snapshot-1": "new-pvname"}, pvName: map[string]string{"new-pvname": "new-pvname"}, }, }, want: []*test.APIResource{ test.PVs( builder.ForPersistentVolume("new-pvname"). ObjectMeta( builder.WithLabels("velero.io/backup-name", "backup-1", "velero.io/restore-name", "restore-1"), builder.WithAnnotations("velero.io/original-pv-name", "source-pv"), ). ClaimRef("target-ns", "pvc-1"). AWSEBSVolumeID("new-pvname"). Result(), ), test.PVCs( builder.ForPersistentVolumeClaim("target-ns", "pvc-1"). ObjectMeta( builder.WithLabels("velero.io/backup-name", "backup-1", "velero.io/restore-name", "restore-1"), ). VolumeName("new-pvname"). Result(), ), }, }, { name: "when a PV with a reclaim policy of retain has a snapshot and exists in-cluster, neither the snapshot nor the PV are restored", restore: defaultRestore().Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1"). ReclaimPolicy(corev1api.PersistentVolumeReclaimRetain). AWSEBSVolumeID("old-volume"). Result(), ). Done(), apiResources: []*test.APIResource{ test.PVs( builder.ForPersistentVolume("pv-1"). ReclaimPolicy(corev1api.PersistentVolumeReclaimRetain). AWSEBSVolumeID("old-volume"). Result(), ), test.PVCs(), }, volumeSnapshots: []*volume.Snapshot{ { Spec: volume.SnapshotSpec{ BackupName: "backup-1", Location: "default", PersistentVolumeName: "pv-1", }, Status: volume.SnapshotStatus{ Phase: volume.SnapshotPhaseCompleted, ProviderSnapshotID: "snapshot-1", }, }, }, volumeSnapshotLocations: []*velerov1api.VolumeSnapshotLocation{ { ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1api.DefaultNamespace, Name: "default", }, Spec: velerov1api.VolumeSnapshotLocationSpec{ Provider: "provider-1", }, }, }, volumeSnapshotterGetter: map[string]vsv1.VolumeSnapshotter{ // the volume snapshotter fake is not configured with any snapshotID -> volumeID // mappings as a way to verify that the snapshot is not restored, since if it were // restored, we'd get an error of "snapshot not found". "provider-1": &volumeSnapshotter{}, }, want: []*test.APIResource{ test.PVs( builder.ForPersistentVolume("pv-1"). ReclaimPolicy(corev1api.PersistentVolumeReclaimRetain). AWSEBSVolumeID("old-volume"). Result(), ), }, }, { name: "when a PV with a snapshot is used by a PVC in a namespace that's being remapped, and the original PV exists in-cluster, the PV is renamed by volumesnapshotter", restore: defaultRestore().NamespaceMappings("source-ns", "target-ns").Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems( "persistentvolumes", builder.ForPersistentVolume("source-pv").AWSEBSVolumeID("source-volume").ClaimRef("source-ns", "pvc-1").Result(), ). AddItems( "persistentvolumeclaims", builder.ForPersistentVolumeClaim("source-ns", "pvc-1").VolumeName("source-pv").Result(), ). Done(), apiResources: []*test.APIResource{ test.PVs( builder.ForPersistentVolume("source-pv").AWSEBSVolumeID("source-volume").ClaimRef("source-ns", "pvc-1").Result(), ), test.PVCs(), }, volumeSnapshots: []*volume.Snapshot{ { Spec: volume.SnapshotSpec{ BackupName: "backup-1", Location: "default", PersistentVolumeName: "source-pv", }, Status: volume.SnapshotStatus{ Phase: volume.SnapshotPhaseCompleted, ProviderSnapshotID: "snapshot-1", }, }, }, volumeSnapshotLocations: []*velerov1api.VolumeSnapshotLocation{ builder.ForVolumeSnapshotLocation(velerov1api.DefaultNamespace, "default").Provider("provider-1").Result(), }, volumeSnapshotterGetter: map[string]vsv1.VolumeSnapshotter{ "provider-1": &volumeSnapshotter{ snapshotVolumes: map[string]string{"snapshot-1": "new-volume"}, pvName: map[string]string{"new-volume": "volumesnapshotter-renamed-source-pv"}, }, }, want: []*test.APIResource{ test.PVs( builder.ForPersistentVolume("source-pv").AWSEBSVolumeID("source-volume").ClaimRef("source-ns", "pvc-1").Result(), builder.ForPersistentVolume("volumesnapshotter-renamed-source-pv"). ObjectMeta( builder.WithAnnotations("velero.io/original-pv-name", "source-pv"), builder.WithLabels("velero.io/backup-name", "backup-1", "velero.io/restore-name", "restore-1"), ). ClaimRef("target-ns", "pvc-1"). AWSEBSVolumeID("new-volume"). Result(), ), test.PVCs( builder.ForPersistentVolumeClaim("target-ns", "pvc-1"). ObjectMeta( builder.WithLabels("velero.io/backup-name", "backup-1", "velero.io/restore-name", "restore-1"), ). VolumeName("volumesnapshotter-renamed-source-pv"). Result(), ), }, }, { name: "when a PV with a reclaim policy of retain has a CSI VolumeSnapshot and does not exist in-cluster, the PV is not restored", restore: defaultRestore().Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1"). ReclaimPolicy(corev1api.PersistentVolumeReclaimRetain). ClaimRef("velero", testPVCName). Result(), ). Done(), apiResources: []*test.APIResource{ test.PVs(), test.PVCs(), }, csiVolumeSnapshots: []*snapshotv1api.VolumeSnapshot{ { ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", Name: "test", }, Spec: snapshotv1api.VolumeSnapshotSpec{ Source: snapshotv1api.VolumeSnapshotSource{ PersistentVolumeClaimName: &testPVCName, }, }, }, }, volumeSnapshotLocations: []*velerov1api.VolumeSnapshotLocation{ builder.ForVolumeSnapshotLocation(velerov1api.DefaultNamespace, "default").Provider("provider-1").Result(), }, volumeSnapshotterGetter: map[string]vsv1.VolumeSnapshotter{ "provider-1": &volumeSnapshotter{ snapshotVolumes: map[string]string{"snapshot-1": "new-volume"}, }, }, want: []*test.APIResource{}, }, { name: "when a PV with a reclaim policy of retain has a DataUpload result CM and does not exist in-cluster, the PV is not restored", restore: defaultRestore().ObjectMeta(builder.WithUID("fakeUID")).Result(), backup: defaultBackup().Result(), tarball: test.NewTarWriter(t). AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1"). ReclaimPolicy(corev1api.PersistentVolumeReclaimRetain). ClaimRef("velero", testPVCName). Result(), ). Done(), apiResources: []*test.APIResource{ test.PVs(), test.PVCs(), test.ConfigMaps(), }, volumeSnapshotLocations: []*velerov1api.VolumeSnapshotLocation{ builder.ForVolumeSnapshotLocation(velerov1api.DefaultNamespace, "default").Provider("provider-1").Result(), }, volumeSnapshotterGetter: map[string]vsv1.VolumeSnapshotter{ "provider-1": &volumeSnapshotter{ snapshotVolumes: map[string]string{"snapshot-1": "new-volume"}, }, }, dataUploadResult: builder.ForConfigMap("velero", "test").ObjectMeta(builder.WithLabelsMap(map[string]string{ velerov1api.RestoreUIDLabel: "fakeUID", velerov1api.PVCNamespaceNameLabel: "velero.testPVC", velerov1api.ResourceUsageLabel: string(velerov1api.VeleroResourceUsageDataUploadResult), })).Result(), want: []*test.APIResource{}, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { h := newHarness(t) h.restorer.resourcePriorities = types.Priorities{HighPriorities: []string{"persistentvolumes", "persistentvolumeclaims"}} h.restorer.pvRenamer = func(oldName string) (string, error) { renamed := "renamed-" + oldName return renamed, nil } // set up the VolumeSnapshotLocation client and add test data to it for _, vsl := range tc.volumeSnapshotLocations { require.NoError(t, h.restorer.kbClient.Create(t.Context(), vsl)) } if tc.dataUploadResult != nil { require.NoError(t, h.restorer.kbClient.Create(t.Context(), tc.dataUploadResult)) } for _, r := range tc.apiResources { h.AddItems(t, r) } // Collect the IDs of all of the wanted resources so we can ensure the // exact set exists in the API after restore. wantIDs := make(map[*test.APIResource][]string) for i, resource := range tc.want { wantIDs[tc.want[i]] = []string{} for _, item := range resource.Items { wantIDs[tc.want[i]] = append(wantIDs[tc.want[i]], fmt.Sprintf("%s/%s", item.GetNamespace(), item.GetName())) } } data := &Request{ Log: h.log, Restore: tc.restore, Backup: tc.backup, VolumeSnapshots: tc.volumeSnapshots, BackupReader: tc.tarball, CSIVolumeSnapshots: tc.csiVolumeSnapshots, RestoreVolumeInfoTracker: volume.NewRestoreVolInfoTracker(tc.restore, h.log, test.NewFakeControllerRuntimeClient(t)), } warnings, errs := h.restorer.Restore( data, nil, // restoreItemActions tc.volumeSnapshotterGetter, ) if tc.wantWarning { assertNonEmptyResults(t, "warning", warnings) } else { assertEmptyResults(t, warnings) } if tc.wantError { assertNonEmptyResults(t, "error", errs) } else { assertEmptyResults(t, errs) } assertAPIContents(t, h, wantIDs) assertRestoredItems(t, h, tc.want) }) } } type fakePodVolumeRestorerFactory struct { restorer *uploadermocks.Restorer } func (f *fakePodVolumeRestorerFactory) NewRestorer(context.Context, *velerov1api.Restore) (podvolume.Restorer, error) { return f.restorer, nil } // TestRestoreWithPodVolume verifies that a call to RestorePodVolumes was made as and when // expected for the given pods by using a mock for the pod volume restorer. func TestRestoreWithPodVolume(t *testing.T) { tests := []struct { name string restore *velerov1api.Restore backup *velerov1api.Backup apiResources []*test.APIResource podVolumeBackups []*velerov1api.PodVolumeBackup podWithPVBs, podWithoutPVBs []*corev1api.Pod want map[*test.APIResource][]string }{ { name: "a pod that exists in given backup and contains associated PVBs should have RestorePodVolumes called", restore: defaultRestore().Result(), backup: defaultBackup().Result(), apiResources: []*test.APIResource{test.Pods()}, podVolumeBackups: []*velerov1api.PodVolumeBackup{ builder.ForPodVolumeBackup("velero", "pvb-1").PodName("pod-1").SnapshotID("foo").Result(), builder.ForPodVolumeBackup("velero", "pvb-2").PodName("pod-2").PodNamespace("ns-1").SnapshotID("foo").Result(), builder.ForPodVolumeBackup("velero", "pvb-3").PodName("pod-4").PodNamespace("ns-2").SnapshotID("foo").Result(), }, podWithPVBs: []*corev1api.Pod{ builder.ForPod("ns-1", "pod-2"). Result(), builder.ForPod("ns-2", "pod-4"). Result(), }, podWithoutPVBs: []*corev1api.Pod{ builder.ForPod("ns-2", "pod-3"). Result(), }, want: map[*test.APIResource][]string{ test.Pods(): {"ns-1/pod-2", "ns-2/pod-3", "ns-2/pod-4"}, }, }, { name: "a pod that exists in given backup but does not contain associated PVBs should not have should have RestorePodVolumes called", restore: defaultRestore().Result(), backup: defaultBackup().Result(), apiResources: []*test.APIResource{test.Pods()}, podVolumeBackups: []*velerov1api.PodVolumeBackup{ builder.ForPodVolumeBackup("velero", "pvb-1").PodName("pod-1").Result(), builder.ForPodVolumeBackup("velero", "pvb-2").PodName("pod-2").Result(), }, podWithPVBs: []*corev1api.Pod{}, podWithoutPVBs: []*corev1api.Pod{ builder.ForPod("ns-1", "pod-3"). Result(), builder.ForPod("ns-2", "pod-4"). Result(), }, want: map[*test.APIResource][]string{ test.Pods(): {"ns-1/pod-3", "ns-2/pod-4"}, }, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { h := newHarness(t) restorer := new(uploadermocks.Restorer) defer restorer.AssertExpectations(t) h.restorer.podVolumeRestorerFactory = &fakePodVolumeRestorerFactory{ restorer: restorer, } // needed only to indicate resource types that can be restored, in this case, pods for _, resource := range tc.apiResources { h.AddItems(t, resource) } tarball := test.NewTarWriter(t) // these backed up pods don't have any PVBs associated with them, so a call to RestorePodVolumes is not expected to be made for them for _, pod := range tc.podWithoutPVBs { tarball.AddItems("pods", pod) } // these backed up pods have PVBs associated with them, so a call to RestorePodVolumes will be made for each of them for _, pod := range tc.podWithPVBs { tarball.AddItems("pods", pod) // the restore process adds these labels before restoring, so we must add them here too otherwise they won't match pod.Labels = map[string]string{"velero.io/backup-name": tc.backup.Name, "velero.io/restore-name": tc.restore.Name} expectedArgs := podvolume.RestoreData{ Restore: tc.restore, Pod: pod, PodVolumeBackups: tc.podVolumeBackups, SourceNamespace: pod.Namespace, BackupLocation: "", } restorer. On("RestorePodVolumes", expectedArgs, mock.Anything). Return(nil) } data := &Request{ Log: h.log, Restore: tc.restore, Backup: tc.backup, PodVolumeBackups: tc.podVolumeBackups, BackupReader: tarball.Done(), } warnings, errs := h.restorer.Restore( data, nil, // restoreItemActions nil, // volume snapshotter getter ) assertEmptyResults(t, warnings, errs) assertAPIContents(t, h, tc.want) }) } } func TestResetMetadata(t *testing.T) { tests := []struct { name string obj *unstructured.Unstructured expectedErr bool expectedRes *unstructured.Unstructured }{ { name: "no metadata causes error", obj: &unstructured.Unstructured{}, expectedErr: true, }, { name: "keep name, namespace, labels, annotations, managedFields, finalizers", obj: newTestUnstructured().WithMetadata("name", "namespace", "labels", "annotations", "managedFields", "finalizers").Unstructured, expectedErr: false, expectedRes: newTestUnstructured().WithMetadata("name", "namespace", "labels", "annotations", "managedFields", "finalizers").Unstructured, }, { name: "remove uid, ownerReferences", obj: newTestUnstructured().WithMetadata("name", "namespace", "uid", "ownerReferences").Unstructured, expectedErr: false, expectedRes: newTestUnstructured().WithMetadata("name", "namespace").Unstructured, }, { name: "keep status", obj: newTestUnstructured().WithMetadata().WithStatus().Unstructured, expectedErr: false, expectedRes: newTestUnstructured().WithMetadata().WithStatus().Unstructured, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { res, err := resetMetadata(test.obj) if assert.Equal(t, test.expectedErr, err != nil) { assert.Equal(t, test.expectedRes, res) } }) } } func TestResetStatus(t *testing.T) { tests := []struct { name string obj *unstructured.Unstructured expectedRes *unstructured.Unstructured }{ { name: "no status don't cause error", obj: &unstructured.Unstructured{}, expectedRes: &unstructured.Unstructured{}, }, { name: "remove status", obj: newTestUnstructured().WithMetadata().WithStatus().Unstructured, expectedRes: newTestUnstructured().WithMetadata().Unstructured, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { resetStatus(test.obj) assert.Equal(t, test.expectedRes, test.obj) }) } } func TestIsCompleted(t *testing.T) { tests := []struct { name string expected bool content string groupResource schema.GroupResource expectedErr bool }{ { name: "Failed pods are complete", expected: true, content: `{"apiVersion":"v1","kind":"Pod","metadata":{"namespace":"ns","name":"pod1"}, "status": {"phase": "Failed"}}`, groupResource: schema.GroupResource{Group: "", Resource: "pods"}, }, { name: "Succeeded pods are complete", expected: true, content: `{"apiVersion":"v1","kind":"Pod","metadata":{"namespace":"ns","name":"pod1"}, "status": {"phase": "Succeeded"}}`, groupResource: schema.GroupResource{Group: "", Resource: "pods"}, }, { name: "Pending pods aren't complete", expected: false, content: `{"apiVersion":"v1","kind":"Pod","metadata":{"namespace":"ns","name":"pod1"}, "status": {"phase": "Pending"}}`, groupResource: schema.GroupResource{Group: "", Resource: "pods"}, }, { name: "Running pods aren't complete", expected: false, content: `{"apiVersion":"v1","kind":"Pod","metadata":{"namespace":"ns","name":"pod1"}, "status": {"phase": "Running"}}`, groupResource: schema.GroupResource{Group: "", Resource: "pods"}, }, { name: "Jobs without a completion time aren't complete", expected: false, content: `{"apiVersion":"v1","kind":"Pod","metadata":{"namespace":"ns","name":"pod1"}}`, groupResource: schema.GroupResource{Group: "batch", Resource: "jobs"}, }, { name: "Jobs with a completion time are completed", expected: true, content: `{"apiVersion":"v1","kind":"Pod","metadata":{"namespace":"ns","name":"pod1"}, "status": {"completionTime": "bar"}}`, groupResource: schema.GroupResource{Group: "batch", Resource: "jobs"}, }, { name: "Jobs with an empty completion time are not completed", expected: false, content: `{"apiVersion":"v1","kind":"Pod","metadata":{"namespace":"ns","name":"pod1"}, "status": {"completionTime": ""}}`, groupResource: schema.GroupResource{Group: "batch", Resource: "jobs"}, }, { name: "Something not a pod or a job may actually be complete, but we're not concerned with that", expected: false, content: `{"apiVersion": "v1", "kind": "Namespace", "metadata": {"name": "ns"}, "status": {"completionTime": "bar", "phase":"Completed"}}`, groupResource: schema.GroupResource{Group: "", Resource: "namespaces"}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { u := test.UnstructuredOrDie(tt.content) backup, err := isCompleted(u, tt.groupResource) if assert.Equal(t, tt.expectedErr, err != nil) { assert.Equal(t, tt.expected, backup) } }) } } func Test_getOrderedResources(t *testing.T) { tests := []struct { name string resourcePriorities types.Priorities backupResources map[string]*archive.ResourceItems want []string }{ { name: "when only priorities are specified, they're returned in order", resourcePriorities: types.Priorities{HighPriorities: []string{"prio-3", "prio-2", "prio-1"}}, backupResources: nil, want: []string{"prio-3", "prio-2", "prio-1"}, }, { name: "when only backup resources are specified, they're returned in alphabetical order", resourcePriorities: types.Priorities{}, backupResources: map[string]*archive.ResourceItems{ "backup-resource-3": nil, "backup-resource-2": nil, "backup-resource-1": nil, }, want: []string{"backup-resource-1", "backup-resource-2", "backup-resource-3"}, }, { name: "when priorities and backup resources are specified, they're returned in the correct order", resourcePriorities: types.Priorities{HighPriorities: []string{"prio-3", "prio-2", "prio-1"}}, backupResources: map[string]*archive.ResourceItems{ "prio-3": nil, "backup-resource-3": nil, "backup-resource-2": nil, "backup-resource-1": nil, }, want: []string{"prio-3", "prio-2", "prio-1", "backup-resource-1", "backup-resource-2", "backup-resource-3"}, }, { name: "when priorities and backup resources are specified, they're returned in the correct order", resourcePriorities: types.Priorities{HighPriorities: []string{"prio-3", "prio-2", "prio-1"}, LowPriorities: []string{"prio-0"}}, backupResources: map[string]*archive.ResourceItems{ "prio-3": nil, "prio-0": nil, "backup-resource-3": nil, "backup-resource-2": nil, "backup-resource-1": nil, }, want: []string{"prio-3", "prio-2", "prio-1", "backup-resource-1", "backup-resource-2", "backup-resource-3", "prio-0"}, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { assert.Equal(t, tc.want, getOrderedResources(tc.resourcePriorities, tc.backupResources)) }) } } // assertResourceCreationOrder ensures that resources were created in the expected // order. Any resources *not* in resourcePriorities are required to come *after* all // resources in any order. func assertResourceCreationOrder(t *testing.T, resourcePriorities []string, createdResources []resourceID) { t.Helper() // lastSeen tracks the index in 'resourcePriorities' of the last resource type // we saw created. Once we've seen a resource in 'resourcePriorities', we should // never see another instance of a prior resource. lastSeen := 0 // Find the index in 'resourcePriorities' of the resource type for // the current item, if it exists. This index ('current') *must* // be greater than or equal to 'lastSeen', which was the last resource // we saw, since otherwise the current resource would be out of order. By // initializing current to len(ordered), we're saying that if the resource // is not explicitly in orderedResources, then it must come *after* // all orderedResources. for _, r := range createdResources { current := len(resourcePriorities) for i, item := range resourcePriorities { if item == r.groupResource { current = i break } } // the index of the current resource must be the same as or greater than the index of // the last resource we saw for the restored order to be correct. assert.GreaterOrEqual(t, current, lastSeen, "%s was restored out of order", r.groupResource) lastSeen = current } } type resourceID struct { groupResource string nsAndName string } // createRecorder provides a Reactor that can be used to capture // resources created in a fake client. type createRecorder struct { t *testing.T resources []resourceID } func (cr *createRecorder) reactor() func(kubetesting.Action) (bool, runtime.Object, error) { return func(action kubetesting.Action) (bool, runtime.Object, error) { createAction, ok := action.(kubetesting.CreateAction) if !ok { return false, nil, nil } accessor, err := meta.Accessor(createAction.GetObject()) require.NoError(cr.t, err) cr.resources = append(cr.resources, resourceID{ groupResource: action.GetResource().GroupResource().String(), nsAndName: fmt.Sprintf("%s/%s", action.GetNamespace(), accessor.GetName()), }) return false, nil, nil } } func defaultRestore() *builder.RestoreBuilder { return builder.ForRestore(velerov1api.DefaultNamespace, "restore-1").Backup("backup-1") } // assertAPIContents asserts that the dynamic client on the provided harness contains // all of the items specified in 'want' (a map from an APIResource definition to a slice // of resource identifiers, formatted as /). func assertAPIContents(t *testing.T, h *harness, want map[*test.APIResource][]string) { t.Helper() for r, want := range want { res, err := h.DynamicClient.Resource(r.GVR()).List(t.Context(), metav1.ListOptions{}) require.NoError(t, err) if err != nil { continue } got := sets.NewString() for _, item := range res.Items { got.Insert(fmt.Sprintf("%s/%s", item.GetNamespace(), item.GetName())) } assert.Equal(t, sets.NewString(want...), got) } } func assertEmptyResults(t *testing.T, res ...Result) { t.Helper() for _, r := range res { assert.Empty(t, r.Cluster) assert.Empty(t, r.Namespaces) assert.Empty(t, r.Velero) } } func assertNonEmptyResults(t *testing.T, typeMsg string, res ...Result) { t.Helper() total := 0 for _, r := range res { total += len(r.Cluster) total += len(r.Namespaces) total += len(r.Velero) } assert.Positive(t, total, "Expected at least one "+typeMsg) } type harness struct { *test.APIServer restorer *kubernetesRestorer log logrus.FieldLogger } func newHarness(t *testing.T) *harness { t.Helper() apiServer := test.NewAPIServer(t) log := logrus.StandardLogger() kbClient := test.NewFakeControllerRuntimeClient(t) discoveryHelper, err := discovery.NewHelper(apiServer.DiscoveryClient, log) require.NoError(t, err) return &harness{ APIServer: apiServer, restorer: &kubernetesRestorer{ discoveryHelper: discoveryHelper, dynamicFactory: client.NewDynamicFactory(apiServer.DynamicClient), namespaceClient: apiServer.KubeClient.CoreV1().Namespaces(), resourceTerminatingTimeout: time.Minute, logger: log, fileSystem: test.NewFakeFileSystem(), // unsupported podVolumeRestorerFactory: nil, podVolumeTimeout: 0, kbClient: kbClient, resourceDeletionStatusTracker: kube.NewResourceDeletionStatusTracker(), }, log: log, } } func (h *harness) AddItems(t *testing.T, resource *test.APIResource) { t.Helper() h.DiscoveryClient.WithAPIResource(resource) require.NoError(t, h.restorer.discoveryHelper.Refresh()) for _, item := range resource.Items { obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(item) require.NoError(t, err) unstructuredObj := &unstructured.Unstructured{Object: obj} // These fields have non-nil zero values in the unstructured objects. We remove // them to make comparison easier in our tests. unstructured.RemoveNestedField(unstructuredObj.Object, "metadata", "creationTimestamp") unstructured.RemoveNestedField(unstructuredObj.Object, "status") if resource.Namespaced { _, err = h.DynamicClient.Resource(resource.GVR()).Namespace(item.GetNamespace()).Create(t.Context(), unstructuredObj, metav1.CreateOptions{}) } else { _, err = h.DynamicClient.Resource(resource.GVR()).Create(t.Context(), unstructuredObj, metav1.CreateOptions{}) } require.NoError(t, err) } } func Test_resetVolumeBindingInfo(t *testing.T) { tests := []struct { name string obj *unstructured.Unstructured expected *unstructured.Unstructured }{ { name: "PVs that are bound have their binding and dynamic provisioning annotations removed", obj: newTestUnstructured().WithMetadataField("kind", "persistentVolume"). WithName("pv-1").WithAnnotations( kubeutil.KubeAnnBindCompleted, kubeutil.KubeAnnBoundByController, kubeutil.KubeAnnDynamicallyProvisioned, ).WithSpecField("claimRef", map[string]any{ "namespace": "ns-1", "name": "pvc-1", "uid": "abc", "resourceVersion": "1"}).Unstructured, expected: newTestUnstructured().WithMetadataField("kind", "persistentVolume"). WithName("pv-1"). WithAnnotations(kubeutil.KubeAnnDynamicallyProvisioned). WithSpecField("claimRef", map[string]any{ "namespace": "ns-1", "name": "pvc-1"}).Unstructured, }, { name: "PVCs that are bound have their binding annotations removed, but the volume name stays", obj: newTestUnstructured().WithMetadataField("kind", "persistentVolumeClaim"). WithName("pvc-1").WithAnnotations( kubeutil.KubeAnnBindCompleted, kubeutil.KubeAnnBoundByController, ).WithSpecField("volumeName", "pv-1").Unstructured, expected: newTestUnstructured().WithMetadataField("kind", "persistentVolumeClaim"). WithName("pvc-1").WithAnnotations(). WithSpecField("volumeName", "pv-1").Unstructured, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { actual := resetVolumeBindingInfo(tc.obj) assert.Equal(t, tc.expected, actual) }) } } func TestIsAlreadyExistsError(t *testing.T) { tests := []struct { name string apiResource *test.APIResource obj *unstructured.Unstructured err error expected bool }{ { name: "The input error is IsAlreadyExists error", err: apierrors.NewAlreadyExists(schema.GroupResource{}, ""), expected: true, }, { name: "The input obj isn't service", obj: &unstructured.Unstructured{ Object: map[string]any{ "kind": "Pod", }, }, expected: false, }, { name: "The StatusError contains no causes", obj: &unstructured.Unstructured{ Object: map[string]any{ "kind": "Service", }, }, err: &apierrors.StatusError{ ErrStatus: metav1.Status{ Reason: metav1.StatusReasonInvalid, }, }, expected: false, }, { name: "The causes contains not only port already allocated error", obj: &unstructured.Unstructured{ Object: map[string]any{ "kind": "Service", }, }, err: &apierrors.StatusError{ ErrStatus: metav1.Status{ Reason: metav1.StatusReasonInvalid, Details: &metav1.StatusDetails{ Causes: []metav1.StatusCause{ {Message: "provided port is already allocated"}, {Message: "other error"}, }, }, }, }, expected: false, }, { name: "Get already allocated error but the service doesn't exist", obj: &unstructured.Unstructured{ Object: map[string]any{ "kind": "Service", "metadata": map[string]any{ "namespace": "default", "name": "test", }, }, }, err: &apierrors.StatusError{ ErrStatus: metav1.Status{ Reason: metav1.StatusReasonInvalid, Details: &metav1.StatusDetails{ Causes: []metav1.StatusCause{ {Message: "provided port is already allocated"}, }, }, }, }, expected: false, }, { name: "Get already allocated error and the service exists", apiResource: test.Services( builder.ForService("default", "test").Result(), ), obj: &unstructured.Unstructured{ Object: map[string]any{ "kind": "Service", "metadata": map[string]any{ "namespace": "default", "name": "test", }, }, }, err: &apierrors.StatusError{ ErrStatus: metav1.Status{ Reason: metav1.StatusReasonInvalid, Details: &metav1.StatusDetails{ Causes: []metav1.StatusCause{ {Message: "provided port is already allocated"}, }, }, }, }, expected: true, }, } for _, test := range tests { h := newHarness(t) ctx := &restoreContext{ log: h.log, dynamicFactory: client.NewDynamicFactory(h.DynamicClient), namespaceClient: h.KubeClient.CoreV1().Namespaces(), resourceDeletionStatusTracker: kube.NewResourceDeletionStatusTracker(), } if test.apiResource != nil { h.AddItems(t, test.apiResource) } client, err := ctx.dynamicFactory.ClientForGroupVersionResource( schema.GroupVersion{Group: "", Version: "v1"}, metav1.APIResource{Name: "services"}, "default", ) require.NoError(t, err) t.Run(test.name, func(t *testing.T) { result, err := isAlreadyExistsError(ctx, test.obj, test.err, client) require.NoError(t, err) assert.Equal(t, test.expected, result) }) } } func TestHasCSIVolumeSnapshot(t *testing.T) { tests := []struct { name string vs *snapshotv1api.VolumeSnapshot obj *unstructured.Unstructured expectedResult bool }{ { name: "Invalid PV, expect false.", obj: &unstructured.Unstructured{ Object: map[string]any{ "kind": 1, }, }, expectedResult: false, }, { name: "Cannot find VS, expect false", obj: &unstructured.Unstructured{ Object: map[string]any{ "kind": "PersistentVolume", "apiVersion": "v1", "metadata": map[string]any{ "namespace": "default", "name": "test", }, }, }, expectedResult: false, }, { name: "VS's source PVC is nil, expect false", obj: &unstructured.Unstructured{ Object: map[string]any{ "kind": "PersistentVolume", "apiVersion": "v1", "metadata": map[string]any{ "namespace": "default", "name": "test", }, "spec": map[string]any{ "claimRef": map[string]any{ "namespace": "velero", "name": "test", }, }, }, }, vs: builder.ForVolumeSnapshot("velero", "test").Result(), expectedResult: false, }, { name: "PVs claimref is nil, expect false.", obj: &unstructured.Unstructured{ Object: map[string]any{ "kind": "PersistentVolume", "apiVersion": "v1", "metadata": map[string]any{ "namespace": "velero", "name": "test", }, }, }, vs: builder.ForVolumeSnapshot("velero", "test").SourcePVC("test").Result(), expectedResult: false, }, { name: "Find VS, expect true.", obj: &unstructured.Unstructured{ Object: map[string]any{ "kind": "PersistentVolume", "apiVersion": "v1", "metadata": map[string]any{ "namespace": "velero", "name": "test", }, "spec": map[string]any{ "claimRef": map[string]any{ "namespace": "velero", "name": "test", }, }, }, }, vs: builder.ForVolumeSnapshot("velero", "test").SourcePVC("test").Result(), expectedResult: true, }, } for _, tc := range tests { h := newHarness(t) ctx := &restoreContext{ log: h.log, resourceDeletionStatusTracker: kube.NewResourceDeletionStatusTracker(), } if tc.vs != nil { ctx.csiVolumeSnapshots = []*snapshotv1api.VolumeSnapshot{tc.vs} } t.Run(tc.name, func(t *testing.T) { require.Equal(t, tc.expectedResult, hasCSIVolumeSnapshot(ctx, tc.obj)) }) } } func TestHasSnapshotDataUpload(t *testing.T) { tests := []struct { name string duResult *corev1api.ConfigMap obj *unstructured.Unstructured expectedResult bool restore *velerov1api.Restore }{ { name: "Invalid PV, expect false.", obj: &unstructured.Unstructured{ Object: map[string]any{ "kind": 1, }, }, expectedResult: false, }, { name: "PV without ClaimRef, expect false", obj: &unstructured.Unstructured{ Object: map[string]any{ "kind": "PersistentVolume", "apiVersion": "v1", "metadata": map[string]any{ "namespace": "default", "name": "test", }, }, }, duResult: builder.ForConfigMap("velero", "test").Result(), restore: builder.ForRestore("velero", "test").ObjectMeta(builder.WithUID("fakeUID")).Result(), expectedResult: false, }, { name: "Cannot find DataUploadResult CM, expect false", obj: &unstructured.Unstructured{ Object: map[string]any{ "kind": "PersistentVolume", "apiVersion": "v1", "metadata": map[string]any{ "namespace": "default", "name": "test", }, "spec": map[string]any{ "claimRef": map[string]any{ "namespace": "velero", "name": "testPVC", }, }, }, }, duResult: builder.ForConfigMap("velero", "test").Result(), restore: builder.ForRestore("velero", "test").ObjectMeta(builder.WithUID("fakeUID")).Result(), expectedResult: false, }, { name: "Find DataUploadResult CM, expect true", obj: &unstructured.Unstructured{ Object: map[string]any{ "kind": "PersistentVolume", "apiVersion": "v1", "metadata": map[string]any{ "namespace": "default", "name": "test", }, "spec": map[string]any{ "claimRef": map[string]any{ "namespace": "velero", "name": "testPVC", }, }, }, }, duResult: builder.ForConfigMap("velero", "test").ObjectMeta(builder.WithLabelsMap(map[string]string{ velerov1api.RestoreUIDLabel: "fakeUID", velerov1api.PVCNamespaceNameLabel: "velero/testPVC", velerov1api.ResourceUsageLabel: string(velerov1api.VeleroResourceUsageDataUploadResult), })).Result(), restore: builder.ForRestore("velero", "test").ObjectMeta(builder.WithUID("fakeUID")).Result(), expectedResult: false, }, } for _, tc := range tests { h := newHarness(t) ctx := &restoreContext{ log: h.log, kbClient: h.restorer.kbClient, restore: tc.restore, resourceDeletionStatusTracker: kube.NewResourceDeletionStatusTracker(), } if tc.duResult != nil { require.NoError(t, ctx.kbClient.Create(t.Context(), tc.duResult)) } t.Run(tc.name, func(t *testing.T) { require.Equal(t, tc.expectedResult, hasSnapshotDataUpload(ctx, tc.obj)) }) } } func TestDetermineRestoreStatus(t *testing.T) { tests := []struct { name string annotations map[string]string restoreSpecIncludes *bool expectedDecision bool }{ { name: "No annotation, fallback to restore spec", annotations: nil, restoreSpecIncludes: boolptr.True(), expectedDecision: true, }, { name: "No annotation, restore spec excludes", annotations: nil, restoreSpecIncludes: boolptr.False(), expectedDecision: false, }, { name: "Annotation explicitly set to true, restore spec is false", annotations: map[string]string{ObjectStatusRestoreAnnotationKey: "true"}, restoreSpecIncludes: boolptr.False(), expectedDecision: true, }, { name: "Annotation explicitly set to false, restore spec is true", annotations: map[string]string{ObjectStatusRestoreAnnotationKey: "false"}, restoreSpecIncludes: boolptr.True(), expectedDecision: false, }, { name: "Invalid annotation value, fallback to restore spec", annotations: map[string]string{ObjectStatusRestoreAnnotationKey: "foo"}, restoreSpecIncludes: boolptr.True(), expectedDecision: true, }, { name: "Empty annotation value, fallback to restore spec", annotations: map[string]string{ObjectStatusRestoreAnnotationKey: ""}, restoreSpecIncludes: boolptr.False(), expectedDecision: false, }, { name: "Mixed-case annotation value 'True' should be treated as true", annotations: map[string]string{ObjectStatusRestoreAnnotationKey: "True"}, restoreSpecIncludes: boolptr.True(), expectedDecision: true, }, { name: "Mixed-case annotation value 'FALSE' should be treated as false", annotations: map[string]string{ObjectStatusRestoreAnnotationKey: "FALSE"}, restoreSpecIncludes: boolptr.True(), expectedDecision: false, }, { name: "Nil IncludesExcludes, but annotation is 'true'", annotations: map[string]string{ObjectStatusRestoreAnnotationKey: "true"}, restoreSpecIncludes: nil, expectedDecision: true, }, { name: "Nil IncludesExcludes, but annotation is 'false'", annotations: map[string]string{ObjectStatusRestoreAnnotationKey: "false"}, restoreSpecIncludes: nil, expectedDecision: false, }, { name: "Nil IncludesExcludes, no annotation (default to false)", annotations: nil, restoreSpecIncludes: nil, expectedDecision: false, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { obj := &unstructured.Unstructured{} obj.SetAnnotations(test.annotations) var includesExcludes *collections.IncludesExcludes if test.restoreSpecIncludes != nil { includesExcludes = collections.NewIncludesExcludes() if *test.restoreSpecIncludes { includesExcludes.Includes("*") } else { includesExcludes.Excludes("*") } } log := logrus.New() result := determineRestoreStatus(obj, includesExcludes, "testGroupResource", log) assert.Equal(t, test.expectedDecision, result) }) } } ================================================ FILE: pkg/restore/restore_wildcard_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package restore import ( "testing" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/archive" ) func TestExpandNamespaceWildcards(t *testing.T) { tests := []struct { name string includeNamespaces []string excludeNamespaces []string backupResources map[string]*archive.ResourceItems expectedIncludeMatches []string expectedExcludeMatches []string expectedWildcardResult []string expectedError string }{ { name: "No wildcards - should not expand", includeNamespaces: []string{"ns1", "ns2"}, excludeNamespaces: []string{"ns3"}, backupResources: map[string]*archive.ResourceItems{ "namespaces": {ItemsByNamespace: map[string][]string{"ns1": {}, "ns2": {}, "ns3": {}}}, }, expectedIncludeMatches: nil, expectedExcludeMatches: nil, expectedWildcardResult: nil, }, { name: "Simple wildcard include pattern", includeNamespaces: []string{"test*"}, excludeNamespaces: []string{}, backupResources: map[string]*archive.ResourceItems{ "namespaces": {ItemsByNamespace: map[string][]string{"test1": {}, "test2": {}, "prod1": {}}}, }, expectedIncludeMatches: []string{"test1", "test2"}, expectedExcludeMatches: []string{}, expectedWildcardResult: []string{"test1", "test2"}, }, { name: "Multiple wildcard patterns", includeNamespaces: []string{"test*", "dev*"}, excludeNamespaces: []string{}, backupResources: map[string]*archive.ResourceItems{ "namespaces": {ItemsByNamespace: map[string][]string{"test1": {}, "test2": {}, "dev1": {}, "prod1": {}}}, }, expectedIncludeMatches: []string{"dev1", "test1", "test2"}, expectedExcludeMatches: []string{}, expectedWildcardResult: []string{"dev1", "test1", "test2"}, }, { name: "Wildcard include with wildcard exclude", includeNamespaces: []string{"test*"}, excludeNamespaces: []string{"*-temp"}, backupResources: map[string]*archive.ResourceItems{ "namespaces": {ItemsByNamespace: map[string][]string{"test1": {}, "test2-temp": {}, "test3": {}}}, }, expectedIncludeMatches: []string{"test1", "test2-temp", "test3"}, expectedExcludeMatches: []string{"test2-temp"}, expectedWildcardResult: []string{"test1", "test3"}, }, { name: "Wildcard include with literal exclude", includeNamespaces: []string{"app-*"}, excludeNamespaces: []string{"app-test"}, backupResources: map[string]*archive.ResourceItems{ "namespaces": {ItemsByNamespace: map[string][]string{"app-prod": {}, "app-test": {}, "app-dev": {}}}, }, expectedIncludeMatches: []string{"app-dev", "app-prod", "app-test"}, expectedExcludeMatches: []string{"app-test"}, expectedWildcardResult: []string{"app-dev", "app-prod"}, }, { name: "Error: wildcard * in excludes", includeNamespaces: []string{"test*"}, excludeNamespaces: []string{"*"}, backupResources: map[string]*archive.ResourceItems{ "namespaces": {ItemsByNamespace: map[string][]string{"test1": {}}}, }, expectedError: "wildcard '*' is not allowed in restore excludes", }, { name: "Empty backup - no matches", includeNamespaces: []string{"test*"}, excludeNamespaces: []string{}, backupResources: map[string]*archive.ResourceItems{}, expectedIncludeMatches: []string{}, expectedExcludeMatches: []string{}, expectedWildcardResult: []string{}, }, { name: "Wildcard with no matches", includeNamespaces: []string{"nonexistent*"}, excludeNamespaces: []string{}, backupResources: map[string]*archive.ResourceItems{ "namespaces": {ItemsByNamespace: map[string][]string{"test1": {}, "test2": {}}}, }, expectedIncludeMatches: []string{}, expectedExcludeMatches: []string{}, expectedWildcardResult: []string{}, }, { name: "Complex pattern with prefix and suffix", includeNamespaces: []string{"app-*-prod"}, excludeNamespaces: []string{}, backupResources: map[string]*archive.ResourceItems{ "namespaces": {ItemsByNamespace: map[string][]string{"app-frontend-prod": {}, "app-backend-prod": {}, "app-frontend-dev": {}}}, }, expectedIncludeMatches: []string{"app-backend-prod", "app-frontend-prod"}, expectedExcludeMatches: []string{}, expectedWildcardResult: []string{"app-backend-prod", "app-frontend-prod"}, }, { name: "Backup with cluster resources", includeNamespaces: []string{"test*"}, excludeNamespaces: []string{}, backupResources: map[string]*archive.ResourceItems{ "namespaces": {ItemsByNamespace: map[string][]string{"test1": {}, "test2": {}}}, "persistentvolumes": {ItemsByNamespace: map[string][]string{"": {}}}, // cluster-scoped "pods.v1": {ItemsByNamespace: map[string][]string{"test1": {"pod1"}}}, }, expectedIncludeMatches: []string{"test1", "test2"}, expectedExcludeMatches: []string{}, expectedWildcardResult: []string{"test1", "test2"}, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { restore := &velerov1api.Restore{ Spec: velerov1api.RestoreSpec{ IncludedNamespaces: tc.includeNamespaces, ExcludedNamespaces: tc.excludeNamespaces, }, } ctx := &restoreContext{ restore: restore, log: logrus.StandardLogger(), } err := ctx.expandNamespaceWildcards(tc.backupResources) if tc.expectedError != "" { require.Error(t, err) assert.Contains(t, err.Error(), tc.expectedError) return } require.NoError(t, err) }) } } func TestExtractNamespacesFromBackup(t *testing.T) { tests := []struct { name string backupResources map[string]*archive.ResourceItems expected []string }{ { name: "Multiple namespaces in backup", backupResources: map[string]*archive.ResourceItems{ "namespaces": {ItemsByNamespace: map[string][]string{"ns1": {}, "ns2": {}, "ns3": {}}}, }, expected: []string{"ns1", "ns2", "ns3"}, }, { name: "Namespaces with resources", backupResources: map[string]*archive.ResourceItems{ "namespaces": {ItemsByNamespace: map[string][]string{"app1": {}, "app2": {}}}, "pods.v1": {ItemsByNamespace: map[string][]string{"app1": {"pod1"}}}, "services.v1": {ItemsByNamespace: map[string][]string{"app2": {"svc1"}}}, }, expected: []string{"app1", "app2"}, }, { name: "Mixed cluster and namespaced resources", backupResources: map[string]*archive.ResourceItems{ "namespaces": {ItemsByNamespace: map[string][]string{"test": {}}}, "persistentvolumes": {ItemsByNamespace: map[string][]string{"": {"pv1"}}}, "clusterroles": {ItemsByNamespace: map[string][]string{"": {"cr1"}}}, "pods.v1": {ItemsByNamespace: map[string][]string{"test": {"pod1"}}}, }, expected: []string{"test"}, }, { name: "Empty backup", backupResources: map[string]*archive.ResourceItems{}, expected: []string{}, }, { name: "Only cluster resources", backupResources: map[string]*archive.ResourceItems{ "persistentvolumes": {ItemsByNamespace: map[string][]string{"": {"pv1"}}}, "clusterroles": {ItemsByNamespace: map[string][]string{"": {"cr1"}}}, "storageclasses": {ItemsByNamespace: map[string][]string{"": {"sc1"}}}, }, expected: []string{}, }, { name: "Duplicate namespace entries", backupResources: map[string]*archive.ResourceItems{ "namespaces": {ItemsByNamespace: map[string][]string{"app": {}}}, "pods.v1": {ItemsByNamespace: map[string][]string{"app": {"pod1"}}}, "configmaps.v1": {ItemsByNamespace: map[string][]string{"app": {"cm1"}}}, }, expected: []string{"app"}, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { result := extractNamespacesFromBackup(tc.backupResources) assert.ElementsMatch(t, tc.expected, result) }) } } ================================================ FILE: pkg/restorehelper/util.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package restorehelper const ( // WaitInitContainer is the name of the init container added // to workload pods to help with restores. // If Velero needs to further process the volume data after PVC is // provisioned, this init container is used to block Pod from running // until the volume data is ready WaitInitContainer = "restore-wait" // This is the name of the init container added by pre-v1.10 for the same // purpose with WaitInitContainer. // For compatibility, we need to check it when restoring backups created by // old releases. The pods backed up by old releases may contain this init container // since the init container is not deleted after pod is restored. WaitInitContainerLegacy = "restic-wait" ) ================================================ FILE: pkg/test/api_server.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package test import ( "testing" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" discoveryfake "k8s.io/client-go/discovery/fake" dynamicfake "k8s.io/client-go/dynamic/fake" kubefake "k8s.io/client-go/kubernetes/fake" ) // APIServer contains in-memory fakes for all of the relevant // Kubernetes API server clients. type APIServer struct { KubeClient *kubefake.Clientset DynamicClient *dynamicfake.FakeDynamicClient DiscoveryClient *DiscoveryClient } // NewAPIServer constructs an APIServer with all of its clients // initialized. func NewAPIServer(t *testing.T) *APIServer { t.Helper() var ( kubeClient = kubefake.NewSimpleClientset() dynamicClient = dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), map[schema.GroupVersionResource]string{ {Group: "", Version: "v1", Resource: "namespaces"}: "NamespacesList", {Group: "", Version: "v1", Resource: "pods"}: "PodsList", {Group: "", Version: "v1", Resource: "persistentvolumes"}: "PVList", {Group: "", Version: "v1", Resource: "persistentvolumeclaims"}: "PVCList", {Group: "", Version: "v1", Resource: "secrets"}: "SecretsList", {Group: "", Version: "v1", Resource: "serviceaccounts"}: "ServiceAccountsList", {Group: "apps", Version: "v1", Resource: "deployments"}: "DeploymentsList", {Group: "apiextensions.k8s.io", Version: "v1beta1", Resource: "customresourcedefinitions"}: "CRDList", {Group: "velero.io", Version: "v1", Resource: "volumesnapshotlocations"}: "VSLList", {Group: "velero.io", Version: "v1", Resource: "backups"}: "BackupList", {Group: "extensions", Version: "v1", Resource: "deployments"}: "ExtDeploymentsList", {Group: "velero.io", Version: "v1", Resource: "deployments"}: "VeleroDeploymentsList", {Group: "velero.io", Version: "v2alpha1", Resource: "datauploads"}: "DataUploadsList", }) discoveryClient = &DiscoveryClient{FakeDiscovery: kubeClient.Discovery().(*discoveryfake.FakeDiscovery)} ) return &APIServer{ KubeClient: kubeClient, DynamicClient: dynamicClient, DiscoveryClient: discoveryClient, } } ================================================ FILE: pkg/test/comparisons.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package test import ( "bytes" "encoding/json" "fmt" "reflect" "testing" "time" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/util/diff" core "k8s.io/client-go/testing" ) // CompareActions checks slices of actual and expected Actions // for equality (ignoring order). It checks that the lengths of // the slices are the same, that each actual Action has a // corresponding expected Action, and that each expected Action // has a corresponding actual Action. func CompareActions(t *testing.T, expected, actual []core.Action) { t.Helper() assert.Len(t, actual, len(expected)) for _, e := range expected { found := false for _, a := range actual { if reflect.DeepEqual(e, a) { found = true break } } if !found { t.Errorf("missing expected action %#v", e) } } for _, a := range actual { found := false for _, e := range expected { if reflect.DeepEqual(e, a) { found = true break } } if !found { t.Errorf("unexpected action %#v", a) } } } // ValidatePatch tests the validity of an action. It checks // that the action is a PatchAction, that the patch decodes from JSON // with the provided decode func and has no extraneous fields, and that // the decoded patch matches the expected. func ValidatePatch(t *testing.T, action core.Action, expected any, decodeFunc func(*json.Decoder) (any, error)) { t.Helper() patchAction, ok := action.(core.PatchAction) require.True(t, ok, "action is not a PatchAction") decoder := json.NewDecoder(bytes.NewReader(patchAction.GetPatch())) decoder.DisallowUnknownFields() actual, err := decodeFunc(decoder) require.NoError(t, err) AssertDeepEqual(t, expected, actual) } // TimesAreEqual compares two times for equality. // This function is used by equality.Semantic.DeepEqual to compare two time objects // without having to call a method. func TimesAreEqual(t1, t2 time.Time) bool { return t1.Equal(t2) } // AssertDeepEqual asserts the semantic equality of objects. // This function exists in order to make sure time.Time and metav1.Time objects // can be compared correctly. See https://github.com/stretchr/testify/issues/502. func AssertDeepEqual(t *testing.T, expected, actual any) bool { t.Helper() // By default, the equality.Semantic object doesn't have a function for comparing time.Times err := equality.Semantic.AddFunc(TimesAreEqual) if err != nil { // Programmer error, the test should die. t.Fatalf("Could not register equality function: %s", err) } if !equality.Semantic.DeepEqual(expected, actual) { s := diff.ObjectDiff(expected, actual) return assert.Fail(t, fmt.Sprintf("Objects not equal:\n\n%s", s)) } return true } // AssertErrorMatches asserts that if expected is the empty string, actual // is nil, otherwise, that actual's error string matches expected. func AssertErrorMatches(t *testing.T, expected string, actual error) bool { t.Helper() if expected != "" { return assert.EqualError(t, actual, expected) } return assert.NoError(t, actual) } func CompareSlice(x, y []string) bool { less := func(a, b string) bool { return a < b } return cmp.Diff(x, y, cmpopts.SortSlices(less)) == "" } ================================================ FILE: pkg/test/discovery_client.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package test import ( "strings" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/discovery" discoveryfake "k8s.io/client-go/discovery/fake" ) // DiscoveryClient is a wrapper for the client-go FakeDiscovery struct. It // adds some extra functionality that's necessary/useful for Velero tests. type DiscoveryClient struct { *discoveryfake.FakeDiscovery } func (c *DiscoveryClient) ServerPreferredResources() ([]*metav1.APIResourceList, error) { return discovery.ServerPreferredResources(c) } // // TEST HELPERS // // WithAPIResource adds the API resource to the discovery client. func (c *DiscoveryClient) WithAPIResource(resource *APIResource) *DiscoveryClient { gv := metav1.GroupVersion{ Group: resource.Group, Version: resource.Version, } var resourceList *metav1.APIResourceList for _, itm := range c.Resources { if itm.GroupVersion == gv.String() { resourceList = itm break } } if resourceList == nil { resourceList = &metav1.APIResourceList{ GroupVersion: gv.String(), } c.Resources = append(c.Resources, resourceList) } for _, itm := range resourceList.APIResources { if itm.Name == resource.Name { return c } } resourceList.APIResources = append(resourceList.APIResources, metav1.APIResource{ Name: resource.Name, SingularName: strings.TrimSuffix(resource.Name, "s"), Namespaced: resource.Namespaced, Group: resource.Group, Version: resource.Version, Kind: resource.Kind, Verbs: metav1.Verbs([]string{"list", "create", "get", "delete"}), ShortNames: []string{resource.ShortName}, }) return c } func (c *DiscoveryClient) GroupsAndMaybeResources() (*metav1.APIGroupList, map[schema.GroupVersion]*metav1.APIResourceList, map[schema.GroupVersion]error, error) { apiGroupList, err := c.ServerGroups() if err != nil { return nil, nil, nil, err } return apiGroupList, nil, nil, nil } ================================================ FILE: pkg/test/fake_controller_runtime_client.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package test import ( "testing" volumegroupsnapshotv1beta1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1beta1" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" "github.com/stretchr/testify/require" appsv1api "k8s.io/api/apps/v1" batchv1api "k8s.io/api/batch/v1" corev1api "k8s.io/api/core/v1" storagev1api "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" k8sfake "sigs.k8s.io/controller-runtime/pkg/client/fake" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" ) func NewFakeControllerRuntimeClientBuilder(t *testing.T) *k8sfake.ClientBuilder { t.Helper() scheme := runtime.NewScheme() require.NoError(t, velerov1api.AddToScheme(scheme)) require.NoError(t, velerov2alpha1api.AddToScheme(scheme)) require.NoError(t, corev1api.AddToScheme(scheme)) require.NoError(t, appsv1api.AddToScheme(scheme)) require.NoError(t, snapshotv1api.AddToScheme(scheme)) require.NoError(t, storagev1api.AddToScheme(scheme)) return k8sfake.NewClientBuilder().WithScheme(scheme) } func NewFakeControllerRuntimeClient(t *testing.T, initObjs ...runtime.Object) client.Client { t.Helper() scheme := runtime.NewScheme() require.NoError(t, velerov1api.AddToScheme(scheme)) require.NoError(t, velerov2alpha1api.AddToScheme(scheme)) require.NoError(t, corev1api.AddToScheme(scheme)) require.NoError(t, appsv1api.AddToScheme(scheme)) require.NoError(t, snapshotv1api.AddToScheme(scheme)) require.NoError(t, storagev1api.AddToScheme(scheme)) require.NoError(t, batchv1api.AddToScheme(scheme)) require.NoError(t, volumegroupsnapshotv1beta1.AddToScheme(scheme)) return k8sfake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(initObjs...).Build() } func NewFakeControllerRuntimeWatchClient( t *testing.T, initObjs ...runtime.Object, ) client.WithWatch { t.Helper() return NewFakeControllerRuntimeClientBuilder(t).WithRuntimeObjects(initObjs...).Build() } ================================================ FILE: pkg/test/fake_credential_file_store.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package test import ( corev1api "k8s.io/api/core/v1" ) // FileStore defines operations for interacting with credentials // that are stored on a file system. type FileStore interface { // Path returns a path on disk where the secret key defined by // the given selector is serialized. Path(selector *corev1api.SecretKeySelector) (string, error) } type fakeCredentialsFileStore struct { path string err error } // Path returns a path on disk where the secret key defined by // the given selector is serialized. func (f *fakeCredentialsFileStore) Path(*corev1api.SecretKeySelector) (string, error) { return f.path, f.err } // NewFakeCredentialsFileStore creates a FileStore which will return the given path // and error when Path is called. func NewFakeCredentialsFileStore(path string, err error) FileStore { return &fakeCredentialsFileStore{ path: path, err: err, } } // SecretStore defines operations for interacting with credentials // that are stored in Secret. type SecretStore interface { // Get returns the secret key defined by the given selector Get(selector *corev1api.SecretKeySelector) (string, error) } type fakeCredentialsSecretStore struct { data string err error } // Get returns the secret data. func (f *fakeCredentialsSecretStore) Get(*corev1api.SecretKeySelector) (string, error) { return f.data, f.err } // NewFakeCredentialsSecretStore creates a SecretStore which will return the given data // and error when Get is called. // data is the secret value to return (e.g., certificate content). // err is the error to return, if any. func NewFakeCredentialsSecretStore(data string, err error) SecretStore { return &fakeCredentialsSecretStore{ data: data, err: err, } } ================================================ FILE: pkg/test/fake_discovery_helper.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package test import ( "errors" "fmt" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/version" "k8s.io/client-go/discovery" ) type FakeDiscoveryHelper struct { ResourceList []*metav1.APIResourceList Mapper meta.RESTMapper AutoReturnResource bool APIGroupsList []metav1.APIGroup ServerVersionData *version.Info } func (dh *FakeDiscoveryHelper) KindFor(input schema.GroupVersionKind) (schema.GroupVersionResource, metav1.APIResource, error) { panic("implement me") } func NewFakeDiscoveryHelper(autoReturnResource bool, resources map[schema.GroupVersionResource]schema.GroupVersionResource) *FakeDiscoveryHelper { helper := &FakeDiscoveryHelper{ AutoReturnResource: autoReturnResource, Mapper: &FakeMapper{ Resources: resources, }, } if resources == nil { return helper } apiResourceMap := make(map[string][]metav1.APIResource) for _, gvr := range resources { var gvString string if gvr.Version != "" && gvr.Group != "" { gvString = fmt.Sprintf("%s/%s", gvr.Group, gvr.Version) } else { gvString = fmt.Sprintf("%s%s", gvr.Group, gvr.Version) } apiResourceMap[gvString] = append(apiResourceMap[gvString], metav1.APIResource{Name: gvr.Resource}) helper.APIGroupsList = append(helper.APIGroupsList, metav1.APIGroup{ Name: gvr.Group, PreferredVersion: metav1.GroupVersionForDiscovery{ GroupVersion: gvString, Version: gvr.Version, }, }) } for group, resources := range apiResourceMap { helper.ResourceList = append(helper.ResourceList, &metav1.APIResourceList{GroupVersion: group, APIResources: resources}) } // FakeTest of version.Info serverVersion := &version.Info{ Major: "1", Minor: "16", GitVersion: "v1.16.4", GitCommit: "FakeTest", GitTreeState: "", BuildDate: "", GoVersion: "", Compiler: "", Platform: "", } helper.ServerVersionData = serverVersion return helper } func (dh *FakeDiscoveryHelper) Resources() []*metav1.APIResourceList { return dh.ResourceList } func (dh *FakeDiscoveryHelper) Refresh() error { return nil } func (dh *FakeDiscoveryHelper) ResourceFor(input schema.GroupVersionResource) (schema.GroupVersionResource, metav1.APIResource, error) { if dh.AutoReturnResource { return schema.GroupVersionResource{ Group: input.Group, Version: input.Version, Resource: input.Resource, }, metav1.APIResource{ Name: input.Resource, }, nil } gvr, err := dh.Mapper.ResourceFor(input) if err != nil { return schema.GroupVersionResource{}, metav1.APIResource{}, err } var gvString string if gvr.Version != "" && gvr.Group != "" { gvString = fmt.Sprintf("%s/%s", gvr.Group, gvr.Version) } else { gvString = gvr.Version + gvr.Group } for _, gr := range dh.ResourceList { if gr.GroupVersion != gvString { continue } for _, resource := range gr.APIResources { if resource.Name == gvr.Resource { return gvr, resource, nil } } } return schema.GroupVersionResource{}, metav1.APIResource{}, errors.New("APIResource not found") } func (dh *FakeDiscoveryHelper) APIGroups() []metav1.APIGroup { return dh.APIGroupsList } type FakeServerResourcesInterface struct { ResourceList []*metav1.APIResourceList APIGroup []*metav1.APIGroup FailedGroups map[schema.GroupVersion]error ReturnError error } func (di *FakeServerResourcesInterface) ServerPreferredResources() ([]*metav1.APIResourceList, error) { if di.ReturnError != nil { return di.ResourceList, di.ReturnError } if len(di.FailedGroups) == 0 { return di.ResourceList, nil } return di.ResourceList, &discovery.ErrGroupDiscoveryFailed{Groups: di.FailedGroups} } func (di *FakeServerResourcesInterface) ServerGroupsAndResources() ([]*metav1.APIGroup, []*metav1.APIResourceList, error) { if di.ReturnError != nil { return di.APIGroup, di.ResourceList, di.ReturnError } if len(di.FailedGroups) == 0 { return di.APIGroup, di.ResourceList, nil } return di.APIGroup, di.ResourceList, &discovery.ErrGroupDiscoveryFailed{Groups: di.FailedGroups} } func NewFakeServerResourcesInterface(resourceList []*metav1.APIResourceList, apiGroup []*metav1.APIGroup, failedGroups map[schema.GroupVersion]error, returnError error) *FakeServerResourcesInterface { helper := &FakeServerResourcesInterface{ ResourceList: resourceList, APIGroup: apiGroup, FailedGroups: failedGroups, ReturnError: returnError, } return helper } func (dh *FakeDiscoveryHelper) ServerVersion() *version.Info { return dh.ServerVersionData } ================================================ FILE: pkg/test/fake_dynamic.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package test import ( "github.com/stretchr/testify/mock" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/dynamic/dynamicinformer" "github.com/vmware-tanzu/velero/pkg/client" ) type FakeDynamicFactory struct { mock.Mock } var _ client.DynamicFactory = &FakeDynamicFactory{} func (df *FakeDynamicFactory) ClientForGroupVersionResource(gv schema.GroupVersion, resource metav1.APIResource, namespace string) (client.Dynamic, error) { args := df.Called(gv, resource, namespace) return args.Get(0).(client.Dynamic), args.Error(1) } func (df *FakeDynamicFactory) DynamicSharedInformerFactory() dynamicinformer.DynamicSharedInformerFactory { args := df.Called() return args.Get(0).(dynamicinformer.DynamicSharedInformerFactory) } type FakeDynamicClient struct { mock.Mock } var _ client.Dynamic = &FakeDynamicClient{} func (c *FakeDynamicClient) List(options metav1.ListOptions) (*unstructured.UnstructuredList, error) { args := c.Called(options) return args.Get(0).(*unstructured.UnstructuredList), args.Error(1) } func (c *FakeDynamicClient) Create(obj *unstructured.Unstructured) (*unstructured.Unstructured, error) { args := c.Called(obj) return args.Get(0).(*unstructured.Unstructured), args.Error(1) } func (c *FakeDynamicClient) Watch(options metav1.ListOptions) (watch.Interface, error) { args := c.Called(options) return args.Get(0).(watch.Interface), args.Error(1) } func (c *FakeDynamicClient) Get(name string, opts metav1.GetOptions) (*unstructured.Unstructured, error) { args := c.Called(name, opts) return args.Get(0).(*unstructured.Unstructured), args.Error(1) } func (c *FakeDynamicClient) Patch(name string, data []byte) (*unstructured.Unstructured, error) { args := c.Called(name, data) return args.Get(0).(*unstructured.Unstructured), args.Error(1) } func (c *FakeDynamicClient) Delete(name string, opts metav1.DeleteOptions) error { args := c.Called(name, opts) return args.Error(1) } func (c *FakeDynamicClient) UpdateStatus(obj *unstructured.Unstructured, opts metav1.UpdateOptions) (*unstructured.Unstructured, error) { args := c.Called(obj, opts) return args.Get(0).(*unstructured.Unstructured), args.Error(1) } func (c *FakeDynamicClient) Apply(name string, obj *unstructured.Unstructured, opts metav1.ApplyOptions) (*unstructured.Unstructured, error) { args := c.Called(name, obj, opts) return args.Get(0).(*unstructured.Unstructured), args.Error(1) } ================================================ FILE: pkg/test/fake_file_system.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package test import ( "io" "io/fs" "os" "github.com/spf13/afero" "github.com/vmware-tanzu/velero/pkg/util/filesystem" ) type FakeFileSystem struct { fs afero.Fs ReadDirCalls []string } func NewFakeFileSystem() *FakeFileSystem { return &FakeFileSystem{ fs: afero.NewMemMapFs(), } } func (fs *FakeFileSystem) Glob(path string) ([]string, error) { return afero.Glob(fs.fs, path) } func (fs *FakeFileSystem) TempDir(dir, prefix string) (string, error) { return afero.TempDir(fs.fs, dir, prefix) } func (fs *FakeFileSystem) MkdirAll(path string, perm os.FileMode) error { return fs.fs.MkdirAll(path, perm) } func (fs *FakeFileSystem) Create(name string) (io.WriteCloser, error) { return fs.fs.Create(name) } func (fs *FakeFileSystem) OpenFile(name string, flag int, perm os.FileMode) (io.WriteCloser, error) { return fs.fs.OpenFile(name, flag, perm) } func (fs *FakeFileSystem) RemoveAll(path string) error { return fs.fs.RemoveAll(path) } func (fs *FakeFileSystem) ReadDir(dirname string) ([]fs.FileInfo, error) { fs.ReadDirCalls = append(fs.ReadDirCalls, dirname) return afero.ReadDir(fs.fs, dirname) } func (fs *FakeFileSystem) ReadFile(filename string) ([]byte, error) { return afero.ReadFile(fs.fs, filename) } func (fs *FakeFileSystem) DirExists(path string) (bool, error) { return afero.DirExists(fs.fs, path) } func (fs *FakeFileSystem) Stat(path string) (os.FileInfo, error) { return fs.fs.Stat(path) } func (fs *FakeFileSystem) WithFile(path string, data []byte) *FakeFileSystem { file, _ := fs.fs.Create(path) _, _ = file.Write(data) file.Close() return fs } func (fs *FakeFileSystem) WithFileAndMode(path string, data []byte, mode os.FileMode) *FakeFileSystem { file, _ := fs.fs.OpenFile(path, os.O_CREATE|os.O_RDWR, mode) _, _ = file.Write(data) file.Close() return fs } func (fs *FakeFileSystem) WithDirectory(path string) *FakeFileSystem { _ = fs.fs.MkdirAll(path, 0755) return fs } func (fs *FakeFileSystem) WithDirectories(path ...string) *FakeFileSystem { for _, dir := range path { fs = fs.WithDirectory(dir) } return fs } func (fs *FakeFileSystem) TempFile(dir, prefix string) (filesystem.NameWriteCloser, error) { return afero.TempFile(fs.fs, dir, prefix) } ================================================ FILE: pkg/test/fake_mapper.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package test import ( "github.com/pkg/errors" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" ) type FakeMapper struct { meta.RESTMapper AutoReturnResource bool Resources map[schema.GroupVersionResource]schema.GroupVersionResource KindToPluralResource map[schema.GroupVersionKind]schema.GroupVersionResource } func (m *FakeMapper) ResourceFor(input schema.GroupVersionResource) (schema.GroupVersionResource, error) { if m.AutoReturnResource { return schema.GroupVersionResource{ Group: input.Group, Version: input.Version, Resource: input.Resource, }, nil } if m.Resources == nil { return schema.GroupVersionResource{}, errors.Errorf("invalid resource %q", input.String()) } if gr, found := m.Resources[input]; found { return gr, nil } if input.Version == "" { input.Version = "v1" if gr, found := m.Resources[input]; found { return gr, nil } input.Version = "v1beta1" if gr, found := m.Resources[input]; found { return gr, nil } } return schema.GroupVersionResource{}, errors.Errorf("invalid resource %q", input.String()) } func (m *FakeMapper) RESTMapping(gk schema.GroupKind, versions ...string) (*meta.RESTMapping, error) { potentialGVK := make([]schema.GroupVersionKind, 0) // Pick an appropriate version for _, version := range versions { if len(version) == 0 || version == runtime.APIVersionInternal { continue } currGVK := gk.WithVersion(version) if _, ok := m.KindToPluralResource[currGVK]; ok { potentialGVK = append(potentialGVK, currGVK) break } } if len(potentialGVK) == 0 { return nil, &meta.NoKindMatchError{GroupKind: gk, SearchedVersions: versions} } for _, gvk := range potentialGVK { //Ensure we have a REST mapping res, ok := m.KindToPluralResource[gvk] if !ok { continue } return &meta.RESTMapping{ Resource: res, GroupVersionKind: gvk, Scope: meta.RESTScopeNamespace, }, nil } return nil, &meta.NoResourceMatchError{PartialResource: schema.GroupVersionResource{Group: gk.Group, Resource: gk.Kind}} } ================================================ FILE: pkg/test/fake_namespace.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package test import ( "context" "github.com/stretchr/testify/mock" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/watch" v1 "k8s.io/client-go/applyconfigurations/core/v1" corev1 "k8s.io/client-go/kubernetes/typed/core/v1" ) type FakeNamespaceClient struct { mock.Mock } var _ corev1.NamespaceInterface = &FakeNamespaceClient{} func (c *FakeNamespaceClient) List(ctx context.Context, options metav1.ListOptions) (*corev1api.NamespaceList, error) { args := c.Called(options) return args.Get(0).(*corev1api.NamespaceList), args.Error(1) } func (c *FakeNamespaceClient) Create(ctx context.Context, obj *corev1api.Namespace, options metav1.CreateOptions) (*corev1api.Namespace, error) { args := c.Called(obj) return args.Get(0).(*corev1api.Namespace), args.Error(1) } func (c *FakeNamespaceClient) Watch(ctx context.Context, options metav1.ListOptions) (watch.Interface, error) { args := c.Called(options) return args.Get(0).(watch.Interface), args.Error(1) } func (c *FakeNamespaceClient) Get(ctx context.Context, name string, opts metav1.GetOptions) (*corev1api.Namespace, error) { args := c.Called(name, opts) return args.Get(0).(*corev1api.Namespace), args.Error(1) } func (c *FakeNamespaceClient) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (*corev1api.Namespace, error) { args := c.Called(name, pt, data, subresources) return args.Get(0).(*corev1api.Namespace), args.Error(1) } func (c *FakeNamespaceClient) Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error { args := c.Called(name, opts) return args.Error(1) } func (c *FakeNamespaceClient) Finalize(ctx context.Context, item *corev1api.Namespace, options metav1.UpdateOptions) (*corev1api.Namespace, error) { args := c.Called(item) return args.Get(0).(*corev1api.Namespace), args.Error(1) } func (c *FakeNamespaceClient) Update(ctx context.Context, namespace *corev1api.Namespace, options metav1.UpdateOptions) (*corev1api.Namespace, error) { args := c.Called(namespace) return args.Get(0).(*corev1api.Namespace), args.Error(1) } func (c *FakeNamespaceClient) UpdateStatus(ctx context.Context, namespace *corev1api.Namespace, options metav1.UpdateOptions) (*corev1api.Namespace, error) { args := c.Called(namespace) return args.Get(0).(*corev1api.Namespace), args.Error(1) } func (c *FakeNamespaceClient) Apply(ctx context.Context, namespace *v1.NamespaceApplyConfiguration, opts metav1.ApplyOptions) (result *corev1api.Namespace, err error) { args := c.Called(namespace) return args.Get(0).(*corev1api.Namespace), args.Error(1) } func (c *FakeNamespaceClient) ApplyStatus(ctx context.Context, namespace *v1.NamespaceApplyConfiguration, opts metav1.ApplyOptions) (result *corev1api.Namespace, err error) { args := c.Called(namespace) return args.Get(0).(*corev1api.Namespace), args.Error(1) } ================================================ FILE: pkg/test/fake_volume_snapshotter.go ================================================ /* Copyright 2017, 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package test import ( "errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" ) type VolumeBackupInfo struct { SnapshotID string Type string Iops *int64 AvailabilityZone string } type FakeVolumeSnapshotter struct { // SnapshotID->VolumeID SnapshotsTaken sets.String // VolumeID -> (SnapshotID, Type, Iops) SnapshottableVolumes map[string]VolumeBackupInfo // VolumeBackupInfo -> VolumeID RestorableVolumes map[VolumeBackupInfo]string VolumeID string VolumeIDSet string Error error } func (bs *FakeVolumeSnapshotter) Init(config map[string]string) error { return nil } func (bs *FakeVolumeSnapshotter) CreateSnapshot(volumeID, volumeAZ string, tags map[string]string) (string, error) { if bs.Error != nil { return "", bs.Error } if _, exists := bs.SnapshottableVolumes[volumeID]; !exists { return "", errors.New("snapshottable volume not found") } if bs.SnapshotsTaken == nil { bs.SnapshotsTaken = sets.NewString() } bs.SnapshotsTaken.Insert(bs.SnapshottableVolumes[volumeID].SnapshotID) return bs.SnapshottableVolumes[volumeID].SnapshotID, nil } func (bs *FakeVolumeSnapshotter) CreateVolumeFromSnapshot(snapshotID, volumeType, volumeAZ string, iops *int64) (string, error) { if bs.Error != nil { return "", bs.Error } key := VolumeBackupInfo{ SnapshotID: snapshotID, Type: volumeType, Iops: iops, AvailabilityZone: volumeAZ, } return bs.RestorableVolumes[key], nil } func (bs *FakeVolumeSnapshotter) DeleteSnapshot(snapshotID string) error { if bs.Error != nil { return bs.Error } if !bs.SnapshotsTaken.Has(snapshotID) { return errors.New("snapshot not found") } bs.SnapshotsTaken.Delete(snapshotID) return nil } func (bs *FakeVolumeSnapshotter) GetVolumeInfo(volumeID, volumeAZ string) (string, *int64, error) { if bs.Error != nil { return "", nil, bs.Error } volumeInfo, exists := bs.SnapshottableVolumes[volumeID] if !exists { return "", nil, errors.New("VolumeID not found") } return volumeInfo.Type, volumeInfo.Iops, nil } func (bs *FakeVolumeSnapshotter) GetVolumeID(pv runtime.Unstructured) (string, error) { return bs.VolumeID, nil } func (bs *FakeVolumeSnapshotter) SetVolumeID(pv runtime.Unstructured, volumeID string) (runtime.Unstructured, error) { bs.VolumeIDSet = volumeID return pv, bs.Error } ================================================ FILE: pkg/test/helpers.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package test import ( "encoding/json" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) func UnstructuredOrDie(data string) *unstructured.Unstructured { o, _, err := unstructured.UnstructuredJSONScheme.Decode([]byte(data), nil, nil) if err != nil { panic(err) } return o.(*unstructured.Unstructured) } func GetAsMap(j string) (map[string]any, error) { m := make(map[string]any) err := json.Unmarshal([]byte(j), &m) return m, err } ================================================ FILE: pkg/test/mock_pod_command_executor.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package test import ( "fmt" "strings" "github.com/sirupsen/logrus" "github.com/stretchr/testify/mock" v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) type MockPodCommandExecutor struct { mock.Mock // hook execution order HookExecutionLog []HookExecutionEntry } type HookExecutionEntry struct { Namespace, Name, HookName string HookCommand []string } func (h HookExecutionEntry) String() string { return fmt.Sprintf("%s.%s.%s.%s", h.Namespace, h.Name, h.HookName, strings.Join(h.HookCommand, ",")) } func (e *MockPodCommandExecutor) ExecutePodCommand(log logrus.FieldLogger, item map[string]any, namespace, name, hookName string, hook *v1.ExecHook) error { e.HookExecutionLog = append(e.HookExecutionLog, HookExecutionEntry{ Namespace: namespace, Name: name, HookName: hookName, HookCommand: hook.Command, }) args := e.Called(log, item, namespace, name, hookName, hook) return args.Error(0) } ================================================ FILE: pkg/test/mocks/VolumeSnapshotLister.go ================================================ // Code generated by mockery v2.35.4. DO NOT EDIT. package mocks import ( mock "github.com/stretchr/testify/mock" labels "k8s.io/apimachinery/pkg/labels" v1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" volumesnapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v8/listers/volumesnapshot/v1" ) // VolumeSnapshotLister is an autogenerated mock type for the VolumeSnapshotLister type type VolumeSnapshotLister struct { mock.Mock } // List provides a mock function with given fields: selector func (_m *VolumeSnapshotLister) List(selector labels.Selector) ([]*v1.VolumeSnapshot, error) { ret := _m.Called(selector) var r0 []*v1.VolumeSnapshot var r1 error if rf, ok := ret.Get(0).(func(labels.Selector) ([]*v1.VolumeSnapshot, error)); ok { return rf(selector) } if rf, ok := ret.Get(0).(func(labels.Selector) []*v1.VolumeSnapshot); ok { r0 = rf(selector) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]*v1.VolumeSnapshot) } } if rf, ok := ret.Get(1).(func(labels.Selector) error); ok { r1 = rf(selector) } else { r1 = ret.Error(1) } return r0, r1 } // VolumeSnapshots provides a mock function with given fields: namespace func (_m *VolumeSnapshotLister) VolumeSnapshots(namespace string) volumesnapshotv1.VolumeSnapshotNamespaceLister { ret := _m.Called(namespace) var r0 volumesnapshotv1.VolumeSnapshotNamespaceLister if rf, ok := ret.Get(0).(func(string) volumesnapshotv1.VolumeSnapshotNamespaceLister); ok { r0 = rf(namespace) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(volumesnapshotv1.VolumeSnapshotNamespaceLister) } } return r0 } // NewVolumeSnapshotLister creates a new instance of VolumeSnapshotLister. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. func NewVolumeSnapshotLister(t interface { mock.TestingT Cleanup(func()) }) *VolumeSnapshotLister { mock := &VolumeSnapshotLister{} mock.Mock.Test(t) t.Cleanup(func() { mock.AssertExpectations(t) }) return mock } ================================================ FILE: pkg/test/mocks.go ================================================ package test import ( snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" snapshotv1listers "github.com/kubernetes-csi/external-snapshotter/client/v8/listers/volumesnapshot/v1" "k8s.io/apimachinery/pkg/labels" ) // VolumeSnapshotLister helps list VolumeSnapshots. // All objects returned here must be treated as read-only. // //go:generate mockery --name VolumeSnapshotLister type VolumeSnapshotLister interface { // List lists all VolumeSnapshots in the indexer. // Objects returned here must be treated as read-only. List(selector labels.Selector) (ret []*snapshotv1.VolumeSnapshot, err error) // VolumeSnapshots returns an object that can list and get VolumeSnapshots. VolumeSnapshots(namespace string) snapshotv1listers.VolumeSnapshotNamespaceLister snapshotv1listers.VolumeSnapshotListerExpansion } ================================================ FILE: pkg/test/resources.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package test import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" ) // APIResource stores information about a specific Kubernetes API // resource. type APIResource struct { Group string Version string Name string Kind string ShortName string Namespaced bool Items []metav1.Object } // GVR returns a GroupVersionResource representing the resource. func (r *APIResource) GVR() schema.GroupVersionResource { return schema.GroupVersionResource{ Group: r.Group, Version: r.Version, Resource: r.Name, } } // Pods returns an APIResource describing core/v1's Pods. func Pods(items ...metav1.Object) *APIResource { return &APIResource{ Group: "", Version: "v1", Name: "pods", ShortName: "po", Namespaced: true, Items: items, Kind: "Pod", } } func PVCs(items ...metav1.Object) *APIResource { return &APIResource{ Group: "", Version: "v1", Name: "persistentvolumeclaims", ShortName: "pvc", Kind: "PersistentVolumeClaim", Namespaced: true, Items: items, } } func PVs(items ...metav1.Object) *APIResource { return &APIResource{ Group: "", Version: "v1", Name: "persistentvolumes", ShortName: "pv", Kind: "PersistentVolume", Namespaced: false, Items: items, } } func Secrets(items ...metav1.Object) *APIResource { return &APIResource{ Group: "", Version: "v1", Name: "secrets", ShortName: "secrets", Kind: "Secret", Namespaced: true, Items: items, } } func Deployments(items ...metav1.Object) *APIResource { return &APIResource{ Group: "apps", Version: "v1", Name: "deployments", ShortName: "deploy", Kind: "Deployment", Namespaced: true, Items: items, } } func ExtensionsDeployments(items ...metav1.Object) *APIResource { return &APIResource{ Group: "extensions", Version: "v1", Name: "deployments", ShortName: "deploy", Kind: "Deployment", Namespaced: true, Items: items, } } // test CRD func VeleroDeployments(items ...metav1.Object) *APIResource { return &APIResource{ Group: "velero.io", Version: "v1", Name: "deployments", ShortName: "deploy", Kind: "Deployment", Namespaced: true, Items: items, } } func Namespaces(items ...metav1.Object) *APIResource { return &APIResource{ Group: "", Version: "v1", Name: "namespaces", ShortName: "ns", Kind: "Namespace", Namespaced: false, Items: items, } } func ServiceAccounts(items ...metav1.Object) *APIResource { return &APIResource{ Group: "", Version: "v1", Name: "serviceaccounts", ShortName: "sa", Kind: "ServiceAccount", Namespaced: true, Items: items, } } func ConfigMaps(items ...metav1.Object) *APIResource { return &APIResource{ Group: "", Version: "v1", Name: "configmaps", ShortName: "cm", Kind: "ConfigMap", Namespaced: true, Items: items, } } func CRDs(items ...metav1.Object) *APIResource { return &APIResource{ Group: "apiextensions.k8s.io", Version: "v1beta1", Name: "customresourcedefinitions", ShortName: "crd", Kind: "CustomResourceDefinition", Namespaced: false, Items: items, } } func VSLs(items ...metav1.Object) *APIResource { return &APIResource{ Group: "velero.io", Version: "v1", Name: "volumesnapshotlocations", Kind: "VolumeSnapshotLocation", Namespaced: true, Items: items, } } func Backups(items ...metav1.Object) *APIResource { return &APIResource{ Group: "velero.io", Version: "v1", Name: "backups", Kind: "Backup", Namespaced: true, Items: items, } } func Services(items ...metav1.Object) *APIResource { return &APIResource{ Group: "", Version: "v1", Name: "services", ShortName: "svc", Kind: "Service", Namespaced: true, Items: items, } } func DataUploads(items ...metav1.Object) *APIResource { return &APIResource{ Group: "velero.io", Version: "v2alpha1", Name: "datauploads", Kind: "DataUpload", Namespaced: true, Items: items, } } ================================================ FILE: pkg/test/tar_writer.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package test import ( "archive/tar" "bytes" "compress/gzip" "encoding/json" "fmt" "testing" "time" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "github.com/vmware-tanzu/velero/pkg/util/encode" ) type TarWriter struct { t *testing.T buf *bytes.Buffer gzw *gzip.Writer tw *tar.Writer } func NewTarWriter(t *testing.T) *TarWriter { t.Helper() tw := new(TarWriter) tw.t = t tw.buf = new(bytes.Buffer) tw.gzw = gzip.NewWriter(tw.buf) tw.tw = tar.NewWriter(tw.gzw) return tw } func (tw *TarWriter) AddItems(groupResource string, items ...metav1.Object) *TarWriter { tw.t.Helper() for _, obj := range items { var path string if obj.GetNamespace() == "" { path = fmt.Sprintf("resources/%s/cluster/%s.json", groupResource, obj.GetName()) } else { path = fmt.Sprintf("resources/%s/namespaces/%s/%s.json", groupResource, obj.GetNamespace(), obj.GetName()) } tw.Add(path, obj) } return tw } func (tw *TarWriter) Add(name string, obj any) *TarWriter { tw.t.Helper() var data []byte var err error switch objType := obj.(type) { case runtime.Object: data, err = encode.Encode(objType, "json") case []byte: data = objType default: data, err = json.Marshal(obj) } require.NoError(tw.t, err) require.NoError(tw.t, tw.tw.WriteHeader(&tar.Header{ Name: name, Size: int64(len(data)), Typeflag: tar.TypeReg, Mode: 0755, ModTime: time.Now(), })) _, err = tw.tw.Write(data) require.NoError(tw.t, err) return tw } func (tw *TarWriter) Done() *bytes.Buffer { require.NoError(tw.t, tw.tw.Close()) require.NoError(tw.t, tw.gzw.Close()) return tw.buf } ================================================ FILE: pkg/test/test_logger.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package test import ( "io" "github.com/sirupsen/logrus" ) func NewLogger() logrus.FieldLogger { logger := logrus.New() logger.Out = io.Discard return logrus.NewEntry(logger) } func NewLoggerWithLevel(level logrus.Level) logrus.FieldLogger { logger := logrus.New() logger.Out = io.Discard logger.Level = level return logrus.NewEntry(logger) } type singleLogRecorder struct { buffer *string } func (s *singleLogRecorder) Write(p []byte) (n int, err error) { *s.buffer = *s.buffer + string(p[:]) return len(p), nil } func NewSingleLogger(buffer *string) logrus.FieldLogger { logger := logrus.New() logger.Out = &singleLogRecorder{buffer: buffer} logger.Level = logrus.TraceLevel return logrus.NewEntry(logger) } func NewSingleLoggerWithHooks(buffer *string, hooks []logrus.Hook) logrus.FieldLogger { logger := logrus.New() logger.Out = &singleLogRecorder{buffer: buffer} logger.Level = logrus.TraceLevel for _, hook := range hooks { logger.Hooks.Add(hook) } return logrus.NewEntry(logger) } type multipleLogRecorder struct { buffer *[]string } func (m *multipleLogRecorder) Write(p []byte) (n int, err error) { *m.buffer = append(*m.buffer, string(p[:])) return len(p), nil } func NewMultipleLogger(buffer *[]string) logrus.FieldLogger { logger := logrus.New() logger.Out = &multipleLogRecorder{buffer} logger.Level = logrus.TraceLevel return logrus.NewEntry(logger) } ================================================ FILE: pkg/types/node_agent.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package types import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/vmware-tanzu/velero/pkg/util/kube" ) type LoadConcurrency struct { // GlobalConfig specifies the concurrency number to all nodes for which per-node config is not specified GlobalConfig int `json:"globalConfig,omitempty"` // PerNodeConfig specifies the concurrency number to nodes matched by rules PerNodeConfig []RuledConfigs `json:"perNodeConfig,omitempty"` // PrepareQueueLength specifies the max number of loads that are under expose PrepareQueueLength int `json:"prepareQueueLength,omitempty"` } type LoadAffinity struct { // NodeSelector specifies the label selector to match nodes NodeSelector metav1.LabelSelector `json:"nodeSelector"` } type RuledConfigs struct { // NodeSelector specifies the label selector to match nodes NodeSelector metav1.LabelSelector `json:"nodeSelector"` // Number specifies the number value associated to the matched nodes Number int `json:"number"` } type BackupPVC struct { // StorageClass is the name of storage class to be used by the backupPVC StorageClass string `json:"storageClass,omitempty"` // ReadOnly sets the backupPVC's access mode as read only ReadOnly bool `json:"readOnly,omitempty"` // SPCNoRelabeling sets Spec.SecurityContext.SELinux.Type to "spc_t" for the pod mounting the backupPVC // ignored if ReadOnly is false SPCNoRelabeling bool `json:"spcNoRelabeling,omitempty"` // Annotations permits setting annotations for the backupPVC Annotations map[string]string `json:"annotations,omitempty"` } type RestorePVC struct { // IgnoreDelayBinding indicates to ignore delay binding the restorePVC when it is in WaitForFirstConsumer mode IgnoreDelayBinding bool `json:"ignoreDelayBinding,omitempty"` } type CachePVC struct { // StorageClass specifies the storage class for cache PVC StorageClass string `json:"storageClass,omitempty"` // ResidentThresholdInMB specifies the minimum size of the backup data to create cache PVC ResidentThresholdInMB int64 `json:"residentThresholdInMB,omitempty"` } type NodeAgentConfigs struct { // LoadConcurrency is the config for data path load concurrency per node. LoadConcurrency *LoadConcurrency `json:"loadConcurrency,omitempty"` // LoadAffinity is the config for data path load affinity. LoadAffinity []*kube.LoadAffinity `json:"loadAffinity,omitempty"` // BackupPVCConfig is the config for backupPVC (intermediate PVC) of snapshot data movement BackupPVCConfig map[string]BackupPVC `json:"backupPVC,omitempty"` // RestoreVCConfig is the config for restorePVC (intermediate PVC) of generic restore RestorePVCConfig *RestorePVC `json:"restorePVC,omitempty"` // PodResources is the resource config for various types of pods launched by node-agent, i.e., data mover pods. PodResources *kube.PodResources `json:"podResources,omitempty"` // PriorityClassName is the priority class name for data mover pods created by the node agent PriorityClassName string `json:"priorityClassName,omitempty"` // PrivilegedFsBackup determines whether to create fs-backup pods as privileged pods PrivilegedFsBackup bool `json:"privilegedFsBackup,omitempty"` // CachePVCConfig is the config for cachePVC CachePVCConfig *CachePVC `json:"cachePVC,omitempty"` // PodAnnotations are annotations to be added to pods created by node-agent, i.e., data mover pods. PodAnnotations map[string]string `json:"podAnnotations,omitempty"` // PodLabels are labels to be added to pods created by node-agent, i.e., data mover pods. PodLabels map[string]string `json:"podLabels,omitempty"` } ================================================ FILE: pkg/types/priority.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package types import ( "fmt" "strings" ) const ( prioritySeparator = "-" ) // Priorities defines the desired order of resource operations: // Resources in the HighPriorities list will be handled first // Resources in the LowPriorities list will be handled last // Other resources will be handled alphabetically after the high prioritized resources and before the low prioritized resources type Priorities struct { HighPriorities []string LowPriorities []string } // String returns a string representation of Priority. func (p *Priorities) String() string { priorities := p.HighPriorities if len(p.LowPriorities) > 0 { priorities = append(priorities, prioritySeparator) priorities = append(priorities, p.LowPriorities...) } return strings.Join(priorities, ",") } // Set parses the provided string to the priority object func (p *Priorities) Set(s string) error { if len(s) == 0 { return nil } strs := strings.Split(s, ",") separatorIndex := -1 for i, str := range strs { if str == prioritySeparator { if separatorIndex > -1 { return fmt.Errorf("multiple priority separator %q found", prioritySeparator) } separatorIndex = i } } // has no separator if separatorIndex == -1 { p.HighPriorities = strs return nil } // start with separator if separatorIndex == 0 { // contain only separator if len(strs) == 1 { return nil } p.LowPriorities = strs[1:] return nil } // end with separator if separatorIndex == len(strs)-1 { p.HighPriorities = strs[:len(strs)-1] return nil } // separator in the middle p.HighPriorities = strs[:separatorIndex] p.LowPriorities = strs[separatorIndex+1:] return nil } // Type specifies the flag type func (p *Priorities) Type() string { return "stringArray" } ================================================ FILE: pkg/types/priority_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package types import ( "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestStringOfPriorities(t *testing.T) { priority := Priorities{ HighPriorities: []string{"high"}, } assert.Equal(t, "high", priority.String()) priority = Priorities{ HighPriorities: []string{"high"}, LowPriorities: []string{"low"}, } assert.Equal(t, "high,-,low", priority.String()) } func TestSetOfPriority(t *testing.T) { cases := []struct { name string input string priorities Priorities hasErr bool }{ { name: "empty input", input: "", priorities: Priorities{}, hasErr: false, }, { name: "only high priorities", input: "p0", priorities: Priorities{ HighPriorities: []string{"p0"}, }, hasErr: false, }, { name: "only low priorities", input: "-,p9", priorities: Priorities{ LowPriorities: []string{"p9"}, }, hasErr: false, }, { name: "only separator", input: "-", priorities: Priorities{}, hasErr: false, }, { name: "multiple separators", input: "-,-", priorities: Priorities{}, hasErr: true, }, { name: "contain both high and low priorities", input: "p0,p1,p2,-,p9", priorities: Priorities{ HighPriorities: []string{"p0", "p1", "p2"}, LowPriorities: []string{"p9"}, }, hasErr: false, }, { name: "end with separator", input: "p0,-", priorities: Priorities{ HighPriorities: []string{"p0"}, }, hasErr: false, }, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { p := Priorities{} err := p.Set(c.input) if c.hasErr { require.Error(t, err) } else { require.NoError(t, err) } assert.Equal(t, c.priorities, p) }) } } ================================================ FILE: pkg/types/repo_maintenance.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package types import "github.com/vmware-tanzu/velero/pkg/util/kube" type JobConfigs struct { // LoadAffinities is the config for repository maintenance job load affinity. LoadAffinities []*kube.LoadAffinity `json:"loadAffinity,omitempty"` // PodResources is the config for the CPU and memory resources setting. PodResources *kube.PodResources `json:"podResources,omitempty"` // KeepLatestMaintenanceJobs is the number of latest maintenance jobs to keep for the repository. KeepLatestMaintenanceJobs *int `json:"keepLatestMaintenanceJobs,omitempty"` // PriorityClassName is the priority class name for the maintenance job pod // Note: This is only read from the global configuration, not per-repository PriorityClassName string `json:"priorityClassName,omitempty"` // PodAnnotations are annotations to be added to maintenance job pods. // Note: This is only read from the global configuration, not per-repository PodAnnotations map[string]string `json:"podAnnotations,omitempty"` // PodLabels are labels to be added to maintenance job pods. // Note: This is only read from the global configuration, not per-repository PodLabels map[string]string `json:"podLabels,omitempty"` } ================================================ FILE: pkg/uploader/kopia/block_backup.go ================================================ //go:build !windows // +build !windows /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package kopia import ( "os" "syscall" "github.com/kopia/kopia/fs" "github.com/kopia/kopia/fs/virtualfs" "github.com/pkg/errors" ) const ErrNotPermitted = "operation not permitted" func getLocalBlockEntry(sourcePath string) (fs.Entry, error) { source, err := resolveSymlink(sourcePath) if err != nil { return nil, errors.Wrap(err, "resolveSymlink") } fileInfo, err := os.Lstat(source) if err != nil { return nil, errors.Wrapf(err, "unable to get the source device information %s", source) } if (fileInfo.Sys().(*syscall.Stat_t).Mode & syscall.S_IFMT) != syscall.S_IFBLK { return nil, errors.Errorf("source path %s is not a block device", source) } device, err := os.Open(source) if err != nil { if os.IsPermission(err) || err.Error() == ErrNotPermitted { return nil, errors.Wrapf(err, "no permission to open the source device %s, make sure that node agent is running in privileged mode", source) } return nil, errors.Wrapf(err, "unable to open the source device %s", source) } sf := virtualfs.StreamingFileFromReader(source, device) return virtualfs.NewStaticDirectory(source, []fs.Entry{sf}), nil } ================================================ FILE: pkg/uploader/kopia/block_backup_windows.go ================================================ //go:build windows // +build windows /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package kopia import ( "fmt" "github.com/kopia/kopia/fs" ) func getLocalBlockEntry(sourcePath string) (fs.Entry, error) { return nil, fmt.Errorf("block mode is not supported for Windows") } ================================================ FILE: pkg/uploader/kopia/block_restore.go ================================================ //go:build !windows // +build !windows /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package kopia import ( "context" "io" "os" "path/filepath" "syscall" "github.com/kopia/kopia/fs" "github.com/kopia/kopia/snapshot/restore" "github.com/pkg/errors" ) type BlockOutput struct { *restore.FilesystemOutput targetFileName string targetFile *os.File } var _ restore.Output = &BlockOutput{} const bufferSize = 128 * 1024 func (o *BlockOutput) WriteFile(ctx context.Context, relativePath string, remoteFile fs.File, progressCb restore.FileWriteProgress) error { remoteReader, err := remoteFile.Open(ctx) if err != nil { return errors.Wrapf(err, "failed to open remote file %s", remoteFile.Name()) } defer remoteReader.Close() targetFile, err := os.Create(o.targetFileName) if err != nil { return errors.Wrapf(err, "failed to open file %s", o.targetFileName) } o.targetFile = targetFile buffer := make([]byte, bufferSize) readData := true for readData { bytesToWrite, err := remoteReader.Read(buffer) if err != nil { if err != io.EOF { return errors.Wrapf(err, "failed to read data from remote file %s", o.targetFileName) } readData = false } if bytesToWrite > 0 { offset := 0 for bytesToWrite > 0 { if bytesWritten, err := targetFile.Write(buffer[offset:bytesToWrite]); err == nil { progressCb(int64(bytesWritten)) bytesToWrite -= bytesWritten offset += bytesWritten } else { return errors.Wrapf(err, "failed to write data to file %s", o.targetFileName) } } } } return nil } func (o *BlockOutput) BeginDirectory(ctx context.Context, relativePath string, e fs.Directory) error { var err error o.targetFileName, err = filepath.EvalSymlinks(o.TargetPath) if err != nil { return errors.Wrapf(err, "unable to evaluate symlinks for %s", o.targetFileName) } fileInfo, err := os.Lstat(o.targetFileName) if err != nil { return errors.Wrapf(err, "unable to get the target device information for %s", o.TargetPath) } if (fileInfo.Sys().(*syscall.Stat_t).Mode & syscall.S_IFMT) != syscall.S_IFBLK { return errors.Errorf("target file %s is not a block device", o.TargetPath) } return nil } func (o *BlockOutput) Flush() error { if o.targetFile != nil { if err := o.targetFile.Sync(); err != nil { return errors.Wrapf(err, "error syncing block dev %v", o.targetFileName) } } return nil } func (o *BlockOutput) Terminate() error { if o.targetFile != nil { if err := o.targetFile.Close(); err != nil { return errors.Wrapf(err, "error closing block dev %v", o.targetFileName) } } return nil } ================================================ FILE: pkg/uploader/kopia/block_restore_windows.go ================================================ //go:build windows // +build windows /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package kopia import ( "context" "fmt" "github.com/kopia/kopia/fs" "github.com/kopia/kopia/snapshot/restore" ) type BlockOutput struct { *restore.FilesystemOutput targetFileName string } func (o *BlockOutput) WriteFile(ctx context.Context, relativePath string, remoteFile fs.File, progressCb restore.FileWriteProgress) error { return fmt.Errorf("block mode is not supported for Windows") } func (o *BlockOutput) BeginDirectory(ctx context.Context, relativePath string, e fs.Directory) error { return fmt.Errorf("block mode is not supported for Windows") } func (o *BlockOutput) Flush() error { return flushVolume(o.targetFileName) } func (o *BlockOutput) Terminate() error { return nil } ================================================ FILE: pkg/uploader/kopia/flush_volume_linux.go ================================================ //go:build linux // +build linux /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package kopia import ( "os" "github.com/pkg/errors" "golang.org/x/sys/unix" ) func flushVolume(dirPath string) error { dir, err := os.Open(dirPath) if err != nil { return errors.Wrapf(err, "error opening dir %v", dirPath) } raw, err := dir.SyscallConn() if err != nil { return errors.Wrapf(err, "error getting handle of dir %v", dirPath) } var syncErr error if err := raw.Control(func(fd uintptr) { if e := unix.Syncfs(int(fd)); e != nil { syncErr = e } }); err != nil { return errors.Wrapf(err, "error calling fs sync from %v", dirPath) } return errors.Wrapf(syncErr, "error syncing fs from %v", dirPath) } ================================================ FILE: pkg/uploader/kopia/flush_volume_other.go ================================================ //go:build !linux // +build !linux /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package kopia func flushVolume(_ string) error { return errFlushUnsupported } ================================================ FILE: pkg/uploader/kopia/progress.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package kopia import ( "sync/atomic" "time" "github.com/sirupsen/logrus" "github.com/vmware-tanzu/velero/pkg/uploader" "github.com/kopia/kopia/snapshot/upload" ) // Throttle throttles controlle the interval of output result type Throttle struct { throttle int64 interval time.Duration } func (t *Throttle) ShouldOutput() bool { nextOutputTimeUnixNano := atomic.LoadInt64(&t.throttle) if nowNano := time.Now().UnixNano(); nowNano > nextOutputTimeUnixNano { if atomic.CompareAndSwapInt64(&t.throttle, nextOutputTimeUnixNano, nowNano+t.interval.Nanoseconds()) { return true } } return false } // Progress represents a backup or restore counters. type Progress struct { // all int64 must precede all int32 due to alignment requirements on ARM // +checkatomic uploadedBytes int64 //the total bytes has uploaded cachedBytes int64 //the total bytes has cached hashededBytes int64 //the total bytes has hashed // +checkatomic uploadedFiles int32 //the total files has ignored // +checkatomic ignoredErrorCount int32 //the total errors has ignored // +checkatomic fatalErrorCount int32 //the total errors has occurred estimatedFileCount int64 // +checklocksignore the total count of files to be processed estimatedTotalBytes int64 // +checklocksignore the total size of files to be processed // +checkatomic processedBytes int64 // which statistic all bytes has been processed currently outputThrottle Throttle // which control the frequency of update progress updater uploader.ProgressUpdater //which kopia progress will call the UpdateProgress interface, the third party will implement the interface to do the progress update log logrus.FieldLogger // output info into log when backup estimationParam upload.EstimationParameters } func NewProgress(updater uploader.ProgressUpdater, interval time.Duration, log logrus.FieldLogger) *Progress { return &Progress{ outputThrottle: Throttle{ throttle: 0, interval: interval, }, updater: updater, estimationParam: upload.EstimationParameters{ Type: upload.EstimationTypeClassic, AdaptiveThreshold: 300000, }, log: log, } } // UploadedBytes the total bytes has uploaded currently func (p *Progress) UploadedBytes(numBytes int64) { atomic.AddInt64(&p.uploadedBytes, numBytes) atomic.AddInt32(&p.uploadedFiles, 1) p.UpdateProgress() } // Error statistic the total Error has occurred func (p *Progress) Error(path string, err error, isIgnored bool) { if isIgnored { atomic.AddInt32(&p.ignoredErrorCount, 1) p.log.Warnf("Ignored error when processing %v: %v", path, err) } else { atomic.AddInt32(&p.fatalErrorCount, 1) p.log.Errorf("Error when processing %v: %v", path, err) } } // EstimatedDataSize statistic the total size of files to be processed and total files to be processed func (p *Progress) EstimatedDataSize(fileCount int64, totalBytes int64) { atomic.StoreInt64(&p.estimatedTotalBytes, totalBytes) atomic.StoreInt64(&p.estimatedFileCount, fileCount) p.UpdateProgress() } // UpdateProgress which calls Updater UpdateProgress interface, update progress by third-party implementation func (p *Progress) UpdateProgress() { if p.outputThrottle.ShouldOutput() { p.updater.UpdateProgress(&uploader.Progress{TotalBytes: p.estimatedTotalBytes, BytesDone: p.processedBytes}) } } // UploadStarted statistic the total Error has occurred func (p *Progress) UploadStarted() {} // CachedFile statistic the total bytes been cached currently func (p *Progress) CachedFile(fname string, numBytes int64) { atomic.AddInt64(&p.cachedBytes, numBytes) atomic.AddInt64(&p.processedBytes, numBytes) p.UpdateProgress() } // HashedBytes statistic the total bytes been hashed currently func (p *Progress) HashedBytes(numBytes int64) { atomic.AddInt64(&p.processedBytes, numBytes) atomic.AddInt64(&p.hashededBytes, numBytes) p.UpdateProgress() } // HashingFile statistic the file been hashed currently func (p *Progress) HashingFile(fname string) {} // ExcludedFile statistic the file been excluded currently func (p *Progress) ExcludedFile(fname string, numBytes int64) {} // ExcludedDir statistic the dir been excluded currently func (p *Progress) ExcludedDir(dirname string) { p.log.Infof("Excluded dir %s", dirname) } // FinishedHashingFile which will called when specific file finished hash func (p *Progress) FinishedHashingFile(fname string, numBytes int64) { p.UpdateProgress() } // StartedDirectory called when begin to upload one directory func (p *Progress) StartedDirectory(dirname string) {} // FinishedDirectory called when finish to upload one directory func (p *Progress) FinishedDirectory(dirname string) { p.UpdateProgress() } // UploadFinished which report the files flushed after the Upload has completed. func (p *Progress) UploadFinished() { p.UpdateProgress() } // ProgressBytes which statistic all bytes has been processed currently func (p *Progress) ProgressBytes(processedBytes int64, totalBytes int64) { atomic.StoreInt64(&p.processedBytes, processedBytes) atomic.StoreInt64(&p.estimatedTotalBytes, totalBytes) p.UpdateProgress() } func (p *Progress) FinishedFile(fname string, err error) {} func (p *Progress) EstimationParameters() upload.EstimationParameters { return p.estimationParam } func (p *Progress) Enabled() bool { return true } func (p *Progress) GetIncrementalSize() int64 { return p.estimatedTotalBytes - p.cachedBytes } ================================================ FILE: pkg/uploader/kopia/progress_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package kopia import ( "testing" "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/vmware-tanzu/velero/pkg/uploader" ) type fakeProgressUpdater struct{} func (f *fakeProgressUpdater) UpdateProgress(p *uploader.Progress) {} func TestThrottle_ShouldOutput(t *testing.T) { testCases := []struct { interval time.Duration throttle int64 expectedOutput bool }{ {interval: time.Second, expectedOutput: true}, {interval: time.Second, throttle: time.Now().UnixNano() + int64(time.Nanosecond*100000000), expectedOutput: false}, } for _, tc := range testCases { // Setup p := &Progress{ outputThrottle: Throttle{ interval: tc.interval, throttle: tc.throttle, }, } // Perform the test output := p.outputThrottle.ShouldOutput() // Verify the result if output != tc.expectedOutput { t.Errorf("Expected ShouldOutput to return %v, but got %v", tc.expectedOutput, output) } } } func TestProgress(t *testing.T) { fileName := "test-filename" var numBytes int64 = 1 testCases := []struct { interval time.Duration throttle int64 }{ {interval: time.Second}, {interval: time.Second, throttle: time.Now().UnixNano() + int64(time.Nanosecond*10000)}, } for _, tc := range testCases { // Setup p := &Progress{ outputThrottle: Throttle{ interval: tc.interval, throttle: tc.throttle, }, updater: &fakeProgressUpdater{}, log: logrus.New(), } // All below calls put together for the implementation are empty or just very simple and just want to cover testing // If wanting to write unit tests for some functions could remove it and with writing new function alone p.UpdateProgress() p.UploadedBytes(numBytes) p.Error("test-path", nil, true) p.Error("test-path", errors.New("processing error"), false) p.UploadStarted() p.EstimatedDataSize(1, numBytes) p.CachedFile(fileName, numBytes) p.HashedBytes(numBytes) p.HashingFile(fileName) p.ExcludedFile(fileName, numBytes) p.ExcludedDir(fileName) p.FinishedHashingFile(fileName, numBytes) p.StartedDirectory(fileName) p.FinishedDirectory(fileName) p.UploadFinished() p.ProgressBytes(numBytes, numBytes) p.FinishedFile(fileName, nil) } } ================================================ FILE: pkg/uploader/kopia/restore_output.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package kopia import ( "github.com/kopia/kopia/snapshot/restore" "github.com/pkg/errors" ) var errFlushUnsupported = errors.New("flush is not supported") type RestoreOutput interface { restore.Output Flush() error Terminate() error } ================================================ FILE: pkg/uploader/kopia/shim.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package kopia import ( "context" "strings" "time" "github.com/pkg/errors" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/content" "github.com/kopia/kopia/repo/content/index" "github.com/kopia/kopia/repo/manifest" "github.com/kopia/kopia/repo/object" ) // shimRepository which is one adapter for unified repo and kopia. // it implement kopia RepositoryWriter interfaces type shimRepository struct { udmRepo udmrepo.BackupRepo } // shimObjectWriter object writer for unifited repo type shimObjectWriter struct { repoWriter udmrepo.ObjectWriter } // shimObjectReader object reader for unifited repo type shimObjectReader struct { repoReader udmrepo.ObjectReader } func NewShimRepo(repo udmrepo.BackupRepo) repo.RepositoryWriter { return &shimRepository{ udmRepo: repo, } } // OpenObject open specific object func (sr *shimRepository) OpenObject(ctx context.Context, id object.ID) (object.Reader, error) { reader, err := sr.udmRepo.OpenObject(ctx, udmrepo.ID(id.String())) if err != nil { return nil, errors.Wrapf(err, "failed to open object with id %v", id) } if reader == nil { return nil, err } return &shimObjectReader{ repoReader: reader, }, err } // VerifyObject not supported func (sr *shimRepository) VerifyObject(ctx context.Context, id object.ID) ([]content.ID, error) { return nil, errors.New("VerifyObject is not supported") } // Get one or more manifest data that match the specific manifest id func (sr *shimRepository) GetManifest(ctx context.Context, id manifest.ID, payload any) (*manifest.EntryMetadata, error) { repoMani := udmrepo.RepoManifest{ Payload: payload, } if err := sr.udmRepo.GetManifest(ctx, udmrepo.ID(id), &repoMani); err != nil { return nil, errors.Wrapf(err, "failed to get manifest with id %v", id) } return GetKopiaManifestEntry(repoMani.Metadata), nil } // Get one or more manifest data that match the given labels func (sr *shimRepository) FindManifests(ctx context.Context, labels map[string]string) ([]*manifest.EntryMetadata, error) { metadata, err := sr.udmRepo.FindManifests(ctx, udmrepo.ManifestFilter{Labels: labels}) if err != nil { return nil, errors.Wrapf(err, "failed to get manifests with labels %v", labels) } return GetKopiaManifestEntries(metadata), nil } // GetKopiaManifestEntry get metadata from specific ManifestEntryMetadata func GetKopiaManifestEntry(uMani *udmrepo.ManifestEntryMetadata) *manifest.EntryMetadata { var ret manifest.EntryMetadata ret.ID = manifest.ID(uMani.ID) ret.Labels = uMani.Labels ret.Length = int(uMani.Length) ret.ModTime = uMani.ModTime return &ret } // GetKopiaManifestEntries get metadata list from specific ManifestEntryMetadata func GetKopiaManifestEntries(uMani []*udmrepo.ManifestEntryMetadata) []*manifest.EntryMetadata { var ret []*manifest.EntryMetadata for _, entry := range uMani { var e manifest.EntryMetadata e.ID = manifest.ID(entry.ID) e.Labels = entry.Labels e.Length = int(entry.Length) e.ModTime = entry.ModTime ret = append(ret, &e) } return ret } // Time Get the local time of the unified repo func (sr *shimRepository) Time() time.Time { return sr.udmRepo.Time() } // ClientOptions is not supported by unified repo func (sr *shimRepository) ClientOptions() repo.ClientOptions { return repo.ClientOptions{} } // Refresh not supported func (sr *shimRepository) Refresh(ctx context.Context) error { return errors.New("Refresh is not supported") } // ContentInfo not supported func (sr *shimRepository) ContentInfo(ctx context.Context, contentID content.ID) (content.Info, error) { return index.Info{}, errors.New("ContentInfo is not supported") } // PrefetchContents is not supported by unified repo func (sr *shimRepository) PrefetchContents(ctx context.Context, contentIDs []content.ID, hint string) []content.ID { return nil } // PrefetchObjects is not supported by unified repo func (sr *shimRepository) PrefetchObjects(ctx context.Context, objectIDs []object.ID, hint string) ([]content.ID, error) { return nil, errors.New("PrefetchObjects is not supported") } // UpdateDescription is not supported by unified repo func (sr *shimRepository) UpdateDescription(d string) { } // NewWriter is not supported by unified repo func (sr *shimRepository) NewWriter(ctx context.Context, option repo.WriteSessionOptions) (context.Context, repo.RepositoryWriter, error) { return nil, nil, errors.New("NewWriter is not supported") } // Close will close unified repo func (sr *shimRepository) Close(ctx context.Context) error { return sr.udmRepo.Close(ctx) } // NewObjectWriter creates an object writer func (sr *shimRepository) NewObjectWriter(ctx context.Context, option object.WriterOptions) object.Writer { var opt udmrepo.ObjectWriteOptions opt.Description = option.Description opt.Prefix = udmrepo.ID(option.Prefix) opt.FullPath = "" opt.AccessMode = udmrepo.ObjectDataAccessModeFile opt.AsyncWrites = option.AsyncWrites if strings.HasPrefix(option.Description, "DIR:") { opt.DataType = udmrepo.ObjectDataTypeMetadata } else { opt.DataType = udmrepo.ObjectDataTypeData } writer := sr.udmRepo.NewObjectWriter(ctx, opt) if writer == nil { return nil } return &shimObjectWriter{ repoWriter: writer, } } // PutManifest saves the given manifest payload with a set of labels. func (sr *shimRepository) PutManifest(ctx context.Context, labels map[string]string, payload any) (manifest.ID, error) { id, err := sr.udmRepo.PutManifest(ctx, udmrepo.RepoManifest{ Payload: payload, Metadata: &udmrepo.ManifestEntryMetadata{ Labels: labels, }, }) return manifest.ID(id), err } // DeleteManifest deletes the manifest with a given ID. func (sr *shimRepository) DeleteManifest(ctx context.Context, id manifest.ID) error { return sr.udmRepo.DeleteManifest(ctx, udmrepo.ID(id)) } func (sr *shimRepository) ReplaceManifests(ctx context.Context, labels map[string]string, payload any) (manifest.ID, error) { const minReplaceManifestTimeDelta = 100 * time.Millisecond md, err := sr.FindManifests(ctx, labels) if err != nil { return "", errors.Wrap(err, "unable to load manifests") } for _, em := range md { age := sr.Time().Sub(em.ModTime) if age < minReplaceManifestTimeDelta { time.Sleep(minReplaceManifestTimeDelta) } if err := sr.DeleteManifest(ctx, em.ID); err != nil { return "", errors.Wrapf(err, "unable to delete previous manifest %v", em.ID) } } return sr.PutManifest(ctx, labels, payload) } // Flush all the unifited repository data func (sr *shimRepository) Flush(ctx context.Context) error { return sr.udmRepo.Flush(ctx) } func (sr *shimRepository) ConcatenateObjects(ctx context.Context, objectIDs []object.ID, opt repo.ConcatenateOptions) (object.ID, error) { if len(objectIDs) == 0 { return object.EmptyID, errors.New("object list is empty") } ids := []udmrepo.ID{} for _, id := range objectIDs { ids = append(ids, udmrepo.ID(id.String())) } id, err := sr.udmRepo.ConcatenateObjects(ctx, ids) if err != nil { return object.EmptyID, err } return object.ParseID(string(id)) } func (sr *shimRepository) OnSuccessfulFlush(callback repo.RepositoryWriterCallback) { } // Flush all the unifited repository data func (sr *shimObjectReader) Read(p []byte) (n int, err error) { return sr.repoReader.Read(p) } func (sr *shimObjectReader) Seek(offset int64, whence int) (int64, error) { return sr.repoReader.Seek(offset, whence) } // Close current io for ObjectReader func (sr *shimObjectReader) Close() error { return sr.repoReader.Close() } // Length returns the logical size of the object func (sr *shimObjectReader) Length() int64 { return sr.repoReader.Length() } // Write data func (sr *shimObjectWriter) Write(p []byte) (n int, err error) { return sr.repoWriter.Write(p) } // Periodically called to preserve the state of data written to the repo so far. func (sr *shimObjectWriter) Checkpoint() (object.ID, error) { id, err := sr.repoWriter.Checkpoint() if err != nil { return object.ID{}, err } objID, err := object.ParseID(string(id)) if err != nil { return object.ID{}, errors.Wrapf(err, "error to parse object ID from %v", id) } return objID, err } // Result returns the object's unified identifier after the write completes. func (sr *shimObjectWriter) Result() (object.ID, error) { id, err := sr.repoWriter.Result() if err != nil { return object.ID{}, err } objID, err := object.ParseID(string(id)) if err != nil { return object.ID{}, errors.Wrapf(err, "error to parse object ID from %v", id) } return objID, err } // Close closes the repository and releases all resources. func (sr *shimObjectWriter) Close() error { return sr.repoWriter.Close() } ================================================ FILE: pkg/uploader/kopia/shim_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package kopia import ( "context" "errors" "testing" "time" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/content" "github.com/kopia/kopia/repo/manifest" "github.com/kopia/kopia/repo/object" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo/mocks" ) func TestShimRepo(t *testing.T) { ctx := t.Context() backupRepo := &mocks.BackupRepo{} backupRepo.On("Time").Return(time.Time{}) shim := NewShimRepo(backupRepo) // All below calls put together for the implementation are empty or just very simple, and just want to cover testing // If wanting to write unit tests for some functions could remove it and with writing new function alone shim.VerifyObject(ctx, object.ID{}) shim.Time() shim.ClientOptions() shim.Refresh(ctx) shim.ContentInfo(ctx, content.ID{}) shim.PrefetchContents(ctx, []content.ID{}, "hint") shim.PrefetchObjects(ctx, []object.ID{}, "hint") shim.UpdateDescription("desc") shim.NewWriter(ctx, repo.WriteSessionOptions{}) shim.OnSuccessfulFlush(func(ctx context.Context, w repo.RepositoryWriter) error { return nil }) backupRepo.On("Close", mock.Anything).Return(nil) NewShimRepo(backupRepo).Close(ctx) var id udmrepo.ID backupRepo.On("PutManifest", mock.Anything, mock.Anything).Return(id, nil) NewShimRepo(backupRepo).PutManifest(ctx, map[string]string{}, nil) var mf manifest.ID backupRepo.On("DeleteManifest", mock.Anything, mock.Anything).Return(nil) NewShimRepo(backupRepo).DeleteManifest(ctx, mf) backupRepo.On("Flush", mock.Anything).Return(nil) NewShimRepo(backupRepo).Flush(ctx) backupRepo.On("NewObjectWriter", mock.Anything, mock.Anything).Return(nil) NewShimRepo(backupRepo).NewObjectWriter(ctx, object.WriterOptions{}) } func TestOpenObject(t *testing.T) { tests := []struct { name string backupRepo *mocks.BackupRepo isOpenObjectError bool isReaderNil bool }{ { name: "Success", backupRepo: func() *mocks.BackupRepo { backupRepo := &mocks.BackupRepo{} backupRepo.On("OpenObject", mock.Anything, mock.Anything).Return(&shimObjectReader{}, nil) return backupRepo }(), }, { name: "Open object error", backupRepo: func() *mocks.BackupRepo { backupRepo := &mocks.BackupRepo{} backupRepo.On("OpenObject", mock.Anything, mock.Anything).Return(&shimObjectReader{}, errors.New("Error open object")) return backupRepo }(), isOpenObjectError: true, }, { name: "Get nil reader", backupRepo: func() *mocks.BackupRepo { backupRepo := &mocks.BackupRepo{} backupRepo.On("OpenObject", mock.Anything, mock.Anything).Return(nil, nil) return backupRepo }(), isReaderNil: true, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { ctx := t.Context() reader, err := NewShimRepo(tc.backupRepo).OpenObject(ctx, object.ID{}) if tc.isOpenObjectError { require.ErrorContains(t, err, "failed to open object") } else if tc.isReaderNil { assert.Nil(t, reader) } else { assert.NotNil(t, reader) assert.NoError(t, err) } }) } } func TestFindManifests(t *testing.T) { meta := []*udmrepo.ManifestEntryMetadata{} tests := []struct { name string backupRepo *mocks.BackupRepo isGetManifestError bool }{ { name: "Success", backupRepo: func() *mocks.BackupRepo { backupRepo := &mocks.BackupRepo{} backupRepo.On("FindManifests", mock.Anything, mock.Anything).Return(meta, nil) return backupRepo }(), }, { name: "Failed to find manifest", isGetManifestError: true, backupRepo: func() *mocks.BackupRepo { backupRepo := &mocks.BackupRepo{} backupRepo.On("FindManifests", mock.Anything, mock.Anything).Return(meta, errors.New("failed to find manifest")) return backupRepo }(), }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { ctx := t.Context() _, err := NewShimRepo(tc.backupRepo).FindManifests(ctx, map[string]string{}) if tc.isGetManifestError { require.ErrorContains(t, err, "failed") } else { assert.NoError(t, err) } }) } } func TestShimObjReader(t *testing.T) { reader := new(shimObjectReader) objReader := &mocks.ObjectReader{} reader.repoReader = objReader // All below calls put together for the implementation are empty or just very simple, and just want to cover testing // If wanting to write unit tests for some functions could remove it and with writing new function alone objReader.On("Seek", mock.Anything, mock.Anything).Return(int64(0), nil) reader.Seek(int64(0), 0) objReader.On("Read", mock.Anything).Return(0, nil) reader.Read(nil) objReader.On("Close").Return(nil) reader.Close() objReader.On("Length").Return(int64(0)) reader.Length() } func TestShimObjWriter(t *testing.T) { writer := new(shimObjectWriter) objWriter := &mocks.ObjectWriter{} writer.repoWriter = objWriter // All below calls put together for the implementation are empty or just very simple, and just want to cover testing // If wanting to write unit tests for some functions could remove it and with writing new function alone var id udmrepo.ID objWriter.On("Checkpoint").Return(id, nil) writer.Checkpoint() objWriter.On("Result").Return(id, nil) writer.Result() objWriter.On("Write", mock.Anything).Return(0, nil) writer.Write(nil) objWriter.On("Close").Return(nil) writer.Close() } func TestReplaceManifests(t *testing.T) { meta1 := udmrepo.ManifestEntryMetadata{ ID: "mani-1", } meta2 := udmrepo.ManifestEntryMetadata{ ID: "mani-2", } tests := []struct { name string backupRepo *mocks.BackupRepo isGetManifestError bool expectedError string expectedID manifest.ID }{ { name: "Failed to find manifest", isGetManifestError: true, backupRepo: func() *mocks.BackupRepo { backupRepo := &mocks.BackupRepo{} backupRepo.On("FindManifests", mock.Anything, mock.Anything).Return([]*udmrepo.ManifestEntryMetadata{}, errors.New("fake-find-error")) return backupRepo }(), expectedError: "unable to load manifests: failed to get manifests with labels map[]: fake-find-error", }, { name: "Failed to delete manifest", isGetManifestError: true, backupRepo: func() *mocks.BackupRepo { backupRepo := &mocks.BackupRepo{} backupRepo.On("FindManifests", mock.Anything, mock.Anything).Return([]*udmrepo.ManifestEntryMetadata{ &meta1, &meta2, }, nil) backupRepo.On("Time").Return(time.Now()) backupRepo.On("DeleteManifest", mock.Anything, mock.Anything).Return(errors.New("fake-delete-error")) return backupRepo }(), expectedError: "unable to delete previous manifest mani-1: fake-delete-error", }, { name: "Failed to put manifest", backupRepo: func() *mocks.BackupRepo { backupRepo := &mocks.BackupRepo{} backupRepo.On("FindManifests", mock.Anything, mock.Anything).Return([]*udmrepo.ManifestEntryMetadata{ &meta1, &meta2, }, nil) backupRepo.On("Time").Return(time.Now()) backupRepo.On("DeleteManifest", mock.Anything, mock.Anything).Return(nil) backupRepo.On("PutManifest", mock.Anything, mock.Anything).Return(udmrepo.ID(""), errors.New("fake-put-error")) return backupRepo }(), expectedError: "fake-put-error", }, { name: "Success", backupRepo: func() *mocks.BackupRepo { backupRepo := &mocks.BackupRepo{} backupRepo.On("FindManifests", mock.Anything, mock.Anything).Return([]*udmrepo.ManifestEntryMetadata{ &meta1, &meta2, }, nil) backupRepo.On("Time").Return(time.Now()) backupRepo.On("DeleteManifest", mock.Anything, mock.Anything).Return(nil) backupRepo.On("PutManifest", mock.Anything, mock.Anything).Return(udmrepo.ID("fake-id"), nil) return backupRepo }(), expectedID: manifest.ID("fake-id"), }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { ctx := t.Context() id, err := NewShimRepo(tc.backupRepo).ReplaceManifests(ctx, map[string]string{}, nil) if tc.expectedError != "" { require.EqualError(t, err, tc.expectedError) } else { require.NoError(t, err) } assert.Equal(t, tc.expectedID, id) }) } } func TestConcatenateObjects(t *testing.T) { tests := []struct { name string backupRepo *mocks.BackupRepo objectIDs []object.ID expectedError string }{ { name: "empty object list", expectedError: "object list is empty", }, { name: "concatenate error", backupRepo: func() *mocks.BackupRepo { backupRepo := &mocks.BackupRepo{} backupRepo.On("ConcatenateObjects", mock.Anything, mock.Anything).Return(udmrepo.ID(""), errors.New("fake-concatenate-error")) return backupRepo }(), objectIDs: []object.ID{ {}, }, expectedError: "fake-concatenate-error", }, { name: "parse error", backupRepo: func() *mocks.BackupRepo { backupRepo := &mocks.BackupRepo{} backupRepo.On("ConcatenateObjects", mock.Anything, mock.Anything).Return(udmrepo.ID("fake-id"), nil) return backupRepo }(), objectIDs: []object.ID{ {}, }, expectedError: "malformed content ID: \"fake-id\": invalid content prefix", }, { name: "success", backupRepo: func() *mocks.BackupRepo { backupRepo := &mocks.BackupRepo{} backupRepo.On("ConcatenateObjects", mock.Anything, mock.Anything).Return(udmrepo.ID("I123456"), nil) return backupRepo }(), objectIDs: []object.ID{ {}, }, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { ctx := t.Context() _, err := NewShimRepo(tc.backupRepo).ConcatenateObjects(ctx, tc.objectIDs, repo.ConcatenateOptions{}) if tc.expectedError != "" { assert.EqualError(t, err, tc.expectedError) } else { assert.NoError(t, err) } }) } } ================================================ FILE: pkg/uploader/kopia/snapshot.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package kopia import ( "context" "fmt" "math" "os" "path/filepath" "runtime" "strings" "time" "github.com/sirupsen/logrus" "github.com/kopia/kopia/fs" "github.com/kopia/kopia/fs/localfs" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/manifest" "github.com/kopia/kopia/snapshot" "github.com/kopia/kopia/snapshot/policy" "github.com/kopia/kopia/snapshot/restore" "github.com/kopia/kopia/snapshot/snapshotfs" "github.com/pkg/errors" "github.com/vmware-tanzu/velero/pkg/kopia" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" "github.com/vmware-tanzu/velero/pkg/uploader" uploaderutil "github.com/vmware-tanzu/velero/pkg/uploader/util" ) // All function mainly used to make testing more convenient var applyRetentionPolicyFunc = policy.ApplyRetentionPolicy var treeForSourceFunc = policy.TreeForSource var setPolicyFunc = policy.SetPolicy var saveSnapshotFunc = snapshot.SaveSnapshot var loadSnapshotFunc = snapshot.LoadSnapshot var listSnapshotsFunc = snapshot.ListSnapshots var filesystemEntryFunc = snapshotfs.FilesystemEntryFromIDWithPath var restoreEntryFunc = restore.Entry var flushVolumeFunc = flushVolume const UploaderConfigMultipartKey = "uploader-multipart" const MaxErrorReported = 10 // SnapshotUploader which mainly used for UT test that could overwrite Upload interface type SnapshotUploader interface { Upload( ctx context.Context, source fs.Entry, policyTree *policy.Tree, sourceInfo snapshot.SourceInfo, previousManifests ...*snapshot.Manifest, ) (*snapshot.Manifest, error) } func newOptionalInt(b int) *policy.OptionalInt { ob := policy.OptionalInt(b) return &ob } func newOptionalInt64(b int64) *policy.OptionalInt64 { ob := policy.OptionalInt64(b) return &ob } func newOptionalBool(b bool) *policy.OptionalBool { ob := policy.OptionalBool(b) return &ob } func getDefaultPolicy() *policy.Policy { return &policy.Policy{ RetentionPolicy: policy.RetentionPolicy{ KeepLatest: newOptionalInt(math.MaxInt32), KeepAnnual: newOptionalInt(math.MaxInt32), KeepDaily: newOptionalInt(math.MaxInt32), KeepHourly: newOptionalInt(math.MaxInt32), KeepMonthly: newOptionalInt(math.MaxInt32), KeepWeekly: newOptionalInt(math.MaxInt32), }, CompressionPolicy: policy.CompressionPolicy{ CompressorName: "none", }, UploadPolicy: policy.UploadPolicy{ MaxParallelFileReads: newOptionalInt(runtime.NumCPU()), ParallelUploadAboveSize: newOptionalInt64(math.MaxInt64), }, SchedulingPolicy: policy.SchedulingPolicy{ Manual: true, }, ErrorHandlingPolicy: policy.ErrorHandlingPolicy{ IgnoreUnknownTypes: newOptionalBool(true), }, } } func setupPolicy(ctx context.Context, rep repo.RepositoryWriter, sourceInfo snapshot.SourceInfo, uploaderCfg map[string]string) (*policy.Tree, error) { // some internal operations from Kopia code retrieves policies from repo directly, so we need to persist the policy to repo curPolicy := getDefaultPolicy() if len(uploaderCfg) > 0 { parallelUpload, err := uploaderutil.GetParallelFilesUpload(uploaderCfg) if err != nil { return nil, errors.Wrap(err, "failed to get uploader config") } if parallelUpload > 0 { curPolicy.UploadPolicy.MaxParallelFileReads = newOptionalInt(parallelUpload) } } if _, ok := uploaderCfg[UploaderConfigMultipartKey]; ok { curPolicy.UploadPolicy.ParallelUploadAboveSize = newOptionalInt64(2 << 30) } if runtime.GOOS == "windows" { curPolicy.FilesPolicy.IgnoreRules = []string{"/System Volume Information/", "/$Recycle.Bin/"} } err := setPolicyFunc(ctx, rep, sourceInfo, curPolicy) if err != nil { return nil, errors.Wrap(err, "error to set policy") } err = rep.Flush(ctx) if err != nil { return nil, errors.Wrap(err, "error to flush repo") } // retrieve policy from repo policyTree, err := treeForSourceFunc(ctx, rep, sourceInfo) if err != nil { return nil, errors.Wrap(err, "error to retrieve policy") } return policyTree, nil } // Backup backup specific sourcePath and update progress func Backup(ctx context.Context, fsUploader SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, tags map[string]string, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error) { if fsUploader == nil { return nil, false, errors.New("get empty kopia uploader") } source, err := filepath.Abs(sourcePath) if err != nil { return nil, false, errors.Wrapf(err, "Invalid source path '%s'", sourcePath) } source = filepath.Clean(source) sourceInfo := snapshot.SourceInfo{ UserName: udmrepo.GetRepoUser(), Host: udmrepo.GetRepoDomain(), Path: filepath.Clean(realSource), } if realSource == "" { sourceInfo.Path = source } var sourceEntry fs.Entry if volMode == uploader.PersistentVolumeBlock { sourceEntry, err = getLocalBlockEntry(source) if err != nil { return nil, false, errors.Wrap(err, "unable to get local block device entry") } } else { sourceEntry, err = getLocalFSEntry(source) if err != nil { return nil, false, errors.Wrap(err, "unable to get local filesystem entry") } } kopiaCtx := kopia.SetupKopiaLog(ctx, log) snapID, snapshotSize, err := SnapshotSource(kopiaCtx, repoWriter, fsUploader, sourceInfo, sourceEntry, forceFull, parentSnapshot, tags, uploaderCfg, log, "Kopia Uploader") snapshotInfo := &uploader.SnapshotInfo{ ID: snapID, Size: snapshotSize, } return snapshotInfo, false, err } func getLocalFSEntry(path0 string) (fs.Entry, error) { path, err := resolveSymlink(path0) if err != nil { return nil, errors.Wrap(err, "resolveSymlink") } e, err := localfs.NewEntry(path) if err != nil { return nil, errors.Wrap(err, "can't get local fs entry") } return e, nil } // resolveSymlink returns the path name after the evaluation of any symbolic links func resolveSymlink(path string) (string, error) { st, err := os.Lstat(path) if err != nil { return "", errors.Wrap(err, "stat") } if (st.Mode() & os.ModeSymlink) == 0 { return path, nil } return filepath.EvalSymlinks(path) } // SnapshotSource which setup policy for snapshot, upload snapshot, update progress func SnapshotSource( ctx context.Context, rep repo.RepositoryWriter, u SnapshotUploader, sourceInfo snapshot.SourceInfo, rootDir fs.Entry, forceFull bool, parentSnapshot string, snapshotTags map[string]string, uploaderCfg map[string]string, log logrus.FieldLogger, description string, ) (string, int64, error) { log.Info("Start to snapshot...") snapshotStartTime := time.Now() var previous []*snapshot.Manifest if !forceFull { if parentSnapshot != "" { log.Infof("Using provided parent snapshot %s", parentSnapshot) mani, err := loadSnapshotFunc(ctx, rep, manifest.ID(parentSnapshot)) if err != nil { log.WithError(err).Warnf("Failed to load previous snapshot %v from kopia, fallback to full backup", parentSnapshot) } else { previous = append(previous, mani) } } else { log.Infof("Searching for parent snapshot") pre, err := findPreviousSnapshotManifest(ctx, rep, sourceInfo, snapshotTags, nil, log) if err != nil { return "", 0, errors.Wrapf(err, "Failed to find previous kopia snapshot manifests for si %v", sourceInfo) } previous = pre } } else { log.Info("Forcing full snapshot") } for i := range previous { log.Infof("Using parent snapshot %s, start time %v, end time %v, description %s", previous[i].ID, previous[i].StartTime.ToTime(), previous[i].EndTime.ToTime(), previous[i].Description) } policyTree, err := setupPolicy(ctx, rep, sourceInfo, uploaderCfg) if err != nil { return "", 0, errors.Wrapf(err, "unable to set policy for si %v", sourceInfo) } manifest, err := u.Upload(ctx, rootDir, policyTree, sourceInfo, previous...) if err != nil { return "", 0, errors.Wrapf(err, "Failed to upload the kopia snapshot for si %v", sourceInfo) } manifest.Tags = snapshotTags manifest.Description = description manifest.Pins = []string{"velero-pin"} if _, err = saveSnapshotFunc(ctx, rep, manifest); err != nil { return "", 0, errors.Wrapf(err, "Failed to save kopia manifest %v", manifest.ID) } _, err = applyRetentionPolicyFunc(ctx, rep, sourceInfo, true) if err != nil { return "", 0, errors.Wrapf(err, "Failed to apply kopia retention policy for si %v", sourceInfo) } if err = rep.Flush(ctx); err != nil { return "", 0, errors.Wrapf(err, "Failed to flush kopia repository") } log.Infof("Created snapshot with root %v and ID %v in %v", manifest.RootObjectID(), manifest.ID, time.Since(snapshotStartTime).Truncate(time.Second)) return reportSnapshotStatus(manifest, policyTree) } func reportSnapshotStatus(manifest *snapshot.Manifest, policyTree *policy.Tree) (string, int64, error) { manifestID := manifest.ID snapSize := manifest.Stats.TotalFileSize var errs []string if ds := manifest.RootEntry.DirSummary; ds != nil { for _, ent := range ds.FailedEntries { if len(errs) > MaxErrorReported { errs = append(errs, "too many errors, ignored...") break } policy := policyTree.EffectivePolicy() if !(policy != nil && bool(*policy.ErrorHandlingPolicy.IgnoreUnknownTypes) && strings.Contains(ent.Error, fs.ErrUnknown.Error())) { errs = append(errs, fmt.Sprintf("Error when processing %v: %v", ent.EntryPath, ent.Error)) } } } if len(errs) != 0 { return string(manifestID), snapSize, errors.New(strings.Join(errs, "\n")) } return string(manifestID), snapSize, nil } // findPreviousSnapshotManifest returns the list of previous snapshots for a given source, including // last complete snapshot following it. func findPreviousSnapshotManifest(ctx context.Context, rep repo.Repository, sourceInfo snapshot.SourceInfo, snapshotTags map[string]string, noLaterThan *fs.UTCTimestamp, log logrus.FieldLogger) ([]*snapshot.Manifest, error) { man, err := listSnapshotsFunc(ctx, rep, sourceInfo) if err != nil { return nil, err } var previousComplete *snapshot.Manifest var result []*snapshot.Manifest for _, p := range man { log.Debugf("Found one snapshot %s, start time %v, incomplete %s, tags %v", p.ID, p.StartTime.ToTime(), p.IncompleteReason, p.Tags) requester, found := p.Tags[uploader.SnapshotRequesterTag] if !found { continue } if requester != snapshotTags[uploader.SnapshotRequesterTag] { continue } uploaderName, found := p.Tags[uploader.SnapshotUploaderTag] if !found { continue } if uploaderName != snapshotTags[uploader.SnapshotUploaderTag] { continue } if noLaterThan != nil && p.StartTime.After(*noLaterThan) { continue } if p.IncompleteReason == "" && (previousComplete == nil || p.StartTime.After(previousComplete.StartTime)) { previousComplete = p } } if previousComplete != nil { result = append(result, previousComplete) } return result, nil } type fileSystemRestoreOutput struct { *restore.FilesystemOutput } func (o *fileSystemRestoreOutput) Flush() error { return flushVolumeFunc(o.TargetPath) } func (o *fileSystemRestoreOutput) Terminate() error { return nil } // Restore restore specific sourcePath with given snapshotID and update progress func Restore(ctx context.Context, rep repo.RepositoryWriter, progress *Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) { log.Info("Start to restore...") kopiaCtx := kopia.SetupKopiaLog(ctx, log) snapshot, err := snapshot.LoadSnapshot(kopiaCtx, rep, manifest.ID(snapshotID)) if err != nil { return 0, 0, errors.Wrapf(err, "Unable to load snapshot %v", snapshotID) } log.Infof("Restore from snapshot %s, description %s, created time %v, tags %v", snapshotID, snapshot.Description, snapshot.EndTime.ToTime(), snapshot.Tags) rootEntry, err := filesystemEntryFunc(kopiaCtx, rep, snapshotID, false) if err != nil { return 0, 0, errors.Wrapf(err, "Unable to get filesystem entry for snapshot %v", snapshotID) } path, err := filepath.Abs(dest) if err != nil { return 0, 0, errors.Wrapf(err, "Unable to resolve path %v", dest) } fsOutput := &restore.FilesystemOutput{ TargetPath: path, OverwriteDirectories: true, OverwriteFiles: true, OverwriteSymlinks: true, IgnorePermissionErrors: true, } restoreConcurrency := runtime.NumCPU() if len(uploaderCfg) > 0 { writeSparseFiles, err := uploaderutil.GetWriteSparseFiles(uploaderCfg) if err != nil { return 0, 0, errors.Wrap(err, "failed to get uploader config") } if writeSparseFiles { fsOutput.WriteSparseFiles = true } concurrency, err := uploaderutil.GetRestoreConcurrency(uploaderCfg) if err != nil { return 0, 0, errors.Wrap(err, "failed to get parallel restore uploader config") } if concurrency > 0 { restoreConcurrency = concurrency } } log.Debugf("Restore filesystem output %v, concurrency %d", fsOutput, restoreConcurrency) err = fsOutput.Init(ctx) if err != nil { return 0, 0, errors.Wrap(err, "error to init output") } var output RestoreOutput if volMode == uploader.PersistentVolumeBlock { output = &BlockOutput{ FilesystemOutput: fsOutput, } } else { output = &fileSystemRestoreOutput{ FilesystemOutput: fsOutput, } } defer func() { if err := output.Terminate(); err != nil { log.Warnf("error terminating restore output for %v", path) } }() stat, err := restoreEntryFunc(kopiaCtx, rep, output, rootEntry, restore.Options{ Parallel: restoreConcurrency, RestoreDirEntryAtDepth: math.MaxInt32, Cancel: cancleCh, ProgressCallback: func(ctx context.Context, stats restore.Stats) { progress.ProgressBytes(stats.RestoredTotalFileSize, stats.EnqueuedTotalFileSize) }, }) if err != nil { return 0, 0, errors.Wrapf(err, "Failed to copy snapshot data to the target") } if err := output.Flush(); err != nil { if err == errFlushUnsupported { log.Warnf("Skip flushing data for %v under the current OS %v", path, runtime.GOOS) } else { return 0, 0, errors.Wrapf(err, "Failed to flush data to target") } } else { log.Infof("Flush done for volume dir %v", path) } return stat.RestoredTotalFileSize, stat.RestoredFileCount, nil } ================================================ FILE: pkg/uploader/kopia/snapshot_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package kopia import ( "context" "strings" "testing" "time" "github.com/kopia/kopia/fs" "github.com/kopia/kopia/fs/virtualfs" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/manifest" "github.com/kopia/kopia/snapshot" "github.com/kopia/kopia/snapshot/policy" "github.com/kopia/kopia/snapshot/restore" "github.com/kopia/kopia/snapshot/snapshotfs" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" repomocks "github.com/vmware-tanzu/velero/pkg/repository/mocks" "github.com/vmware-tanzu/velero/pkg/uploader" uploadermocks "github.com/vmware-tanzu/velero/pkg/uploader/mocks" ) type snapshotMockes struct { policyMock *uploadermocks.Policy snapshotMock *uploadermocks.Snapshot uploderMock *uploadermocks.Uploader repoWriterMock *repomocks.RepositoryWriter } type mockArgs struct { methodName string returns []any } func injectSnapshotFuncs() *snapshotMockes { s := &snapshotMockes{ policyMock: &uploadermocks.Policy{}, snapshotMock: &uploadermocks.Snapshot{}, uploderMock: &uploadermocks.Uploader{}, repoWriterMock: &repomocks.RepositoryWriter{}, } applyRetentionPolicyFunc = s.policyMock.ApplyRetentionPolicy setPolicyFunc = s.policyMock.SetPolicy treeForSourceFunc = s.policyMock.TreeForSource loadSnapshotFunc = s.snapshotMock.LoadSnapshot saveSnapshotFunc = s.snapshotMock.SaveSnapshot return s } func MockFuncs(s *snapshotMockes, args []mockArgs) { s.snapshotMock.On("LoadSnapshot", mock.Anything, mock.Anything, mock.Anything).Return(args[0].returns...) s.snapshotMock.On("SaveSnapshot", mock.Anything, mock.Anything, mock.Anything).Return(args[1].returns...) s.policyMock.On("TreeForSource", mock.Anything, mock.Anything, mock.Anything).Return(args[2].returns...) s.policyMock.On("ApplyRetentionPolicy", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(args[3].returns...) s.policyMock.On("SetPolicy", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(args[4].returns...) s.uploderMock.On("Upload", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(args[5].returns...) s.repoWriterMock.On("Flush", mock.Anything).Return(args[6].returns...) } func TestSnapshotSource(t *testing.T) { ctx := t.Context() sourceInfo := snapshot.SourceInfo{ UserName: "testUserName", Host: "testHost", Path: "/var", } rootDir, err := getLocalFSEntry(sourceInfo.Path) require.NoError(t, err) log := logrus.New() manifest := &snapshot.Manifest{ ID: "test", RootEntry: &snapshot.DirEntry{}, } testCases := []struct { name string args []mockArgs uploaderCfg map[string]string notError bool }{ { name: "regular test", args: []mockArgs{ {methodName: "LoadSnapshot", returns: []any{manifest, nil}}, {methodName: "SaveSnapshot", returns: []any{manifest.ID, nil}}, {methodName: "TreeForSource", returns: []any{nil, nil}}, {methodName: "ApplyRetentionPolicy", returns: []any{nil, nil}}, {methodName: "SetPolicy", returns: []any{nil}}, {methodName: "Upload", returns: []any{manifest, nil}}, {methodName: "Flush", returns: []any{nil}}, }, notError: true, }, { name: "failed to load snapshot, should fallback to full backup and not error", args: []mockArgs{ {methodName: "LoadSnapshot", returns: []any{manifest, errors.New("failed to load snapshot")}}, {methodName: "SaveSnapshot", returns: []any{manifest.ID, nil}}, {methodName: "TreeForSource", returns: []any{nil, nil}}, {methodName: "ApplyRetentionPolicy", returns: []any{nil, nil}}, {methodName: "SetPolicy", returns: []any{nil}}, {methodName: "Upload", returns: []any{manifest, nil}}, {methodName: "Flush", returns: []any{nil}}, }, notError: true, }, { name: "failed to save snapshot", args: []mockArgs{ {methodName: "LoadSnapshot", returns: []any{manifest, nil}}, {methodName: "SaveSnapshot", returns: []any{manifest.ID, errors.New("failed to save snapshot")}}, {methodName: "TreeForSource", returns: []any{nil, nil}}, {methodName: "ApplyRetentionPolicy", returns: []any{nil, nil}}, {methodName: "SetPolicy", returns: []any{nil}}, {methodName: "Upload", returns: []any{manifest, nil}}, {methodName: "Flush", returns: []any{nil}}, }, notError: false, }, { name: "failed to set policy", args: []mockArgs{ {methodName: "LoadSnapshot", returns: []any{manifest, nil}}, {methodName: "SaveSnapshot", returns: []any{manifest.ID, nil}}, {methodName: "TreeForSource", returns: []any{nil, nil}}, {methodName: "ApplyRetentionPolicy", returns: []any{nil, nil}}, {methodName: "SetPolicy", returns: []any{errors.New("failed to set policy")}}, {methodName: "Upload", returns: []any{manifest, nil}}, {methodName: "Flush", returns: []any{nil}}, }, notError: false, }, { name: "set policy with parallel files upload", args: []mockArgs{ {methodName: "LoadSnapshot", returns: []any{manifest, nil}}, {methodName: "SaveSnapshot", returns: []any{manifest.ID, nil}}, {methodName: "TreeForSource", returns: []any{nil, nil}}, {methodName: "ApplyRetentionPolicy", returns: []any{nil, nil}}, {methodName: "SetPolicy", returns: []any{nil}}, {methodName: "Upload", returns: []any{manifest, nil}}, {methodName: "Flush", returns: []any{nil}}, }, uploaderCfg: map[string]string{ "ParallelFilesUpload": "10", }, notError: true, }, { name: "failed to upload snapshot", args: []mockArgs{ {methodName: "LoadSnapshot", returns: []any{manifest, nil}}, {methodName: "SaveSnapshot", returns: []any{manifest.ID, nil}}, {methodName: "TreeForSource", returns: []any{nil, nil}}, {methodName: "ApplyRetentionPolicy", returns: []any{nil, nil}}, {methodName: "SetPolicy", returns: []any{nil}}, {methodName: "Upload", returns: []any{manifest, errors.New("failed to upload snapshot")}}, {methodName: "Flush", returns: []any{nil}}, }, notError: false, }, { name: "failed to flush repo", args: []mockArgs{ {methodName: "LoadSnapshot", returns: []any{manifest, nil}}, {methodName: "SaveSnapshot", returns: []any{manifest.ID, errors.New("failed to save snapshot")}}, {methodName: "TreeForSource", returns: []any{nil, nil}}, {methodName: "ApplyRetentionPolicy", returns: []any{nil, nil}}, {methodName: "SetPolicy", returns: []any{nil}}, {methodName: "Upload", returns: []any{manifest, nil}}, {methodName: "Flush", returns: []any{errors.New("failed to flush repo")}}, }, notError: false, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { s := injectSnapshotFuncs() MockFuncs(s, tc.args) _, _, err = SnapshotSource(ctx, s.repoWriterMock, s.uploderMock, sourceInfo, rootDir, false, "/", nil, tc.uploaderCfg, log, "TestSnapshotSource") if tc.notError { assert.NoError(t, err) } else { assert.Error(t, err) } }) } } func TestReportSnapshotStatus(t *testing.T) { testCases := []struct { shouldError bool expectedResult string expectedSize int64 directorySummary *fs.DirectorySummary expectedErrors []string }{ { shouldError: false, expectedResult: "sample-manifest-id", expectedSize: 1024, directorySummary: &fs.DirectorySummary{ TotalFileSize: 1024, }, }, { shouldError: true, expectedResult: "sample-manifest-id", expectedSize: 1024, directorySummary: &fs.DirectorySummary{ FailedEntries: []*fs.EntryWithError{ { EntryPath: "/path/to/file.txt", Error: "Unknown file error", }, }, }, expectedErrors: []string{"Error when processing /path/to/file.txt: Unknown file error"}, }, } for _, tc := range testCases { manifest := &snapshot.Manifest{ ID: manifest.ID("sample-manifest-id"), Stats: snapshot.Stats{ TotalFileSize: 1024, }, RootEntry: &snapshot.DirEntry{ DirSummary: tc.directorySummary, }, } result, size, err := reportSnapshotStatus(manifest, policy.BuildTree(nil, getDefaultPolicy())) switch { case tc.shouldError && err == nil: t.Errorf("expected error, but got nil") case !tc.shouldError && err != nil: t.Errorf("unexpected error: %v", err) case tc.shouldError && err != nil: expectedErr := strings.Join(tc.expectedErrors, "\n") if err.Error() != expectedErr { t.Errorf("unexpected error: got %v, want %v", err, expectedErr) } } if result != tc.expectedResult { t.Errorf("unexpected result: got %v, want %v", result, tc.expectedResult) } if size != tc.expectedSize { t.Errorf("unexpected size: got %v, want %v", size, tc.expectedSize) } } } func TestFindPreviousSnapshotManifest(t *testing.T) { // Prepare test data sourceInfo := snapshot.SourceInfo{ UserName: "user1", Host: "host1", Path: "/path/to/dir1", } snapshotTags := map[string]string{ uploader.SnapshotRequesterTag: "user1", uploader.SnapshotUploaderTag: "uploader1", } noLaterThan := fs.UTCTimestampFromTime(time.Now()) testCases := []struct { name string listSnapshotsFunc func(ctx context.Context, rep repo.Repository, si snapshot.SourceInfo) ([]*snapshot.Manifest, error) expectedSnapshots []*snapshot.Manifest expectedError error }{ // No matching snapshots { name: "No matching snapshots", listSnapshotsFunc: func(ctx context.Context, rep repo.Repository, si snapshot.SourceInfo) ([]*snapshot.Manifest, error) { return []*snapshot.Manifest{}, nil }, expectedSnapshots: []*snapshot.Manifest{}, expectedError: nil, }, { name: "Error getting manifest", listSnapshotsFunc: func(ctx context.Context, rep repo.Repository, si snapshot.SourceInfo) ([]*snapshot.Manifest, error) { return []*snapshot.Manifest{}, errors.New("Error getting manifest") }, expectedSnapshots: []*snapshot.Manifest{}, expectedError: errors.New("Error getting manifest"), }, // Only one matching snapshot { name: "One matching snapshot", listSnapshotsFunc: func(ctx context.Context, rep repo.Repository, si snapshot.SourceInfo) ([]*snapshot.Manifest, error) { return []*snapshot.Manifest{ { Tags: map[string]string{ uploader.SnapshotRequesterTag: "user1", uploader.SnapshotUploaderTag: "uploader1", "otherTag": "value", "anotherCustomTag": "123", "snapshotRequestor": "user1", "snapshotUploader": "uploader1", }, }, }, nil }, expectedSnapshots: []*snapshot.Manifest{ { Tags: map[string]string{ uploader.SnapshotRequesterTag: "user1", uploader.SnapshotUploaderTag: "uploader1", "otherTag": "value", "anotherCustomTag": "123", "snapshotRequestor": "user1", "snapshotUploader": "uploader1", }, }, }, expectedError: nil, }, // Multiple matching snapshots { name: "Multiple matching snapshots", listSnapshotsFunc: func(ctx context.Context, rep repo.Repository, si snapshot.SourceInfo) ([]*snapshot.Manifest, error) { return []*snapshot.Manifest{ { Tags: map[string]string{ uploader.SnapshotRequesterTag: "user1", uploader.SnapshotUploaderTag: "uploader1", "otherTag": "value1", "snapshotRequestor": "user1", "snapshotUploader": "uploader1", }, }, { Tags: map[string]string{ uploader.SnapshotRequesterTag: "user1", uploader.SnapshotUploaderTag: "uploader1", "otherTag": "value2", "snapshotRequestor": "user1", "snapshotUploader": "uploader1", }, }, }, nil }, expectedSnapshots: []*snapshot.Manifest{ { Tags: map[string]string{ uploader.SnapshotRequesterTag: "user1", uploader.SnapshotUploaderTag: "uploader1", "otherTag": "value1", "snapshotRequestor": "user1", "snapshotUploader": "uploader1", }, }, }, expectedError: nil, }, // Snapshot with different requester { name: "Snapshot with different requester", listSnapshotsFunc: func(ctx context.Context, rep repo.Repository, si snapshot.SourceInfo) ([]*snapshot.Manifest, error) { return []*snapshot.Manifest{ { Tags: map[string]string{ uploader.SnapshotRequesterTag: "user2", uploader.SnapshotUploaderTag: "uploader1", "otherTag": "value", "snapshotRequestor": "user2", "snapshotUploader": "uploader1", }, }, }, nil }, expectedSnapshots: []*snapshot.Manifest{}, expectedError: nil, }, // Snapshot with different uploader { name: "Snapshot with different uploader", listSnapshotsFunc: func(ctx context.Context, rep repo.Repository, si snapshot.SourceInfo) ([]*snapshot.Manifest, error) { return []*snapshot.Manifest{ { Tags: map[string]string{ uploader.SnapshotRequesterTag: "user1", uploader.SnapshotUploaderTag: "uploader2", "otherTag": "value", "snapshotRequestor": "user1", "snapshotUploader": "uploader2", }, }, }, nil }, expectedSnapshots: []*snapshot.Manifest{}, expectedError: nil, }, // Snapshot with a later start time { name: "Snapshot with a later start time", listSnapshotsFunc: func(ctx context.Context, rep repo.Repository, si snapshot.SourceInfo) ([]*snapshot.Manifest, error) { return []*snapshot.Manifest{ { Tags: map[string]string{ uploader.SnapshotRequesterTag: "user1", uploader.SnapshotUploaderTag: "uploader1", "otherTag": "value", "snapshotRequestor": "user1", "snapshotUploader": "uploader1", }, StartTime: fs.UTCTimestampFromTime(time.Now().Add(time.Hour)), }, }, nil }, expectedSnapshots: []*snapshot.Manifest{}, expectedError: nil, }, // Snapshot with incomplete reason { name: "Snapshot with incomplete reason", listSnapshotsFunc: func(ctx context.Context, rep repo.Repository, si snapshot.SourceInfo) ([]*snapshot.Manifest, error) { return []*snapshot.Manifest{ { Tags: map[string]string{ uploader.SnapshotRequesterTag: "user1", uploader.SnapshotUploaderTag: "uploader1", "otherTag": "value", "snapshotRequestor": "user1", "snapshotUploader": "uploader1", }, IncompleteReason: "reason", }, }, nil }, expectedSnapshots: []*snapshot.Manifest{}, expectedError: nil, }, // Multiple snapshots with some matching conditions { name: "Multiple snapshots with matching conditions", listSnapshotsFunc: func(ctx context.Context, rep repo.Repository, si snapshot.SourceInfo) ([]*snapshot.Manifest, error) { return []*snapshot.Manifest{ { Tags: map[string]string{ uploader.SnapshotRequesterTag: "user1", uploader.SnapshotUploaderTag: "uploader1", "otherTag": "value1", "snapshotRequestor": "user1", "snapshotUploader": "uploader1", }, }, { Tags: map[string]string{ uploader.SnapshotRequesterTag: "user1", uploader.SnapshotUploaderTag: "uploader1", "otherTag": "value2", "snapshotRequestor": "user1", "snapshotUploader": "uploader1", }, StartTime: fs.UTCTimestampFromTime(time.Now().Add(-time.Hour)), IncompleteReason: "reason", }, { Tags: map[string]string{ uploader.SnapshotRequesterTag: "user1", uploader.SnapshotUploaderTag: "uploader1", "otherTag": "value3", "snapshotRequestor": "user1", "snapshotUploader": "uploader1", }, StartTime: fs.UTCTimestampFromTime(time.Now().Add(-time.Hour)), }, }, nil }, expectedSnapshots: []*snapshot.Manifest{ { Tags: map[string]string{ uploader.SnapshotRequesterTag: "user1", uploader.SnapshotUploaderTag: "uploader1", "otherTag": "value3", "snapshotRequestor": "user1", "snapshotUploader": "uploader1", }, StartTime: fs.UTCTimestampFromTime(time.Now().Add(-time.Hour)), }, }, expectedError: nil, }, // Snapshot with manifest SnapshotRequesterTag not found { name: "Snapshot with manifest SnapshotRequesterTag not found", listSnapshotsFunc: func(ctx context.Context, rep repo.Repository, si snapshot.SourceInfo) ([]*snapshot.Manifest, error) { return []*snapshot.Manifest{ { Tags: map[string]string{ "requester": "user1", uploader.SnapshotUploaderTag: "uploader1", "otherTag": "value", "snapshotRequestor": "user1", "snapshotUploader": "uploader1", }, IncompleteReason: "reason", }, }, nil }, expectedSnapshots: []*snapshot.Manifest{}, expectedError: nil, }, // Snapshot with manifest SnapshotRequesterTag not found { name: "Snapshot with manifest SnapshotUploaderTag not found", listSnapshotsFunc: func(ctx context.Context, rep repo.Repository, si snapshot.SourceInfo) ([]*snapshot.Manifest, error) { return []*snapshot.Manifest{ { Tags: map[string]string{ uploader.SnapshotRequesterTag: "user1", "uploader": "uploader1", "otherTag": "value", "snapshotRequestor": "user1", "snapshotUploader": "uploader1", }, IncompleteReason: "reason", }, }, nil }, expectedSnapshots: []*snapshot.Manifest{}, expectedError: nil, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { var repo repo.Repository listSnapshotsFunc = tc.listSnapshotsFunc snapshots, err := findPreviousSnapshotManifest(t.Context(), repo, sourceInfo, snapshotTags, &noLaterThan, logrus.New()) // Check if the returned error matches the expected error if tc.expectedError != nil { require.ErrorContains(t, err, tc.expectedError.Error()) } else { require.NoError(t, err) } // Check the number of returned snapshots if len(snapshots) != len(tc.expectedSnapshots) { t.Errorf("Expected %d snapshots, got %d", len(tc.expectedSnapshots), len(snapshots)) } }) } } func TestBackup(t *testing.T) { type testCase struct { name string sourcePath string forceFull bool parentSnapshot string tags map[string]string isEmptyUploader bool isSnapshotSourceError bool expectedError error expectedEmpty bool volMode uploader.PersistentVolumeMode } manifest := &snapshot.Manifest{ ID: "test", RootEntry: &snapshot.DirEntry{}, } // Define test cases testCases := []testCase{ { name: "Successful backup", sourcePath: "/", tags: map[string]string{}, expectedError: nil, }, { name: "Empty fsUploader", isEmptyUploader: true, sourcePath: "/", tags: nil, expectedError: errors.New("get empty kopia uploader"), }, { name: "Unable to read directory", sourcePath: "/invalid/path", tags: nil, expectedError: errors.New("no such file or directory"), }, { name: "Source path is not a block device", sourcePath: "/", tags: nil, volMode: uploader.PersistentVolumeBlock, expectedError: errors.New("source path / is not a block device"), }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { if tc.volMode == "" { tc.volMode = uploader.PersistentVolumeFilesystem } s := injectSnapshotFuncs() args := []mockArgs{ {methodName: "LoadSnapshot", returns: []any{manifest, nil}}, {methodName: "SaveSnapshot", returns: []any{manifest.ID, nil}}, {methodName: "TreeForSource", returns: []any{nil, nil}}, {methodName: "ApplyRetentionPolicy", returns: []any{nil, nil}}, {methodName: "SetPolicy", returns: []any{nil}}, {methodName: "Upload", returns: []any{manifest, nil}}, {methodName: "Flush", returns: []any{nil}}, } MockFuncs(s, args) if tc.isSnapshotSourceError { s.repoWriterMock.On("FindManifests", mock.Anything, mock.Anything).Return(nil, errors.New("Failed to get manifests")) s.repoWriterMock.On("Flush", mock.Anything).Return(errors.New("Failed to get manifests")) } else { s.repoWriterMock.On("FindManifests", mock.Anything, mock.Anything).Return(nil, nil) } var isSnapshotEmpty bool var snapshotInfo *uploader.SnapshotInfo var err error if tc.isEmptyUploader { snapshotInfo, isSnapshotEmpty, err = Backup(t.Context(), nil, s.repoWriterMock, tc.sourcePath, "", tc.forceFull, tc.parentSnapshot, tc.volMode, map[string]string{}, tc.tags, &logrus.Logger{}) } else { snapshotInfo, isSnapshotEmpty, err = Backup(t.Context(), s.uploderMock, s.repoWriterMock, tc.sourcePath, "", tc.forceFull, tc.parentSnapshot, tc.volMode, map[string]string{}, tc.tags, &logrus.Logger{}) } // Check if the returned error matches the expected error if tc.expectedError != nil { require.ErrorContains(t, err, tc.expectedError.Error()) } else { require.NoError(t, err) } assert.Equal(t, tc.expectedEmpty, isSnapshotEmpty) if err == nil { assert.NotNil(t, snapshotInfo) } }) } } func TestRestore(t *testing.T) { type testCase struct { name string snapshotID string invalidManifestType bool filesystemEntryFunc func(ctx context.Context, rep repo.Repository, rootID string, consistentAttributes bool) (fs.Entry, error) restoreEntryFunc func(ctx context.Context, rep repo.Repository, output restore.Output, rootEntry fs.Entry, options restore.Options) (restore.Stats, error) flushVolumeFunc func(string) error dest string expectedBytes int64 expectedCount int32 expectedError error volMode uploader.PersistentVolumeMode } // Define test cases testCases := []testCase{ { name: "manifest is not a snapshot", invalidManifestType: true, dest: "/path/to/destination", expectedError: errors.New("Unable to load snapshot"), }, { name: "Failed to get filesystem entry", snapshotID: "snapshot-123", expectedError: errors.New("Unable to get filesystem entry"), }, { name: "Failed to restore with filesystem entry", filesystemEntryFunc: func(ctx context.Context, rep repo.Repository, rootID string, consistentAttributes bool) (fs.Entry, error) { return snapshotfs.EntryFromDirEntry(rep, &snapshot.DirEntry{Type: snapshot.EntryTypeFile}), nil }, restoreEntryFunc: func(ctx context.Context, rep repo.Repository, output restore.Output, rootEntry fs.Entry, options restore.Options) (restore.Stats, error) { return restore.Stats{}, errors.New("Unable to get filesystem entry") }, snapshotID: "snapshot-123", expectedError: errors.New("Unable to get filesystem entry"), }, { name: "Expect successful", filesystemEntryFunc: func(ctx context.Context, rep repo.Repository, rootID string, consistentAttributes bool) (fs.Entry, error) { return snapshotfs.EntryFromDirEntry(rep, &snapshot.DirEntry{Type: snapshot.EntryTypeFile}), nil }, restoreEntryFunc: func(ctx context.Context, rep repo.Repository, output restore.Output, rootEntry fs.Entry, options restore.Options) (restore.Stats, error) { return restore.Stats{}, nil }, snapshotID: "snapshot-123", expectedError: nil, }, { name: "Expect block volume successful", filesystemEntryFunc: func(ctx context.Context, rep repo.Repository, rootID string, consistentAttributes bool) (fs.Entry, error) { return snapshotfs.EntryFromDirEntry(rep, &snapshot.DirEntry{Type: snapshot.EntryTypeFile}), nil }, restoreEntryFunc: func(ctx context.Context, rep repo.Repository, output restore.Output, rootEntry fs.Entry, options restore.Options) (restore.Stats, error) { return restore.Stats{}, nil }, snapshotID: "snapshot-123", expectedError: nil, volMode: uploader.PersistentVolumeBlock, }, { name: "Unable to evaluate symlinks for block volume", filesystemEntryFunc: func(ctx context.Context, rep repo.Repository, rootID string, consistentAttributes bool) (fs.Entry, error) { return snapshotfs.EntryFromDirEntry(rep, &snapshot.DirEntry{Type: snapshot.EntryTypeFile}), nil }, restoreEntryFunc: func(ctx context.Context, rep repo.Repository, output restore.Output, rootEntry fs.Entry, options restore.Options) (restore.Stats, error) { err := output.BeginDirectory(ctx, "fake-dir", virtualfs.NewStaticDirectory("fake-dir", nil)) return restore.Stats{}, err }, snapshotID: "snapshot-123", expectedError: errors.New("unable to evaluate symlinks for"), volMode: uploader.PersistentVolumeBlock, dest: "/wrong-dest", }, { name: "Target file is not a block device", filesystemEntryFunc: func(ctx context.Context, rep repo.Repository, rootID string, consistentAttributes bool) (fs.Entry, error) { return snapshotfs.EntryFromDirEntry(rep, &snapshot.DirEntry{Type: snapshot.EntryTypeFile}), nil }, restoreEntryFunc: func(ctx context.Context, rep repo.Repository, output restore.Output, rootEntry fs.Entry, options restore.Options) (restore.Stats, error) { err := output.BeginDirectory(ctx, "fake-dir", virtualfs.NewStaticDirectory("fake-dir", nil)) return restore.Stats{}, err }, snapshotID: "snapshot-123", expectedError: errors.New("target file /tmp is not a block device"), volMode: uploader.PersistentVolumeBlock, dest: "/tmp", }, { name: "Flush is not supported", filesystemEntryFunc: func(ctx context.Context, rep repo.Repository, rootID string, consistentAttributes bool) (fs.Entry, error) { return snapshotfs.EntryFromDirEntry(rep, &snapshot.DirEntry{Type: snapshot.EntryTypeFile}), nil }, restoreEntryFunc: func(ctx context.Context, rep repo.Repository, output restore.Output, rootEntry fs.Entry, options restore.Options) (restore.Stats, error) { return restore.Stats{}, nil }, flushVolumeFunc: func(string) error { return errFlushUnsupported }, snapshotID: "snapshot-123", expectedError: nil, }, { name: "Flush fails", filesystemEntryFunc: func(ctx context.Context, rep repo.Repository, rootID string, consistentAttributes bool) (fs.Entry, error) { return snapshotfs.EntryFromDirEntry(rep, &snapshot.DirEntry{Type: snapshot.EntryTypeFile}), nil }, restoreEntryFunc: func(ctx context.Context, rep repo.Repository, output restore.Output, rootEntry fs.Entry, options restore.Options) (restore.Stats, error) { return restore.Stats{}, nil }, flushVolumeFunc: func(string) error { return errors.New("fake-flush-error") }, snapshotID: "snapshot-123", expectedError: errors.New("fake-flush-error"), }, } em := &manifest.EntryMetadata{ ID: "test", Labels: map[string]string{}, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { if tc.volMode == "" { tc.volMode = uploader.PersistentVolumeFilesystem } if tc.invalidManifestType { em.Labels[manifest.TypeLabelKey] = "" } else { em.Labels[manifest.TypeLabelKey] = snapshot.ManifestType } if tc.filesystemEntryFunc != nil { filesystemEntryFunc = tc.filesystemEntryFunc } if tc.restoreEntryFunc != nil { restoreEntryFunc = tc.restoreEntryFunc } if tc.flushVolumeFunc != nil { flushVolumeFunc = tc.flushVolumeFunc } repoWriterMock := &repomocks.RepositoryWriter{} repoWriterMock.On("GetManifest", mock.Anything, mock.Anything, mock.Anything).Return(em, nil) repoWriterMock.On("OpenObject", mock.Anything, mock.Anything).Return(em, nil) progress := new(Progress) bytesRestored, fileCount, err := Restore(t.Context(), repoWriterMock, progress, tc.snapshotID, tc.dest, tc.volMode, map[string]string{}, logrus.New(), nil) // Check if the returned error matches the expected error if tc.expectedError != nil { require.ErrorContains(t, err, tc.expectedError.Error()) } else { require.NoError(t, err) } // Check the number of bytes restored assert.Equal(t, tc.expectedBytes, bytesRestored) // Check the number of files restored assert.Equal(t, tc.expectedCount, fileCount) }) } } ================================================ FILE: pkg/uploader/mocks/policy.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package mocks import ( "context" "github.com/kopia/kopia/repo/manifest" "github.com/kopia/kopia/snapshot/policy" "github.com/stretchr/testify/mock" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/snapshot" ) // policy is an autogenerated mock type for the TreeForSource type type Policy struct { mock.Mock } // Execute provides a mock function with given fields: ctx, rep, si func (_m *Policy) TreeForSource(ctx context.Context, rep repo.Repository, si snapshot.SourceInfo) (*policy.Tree, error) { ret := _m.Called(ctx, rep, si) var r0 *policy.Tree if rf, ok := ret.Get(0).(func(context.Context, repo.Repository, snapshot.SourceInfo) *policy.Tree); ok { r0 = rf(ctx, rep, si) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(*policy.Tree) } } var r1 error if rf, ok := ret.Get(1).(func(context.Context, repo.Repository, snapshot.SourceInfo) error); ok { r1 = rf(ctx, rep, si) } else { r1 = ret.Error(1) } return r0, r1 } // ApplyRetentionPolicy provides a mock function with given fields: ctx, rep, sourceInfo, reallyDelete func (_m *Policy) ApplyRetentionPolicy(ctx context.Context, rep repo.RepositoryWriter, sourceInfo snapshot.SourceInfo, reallyDelete bool) ([]manifest.ID, error) { ret := _m.Called(ctx, rep, sourceInfo, reallyDelete) var r0 []manifest.ID if rf, ok := ret.Get(0).(func(context.Context, repo.RepositoryWriter, snapshot.SourceInfo, bool) []manifest.ID); ok { r0 = rf(ctx, rep, sourceInfo, reallyDelete) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]manifest.ID) } } var r1 error if rf, ok := ret.Get(1).(func(context.Context, repo.RepositoryWriter, snapshot.SourceInfo, bool) error); ok { r1 = rf(ctx, rep, sourceInfo, reallyDelete) } else { r1 = ret.Error(1) } return r0, r1 } func (_m *Policy) SetPolicy(ctx context.Context, rep repo.RepositoryWriter, si snapshot.SourceInfo, pol *policy.Policy) error { ret := _m.Called(ctx, rep, si, pol) var r0 error if rf, ok := ret.Get(0).(func(context.Context, repo.RepositoryWriter, snapshot.SourceInfo, *policy.Policy) error); ok { r0 = rf(ctx, rep, si, pol) } else { r0 = ret.Error(0) } return r0 } ================================================ FILE: pkg/uploader/mocks/shim.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package mocks import ( context "context" mock "github.com/stretchr/testify/mock" ) // shimRepository is an autogenerated mock type for the shimRepository type type ShimRepository struct { mock.Mock } // Flush provides a mock function with given fields: ctx func (_m *ShimRepository) Flush(ctx context.Context) error { ret := _m.Called(ctx) var r0 error if rf, ok := ret.Get(0).(func(context.Context) error); ok { r0 = rf(ctx) } else { r0 = ret.Error(0) } return r0 } ================================================ FILE: pkg/uploader/mocks/snapshot.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package mocks import ( "context" "github.com/kopia/kopia/repo/manifest" "github.com/kopia/kopia/snapshot" "github.com/stretchr/testify/mock" "github.com/kopia/kopia/repo" ) // snapshot is an autogenerated mock type for the snapshot type type Snapshot struct { mock.Mock } // LoadSnapshot provides a mock function with given fields: ctx, rep, manifestID func (_m *Snapshot) LoadSnapshot(ctx context.Context, rep repo.Repository, manifestID manifest.ID) (*snapshot.Manifest, error) { ret := _m.Called(ctx, rep, manifestID) var r0 *snapshot.Manifest if rf, ok := ret.Get(0).(func(context.Context, repo.Repository, manifest.ID) *snapshot.Manifest); ok { r0 = rf(ctx, rep, manifestID) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(*snapshot.Manifest) } } var r1 error if rf, ok := ret.Get(1).(func(context.Context, repo.Repository, manifest.ID) error); ok { r1 = rf(ctx, rep, manifestID) } else { r1 = ret.Error(1) } return r0, r1 } // SaveSnapshot provides a mock function with given fields: ctx, rep, man func (_m *Snapshot) SaveSnapshot(ctx context.Context, rep repo.RepositoryWriter, man *snapshot.Manifest) (manifest.ID, error) { ret := _m.Called(ctx, rep, man) var r0 manifest.ID if rf, ok := ret.Get(0).(func(context.Context, repo.RepositoryWriter, *snapshot.Manifest) manifest.ID); ok { r0 = rf(ctx, rep, man) } else { r0 = ret.Get(0).(manifest.ID) } var r1 error if rf, ok := ret.Get(1).(func(context.Context, repo.RepositoryWriter, *snapshot.Manifest) error); ok { r1 = rf(ctx, rep, man) } else { r1 = ret.Error(1) } return r0, r1 } ================================================ FILE: pkg/uploader/mocks/uploader.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package mocks import ( "context" "github.com/kopia/kopia/fs" "github.com/stretchr/testify/mock" "github.com/kopia/kopia/snapshot/policy" "github.com/kopia/kopia/snapshot" ) // Upload is an autogenerated mock type for the Upload type type Uploader struct { mock.Mock } // Execute provides a mock function with given fields: ctx, source, policyTree, sourceInfo, previousManifests func (_m *Uploader) Upload(ctx context.Context, source fs.Entry, policyTree *policy.Tree, sourceInfo snapshot.SourceInfo, previousManifests ...*snapshot.Manifest) (*snapshot.Manifest, error) { _va := make([]any, len(previousManifests)) for _i := range previousManifests { _va[_i] = previousManifests[_i] } var _ca []any _ca = append(_ca, ctx, source, policyTree, sourceInfo) _ca = append(_ca, _va...) ret := _m.Called(_ca...) var r0 *snapshot.Manifest if rf, ok := ret.Get(0).(func(context.Context, fs.Entry, *policy.Tree, snapshot.SourceInfo, ...*snapshot.Manifest) *snapshot.Manifest); ok { r0 = rf(ctx, source, policyTree, sourceInfo, previousManifests...) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(*snapshot.Manifest) } } var r1 error if rf, ok := ret.Get(1).(func(context.Context, fs.Entry, *policy.Tree, snapshot.SourceInfo, ...*snapshot.Manifest) error); ok { r1 = rf(ctx, source, policyTree, sourceInfo, previousManifests...) } else { r1 = ret.Error(1) } return r0, r1 } ================================================ FILE: pkg/uploader/provider/kopia.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package provider import ( "context" "fmt" "strings" "sync/atomic" "github.com/kopia/kopia/snapshot/upload" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/vmware-tanzu/velero/pkg/uploader" "github.com/vmware-tanzu/velero/pkg/uploader/kopia" "github.com/vmware-tanzu/velero/internal/credentials" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" repokeys "github.com/vmware-tanzu/velero/pkg/repository/keys" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo/service" ) // BackupFunc mainly used to make testing more convenient var BackupFunc = kopia.Backup var RestoreFunc = kopia.Restore var BackupRepoServiceCreateFunc = service.Create // kopiaProvider recorded info related with kopiaProvider type kopiaProvider struct { requestorType string bkRepo udmrepo.BackupRepo credGetter *credentials.CredentialGetter log logrus.FieldLogger canceling int32 } // NewKopiaUploaderProvider initialized with open or create a repository func NewKopiaUploaderProvider( requestorType string, ctx context.Context, credGetter *credentials.CredentialGetter, backupRepo *velerov1api.BackupRepository, log logrus.FieldLogger, ) (Provider, error) { kp := &kopiaProvider{ requestorType: requestorType, log: log, credGetter: credGetter, } //repoUID which is used to generate kopia repository config with unique directory path repoUID := string(backupRepo.GetUID()) repoOpt, err := udmrepo.NewRepoOptions( udmrepo.WithPassword(kp, ""), udmrepo.WithConfigFile("", repoUID), udmrepo.WithDescription("Initial kopia uploader provider"), ) if err != nil { return nil, errors.Wrapf(err, "error to get repo options") } repoSvc := BackupRepoServiceCreateFunc(backupRepo.Spec.RepositoryType, log) log.WithField("repoUID", repoUID).Info("Opening backup repo") kp.bkRepo, err = repoSvc.Open(ctx, *repoOpt) if err != nil { return nil, errors.Wrapf(err, "Failed to find kopia repository") } return kp, nil } // CheckContext check context status check if context is timeout or cancel and backup restore once finished it will quit and return func (kp *kopiaProvider) CheckContext(ctx context.Context, finishChan chan struct{}, restoreChan chan struct{}, uploader *upload.Uploader) { select { case <-finishChan: kp.log.Infof("Action finished") return case <-ctx.Done(): atomic.StoreInt32(&kp.canceling, 1) if uploader != nil { uploader.Cancel() kp.log.Infof("Backup is been canceled") } if restoreChan != nil { close(restoreChan) kp.log.Infof("Restore is been canceled") } return } } func (kp *kopiaProvider) Close(ctx context.Context) error { return kp.bkRepo.Close(ctx) } // RunBackup which will backup specific path and update backup progress // return snapshotID, isEmptySnapshot, error func (kp *kopiaProvider) RunBackup( ctx context.Context, path string, realSource string, tags map[string]string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, updater uploader.ProgressUpdater) (string, bool, int64, int64, error) { if updater == nil { return "", false, 0, 0, errors.New("Need to initial backup progress updater first") } if path == "" { return "", false, 0, 0, errors.New("path is empty") } log := kp.log.WithFields(logrus.Fields{ "path": path, "realSource": realSource, "parentSnapshot": parentSnapshot, }) repoWriter := kopia.NewShimRepo(kp.bkRepo) kpUploader := upload.NewUploader(repoWriter) progress := kopia.NewProgress(updater, backupProgressCheckInterval, log) kpUploader.Progress = progress kpUploader.FailFast = true quit := make(chan struct{}) log.Info("Starting backup") go kp.CheckContext(ctx, quit, nil, kpUploader) defer func() { close(quit) }() if tags == nil { tags = make(map[string]string) } tags[uploader.SnapshotRequesterTag] = kp.requestorType tags[uploader.SnapshotUploaderTag] = uploader.KopiaType if realSource != "" { realSource = fmt.Sprintf("%s/%s/%s", kp.requestorType, uploader.KopiaType, realSource) } if kp.bkRepo.GetAdvancedFeatures().MultiPartBackup { if uploaderCfg == nil { uploaderCfg = make(map[string]string) } uploaderCfg[kopia.UploaderConfigMultipartKey] = "true" } snapshotInfo, _, err := BackupFunc(ctx, kpUploader, repoWriter, path, realSource, forceFull, parentSnapshot, volMode, uploaderCfg, tags, log) if err != nil { snapshotID := "" if snapshotInfo != nil { snapshotID = snapshotInfo.ID } else { log.Infof("Kopia backup failed with %v and get empty snapshot ID", err) } if kpUploader.IsCanceled() { log.Warn("Kopia backup is canceled") return snapshotID, false, 0, 0, ErrorCanceled } return snapshotID, false, 0, 0, errors.Wrapf(err, "Failed to run kopia backup") } // which ensure that the statistic data of TotalBytes equal to BytesDone when finished updater.UpdateProgress( &uploader.Progress{ TotalBytes: snapshotInfo.Size, BytesDone: snapshotInfo.Size, }, ) log.Debugf("Kopia backup finished, snapshot ID %s, backup size %d", snapshotInfo.ID, snapshotInfo.Size) return snapshotInfo.ID, false, snapshotInfo.Size, progress.GetIncrementalSize(), nil } func (kp *kopiaProvider) GetPassword(param any) (string, error) { if kp.credGetter.FromSecret == nil { return "", errors.New("invalid credentials interface") } rawPass, err := kp.credGetter.FromSecret.Get(repokeys.RepoKeySelector()) if err != nil { return "", errors.Wrap(err, "error to get password") } return strings.TrimSpace(rawPass), nil } // RunRestore which will restore specific path and update restore progress func (kp *kopiaProvider) RunRestore( ctx context.Context, snapshotID string, volumePath string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, updater uploader.ProgressUpdater) (int64, error) { log := kp.log.WithFields(logrus.Fields{ "snapshotID": snapshotID, "volumePath": volumePath, }) repoWriter := kopia.NewShimRepo(kp.bkRepo) progress := kopia.NewProgress(updater, restoreProgressCheckInterval, log) restoreCancel := make(chan struct{}) quit := make(chan struct{}) log.Info("Starting restore") defer func() { close(quit) }() go kp.CheckContext(ctx, quit, restoreCancel, nil) // We use the cancel channel to control the restore cancel, so don't pass a context with cancel to Kopia restore. // Otherwise, Kopia restore will not response to the cancel control but return an arbitrary error. // Kopia restore cancel is not designed as well as Kopia backup which uses the context to control backup cancel all the way. size, fileCount, err := RestoreFunc(context.Background(), repoWriter, progress, snapshotID, volumePath, volMode, uploaderCfg, log, restoreCancel) if err != nil { return 0, errors.Wrapf(err, "Failed to run kopia restore") } if atomic.LoadInt32(&kp.canceling) == 1 { log.Error("Kopia restore is canceled") return 0, ErrorCanceled } // which ensure that the statistic data of TotalBytes equal to BytesDone when finished updater.UpdateProgress(&uploader.Progress{ TotalBytes: size, BytesDone: size, }) output := fmt.Sprintf("Kopia restore finished, restore size %d, file count %d", size, fileCount) log.Info(output) return size, nil } ================================================ FILE: pkg/uploader/provider/kopia_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package provider import ( "context" "sync" "testing" "time" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/snapshot/upload" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "github.com/vmware-tanzu/velero/internal/credentials" "github.com/vmware-tanzu/velero/internal/credentials/mocks" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/repository" udmrepo "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" udmrepomocks "github.com/vmware-tanzu/velero/pkg/repository/udmrepo/mocks" "github.com/vmware-tanzu/velero/pkg/uploader" "github.com/vmware-tanzu/velero/pkg/uploader/kopia" "github.com/vmware-tanzu/velero/pkg/util" ) type FakeBackupProgressUpdater struct { PodVolumeBackup *velerov1api.PodVolumeBackup Log logrus.FieldLogger Ctx context.Context Cli client.Client } func (f *FakeBackupProgressUpdater) UpdateProgress(p *uploader.Progress) {} type FakeRestoreProgressUpdater struct { PodVolumeRestore *velerov1api.PodVolumeRestore Log logrus.FieldLogger Ctx context.Context Cli client.Client } func (f *FakeRestoreProgressUpdater) UpdateProgress(p *uploader.Progress) {} func TestRunBackup(t *testing.T) { testCases := []struct { name string hookBackupFunc func(ctx context.Context, fsUploader kopia.SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, tags map[string]string, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error) volMode uploader.PersistentVolumeMode notError bool }{ { name: "success to backup", hookBackupFunc: func(ctx context.Context, fsUploader kopia.SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, tags map[string]string, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error) { return &uploader.SnapshotInfo{}, false, nil }, notError: true, }, { name: "get error to backup", hookBackupFunc: func(ctx context.Context, fsUploader kopia.SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, tags map[string]string, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error) { return &uploader.SnapshotInfo{}, false, errors.New("failed to backup") }, notError: false, }, { name: "success to backup block mode volume", hookBackupFunc: func(ctx context.Context, fsUploader kopia.SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, tags map[string]string, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error) { return &uploader.SnapshotInfo{}, false, nil }, volMode: uploader.PersistentVolumeBlock, notError: true, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { mockBRepo := udmrepomocks.NewBackupRepo(t) mockBRepo.On("GetAdvancedFeatures").Return(udmrepo.AdvancedFeatureInfo{}) var kp kopiaProvider kp.log = logrus.New() kp.bkRepo = mockBRepo updater := FakeBackupProgressUpdater{PodVolumeBackup: &velerov1api.PodVolumeBackup{}, Log: kp.log, Ctx: t.Context(), Cli: fake.NewClientBuilder().WithScheme(util.VeleroScheme).Build()} if tc.volMode == "" { tc.volMode = uploader.PersistentVolumeFilesystem } BackupFunc = tc.hookBackupFunc _, _, _, _, err := kp.RunBackup(t.Context(), "var", "", nil, false, "", tc.volMode, map[string]string{}, &updater) if tc.notError { assert.NoError(t, err) } else { assert.Error(t, err) } }) } } func TestRunRestore(t *testing.T) { testCases := []struct { name string hookRestoreFunc func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) notError bool volMode uploader.PersistentVolumeMode }{ { name: "normal restore", hookRestoreFunc: func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) { return 0, 0, nil }, notError: true, }, { name: "normal block mode restore", hookRestoreFunc: func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) { return 0, 0, nil }, volMode: uploader.PersistentVolumeBlock, notError: true, }, { name: "failed to restore", hookRestoreFunc: func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) { return 0, 0, errors.New("failed to restore") }, notError: false, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { var kp kopiaProvider kp.log = logrus.New() updater := FakeRestoreProgressUpdater{PodVolumeRestore: &velerov1api.PodVolumeRestore{}, Log: kp.log, Ctx: t.Context(), Cli: fake.NewClientBuilder().WithScheme(util.VeleroScheme).Build()} if tc.volMode == "" { tc.volMode = uploader.PersistentVolumeFilesystem } RestoreFunc = tc.hookRestoreFunc _, err := kp.RunRestore(t.Context(), "", "/var", tc.volMode, map[string]string{}, &updater) if tc.notError { assert.NoError(t, err) } else { assert.Error(t, err) } }) } } func TestCheckContext(t *testing.T) { testCases := []struct { name string finishChan chan struct{} restoreChan chan struct{} uploader *upload.Uploader expectCancel bool expectBackup bool expectRestore bool }{ { name: "FinishChan", finishChan: make(chan struct{}), restoreChan: make(chan struct{}), uploader: &upload.Uploader{}, expectCancel: false, expectBackup: false, expectRestore: false, }, { name: "nil uploader", finishChan: make(chan struct{}), restoreChan: make(chan struct{}), uploader: nil, expectCancel: true, expectBackup: false, expectRestore: true, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { var wg sync.WaitGroup wg.Add(1) if tc.expectBackup { go func() { wg.Wait() tc.restoreChan <- struct{}{} }() } ctx, cancel := context.WithCancel(t.Context()) defer cancel() go func() { time.Sleep(100 * time.Millisecond) cancel() wg.Done() }() kp := &kopiaProvider{log: logrus.New()} kp.CheckContext(ctx, tc.finishChan, tc.restoreChan, tc.uploader) if tc.expectCancel && tc.uploader != nil { t.Error("Expected the uploader to be canceled") } if tc.expectBackup && tc.uploader == nil && len(tc.restoreChan) > 0 { t.Error("Expected the restore channel to be closed") } }) } } func TestGetPassword(t *testing.T) { testCases := []struct { name string empytSecret bool credGetterFunc func(*mocks.SecretStore, *corev1api.SecretKeySelector) expectError bool expectedPass string }{ { name: "valid credentials interface", credGetterFunc: func(ss *mocks.SecretStore, selector *corev1api.SecretKeySelector) { ss.On("Get", selector).Return("test", nil) }, expectError: false, expectedPass: "test", }, { name: "empty from secret", empytSecret: true, expectError: true, expectedPass: "", }, { name: "ErrorGettingPassword", credGetterFunc: func(ss *mocks.SecretStore, selector *corev1api.SecretKeySelector) { ss.On("Get", selector).Return("", errors.New("error getting password")) }, expectError: true, expectedPass: "", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { // Mock CredentialGetter credGetter := &credentials.CredentialGetter{} mockCredGetter := &mocks.SecretStore{} if !tc.empytSecret { credGetter.FromSecret = mockCredGetter } repoKeySelector := &corev1api.SecretKeySelector{LocalObjectReference: corev1api.LocalObjectReference{Name: "velero-repo-credentials"}, Key: "repository-password"} if tc.credGetterFunc != nil { tc.credGetterFunc(mockCredGetter, repoKeySelector) } kp := &kopiaProvider{ credGetter: credGetter, } password, err := kp.GetPassword(nil) if tc.expectError { require.Error(t, err, "Expected an error") } else { require.NoError(t, err, "Expected no error") } assert.Equal(t, tc.expectedPass, password, "Expected password to match") }) } } func (m *MockCredentialGetter) GetCredentials() (string, error) { args := m.Called() return args.String(0), args.Error(1) } // MockRepoSvc is a mock implementation of the RepoService interface. type MockRepoSvc struct { mock.Mock } func (m *MockRepoSvc) Open(ctx context.Context, opts udmrepo.RepoOptions) (udmrepo.BackupRepo, error) { args := m.Called(ctx, opts) return args.Get(0).(udmrepo.BackupRepo), args.Error(1) } func TestNewKopiaUploaderProvider(t *testing.T) { requestorType := "testRequestor" ctx := t.Context() backupRepo := repository.NewBackupRepository(velerov1api.DefaultNamespace, repository.BackupRepositoryKey{VolumeNamespace: "fake-volume-ns-02", BackupLocation: "fake-bsl-02", RepositoryType: "fake-repository-type-02"}) mockLog := logrus.New() // Define test cases testCases := []struct { name string mockCredGetter *mocks.SecretStore mockBackupRepoService udmrepo.BackupRepoService expectedError string }{ { name: "Success", mockCredGetter: func() *mocks.SecretStore { mockCredGetter := &mocks.SecretStore{} mockCredGetter.On("Get", mock.Anything).Return("test", nil) return mockCredGetter }(), mockBackupRepoService: func() udmrepo.BackupRepoService { backupRepoService := &udmrepomocks.BackupRepoService{} var backupRepo udmrepo.BackupRepo backupRepoService.On("Open", t.Context(), mock.Anything).Return(backupRepo, nil) return backupRepoService }(), expectedError: "", }, { name: "Error to get repo options", mockCredGetter: func() *mocks.SecretStore { mockCredGetter := &mocks.SecretStore{} mockCredGetter.On("Get", mock.Anything).Return("test", errors.New("failed to get password")) return mockCredGetter }(), mockBackupRepoService: func() udmrepo.BackupRepoService { backupRepoService := &udmrepomocks.BackupRepoService{} var backupRepo udmrepo.BackupRepo backupRepoService.On("Open", t.Context(), mock.Anything).Return(backupRepo, nil) return backupRepoService }(), expectedError: "error to get repo options", }, { name: "Error open repository service", mockCredGetter: func() *mocks.SecretStore { mockCredGetter := &mocks.SecretStore{} mockCredGetter.On("Get", mock.Anything).Return("test", nil) return mockCredGetter }(), mockBackupRepoService: func() udmrepo.BackupRepoService { backupRepoService := &udmrepomocks.BackupRepoService{} var backupRepo udmrepo.BackupRepo backupRepoService.On("Open", t.Context(), mock.Anything).Return(backupRepo, errors.New("failed to init repository")) return backupRepoService }(), expectedError: "Failed to find kopia repository", }, } // Iterate through test cases for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { credGetter := &credentials.CredentialGetter{FromSecret: tc.mockCredGetter} BackupRepoServiceCreateFunc = func(string, logrus.FieldLogger) udmrepo.BackupRepoService { return tc.mockBackupRepoService } // Call the function being tested. _, err := NewKopiaUploaderProvider(requestorType, ctx, credGetter, backupRepo, mockLog) // Assertions if tc.expectedError != "" { require.ErrorContains(t, err, tc.expectedError) } else { require.NoError(t, err) } // Verify that the expected methods were called on the mocks. tc.mockCredGetter.AssertExpectations(t) }) } } ================================================ FILE: pkg/uploader/provider/mocks/Provider.go ================================================ // Code generated by mockery v2.53.5. DO NOT EDIT. package mocks import ( context "context" mock "github.com/stretchr/testify/mock" uploader "github.com/vmware-tanzu/velero/pkg/uploader" ) // Provider is an autogenerated mock type for the Provider type type Provider struct { mock.Mock } // Close provides a mock function with given fields: ctx func (_m *Provider) Close(ctx context.Context) error { ret := _m.Called(ctx) if len(ret) == 0 { panic("no return value specified for Close") } var r0 error if rf, ok := ret.Get(0).(func(context.Context) error); ok { r0 = rf(ctx) } else { r0 = ret.Error(0) } return r0 } // RunBackup provides a mock function with given fields: ctx, path, realSource, tags, forceFull, parentSnapshot, volMode, uploaderCfg, updater func (_m *Provider) RunBackup(ctx context.Context, path string, realSource string, tags map[string]string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, updater uploader.ProgressUpdater) (string, bool, int64, int64, error) { ret := _m.Called(ctx, path, realSource, tags, forceFull, parentSnapshot, volMode, uploaderCfg, updater) if len(ret) == 0 { panic("no return value specified for RunBackup") } var r0 string var r1 bool var r2 int64 var r3 int64 var r4 error if rf, ok := ret.Get(0).(func(context.Context, string, string, map[string]string, bool, string, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) (string, bool, int64, int64, error)); ok { return rf(ctx, path, realSource, tags, forceFull, parentSnapshot, volMode, uploaderCfg, updater) } if rf, ok := ret.Get(0).(func(context.Context, string, string, map[string]string, bool, string, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) string); ok { r0 = rf(ctx, path, realSource, tags, forceFull, parentSnapshot, volMode, uploaderCfg, updater) } else { r0 = ret.Get(0).(string) } if rf, ok := ret.Get(1).(func(context.Context, string, string, map[string]string, bool, string, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) bool); ok { r1 = rf(ctx, path, realSource, tags, forceFull, parentSnapshot, volMode, uploaderCfg, updater) } else { r1 = ret.Get(1).(bool) } if rf, ok := ret.Get(2).(func(context.Context, string, string, map[string]string, bool, string, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) int64); ok { r2 = rf(ctx, path, realSource, tags, forceFull, parentSnapshot, volMode, uploaderCfg, updater) } else { r2 = ret.Get(2).(int64) } if rf, ok := ret.Get(3).(func(context.Context, string, string, map[string]string, bool, string, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) int64); ok { r3 = rf(ctx, path, realSource, tags, forceFull, parentSnapshot, volMode, uploaderCfg, updater) } else { r3 = ret.Get(3).(int64) } if rf, ok := ret.Get(4).(func(context.Context, string, string, map[string]string, bool, string, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) error); ok { r4 = rf(ctx, path, realSource, tags, forceFull, parentSnapshot, volMode, uploaderCfg, updater) } else { r4 = ret.Error(4) } return r0, r1, r2, r3, r4 } // RunRestore provides a mock function with given fields: ctx, snapshotID, volumePath, volMode, uploaderConfig, updater func (_m *Provider) RunRestore(ctx context.Context, snapshotID string, volumePath string, volMode uploader.PersistentVolumeMode, uploaderConfig map[string]string, updater uploader.ProgressUpdater) (int64, error) { ret := _m.Called(ctx, snapshotID, volumePath, volMode, uploaderConfig, updater) if len(ret) == 0 { panic("no return value specified for RunRestore") } var r0 int64 var r1 error if rf, ok := ret.Get(0).(func(context.Context, string, string, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) (int64, error)); ok { return rf(ctx, snapshotID, volumePath, volMode, uploaderConfig, updater) } if rf, ok := ret.Get(0).(func(context.Context, string, string, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) int64); ok { r0 = rf(ctx, snapshotID, volumePath, volMode, uploaderConfig, updater) } else { r0 = ret.Get(0).(int64) } if rf, ok := ret.Get(1).(func(context.Context, string, string, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) error); ok { r1 = rf(ctx, snapshotID, volumePath, volMode, uploaderConfig, updater) } else { r1 = ret.Error(1) } return r0, r1 } // NewProvider creates a new instance of Provider. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. func NewProvider(t interface { mock.TestingT Cleanup(func()) }) *Provider { mock := &Provider{} mock.Mock.Test(t) t.Cleanup(func() { mock.AssertExpectations(t) }) return mock } ================================================ FILE: pkg/uploader/provider/provider.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package provider import ( "context" "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/internal/credentials" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/uploader" ) const restoreProgressCheckInterval = 10 * time.Second const backupProgressCheckInterval = 10 * time.Second var ErrorCanceled error = errors.New("uploader is canceled") // Provider which is designed for one pod volume to do the backup or restore type Provider interface { // RunBackup which will do backup for one specific volume and return snapshotID, isSnapshotEmpty, error // updater is used for updating backup progress which implement by third-party RunBackup( ctx context.Context, path string, realSource string, tags map[string]string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, updater uploader.ProgressUpdater) (string, bool, int64, int64, error) // RunRestore which will do restore for one specific volume with given snapshot id and return error // updater is used for updating backup progress which implement by third-party RunRestore( ctx context.Context, snapshotID string, volumePath string, volMode uploader.PersistentVolumeMode, uploaderConfig map[string]string, updater uploader.ProgressUpdater) (int64, error) // Close which will close related repository Close(ctx context.Context) error } // NewUploaderProvider initialize provider with specific uploaderType func NewUploaderProvider( ctx context.Context, client client.Client, uploaderType string, requesterType string, repoIdentifier string, bsl *velerov1api.BackupStorageLocation, backupRepo *velerov1api.BackupRepository, credGetter *credentials.CredentialGetter, repoKeySelector *corev1api.SecretKeySelector, log logrus.FieldLogger, ) (Provider, error) { if requesterType == "" { return nil, errors.New("requester type is empty") } if credGetter.FromFile == nil { return nil, errors.New("uninitialized FileStore credential is not supported") } if uploaderType == uploader.KopiaType { return NewKopiaUploaderProvider(requesterType, ctx, credGetter, backupRepo, log) } else { return NewResticUploaderProvider(repoIdentifier, bsl, credGetter, repoKeySelector, log) } } ================================================ FILE: pkg/uploader/provider/provider_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package provider import ( "testing" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client/fake" "github.com/vmware-tanzu/velero/internal/credentials" "github.com/vmware-tanzu/velero/internal/credentials/mocks" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/util" ) type NewUploaderProviderTestCase struct { Description string UploaderType string RequestorType string ExpectedError string needFromFile bool } func TestNewUploaderProvider(t *testing.T) { // Mock objects or dependencies ctx := t.Context() client := fake.NewClientBuilder().WithScheme(util.VeleroScheme).Build() repoIdentifier := "repoIdentifier" bsl := &velerov1api.BackupStorageLocation{} backupRepo := &velerov1api.BackupRepository{} credGetter := &credentials.CredentialGetter{} repoKeySelector := &corev1api.SecretKeySelector{} log := logrus.New() testCases := []NewUploaderProviderTestCase{ { Description: "When requestorType is empty, it should return an error", UploaderType: "kopia", RequestorType: "", ExpectedError: "requester type is empty", }, { Description: "When FileStore credential is uninitialized, it should return an error", UploaderType: "kopia", RequestorType: "requester", ExpectedError: "uninitialized FileStore credential", }, { Description: "When uploaderType is kopia, it should return a KopiaUploaderProvider", UploaderType: "kopia", RequestorType: "requester", needFromFile: true, ExpectedError: "invalid credentials interface", }, { Description: "When uploaderType is not kopia, it should return a ResticUploaderProvider", UploaderType: "restic", RequestorType: "requester", needFromFile: true, ExpectedError: "", }, } for _, testCase := range testCases { t.Run(testCase.Description, func(t *testing.T) { if testCase.needFromFile { mockFileGetter := &mocks.FileStore{} mockFileGetter.On("Path", &corev1api.SecretKeySelector{}).Return("", nil) credGetter.FromFile = mockFileGetter } _, err := NewUploaderProvider(ctx, client, testCase.UploaderType, testCase.RequestorType, repoIdentifier, bsl, backupRepo, credGetter, repoKeySelector, log) if testCase.ExpectedError == "" { assert.NoError(t, err) } else { require.ErrorContains(t, err, testCase.ExpectedError) } }) } } ================================================ FILE: pkg/uploader/provider/restic.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package provider import ( "context" "fmt" "os" "strings" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "github.com/vmware-tanzu/velero/internal/credentials" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/restic" "github.com/vmware-tanzu/velero/pkg/uploader" uploaderutil "github.com/vmware-tanzu/velero/pkg/uploader/util" "github.com/vmware-tanzu/velero/pkg/util/filesystem" ) // resticBackupCMDFunc and resticRestoreCMDFunc are mainly used to make testing more convenient var resticBackupCMDFunc = restic.BackupCommand var resticBackupFunc = restic.RunBackup var resticGetSnapshotFunc = restic.GetSnapshotCommand var resticGetSnapshotIDFunc = restic.GetSnapshotID var resticRestoreCMDFunc = restic.RestoreCommand var resticTempCACertFileFunc = restic.TempCACertFile var resticCmdEnvFunc = restic.CmdEnv type resticProvider struct { repoIdentifier string credentialsFile string caCertFile string cmdEnv []string extraFlags []string bsl *velerov1api.BackupStorageLocation log logrus.FieldLogger } func NewResticUploaderProvider( repoIdentifier string, bsl *velerov1api.BackupStorageLocation, credGetter *credentials.CredentialGetter, repoKeySelector *corev1api.SecretKeySelector, log logrus.FieldLogger, ) (Provider, error) { provider := resticProvider{ repoIdentifier: repoIdentifier, bsl: bsl, log: log, } var err error provider.credentialsFile, err = credGetter.FromFile.Path(repoKeySelector) if err != nil { return nil, errors.Wrap(err, "error creating temp restic credentials file") } // if there's a caCert on the ObjectStorage, write it to disk so that it can be passed to restic if bsl.Spec.ObjectStorage != nil { var caCertData []byte // Try CACertRef first (new method), then fall back to CACert (deprecated) if bsl.Spec.ObjectStorage.CACertRef != nil { caCertString, err := credGetter.FromSecret.Get(bsl.Spec.ObjectStorage.CACertRef) if err != nil { return nil, errors.Wrap(err, "error getting CA certificate from secret") } caCertData = []byte(caCertString) } else if bsl.Spec.ObjectStorage.CACert != nil { caCertData = bsl.Spec.ObjectStorage.CACert } if caCertData != nil { provider.caCertFile, err = resticTempCACertFileFunc(caCertData, bsl.Name, filesystem.NewFileSystem()) if err != nil { return nil, errors.Wrap(err, "error create temp cert file") } } } provider.cmdEnv, err = resticCmdEnvFunc(bsl, credGetter.FromFile) if err != nil { return nil, errors.Wrap(err, "error generating repository cmnd env") } // #4820: restrieve insecureSkipTLSVerify from BSL configuration for // AWS plugin. If nothing is return, that means insecureSkipTLSVerify // is not enable for Restic command. skipTLSRet := restic.GetInsecureSkipTLSVerifyFromBSL(bsl, log) if len(skipTLSRet) > 0 { provider.extraFlags = append(provider.extraFlags, skipTLSRet) } return &provider, nil } func (rp *resticProvider) Close(ctx context.Context) error { _, err := os.Stat(rp.credentialsFile) if err == nil { return os.Remove(rp.credentialsFile) } else if !os.IsNotExist(err) { return errors.Errorf("failed to get file %s info with error %v", rp.credentialsFile, err) } _, err = os.Stat(rp.caCertFile) if err == nil { return os.Remove(rp.caCertFile) } else if !os.IsNotExist(err) { return errors.Errorf("failed to get file %s info with error %v", rp.caCertFile, err) } return nil } // RunBackup runs a `backup` command and watches the output to provide // progress updates to the caller and return snapshotID, isEmptySnapshot, error func (rp *resticProvider) RunBackup( ctx context.Context, path string, realSource string, tags map[string]string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, updater uploader.ProgressUpdater) (string, bool, int64, int64, error) { if updater == nil { return "", false, 0, 0, errors.New("Need to initial backup progress updater first") } if path == "" { return "", false, 0, 0, errors.New("path is empty") } if realSource != "" { return "", false, 0, 0, errors.New("real source is not empty, this is not supported by restic uploader") } if volMode == uploader.PersistentVolumeBlock { return "", false, 0, 0, errors.New("unable to support block mode") } log := rp.log.WithFields(logrus.Fields{ "path": path, "parentSnapshot": parentSnapshot, }) if len(uploaderCfg) > 0 { parallelFilesUpload, err := uploaderutil.GetParallelFilesUpload(uploaderCfg) if err != nil { return "", false, 0, 0, errors.Wrap(err, "failed to get uploader config") } if parallelFilesUpload > 0 { log.Warnf("ParallelFilesUpload is set to %d, but restic does not support parallel file uploads. Ignoring.", parallelFilesUpload) } } backupCmd := resticBackupCMDFunc(rp.repoIdentifier, rp.credentialsFile, path, tags) backupCmd.Env = rp.cmdEnv backupCmd.CACertFile = rp.caCertFile if len(rp.extraFlags) != 0 { backupCmd.ExtraFlags = append(backupCmd.ExtraFlags, rp.extraFlags...) } if parentSnapshot != "" { backupCmd.ExtraFlags = append(backupCmd.ExtraFlags, fmt.Sprintf("--parent=%s", parentSnapshot)) } summary, stderrBuf, err := resticBackupFunc(backupCmd, log, updater) if err != nil { if strings.Contains(stderrBuf, "snapshot is empty") { log.Debugf("Restic backup got empty dir with %s path", path) return "", true, 0, 0, nil } return "", false, 0, 0, errors.WithStack(fmt.Errorf("error running restic backup command %s with error: %v stderr: %v", backupCmd.String(), err, stderrBuf)) } // GetSnapshotID snapshotIDCmd := resticGetSnapshotFunc(rp.repoIdentifier, rp.credentialsFile, tags) snapshotIDCmd.Env = rp.cmdEnv snapshotIDCmd.CACertFile = rp.caCertFile if len(rp.extraFlags) != 0 { snapshotIDCmd.ExtraFlags = append(snapshotIDCmd.ExtraFlags, rp.extraFlags...) } snapshotID, err := resticGetSnapshotIDFunc(snapshotIDCmd) if err != nil { return "", false, 0, 0, errors.WithStack(fmt.Errorf("error getting snapshot id with error: %v", err)) } log.Infof("Run command=%s, stdout=%s, stderr=%s", backupCmd.String(), summary, stderrBuf) return snapshotID, false, 0, 0, nil } // RunRestore runs a `restore` command and monitors the volume size to // provide progress updates to the caller. func (rp *resticProvider) RunRestore( ctx context.Context, snapshotID string, volumePath string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, updater uploader.ProgressUpdater) (int64, error) { if updater == nil { return 0, errors.New("Need to initial backup progress updater first") } log := rp.log.WithFields(logrus.Fields{ "snapshotID": snapshotID, "volumePath": volumePath, }) if volMode == uploader.PersistentVolumeBlock { return 0, errors.New("unable to support block mode") } restoreCmd := resticRestoreCMDFunc(rp.repoIdentifier, rp.credentialsFile, snapshotID, volumePath) restoreCmd.Env = rp.cmdEnv restoreCmd.CACertFile = rp.caCertFile if len(rp.extraFlags) != 0 { restoreCmd.ExtraFlags = append(restoreCmd.ExtraFlags, rp.extraFlags...) } extraFlags, err := rp.parseRestoreExtraFlags(uploaderCfg) if err != nil { return 0, errors.Wrap(err, "failed to parse uploader config") } else if len(extraFlags) != 0 { restoreCmd.ExtraFlags = append(restoreCmd.ExtraFlags, extraFlags...) } stdout, stderr, err := restic.RunRestore(restoreCmd, log, updater) log.Infof("Run command=%v, stdout=%s, stderr=%s", restoreCmd, stdout, stderr) return 0, err } func (rp *resticProvider) parseRestoreExtraFlags(uploaderCfg map[string]string) ([]string, error) { extraFlags := []string{} if len(uploaderCfg) == 0 { return extraFlags, nil } writeSparseFiles, err := uploaderutil.GetWriteSparseFiles(uploaderCfg) if err != nil { return extraFlags, errors.Wrap(err, "failed to get uploader config") } if writeSparseFiles { extraFlags = append(extraFlags, "--sparse") } if restoreConcurrency, err := uploaderutil.GetRestoreConcurrency(uploaderCfg); err == nil && restoreConcurrency > 0 { return extraFlags, errors.New("restic does not support parallel restore") } return extraFlags, nil } ================================================ FILE: pkg/uploader/provider/restic_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package provider import ( "errors" "os" "reflect" "strings" "testing" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client/fake" "github.com/vmware-tanzu/velero/internal/credentials" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/restic" "github.com/vmware-tanzu/velero/pkg/uploader" "github.com/vmware-tanzu/velero/pkg/util" "github.com/vmware-tanzu/velero/pkg/util/filesystem" ) func TestResticRunBackup(t *testing.T) { testCases := []struct { name string nilUpdater bool parentSnapshot string rp *resticProvider volMode uploader.PersistentVolumeMode hookBackupFunc func(string, string, string, map[string]string) *restic.Command hookResticBackupFunc func(*restic.Command, logrus.FieldLogger, uploader.ProgressUpdater) (string, string, error) hookResticGetSnapshotFunc func(string, string, map[string]string) *restic.Command hookResticGetSnapshotIDFunc func(*restic.Command) (string, error) errorHandleFunc func(err error) bool }{ { name: "nil uploader", rp: &resticProvider{log: logrus.New()}, nilUpdater: true, hookBackupFunc: func(repoIdentifier string, passwordFile string, path string, tags map[string]string) *restic.Command { return &restic.Command{Command: "date"} }, errorHandleFunc: func(err error) bool { return strings.Contains(err.Error(), "Need to initial backup progress updater first") }, }, { name: "wrong restic execute command", rp: &resticProvider{log: logrus.New()}, hookBackupFunc: func(repoIdentifier string, passwordFile string, path string, tags map[string]string) *restic.Command { return &restic.Command{Command: "date"} }, errorHandleFunc: func(err error) bool { return strings.Contains(err.Error(), "error running") }, }, { name: "has parent snapshot", rp: &resticProvider{log: logrus.New()}, parentSnapshot: "parentSnapshot", hookBackupFunc: func(repoIdentifier string, passwordFile string, path string, tags map[string]string) *restic.Command { return &restic.Command{Command: "date"} }, hookResticBackupFunc: func(*restic.Command, logrus.FieldLogger, uploader.ProgressUpdater) (string, string, error) { return "", "", nil }, hookResticGetSnapshotIDFunc: func(*restic.Command) (string, error) { return "test-snapshot-id", nil }, errorHandleFunc: func(err error) bool { return err == nil }, }, { name: "has extra flags", rp: &resticProvider{log: logrus.New(), extraFlags: []string{"testFlags"}}, hookBackupFunc: func(string, string, string, map[string]string) *restic.Command { return &restic.Command{Command: "date"} }, hookResticBackupFunc: func(*restic.Command, logrus.FieldLogger, uploader.ProgressUpdater) (string, string, error) { return "", "", nil }, hookResticGetSnapshotIDFunc: func(*restic.Command) (string, error) { return "test-snapshot-id", nil }, errorHandleFunc: func(err error) bool { return err == nil }, }, { name: "failed to get snapshot id", rp: &resticProvider{log: logrus.New(), extraFlags: []string{"testFlags"}}, hookBackupFunc: func(string, string, string, map[string]string) *restic.Command { return &restic.Command{Command: "date"} }, hookResticBackupFunc: func(*restic.Command, logrus.FieldLogger, uploader.ProgressUpdater) (string, string, error) { return "", "", nil }, hookResticGetSnapshotIDFunc: func(*restic.Command) (string, error) { return "test-snapshot-id", errors.New("failed to get snapshot id") }, errorHandleFunc: func(err error) bool { return strings.Contains(err.Error(), "failed to get snapshot id") }, }, { name: "failed to use block mode", rp: &resticProvider{log: logrus.New(), extraFlags: []string{"testFlags"}}, volMode: uploader.PersistentVolumeBlock, errorHandleFunc: func(err error) bool { return strings.Contains(err.Error(), "unable to support block mode") }, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { var err error parentSnapshot := tc.parentSnapshot if tc.hookBackupFunc != nil { resticBackupCMDFunc = tc.hookBackupFunc } if tc.hookResticBackupFunc != nil { resticBackupFunc = tc.hookResticBackupFunc } if tc.hookResticGetSnapshotFunc != nil { resticGetSnapshotFunc = tc.hookResticGetSnapshotFunc } if tc.hookResticGetSnapshotIDFunc != nil { resticGetSnapshotIDFunc = tc.hookResticGetSnapshotIDFunc } if tc.volMode == "" { tc.volMode = uploader.PersistentVolumeFilesystem } if !tc.nilUpdater { updater := FakeBackupProgressUpdater{PodVolumeBackup: &velerov1api.PodVolumeBackup{}, Log: tc.rp.log, Ctx: t.Context(), Cli: fake.NewClientBuilder().WithScheme(util.VeleroScheme).Build()} _, _, _, _, err = tc.rp.RunBackup(t.Context(), "var", "", map[string]string{}, false, parentSnapshot, tc.volMode, map[string]string{}, &updater) } else { _, _, _, _, err = tc.rp.RunBackup(t.Context(), "var", "", map[string]string{}, false, parentSnapshot, tc.volMode, map[string]string{}, nil) } tc.rp.log.Infof("test name %v error %v", tc.name, err) require.True(t, tc.errorHandleFunc(err)) }) } } func TestResticRunRestore(t *testing.T) { resticRestoreCMDFunc = func(repoIdentifier, passwordFile, snapshotID, target string) *restic.Command { return &restic.Command{Args: []string{""}} } testCases := []struct { name string rp *resticProvider nilUpdater bool hookResticRestoreFunc func(repoIdentifier, passwordFile, snapshotID, target string) *restic.Command errorHandleFunc func(err error) bool volMode uploader.PersistentVolumeMode }{ { name: "wrong restic execute command", rp: &resticProvider{log: logrus.New()}, nilUpdater: true, errorHandleFunc: func(err error) bool { return strings.Contains(err.Error(), "Need to initial backup progress updater first") }, }, { name: "has extral flags", rp: &resticProvider{log: logrus.New(), extraFlags: []string{"test-extra-flags"}}, hookResticRestoreFunc: func(repoIdentifier, passwordFile, snapshotID, target string) *restic.Command { return &restic.Command{Args: []string{"date"}} }, errorHandleFunc: func(err error) bool { return strings.Contains(err.Error(), "error running command") }, }, { name: "wrong restic execute command", rp: &resticProvider{log: logrus.New()}, hookResticRestoreFunc: func(repoIdentifier, passwordFile, snapshotID, target string) *restic.Command { return &restic.Command{Args: []string{"date"}} }, errorHandleFunc: func(err error) bool { return strings.Contains(err.Error(), "error running command") }, }, { name: "error block volume mode", rp: &resticProvider{log: logrus.New()}, errorHandleFunc: func(err error) bool { return strings.Contains(err.Error(), "unable to support block mode") }, volMode: uploader.PersistentVolumeBlock, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { if tc.volMode == "" { tc.volMode = uploader.PersistentVolumeFilesystem } resticRestoreCMDFunc = tc.hookResticRestoreFunc if tc.volMode == "" { tc.volMode = uploader.PersistentVolumeFilesystem } var err error if !tc.nilUpdater { updater := FakeBackupProgressUpdater{PodVolumeBackup: &velerov1api.PodVolumeBackup{}, Log: tc.rp.log, Ctx: t.Context(), Cli: fake.NewClientBuilder().WithScheme(util.VeleroScheme).Build()} _, err = tc.rp.RunRestore(t.Context(), "", "var", tc.volMode, map[string]string{}, &updater) } else { _, err = tc.rp.RunRestore(t.Context(), "", "var", tc.volMode, map[string]string{}, nil) } tc.rp.log.Infof("test name %v error %v", tc.name, err) require.True(t, tc.errorHandleFunc(err)) }) } } func TestClose(t *testing.T) { t.Run("Delete existing credentials file", func(t *testing.T) { // Create temporary files for the credentials and caCert credentialsFile, err := os.CreateTemp(t.TempDir(), "credentialsFile") if err != nil { t.Fatalf("failed to create temp file: %v", err) } defer os.Remove(credentialsFile.Name()) caCertFile, err := os.CreateTemp(t.TempDir(), "caCertFile") if err != nil { t.Fatalf("failed to create temp file: %v", err) } defer os.Remove(caCertFile.Name()) rp := &resticProvider{ credentialsFile: credentialsFile.Name(), caCertFile: caCertFile.Name(), } // Test deleting an existing credentials file err = rp.Close(t.Context()) if err != nil { t.Errorf("unexpected error: %v", err) } _, err = os.Stat(rp.credentialsFile) if !os.IsNotExist(err) { t.Errorf("expected credentials file to be deleted, got error: %v", err) } }) t.Run("Delete existing caCert file", func(t *testing.T) { // Create temporary files for the credentials and caCert caCertFile, err := os.CreateTemp(t.TempDir(), "caCertFile") if err != nil { t.Fatalf("failed to create temp file: %v", err) } defer os.Remove(caCertFile.Name()) rp := &resticProvider{ credentialsFile: "", caCertFile: "", } err = rp.Close(t.Context()) // Test deleting an existing caCert file if err != nil { t.Errorf("unexpected error: %v", err) } _, err = os.Stat(rp.caCertFile) if !os.IsNotExist(err) { t.Errorf("expected caCert file to be deleted, got error: %v", err) } }) } type MockCredentialGetter struct { mock.Mock } func (m *MockCredentialGetter) Path(selector *corev1api.SecretKeySelector) (string, error) { args := m.Called(selector) return args.Get(0).(string), args.Error(1) } func TestNewResticUploaderProvider(t *testing.T) { testCases := []struct { name string emptyBSL bool mockCredFunc func(*MockCredentialGetter, *corev1api.SecretKeySelector) resticCmdEnvFunc func(backupLocation *velerov1api.BackupStorageLocation, credentialFileStore credentials.FileStore) ([]string, error) resticTempCACertFileFunc func(caCert []byte, bsl string, fs filesystem.Interface) (string, error) checkFunc func(t *testing.T, provider Provider, err error) }{ { name: "No error in creating temp credentials file", mockCredFunc: func(credGetter *MockCredentialGetter, repoKeySelector *corev1api.SecretKeySelector) { credGetter.On("Path", repoKeySelector).Return("temp-credentials", nil) }, checkFunc: func(t *testing.T, provider Provider, err error) { t.Helper() require.NoError(t, err) assert.NotNil(t, provider) }, }, { name: "Error in creating temp credentials file", mockCredFunc: func(credGetter *MockCredentialGetter, repoKeySelector *corev1api.SecretKeySelector) { credGetter.On("Path", repoKeySelector).Return("", errors.New("error creating temp credentials file")) }, checkFunc: func(t *testing.T, provider Provider, err error) { t.Helper() require.Error(t, err) assert.Nil(t, provider) }, }, { name: "ObjectStorage with CACert present and creating CACert file failed", mockCredFunc: func(credGetter *MockCredentialGetter, repoKeySelector *corev1api.SecretKeySelector) { credGetter.On("Path", repoKeySelector).Return("temp-credentials", nil) }, resticTempCACertFileFunc: func(caCert []byte, bsl string, fs filesystem.Interface) (string, error) { return "", errors.New("error writing CACert file") }, checkFunc: func(t *testing.T, provider Provider, err error) { t.Helper() require.Error(t, err) assert.Nil(t, provider) }, }, { name: "Generating repository cmd failed", mockCredFunc: func(credGetter *MockCredentialGetter, repoKeySelector *corev1api.SecretKeySelector) { credGetter.On("Path", repoKeySelector).Return("temp-credentials", nil) }, resticTempCACertFileFunc: func(caCert []byte, bsl string, fs filesystem.Interface) (string, error) { return "test-ca", nil }, resticCmdEnvFunc: func(backupLocation *velerov1api.BackupStorageLocation, credentialFileStore credentials.FileStore) ([]string, error) { return nil, errors.New("error generating repository cmnd env") }, checkFunc: func(t *testing.T, provider Provider, err error) { t.Helper() require.Error(t, err) assert.Nil(t, provider) }, }, { name: "New provider with not nil bsl", mockCredFunc: func(credGetter *MockCredentialGetter, repoKeySelector *corev1api.SecretKeySelector) { credGetter.On("Path", repoKeySelector).Return("temp-credentials", nil) }, resticTempCACertFileFunc: func(caCert []byte, bsl string, fs filesystem.Interface) (string, error) { return "test-ca", nil }, resticCmdEnvFunc: func(backupLocation *velerov1api.BackupStorageLocation, credentialFileStore credentials.FileStore) ([]string, error) { return nil, nil }, checkFunc: func(t *testing.T, provider Provider, err error) { t.Helper() require.NoError(t, err) assert.NotNil(t, provider) }, }, { name: "New provider with nil bsl", emptyBSL: true, mockCredFunc: func(credGetter *MockCredentialGetter, repoKeySelector *corev1api.SecretKeySelector) { credGetter.On("Path", repoKeySelector).Return("temp-credentials", nil) }, resticTempCACertFileFunc: func(caCert []byte, bsl string, fs filesystem.Interface) (string, error) { return "test-ca", nil }, resticCmdEnvFunc: func(backupLocation *velerov1api.BackupStorageLocation, credentialFileStore credentials.FileStore) ([]string, error) { return nil, nil }, checkFunc: func(t *testing.T, provider Provider, err error) { t.Helper() require.NoError(t, err) assert.NotNil(t, provider) }, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { repoIdentifier := "my-repo" bsl := &velerov1api.BackupStorageLocation{} if !tc.emptyBSL { bsl = builder.ForBackupStorageLocation("test-ns", "test-name").CACert([]byte("my-cert")).Result() } credGetter := &credentials.CredentialGetter{} repoKeySelector := &corev1api.SecretKeySelector{} log := logrus.New() // Mock CredentialGetter mockCredGetter := &MockCredentialGetter{} credGetter.FromFile = mockCredGetter tc.mockCredFunc(mockCredGetter, repoKeySelector) if tc.resticCmdEnvFunc != nil { resticCmdEnvFunc = tc.resticCmdEnvFunc } if tc.resticTempCACertFileFunc != nil { resticTempCACertFileFunc = tc.resticTempCACertFileFunc } provider, err := NewResticUploaderProvider(repoIdentifier, bsl, credGetter, repoKeySelector, log) tc.checkFunc(t, provider, err) }) } } func TestParseUploaderConfig(t *testing.T) { rp := &resticProvider{} testCases := []struct { name string uploaderConfig map[string]string expectedFlags []string }{ { name: "SparseFilesEnabled", uploaderConfig: map[string]string{ "WriteSparseFiles": "true", }, expectedFlags: []string{"--sparse"}, }, { name: "SparseFilesDisabled", uploaderConfig: map[string]string{ "writeSparseFiles": "false", }, expectedFlags: []string{}, }, { name: "RestoreConcorrency", uploaderConfig: map[string]string{ "Parallel": "5", }, expectedFlags: []string{}, }, } for _, testCase := range testCases { t.Run(testCase.name, func(t *testing.T) { result, err := rp.parseRestoreExtraFlags(testCase.uploaderConfig) if err != nil { t.Errorf("Test case %s failed with error: %v", testCase.name, err) return } if !reflect.DeepEqual(result, testCase.expectedFlags) { t.Errorf("Test case %s failed. Expected: %v, Got: %v", testCase.name, testCase.expectedFlags, result) } }) } } ================================================ FILE: pkg/uploader/types.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package uploader import ( "fmt" "strings" ) const ( ResticType = "restic" KopiaType = "kopia" SnapshotRequesterTag = "snapshot-requester" SnapshotUploaderTag = "snapshot-uploader" ) type PersistentVolumeMode string const ( // PersistentVolumeBlock means the volume will not be formatted with a filesystem and will remain a raw block device. PersistentVolumeBlock PersistentVolumeMode = "Block" // PersistentVolumeFilesystem means the volume will be or is formatted with a filesystem. PersistentVolumeFilesystem PersistentVolumeMode = "Filesystem" ) // ValidateUploaderType validates if the input param is a valid uploader type. // It will return an error if it's invalid. func ValidateUploaderType(t string) (string, error) { t = strings.TrimSpace(t) if t != KopiaType { return "", fmt.Errorf("invalid uploader type '%s', valid type: '%s'", t, KopiaType) } return "", nil } type SnapshotInfo struct { ID string `json:"id"` Size int64 `json:"Size"` } // Progress which defined two variables to record progress type Progress struct { TotalBytes int64 `json:"totalBytes,omitempty"` BytesDone int64 `json:"doneBytes,omitempty"` } // UploaderProgress which defined generic interface to update progress type ProgressUpdater interface { UpdateProgress(p *Progress) } ================================================ FILE: pkg/uploader/types_test.go ================================================ package uploader import ( "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestValidateUploaderType(t *testing.T) { tests := []struct { name string input string wantErr string wantMsg string }{ { "' kopia ' is a valid type (space will be trimmed)", " kopia ", "", "", }, { "'anything_else' is invalid", "anything_else", "invalid uploader type 'anything_else', valid type: 'kopia'", "", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { msg, err := ValidateUploaderType(tt.input) if tt.wantErr != "" { require.EqualError(t, err, tt.wantErr) } else { require.NoError(t, err) } assert.Equal(t, tt.wantMsg, msg) }) } } ================================================ FILE: pkg/uploader/util/uploader_config.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package util import ( "strconv" "github.com/pkg/errors" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) const ( ParallelFilesUpload = "ParallelFilesUpload" WriteSparseFiles = "WriteSparseFiles" RestoreConcurrency = "ParallelFilesDownload" ) func StoreBackupConfig(config *velerov1api.UploaderConfigForBackup) map[string]string { data := make(map[string]string) data[ParallelFilesUpload] = strconv.Itoa(config.ParallelFilesUpload) return data } func StoreRestoreConfig(config *velerov1api.UploaderConfigForRestore) map[string]string { data := make(map[string]string) if config.WriteSparseFiles != nil { data[WriteSparseFiles] = strconv.FormatBool(*config.WriteSparseFiles) } else { data[WriteSparseFiles] = strconv.FormatBool(false) } if config.ParallelFilesDownload > 0 { data[RestoreConcurrency] = strconv.Itoa(config.ParallelFilesDownload) } return data } func GetParallelFilesUpload(uploaderCfg map[string]string) (int, error) { parallelFilesUpload, ok := uploaderCfg[ParallelFilesUpload] if ok { parallelFilesUploadInt, err := strconv.Atoi(parallelFilesUpload) if err != nil { return 0, errors.Wrap(err, "failed to parse ParallelFilesUpload config") } return parallelFilesUploadInt, nil } return 0, nil } func GetWriteSparseFiles(uploaderCfg map[string]string) (bool, error) { writeSparseFiles, ok := uploaderCfg[WriteSparseFiles] if ok { writeSparseFilesBool, err := strconv.ParseBool(writeSparseFiles) if err != nil { return false, errors.Wrap(err, "failed to parse WriteSparseFiles config") } return writeSparseFilesBool, nil } return false, nil } func GetRestoreConcurrency(uploaderCfg map[string]string) (int, error) { restoreConcurrency, ok := uploaderCfg[RestoreConcurrency] if ok { restoreConcurrencyInt, err := strconv.Atoi(restoreConcurrency) if err != nil { return 0, errors.Wrap(err, "failed to parse RestoreConcurrency config") } return restoreConcurrencyInt, nil } return 0, nil } ================================================ FILE: pkg/uploader/util/uploader_config_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package util import ( "reflect" "testing" "github.com/pkg/errors" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) func TestStoreBackupConfig(t *testing.T) { config := &velerov1api.UploaderConfigForBackup{ ParallelFilesUpload: 3, } expectedData := map[string]string{ ParallelFilesUpload: "3", } result := StoreBackupConfig(config) if !reflect.DeepEqual(result, expectedData) { t.Errorf("Expected: %v, but got: %v", expectedData, result) } } func TestStoreRestoreConfig(t *testing.T) { var ( boolTrue = true boolFalse = false ) testCases := []struct { name string config *velerov1api.UploaderConfigForRestore expectedData map[string]string }{ { name: "WriteSparseFiles is true", config: &velerov1api.UploaderConfigForRestore{ WriteSparseFiles: &boolTrue, }, expectedData: map[string]string{ WriteSparseFiles: "true", }, }, { name: "WriteSparseFiles is false", config: &velerov1api.UploaderConfigForRestore{ WriteSparseFiles: &boolFalse, }, expectedData: map[string]string{ WriteSparseFiles: "false", }, }, { name: "WriteSparseFiles is nil", config: &velerov1api.UploaderConfigForRestore{ WriteSparseFiles: nil, }, expectedData: map[string]string{ WriteSparseFiles: "false", // Assuming default value is false for nil case }, }, { name: "Parallel is set", config: &velerov1api.UploaderConfigForRestore{ ParallelFilesDownload: 5, }, expectedData: map[string]string{ RestoreConcurrency: "5", WriteSparseFiles: "false", }, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { result := StoreRestoreConfig(tc.config) if !reflect.DeepEqual(result, tc.expectedData) { t.Errorf("Expected: %v, but got: %v", tc.expectedData, result) } }) } } func TestGetParallelFilesUpload(t *testing.T) { tests := []struct { name string uploaderCfg map[string]string expectedResult int expectedError error }{ { name: "Valid ParallelFilesUpload", uploaderCfg: map[string]string{ParallelFilesUpload: "5"}, expectedResult: 5, expectedError: nil, }, { name: "Missing ParallelFilesUpload", uploaderCfg: map[string]string{}, expectedResult: 0, expectedError: nil, }, { name: "Invalid ParallelFilesUpload (not a number)", uploaderCfg: map[string]string{ParallelFilesUpload: "invalid"}, expectedResult: 0, expectedError: errors.Wrap(errors.New("strconv.Atoi: parsing \"invalid\": invalid syntax"), "failed to parse ParallelFilesUpload config"), }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { result, err := GetParallelFilesUpload(test.uploaderCfg) if result != test.expectedResult { t.Errorf("Expected result %d, but got %d", test.expectedResult, result) } if (err == nil && test.expectedError != nil) || (err != nil && test.expectedError == nil) || (err != nil && test.expectedError != nil && err.Error() != test.expectedError.Error()) { t.Errorf("Expected error '%v', but got '%v'", test.expectedError, err) } }) } } func TestGetWriteSparseFiles(t *testing.T) { tests := []struct { name string uploaderCfg map[string]string expectedResult bool expectedError error }{ { name: "Valid WriteSparseFiles (true)", uploaderCfg: map[string]string{WriteSparseFiles: "true"}, expectedResult: true, expectedError: nil, }, { name: "Valid WriteSparseFiles (false)", uploaderCfg: map[string]string{WriteSparseFiles: "false"}, expectedResult: false, expectedError: nil, }, { name: "Invalid WriteSparseFiles (not a boolean)", uploaderCfg: map[string]string{WriteSparseFiles: "invalid"}, expectedResult: false, expectedError: errors.Wrap(errors.New("strconv.ParseBool: parsing \"invalid\": invalid syntax"), "failed to parse WriteSparseFiles config"), }, { name: "Missing WriteSparseFiles", uploaderCfg: map[string]string{}, expectedResult: false, expectedError: nil, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { result, err := GetWriteSparseFiles(test.uploaderCfg) if result != test.expectedResult { t.Errorf("Expected result %t, but got %t", test.expectedResult, result) } if (err == nil && test.expectedError != nil) || (err != nil && test.expectedError == nil) || (err != nil && test.expectedError != nil && err.Error() != test.expectedError.Error()) { t.Errorf("Expected error '%v', but got '%v'", test.expectedError, err) } }) } } func TestGetRestoreConcurrency(t *testing.T) { testCases := []struct { Name string UploaderCfg map[string]string ExpectedResult int ExpectedError bool ExpectedErrorMsg string }{ { Name: "Valid Configuration", UploaderCfg: map[string]string{RestoreConcurrency: "10"}, ExpectedResult: 10, ExpectedError: false, }, { Name: "Missing Configuration", UploaderCfg: map[string]string{}, ExpectedResult: 0, ExpectedError: false, }, { Name: "Invalid Configuration", UploaderCfg: map[string]string{RestoreConcurrency: "not_an_integer"}, ExpectedResult: 0, ExpectedError: true, ExpectedErrorMsg: "failed to parse RestoreConcurrency config: strconv.Atoi: parsing \"not_an_integer\": invalid syntax", }, } for _, tc := range testCases { t.Run(tc.Name, func(t *testing.T) { result, err := GetRestoreConcurrency(tc.UploaderCfg) if tc.ExpectedError { if err.Error() != tc.ExpectedErrorMsg { t.Errorf("Expected error message %s, but got %s", tc.ExpectedErrorMsg, err.Error()) } } else { if err != nil { t.Errorf("Expected no error, but got %v", err) } } if result != tc.ExpectedResult { t.Errorf("Expected result %d, but got %d", tc.ExpectedResult, result) } }) } } ================================================ FILE: pkg/util/actionhelpers/pod_helper.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actionhelpers import ( "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/plugin/velero" ) func RelatedItemsForPod(pod *corev1api.Pod, log logrus.FieldLogger) []velero.ResourceIdentifier { var additionalItems []velero.ResourceIdentifier if pod.Spec.PriorityClassName != "" { log.Infof("Adding priorityclass %s to additionalItems", pod.Spec.PriorityClassName) additionalItems = append(additionalItems, velero.ResourceIdentifier{ GroupResource: kuberesource.PriorityClasses, Name: pod.Spec.PriorityClassName, }) } if len(pod.Spec.Volumes) == 0 { log.Info("pod has no volumes") } for _, volume := range pod.Spec.Volumes { if volume.PersistentVolumeClaim != nil && volume.PersistentVolumeClaim.ClaimName != "" { log.Infof("Adding pvc %s to additionalItems", volume.PersistentVolumeClaim.ClaimName) additionalItems = append(additionalItems, velero.ResourceIdentifier{ GroupResource: kuberesource.PersistentVolumeClaims, Namespace: pod.Namespace, Name: volume.PersistentVolumeClaim.ClaimName, }) } } return additionalItems } ================================================ FILE: pkg/util/actionhelpers/pvc_helper.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actionhelpers import ( "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/plugin/velero" ) func RelatedItemsForPVC(pvc *corev1api.PersistentVolumeClaim, log logrus.FieldLogger) []velero.ResourceIdentifier { return []velero.ResourceIdentifier{ { GroupResource: kuberesource.PersistentVolumes, Name: pvc.Spec.VolumeName, }, } } ================================================ FILE: pkg/util/actionhelpers/rbac.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actionhelpers import ( "context" "github.com/pkg/errors" rbacv1 "k8s.io/api/rbac/v1" rbacbeta "k8s.io/api/rbac/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" rbacclient "k8s.io/client-go/kubernetes/typed/rbac/v1" rbacbetaclient "k8s.io/client-go/kubernetes/typed/rbac/v1beta1" ) // ClusterRoleBindingLister allows for listing ClusterRoleBindings in a version-independent way. type ClusterRoleBindingLister interface { // List returns a slice of ClusterRoleBindings which can represent either v1 or v1beta1 ClusterRoleBindings. List() ([]ClusterRoleBinding, error) } // noopClusterRoleBindingLister exists to handle clusters where RBAC is disabled. type NoopClusterRoleBindingLister struct { } func (noop NoopClusterRoleBindingLister) List() ([]ClusterRoleBinding, error) { return []ClusterRoleBinding{}, nil } type V1ClusterRoleBindingLister struct { client rbacclient.ClusterRoleBindingInterface } func (v1 V1ClusterRoleBindingLister) List() ([]ClusterRoleBinding, error) { crbList, err := v1.client.List(context.TODO(), metav1.ListOptions{}) if err != nil { return nil, errors.WithStack(err) } var crbs []ClusterRoleBinding for _, crb := range crbList.Items { crbs = append(crbs, V1ClusterRoleBinding{Crb: crb}) } return crbs, nil } type V1beta1ClusterRoleBindingLister struct { client rbacbetaclient.ClusterRoleBindingInterface } func (v1beta1 V1beta1ClusterRoleBindingLister) List() ([]ClusterRoleBinding, error) { crbList, err := v1beta1.client.List(context.TODO(), metav1.ListOptions{}) if err != nil { return nil, errors.WithStack(err) } var crbs []ClusterRoleBinding for _, crb := range crbList.Items { crbs = append(crbs, V1beta1ClusterRoleBinding{Crb: crb}) } return crbs, nil } // NewClusterRoleBindingListerMap creates a map of RBAC version strings to their associated // ClusterRoleBindingLister structs. // Necessary so that callers to the ClusterRoleBindingLister interfaces don't need the kubernetes.Interface. func NewClusterRoleBindingListerMap(clientset kubernetes.Interface) map[string]ClusterRoleBindingLister { return map[string]ClusterRoleBindingLister{ rbacv1.SchemeGroupVersion.Version: V1ClusterRoleBindingLister{client: clientset.RbacV1().ClusterRoleBindings()}, rbacbeta.SchemeGroupVersion.Version: V1beta1ClusterRoleBindingLister{client: clientset.RbacV1beta1().ClusterRoleBindings()}, "": NoopClusterRoleBindingLister{}, } } // ClusterRoleBinding abstracts access to ClusterRoleBindings whether they're v1 or v1beta1. type ClusterRoleBinding interface { // Name returns the name of a ClusterRoleBinding. Name() string // ServiceAccountSubjects returns the names of subjects that are service accounts in the given namespace. ServiceAccountSubjects(namespace string) []string // RoleRefName returns the name of a ClusterRoleBinding's RoleRef. RoleRefName() string } type V1ClusterRoleBinding struct { Crb rbacv1.ClusterRoleBinding } func (c V1ClusterRoleBinding) Name() string { return c.Crb.Name } func (c V1ClusterRoleBinding) RoleRefName() string { return c.Crb.RoleRef.Name } func (c V1ClusterRoleBinding) ServiceAccountSubjects(namespace string) []string { var saSubjects []string for _, s := range c.Crb.Subjects { if s.Kind == rbacv1.ServiceAccountKind && s.Namespace == namespace { saSubjects = append(saSubjects, s.Name) } } return saSubjects } type V1beta1ClusterRoleBinding struct { Crb rbacbeta.ClusterRoleBinding } func (c V1beta1ClusterRoleBinding) Name() string { return c.Crb.Name } func (c V1beta1ClusterRoleBinding) RoleRefName() string { return c.Crb.RoleRef.Name } func (c V1beta1ClusterRoleBinding) ServiceAccountSubjects(namespace string) []string { var saSubjects []string for _, s := range c.Crb.Subjects { if s.Kind == rbacv1.ServiceAccountKind && s.Namespace == namespace { saSubjects = append(saSubjects, s.Name) } } return saSubjects } ================================================ FILE: pkg/util/actionhelpers/service_account_helper.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package actionhelpers import ( "github.com/sirupsen/logrus" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" velerodiscovery "github.com/vmware-tanzu/velero/pkg/discovery" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/plugin/velero" ) func ClusterRoleBindingsForAction(clusterRoleBindingListers map[string]ClusterRoleBindingLister, discoveryHelper velerodiscovery.Helper) ([]ClusterRoleBinding, error) { // Look up the supported RBAC version var supportedAPI metav1.GroupVersionForDiscovery for _, ag := range discoveryHelper.APIGroups() { if ag.Name == rbacv1.GroupName { supportedAPI = ag.PreferredVersion break } } crbLister := clusterRoleBindingListers[supportedAPI.Version] // This should be safe because the List call will return a 0-item slice // if there's no matching API version. return crbLister.List() } func RelatedItemsForServiceAccount(objectMeta metav1.Object, clusterRoleBindings []ClusterRoleBinding, log logrus.FieldLogger) []velero.ResourceIdentifier { var ( namespace = objectMeta.GetNamespace() name = objectMeta.GetName() bindings = sets.NewString() roles = sets.NewString() ) for _, crb := range clusterRoleBindings { for _, s := range crb.ServiceAccountSubjects(namespace) { if s == name { log.Infof("Adding clusterrole %s and clusterrolebinding %s to relatedItems since serviceaccount %s/%s is a subject", crb.RoleRefName(), crb.Name(), namespace, name) bindings.Insert(crb.Name()) roles.Insert(crb.RoleRefName()) break } } } var relatedItems []velero.ResourceIdentifier for binding := range bindings { relatedItems = append(relatedItems, velero.ResourceIdentifier{ GroupResource: kuberesource.ClusterRoleBindings, Name: binding, }) } for role := range roles { relatedItems = append(relatedItems, velero.ResourceIdentifier{ GroupResource: kuberesource.ClusterRoles, Name: role, }) } return relatedItems } ================================================ FILE: pkg/util/azure/credential.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package azure import ( "os" "strings" "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/pkg/errors" ) // NewCredential constructs a Credential that tries the config credential, workload identity credential // and managed identity credential according to the provided creds. func NewCredential(creds map[string]string, options policy.ClientOptions) (azcore.TokenCredential, error) { additionalTenants := []string{} if tenants := creds[CredentialKeyAdditionallyAllowedTenants]; tenants != "" { additionalTenants = strings.Split(tenants, ";") } // config credential if len(creds[CredentialKeyClientSecret]) > 0 || len(creds[CredentialKeyClientCertificate]) > 0 || len(creds[CredentialKeyClientCertificatePath]) > 0 || len(creds[CredentialKeyUsername]) > 0 { return newConfigCredential(creds, configCredentialOptions{ ClientOptions: options, AdditionallyAllowedTenants: additionalTenants, }) } // workload identity credential if len(os.Getenv("AZURE_FEDERATED_TOKEN_FILE")) > 0 { return azidentity.NewWorkloadIdentityCredential(&azidentity.WorkloadIdentityCredentialOptions{ AdditionallyAllowedTenants: additionalTenants, ClientOptions: options, }) } // managed identity credential o := &azidentity.ManagedIdentityCredentialOptions{ClientOptions: options, ID: azidentity.ClientID(creds[CredentialKeyClientID])} return azidentity.NewManagedIdentityCredential(o) } type configCredentialOptions struct { azcore.ClientOptions AdditionallyAllowedTenants []string } // newConfigCredential works similar as the azidentity.EnvironmentCredential but reads the credentials from a map // rather than environment variables. This is required for Velero to run B/R concurrently // https://github.com/Azure/azure-sdk-for-go/blob/sdk/azidentity/v1.3.0/sdk/azidentity/environment_credential.go#L80 func newConfigCredential(creds map[string]string, options configCredentialOptions) (azcore.TokenCredential, error) { tenantID := creds[CredentialKeyTenantID] if tenantID == "" { return nil, errors.Errorf("%s is required", CredentialKeyTenantID) } clientID := creds[CredentialKeyClientID] if clientID == "" { return nil, errors.Errorf("%s is required", CredentialKeyClientID) } // client secret if clientSecret := creds[CredentialKeyClientSecret]; clientSecret != "" { return azidentity.NewClientSecretCredential(tenantID, clientID, clientSecret, &azidentity.ClientSecretCredentialOptions{ AdditionallyAllowedTenants: options.AdditionallyAllowedTenants, ClientOptions: options.ClientOptions, }) } // raw certificate or certificate file if rawCerts, certsPath := []byte(creds[CredentialKeyClientCertificate]), creds[CredentialKeyClientCertificatePath]; len(rawCerts) > 0 || len(certsPath) > 0 { var err error // raw certificate isn't specified while certificate path is specified if len(rawCerts) == 0 { rawCerts, err = os.ReadFile(certsPath) if err != nil { return nil, errors.Wrapf(err, "failed to read certificate file %s", certsPath) } } var password []byte if v := creds[CredentialKeyClientCertificatePassword]; v != "" { password = []byte(v) } certs, key, err := azidentity.ParseCertificates(rawCerts, password) if err != nil { return nil, errors.Wrap(err, "failed to parse certificate") } o := &azidentity.ClientCertificateCredentialOptions{ AdditionallyAllowedTenants: options.AdditionallyAllowedTenants, ClientOptions: options.ClientOptions, } if v, ok := creds[CredentialKeySendCertChain]; ok { o.SendCertificateChain = v == "1" || strings.ToLower(v) == "true" } return azidentity.NewClientCertificateCredential(tenantID, clientID, certs, key, o) } return nil, errors.New("incomplete credential configuration. Only AZURE_TENANT_ID and AZURE_CLIENT_ID are set") } ================================================ FILE: pkg/util/azure/credential_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package azure import ( "os" "testing" "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestNewCredential(t *testing.T) { options := policy.ClientOptions{} // invalid client secret credential (missing tenant ID) creds := map[string]string{ CredentialKeyClientID: "clientid", CredentialKeyClientSecret: "secret", } _, err := NewCredential(creds, options) require.Error(t, err) // valid client secret credential creds = map[string]string{ CredentialKeyTenantID: "tenantid", CredentialKeyClientID: "clientid", CredentialKeyClientSecret: "secret", } tokenCredential, err := NewCredential(creds, options) require.NoError(t, err) assert.IsType(t, &azidentity.ClientSecretCredential{}, tokenCredential) // client certificate credential certData, err := readCertData() require.NoError(t, err) creds = map[string]string{ CredentialKeyTenantID: "tenantid", CredentialKeyClientID: "clientid", CredentialKeyClientCertificate: certData, } tokenCredential, err = NewCredential(creds, options) require.NoError(t, err) assert.IsType(t, &azidentity.ClientCertificateCredential{}, tokenCredential) // workload identity credential os.Setenv(CredentialKeyTenantID, "tenantid") os.Setenv(CredentialKeyClientID, "clientid") os.Setenv("AZURE_FEDERATED_TOKEN_FILE", "/tmp/token") creds = map[string]string{} tokenCredential, err = NewCredential(creds, options) require.NoError(t, err) assert.IsType(t, &azidentity.WorkloadIdentityCredential{}, tokenCredential) os.Clearenv() // managed identity credential creds = map[string]string{CredentialKeyClientID: "clientid"} tokenCredential, err = NewCredential(creds, options) require.NoError(t, err) assert.IsType(t, &azidentity.ManagedIdentityCredential{}, tokenCredential) } func Test_newConfigCredential(t *testing.T) { options := configCredentialOptions{} // tenantID not specified creds := map[string]string{} _, err := newConfigCredential(creds, options) require.Error(t, err) // clientID not specified creds = map[string]string{ CredentialKeyTenantID: "clientid", } _, err = newConfigCredential(creds, options) require.Error(t, err) // client secret creds = map[string]string{ CredentialKeyTenantID: "clientid", CredentialKeyClientID: "clientid", CredentialKeyClientSecret: "secret", } credential, err := newConfigCredential(creds, options) require.NoError(t, err) require.NotNil(t, credential) _, ok := credential.(*azidentity.ClientSecretCredential) require.True(t, ok) // client certificate certData, err := readCertData() require.NoError(t, err) creds = map[string]string{ CredentialKeyTenantID: "clientid", CredentialKeyClientID: "clientid", CredentialKeyClientCertificate: certData, } credential, err = newConfigCredential(creds, options) require.NoError(t, err) require.NotNil(t, credential) _, ok = credential.(*azidentity.ClientCertificateCredential) require.True(t, ok) } func readCertData() (string, error) { data, err := os.ReadFile("testdata/certificate.pem") if err != nil { return "", err } return string(data), nil } ================================================ FILE: pkg/util/azure/storage.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package azure import ( "context" "fmt" "strconv" "strings" "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" _ "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm/runtime" "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) const ( // the keys of Azure BSL config: // https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure/blob/main/backupstoragelocation.md BSLConfigResourceGroup = "resourceGroup" BSLConfigStorageAccount = "storageAccount" BSLConfigStorageAccountAccessKeyName = "storageAccountKeyEnvVar" BSLConfigSubscriptionID = "subscriptionId" BSLConfigStorageAccountURI = "storageAccountURI" BSLConfigUseAAD = "useAAD" BSLConfigActiveDirectoryAuthorityURI = "activeDirectoryAuthorityURI" serviceNameBlob cloud.ServiceName = "blob" ) func init() { cloud.AzureChina.Services[serviceNameBlob] = cloud.ServiceConfiguration{ Endpoint: "blob.core.chinacloudapi.cn", } cloud.AzureGovernment.Services[serviceNameBlob] = cloud.ServiceConfiguration{ Endpoint: "blob.core.usgovcloudapi.net", } cloud.AzurePublic.Services[serviceNameBlob] = cloud.ServiceConfiguration{ Endpoint: "blob.core.windows.net", } } // NewStorageClient creates a blob storage client(data plane) with the provided config which contains BSL config and the credential file name. // The returned azblob.SharedKeyCredential is needed for Azure plugin to generate the SAS URL when auth with storage // account access key func NewStorageClient(log logrus.FieldLogger, config map[string]string) (*azblob.Client, *azblob.SharedKeyCredential, error) { // rename to bslCfg for easy understanding bslCfg := config // storage account is required storageAccount := bslCfg[BSLConfigStorageAccount] if storageAccount == "" { return nil, nil, errors.Errorf("%s is required in BSL", BSLConfigStorageAccount) } // read the credentials provided by users creds, err := LoadCredentials(config) if err != nil { return nil, nil, err } // exchange the storage account access key if needed creds, err = GetStorageAccountCredentials(bslCfg, creds) if err != nil { return nil, nil, err } // get the storage account URI uri, err := getStorageAccountURI(log, bslCfg, creds) if err != nil { return nil, nil, err } clientOptions, err := GetClientOptions(bslCfg, creds) if err != nil { return nil, nil, err } blobClientOptions := &azblob.ClientOptions{ ClientOptions: clientOptions, } // auth with storage account access key accessKey := creds[CredentialKeyStorageAccountAccessKey] if accessKey != "" { log.Info("auth with the storage account access key") cred, err := azblob.NewSharedKeyCredential(storageAccount, accessKey) if err != nil { return nil, nil, errors.Wrap(err, "failed to create storage account access key credential") } client, err := azblob.NewClientWithSharedKeyCredential(uri, cred, blobClientOptions) if err != nil { return nil, nil, errors.Wrap(err, "failed to create blob client with the storage account access key") } return client, cred, nil } // auth with Azure AD log.Info("auth with Azure AD") cred, err := NewCredential(creds, clientOptions) if err != nil { return nil, nil, err } client, err := azblob.NewClient(uri, cred, blobClientOptions) if err != nil { return nil, nil, errors.Wrap(err, "failed to create blob client with the Azure AD credential") } return client, nil, nil } // GetStorageAccountCredentials returns the credentials to interactive with storage account according to the config of BSL // and credential file by the following order: // 1. Return the storage account access key directly if it is provided // 2. Return the content of the credential file directly if "userAAD" is set as true in BSL config // 3. Call Azure API to exchange the storage account access key func GetStorageAccountCredentials(bslCfg map[string]string, creds map[string]string) (map[string]string, error) { // use storage account access key if specified if name := bslCfg[BSLConfigStorageAccountAccessKeyName]; name != "" { accessKey := creds[name] if accessKey == "" { return nil, errors.Errorf("no storage account access key with key %s found in credential", name) } creds[CredentialKeyStorageAccountAccessKey] = accessKey return creds, nil } // use AAD if bslCfg[BSLConfigUseAAD] != "" { useAAD, err := strconv.ParseBool(bslCfg[BSLConfigUseAAD]) if err != nil { return nil, errors.Errorf("failed to parse bool from useAAD string: %s", bslCfg[BSLConfigUseAAD]) } if useAAD { return creds, nil } } // exchange the storage account access key accessKey, err := exchangeStorageAccountAccessKey(bslCfg, creds) if err != nil { return nil, errors.WithMessage(err, "failed to get storage account access key") } creds[CredentialKeyStorageAccountAccessKey] = accessKey return creds, nil } // getStorageAccountURI returns the storage account URI by the following order: // 1. Return the storage account URI directly if it is specified in BSL config // 2. Try to call Azure API to get the storage account URI if possible(Background: https://github.com/vmware-tanzu/velero/issues/6163) // 3. Fall back to return the default URI func getStorageAccountURI(log logrus.FieldLogger, bslCfg map[string]string, creds map[string]string) (string, error) { // if the URI is specified in the BSL, return it directly uri := bslCfg[BSLConfigStorageAccountURI] if uri != "" { log.Infof("the storage account URI %q is specified in the BSL, use it directly", uri) return uri, nil } storageAccount := bslCfg[BSLConfigStorageAccount] cloudCfg, err := getCloudConfiguration(bslCfg, creds) if err != nil { return "", err } // the default URI uri = fmt.Sprintf("https://%s.%s", storageAccount, cloudCfg.Services[serviceNameBlob].Endpoint) // the storage account access key cannot be used to get the storage account properties, // so fallback to the default URI if name := bslCfg[BSLConfigStorageAccountAccessKeyName]; name != "" && creds[name] != "" { log.Infof("auth with the storage account access key, cannot retrieve the storage account properties, fallback to use the default URI %q", uri) return uri, nil } client, err := newStorageAccountManagemenClient(bslCfg, creds) if err != nil { log.Infof("failed to create the storage account management client: %v, fallback to use the default URI %q", err, uri) return uri, nil } resourceGroup := GetFromLocationConfigOrCredential(bslCfg, creds, BSLConfigResourceGroup, CredentialKeyResourceGroup) // we cannot get the storage account properties without the resource group, so fallback to the default URI if resourceGroup == "" { log.Infof("resource group isn't set which is required to retrieve the storage account properties, fallback to use the default URI %q", uri) return uri, nil } properties, err := client.GetProperties(context.Background(), resourceGroup, storageAccount, nil) // get error, fallback to the default URI if err != nil { log.Infof("failed to retrieve the storage account properties: %v, fallback to use the default URI %q", err, uri) return uri, nil } uri = *properties.Account.Properties.PrimaryEndpoints.Blob log.Infof("use the storage account URI retrieved from the storage account properties %q", uri) return uri, nil } // try to exchange the storage account access key with the provided credentials func exchangeStorageAccountAccessKey(bslCfg, creds map[string]string) (string, error) { client, err := newStorageAccountManagemenClient(bslCfg, creds) if err != nil { return "", err } resourceGroup := GetFromLocationConfigOrCredential(bslCfg, creds, BSLConfigResourceGroup, CredentialKeyResourceGroup) if resourceGroup == "" { return "", errors.New("resource group is required in BSL or credential to exchange the storage account access key") } storageAccount := bslCfg[BSLConfigStorageAccount] if storageAccount == "" { return "", errors.Errorf("%s is required in the BSL to exchange the storage account access key", BSLConfigStorageAccount) } expand := "kerb" resp, err := client.ListKeys(context.Background(), resourceGroup, storageAccount, &armstorage.AccountsClientListKeysOptions{ Expand: &expand, }) if err != nil { return "", errors.Wrap(err, "failed to list storage account access keys") } for _, key := range resp.Keys { if key == nil || key.Permissions == nil { continue } if strings.EqualFold(string(*key.Permissions), string(armstorage.KeyPermissionFull)) { return *key.Value, nil } } return "", errors.New("no storage key with Full permissions found") } // new a management client for the storage account func newStorageAccountManagemenClient(bslCfg map[string]string, creds map[string]string) (*armstorage.AccountsClient, error) { clientOptions, err := GetClientOptions(bslCfg, creds) if err != nil { return nil, err } cred, err := NewCredential(creds, clientOptions) if err != nil { return nil, errors.WithMessage(err, "failed to create Azure AD credential") } subID := GetFromLocationConfigOrCredential(bslCfg, creds, BSLConfigSubscriptionID, CredentialKeySubscriptionID) if subID == "" { return nil, errors.New("subscription ID is required in BSL or credential to create the storage account client") } client, err := armstorage.NewAccountsClient(subID, cred, &arm.ClientOptions{ ClientOptions: clientOptions, }) if err != nil { return nil, errors.Wrap(err, "failed to create the storage account client") } return client, nil } ================================================ FILE: pkg/util/azure/storage_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package azure import ( "os" "path/filepath" "testing" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestNewStorageClient(t *testing.T) { log := logrus.New() config := map[string]string{} name := filepath.Join(os.TempDir(), "credential") file, err := os.Create(name) require.NoError(t, err) defer file.Close() defer os.Remove(name) _, err = file.WriteString("AccessKey: YWNjZXNza2V5\nAZURE_TENANT_ID: tenantid\nAZURE_CLIENT_ID: clientid\nAZURE_CLIENT_SECRET: secret") require.NoError(t, err) // storage account isn't specified _, _, err = NewStorageClient(log, config) require.Error(t, err) // auth with storage account access key config = map[string]string{ BSLConfigStorageAccount: "storage-account", "credentialsFile": name, BSLConfigStorageAccountAccessKeyName: "AccessKey", } client, credential, err := NewStorageClient(log, config) require.NoError(t, err) assert.NotNil(t, client) assert.NotNil(t, credential) // auth with Azure AD config = map[string]string{ BSLConfigStorageAccount: "storage-account", "credentialsFile": name, "useAAD": "true", } client, credential, err = NewStorageClient(log, config) require.NoError(t, err) assert.NotNil(t, client) assert.Nil(t, credential) } func TestGetStorageAccountCredentials(t *testing.T) { // use access secret but no secret specified cfg := map[string]string{ BSLConfigStorageAccountAccessKeyName: "KEY", } creds := map[string]string{} _, err := GetStorageAccountCredentials(cfg, creds) require.Error(t, err) // use access secret cfg = map[string]string{ BSLConfigStorageAccountAccessKeyName: "KEY", } creds = map[string]string{ "KEY": "key", } m, err := GetStorageAccountCredentials(cfg, creds) require.NoError(t, err) assert.Equal(t, "key", m[CredentialKeyStorageAccountAccessKey]) // use AAD, but useAAD invalid cfg = map[string]string{ "useAAD": "invalid", } creds = map[string]string{} _, err = GetStorageAccountCredentials(cfg, creds) require.Error(t, err) // use AAD cfg = map[string]string{ "useAAD": "true", } creds = map[string]string{ "KEY": "key", } m, err = GetStorageAccountCredentials(cfg, creds) require.NoError(t, err) assert.Equal(t, creds, m) } func Test_getStorageAccountURI(t *testing.T) { log := logrus.New() // URI specified bslCfg := map[string]string{ BSLConfigStorageAccountURI: "uri", } creds := map[string]string{} uri, err := getStorageAccountURI(log, bslCfg, creds) require.NoError(t, err) assert.Equal(t, "uri", uri) // no URI specified, and auth with access key bslCfg = map[string]string{ BSLConfigStorageAccountAccessKeyName: "KEY", } creds = map[string]string{ "KEY": "value", } uri, err = getStorageAccountURI(log, bslCfg, creds) require.NoError(t, err) assert.Equal(t, "https://.blob.core.windows.net", uri) // no URI specified, auth with AAD, resource group isn't specified bslCfg = map[string]string{ BSLConfigSubscriptionID: "subscriptionid", } creds = map[string]string{ "AZURE_TENANT_ID": "tenantid", "AZURE_CLIENT_ID": "clientid", "AZURE_CLIENT_SECRET": "secret", } uri, err = getStorageAccountURI(log, bslCfg, creds) require.NoError(t, err) assert.Equal(t, "https://.blob.core.windows.net", uri) // no URI specified, auth with AAD, resource group specified bslCfg = map[string]string{ BSLConfigSubscriptionID: "subscriptionid", BSLConfigResourceGroup: "resourcegroup", BSLConfigStorageAccount: "account", } creds = map[string]string{ "AZURE_TENANT_ID": "tenantid", "AZURE_CLIENT_ID": "clientid", "AZURE_CLIENT_SECRET": "secret", } uri, err = getStorageAccountURI(log, bslCfg, creds) require.NoError(t, err) assert.Equal(t, "https://account.blob.core.windows.net", uri) } func Test_exchangeStorageAccountAccessKey(t *testing.T) { // resource group isn't specified bslCfg := map[string]string{ BSLConfigSubscriptionID: "subscriptionid", } creds := map[string]string{ "AZURE_TENANT_ID": "tenantid", "AZURE_CLIENT_ID": "clientid", "AZURE_CLIENT_SECRET": "secret", } _, err := exchangeStorageAccountAccessKey(bslCfg, creds) require.Error(t, err) // storage account isn't specified bslCfg = map[string]string{ BSLConfigSubscriptionID: "subscriptionid", BSLConfigResourceGroup: "resourcegroup", } creds = map[string]string{ "AZURE_TENANT_ID": "tenantid", "AZURE_CLIENT_ID": "clientid", "AZURE_CLIENT_SECRET": "secret", } _, err = exchangeStorageAccountAccessKey(bslCfg, creds) require.Error(t, err) // storage account specified bslCfg = map[string]string{ BSLConfigSubscriptionID: "subscriptionid", BSLConfigResourceGroup: "resourcegroup", BSLConfigStorageAccount: "account", } creds = map[string]string{ "AZURE_TENANT_ID": "tenantid", "AZURE_CLIENT_ID": "clientid", "AZURE_CLIENT_SECRET": "secret", } _, err = exchangeStorageAccountAccessKey(bslCfg, creds) require.Error(t, err) } func Test_newStorageAccountManagemenClient(t *testing.T) { // subscription ID isn't specified bslCfg := map[string]string{} creds := map[string]string{ "AZURE_TENANT_ID": "tenantid", "AZURE_CLIENT_ID": "clientid", "AZURE_CLIENT_SECRET": "secret", } _, err := newStorageAccountManagemenClient(bslCfg, creds) require.Error(t, err) // subscription ID isn't specified bslCfg = map[string]string{ BSLConfigSubscriptionID: "subscriptionid", } creds = map[string]string{ "AZURE_TENANT_ID": "tenantid", "AZURE_CLIENT_ID": "clientid", "AZURE_CLIENT_SECRET": "secret", } _, err = newStorageAccountManagemenClient(bslCfg, creds) require.NoError(t, err) } ================================================ FILE: pkg/util/azure/testdata/certificate.pem ================================================ -----BEGIN PRIVATE KEY----- MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDL1hG+JYCfIPp3 tlZ05J4pYIJ3Ckfs432bE3rYuWlR2w9KqdjWkKxuAxpjJ+T+uoqVaT3BFMfi4ZRY OCI69s4+lP3DwR8uBCp9xyVkF8thXfS3iui0liGDviVBoBJJWvjDFU8a/Hseg+Qf oxAb6tx0kEc7V3ozBLWoIDJjfwJ3NdsLZGVtAC34qCWeEIvS97CDA4g3Kc6hYJIr Aa7pxHzo/Nd0U3e7z+DlBcJV7dY6TZUyjBVTpzppWe+XQEOfKsjkDNykHEC1C1bC lG0u7unS7QOBMd6bOGkeL+Bc+n22slTzs5amsbDLNuobSaUsFt9vgD5jRD6FwhpX wj/Ek0F7AgMBAAECggEAblU3UWdXUcs2CCqIbcl52wfEVs8X05/n01MeAcWKvqYG hvGcz7eLvhir5dQoXcF3VhybMrIe6C4WcBIiZSxGwxU+rwEP8YaLwX1UPfOrQM7s sZTdFTLWfUslO3p7q300fdRA92iG9COMDZvkElh0cBvQksxs9sSr149l9vk+ymtC uBhZtHG6Ki0BIMBNC9jGUqDuOatXl/dkK4tNjXrNJT7tVwzPaqnNALIWl6B+k9oQ m1oNhSH2rvs9tw2ITXfIoIk9KdOMjQVUD43wKOaz0hNZhUsb1OFuls7UtRzaFcZH rMd/M8DtA104QTTlHK+XS7r+nqdv7+ZyB+suTdM+oQKBgQDxCrJZU3hJ0eJ4VYhK xGDfVGNpYxNkQ4CDB9fwRNbFr/Ck3kgzfE9QxTx1pJOolVmfuFmk9B86in4UNy91 KdaqT79AU5RdOBXNN6tuMbLC0AVqe8sZq+1vWVVwbCstffxEMmyW1Ju/FLYPl2Zp e5P96dBh5B3mXrQtpDJ0RkxxaQKBgQDYfE6tQQnQSs2ewD6ae8Mu6j8ueDlVoZ37 vze1QdBasR26xu2H8XBt3u41zc524BwQsB1GE1tnC8ZylrqwVEayK4FesSQRCO6o yK8QSdb06I5J4TaN+TppCDPLzstOh0Dmxp+iFUGoErb7AEOLAJ/VebhF9kBZObL/ HYy4Es+bQwKBgHW/4vYuB3IQXNCp/+V+X1BZ+iJOaves3gekekF+b2itFSKFD8JO 9LQhVfKmTheptdmHhgtF0keXxhV8C+vxX1Ndl7EF41FSh5vzmQRAtPHkCvFEviex TFD70/gSb1lO1UA/Xbqk69yBcprVPAtFejss0EYx2MVj+CLftmIEwW0ZAoGBAIMG EVQ45eikLXjkn78+Iq7VZbIJX6IdNBH29I+GqsUJJ5Yw6fh6P3KwF3qG+mvmTfYn sUAFXS+r58rYwVsRVsxlGmKmUc7hmhibhaEVH72QtvWuEiexbRG+viKfIVuA7t39 3wXpWZiQ4yBdU4Pgt9wrVEU7ukyGaHiReOa7s90jAoGAJc0K7smn98YutQQ+g2ur ybfnsl0YdsksaP2S2zvZUmNevKPrgnaIDDabOlhYYga+AK1G3FQ7/nefUgiIg1Nd kr+T6Q4osS3xHB6Az9p/jaF4R2KaWN2nNVCn7ecsmPxDdM7k1vLxaT26vwO9OP5f YU/5CeIzrfA5nQyPZkOXZBk= -----END PRIVATE KEY----- -----BEGIN CERTIFICATE----- MIIDazCCAlOgAwIBAgIUF2VIP4+AnEtb52KTCHbo4+fESfswDQYJKoZIhvcNAQEL BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0xOTEwMzAyMjQ2MjBaFw0yMjA4 MTkyMjQ2MjBaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEB AQUAA4IBDwAwggEKAoIBAQDL1hG+JYCfIPp3tlZ05J4pYIJ3Ckfs432bE3rYuWlR 2w9KqdjWkKxuAxpjJ+T+uoqVaT3BFMfi4ZRYOCI69s4+lP3DwR8uBCp9xyVkF8th XfS3iui0liGDviVBoBJJWvjDFU8a/Hseg+QfoxAb6tx0kEc7V3ozBLWoIDJjfwJ3 NdsLZGVtAC34qCWeEIvS97CDA4g3Kc6hYJIrAa7pxHzo/Nd0U3e7z+DlBcJV7dY6 TZUyjBVTpzppWe+XQEOfKsjkDNykHEC1C1bClG0u7unS7QOBMd6bOGkeL+Bc+n22 slTzs5amsbDLNuobSaUsFt9vgD5jRD6FwhpXwj/Ek0F7AgMBAAGjUzBRMB0GA1Ud DgQWBBT6Mf9uXFB67bY2PeW3GCTKfkO7vDAfBgNVHSMEGDAWgBT6Mf9uXFB67bY2 PeW3GCTKfkO7vDAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQCZ 1+kTISX85v9/ag7glavaPFUYsOSOOofl8gSzov7L01YL+srq7tXdvZmWrjQ/dnOY h18rp9rb24vwIYxNioNG/M2cW1jBJwEGsDPOwdPV1VPcRmmUJW9kY130gRHBCd/N qB7dIkcQnpNsxPIIWI+sRQp73U0ijhOByDnCNHLHon6vbfFTwkO1XggmV5BdZ3uQ JNJyckILyNzlhmf6zhonMp4lVzkgxWsAm2vgdawd6dmBa+7Avb2QK9s+IdUSutFh DgW2L12Obgh12Y4sf1iKQXA0RbZ2k+XQIz8EKZa7vJQY0ciYXSgB/BV3a96xX3cx LIPL8Vam8Ytkopi3gsGA -----END CERTIFICATE----- ================================================ FILE: pkg/util/azure/util.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package azure import ( "crypto/tls" "crypto/x509" "encoding/base64" "fmt" "net" "net/http" "os" "strings" "time" "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/joho/godotenv" "github.com/pkg/errors" ) const ( // the keys of Azure variables in credential CredentialKeySubscriptionID = "AZURE_SUBSCRIPTION_ID" // #nosec CredentialKeyResourceGroup = "AZURE_RESOURCE_GROUP" // #nosec CredentialKeyCloudName = "AZURE_CLOUD_NAME" // #nosec CredentialKeyStorageAccountAccessKey = "AZURE_STORAGE_KEY" // #nosec CredentialKeyAdditionallyAllowedTenants = "AZURE_ADDITIONALLY_ALLOWED_TENANTS" // #nosec CredentialKeyTenantID = "AZURE_TENANT_ID" // #nosec CredentialKeyClientID = "AZURE_CLIENT_ID" // #nosec CredentialKeyClientSecret = "AZURE_CLIENT_SECRET" // #nosec CredentialKeyClientCertificate = "AZURE_CLIENT_CERTIFICATE" // #nosec CredentialKeyClientCertificatePath = "AZURE_CLIENT_CERTIFICATE_PATH" // #nosec CredentialKeyClientCertificatePassword = "AZURE_CLIENT_CERTIFICATE_PASSWORD" // #nosec CredentialKeySendCertChain = "AZURE_CLIENT_SEND_CERTIFICATE_CHAIN" // #nosec CredentialKeyUsername = "AZURE_USERNAME" // #nosec CredentialKeyPassword = "AZURE_PASSWORD" // #nosec credentialFile = "credentialsFile" ) // LoadCredentials gets the credential file from config and loads it into a map func LoadCredentials(config map[string]string) (map[string]string, error) { // the default credential file credFile := os.Getenv("AZURE_CREDENTIALS_FILE") // use the credential file specified in the BSL spec if provided if config != nil && config[credentialFile] != "" { credFile = config[credentialFile] } if len(credFile) == 0 { return map[string]string{}, nil } // put the credential file content into a map creds, err := godotenv.Read(credFile) if err != nil { return nil, errors.Wrapf(err, "failed to read credentials from file %s", credFile) } return creds, nil } // GetClientOptions returns the client options based on the BSL/VSL config and credentials func GetClientOptions(locationCfg, creds map[string]string) (policy.ClientOptions, error) { options := policy.ClientOptions{} cloudCfg, err := getCloudConfiguration(locationCfg, creds) if err != nil { return options, err } options.Cloud = cloudCfg if locationCfg["caCert"] != "" { certPool, _ := x509.SystemCertPool() if certPool == nil { certPool = x509.NewCertPool() } var caCert []byte // As this function is used in both repository and plugin, the caCert isn't encoded // when passing to the plugin while is encoded when works with repository, use one // config item to distinguish these two cases if locationCfg["caCertEncoded"] != "" { caCert, err = base64.StdEncoding.DecodeString(locationCfg["caCert"]) if err != nil { return options, err } } else { caCert = []byte(locationCfg["caCert"]) } certPool.AppendCertsFromPEM(caCert) // https://github.com/Azure/azure-sdk-for-go/blob/sdk/azcore/v1.6.1/sdk/azcore/runtime/transport_default_http_client.go#L19 transport := &http.Transport{ Proxy: http.ProxyFromEnvironment, DialContext: (&net.Dialer{ Timeout: 30 * time.Second, KeepAlive: 30 * time.Second, }).DialContext, ForceAttemptHTTP2: true, MaxIdleConns: 100, IdleConnTimeout: 90 * time.Second, TLSHandshakeTimeout: 10 * time.Second, ExpectContinueTimeout: 1 * time.Second, TLSClientConfig: &tls.Config{ MinVersion: tls.VersionTLS12, RootCAs: certPool, }, } options.Transport = &http.Client{ Transport: transport, } } return options, nil } // getCloudConfiguration based on the BSL/VSL config and credentials func getCloudConfiguration(locationCfg, creds map[string]string) (cloud.Configuration, error) { name := creds[CredentialKeyCloudName] activeDirectoryAuthorityURI := locationCfg[BSLConfigActiveDirectoryAuthorityURI] var cfg cloud.Configuration switch strings.ToUpper(name) { case "", "AZURECLOUD", "AZUREPUBLICCLOUD": cfg = cloud.AzurePublic case "AZURECHINACLOUD": cfg = cloud.AzureChina case "AZUREUSGOVERNMENT", "AZUREUSGOVERNMENTCLOUD": cfg = cloud.AzureGovernment default: return cloud.Configuration{}, errors.New(fmt.Sprintf("unknown cloud: %s", name)) } if activeDirectoryAuthorityURI != "" { cfg.ActiveDirectoryAuthorityHost = activeDirectoryAuthorityURI } return cfg, nil } // GetFromLocationConfigOrCredential returns the value of the specified key from BSL/VSL config or credentials // as some common configuration items can be set in BSL/VSL config or credential file(such as the subscription ID or resource group) // Reading from BSL/VSL config takes first. func GetFromLocationConfigOrCredential(cfg, creds map[string]string, cfgKey, credKey string) string { value := cfg[cfgKey] if value != "" { return value } return creds[credKey] } ================================================ FILE: pkg/util/azure/util_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package azure import ( "os" "path/filepath" "testing" "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestLoadCredentials(t *testing.T) { // no credential file credentials, err := LoadCredentials(nil) require.NoError(t, err) assert.NotNil(t, credentials) // specified credential file in the config name := filepath.Join(os.TempDir(), "credential") file, err := os.Create(name) require.NoError(t, err) defer file.Close() defer os.Remove(name) _, err = file.WriteString("key: value") require.NoError(t, err) config := map[string]string{ "credentialsFile": name, } credentials, err = LoadCredentials(config) require.NoError(t, err) assert.Equal(t, "value", credentials["key"]) // use the default path defined via env variable config = nil os.Setenv("AZURE_CREDENTIALS_FILE", name) credentials, err = LoadCredentials(config) require.NoError(t, err) assert.Equal(t, "value", credentials["key"]) } func TestGetClientOptions(t *testing.T) { // invalid cloud name bslCfg := map[string]string{} creds := map[string]string{ CredentialKeyCloudName: "invalid", } _, err := GetClientOptions(bslCfg, creds) require.Error(t, err) // specify caCert bslCfg = map[string]string{ CredentialKeyCloudName: "", "caCert": "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUZTakNDQXpLZ0F3SUJBZ0lVWmcxbzRpWld2bVh5ekJrQ0J6SGdiODZGemtFd0RRWUpLb1pJaHZjTkFRRUwKQlFBd1VqRUxNQWtHQTFVRUJoTUNRMDR4RERBS0JnTlZCQWdNQTFCRlN6RVJNQThHQTFVRUJ3d0lRbVZwSUVwcApibWN4RHpBTkJnTlZCQW9NQmxaTmQyRnlaVEVSTUE4R0ExVUVBd3dJU0dGeVltOXlRMEV3SGhjTk1qTXdPVEEyCk1ESXpOakUyV2hjTk1qUXdPVEExTURJek5qRTJXakJYTVFzd0NRWURWUVFHRXdKRFRqRU1NQW9HQTFVRUNBd0QKVUVWTE1SRXdEd1lEVlFRSERBaENaV2tnU21sdVp6RVBNQTBHQTFVRUNnd0dWazEzWVhKbE1SWXdGQVlEVlFRRApEQTFJWVhKaWIzSk5ZVzVoWjJWeU1JSUNJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBZzhBTUlJQ0NnS0NBZ0VBCnIrK1FHaHYvUnBDUTFIcncrMnYyQWNoaGhUVTVQL3hCd2RIWkZHWWJzMmxGbGtiL3oycEs2Y05ycFZmNUtmdjIKVUNpZEovMjFhZHc2SWNGZWxkSnFudU4rSlJaWXh5S0w0bzdRRGNVSk1sUTZJZk5kbEI0NUNwcGFBZVA4blVVTgo0YUwyV244b094L1pROTd2YmRXeERIR1FqZGR4N3p0Q09PaVZ0SEk4NS9Ka3kydTJnNmVhMklndmh1ZEVPZ3JtCjJzNU8zZlVtdHhSTEhwNnpDbURYZGZFUWg4ZFpndCs1d0RlazRWR2t4Zk81VG1tUHJ0LzBPTnVGYjJMWGVWV0sKeXkzVDFFTGNOSWhPSzQ1amhEejNnb2JhQzAwK0JTdzJMejVocXRxdUQ2RGxmME53TWtBQkt6d1dMZkROOXNrRApVazVYTmZNa2c0L3JhblRIWHlCNUNKSlk1akhORUtBQWlnM2NFSFNvejVjeGJqTE14VDhoMEk3MitEWldmUzFTCjU3Rm1SN2ZTNXk0QUcrU3Z1U3kvbktCUktJS2dQZ0t2Rjl6NktuL2ZPMTNsbk5LbHQwWU5mWFBFV2hZQytmMUoKTWpTOWc5eHBpYkZhM2Q0aHpOeWZhMWJHcUxtbkUwNVNpbWZNMVI1Z21Tamw1Z1FKQlltTHA3dWRLdjFDSUNRSwptQng2WG4vcnJEZHFiMndCRUNRSjBMbUo2SW5SaFZtT0s0WUdFeFRqZ1FRMldSWHYxMnhVK05GYWlZS3cxZkp0CkdaemFQeENxaG5JZXM5cGNPY0FjdmFHVngwSjlFYnRod0ltekdoTjBBREdCOVZaL1dFdHYvN0gwQ2xjOVlyT0gKNnRMb212b2pjQUZnN0xFbXZxeFFEOFFSTzlZZVdTTkgvV1REY1hVb3R5a0NBd0VBQWFNVE1CRXdEd1lEVlIwUgpCQWd3Qm9jRUNycFFxakFOQmdrcWhraUc5dzBCQVFzRkFBT0NBZ0VBZnRVdmR1UFMvajhSaWx4ME5aelhSeEY3Ck9HZW9qU2JaQ1ZvamNPdnRNTVlMaFkxdDE2Y2laY1VWMGF4Z2FUWkkvak9WMGJPTEl3dmEvcVB2Z1RmSWZqL0gKVzhiMlNTRVdIUzZPSFFaR1BYNy9zVFVwQzB6QVcva2haN1FWR1BoWEcyK0V1NjFaNE95ejZ5dTRPdi9MYjlMUQpmMU9zTXhwandkbmhxazFKaERxUkpZbGIvZ05TRGZnVlN0YmhHVzVhb3paUlBBMUtqVXVaT3QzR2xQR09Wd3ZLCnpUcFFMdGVTUHNibTJMcUl2ZEg4dlgzK1kwcHIzdEdtdnExbWtIWUhYQTlBZWtYRkVsRHc4dGtZVHdLaEFqblUKZEFjWTFkTis5ODNiMDI0L0JQUXZKQlRTVjd4blEyUnlrUmMrVGxIL3B5RlM1cEtVbUF0aU9qTElxL2ZEMmJVagorTzlxT1hjK0c1b0xEaXlXWDRXSG9XdkZZdTdva1gwT1dGcHFETXFOcHlLUkRzQ1FENXViMEVQaVlVS0hnWEhiCnV3UXVtK0pRRUREdzRXL1kzZktnMW9TWW1XOHJndFNPZmtRQlQ0UnlaTUg2SzN6cFp5dVVsbmJUV0NWeEcyYVoKWVo0T2JpbUFGbVlveGRYdktWdFU0YUdlTjRoaXBvb2dzaXVXKzZYQ3Bqa2pWZlZuUEY4elZVNlZ3anRQVkkzKwpxdWxRNWJLS3lKYng3bk9NNXFob2svSmk2N1pyZDhob3ZwclhhRUdvakNDTVI3MllPWGVuMlB3bVlZZWNkQ2pyCnErSDdHNUV3ZXBoRWxrN3RWRWY4RVV4OEc1Mk9SVEtZMkF1dlRGVlliUC8yaTROS1FlMWdEWWZrWnNzUk1MajEKK0JCQVVJcnFVMnRuUHhwZW4vMD0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=", } creds = map[string]string{} options, err := GetClientOptions(bslCfg, creds) require.NoError(t, err) assert.Equal(t, options.Cloud, cloud.AzurePublic) assert.NotNil(t, options.Transport) // doesn't specify caCert bslCfg = map[string]string{ CredentialKeyCloudName: "", } creds = map[string]string{} options, err = GetClientOptions(bslCfg, creds) require.NoError(t, err) assert.Equal(t, options.Cloud, cloud.AzurePublic) assert.Nil(t, options.Transport) } func Test_getCloudConfiguration(t *testing.T) { publicCloudWithADURI := cloud.AzurePublic publicCloudWithADURI.ActiveDirectoryAuthorityHost = "https://example.com" cases := []struct { name string bslCfg map[string]string creds map[string]string err bool expected cloud.Configuration }{ { name: "invalid cloud name", bslCfg: map[string]string{}, creds: map[string]string{ CredentialKeyCloudName: "invalid", }, err: true, }, { name: "null cloud name", bslCfg: map[string]string{}, creds: map[string]string{ CredentialKeyCloudName: "", }, err: false, expected: cloud.AzurePublic, }, { name: "azure public cloud", bslCfg: map[string]string{}, creds: map[string]string{ CredentialKeyCloudName: "AZURECLOUD", }, err: false, expected: cloud.AzurePublic, }, { name: "azure public cloud", bslCfg: map[string]string{}, creds: map[string]string{ CredentialKeyCloudName: "AZUREPUBLICCLOUD", }, err: false, expected: cloud.AzurePublic, }, { name: "azure public cloud", bslCfg: map[string]string{}, creds: map[string]string{ CredentialKeyCloudName: "azurecloud", }, err: false, expected: cloud.AzurePublic, }, { name: "azure China cloud", bslCfg: map[string]string{}, creds: map[string]string{ CredentialKeyCloudName: "AZURECHINACLOUD", }, err: false, expected: cloud.AzureChina, }, { name: "azure US government cloud", bslCfg: map[string]string{}, creds: map[string]string{ CredentialKeyCloudName: "AZUREUSGOVERNMENT", }, err: false, expected: cloud.AzureGovernment, }, { name: "azure US government cloud", bslCfg: map[string]string{}, creds: map[string]string{ CredentialKeyCloudName: "AZUREUSGOVERNMENTCLOUD", }, err: false, expected: cloud.AzureGovernment, }, { name: "AD authority URI provided", bslCfg: map[string]string{ BSLConfigActiveDirectoryAuthorityURI: "https://example.com", }, creds: map[string]string{ CredentialKeyCloudName: "", }, err: false, expected: publicCloudWithADURI, }, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { cfg, err := getCloudConfiguration(c.bslCfg, c.creds) require.Equal(t, c.err, err != nil) if !c.err { assert.Equal(t, c.expected, cfg) } }) } } func TestGetFromLocationConfigOrCredential(t *testing.T) { // from cfg cfg := map[string]string{ "cfgkey": "value", } creds := map[string]string{} cfgKey, credKey := "cfgkey", "credkey" str := GetFromLocationConfigOrCredential(cfg, creds, cfgKey, credKey) assert.Equal(t, "value", str) // from cred cfg = map[string]string{} creds = map[string]string{ "credkey": "value", } str = GetFromLocationConfigOrCredential(cfg, creds, cfgKey, credKey) assert.Equal(t, "value", str) } ================================================ FILE: pkg/util/boolptr/boolptr.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package boolptr // IsSetToTrue returns true if and only if the bool pointer is non-nil and set to true. func IsSetToTrue(b *bool) bool { return b != nil && *b } // IsSetToFalse returns true if and only if the bool pointer is non-nil and set to false. func IsSetToFalse(b *bool) bool { return b != nil && !*b } // True returns a *bool whose underlying value is true. func True() *bool { t := true return &t } // False returns a *bool whose underlying value is false. func False() *bool { t := false return &t } ================================================ FILE: pkg/util/collections/includes_excludes.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package collections import ( "strings" "github.com/vmware-tanzu/velero/internal/resourcepolicies" "github.com/gobwas/glob" "github.com/pkg/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/api/validation" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/sets" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/discovery" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/util/boolptr" "github.com/vmware-tanzu/velero/pkg/util/wildcard" ) type globStringSet struct { sets.String } func newGlobStringSet() globStringSet { return globStringSet{sets.NewString()} } func (gss globStringSet) match(match string) bool { for _, item := range gss.List() { g, err := glob.Compile(item) if err != nil { return false } if g.Match(match) { return true } } return false } // NamespaceIncludesExcludes adds some features to IncludesExcludes // to handle namespace-specific functionality. In particular, it // provides a way to list all namespaces included in order to determine // overlap between backups, and it will be expanded in the future to // handle namespace wildcard values type NamespaceIncludesExcludes struct { activeNamespaces []string includesExcludes *IncludesExcludes wildcardExpanded bool wildcardResult []string } func NewNamespaceIncludesExcludes() *NamespaceIncludesExcludes { return &NamespaceIncludesExcludes{ activeNamespaces: []string{}, includesExcludes: NewIncludesExcludes(), } } func (nie *NamespaceIncludesExcludes) ActiveNamespaces(activeNamespaces []string) *NamespaceIncludesExcludes { nie.activeNamespaces = activeNamespaces return nie } func (nie *NamespaceIncludesExcludes) IsWildcardExpanded() bool { return nie.wildcardExpanded } // Includes adds items to the includes list. '*' is a wildcard // value meaning "include everything". func (nie *NamespaceIncludesExcludes) Includes(includes ...string) *NamespaceIncludesExcludes { nie.includesExcludes.Includes(includes...) return nie } // GetIncludes returns the items in the includes list func (nie *NamespaceIncludesExcludes) GetIncludes() []string { return nie.includesExcludes.GetIncludes() } func (nie *NamespaceIncludesExcludes) GetExcludes() []string { return nie.includesExcludes.GetExcludes() } // SetIncludes sets the includes list to the given list func (nie *NamespaceIncludesExcludes) SetIncludes(includes []string) *NamespaceIncludesExcludes { nie.includesExcludes.includes = newGlobStringSet() nie.includesExcludes.includes.Insert(includes...) return nie } // SetExcludes sets the excludes list to the given list func (nie *NamespaceIncludesExcludes) SetExcludes(excludes []string) *NamespaceIncludesExcludes { nie.includesExcludes.excludes = newGlobStringSet() nie.includesExcludes.excludes.Insert(excludes...) return nie } // IncludesString returns a string containing all of the includes, separated by commas, or * if the // list is empty. func (nie *NamespaceIncludesExcludes) IncludesString() string { return nie.includesExcludes.IncludesString() } // Excludes adds items to the includes list. '*' is a wildcard // value meaning "include everything". func (nie *NamespaceIncludesExcludes) Excludes(excludes ...string) *NamespaceIncludesExcludes { nie.includesExcludes.Excludes(excludes...) return nie } // IncludesString returns a string containing all of the excludes, separated by commas, or * if the // list is empty. func (nie *NamespaceIncludesExcludes) ExcludesString() string { return nie.includesExcludes.ExcludesString() } // ShouldInclude returns whether the specified item should be // included or not. Everything in the includes list except those // items in the excludes list should be included. func (nie *NamespaceIncludesExcludes) ShouldInclude(s string) bool { // Special case: if wildcard expansion occurred and resulted in an empty includes list, // it means the wildcard pattern matched nothing, so we should include nothing. // This differs from the default behavior where an empty includes list means "include everything". if nie.wildcardExpanded && nie.includesExcludes.includes.Len() == 0 { return false } return nie.includesExcludes.ShouldInclude(s) } // IncludeEverything returns true if the includes list is empty or '*' // and the excludes list is empty, or false otherwise. func (nie *NamespaceIncludesExcludes) IncludeEverything() bool { return nie.includesExcludes.IncludeEverything() } // Attempts to expand wildcard patterns, if any, in the includes and excludes lists. func (nie *NamespaceIncludesExcludes) ExpandIncludesExcludes() error { includes := nie.GetIncludes() excludes := nie.GetExcludes() if wildcard.ShouldExpandWildcards(includes, excludes) { expandedIncludes, expandedExcludes, err := wildcard.ExpandWildcards( nie.activeNamespaces, includes, excludes) if err != nil { return err } nie.SetIncludes(expandedIncludes) nie.SetExcludes(expandedExcludes) nie.wildcardExpanded = true } return nil } // ResolveNamespaceList returns a list of all namespaces which will be backed up. // The second return value indicates whether wildcard expansion was performed. func (nie *NamespaceIncludesExcludes) ResolveNamespaceList() ([]string, error) { // Check if this is being called by non-backup processing e.g. backup queue controller if !nie.wildcardExpanded { err := nie.ExpandIncludesExcludes() if err != nil { return nil, err } } outNamespaces := []string{} for _, ns := range nie.activeNamespaces { if nie.ShouldInclude(ns) { outNamespaces = append(outNamespaces, ns) } } nie.wildcardResult = outNamespaces return nie.wildcardResult, nil } // IncludesExcludes is a type that manages lists of included // and excluded items. The logic implemented is that everything // in the included list except those items in the excluded list // should be included. '*' in the includes list means "include // everything", but it is not valid in the exclude list. type IncludesExcludes struct { includes globStringSet excludes globStringSet } func NewIncludesExcludes() *IncludesExcludes { return &IncludesExcludes{ includes: newGlobStringSet(), excludes: newGlobStringSet(), } } // Includes adds items to the includes list. '*' is a wildcard // value meaning "include everything". func (ie *IncludesExcludes) Includes(includes ...string) *IncludesExcludes { ie.includes.Insert(includes...) return ie } // GetIncludes returns the items in the includes list func (ie *IncludesExcludes) GetIncludes() []string { return ie.includes.List() } // Excludes adds items to the excludes list func (ie *IncludesExcludes) Excludes(excludes ...string) *IncludesExcludes { ie.excludes.Insert(excludes...) return ie } // GetExcludes returns the items in the excludes list func (ie *IncludesExcludes) GetExcludes() []string { return ie.excludes.List() } // ShouldInclude returns whether the specified item should be // included or not. Everything in the includes list except those // items in the excludes list should be included. func (ie *IncludesExcludes) ShouldInclude(s string) bool { if ie.excludes.match(s) { return false } // len=0 means include everything return ie.includes.Len() == 0 || ie.includes.Has("*") || ie.includes.match(s) } // IncludesString returns a string containing all of the includes, separated by commas, or * if the // list is empty. func (ie *IncludesExcludes) IncludesString() string { return asString(ie.GetIncludes(), "*") } // ExcludesString returns a string containing all of the excludes, separated by commas, or if the // list is empty. func (ie *IncludesExcludes) ExcludesString() string { return asString(ie.GetExcludes(), "") } // IncludeEverything returns true if the includes list is empty or '*' // and the excludes list is empty, or false otherwise. func (ie *IncludesExcludes) IncludeEverything() bool { return ie.excludes.Len() == 0 && (ie.includes.Len() == 0 || (ie.includes.Len() == 1 && ie.includes.Has("*"))) } // GetResourceIncludesExcludes takes the lists of resources to include and exclude, uses the // discovery helper to resolve them to fully-qualified group-resource names, and returns an // IncludesExcludes list. func GetResourceIncludesExcludes(helper discovery.Helper, includes, excludes []string) *IncludesExcludes { resources := generateIncludesExcludes( includes, excludes, func(item string) string { gvr, _, err := helper.ResourceFor(schema.ParseGroupResource(item).WithVersion("")) if err != nil { // If we can't resolve it, return it as-is. This prevents the generated // includes-excludes list from including *everything*, if none of the includes // can be resolved. ref. https://github.com/vmware-tanzu/velero/issues/2461 return item } gr := gvr.GroupResource() return gr.String() }, ) return resources } func asString(in []string, empty string) string { if len(in) == 0 { return empty } return strings.Join(in, ", ") } // IncludesExcludesInterface is used as polymorphic IncludesExcludes for Global and scope // resources Include/Exclude. type IncludesExcludesInterface interface { // ShouldInclude checks whether the type name passed in by parameter should be included. // typeName should be k8s.io/apimachinery/pkg/runtime/schema GroupResource's String() result. ShouldInclude(typeName string) bool // ShouldExclude checks whether the type name passed in by parameter should be excluded. // typeName should be k8s.io/apimachinery/pkg/runtime/schema GroupResource's String() result. ShouldExclude(typeName string) bool } type GlobalIncludesExcludes struct { resourceFilter IncludesExcludes includeClusterResources *bool namespaceFilter NamespaceIncludesExcludes helper discovery.Helper logger logrus.FieldLogger } // ShouldInclude returns whether the specified item should be // included or not. Everything in the includes list except those // items in the excludes list should be included. // It has some exceptional cases. When IncludeClusterResources is set to false, // no need to check the filter, all cluster resources are excluded. func (ie *GlobalIncludesExcludes) ShouldInclude(typeName string) bool { _, resource, err := ie.helper.ResourceFor(schema.ParseGroupResource(typeName).WithVersion("")) if err != nil { ie.logger.Errorf("fail to get resource %s. %s", typeName, err.Error()) return false } if !resource.Namespaced && boolptr.IsSetToFalse(ie.includeClusterResources) { ie.logger.Info("Skipping resource %s, because it's cluster-scoped, and IncludeClusterResources is set to false.", typeName) return false } // when IncludeClusterResources == nil (auto), only directly // back up cluster-scoped resources if we're doing a full-cluster // (all namespaces and all namespace scope types) backup. Note that in the case of a subset of // namespaces being backed up, some related cluster-scoped resources // may still be backed up if triggered by a custom action (e.g. PVC->PV). // If we're processing namespaces themselves, we will not skip here, they may be // filtered out later. if typeName != kuberesource.Namespaces.String() && !resource.Namespaced && ie.includeClusterResources == nil && !ie.namespaceFilter.IncludeEverything() { ie.logger.Infof("Skipping resource %s, because it's cluster-scoped and only specific namespaces or namespace scope types are included in the backup.", typeName) return false } return ie.resourceFilter.ShouldInclude(typeName) } // ShouldExclude returns whether the resource type should be excluded or not. func (ie *GlobalIncludesExcludes) ShouldExclude(typeName string) bool { // if the type name is specified in excluded list, it's excluded. if ie.resourceFilter.excludes.match(typeName) { return true } _, resource, err := ie.helper.ResourceFor(schema.ParseGroupResource(typeName).WithVersion("")) if err != nil { ie.logger.Errorf("fail to get resource %s. %s", typeName, err.Error()) return true } // the resource type is cluster scope if !resource.Namespaced { // if includeClusterResources is set to false, cluster resource should be excluded. if boolptr.IsSetToFalse(ie.includeClusterResources) { return true } // if includeClusterResources is set to nil, check whether it's included by resource // filter. if ie.includeClusterResources == nil && !ie.resourceFilter.ShouldInclude(typeName) { return true } } return false } func GetGlobalResourceIncludesExcludes(helper discovery.Helper, logger logrus.FieldLogger, includes, excludes []string, includeClusterResources *bool, nsIncludesExcludes NamespaceIncludesExcludes) *GlobalIncludesExcludes { ret := &GlobalIncludesExcludes{ resourceFilter: *GetResourceIncludesExcludes(helper, includes, excludes), includeClusterResources: includeClusterResources, namespaceFilter: nsIncludesExcludes, helper: helper, logger: logger, } logger.Infof("Including resources: %s", ret.resourceFilter.IncludesString()) logger.Infof("Excluding resources: %s", ret.resourceFilter.ExcludesString()) return ret } type ScopeIncludesExcludes struct { namespaceScopedResourceFilter IncludesExcludes // namespace-scoped resource filter clusterScopedResourceFilter IncludesExcludes // cluster-scoped resource filter namespaceFilter NamespaceIncludesExcludes // namespace filter helper discovery.Helper logger logrus.FieldLogger } // ShouldInclude returns whether the specified resource should be included or not. // The function will check whether the resource is namespace-scoped resource first. // For namespace-scoped resource, except resources listed in excludes, other things should be included. // For cluster-scoped resource, except resources listed in excludes, only include the resource specified by the included. // It also has some exceptional checks. For namespace, as long as it's not excluded, it is involved. // If all namespace-scoped resources are included, all cluster-scoped resource are returned to get a full backup. func (ie *ScopeIncludesExcludes) ShouldInclude(typeName string) bool { _, resource, err := ie.helper.ResourceFor(schema.ParseGroupResource(typeName).WithVersion("")) if err != nil { ie.logger.Errorf("fail to get resource %s. %s", typeName, err.Error()) return false } if resource.Namespaced { if ie.namespaceScopedResourceFilter.excludes.Has("*") || ie.namespaceScopedResourceFilter.excludes.match(typeName) { return false } // len=0 means include everything return ie.namespaceScopedResourceFilter.includes.Len() == 0 || ie.namespaceScopedResourceFilter.includes.Has("*") || ie.namespaceScopedResourceFilter.includes.match(typeName) } if ie.clusterScopedResourceFilter.excludes.Has("*") || ie.clusterScopedResourceFilter.excludes.match(typeName) { return false } // when IncludedClusterScopedResources and ExcludedClusterScopedResources are not specified, // only directly back up cluster-scoped resources if we're doing a full-cluster // (all namespaces and all namespace-scoped types) backup. if len(ie.clusterScopedResourceFilter.includes.List()) == 0 && len(ie.clusterScopedResourceFilter.excludes.List()) == 0 && ie.namespaceFilter.IncludeEverything() && ie.namespaceScopedResourceFilter.IncludeEverything() { return true } // Also include namespace resource by default. return ie.clusterScopedResourceFilter.includes.Has("*") || ie.clusterScopedResourceFilter.includes.match(typeName) || typeName == kuberesource.Namespaces.String() } // ShouldExclude returns whether the resource type should be excluded or not. // For ScopeIncludesExcludes, if the resource type is specified in the exclude // list, it should be excluded. func (ie *ScopeIncludesExcludes) ShouldExclude(typeName string) bool { _, resource, err := ie.helper.ResourceFor(schema.ParseGroupResource(typeName).WithVersion("")) if err != nil { ie.logger.Errorf("fail to get resource %s. %s", typeName, err.Error()) return true } if resource.Namespaced { if ie.namespaceScopedResourceFilter.excludes.match(typeName) { return true } } else { if ie.clusterScopedResourceFilter.excludes.match(typeName) { return true } } return false } func (ie *ScopeIncludesExcludes) CombineWithPolicy(policy *resourcepolicies.IncludeExcludePolicy) { if policy == nil { return } mapFunc := scopeResourceMapFunc(ie.helper) for _, item := range policy.ExcludedNamespaceScopedResources { resolvedItem := mapFunc(item, true) if resolvedItem == "" { continue } // The existing includeExcludes in the struct has higher priority, therefore, we should only add the item to the filter // when the struct does not include this item and this item is not yet in the excludes filter. if !ie.namespaceScopedResourceFilter.includes.match(resolvedItem) && !ie.namespaceScopedResourceFilter.excludes.match(resolvedItem) { ie.namespaceScopedResourceFilter.Excludes(resolvedItem) } } for _, item := range policy.IncludedNamespaceScopedResources { resolvedItem := mapFunc(item, true) if resolvedItem == "" { continue } // The existing includeExcludes in the struct has higher priority, therefore, we should only add the item to the filter // when the struct does not exclude this item and this item is not yet in the includes filter. if !ie.namespaceScopedResourceFilter.includes.match(resolvedItem) && !ie.namespaceScopedResourceFilter.excludes.match(resolvedItem) { ie.namespaceScopedResourceFilter.Includes(resolvedItem) } } for _, item := range policy.ExcludedClusterScopedResources { resolvedItem := mapFunc(item, false) if resolvedItem == "" { continue } if !ie.clusterScopedResourceFilter.includes.match(resolvedItem) && !ie.clusterScopedResourceFilter.excludes.match(resolvedItem) { // The existing includeExcludes in the struct has higher priority, therefore, we should only add the item to the filter // when the struct does not exclude this item and this item is not yet in the includes filter. ie.clusterScopedResourceFilter.Excludes(resolvedItem) } } for _, item := range policy.IncludedClusterScopedResources { resolvedItem := mapFunc(item, false) if resolvedItem == "" { continue } if !ie.clusterScopedResourceFilter.includes.match(resolvedItem) && !ie.clusterScopedResourceFilter.excludes.match(resolvedItem) { // The existing includeExcludes in the struct has higher priority, therefore, we should only add the item to the filter // when the struct does not exclude this item and this item is not yet in the includes filter. ie.clusterScopedResourceFilter.Includes(resolvedItem) } } ie.logger.Infof("Scoped resource includes/excludes after combining with resource policy") ie.logger.Infof("Including namespace-scoped resources: %s", ie.namespaceScopedResourceFilter.IncludesString()) ie.logger.Infof("Excluding namespace-scoped resources: %s", ie.namespaceScopedResourceFilter.ExcludesString()) ie.logger.Infof("Including cluster-scoped resources: %s", ie.clusterScopedResourceFilter.GetIncludes()) ie.logger.Infof("Excluding cluster-scoped resources: %s", ie.clusterScopedResourceFilter.ExcludesString()) } func newScopeIncludesExcludes(nsIncludesExcludes NamespaceIncludesExcludes, helper discovery.Helper, logger logrus.FieldLogger) *ScopeIncludesExcludes { ret := &ScopeIncludesExcludes{ namespaceScopedResourceFilter: IncludesExcludes{ includes: newGlobStringSet(), excludes: newGlobStringSet(), }, clusterScopedResourceFilter: IncludesExcludes{ includes: newGlobStringSet(), excludes: newGlobStringSet(), }, namespaceFilter: nsIncludesExcludes, helper: helper, logger: logger, } return ret } // GetScopeResourceIncludesExcludes function is similar with GetResourceIncludesExcludes, // but it's used for scoped Includes/Excludes, and can handle both cluster-scoped and namespace-scoped resources. func GetScopeResourceIncludesExcludes(helper discovery.Helper, logger logrus.FieldLogger, namespaceIncludes, namespaceExcludes, clusterIncludes, clusterExcludes []string, nsIncludesExcludes NamespaceIncludesExcludes) *ScopeIncludesExcludes { ret := generateScopedIncludesExcludes( namespaceIncludes, namespaceExcludes, clusterIncludes, clusterExcludes, scopeResourceMapFunc(helper), nsIncludesExcludes, helper, logger, ) logger.Infof("Scoped resource includes/excludes after initialization") logger.Infof("Including namespace-scoped resources: %s", ret.namespaceScopedResourceFilter.IncludesString()) logger.Infof("Excluding namespace-scoped resources: %s", ret.namespaceScopedResourceFilter.ExcludesString()) logger.Infof("Including cluster-scoped resources: %s", ret.clusterScopedResourceFilter.GetIncludes()) logger.Infof("Excluding cluster-scoped resources: %s", ret.clusterScopedResourceFilter.ExcludesString()) return ret } func scopeResourceMapFunc(helper discovery.Helper) func(string, bool) string { return func(item string, namespaced bool) string { gvr, resource, err := helper.ResourceFor(schema.ParseGroupResource(item).WithVersion("")) if err != nil { return item } if resource.Namespaced != namespaced { return "" } gr := gvr.GroupResource() return gr.String() } } // ValidateIncludesExcludes checks provided lists of included and excluded // items to ensure they are a valid set of IncludesExcludes data. func ValidateIncludesExcludes(includesList, excludesList []string) []error { // TODO we should not allow an IncludesExcludes object to be created that // does not meet these criteria. Do a more significant refactoring to embed // this logic in object creation/modification. var errs []error includes := sets.NewString(includesList...) excludes := sets.NewString(excludesList...) if includes.Len() > 1 && includes.Has("*") { errs = append(errs, errors.New("includes list must either contain '*' only, or a non-empty list of items")) } if excludes.Has("*") { errs = append(errs, errors.New("excludes list cannot contain '*'")) } for _, itm := range excludes.List() { if includes.Has(itm) { errs = append(errs, errors.Errorf("excludes list cannot contain an item in the includes list: %v", itm)) } } return errs } // ValidateNamespaceIncludesExcludes checks provided lists of included and // excluded namespaces to ensure they are a valid set of IncludesExcludes data. func ValidateNamespaceIncludesExcludes(includesList, excludesList []string) []error { errs := ValidateIncludesExcludes(includesList, excludesList) includes := sets.NewString(includesList...) excludes := sets.NewString(excludesList...) for _, itm := range includes.List() { if nsErrs := validateNamespaceName(itm); nsErrs != nil { errs = append(errs, nsErrs...) } } for _, itm := range excludes.List() { if nsErrs := validateNamespaceName(itm); nsErrs != nil { errs = append(errs, nsErrs...) } } return errs } // ValidateScopedIncludesExcludes checks provided lists of namespace-scoped or cluster-scoped // included and excluded items to ensure they are a valid set of IncludesExcludes data. func ValidateScopedIncludesExcludes(includesList, excludesList []string) []error { var errs []error includes := sets.NewString(includesList...) excludes := sets.NewString(excludesList...) if includes.Len() > 1 && includes.Has("*") { errs = append(errs, errors.New("includes list must either contain '*' only, or a non-empty list of items")) } if excludes.Len() > 1 && excludes.Has("*") { errs = append(errs, errors.New("excludes list must either contain '*' only, or a non-empty list of items")) } if includes.Len() > 0 && excludes.Has("*") { errs = append(errs, errors.New("when exclude is '*', include cannot have value")) } for _, itm := range excludes.List() { if includes.Has(itm) { errs = append(errs, errors.Errorf("excludes list cannot contain an item in the includes list: %v", itm)) } } return errs } func validateNamespaceName(ns string) []error { var errs []error // Velero interprets empty string as "no namespace", so allow it even though // it is not a valid Kubernetes name. if ns == "" { return nil } // Validate the namespace name to ensure it is a valid wildcard pattern if err := wildcard.ValidateNamespaceName(ns); err != nil { return []error{err} } // Kubernetes does not allow wildcard characters in namespaces but Velero uses them // for glob patterns. Replace wildcard characters with valid characters to pass // Kubernetes validation. tmpNamespace := ns // Replace glob wildcard characters with valid alphanumeric characters // Note: Validation of wildcard patterns is handled by the wildcard package. tmpNamespace = strings.ReplaceAll(tmpNamespace, "*", "x") // matches any sequence tmpNamespace = strings.ReplaceAll(tmpNamespace, "?", "x") // matches single character tmpNamespace = strings.ReplaceAll(tmpNamespace, "[", "x") // character class start tmpNamespace = strings.ReplaceAll(tmpNamespace, "]", "x") // character class end if errMsgs := validation.ValidateNamespaceName(tmpNamespace, false); errMsgs != nil { for _, msg := range errMsgs { errs = append(errs, errors.Errorf("invalid namespace %q: %s", ns, msg)) } } return errs } // generateIncludesExcludes constructs an IncludesExcludes struct by taking the provided // include/exclude slices, applying the specified mapping function to each item in them, // and adding the output of the function to the new struct. If the mapping function returns // an empty string for an item, it is omitted from the result. func generateIncludesExcludes(includes, excludes []string, mapFunc func(string) string) *IncludesExcludes { res := NewIncludesExcludes() for _, item := range includes { if item == "*" { res.Includes(item) continue } key := mapFunc(item) if key == "" { continue } res.Includes(key) } for _, item := range excludes { // wildcards are invalid for excludes, // so ignore them. if item == "*" { continue } key := mapFunc(item) if key == "" { continue } res.Excludes(key) } return res } // generateScopedIncludesExcludes function is similar with generateIncludesExcludes, // but it's used for scoped Includes/Excludes. func generateScopedIncludesExcludes(namespacedIncludes, namespacedExcludes, clusterIncludes, clusterExcludes []string, mapFunc func(string, bool) string, nsIncludesExcludes NamespaceIncludesExcludes, helper discovery.Helper, logger logrus.FieldLogger) *ScopeIncludesExcludes { res := newScopeIncludesExcludes(nsIncludesExcludes, helper, logger) generateFilter(res.namespaceScopedResourceFilter.includes, namespacedIncludes, mapFunc, true) generateFilter(res.namespaceScopedResourceFilter.excludes, namespacedExcludes, mapFunc, true) generateFilter(res.clusterScopedResourceFilter.includes, clusterIncludes, mapFunc, false) generateFilter(res.clusterScopedResourceFilter.excludes, clusterExcludes, mapFunc, false) return res } func generateFilter(filter globStringSet, resources []string, mapFunc func(string, bool) string, namespaced bool) { for _, item := range resources { if item == "*" { filter.Insert(item) continue } key := mapFunc(item, namespaced) if key == "" { continue } filter.Insert(key) } } // UseOldResourceFilters checks whether to use old resource filters (IncludeClusterResources, // IncludedResources and ExcludedResources), depending the backup's filters setting. // New filters are IncludedClusterScopedResources, ExcludedClusterScopedResources, // IncludedNamespaceScopedResources and ExcludedNamespaceScopedResources. // If all resource filters are none, it is treated as using new parameter filters. func UseOldResourceFilters(backupSpec velerov1api.BackupSpec) bool { if backupSpec.IncludeClusterResources != nil || len(backupSpec.IncludedResources) > 0 || len(backupSpec.ExcludedResources) > 0 { return true } return false } ================================================ FILE: pkg/util/collections/includes_excludes_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package collections import ( "testing" "github.com/vmware-tanzu/velero/internal/resourcepolicies" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/sets" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/test" ) func TestShouldInclude(t *testing.T) { tests := []struct { name string includes []string excludes []string item string want bool }{ { name: "empty string should include every item", item: "foo", want: true, }, { name: "include * should include every item", includes: []string{"*"}, item: "foo", want: true, }, { name: "item in includes list should include item", includes: []string{"foo", "bar", "baz"}, item: "foo", want: true, }, { name: "item not in includes list should not include item", includes: []string{"foo", "baz"}, item: "bar", want: false, }, { name: "include *, excluded item should not include item", includes: []string{"*"}, excludes: []string{"foo"}, item: "foo", want: false, }, { name: "include *, exclude foo, bar should be included", includes: []string{"*"}, excludes: []string{"foo"}, item: "bar", want: true, }, { name: "an item both included and excluded should not be included", includes: []string{"foo"}, excludes: []string{"foo"}, item: "foo", want: false, }, { name: "wildcard should include item", includes: []string{"*.bar"}, item: "foo.bar", want: true, }, { name: "wildcard mismatch should not include item", includes: []string{"*.bar"}, item: "bar.foo", want: false, }, { name: "wildcard exclude should not include item", includes: []string{"*"}, excludes: []string{"*.bar"}, item: "foo.bar", want: false, }, { name: "wildcard mismatch should include item", includes: []string{"*"}, excludes: []string{"*.bar"}, item: "bar.foo", want: true, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { includesExcludes := NewIncludesExcludes().Includes(tc.includes...).Excludes(tc.excludes...) if got := includesExcludes.ShouldInclude((tc.item)); got != tc.want { t.Errorf("want %t, got %t", tc.want, got) } }) } } func TestValidateIncludesExcludes(t *testing.T) { tests := []struct { name string includes []string excludes []string want []error }{ { name: "empty includes (everything) is allowed", includes: []string{}, }, { name: "include everything", includes: []string{"*"}, }, { name: "include everything not allowed with other includes", includes: []string{"*", "foo"}, want: []error{errors.New("includes list must either contain '*' only, or a non-empty list of items")}, }, { name: "exclude everything not allowed", includes: []string{"foo"}, excludes: []string{"*"}, want: []error{errors.New("excludes list cannot contain '*'")}, }, { name: "excludes cannot contain items in includes", includes: []string{"foo", "bar"}, excludes: []string{"bar"}, want: []error{errors.New("excludes list cannot contain an item in the includes list: bar")}, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { errs := ValidateIncludesExcludes(tc.includes, tc.excludes) require.Len(t, errs, len(tc.want)) for i := 0; i < len(tc.want); i++ { assert.Equal(t, tc.want[i].Error(), errs[i].Error()) } }) } } func TestIncludeExcludeString(t *testing.T) { tests := []struct { name string includes []string excludes []string wantIncludes string wantExcludes string }{ { name: "unspecified includes/excludes should return '*'/''", includes: nil, excludes: nil, wantIncludes: "*", wantExcludes: "", }, { name: "specific resources should result in sorted joined string", includes: []string{"foo", "bar"}, excludes: []string{"baz", "xyz"}, wantIncludes: "bar, foo", wantExcludes: "baz, xyz", }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { includesExcludes := NewIncludesExcludes().Includes(tc.includes...).Excludes(tc.excludes...) assert.Equal(t, tc.wantIncludes, includesExcludes.IncludesString()) assert.Equal(t, tc.wantExcludes, includesExcludes.ExcludesString()) }) } } func TestValidateNamespaceIncludesExcludes(t *testing.T) { tests := []struct { name string includes []string excludes []string wantErr bool }{ { name: "empty slice doesn't return error", includes: []string{}, wantErr: false, }, { name: "asterisk by itself is valid", includes: []string{"*"}, wantErr: false, }, { name: "alphanumeric names with optional dash inside are valid", includes: []string{"foobar", "bar-321", "foo123bar"}, excludes: []string{"123bar", "barfoo", "foo-321", "bar123foo"}, wantErr: false, }, { name: "not starting or ending with an alphanumeric character is invalid", includes: []string{"-123foo"}, excludes: []string{"foo321-", "foo321-"}, wantErr: true, }, { name: "special characters in name is invalid", includes: []string{"foo?", "foo.bar", "bar_321"}, excludes: []string{"$foo", "foo>bar", "bar=321"}, wantErr: true, }, { name: "empty includes (everything) is valid", includes: []string{}, wantErr: false, }, { name: "empty string includes is valid (includes nothing)", includes: []string{""}, wantErr: false, }, { name: "empty string excludes is valid (excludes nothing)", excludes: []string{""}, wantErr: false, }, { name: "include everything using asterisk is valid", includes: []string{"*"}, wantErr: false, }, { name: "excludes can contain wildcard", includes: []string{"foo", "bar"}, excludes: []string{"nginx-ingress-*", "*-bar", "*-ingress-*"}, wantErr: false, }, { name: "includes can contain wildcard", includes: []string{"*-foo", "kube-*", "*kube*"}, excludes: []string{"bar"}, wantErr: false, }, { name: "include everything not allowed with other includes", includes: []string{"*", "foo"}, wantErr: true, }, { name: "exclude everything not allowed", includes: []string{"foo"}, excludes: []string{"*"}, wantErr: true, }, { name: "excludes cannot contain items in includes", includes: []string{"foo", "bar"}, excludes: []string{"bar"}, wantErr: true, }, { name: "glob characters in includes should not error", includes: []string{"kube-*", "test-?", "ns-[0-9]"}, excludes: []string{}, wantErr: false, }, { name: "glob characters in excludes should not error", includes: []string{"default"}, excludes: []string{"test-*", "app-?", "ns-[1-5]"}, wantErr: false, }, { name: "character class in includes should not error", includes: []string{"ns-[abc]", "test-[0-9]"}, excludes: []string{}, wantErr: false, }, { name: "mixed glob patterns should not error", includes: []string{"kube-*", "test-?"}, excludes: []string{"*-test", "debug-[0-9]"}, wantErr: false, }, { name: "pipe character in includes should error", includes: []string{"namespace|other"}, excludes: []string{}, wantErr: true, }, { name: "parentheses in includes should error", includes: []string{"namespace(prod)", "test-(dev)"}, excludes: []string{}, wantErr: true, }, { name: "exclamation mark in includes should error", includes: []string{"!namespace", "test!"}, excludes: []string{}, wantErr: true, }, { name: "unsupported characters in excludes should error", includes: []string{"default"}, excludes: []string{"test|prod", "app(staging)"}, wantErr: true, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { errs := ValidateNamespaceIncludesExcludes(tc.includes, tc.excludes) if tc.wantErr && len(errs) == 0 { t.Errorf("%s: wanted errors but got none", tc.name) } if !tc.wantErr && len(errs) != 0 { t.Errorf("%s: wanted no errors but got: %v", tc.name, errs) } }) } } func TestValidateScopedIncludesExcludes(t *testing.T) { tests := []struct { name string includes []string excludes []string wantErr []error }{ // includes testing { name: "empty includes is valid", includes: []string{}, wantErr: []error{}, }, { name: "asterisk includes is valid", includes: []string{"*"}, wantErr: []error{}, }, { name: "include everything not allowed with other includes", includes: []string{"*", "foo"}, wantErr: []error{errors.New("includes list must either contain '*' only, or a non-empty list of items")}, }, // excludes testing { name: "empty excludes is valid", excludes: []string{}, wantErr: []error{}, }, { name: "asterisk excludes is valid", excludes: []string{"*"}, wantErr: []error{}, }, { name: "exclude everything not allowed with other excludes", excludes: []string{"*", "foo"}, wantErr: []error{errors.New("excludes list must either contain '*' only, or a non-empty list of items")}, }, // includes and excludes combination testing { name: "asterisk excludes doesn't work with non-empty includes", includes: []string{"foo"}, excludes: []string{"*"}, wantErr: []error{errors.New("when exclude is '*', include cannot have value")}, }, { name: "excludes cannot contain items in includes", includes: []string{"foo", "bar"}, excludes: []string{"bar"}, wantErr: []error{errors.New("excludes list cannot contain an item in the includes list: bar")}, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { errs := ValidateScopedIncludesExcludes(tc.includes, tc.excludes) require.Len(t, errs, len(tc.wantErr)) for i := 0; i < len(tc.wantErr); i++ { assert.Equal(t, tc.wantErr[i].Error(), errs[i].Error()) } }) } } func TestNamespaceScopedShouldInclude(t *testing.T) { tests := []struct { name string namespaceScopedIncludes []string namespaceScopedExcludes []string item string want bool apiResources []*test.APIResource }{ { name: "empty string should include every item", item: "pods", want: true, apiResources: []*test.APIResource{ test.Pods(), }, }, { name: "include * should include every item", namespaceScopedIncludes: []string{"*"}, item: "pods", want: true, apiResources: []*test.APIResource{ test.Pods(), }, }, { name: "item in includes list should include item", namespaceScopedIncludes: []string{"foo", "bar", "pods"}, item: "pods", want: true, apiResources: []*test.APIResource{ test.Pods(), }, }, { name: "item not in includes list should not include item", namespaceScopedIncludes: []string{"foo", "baz"}, item: "pods", want: false, apiResources: []*test.APIResource{ test.Pods(), }, }, { name: "include *, excluded item should not include item", namespaceScopedIncludes: []string{"*"}, namespaceScopedExcludes: []string{"pods"}, item: "pods", want: false, apiResources: []*test.APIResource{ test.Pods(), }, }, { name: "include *, exclude foo, bar should be included", namespaceScopedIncludes: []string{"*"}, namespaceScopedExcludes: []string{"foo"}, item: "pods", want: true, apiResources: []*test.APIResource{ test.Pods(), }, }, { name: "an item both included and excluded should not be included", namespaceScopedIncludes: []string{"pods"}, namespaceScopedExcludes: []string{"pods"}, item: "pods", want: false, apiResources: []*test.APIResource{ test.Pods(), }, }, { name: "wildcard should include item", namespaceScopedIncludes: []string{"*s"}, item: "pods", want: true, apiResources: []*test.APIResource{ test.Pods(), }, }, { name: "wildcard mismatch should not include item", namespaceScopedIncludes: []string{"*.bar"}, item: "pods", want: false, apiResources: []*test.APIResource{ test.Pods(), }, }, { name: "exclude * should include nothing", namespaceScopedExcludes: []string{"*"}, item: "pods", want: false, apiResources: []*test.APIResource{ test.Pods(), }, }, { name: "wildcard exclude should not include item", namespaceScopedIncludes: []string{"*"}, namespaceScopedExcludes: []string{"*s"}, item: "pods", want: false, apiResources: []*test.APIResource{ test.Pods(), }, }, { name: "wildcard exclude mismatch should include item", namespaceScopedExcludes: []string{"*.bar"}, item: "pods", want: true, apiResources: []*test.APIResource{ test.Pods(), }, }, { name: "resource cannot be found by discovery client should not be include", item: "pods", want: false, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { discoveryHelper := setupDiscoveryClientWithResources(tc.apiResources) logger := logrus.StandardLogger() scopeIncludesExcludes := GetScopeResourceIncludesExcludes(discoveryHelper, logger, tc.namespaceScopedIncludes, tc.namespaceScopedExcludes, []string{}, []string{}, *NewNamespaceIncludesExcludes()) if got := scopeIncludesExcludes.ShouldInclude((tc.item)); got != tc.want { t.Errorf("want %t, got %t", tc.want, got) } }) } } func TestClusterScopedShouldInclude(t *testing.T) { tests := []struct { name string clusterScopedIncludes []string clusterScopedExcludes []string nsIncludes []string item string want bool apiResources []*test.APIResource }{ { name: "empty string should include nothing", nsIncludes: []string{"default"}, item: "persistentvolumes", want: false, apiResources: []*test.APIResource{ test.PVs(), }, }, { name: "include * should include every item", clusterScopedIncludes: []string{"*"}, item: "persistentvolumes", want: true, apiResources: []*test.APIResource{ test.PVs(), }, }, { name: "item in includes list should include item", clusterScopedIncludes: []string{"namespaces", "bar", "baz"}, item: "namespaces", want: true, apiResources: []*test.APIResource{ test.Namespaces(), }, }, { name: "item not in includes list should not include item", clusterScopedIncludes: []string{"foo", "baz"}, nsIncludes: []string{"default"}, item: "persistentvolumes", want: false, apiResources: []*test.APIResource{ test.PVs(), }, }, { name: "include *, excluded item should not include item", clusterScopedIncludes: []string{"*"}, clusterScopedExcludes: []string{"namespaces"}, item: "namespaces", want: false, apiResources: []*test.APIResource{ test.Namespaces(), }, }, { name: "include *, exclude foo, bar should be included", clusterScopedIncludes: []string{"*"}, clusterScopedExcludes: []string{"foo"}, item: "namespaces", want: true, apiResources: []*test.APIResource{ test.Namespaces(), }, }, { name: "an item both included and excluded should not be included", clusterScopedIncludes: []string{"namespaces"}, clusterScopedExcludes: []string{"namespaces"}, item: "namespaces", want: false, apiResources: []*test.APIResource{ test.Namespaces(), }, }, { name: "wildcard should include item", clusterScopedIncludes: []string{"*spaces"}, item: "namespaces", want: true, apiResources: []*test.APIResource{ test.Namespaces(), }, }, { name: "wildcard mismatch should not include item", clusterScopedIncludes: []string{"*.bar"}, nsIncludes: []string{"default"}, item: "persistentvolumes", want: false, apiResources: []*test.APIResource{ test.PVs(), }, }, { name: "exclude * should include nothing", clusterScopedExcludes: []string{"*"}, item: "namespaces", want: false, apiResources: []*test.APIResource{ test.Namespaces(), }, }, { name: "wildcard exclude should not include item", clusterScopedIncludes: []string{"*"}, clusterScopedExcludes: []string{"*spaces"}, item: "namespaces", want: false, apiResources: []*test.APIResource{ test.Namespaces(), }, }, { name: "wildcard exclude mismatch should not include item", clusterScopedExcludes: []string{"*spaces"}, item: "namespaces", want: false, apiResources: []*test.APIResource{ test.Namespaces(), }, }, { name: "resource cannot be found by discovery client should not be include", item: "namespaces", want: false, }, { name: "even namespaces is not in the include list, it should also be involved.", clusterScopedIncludes: []string{"foo", "baz"}, item: "namespaces", want: true, apiResources: []*test.APIResource{ test.Namespaces(), }, }, { name: "When all namespaces and namespace scope resources are included, cluster resource should be included.", clusterScopedIncludes: []string{}, nsIncludes: []string{"*"}, item: "persistentvolumes", want: true, apiResources: []*test.APIResource{ test.PVs(), }, }, { name: "When all namespaces and namespace scope resources are included, but cluster resource is excluded.", clusterScopedIncludes: []string{}, clusterScopedExcludes: []string{"persistentvolumes"}, nsIncludes: []string{"*"}, item: "persistentvolumes", want: false, apiResources: []*test.APIResource{ test.PVs(), }, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { discoveryHelper := setupDiscoveryClientWithResources(tc.apiResources) logger := logrus.StandardLogger() nsIncludeExclude := NewNamespaceIncludesExcludes().Includes(tc.nsIncludes...) scopeIncludesExcludes := GetScopeResourceIncludesExcludes(discoveryHelper, logger, []string{}, []string{}, tc.clusterScopedIncludes, tc.clusterScopedExcludes, *nsIncludeExclude) if got := scopeIncludesExcludes.ShouldInclude((tc.item)); got != tc.want { t.Errorf("want %t, got %t", tc.want, got) } }) } } func TestGetScopedResourceIncludesExcludes(t *testing.T) { tests := []struct { name string namespaceScopedIncludes []string namespaceScopedExcludes []string clusterScopedIncludes []string clusterScopedExcludes []string expectedNamespaceScopedIncludes []string expectedNamespaceScopedExcludes []string expectedClusterScopedIncludes []string expectedClusterScopedExcludes []string apiResources []*test.APIResource }{ { name: "only include namespace-scoped resources in IncludesExcludes", namespaceScopedIncludes: []string{"deployments.apps", "persistentvolumes"}, namespaceScopedExcludes: []string{"pods", "persistentvolumes"}, expectedNamespaceScopedIncludes: []string{"deployments.apps"}, expectedNamespaceScopedExcludes: []string{"pods"}, expectedClusterScopedIncludes: []string{}, expectedClusterScopedExcludes: []string{}, apiResources: []*test.APIResource{ test.Deployments(), test.PVs(), test.Pods(), }, }, { name: "only include cluster-scoped resources in IncludesExcludes", clusterScopedIncludes: []string{"deployments.apps", "persistentvolumes"}, clusterScopedExcludes: []string{"pods", "persistentvolumes"}, expectedNamespaceScopedIncludes: []string{}, expectedNamespaceScopedExcludes: []string{}, expectedClusterScopedIncludes: []string{"persistentvolumes"}, expectedClusterScopedExcludes: []string{"persistentvolumes"}, apiResources: []*test.APIResource{ test.Deployments(), test.PVs(), test.Pods(), }, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { logger := logrus.StandardLogger() nsIncludeExclude := NewNamespaceIncludesExcludes() resources := GetScopeResourceIncludesExcludes(setupDiscoveryClientWithResources(tc.apiResources), logger, tc.namespaceScopedIncludes, tc.namespaceScopedExcludes, tc.clusterScopedIncludes, tc.clusterScopedExcludes, *nsIncludeExclude) assert.Equal(t, tc.expectedNamespaceScopedIncludes, resources.namespaceScopedResourceFilter.includes.List()) assert.Equal(t, tc.expectedNamespaceScopedExcludes, resources.namespaceScopedResourceFilter.excludes.List()) assert.Equal(t, tc.expectedClusterScopedIncludes, resources.clusterScopedResourceFilter.includes.List()) assert.Equal(t, tc.expectedClusterScopedExcludes, resources.clusterScopedResourceFilter.excludes.List()) }) } } func TestScopeIncludesExcludes_CombineWithPolicy(t *testing.T) { apiResources := []*test.APIResource{test.Deployments(), test.Pods(), test.ConfigMaps(), test.Secrets(), test.PVs(), test.CRDs(), test.ServiceAccounts()} tests := []struct { name string namespaceScopedIncludes []string namespaceScopedExcludes []string clusterScopedIncludes []string clusterScopedExcludes []string policy *resourcepolicies.IncludeExcludePolicy verify func(sie ScopeIncludesExcludes) bool }{ { name: "When policy is nil, the original includes excludes filters should not change", namespaceScopedIncludes: []string{"deployments", "pods"}, namespaceScopedExcludes: []string{"configmaps"}, clusterScopedIncludes: []string{"persistentvolumes"}, clusterScopedExcludes: []string{"crds"}, policy: nil, verify: func(sie ScopeIncludesExcludes) bool { return sie.clusterScopedResourceFilter.ShouldInclude("persistentvolumes") && !sie.clusterScopedResourceFilter.ShouldInclude("crds") && sie.namespaceScopedResourceFilter.ShouldInclude("deployments") && !sie.namespaceScopedResourceFilter.ShouldInclude("configmaps") }, }, { name: "policy includes excludes should be merged to the original includes excludes when there's no conflict", namespaceScopedIncludes: []string{"pods"}, namespaceScopedExcludes: []string{"configmaps"}, clusterScopedIncludes: []string{}, clusterScopedExcludes: []string{"crds"}, policy: &resourcepolicies.IncludeExcludePolicy{ IncludedNamespaceScopedResources: []string{"deployments"}, ExcludedNamespaceScopedResources: []string{"secrets"}, IncludedClusterScopedResources: []string{"persistentvolumes"}, ExcludedClusterScopedResources: []string{}, }, verify: func(sie ScopeIncludesExcludes) bool { return sie.clusterScopedResourceFilter.ShouldInclude("persistentvolumes") && !sie.clusterScopedResourceFilter.ShouldInclude("crds") && sie.namespaceScopedResourceFilter.ShouldInclude("deployments") && !sie.namespaceScopedResourceFilter.ShouldInclude("configmaps") && !sie.namespaceScopedResourceFilter.ShouldInclude("secrets") }, }, { name: "when there are conflicts, the existing includes excludes filters have higher priorities", namespaceScopedIncludes: []string{"pods", "deployments"}, namespaceScopedExcludes: []string{"configmaps"}, clusterScopedIncludes: []string{"crds"}, clusterScopedExcludes: []string{"persistentvolumes"}, policy: &resourcepolicies.IncludeExcludePolicy{ IncludedNamespaceScopedResources: []string{"configmaps"}, ExcludedNamespaceScopedResources: []string{"pods", "secrets"}, IncludedClusterScopedResources: []string{"persistentvolumes"}, ExcludedClusterScopedResources: []string{"crds"}, }, verify: func(sie ScopeIncludesExcludes) bool { return sie.clusterScopedResourceFilter.ShouldInclude("crds") && !sie.clusterScopedResourceFilter.ShouldInclude("persistentvolumes") && sie.namespaceScopedResourceFilter.ShouldInclude("pods") && !sie.namespaceScopedResourceFilter.ShouldInclude("configmaps") && !sie.namespaceScopedResourceFilter.ShouldInclude("secrets") }, }, { name: "verify the case when there's '*' in the original include filter", namespaceScopedIncludes: []string{"*"}, namespaceScopedExcludes: []string{}, clusterScopedIncludes: []string{}, clusterScopedExcludes: []string{}, policy: &resourcepolicies.IncludeExcludePolicy{ IncludedNamespaceScopedResources: []string{"deployments", "pods"}, ExcludedNamespaceScopedResources: []string{"configmaps", "secrets"}, IncludedClusterScopedResources: []string{}, ExcludedClusterScopedResources: []string{}, }, verify: func(sie ScopeIncludesExcludes) bool { return sie.namespaceScopedResourceFilter.ShouldInclude("configmaps") && sie.namespaceScopedResourceFilter.ShouldInclude("secrets") }, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { logger := logrus.StandardLogger() discoveryHelper := setupDiscoveryClientWithResources(apiResources) sie := GetScopeResourceIncludesExcludes(discoveryHelper, logger, tc.namespaceScopedIncludes, tc.namespaceScopedExcludes, tc.clusterScopedIncludes, tc.clusterScopedExcludes, *NewNamespaceIncludesExcludes()) sie.CombineWithPolicy(tc.policy) assert.True(t, tc.verify(*sie)) }) } } func TestUseOldResourceFilters(t *testing.T) { tests := []struct { name string backup velerov1api.Backup useOldResourceFilters bool }{ { name: "backup with no filters should use new filters", backup: *defaultBackup().Result(), useOldResourceFilters: false, }, { name: "backup with only old filters should use old filters", backup: *defaultBackup().IncludeClusterResources(true).Result(), useOldResourceFilters: true, }, { name: "backup with only new filters should use new filters", backup: *defaultBackup().IncludedClusterScopedResources("StorageClass").Result(), useOldResourceFilters: false, }, { // This case should not happen in Velero workflow, because filter validation not old and new // filters used together. So this is only used for UT checking, and I assume old filters // have higher priority, because old parameter should be the default one. name: "backup with both old and new filters should use old filters", backup: *defaultBackup().IncludeClusterResources(true).IncludedClusterScopedResources("StorageClass").Result(), useOldResourceFilters: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { assert.Equal(t, test.useOldResourceFilters, UseOldResourceFilters(test.backup.Spec)) }) } } func defaultBackup() *builder.BackupBuilder { return builder.ForBackup(velerov1api.DefaultNamespace, "backup-1").DefaultVolumesToFsBackup(false) } func TestShouldExcluded(t *testing.T) { falseBoolean := false trueBoolean := true tests := []struct { name string clusterIncludes []string clusterExcludes []string includeClusterResources *bool filterType string resourceName string apiResources []*test.APIResource resourceIsExcluded bool }{ { name: "GlobalResourceIncludesExcludes: filters are all default", clusterIncludes: []string{}, clusterExcludes: []string{}, includeClusterResources: nil, filterType: "global", resourceName: "persistentvolumes", apiResources: []*test.APIResource{ test.PVs(), }, resourceIsExcluded: false, }, { name: "GlobalResourceIncludesExcludes: IncludeClusterResources is set to true", clusterIncludes: []string{}, clusterExcludes: []string{}, includeClusterResources: &trueBoolean, filterType: "global", resourceName: "persistentvolumes", apiResources: []*test.APIResource{ test.PVs(), }, resourceIsExcluded: false, }, { name: "GlobalResourceIncludesExcludes: IncludeClusterResources is set to false", clusterIncludes: []string{"persistentvolumes"}, clusterExcludes: []string{}, includeClusterResources: &falseBoolean, filterType: "global", resourceName: "persistentvolumes", apiResources: []*test.APIResource{ test.PVs(), }, resourceIsExcluded: true, }, { name: "GlobalResourceIncludesExcludes: resource is in the include list", clusterIncludes: []string{"persistentvolumes"}, clusterExcludes: []string{}, includeClusterResources: nil, filterType: "global", resourceName: "persistentvolumes", apiResources: []*test.APIResource{ test.PVs(), }, resourceIsExcluded: false, }, { name: "ScopeResourceIncludesExcludes: resource is in the include list", clusterIncludes: []string{"persistentvolumes"}, clusterExcludes: []string{}, filterType: "scope", resourceName: "persistentvolumes", apiResources: []*test.APIResource{ test.PVs(), }, resourceIsExcluded: false, }, { name: "ScopeResourceIncludesExcludes: filters are all default", clusterIncludes: []string{}, clusterExcludes: []string{}, filterType: "scope", resourceName: "persistentvolumes", apiResources: []*test.APIResource{ test.PVs(), }, resourceIsExcluded: false, }, { name: "ScopeResourceIncludesExcludes: resource is not in the exclude list", clusterIncludes: []string{}, clusterExcludes: []string{"namespaces"}, filterType: "scope", resourceName: "persistentvolumes", apiResources: []*test.APIResource{ test.PVs(), }, resourceIsExcluded: false, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { logger := logrus.StandardLogger() var ie IncludesExcludesInterface if tc.filterType == "global" { ie = GetGlobalResourceIncludesExcludes(setupDiscoveryClientWithResources(tc.apiResources), logger, tc.clusterIncludes, tc.clusterExcludes, tc.includeClusterResources, *NewNamespaceIncludesExcludes()) } else if tc.filterType == "scope" { ie = GetScopeResourceIncludesExcludes(setupDiscoveryClientWithResources(tc.apiResources), logger, []string{}, []string{}, tc.clusterIncludes, tc.clusterExcludes, *NewNamespaceIncludesExcludes()) } assert.Equal(t, tc.resourceIsExcluded, ie.ShouldExclude(tc.resourceName)) }) } } func TestExpandIncludesExcludes(t *testing.T) { tests := []struct { name string includes []string excludes []string activeNamespaces []string expectedIncludes []string expectedExcludes []string expectedWildcardExpanded bool expectError bool }{ { name: "no wildcards - should not expand", includes: []string{"default", "kube-system"}, excludes: []string{"kube-public"}, activeNamespaces: []string{"default", "kube-system", "kube-public", "test"}, expectedIncludes: []string{"default", "kube-system"}, expectedExcludes: []string{"kube-public"}, expectedWildcardExpanded: false, expectError: false, }, { name: "asterisk alone - should not expand", includes: []string{"*"}, excludes: []string{}, activeNamespaces: []string{"default", "kube-system", "test"}, expectedIncludes: []string{"*"}, expectedExcludes: []string{}, expectedWildcardExpanded: false, expectError: false, }, { name: "wildcard in includes - should expand", includes: []string{"kube-*"}, excludes: []string{}, activeNamespaces: []string{"default", "kube-system", "kube-public", "test"}, expectedIncludes: []string{"kube-system", "kube-public"}, expectedExcludes: []string{}, expectedWildcardExpanded: true, expectError: false, }, { name: "wildcard in excludes - should expand", includes: []string{"default"}, excludes: []string{"*-test"}, activeNamespaces: []string{"default", "kube-test", "app-test", "prod"}, expectedIncludes: []string{"default"}, expectedExcludes: []string{"kube-test", "app-test"}, expectedWildcardExpanded: true, expectError: false, }, { name: "wildcards in both includes and excludes", includes: []string{"kube-*", "app-*"}, excludes: []string{"*-test"}, activeNamespaces: []string{"kube-system", "kube-test", "app-prod", "app-test", "default"}, expectedIncludes: []string{"kube-system", "kube-test", "app-prod", "app-test"}, expectedExcludes: []string{"kube-test", "app-test"}, expectedWildcardExpanded: true, expectError: false, }, { name: "wildcard pattern matches nothing", includes: []string{"nonexistent-*"}, excludes: []string{}, activeNamespaces: []string{"default", "kube-system"}, expectedIncludes: []string{}, expectedExcludes: []string{}, expectedWildcardExpanded: true, expectError: false, }, { name: "mix of wildcards and non-wildcards in includes", includes: []string{"default", "kube-*"}, excludes: []string{}, activeNamespaces: []string{"default", "kube-system", "kube-public", "test"}, expectedIncludes: []string{"default", "kube-system", "kube-public"}, expectedExcludes: []string{}, expectedWildcardExpanded: true, expectError: false, }, { name: "question mark wildcard", includes: []string{"test-?"}, excludes: []string{}, activeNamespaces: []string{"test-1", "test-2", "test-10", "default"}, expectedIncludes: []string{"test-1", "test-2"}, expectedExcludes: []string{}, expectedWildcardExpanded: true, expectError: false, }, { name: "empty activeNamespaces with wildcards", includes: []string{"kube-*"}, excludes: []string{}, activeNamespaces: []string{}, expectedIncludes: []string{}, expectedExcludes: []string{}, expectedWildcardExpanded: true, expectError: false, }, { name: "invalid wildcard pattern - consecutive asterisks", includes: []string{"kube-**"}, excludes: []string{}, activeNamespaces: []string{"default"}, expectedIncludes: []string{"kube-**"}, expectedExcludes: []string{}, expectedWildcardExpanded: false, expectError: true, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { nie := NewNamespaceIncludesExcludes(). ActiveNamespaces(tc.activeNamespaces). Includes(tc.includes...). Excludes(tc.excludes...) err := nie.ExpandIncludesExcludes() if tc.expectError { assert.Error(t, err) return } require.NoError(t, err) assert.Equal(t, tc.expectedWildcardExpanded, nie.IsWildcardExpanded()) // Check includes - convert to sets for order-independent comparison actualIncludes := sets.NewString(nie.GetIncludes()...) expectedIncludes := sets.NewString(tc.expectedIncludes...) assert.True(t, actualIncludes.Equal(expectedIncludes), "includes mismatch: expected %v, got %v", tc.expectedIncludes, nie.GetIncludes()) // Check excludes actualExcludes := sets.NewString(nie.GetExcludes()...) expectedExcludes := sets.NewString(tc.expectedExcludes...) assert.True(t, actualExcludes.Equal(expectedExcludes), "excludes mismatch: expected %v, got %v", tc.expectedExcludes, nie.GetExcludes()) }) } } func TestResolveNamespaceList(t *testing.T) { tests := []struct { name string includes []string excludes []string activeNamespaces []string expectedNamespaces []string preExpandWildcards bool }{ { name: "no includes/excludes - all active namespaces", includes: []string{}, excludes: []string{}, activeNamespaces: []string{"default", "kube-system", "test"}, expectedNamespaces: []string{"default", "kube-system", "test"}, }, { name: "asterisk includes - all active namespaces", includes: []string{"*"}, excludes: []string{}, activeNamespaces: []string{"default", "kube-system", "test"}, expectedNamespaces: []string{"default", "kube-system", "test"}, }, { name: "specific includes - only those namespaces", includes: []string{"default", "test"}, excludes: []string{}, activeNamespaces: []string{"default", "kube-system", "test"}, expectedNamespaces: []string{"default", "test"}, }, { name: "includes with excludes", includes: []string{"*"}, excludes: []string{"kube-system"}, activeNamespaces: []string{"default", "kube-system", "test"}, expectedNamespaces: []string{"default", "test"}, }, { name: "wildcard includes - expands and filters", includes: []string{"kube-*"}, excludes: []string{}, activeNamespaces: []string{"default", "kube-system", "kube-public", "test"}, expectedNamespaces: []string{"kube-system", "kube-public"}, }, { name: "wildcard includes with wildcard excludes", includes: []string{"app-*"}, excludes: []string{"*-test"}, activeNamespaces: []string{"app-prod", "app-dev", "app-test", "default"}, expectedNamespaces: []string{"app-prod", "app-dev"}, }, { name: "wildcard matches nothing - empty result", includes: []string{"nonexistent-*"}, excludes: []string{}, activeNamespaces: []string{"default", "kube-system"}, expectedNamespaces: []string{}, }, { name: "empty active namespaces", includes: []string{"*"}, excludes: []string{}, activeNamespaces: []string{}, expectedNamespaces: []string{}, }, { name: "includes namespace not in active namespaces", includes: []string{"default", "nonexistent"}, excludes: []string{}, activeNamespaces: []string{"default", "test"}, expectedNamespaces: []string{"default"}, }, { name: "excludes all namespaces from includes", includes: []string{"default", "test"}, excludes: []string{"default", "test"}, activeNamespaces: []string{"default", "test", "prod"}, expectedNamespaces: []string{}, }, { name: "pre-expanded wildcards - should not expand again", includes: []string{"kube-*"}, excludes: []string{}, activeNamespaces: []string{"default", "kube-system", "kube-public"}, expectedNamespaces: []string{"kube-system", "kube-public"}, preExpandWildcards: true, }, { name: "question mark wildcard pattern", includes: []string{"ns-?"}, excludes: []string{}, activeNamespaces: []string{"ns-1", "ns-2", "ns-10", "default"}, expectedNamespaces: []string{"ns-1", "ns-2"}, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { nie := NewNamespaceIncludesExcludes(). ActiveNamespaces(tc.activeNamespaces). Includes(tc.includes...). Excludes(tc.excludes...) // Pre-expand wildcards if requested if tc.preExpandWildcards { err := nie.ExpandIncludesExcludes() require.NoError(t, err) } namespaces, err := nie.ResolveNamespaceList() require.NoError(t, err) // Convert to sets for order-independent comparison actualNs := sets.NewString(namespaces...) expectedNs := sets.NewString(tc.expectedNamespaces...) assert.True(t, actualNs.Equal(expectedNs), "namespaces mismatch: expected %v, got %v", tc.expectedNamespaces, namespaces) }) } } func TestResolveNamespaceListError(t *testing.T) { tests := []struct { name string includes []string excludes []string activeNamespaces []string }{ { name: "invalid wildcard pattern in includes", includes: []string{"kube-**"}, excludes: []string{}, activeNamespaces: []string{"default"}, }, { name: "invalid wildcard pattern in excludes", includes: []string{"default"}, excludes: []string{"test-**"}, activeNamespaces: []string{"default"}, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { nie := NewNamespaceIncludesExcludes(). ActiveNamespaces(tc.activeNamespaces). Includes(tc.includes...). Excludes(tc.excludes...) _, err := nie.ResolveNamespaceList() assert.Error(t, err) }) } } func TestNamespaceIncludesExcludesShouldIncludeAfterWildcardExpansion(t *testing.T) { tests := []struct { name string includes []string excludes []string activeNamespaces []string testNamespace string expectedResult bool }{ { name: "wildcard expanded to empty includes - should not include anything", includes: []string{"nonexistent-*"}, excludes: []string{}, activeNamespaces: []string{"default", "kube-system"}, testNamespace: "default", expectedResult: false, }, { name: "wildcard expanded with matches - should include matched namespace", includes: []string{"kube-*"}, excludes: []string{}, activeNamespaces: []string{"default", "kube-system", "kube-public"}, testNamespace: "kube-system", expectedResult: true, }, { name: "wildcard expanded with matches - should not include unmatched namespace", includes: []string{"kube-*"}, excludes: []string{}, activeNamespaces: []string{"default", "kube-system", "kube-public"}, testNamespace: "default", expectedResult: false, }, { name: "no wildcard expansion - empty includes means include all", includes: []string{}, excludes: []string{}, activeNamespaces: []string{"default", "kube-system"}, testNamespace: "default", expectedResult: true, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { nie := NewNamespaceIncludesExcludes(). ActiveNamespaces(tc.activeNamespaces). Includes(tc.includes...). Excludes(tc.excludes...) err := nie.ExpandIncludesExcludes() require.NoError(t, err) result := nie.ShouldInclude(tc.testNamespace) assert.Equal(t, tc.expectedResult, result) }) } } func setupDiscoveryClientWithResources(APIResources []*test.APIResource) *test.FakeDiscoveryHelper { resourcesMap := make(map[schema.GroupVersionResource]schema.GroupVersionResource) resourceList := make([]*metav1.APIResourceList, 0) for _, resource := range APIResources { gvr := schema.GroupVersionResource{ Group: resource.Group, Version: resource.Version, Resource: resource.Name, } resourcesMap[gvr] = gvr resourceList = append(resourceList, &metav1.APIResourceList{ GroupVersion: gvr.GroupVersion().String(), APIResources: []metav1.APIResource{ { Name: resource.Name, Kind: resource.Name, Namespaced: resource.Namespaced, }, }, }, ) } discoveryHelper := test.NewFakeDiscoveryHelper(false, resourcesMap) discoveryHelper.ResourceList = resourceList return discoveryHelper } ================================================ FILE: pkg/util/csi/util.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package csi import ( "strings" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/features" ) const ( csiPluginNamePrefix = "velero.io/csi-" ) func ShouldSkipAction(actionName string) bool { return !features.IsEnabled(velerov1api.CSIFeatureFlag) && strings.Contains(actionName, csiPluginNamePrefix) } ================================================ FILE: pkg/util/csi/util_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package csi import ( "testing" "github.com/stretchr/testify/require" "github.com/vmware-tanzu/velero/pkg/features" ) func TestCSIFeatureNotEnabledAndPluginIsFromCSI(t *testing.T) { features.NewFeatureFlagSet("EnableCSI") require.False(t, ShouldSkipAction("abc")) require.False(t, ShouldSkipAction("velero.io/csi-pvc-backupper")) features.NewFeatureFlagSet("") require.True(t, ShouldSkipAction("velero.io/csi-pvc-backupper")) require.False(t, ShouldSkipAction("abc")) } ================================================ FILE: pkg/util/csi/volume_snapshot.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package csi import ( "context" "encoding/json" "fmt" "strings" "time" jsonpatch "github.com/evanphx/json-patch/v5" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" snapshotter "github.com/kubernetes-csi/external-snapshotter/client/v8/clientset/versioned/typed/volumesnapshot/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" crclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/util/boolptr" "github.com/vmware-tanzu/velero/pkg/util/stringptr" "github.com/vmware-tanzu/velero/pkg/util/stringslice" ) const ( waitInternal = 2 * time.Second volumeSnapshotContentProtectFinalizer = "velero.io/volume-snapshot-content-protect-finalizer" ) // WaitVolumeSnapshotReady waits a VS to become ready to use until the timeout reaches func WaitVolumeSnapshotReady( ctx context.Context, snapshotClient snapshotter.SnapshotV1Interface, volumeSnapshot string, volumeSnapshotNS string, timeout time.Duration, log logrus.FieldLogger, ) (*snapshotv1api.VolumeSnapshot, error) { var updated *snapshotv1api.VolumeSnapshot errMessage := sets.NewString() err := wait.PollUntilContextTimeout( ctx, waitInternal, timeout, true, func(ctx context.Context) (bool, error) { tmpVS, err := snapshotClient.VolumeSnapshots(volumeSnapshotNS).Get( ctx, volumeSnapshot, metav1.GetOptions{}) if err != nil { return false, errors.Wrapf( err, "error to get VolumeSnapshot %s/%s", volumeSnapshotNS, volumeSnapshot, ) } if tmpVS.Status == nil { return false, nil } if tmpVS.Status.Error != nil { errMessage.Insert(stringptr.GetString(tmpVS.Status.Error.Message)) } if !boolptr.IsSetToTrue(tmpVS.Status.ReadyToUse) { return false, nil } updated = tmpVS return true, nil }, ) if wait.Interrupted(err) { err = errors.Errorf( "volume snapshot is not ready until timeout, errors: %v", errMessage.List(), ) } if errMessage.Len() > 0 { log.Warnf("Some errors happened during waiting for ready snapshot, errors: %v", errMessage.List()) } return updated, err } // GetVolumeSnapshotContentForVolumeSnapshot returns the VolumeSnapshotContent // object associated with the VolumeSnapshot. func GetVolumeSnapshotContentForVolumeSnapshot( volSnap *snapshotv1api.VolumeSnapshot, snapshotClient snapshotter.SnapshotV1Interface, ) (*snapshotv1api.VolumeSnapshotContent, error) { if volSnap.Status == nil || volSnap.Status.BoundVolumeSnapshotContentName == nil { return nil, errors.Errorf("invalid snapshot info in volume snapshot %s", volSnap.Name) } vsc, err := snapshotClient.VolumeSnapshotContents().Get( context.TODO(), *volSnap.Status.BoundVolumeSnapshotContentName, metav1.GetOptions{}, ) if err != nil { return nil, errors.Wrap(err, "error getting volume snapshot content from API") } return vsc, nil } // RetainVSC updates the VSC's deletion policy to Retain and then return the update VSC func RetainVSC(ctx context.Context, snapshotClient snapshotter.SnapshotV1Interface, vsc *snapshotv1api.VolumeSnapshotContent) (*snapshotv1api.VolumeSnapshotContent, error) { if vsc.Spec.DeletionPolicy == snapshotv1api.VolumeSnapshotContentRetain { return vsc, nil } return patchVSC(ctx, snapshotClient, vsc, func(updated *snapshotv1api.VolumeSnapshotContent) { updated.Spec.DeletionPolicy = snapshotv1api.VolumeSnapshotContentRetain }) } // DeleteVolumeSnapshotContentIfAny deletes a VSC by name if it exists, // and log an error when the deletion fails. func DeleteVolumeSnapshotContentIfAny( ctx context.Context, snapshotClient snapshotter.SnapshotV1Interface, vscName string, log logrus.FieldLogger, ) { err := snapshotClient.VolumeSnapshotContents().Delete(ctx, vscName, metav1.DeleteOptions{}) if err != nil { if apierrors.IsNotFound(err) { log.WithError(err).Debugf("Abort deleting VSC, it doesn't exist %s", vscName) } else { log.WithError(err).Errorf("Failed to delete volume snapshot content %s", vscName) } } } // EnsureDeleteVS asserts the existence of a VS by name, deletes it and waits for its // disappearance and returns errors on any failure. func EnsureDeleteVS(ctx context.Context, snapshotClient snapshotter.SnapshotV1Interface, vsName string, vsNamespace string, timeout time.Duration) error { err := snapshotClient.VolumeSnapshots(vsNamespace).Delete(ctx, vsName, metav1.DeleteOptions{}) if err != nil { return errors.Wrap(err, "error to delete volume snapshot") } var updated *snapshotv1api.VolumeSnapshot err = wait.PollUntilContextTimeout(ctx, waitInternal, timeout, true, func(ctx context.Context) (bool, error) { vs, err := snapshotClient.VolumeSnapshots(vsNamespace).Get(ctx, vsName, metav1.GetOptions{}) if err != nil { if apierrors.IsNotFound(err) { return true, nil } return false, errors.Wrapf(err, "error to get VolumeSnapshot %s", vsName) } updated = vs return false, nil }) if err != nil { if errors.Is(err, context.DeadlineExceeded) { return errors.Errorf("timeout to assure VolumeSnapshot %s is deleted, finalizers in VS %v", vsName, updated.Finalizers) } else { return errors.Wrapf(err, "error to assure VolumeSnapshot is deleted, %s", vsName) } } return nil } func RemoveVSCProtect(ctx context.Context, snapshotClient snapshotter.SnapshotV1Interface, vscName string, timeout time.Duration) error { err := wait.PollUntilContextTimeout(ctx, waitInternal, timeout, true, func(ctx context.Context) (bool, error) { vsc, err := snapshotClient.VolumeSnapshotContents().Get(ctx, vscName, metav1.GetOptions{}) if err != nil { return false, errors.Wrapf(err, "error to get VolumeSnapshotContent %s", vscName) } vsc.Finalizers = stringslice.Except(vsc.Finalizers, volumeSnapshotContentProtectFinalizer) _, err = snapshotClient.VolumeSnapshotContents().Update(ctx, vsc, metav1.UpdateOptions{}) if err == nil { return true, nil } if !apierrors.IsConflict(err) { return false, errors.Wrapf(err, "error to update VolumeSnapshotContent %s", vscName) } return false, nil }) return err } // EnsureDeleteVSC asserts the existence of a VSC by name, deletes it and waits for its // disappearance and returns errors on any failure. func EnsureDeleteVSC(ctx context.Context, snapshotClient snapshotter.SnapshotV1Interface, vscName string, timeout time.Duration) error { err := snapshotClient.VolumeSnapshotContents().Delete(ctx, vscName, metav1.DeleteOptions{}) if err != nil && !apierrors.IsNotFound(err) { return errors.Wrap(err, "error to delete volume snapshot content") } var updated *snapshotv1api.VolumeSnapshotContent err = wait.PollUntilContextTimeout(ctx, waitInternal, timeout, true, func(ctx context.Context) (bool, error) { vsc, err := snapshotClient.VolumeSnapshotContents().Get(ctx, vscName, metav1.GetOptions{}) if err != nil { if apierrors.IsNotFound(err) { return true, nil } return false, errors.Wrapf(err, "error to get VolumeSnapshotContent %s", vscName) } updated = vsc return false, nil }) if err != nil { if errors.Is(err, context.DeadlineExceeded) { return errors.Errorf("timeout to assure VolumeSnapshotContent %s is deleted, finalizers in VSC %v", vscName, updated.Finalizers) } else { return errors.Wrapf(err, "error to assure VolumeSnapshotContent is deleted, %s", vscName) } } return nil } // DeleteVolumeSnapshotIfAny deletes a VS by name if it exists, // and log an error when the deletion fails func DeleteVolumeSnapshotIfAny( ctx context.Context, snapshotClient snapshotter.SnapshotV1Interface, vsName string, vsNamespace string, log logrus.FieldLogger, ) { err := snapshotClient.VolumeSnapshots(vsNamespace).Delete(ctx, vsName, metav1.DeleteOptions{}) if err != nil { if apierrors.IsNotFound(err) { log.WithError(err).Debugf( "Abort deleting volume snapshot, it doesn't exist %s/%s", vsNamespace, vsName) } else { log.WithError(err).Errorf( "Failed to delete volume snapshot %s/%s", vsNamespace, vsName) } } } func patchVSC( ctx context.Context, snapshotClient snapshotter.SnapshotV1Interface, vsc *snapshotv1api.VolumeSnapshotContent, updateFunc func(*snapshotv1api.VolumeSnapshotContent), ) (*snapshotv1api.VolumeSnapshotContent, error) { origBytes, err := json.Marshal(vsc) if err != nil { return nil, errors.Wrap(err, "error marshaling original VSC") } updated := vsc.DeepCopy() updateFunc(updated) updatedBytes, err := json.Marshal(updated) if err != nil { return nil, errors.Wrap(err, "error marshaling updated VSC") } patchBytes, err := jsonpatch.CreateMergePatch(origBytes, updatedBytes) if err != nil { return nil, errors.Wrap(err, "error creating json merge patch for VSC") } patched, err := snapshotClient.VolumeSnapshotContents().Patch(ctx, vsc.Name, types.MergePatchType, patchBytes, metav1.PatchOptions{}) if err != nil { return nil, errors.Wrap(err, "error patching VSC") } return patched, nil } func GetVolumeSnapshotClass( provisioner string, backup *velerov1api.Backup, pvc *corev1api.PersistentVolumeClaim, log logrus.FieldLogger, crClient crclient.Client, ) (*snapshotv1api.VolumeSnapshotClass, error) { snapshotClasses := new(snapshotv1api.VolumeSnapshotClassList) err := crClient.List(context.TODO(), snapshotClasses) if err != nil { return nil, errors.Wrap(err, "error listing VolumeSnapshotClass") } // If a snapshot class is set for provider in PVC annotations, use that snapshotClass, err := GetVolumeSnapshotClassFromPVCAnnotationsForDriver( pvc, provisioner, snapshotClasses, ) if err != nil { log.Debugf("Didn't find VolumeSnapshotClass from PVC annotations: %v", err) } if snapshotClass != nil { return snapshotClass, nil } // If there is no annotation in PVC, attempt to fetch it from backup annotations snapshotClass, err = GetVolumeSnapshotClassFromBackupAnnotationsForDriver( backup, provisioner, snapshotClasses) if err != nil { log.Debugf("Didn't find VolumeSnapshotClass from Backup annotations: %v", err) } if snapshotClass != nil { return snapshotClass, nil } // fallback to default behavior of fetching snapshot class based on label snapshotClass, err = GetVolumeSnapshotClassForStorageClass( provisioner, snapshotClasses) if err != nil || snapshotClass == nil { return nil, errors.Wrap(err, "error getting VolumeSnapshotClass") } return snapshotClass, nil } func GetVolumeSnapshotClassFromPVCAnnotationsForDriver( pvc *corev1api.PersistentVolumeClaim, provisioner string, snapshotClasses *snapshotv1api.VolumeSnapshotClassList, ) (*snapshotv1api.VolumeSnapshotClass, error) { annotationKey := velerov1api.VolumeSnapshotClassDriverPVCAnnotation snapshotClassName, ok := pvc.ObjectMeta.Annotations[annotationKey] if !ok { return nil, nil } for _, sc := range snapshotClasses.Items { if strings.EqualFold(snapshotClassName, sc.ObjectMeta.Name) { if !strings.EqualFold(sc.Driver, provisioner) { return nil, errors.Errorf( "Incorrect VolumeSnapshotClass %s is not for driver %s", sc.ObjectMeta.Name, provisioner, ) } return &sc, nil } } return nil, errors.Errorf( "No CSI VolumeSnapshotClass found with name %s for provisioner %s for PVC %s", snapshotClassName, provisioner, pvc.Name, ) } // GetVolumeSnapshotClassFromAnnotationsForDriver returns a // VolumeSnapshotClass for the supplied volume provisioner/driver // name from the annotation of the backup. func GetVolumeSnapshotClassFromBackupAnnotationsForDriver( backup *velerov1api.Backup, provisioner string, snapshotClasses *snapshotv1api.VolumeSnapshotClassList, ) (*snapshotv1api.VolumeSnapshotClass, error) { annotationKey := fmt.Sprintf( "%s_%s", velerov1api.VolumeSnapshotClassDriverBackupAnnotationPrefix, strings.ToLower(provisioner), ) snapshotClassName, ok := backup.ObjectMeta.Annotations[annotationKey] if !ok { return nil, nil } for _, sc := range snapshotClasses.Items { if strings.EqualFold(snapshotClassName, sc.ObjectMeta.Name) { if !strings.EqualFold(sc.Driver, provisioner) { return nil, errors.Errorf( "Incorrect VolumeSnapshotClass %s is not for driver %s for backup %s", sc.ObjectMeta.Name, provisioner, backup.Name, ) } return &sc, nil } } return nil, errors.Errorf( "No CSI VolumeSnapshotClass found with name %s for driver %s for backup %s", snapshotClassName, provisioner, backup.Name, ) } // GetVolumeSnapshotClassForStorageClass returns a VolumeSnapshotClass // for the supplied volume provisioner/ driver name. func GetVolumeSnapshotClassForStorageClass( provisioner string, snapshotClasses *snapshotv1api.VolumeSnapshotClassList, ) (*snapshotv1api.VolumeSnapshotClass, error) { n := 0 var vsClass snapshotv1api.VolumeSnapshotClass // We pick the VolumeSnapshotClass that matches the CSI driver name // and has a 'velero.io/csi-volumesnapshot-class' label. This allows // multiple VolumeSnapshotClasses for the same driver with different // values for the other fields in the spec. for _, sc := range snapshotClasses.Items { _, hasLabelSelector := sc.Labels[velerov1api.VolumeSnapshotClassSelectorLabel] if sc.Driver == provisioner { n++ vsClass = sc if hasLabelSelector { return &sc, nil } } } // not found by label, pick by annotation for _, sc := range snapshotClasses.Items { _, hasDefaultAnnotation := sc.Annotations[velerov1api.VolumeSnapshotClassKubernetesAnnotation] if sc.Driver == provisioner { vsClass = sc if hasDefaultAnnotation { return &sc, nil } } } // If there's only one volumesnapshotclass for the driver, return it. if n == 1 { return &vsClass, nil } return nil, fmt.Errorf( "failed to get VolumeSnapshotClass for provisioner %s: "+ "ensure that the desired VolumeSnapshotClass has the %s label or %s annotation, "+ "and that its driver matches the StorageClass provisioner", provisioner, velerov1api.VolumeSnapshotClassSelectorLabel, velerov1api.VolumeSnapshotClassKubernetesAnnotation, ) } // IsVolumeSnapshotClassHasListerSecret returns whether a volumesnapshotclass has a snapshotlister secret func IsVolumeSnapshotClassHasListerSecret(vc *snapshotv1api.VolumeSnapshotClass) bool { // https://github.com/kubernetes-csi/external-snapshotter/blob/master/pkg/utils/util.go#L59-L60 // There is no release w/ these constants exported. Using the strings for now. _, nameExists := vc.Annotations[velerov1api.PrefixedListSecretNameAnnotation] _, nsExists := vc.Annotations[velerov1api.PrefixedListSecretNamespaceAnnotation] return nameExists && nsExists } // IsVolumeSnapshotContentHasDeleteSecret returns whether a volumesnapshotcontent has a deletesnapshot secret func IsVolumeSnapshotContentHasDeleteSecret(vsc *snapshotv1api.VolumeSnapshotContent) bool { // https://github.com/kubernetes-csi/external-snapshotter/blob/master/pkg/utils/util.go#L56-L57 // use exported constants in the next release _, nameExists := vsc.Annotations[velerov1api.PrefixedSecretNameAnnotation] _, nsExists := vsc.Annotations[velerov1api.PrefixedSecretNamespaceAnnotation] return nameExists && nsExists } // IsVolumeSnapshotExists returns whether a specific volumesnapshot object exists. func IsVolumeSnapshotExists( ns, name string, crClient crclient.Client, ) bool { vs := new(snapshotv1api.VolumeSnapshot) err := crClient.Get( context.TODO(), crclient.ObjectKey{Namespace: ns, Name: name}, vs, ) return err == nil } func SetVolumeSnapshotContentDeletionPolicy( vscName string, crClient crclient.Client, policy snapshotv1api.DeletionPolicy, ) (*snapshotv1api.VolumeSnapshotContent, error) { vsc := new(snapshotv1api.VolumeSnapshotContent) if err := crClient.Get(context.TODO(), crclient.ObjectKey{Name: vscName}, vsc); err != nil { return nil, err } originVSC := vsc.DeepCopy() vsc.Spec.DeletionPolicy = policy return vsc, crClient.Patch(context.TODO(), vsc, crclient.MergeFrom(originVSC)) } // CleanupVolumeSnapshot deletes the VolumeSnapshot and the associated VolumeSnapshotContent. It will make sure the // physical snapshot is also deleted. func CleanupVolumeSnapshot( volSnap *snapshotv1api.VolumeSnapshot, crClient crclient.Client, log logrus.FieldLogger, ) { log.Infof("Deleting Volumesnapshot %s/%s", volSnap.Namespace, volSnap.Name) vs := new(snapshotv1api.VolumeSnapshot) err := crClient.Get( context.TODO(), crclient.ObjectKey{Name: volSnap.Name, Namespace: volSnap.Namespace}, vs, ) if err != nil { log.Debugf("Failed to get volumesnapshot %s/%s", volSnap.Namespace, volSnap.Name) return } if vs.Status != nil && vs.Status.BoundVolumeSnapshotContentName != nil { // we patch the DeletionPolicy of the VolumeSnapshotContent to set it to Delete. // This ensures that the volume snapshot in the storage provider is also deleted. _, err := SetVolumeSnapshotContentDeletionPolicy( *vs.Status.BoundVolumeSnapshotContentName, crClient, snapshotv1api.VolumeSnapshotContentDelete, ) if err != nil { log.Debugf("Failed to patch DeletionPolicy of volume snapshot %s/%s", vs.Namespace, vs.Name) } } err = crClient.Delete(context.TODO(), vs) if err != nil { log.Debugf("Failed to delete volumesnapshot %s/%s: %v", vs.Namespace, vs.Name, err) } else { log.Infof("Deleted volumesnapshot with volumesnapshotContent %s/%s", vs.Namespace, vs.Name) } } func DeleteReadyVolumeSnapshot( vs snapshotv1api.VolumeSnapshot, client crclient.Client, logger logrus.FieldLogger, ) { logger.Infof("Deleting Volumesnapshot %s/%s", vs.Namespace, vs.Name) if vs.Status == nil || vs.Status.BoundVolumeSnapshotContentName == nil || len(*vs.Status.BoundVolumeSnapshotContentName) <= 0 { logger.Errorf("VolumeSnapshot %s/%s is not ready. This is not expected.", vs.Namespace, vs.Name) return } var vsc *snapshotv1api.VolumeSnapshotContent if vs.Status != nil && vs.Status.BoundVolumeSnapshotContentName != nil { var err error // Patch the DeletionPolicy of the VolumeSnapshotContent to set it to Retain. // This ensures that the volume snapshot in the storage provider is kept. if vsc, err = SetVolumeSnapshotContentDeletionPolicy( *vs.Status.BoundVolumeSnapshotContentName, client, snapshotv1api.VolumeSnapshotContentRetain, ); err != nil { logger.Warnf("Failed to patch DeletionPolicy of VolumeSnapshot %s/%s", vs.Namespace, vs.Name) return } if err := client.Delete(context.TODO(), vsc); err != nil { logger.WithError(err).Warnf("Failed to delete the VolumeSnapshotContent %s", vsc.Name) } } if err := client.Delete(context.TODO(), &vs); err != nil { logger.WithError(err).Warnf("Failed to delete VolumeSnapshot %s", vs.Namespace+"/"+vs.Name) } else { logger.Infof("Deleted VolumeSnapshot %s and VolumeSnapshotContent %s", vs.Namespace+"/"+vs.Name, vsc.Name) } } // WaitUntilVSCHandleIsReady returns the VolumeSnapshotContent // object associated with the volumesnapshot func WaitUntilVSCHandleIsReady( volSnap *snapshotv1api.VolumeSnapshot, crClient crclient.Client, log logrus.FieldLogger, csiSnapshotTimeout time.Duration, ) (*snapshotv1api.VolumeSnapshotContent, error) { // We'll wait 10m for the VSC to be reconciled polling // every 5s unless backup's csiSnapshotTimeout is set interval := 5 * time.Second vsc := new(snapshotv1api.VolumeSnapshotContent) err := wait.PollUntilContextTimeout( context.Background(), interval, csiSnapshotTimeout, true, func(ctx context.Context) (bool, error) { vs := new(snapshotv1api.VolumeSnapshot) if err := crClient.Get( ctx, crclient.ObjectKeyFromObject(volSnap), vs, ); err != nil { return false, errors.Wrapf( err, "failed to get volumesnapshot %s/%s", volSnap.Namespace, volSnap.Name, ) } if vs.Status == nil || vs.Status.BoundVolumeSnapshotContentName == nil { log.Infof("Waiting for CSI driver to reconcile volumesnapshot %s/%s. Retrying in %ds", volSnap.Namespace, volSnap.Name, interval/time.Second) return false, nil } if err := crClient.Get( ctx, crclient.ObjectKey{ Name: *vs.Status.BoundVolumeSnapshotContentName, }, vsc, ); err != nil { return false, errors.Wrapf( err, "failed to get VolumeSnapshotContent %s for VolumeSnapshot %s/%s", *vs.Status.BoundVolumeSnapshotContentName, vs.Namespace, vs.Name, ) } // we need to wait for the VolumeSnapshotContent // to have a snapshot handle because during restore, // we'll use that snapshot handle as the source for // the VolumeSnapshotContent so it's statically // bound to the existing snapshot. if vsc.Status == nil || vsc.Status.SnapshotHandle == nil { log.Infof( "Waiting for VolumeSnapshotContents %s to have snapshot handle. Retrying in %ds", vsc.Name, interval/time.Second) if vsc.Status != nil && vsc.Status.Error != nil { log.Warnf("VolumeSnapshotContent %s has error: %v", vsc.Name, *vsc.Status.Error.Message) } return false, nil } return true, nil }, ) if err != nil { if wait.Interrupted(err) { if vsc != nil && vsc.Status != nil && vsc.Status.Error != nil { log.Errorf( "Timed out awaiting reconciliation of VolumeSnapshot, VolumeSnapshotContent %s has error: %v", vsc.Name, *vsc.Status.Error.Message) return nil, errors.Errorf("CSI got timed out with error: %v", *vsc.Status.Error.Message) } else { log.Errorf( "Timed out awaiting reconciliation of volumesnapshot %s/%s", volSnap.Namespace, volSnap.Name) } } return nil, err } return vsc, nil } func DiagnoseVS(vs *snapshotv1api.VolumeSnapshot, events *corev1api.EventList) string { vscName := "" readyToUse := false errMessage := "" if vs.Status != nil { if vs.Status.BoundVolumeSnapshotContentName != nil { vscName = *vs.Status.BoundVolumeSnapshotContentName } if vs.Status.ReadyToUse != nil { readyToUse = *vs.Status.ReadyToUse } if vs.Status.Error != nil && vs.Status.Error.Message != nil { errMessage = *vs.Status.Error.Message } } diag := fmt.Sprintf("VS %s/%s, bind to %s, readyToUse %v, errMessage %s\n", vs.Namespace, vs.Name, vscName, readyToUse, errMessage) if events != nil { for _, e := range events.Items { if e.InvolvedObject.UID == vs.UID && e.Type == corev1api.EventTypeWarning { diag += fmt.Sprintf("VS event reason %s, message %s\n", e.Reason, e.Message) } } } return diag } func DiagnoseVSC(vsc *snapshotv1api.VolumeSnapshotContent) string { handle := "" readyToUse := false errMessage := "" if vsc.Status != nil { if vsc.Status.SnapshotHandle != nil { handle = *vsc.Status.SnapshotHandle } if vsc.Status.ReadyToUse != nil { readyToUse = *vsc.Status.ReadyToUse } if vsc.Status.Error != nil && vsc.Status.Error.Message != nil { errMessage = *vsc.Status.Error.Message } } diag := fmt.Sprintf("VSC %s, readyToUse %v, errMessage %s, handle %s\n", vsc.Name, readyToUse, errMessage, handle) return diag } // GetVSCForVS returns the VolumeSnapshotContent object associated with the VolumeSnapshot. func GetVSCForVS( ctx context.Context, vs *snapshotv1api.VolumeSnapshot, client crclient.Client, ) (*snapshotv1api.VolumeSnapshotContent, error) { if vs.Status == nil || vs.Status.BoundVolumeSnapshotContentName == nil { return nil, errors.Errorf("invalid snapshot info in volume snapshot %s", vs.Name) } vsc := new(snapshotv1api.VolumeSnapshotContent) if err := client.Get( ctx, crclient.ObjectKey{ Name: *vs.Status.BoundVolumeSnapshotContentName, }, vsc, ); err != nil { return nil, errors.Wrap(err, "error getting volume snapshot content from API") } return vsc, nil } ================================================ FILE: pkg/util/csi/volume_snapshot_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package csi import ( "errors" "testing" "time" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" snapshotFake "github.com/kubernetes-csi/external-snapshotter/client/v8/clientset/versioned/fake" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" clientTesting "k8s.io/client-go/testing" crclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/test" velerotest "github.com/vmware-tanzu/velero/pkg/test" "github.com/vmware-tanzu/velero/pkg/util/boolptr" "github.com/vmware-tanzu/velero/pkg/util/logging" "github.com/vmware-tanzu/velero/pkg/util/stringptr" ) type reactor struct { verb string resource string reactorFunc clientTesting.ReactionFunc } // expected: &v1.VolumeSnapshot{TypeMeta:v1.TypeMeta{Kind:"", APIVersion:""}, ObjectMeta:v1.ObjectMeta{Name:"fake-vs", GenerateName:"", Namespace:"fake-ns", SelfLink:"", UID:"", ResourceVersion:"999", Generation:0, CreationTimestamp:time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC), DeletionTimestamp:, DeletionGracePeriodSeconds:(*int64)(nil), Labels:map[string]string(nil), Annotations:map[string]string(nil), OwnerReferences:[]v1.OwnerReference(nil), Finalizers:[]string(nil), ManagedFields:[]v1.ManagedFieldsEntry(nil)}, Spec:v1.VolumeSnapshotSpec{Source:v1.VolumeSnapshotSource{PersistentVolumeClaimName:(*string)(nil), VolumeSnapshotContentName:(*string)(nil)}, VolumeSnapshotClassName:(*string)(nil)}, Status:(*v1.VolumeSnapshotStatus)(0x140000af8c0)} // actual : &v1.VolumeSnapshot{TypeMeta:v1.TypeMeta{Kind:"", APIVersion:""}, ObjectMeta:v1.ObjectMeta{Name:"fake-vs", GenerateName:"", Namespace:"fake-ns", SelfLink:"", UID:"", ResourceVersion:"999", Generation:0, CreationTimestamp:time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC), DeletionTimestamp:, DeletionGracePeriodSeconds:(*int64)(nil), Labels:map[string]string(nil), Annotations:map[string]string(nil), OwnerReferences:[]v1.OwnerReference(nil), Finalizers:[]string(nil), ManagedFields:[]v1.ManagedFieldsEntry(nil)}, Spec:v1.VolumeSnapshotSpec{Source:v1.VolumeSnapshotSource{PersistentVolumeClaimName:(*string)(nil), VolumeSnapshotContentName:(*string)(nil)}, VolumeSnapshotClassName:(*string)(nil)}, Status:(*v1.VolumeSnapshotStatus)(0x1400024ed20)} func TestWaitVolumeSnapshotReady(t *testing.T) { vscName := "fake-vsc" quantity := resource.MustParse("0") vsObj := &snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-vs", Namespace: "fake-ns", }, Status: &snapshotv1api.VolumeSnapshotStatus{ BoundVolumeSnapshotContentName: &vscName, ReadyToUse: boolptr.True(), RestoreSize: &quantity, }, } errMessage := "fake-snapshot-creation-error" tests := []struct { name string clientObj []runtime.Object vsName string namespace string err string expect *snapshotv1api.VolumeSnapshot }{ { name: "get vs error", vsName: "fake-vs-1", namespace: "fake-ns-1", err: "error to get VolumeSnapshot fake-ns-1/fake-vs-1: volumesnapshots.snapshot.storage.k8s.io \"fake-vs-1\" not found", }, { name: "vs status is nil", vsName: "fake-vs", namespace: "fake-ns", clientObj: []runtime.Object{ &snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-vs", Namespace: "fake-ns", }, }, }, err: "volume snapshot is not ready until timeout, errors: []", }, { name: "vsc is nil in status", vsName: "fake-vs", namespace: "fake-ns", clientObj: []runtime.Object{ &snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-vs", Namespace: "fake-ns", }, Status: &snapshotv1api.VolumeSnapshotStatus{}, }, }, err: "volume snapshot is not ready until timeout, errors: []", }, { name: "ready to use is nil in status", vsName: "fake-vs", namespace: "fake-ns", clientObj: []runtime.Object{ &snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-vs", Namespace: "fake-ns", }, Status: &snapshotv1api.VolumeSnapshotStatus{ BoundVolumeSnapshotContentName: &vscName, }, }, }, err: "volume snapshot is not ready until timeout, errors: []", }, { name: "ready to use is false", vsName: "fake-vs", namespace: "fake-ns", clientObj: []runtime.Object{ &snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-vs", Namespace: "fake-ns", }, Status: &snapshotv1api.VolumeSnapshotStatus{ BoundVolumeSnapshotContentName: &vscName, ReadyToUse: boolptr.False(), }, }, }, err: "volume snapshot is not ready until timeout, errors: []", }, { name: "snapshot creation error with message", vsName: "fake-vs", namespace: "fake-ns", clientObj: []runtime.Object{ &snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-vs", Namespace: "fake-ns", }, Status: &snapshotv1api.VolumeSnapshotStatus{ Error: &snapshotv1api.VolumeSnapshotError{ Message: &errMessage, }, }, }, }, err: "volume snapshot is not ready until timeout, errors: [fake-snapshot-creation-error]", }, { name: "snapshot creation error without message", vsName: "fake-vs", namespace: "fake-ns", clientObj: []runtime.Object{ &snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-vs", Namespace: "fake-ns", }, Status: &snapshotv1api.VolumeSnapshotStatus{ Error: &snapshotv1api.VolumeSnapshotError{}, }, }, }, err: "volume snapshot is not ready until timeout, errors: [" + stringptr.NilString + "]", }, { name: "success", vsName: "fake-vs", namespace: "fake-ns", clientObj: []runtime.Object{ vsObj, }, expect: vsObj, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeClient := snapshotFake.NewSimpleClientset(test.clientObj...) vs, err := WaitVolumeSnapshotReady(t.Context(), fakeClient.SnapshotV1(), test.vsName, test.namespace, time.Millisecond, velerotest.NewLogger()) if err != nil { require.EqualError(t, err, test.err) } else { require.NoError(t, err) } assert.Equal(t, test.expect, vs) }) } } func TestGetVolumeSnapshotContentForVolumeSnapshot(t *testing.T) { vscName := "fake-vsc" vsObj := &snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-vs", Namespace: "fake-ns", }, Status: &snapshotv1api.VolumeSnapshotStatus{ BoundVolumeSnapshotContentName: &vscName, ReadyToUse: boolptr.True(), RestoreSize: &resource.Quantity{}, }, } vscObj := &snapshotv1api.VolumeSnapshotContent{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-vsc", }, } tests := []struct { name string snapshotObj *snapshotv1api.VolumeSnapshot clientObj []runtime.Object vsName string namespace string err string expect *snapshotv1api.VolumeSnapshotContent }{ { name: "vs status is nil", vsName: "fake-vs", namespace: "fake-ns", snapshotObj: &snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-vs", Namespace: "fake-ns", }, }, err: "invalid snapshot info in volume snapshot fake-vs", }, { name: "vsc is nil in status", vsName: "fake-vs", namespace: "fake-ns", snapshotObj: &snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-vs", Namespace: "fake-ns", }, }, err: "invalid snapshot info in volume snapshot fake-vs", }, { name: "get vsc fail", vsName: "fake-vs", namespace: "fake-ns", snapshotObj: vsObj, err: "error getting volume snapshot content from API: volumesnapshotcontents.snapshot.storage.k8s.io \"fake-vsc\" not found", }, { name: "success", vsName: "fake-vs", namespace: "fake-ns", snapshotObj: vsObj, clientObj: []runtime.Object{vscObj}, expect: vscObj, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeClient := snapshotFake.NewSimpleClientset(test.clientObj...) vs, err := GetVolumeSnapshotContentForVolumeSnapshot(test.snapshotObj, fakeClient.SnapshotV1()) if err != nil { require.EqualError(t, err, test.err) } else { require.NoError(t, err) } assert.Equal(t, test.expect, vs) }) } } func TestEnsureDeleteVS(t *testing.T) { vsObj := &snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-vs", Namespace: "fake-ns", }, } vsObjWithFinalizer := &snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-vs", Namespace: "fake-ns", Finalizers: []string{"fake-finalizer-1", "fake-finalizer-2"}, }, } tests := []struct { name string clientObj []runtime.Object vsName string namespace string reactors []reactor err string }{ { name: "delete fail", vsName: "fake-vs", namespace: "fake-ns", err: "error to delete volume snapshot: volumesnapshots.snapshot.storage.k8s.io \"fake-vs\" not found", }, { name: "wait fail", vsName: "fake-vs", namespace: "fake-ns", clientObj: []runtime.Object{vsObj}, reactors: []reactor{ { verb: "get", resource: "volumesnapshots", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, errors.New("fake-get-error") }, }, }, err: "error to assure VolumeSnapshot is deleted, fake-vs: error to get VolumeSnapshot fake-vs: fake-get-error", }, { name: "wait timeout", vsName: "fake-vs", namespace: "fake-ns", clientObj: []runtime.Object{vsObjWithFinalizer}, reactors: []reactor{ { verb: "delete", resource: "volumesnapshots", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, nil }, }, }, err: "timeout to assure VolumeSnapshot fake-vs is deleted, finalizers in VS [fake-finalizer-1 fake-finalizer-2]", }, { name: "wait timeout, no finalizer", vsName: "fake-vs", namespace: "fake-ns", clientObj: []runtime.Object{vsObj}, reactors: []reactor{ { verb: "delete", resource: "volumesnapshots", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, nil }, }, }, err: "timeout to assure VolumeSnapshot fake-vs is deleted, finalizers in VS []", }, { name: "success", vsName: "fake-vs", namespace: "fake-ns", clientObj: []runtime.Object{vsObj}, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeSnapshotClient := snapshotFake.NewSimpleClientset(test.clientObj...) for _, reactor := range test.reactors { fakeSnapshotClient.Fake.PrependReactor(reactor.verb, reactor.resource, reactor.reactorFunc) } err := EnsureDeleteVS(t.Context(), fakeSnapshotClient.SnapshotV1(), test.vsName, test.namespace, time.Millisecond) if err != nil { assert.EqualError(t, err, test.err) } else { assert.NoError(t, err) } }) } } func TestEnsureDeleteVSC(t *testing.T) { vscObj := &snapshotv1api.VolumeSnapshotContent{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-vsc", }, } vscObjWithFinalizer := &snapshotv1api.VolumeSnapshotContent{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-vsc", Finalizers: []string{"fake-finalizer-1", "fake-finalizer-2"}, }, } tests := []struct { name string clientObj []runtime.Object reactors []reactor vscName string err string }{ { name: "delete fail on VSC not found", vscName: "fake-vsc", }, { name: "delete fail on others", vscName: "fake-vsc", clientObj: []runtime.Object{vscObj}, reactors: []reactor{ { verb: "delete", resource: "volumesnapshotcontents", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, errors.New("fake-delete-error") }, }, }, err: "error to delete volume snapshot content: fake-delete-error", }, { name: "wait fail", vscName: "fake-vsc", clientObj: []runtime.Object{vscObj}, reactors: []reactor{ { verb: "get", resource: "volumesnapshotcontents", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, errors.New("fake-get-error") }, }, }, err: "error to assure VolumeSnapshotContent is deleted, fake-vsc: error to get VolumeSnapshotContent fake-vsc: fake-get-error", }, { name: "wait timeout", vscName: "fake-vsc", clientObj: []runtime.Object{vscObjWithFinalizer}, reactors: []reactor{ { verb: "delete", resource: "volumesnapshotcontents", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, nil }, }, }, err: "timeout to assure VolumeSnapshotContent fake-vsc is deleted, finalizers in VSC [fake-finalizer-1 fake-finalizer-2]", }, { name: "wait timeout, no finalizer", vscName: "fake-vsc", clientObj: []runtime.Object{vscObj}, reactors: []reactor{ { verb: "delete", resource: "volumesnapshotcontents", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, nil }, }, }, err: "timeout to assure VolumeSnapshotContent fake-vsc is deleted, finalizers in VSC []", }, { name: "success", vscName: "fake-vsc", clientObj: []runtime.Object{vscObj}, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeSnapshotClient := snapshotFake.NewSimpleClientset(test.clientObj...) for _, reactor := range test.reactors { fakeSnapshotClient.Fake.PrependReactor(reactor.verb, reactor.resource, reactor.reactorFunc) } err := EnsureDeleteVSC(t.Context(), fakeSnapshotClient.SnapshotV1(), test.vscName, time.Millisecond) if test.err != "" { assert.EqualError(t, err, test.err) } else { assert.NoError(t, err) } }) } } func TestDeleteVolumeSnapshotContentIfAny(t *testing.T) { tests := []struct { name string clientObj []runtime.Object reactors []reactor vscName string logMessage string logLevel string logError string }{ { name: "vsc not exist", vscName: "fake-vsc", logMessage: "Abort deleting VSC, it doesn't exist fake-vsc", logLevel: "level=debug", }, { name: "deleete fail", vscName: "fake-vsc", reactors: []reactor{ { verb: "delete", resource: "volumesnapshotcontents", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, errors.New("fake-delete-error") }, }, }, logMessage: "Failed to delete volume snapshot content fake-vsc", logLevel: "level=error", logError: "error=fake-delete-error", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeSnapshotClient := snapshotFake.NewSimpleClientset(test.clientObj...) for _, reactor := range test.reactors { fakeSnapshotClient.Fake.PrependReactor(reactor.verb, reactor.resource, reactor.reactorFunc) } logMessage := "" DeleteVolumeSnapshotContentIfAny(t.Context(), fakeSnapshotClient.SnapshotV1(), test.vscName, velerotest.NewSingleLogger(&logMessage)) if len(test.logMessage) > 0 { assert.Contains(t, logMessage, test.logMessage) } if len(test.logLevel) > 0 { assert.Contains(t, logMessage, test.logLevel) } if len(test.logError) > 0 { assert.Contains(t, logMessage, test.logError) } }) } } func TestDeleteVolumeSnapshotIfAny(t *testing.T) { tests := []struct { name string clientObj []runtime.Object reactors []reactor vsName string vsNamespace string logMessage string logLevel string logError string }{ { name: "vs not exist", vsName: "fake-vs", vsNamespace: "fake-ns", logMessage: "Abort deleting volume snapshot, it doesn't exist fake-ns/fake-vs", logLevel: "level=debug", }, { name: "delete fail", vsName: "fake-vs", vsNamespace: "fake-ns", reactors: []reactor{ { verb: "delete", resource: "volumesnapshots", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, errors.New("fake-delete-error") }, }, }, logMessage: "Failed to delete volume snapshot fake-ns/fake-vs", logLevel: "level=error", logError: "error=fake-delete-error", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeSnapshotClient := snapshotFake.NewSimpleClientset(test.clientObj...) for _, reactor := range test.reactors { fakeSnapshotClient.Fake.PrependReactor(reactor.verb, reactor.resource, reactor.reactorFunc) } logMessage := "" DeleteVolumeSnapshotIfAny(t.Context(), fakeSnapshotClient.SnapshotV1(), test.vsName, test.vsNamespace, velerotest.NewSingleLogger(&logMessage)) if len(test.logMessage) > 0 { assert.Contains(t, logMessage, test.logMessage) } if len(test.logLevel) > 0 { assert.Contains(t, logMessage, test.logLevel) } if len(test.logError) > 0 { assert.Contains(t, logMessage, test.logError) } }) } } func TestRetainVSC(t *testing.T) { vscObj := &snapshotv1api.VolumeSnapshotContent{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-vsc", }, } tests := []struct { name string clientObj []runtime.Object reactors []reactor vsc *snapshotv1api.VolumeSnapshotContent updated *snapshotv1api.VolumeSnapshotContent err string }{ { name: "already retained", vsc: &snapshotv1api.VolumeSnapshotContent{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-vsc", }, Spec: snapshotv1api.VolumeSnapshotContentSpec{ DeletionPolicy: snapshotv1api.VolumeSnapshotContentRetain, }, }, updated: &snapshotv1api.VolumeSnapshotContent{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-vsc", }, Spec: snapshotv1api.VolumeSnapshotContentSpec{ DeletionPolicy: snapshotv1api.VolumeSnapshotContentRetain, }, }, }, { name: "path vsc fail", vsc: &snapshotv1api.VolumeSnapshotContent{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-vsc", }, Spec: snapshotv1api.VolumeSnapshotContentSpec{ DeletionPolicy: snapshotv1api.VolumeSnapshotContentDelete, }, }, reactors: []reactor{ { verb: "patch", resource: "volumesnapshotcontents", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, errors.New("fake-patch-error") }, }, }, err: "error patching VSC: fake-patch-error", }, { name: "success", vsc: &snapshotv1api.VolumeSnapshotContent{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-vsc", }, Spec: snapshotv1api.VolumeSnapshotContentSpec{ DeletionPolicy: snapshotv1api.VolumeSnapshotContentDelete, }, }, clientObj: []runtime.Object{vscObj}, updated: &snapshotv1api.VolumeSnapshotContent{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-vsc", }, Spec: snapshotv1api.VolumeSnapshotContentSpec{ DeletionPolicy: snapshotv1api.VolumeSnapshotContentRetain, }, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeSnapshotClient := snapshotFake.NewSimpleClientset(test.clientObj...) for _, reactor := range test.reactors { fakeSnapshotClient.Fake.PrependReactor(reactor.verb, reactor.resource, reactor.reactorFunc) } returned, err := RetainVSC(t.Context(), fakeSnapshotClient.SnapshotV1(), test.vsc) if len(test.err) == 0 { require.NoError(t, err) } else { require.EqualError(t, err, test.err) } if test.updated != nil { assert.Equal(t, *test.updated, *returned) } else { assert.Nil(t, returned) } }) } } func TestRemoveVSCProtect(t *testing.T) { vscObj := &snapshotv1api.VolumeSnapshotContent{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-vsc", Finalizers: []string{volumeSnapshotContentProtectFinalizer}, }, } tests := []struct { name string clientObj []runtime.Object reactors []reactor vsc string updated *snapshotv1api.VolumeSnapshotContent timeout time.Duration err string }{ { name: "get vsc error", vsc: "fake-vsc", err: "error to get VolumeSnapshotContent fake-vsc: volumesnapshotcontents.snapshot.storage.k8s.io \"fake-vsc\" not found", }, { name: "update vsc fail", vsc: "fake-vsc", clientObj: []runtime.Object{vscObj}, reactors: []reactor{ { verb: "update", resource: "volumesnapshotcontents", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, errors.New("fake-update-error") }, }, }, err: "error to update VolumeSnapshotContent fake-vsc: fake-update-error", }, { name: "update vsc timeout", vsc: "fake-vsc", clientObj: []runtime.Object{vscObj}, reactors: []reactor{ { verb: "update", resource: "volumesnapshotcontents", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, &apierrors.StatusError{ErrStatus: metav1.Status{ Reason: metav1.StatusReasonConflict, }} }, }, }, timeout: time.Second, err: "context deadline exceeded", }, { name: "succeed", vsc: "fake-vsc", clientObj: []runtime.Object{vscObj}, timeout: time.Second, updated: &snapshotv1api.VolumeSnapshotContent{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-vsc", Finalizers: []string{}, }, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeSnapshotClient := snapshotFake.NewSimpleClientset(test.clientObj...) for _, reactor := range test.reactors { fakeSnapshotClient.Fake.PrependReactor(reactor.verb, reactor.resource, reactor.reactorFunc) } err := RemoveVSCProtect(t.Context(), fakeSnapshotClient.SnapshotV1(), test.vsc, test.timeout) if len(test.err) == 0 { require.NoError(t, err) } else { require.EqualError(t, err, test.err) } if test.updated != nil { updated, err := fakeSnapshotClient.SnapshotV1().VolumeSnapshotContents().Get(t.Context(), test.vsc, metav1.GetOptions{}) require.NoError(t, err) assert.Equal(t, test.updated.Finalizers, updated.Finalizers) } }) } } func TestGetVolumeSnapshotClass(t *testing.T) { // backups backupFoo := &velerov1api.Backup{ ObjectMeta: metav1.ObjectMeta{ Name: "foo", Annotations: map[string]string{ "velero.io/csi-volumesnapshot-class_foo.csi.k8s.io": "foowithoutlabel", }, }, Spec: velerov1api.BackupSpec{ IncludedNamespaces: []string{"ns1", "ns2"}, }, } backupFoo2 := &velerov1api.Backup{ ObjectMeta: metav1.ObjectMeta{ Name: "foo2", Annotations: map[string]string{ "velero.io/csi-volumesnapshot-class_foo.csi.k8s.io": "foo2", }, }, Spec: velerov1api.BackupSpec{ IncludedNamespaces: []string{"ns1", "ns2"}, }, } backupBar2 := &velerov1api.Backup{ ObjectMeta: metav1.ObjectMeta{ Name: "bar", Annotations: map[string]string{ "velero.io/csi-volumesnapshot-class_bar.csi.k8s.io": "bar2", }, }, Spec: velerov1api.BackupSpec{ IncludedNamespaces: []string{"ns1", "ns2"}, }, } backupNone := &velerov1api.Backup{ ObjectMeta: metav1.ObjectMeta{ Name: "none", }, Spec: velerov1api.BackupSpec{ IncludedNamespaces: []string{"ns1", "ns2"}, }, } // pvcs pvcFoo := &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: "foo", Annotations: map[string]string{ "velero.io/csi-volumesnapshot-class": "foowithoutlabel", }, }, Spec: corev1api.PersistentVolumeClaimSpec{}, } pvcFoo2 := &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: "foo", Annotations: map[string]string{ "velero.io/csi-volumesnapshot-class": "foo2", }, }, Spec: corev1api.PersistentVolumeClaimSpec{}, } pvcNone := &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: "none", }, Spec: corev1api.PersistentVolumeClaimSpec{}, } // vsclasses hostpathClass := &snapshotv1api.VolumeSnapshotClass{ ObjectMeta: metav1.ObjectMeta{ Name: "hostpath", Labels: map[string]string{ velerov1api.VolumeSnapshotClassSelectorLabel: "foo", }, }, Driver: "hostpath.csi.k8s.io", } fooClass := &snapshotv1api.VolumeSnapshotClass{ ObjectMeta: metav1.ObjectMeta{ Name: "foo", Labels: map[string]string{ velerov1api.VolumeSnapshotClassSelectorLabel: "foo", }, }, Driver: "foo.csi.k8s.io", } fooClassWithoutLabel := &snapshotv1api.VolumeSnapshotClass{ ObjectMeta: metav1.ObjectMeta{ Name: "foowithoutlabel", }, Driver: "foo.csi.k8s.io", } barClass := &snapshotv1api.VolumeSnapshotClass{ ObjectMeta: metav1.ObjectMeta{ Name: "bar", Labels: map[string]string{ velerov1api.VolumeSnapshotClassSelectorLabel: "true", }, }, Driver: "bar.csi.k8s.io", } barClass2 := &snapshotv1api.VolumeSnapshotClass{ ObjectMeta: metav1.ObjectMeta{ Name: "bar2", Labels: map[string]string{ velerov1api.VolumeSnapshotClassSelectorLabel: "true", }, }, Driver: "bar.csi.k8s.io", } objs := []runtime.Object{hostpathClass, fooClass, barClass, fooClassWithoutLabel, barClass2} fakeClient := velerotest.NewFakeControllerRuntimeClient(t, objs...) testCases := []struct { name string driverName string pvc *corev1api.PersistentVolumeClaim backup *velerov1api.Backup expectedVSC *snapshotv1api.VolumeSnapshotClass expectError bool }{ { name: "no annotations on pvc and backup, should find hostpath volumesnapshotclass using default behavior of labels", driverName: "hostpath.csi.k8s.io", pvc: pvcNone, backup: backupNone, expectedVSC: hostpathClass, expectError: false, }, { name: "foowithoutlabel VSC annotations on pvc", driverName: "foo.csi.k8s.io", pvc: pvcFoo, backup: backupNone, expectedVSC: fooClassWithoutLabel, expectError: false, }, { name: "foowithoutlabel VSC annotations on pvc, but csi driver does not match, no annotation on backup so fallback to default behavior of labels", driverName: "bar.csi.k8s.io", pvc: pvcFoo, backup: backupNone, expectedVSC: barClass, expectError: false, }, { name: "foowithoutlabel VSC annotations on pvc, but csi driver does not match so fallback to fetch from backupAnnotations ", driverName: "bar.csi.k8s.io", pvc: pvcFoo, backup: backupBar2, expectedVSC: barClass2, expectError: false, }, { name: "foowithoutlabel VSC annotations on backup for foo.csi.k8s.io", driverName: "foo.csi.k8s.io", pvc: pvcNone, backup: backupFoo, expectedVSC: fooClassWithoutLabel, expectError: false, }, { name: "foowithoutlabel VSC annotations on backup for bar.csi.k8s.io, no annotation corresponding to foo.csi.k8s.io, so fallback to default behavior of labels", driverName: "bar.csi.k8s.io", pvc: pvcNone, backup: backupFoo, expectedVSC: barClass, expectError: false, }, { name: "no snapshotClass for given driver", driverName: "blah.csi.k8s.io", pvc: pvcNone, backup: backupNone, expectedVSC: nil, expectError: true, }, { name: "foo2 VSC annotations on pvc, but doesn't exist in cluster, fallback to default behavior of labels", driverName: "foo.csi.k8s.io", pvc: pvcFoo2, backup: backupFoo2, expectedVSC: fooClass, expectError: false, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { actualSnapshotClass, actualError := GetVolumeSnapshotClass( tc.driverName, tc.backup, tc.pvc, logrus.New(), fakeClient) if tc.expectError { require.Error(t, actualError) assert.Nil(t, actualSnapshotClass) return } assert.Equal(t, tc.expectedVSC, actualSnapshotClass) }) } } func TestGetVolumeSnapshotClassForStorageClass(t *testing.T) { hostpathClass := &snapshotv1api.VolumeSnapshotClass{ ObjectMeta: metav1.ObjectMeta{ Name: "hostpath", Labels: map[string]string{ velerov1api.VolumeSnapshotClassSelectorLabel: "foo", }, }, Driver: "hostpath.csi.k8s.io", } fooClass := &snapshotv1api.VolumeSnapshotClass{ ObjectMeta: metav1.ObjectMeta{ Name: "foo", Labels: map[string]string{ velerov1api.VolumeSnapshotClassSelectorLabel: "foo", }, }, Driver: "foo.csi.k8s.io", } barClass := &snapshotv1api.VolumeSnapshotClass{ ObjectMeta: metav1.ObjectMeta{ Name: "bar", Labels: map[string]string{ velerov1api.VolumeSnapshotClassSelectorLabel: "foo", }, }, Driver: "bar.csi.k8s.io", } bazClass := &snapshotv1api.VolumeSnapshotClass{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", }, Driver: "baz.csi.k8s.io", } ambClass1 := &snapshotv1api.VolumeSnapshotClass{ ObjectMeta: metav1.ObjectMeta{ Name: "amb1", }, Driver: "amb.csi.k8s.io", } ambClass2 := &snapshotv1api.VolumeSnapshotClass{ ObjectMeta: metav1.ObjectMeta{ Name: "amb2", }, Driver: "amb.csi.k8s.io", } snapshotClasses := &snapshotv1api.VolumeSnapshotClassList{ Items: []snapshotv1api.VolumeSnapshotClass{ *hostpathClass, *fooClass, *barClass, *bazClass, *ambClass1, *ambClass2}, } testCases := []struct { name string driverName string expectedVSC *snapshotv1api.VolumeSnapshotClass expectError bool }{ { name: "should find hostpath volumesnapshotclass", driverName: "hostpath.csi.k8s.io", expectedVSC: hostpathClass, expectError: false, }, { name: "should find foo volumesnapshotclass", driverName: "foo.csi.k8s.io", expectedVSC: fooClass, expectError: false, }, { name: "should find bar volumesnapshotclass", driverName: "bar.csi.k8s.io", expectedVSC: barClass, expectError: false, }, { name: "should find baz volumesnapshotclass without \"velero.io/csi-volumesnapshot-class\" label, b/c there's only one vsclass matching the driver name", driverName: "baz.csi.k8s.io", expectedVSC: bazClass, expectError: false, }, { name: "should not find amb volumesnapshotclass without \"velero.io/csi-volumesnapshot-class\" label, b/c there're more than one vsclass matching the driver name", driverName: "amb.csi.k8s.io", expectedVSC: nil, expectError: true, }, { name: "should not find does-not-exist volumesnapshotclass", driverName: "not-found.csi.k8s.io", expectedVSC: nil, expectError: true, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { actualVSC, actualError := GetVolumeSnapshotClassForStorageClass(tc.driverName, snapshotClasses) if tc.expectError { require.Error(t, actualError) assert.Nil(t, actualVSC) return } assert.Equalf(t, tc.expectedVSC.Name, actualVSC.Name, "unexpected volumesnapshotclass name returned. Want: %s; Got:%s", tc.expectedVSC.Name, actualVSC.Name) assert.Equalf(t, tc.expectedVSC.Driver, actualVSC.Driver, "unexpected driver name returned. Want: %s; Got:%s", tc.expectedVSC.Driver, actualVSC.Driver) }) } } func TestIsVolumeSnapshotClassHasListerSecret(t *testing.T) { testCases := []struct { name string snapClass snapshotv1api.VolumeSnapshotClass expected bool }{ { name: "should find both annotations", snapClass: snapshotv1api.VolumeSnapshotClass{ ObjectMeta: metav1.ObjectMeta{ Name: "class-1", Annotations: map[string]string{ velerov1api.PrefixedListSecretNameAnnotation: "snapListSecret", velerov1api.PrefixedListSecretNamespaceAnnotation: "awesome-ns", }, }, }, expected: true, }, { name: "should not find both annotations name is missing", snapClass: snapshotv1api.VolumeSnapshotClass{ ObjectMeta: metav1.ObjectMeta{ Name: "class-1", Annotations: map[string]string{ "foo": "snapListSecret", velerov1api.PrefixedListSecretNamespaceAnnotation: "awesome-ns", }, }, }, expected: false, }, { name: "should not find both annotations namespace is missing", snapClass: snapshotv1api.VolumeSnapshotClass{ ObjectMeta: metav1.ObjectMeta{ Name: "class-1", Annotations: map[string]string{ velerov1api.PrefixedListSecretNameAnnotation: "snapListSecret", "foo": "awesome-ns", }, }, }, expected: false, }, { name: "should not find expected annotation non-empty annotation", snapClass: snapshotv1api.VolumeSnapshotClass{ ObjectMeta: metav1.ObjectMeta{ Name: "class-2", Annotations: map[string]string{ "foo": "snapListSecret", "bar": "awesome-ns", }, }, }, expected: false, }, { name: "should not find expected annotation nil annotation", snapClass: snapshotv1api.VolumeSnapshotClass{ ObjectMeta: metav1.ObjectMeta{ Name: "class-3", Annotations: nil, }, }, expected: false, }, { name: "should not find expected annotation empty annotation", snapClass: snapshotv1api.VolumeSnapshotClass{ ObjectMeta: metav1.ObjectMeta{ Name: "class-3", Annotations: map[string]string{}, }, }, expected: false, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { actual := IsVolumeSnapshotClassHasListerSecret(&tc.snapClass) assert.Equal(t, tc.expected, actual) }) } } func TestIsVolumeSnapshotContentHasDeleteSecret(t *testing.T) { testCases := []struct { name string vsc snapshotv1api.VolumeSnapshotContent expected bool }{ { name: "should find both annotations", vsc: snapshotv1api.VolumeSnapshotContent{ ObjectMeta: metav1.ObjectMeta{ Name: "vsc-1", Annotations: map[string]string{ velerov1api.PrefixedSecretNameAnnotation: "delSnapSecret", velerov1api.PrefixedSecretNamespaceAnnotation: "awesome-ns", }, }, }, expected: true, }, { name: "should not find both annotations name is missing", vsc: snapshotv1api.VolumeSnapshotContent{ ObjectMeta: metav1.ObjectMeta{ Name: "vsc-2", Annotations: map[string]string{ "foo": "delSnapSecret", velerov1api.PrefixedSecretNamespaceAnnotation: "awesome-ns", }, }, }, expected: false, }, { name: "should not find both annotations namespace is missing", vsc: snapshotv1api.VolumeSnapshotContent{ ObjectMeta: metav1.ObjectMeta{ Name: "vsc-3", Annotations: map[string]string{ velerov1api.PrefixedSecretNameAnnotation: "delSnapSecret", "foo": "awesome-ns", }, }, }, expected: false, }, { name: "should not find expected annotation non-empty annotation", vsc: snapshotv1api.VolumeSnapshotContent{ ObjectMeta: metav1.ObjectMeta{ Name: "vsc-4", Annotations: map[string]string{ "foo": "delSnapSecret", "bar": "awesome-ns", }, }, }, expected: false, }, { name: "should not find expected annotation empty annotation", vsc: snapshotv1api.VolumeSnapshotContent{ ObjectMeta: metav1.ObjectMeta{ Name: "vsc-5", Annotations: map[string]string{}, }, }, expected: false, }, { name: "should not find expected annotation nil annotation", vsc: snapshotv1api.VolumeSnapshotContent{ ObjectMeta: metav1.ObjectMeta{ Name: "vsc-6", Annotations: nil, }, }, expected: false, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { actual := IsVolumeSnapshotContentHasDeleteSecret(&tc.vsc) assert.Equal(t, tc.expected, actual) }) } } func TestIsVolumeSnapshotExists(t *testing.T) { vsExists := &snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: "vs-exists", Namespace: "default", }, } vsNotExists := &snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: "vs-does-not-exists", Namespace: "default", }, } objs := []runtime.Object{vsExists} fakeClient := velerotest.NewFakeControllerRuntimeClient(t, objs...) testCases := []struct { name string expected bool vs *snapshotv1api.VolumeSnapshot }{ { name: "should find existing VolumeSnapshot object", expected: true, vs: vsExists, }, { name: "should not find non-existing VolumeSnapshot object", expected: false, vs: vsNotExists, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { actual := IsVolumeSnapshotExists(tc.vs.Namespace, tc.vs.Name, fakeClient) assert.Equal(t, tc.expected, actual) }) } } func TestSetVolumeSnapshotContentDeletionPolicy(t *testing.T) { testCases := []struct { name string inputVSCName string policy snapshotv1api.DeletionPolicy objs []runtime.Object expectError bool }{ { name: "should update DeletionPolicy of a VSC from retain to delete", inputVSCName: "retainVSC", policy: snapshotv1api.VolumeSnapshotContentDelete, objs: []runtime.Object{ &snapshotv1api.VolumeSnapshotContent{ ObjectMeta: metav1.ObjectMeta{ Name: "retainVSC", }, Spec: snapshotv1api.VolumeSnapshotContentSpec{ DeletionPolicy: snapshotv1api.VolumeSnapshotContentRetain, }, }, }, expectError: false, }, { name: "should be a no-op updating if DeletionPolicy of a VSC is already Delete", inputVSCName: "deleteVSC", policy: snapshotv1api.VolumeSnapshotContentDelete, objs: []runtime.Object{ &snapshotv1api.VolumeSnapshotContent{ ObjectMeta: metav1.ObjectMeta{ Name: "deleteVSC", }, Spec: snapshotv1api.VolumeSnapshotContentSpec{ DeletionPolicy: snapshotv1api.VolumeSnapshotContentDelete, }, }, }, expectError: false, }, { name: "should update DeletionPolicy of a VSC with no DeletionPolicy", inputVSCName: "nothingVSC", policy: snapshotv1api.VolumeSnapshotContentDelete, objs: []runtime.Object{ &snapshotv1api.VolumeSnapshotContent{ ObjectMeta: metav1.ObjectMeta{ Name: "nothingVSC", }, Spec: snapshotv1api.VolumeSnapshotContentSpec{}, }, }, expectError: false, }, { name: "should return not found error if supplied VSC does not exist", inputVSCName: "does-not-exist", objs: []runtime.Object{}, expectError: true, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { fakeClient := velerotest.NewFakeControllerRuntimeClient(t, tc.objs...) _, err := SetVolumeSnapshotContentDeletionPolicy(tc.inputVSCName, fakeClient, tc.policy) if tc.expectError { assert.Error(t, err) } else { require.NoError(t, err) actual := new(snapshotv1api.VolumeSnapshotContent) err := fakeClient.Get( t.Context(), crclient.ObjectKey{Name: tc.inputVSCName}, actual, ) require.NoError(t, err) assert.Equal( t, tc.policy, actual.Spec.DeletionPolicy, ) } }) } } func TestDeleteVolumeSnapshots(t *testing.T) { tests := []struct { name string vs snapshotv1api.VolumeSnapshot vsc snapshotv1api.VolumeSnapshotContent keepVSAndVSC bool }{ { name: "VS is ReadyToUse, and VS has corresponding VSC. VS should be deleted.", vs: *builder.ForVolumeSnapshot("velero", "vs1"). ObjectMeta(builder.WithLabels("testing-vs", "vs1")). Status().BoundVolumeSnapshotContentName("vsc1").Result(), vsc: *builder.ForVolumeSnapshotContent("vsc1"). DeletionPolicy(snapshotv1api.VolumeSnapshotContentDelete). Status(&snapshotv1api.VolumeSnapshotContentStatus{}).Result(), }, { name: "VS status is nil. VSC should not be modified.", vs: *builder.ForVolumeSnapshot("velero", "vs1"). ObjectMeta(builder.WithLabels("testing-vs", "vs1")).Result(), vsc: *builder.ForVolumeSnapshotContent("vsc1"). DeletionPolicy(snapshotv1api.VolumeSnapshotContentDelete). Status(&snapshotv1api.VolumeSnapshotContentStatus{}).Result(), keepVSAndVSC: true, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { client := velerotest.NewFakeControllerRuntimeClient( t, []runtime.Object{&tc.vs, &tc.vsc}..., ) logger := logging.DefaultLogger(logrus.DebugLevel, logging.FormatText) DeleteReadyVolumeSnapshot(tc.vs, client, logger) vsList := new(snapshotv1api.VolumeSnapshotList) err := client.List( t.Context(), vsList, &crclient.ListOptions{ Namespace: "velero", }, ) require.NoError(t, err) vscList := new(snapshotv1api.VolumeSnapshotContentList) err = client.List( t.Context(), vscList, ) require.NoError(t, err) if tc.keepVSAndVSC { require.Equal(t, crclient.ObjectKeyFromObject(&tc.vs), crclient.ObjectKeyFromObject(&vsList.Items[0])) require.Equal(t, crclient.ObjectKeyFromObject(&tc.vsc), crclient.ObjectKeyFromObject(&vscList.Items[0])) } else { require.Empty(t, vsList.Items) require.Empty(t, vscList.Items) } }) } } func TestWaitUntilVSCHandleIsReady(t *testing.T) { vscName := "snapcontent-7d1bdbd1-d10d-439c-8d8e-e1c2565ddc53" snapshotHandle := "snapshot-handle" vscObj := &snapshotv1api.VolumeSnapshotContent{ ObjectMeta: metav1.ObjectMeta{ Name: vscName, }, Spec: snapshotv1api.VolumeSnapshotContentSpec{ VolumeSnapshotRef: corev1api.ObjectReference{ Name: "vol-snap-1", APIVersion: snapshotv1api.SchemeGroupVersion.String(), }, }, Status: &snapshotv1api.VolumeSnapshotContentStatus{ SnapshotHandle: &snapshotHandle, }, } validVS := &snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: "vs", Namespace: "default", }, Status: &snapshotv1api.VolumeSnapshotStatus{ BoundVolumeSnapshotContentName: &vscName, }, } notFound := "does-not-exist" vsWithVSCNotFound := &snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: notFound, Namespace: "default", }, Status: &snapshotv1api.VolumeSnapshotStatus{ BoundVolumeSnapshotContentName: ¬Found, }, } vsWithNilStatus := &snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: "nil-status-vs", Namespace: "default", }, Status: nil, } vsWithNilStatusField := &snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: "nil-status-field-vs", Namespace: "default", }, Status: &snapshotv1api.VolumeSnapshotStatus{ BoundVolumeSnapshotContentName: nil, }, } nilStatusVsc := "nil-status-vsc" vscWithNilStatus := &snapshotv1api.VolumeSnapshotContent{ ObjectMeta: metav1.ObjectMeta{ Name: nilStatusVsc, }, Spec: snapshotv1api.VolumeSnapshotContentSpec{ VolumeSnapshotRef: corev1api.ObjectReference{ Name: "vol-snap-1", APIVersion: snapshotv1api.SchemeGroupVersion.String(), }, }, Status: nil, } vsForNilStatusVsc := &snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: "vs-for-nil-status-vsc", Namespace: "default", }, Status: &snapshotv1api.VolumeSnapshotStatus{ BoundVolumeSnapshotContentName: &nilStatusVsc, }, } nilStatusFieldVsc := "nil-status-field-vsc" vscWithNilStatusField := &snapshotv1api.VolumeSnapshotContent{ ObjectMeta: metav1.ObjectMeta{ Name: nilStatusFieldVsc, }, Spec: snapshotv1api.VolumeSnapshotContentSpec{ VolumeSnapshotRef: corev1api.ObjectReference{ Name: "vol-snap-1", APIVersion: snapshotv1api.SchemeGroupVersion.String(), }, }, Status: &snapshotv1api.VolumeSnapshotContentStatus{ SnapshotHandle: nil, }, } vsForNilStatusFieldVsc := &snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: "vs-for-nil-status-field", Namespace: "default", }, Status: &snapshotv1api.VolumeSnapshotStatus{ BoundVolumeSnapshotContentName: &nilStatusFieldVsc, }, } objs := []runtime.Object{ vscObj, validVS, vsWithVSCNotFound, vsWithNilStatus, vsWithNilStatusField, vscWithNilStatus, vsForNilStatusVsc, vscWithNilStatusField, vsForNilStatusFieldVsc, } fakeClient := velerotest.NewFakeControllerRuntimeClient(t, objs...) testCases := []struct { name string volSnap *snapshotv1api.VolumeSnapshot exepctedVSC *snapshotv1api.VolumeSnapshotContent expectError bool }{ { name: "waitEnabled should find volumesnapshotcontent for volumesnapshot", volSnap: validVS, exepctedVSC: vscObj, expectError: false, }, { name: "waitEnabled should not find volumesnapshotcontent for volumesnapshot with non-existing snapshotcontent name in status.BoundVolumeSnapshotContentName", volSnap: vsWithVSCNotFound, exepctedVSC: nil, expectError: true, }, { name: "waitEnabled should not find volumesnapshotcontent for a non-existent volumesnapshot", exepctedVSC: nil, expectError: true, volSnap: &snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: "not-found", Namespace: "default", }, Status: &snapshotv1api.VolumeSnapshotStatus{ BoundVolumeSnapshotContentName: &nilStatusVsc, }, }, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { actualVSC, actualError := WaitUntilVSCHandleIsReady(tc.volSnap, fakeClient, logrus.New().WithField("fake", "test"), 0) if tc.expectError && actualError == nil { require.Error(t, actualError) assert.Nil(t, actualVSC) return } assert.Equal(t, tc.exepctedVSC, actualVSC) }) } } func TestDiagnoseVS(t *testing.T) { vscName := "fake-vsc" readyToUse := true message := "fake-message" testCases := []struct { name string vs *snapshotv1api.VolumeSnapshot events *corev1api.EventList expected string }{ { name: "VS with no status", vs: &snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-vs", Namespace: "fake-ns", }, }, expected: "VS fake-ns/fake-vs, bind to , readyToUse false, errMessage \n", }, { name: "VS with empty status", vs: &snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-vs", Namespace: "fake-ns", }, Status: &snapshotv1api.VolumeSnapshotStatus{}, }, expected: "VS fake-ns/fake-vs, bind to , readyToUse false, errMessage \n", }, { name: "VS with VSC name", vs: &snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-vs", Namespace: "fake-ns", }, Status: &snapshotv1api.VolumeSnapshotStatus{ BoundVolumeSnapshotContentName: &vscName, }, }, expected: "VS fake-ns/fake-vs, bind to fake-vsc, readyToUse false, errMessage \n", }, { name: "VS with VSC name+ready", vs: &snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-vs", Namespace: "fake-ns", }, Status: &snapshotv1api.VolumeSnapshotStatus{ BoundVolumeSnapshotContentName: &vscName, ReadyToUse: &readyToUse, }, }, expected: "VS fake-ns/fake-vs, bind to fake-vsc, readyToUse true, errMessage \n", }, { name: "VS with VSC name+ready+empty error", vs: &snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-vs", Namespace: "fake-ns", }, Status: &snapshotv1api.VolumeSnapshotStatus{ BoundVolumeSnapshotContentName: &vscName, ReadyToUse: &readyToUse, Error: &snapshotv1api.VolumeSnapshotError{}, }, }, expected: "VS fake-ns/fake-vs, bind to fake-vsc, readyToUse true, errMessage \n", }, { name: "VS with VSC name+ready+error", vs: &snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-vs", Namespace: "fake-ns", }, Status: &snapshotv1api.VolumeSnapshotStatus{ BoundVolumeSnapshotContentName: &vscName, ReadyToUse: &readyToUse, Error: &snapshotv1api.VolumeSnapshotError{ Message: &message, }, }, }, expected: "VS fake-ns/fake-vs, bind to fake-vsc, readyToUse true, errMessage fake-message\n", }, { name: "VS with VSC and empty event", vs: &snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-vs", Namespace: "fake-ns", }, Status: &snapshotv1api.VolumeSnapshotStatus{ BoundVolumeSnapshotContentName: &vscName, ReadyToUse: &readyToUse, Error: &snapshotv1api.VolumeSnapshotError{}, }, }, events: &corev1api.EventList{}, expected: "VS fake-ns/fake-vs, bind to fake-vsc, readyToUse true, errMessage \n", }, { name: "VS with VSC and events", vs: &snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-vs", Namespace: "fake-ns", UID: "fake-vs-uid", }, Status: &snapshotv1api.VolumeSnapshotStatus{ BoundVolumeSnapshotContentName: &vscName, ReadyToUse: &readyToUse, Error: &snapshotv1api.VolumeSnapshotError{}, }, }, events: &corev1api.EventList{Items: []corev1api.Event{ { InvolvedObject: corev1api.ObjectReference{UID: "fake-uid-1"}, Type: corev1api.EventTypeWarning, Reason: "reason-1", Message: "message-1", }, { InvolvedObject: corev1api.ObjectReference{UID: "fake-uid-2"}, Type: corev1api.EventTypeWarning, Reason: "reason-2", Message: "message-2", }, { InvolvedObject: corev1api.ObjectReference{UID: "fake-vs-uid"}, Type: corev1api.EventTypeWarning, Reason: "reason-3", Message: "message-3", }, { InvolvedObject: corev1api.ObjectReference{UID: "fake-vs-uid"}, Type: corev1api.EventTypeNormal, Reason: "reason-4", Message: "message-4", }, { InvolvedObject: corev1api.ObjectReference{UID: "fake-vs-uid"}, Type: corev1api.EventTypeNormal, Reason: "reason-5", Message: "message-5", }, { InvolvedObject: corev1api.ObjectReference{UID: "fake-vs-uid"}, Type: corev1api.EventTypeWarning, Reason: "reason-6", Message: "message-6", }, }}, expected: "VS fake-ns/fake-vs, bind to fake-vsc, readyToUse true, errMessage \nVS event reason reason-3, message message-3\nVS event reason reason-6, message message-6\n", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { diag := DiagnoseVS(tc.vs, tc.events) assert.Equal(t, tc.expected, diag) }) } } func TestDiagnoseVSC(t *testing.T) { readyToUse := true message := "fake-message" handle := "fake-handle" testCases := []struct { name string vsc *snapshotv1api.VolumeSnapshotContent expected string }{ { name: "VS with no status", vsc: &snapshotv1api.VolumeSnapshotContent{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-vsc", }, }, expected: "VSC fake-vsc, readyToUse false, errMessage , handle \n", }, { name: "VSC with empty status", vsc: &snapshotv1api.VolumeSnapshotContent{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-vsc", }, Status: &snapshotv1api.VolumeSnapshotContentStatus{}, }, expected: "VSC fake-vsc, readyToUse false, errMessage , handle \n", }, { name: "VSC with ready", vsc: &snapshotv1api.VolumeSnapshotContent{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-vsc", }, Status: &snapshotv1api.VolumeSnapshotContentStatus{ ReadyToUse: &readyToUse, }, }, expected: "VSC fake-vsc, readyToUse true, errMessage , handle \n", }, { name: "VSC with ready+handle", vsc: &snapshotv1api.VolumeSnapshotContent{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-vsc", }, Status: &snapshotv1api.VolumeSnapshotContentStatus{ ReadyToUse: &readyToUse, SnapshotHandle: &handle, }, }, expected: "VSC fake-vsc, readyToUse true, errMessage , handle fake-handle\n", }, { name: "VSC with ready+handle+empty error", vsc: &snapshotv1api.VolumeSnapshotContent{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-vsc", }, Status: &snapshotv1api.VolumeSnapshotContentStatus{ ReadyToUse: &readyToUse, SnapshotHandle: &handle, Error: &snapshotv1api.VolumeSnapshotError{}, }, }, expected: "VSC fake-vsc, readyToUse true, errMessage , handle fake-handle\n", }, { name: "VSC with ready+handle+error", vsc: &snapshotv1api.VolumeSnapshotContent{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-vsc", }, Status: &snapshotv1api.VolumeSnapshotContentStatus{ ReadyToUse: &readyToUse, SnapshotHandle: &handle, Error: &snapshotv1api.VolumeSnapshotError{ Message: &message, }, }, }, expected: "VSC fake-vsc, readyToUse true, errMessage fake-message, handle fake-handle\n", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { diag := DiagnoseVSC(tc.vsc) assert.Equal(t, tc.expected, diag) }) } } func TestGetVSCForVS(t *testing.T) { testCases := []struct { name string vs *snapshotv1api.VolumeSnapshot vsc *snapshotv1api.VolumeSnapshotContent expectedErr string expectedVSC *snapshotv1api.VolumeSnapshotContent }{ { name: "vs has no status", vs: builder.ForVolumeSnapshot("ns1", "vs1").Result(), expectedErr: "invalid snapshot info in volume snapshot vs1", }, { name: "vs has no bound vsc", vs: builder.ForVolumeSnapshot("ns1", "vs1").Status().Result(), expectedErr: "invalid snapshot info in volume snapshot vs1", }, { name: "vs bound vsc cannot be found", vs: builder.ForVolumeSnapshot("ns1", "vs1").Status().BoundVolumeSnapshotContentName("vsc1").Result(), expectedErr: "error getting volume snapshot content from API: volumesnapshotcontents.snapshot.storage.k8s.io \"vsc1\" not found", }, { name: "normal case", vs: builder.ForVolumeSnapshot("ns1", "vs1").Status().BoundVolumeSnapshotContentName("vsc1").Result(), vsc: builder.ForVolumeSnapshotContent("vsc1").Result(), expectedVSC: builder.ForVolumeSnapshotContent("vsc1").Result(), }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { objs := []runtime.Object{tc.vs} if tc.vsc != nil { objs = append(objs, tc.vsc) } client := test.NewFakeControllerRuntimeClient(t, objs...) vsc, err := GetVSCForVS(t.Context(), tc.vs, client) if tc.expectedErr != "" { require.EqualError(t, err, tc.expectedErr) } else { require.NoError(t, err) } if tc.expectedVSC != nil { require.True(t, cmp.Equal(tc.expectedVSC, vsc, cmpopts.IgnoreFields(snapshotv1api.VolumeSnapshotContent{}, "ResourceVersion"))) } }) } } ================================================ FILE: pkg/util/encode/encode.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package encode import ( "bytes" "compress/gzip" "encoding/json" "fmt" "io" "github.com/pkg/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/util" ) // Encode converts the provided object to the specified format // and returns a byte slice of the encoded data. func Encode(obj runtime.Object, format string) ([]byte, error) { buf := new(bytes.Buffer) if err := To(obj, format, buf); err != nil { return nil, err } return buf.Bytes(), nil } // To converts the provided object to the specified format and // writes the encoded data to the provided io.Writer. func To(obj runtime.Object, format string, w io.Writer) error { encoder, err := EncoderFor(format, obj) if err != nil { return err } return errors.WithStack(encoder.Encode(obj, w)) } // EncoderFor gets the appropriate encoder for the specified format. // Only objects registered in the velero scheme, or objects with their TypeMeta set will have valid encoders. func EncoderFor(format string, obj runtime.Object) (runtime.Encoder, error) { var encoder runtime.Encoder codecFactory := serializer.NewCodecFactory(util.VeleroScheme) desiredMediaType := fmt.Sprintf("application/%s", format) serializerInfo, found := runtime.SerializerInfoForMediaType(codecFactory.SupportedMediaTypes(), desiredMediaType) if !found { return nil, errors.Errorf("unable to locate an encoder for %q", desiredMediaType) } if serializerInfo.PrettySerializer != nil { encoder = serializerInfo.PrettySerializer } else { encoder = serializerInfo.Serializer } if !obj.GetObjectKind().GroupVersionKind().Empty() { return encoder, nil } encoder = codecFactory.EncoderForVersion(encoder, v1.SchemeGroupVersion) return encoder, nil } // ToJSONGzip takes arbitrary Go data and encodes it to GZip compressed JSON in a buffer, as well as a description of the data to put into an error should encoding fail. func ToJSONGzip(data any, desc string) (*bytes.Buffer, []error) { buf := new(bytes.Buffer) gzw := gzip.NewWriter(buf) // Since both encoding and closing the gzip writer could fail separately and both errors are useful, // collect both errors to report back. errs := []error{} if err := json.NewEncoder(gzw).Encode(data); err != nil { errs = append(errs, errors.Wrapf(err, "error encoding %s", desc)) } if err := gzw.Close(); err != nil { errs = append(errs, errors.Wrapf(err, "error closing gzip writer for %s", desc)) } if len(errs) > 0 { return nil, errs } return buf, nil } ================================================ FILE: pkg/util/exec/exec.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package exec import ( "bytes" "io" "os/exec" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) // RunCommand runs a command and returns its stdout, stderr, and its returned // error (if any). If there are errors reading stdout or stderr, their return // value(s) will contain the error as a string. func RunCommand(cmd *exec.Cmd) (string, string, error) { stdoutBuf := new(bytes.Buffer) stderrBuf := new(bytes.Buffer) cmd.Stdout = stdoutBuf cmd.Stderr = stderrBuf runErr := cmd.Run() var stdout, stderr string if res, readErr := io.ReadAll(stdoutBuf); readErr != nil { stdout = errors.Wrap(readErr, "error reading command's stdout").Error() } else { stdout = string(res) } if res, readErr := io.ReadAll(stderrBuf); readErr != nil { stderr = errors.Wrap(readErr, "error reading command's stderr").Error() } else { stderr = string(res) } return stdout, stderr, runErr } func RunCommandWithLog(cmd *exec.Cmd, log logrus.FieldLogger) (string, string, error) { stdout, stderr, err := RunCommand(cmd) LogErrorAsExitCode(err, log) return stdout, stderr, err } func LogErrorAsExitCode(err error, log logrus.FieldLogger) { if err != nil { if exitError, ok := err.(*exec.ExitError); ok { log.Errorf("Restic command fail with ExitCode: %d. Process ID is %d, Exit error is: %s", exitError.ExitCode(), exitError.Pid(), exitError.String()) // Golang's os.exec -1 ExitCode means signal kill. Usually this is caused // by CGroup's OOM. Log a warning to notice user. // https://github.com/golang/go/blob/master/src/os/exec_posix.go#L128-L136 if exitError.ExitCode() == -1 { log.Warnf("The ExitCode is -1, which means the process is terminated by signal. Usually this is caused by CGroup kill due to out of memory. Please check whether there is such information in the work nodes' dmesg log.") } } else { log.WithError(err).Info("Error cannot be convert to ExitError format.") } } } ================================================ FILE: pkg/util/filesystem/file_system.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package filesystem import ( "io" "os" "path/filepath" ) // Interface defines methods for interacting with an // underlying file system. type Interface interface { TempDir(dir, prefix string) (string, error) MkdirAll(path string, perm os.FileMode) error Create(name string) (io.WriteCloser, error) OpenFile(name string, flag int, perm os.FileMode) (io.WriteCloser, error) RemoveAll(path string) error ReadDir(dirname string) ([]os.FileInfo, error) ReadFile(filename string) ([]byte, error) DirExists(path string) (bool, error) TempFile(dir, prefix string) (NameWriteCloser, error) Stat(path string) (os.FileInfo, error) Glob(path string) ([]string, error) } type NameWriteCloser interface { io.WriteCloser Name() string } func NewFileSystem() Interface { return &osFileSystem{} } type osFileSystem struct{} func (fs *osFileSystem) Glob(path string) ([]string, error) { return filepath.Glob(path) } func (fs *osFileSystem) TempDir(dir, prefix string) (string, error) { return os.MkdirTemp(dir, prefix) } func (fs *osFileSystem) MkdirAll(path string, perm os.FileMode) error { return os.MkdirAll(path, perm) } func (fs *osFileSystem) Create(name string) (io.WriteCloser, error) { return os.Create(name) } func (fs *osFileSystem) OpenFile(name string, flag int, perm os.FileMode) (io.WriteCloser, error) { return os.OpenFile(name, flag, perm) } func (fs *osFileSystem) RemoveAll(path string) error { return os.RemoveAll(path) } func (fs *osFileSystem) ReadDir(dirname string) ([]os.FileInfo, error) { var fileInfos []os.FileInfo dirInfos, ise := os.ReadDir(dirname) if ise != nil { return fileInfos, ise } for _, dirInfo := range dirInfos { fileInfo, ise := dirInfo.Info() if ise == nil { fileInfos = append(fileInfos, fileInfo) } } return fileInfos, nil } func (fs *osFileSystem) ReadFile(filename string) ([]byte, error) { return os.ReadFile(filename) } func (fs *osFileSystem) DirExists(path string) (bool, error) { _, err := os.Stat(path) if err == nil { return true, nil } if os.IsNotExist(err) { return false, nil } return false, err } func (fs *osFileSystem) TempFile(dir, prefix string) (NameWriteCloser, error) { return os.CreateTemp(dir, prefix) } func (fs *osFileSystem) Stat(path string) (os.FileInfo, error) { return os.Stat(path) } ================================================ FILE: pkg/util/kube/client.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package kube import ( "context" "time" "sigs.k8s.io/controller-runtime/pkg/client" veleroPkgClient "github.com/vmware-tanzu/velero/pkg/client" ) func PatchResource(original, updated client.Object, kbClient client.Client) error { err := kbClient.Patch(context.Background(), updated, client.MergeFrom(original)) return err } // PatchResourceWithRetries patches the original resource with the updated resource, retrying when the provided retriable function returns true. func PatchResourceWithRetries(maxDuration time.Duration, original, updated client.Object, kbClient client.Client, retriable func(error) bool) error { return veleroPkgClient.RetryOnRetriableMaxBackOff(maxDuration, func() error { return PatchResource(original, updated, kbClient) }, retriable) } // PatchResourceWithRetriesOnErrors patches the original resource with the updated resource, retrying when the operation returns an error. func PatchResourceWithRetriesOnErrors(maxDuration time.Duration, original, updated client.Object, kbClient client.Client) error { return PatchResourceWithRetries(maxDuration, original, updated, kbClient, func(err error) bool { // retry using DefaultBackoff to resolve connection refused error that may occur when the server is under heavy load // TODO: consider using a more specific error type to retry, for now, we retry on all errors // specific errors: // - connection refused: https://pkg.go.dev/syscall#:~:text=Errno(0x67)-,ECONNREFUSED,-%3D%20Errno(0x6f return err != nil }) } ================================================ FILE: pkg/util/kube/event.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package kube import ( "context" "math" "sync" "time" "github.com/google/uuid" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes" typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/tools/record" ) type EventRecorder interface { Event(object runtime.Object, warning bool, reason string, message string, a ...any) EndingEvent(object runtime.Object, warning bool, reason string, message string, a ...any) Shutdown() } type eventRecorder struct { broadcaster record.EventBroadcaster recorder record.EventRecorder lock sync.Mutex endingSentinel *eventElement log logrus.FieldLogger } type eventElement struct { t string r string m string sinked chan struct{} } type eventSink struct { recorder *eventRecorder sink typedcorev1.EventInterface } func NewEventRecorder(kubeClient kubernetes.Interface, scheme *runtime.Scheme, eventSource string, eventNode string, log logrus.FieldLogger) EventRecorder { res := eventRecorder{ log: log, } res.broadcaster = record.NewBroadcasterWithCorrelatorOptions(record.CorrelatorOptions{ // Bypass the built-in EventCorrelator's rate filtering, otherwise, the event will be abandoned if the rate exceeds. // The callers (i.e., data mover pods) have controlled the rate and total number outside. E.g., the progress is designed to be updated every 10 seconds and is changeable. BurstSize: math.MaxInt32, MaxEvents: 1, MessageFunc: func(event *corev1api.Event) string { return event.Message }, }) res.broadcaster.StartRecordingToSink(&eventSink{ recorder: &res, sink: kubeClient.CoreV1().Events(""), }) res.recorder = res.broadcaster.NewRecorder(scheme, corev1api.EventSource{ Component: eventSource, Host: eventNode, }) return &res } func (er *eventRecorder) Event(object runtime.Object, warning bool, reason string, message string, a ...any) { if er.broadcaster == nil { return } eventType := corev1api.EventTypeNormal if warning { eventType = corev1api.EventTypeWarning } if len(a) > 0 { er.recorder.Eventf(object, eventType, reason, message, a...) } else { er.recorder.Event(object, eventType, reason, message) } } func (er *eventRecorder) EndingEvent(object runtime.Object, warning bool, reason string, message string, a ...any) { if er.broadcaster == nil { return } er.Event(object, warning, reason, message, a...) var sentinelEvent string er.lock.Lock() if er.endingSentinel == nil { sentinelEvent = uuid.NewString() er.endingSentinel = &eventElement{ t: corev1api.EventTypeNormal, r: sentinelEvent, m: sentinelEvent, sinked: make(chan struct{}), } } er.lock.Unlock() if sentinelEvent != "" { er.Event(object, false, sentinelEvent, sentinelEvent) } else { er.log.Warn("More than one ending events, ignore") } } var shutdownTimeout = time.Minute func (er *eventRecorder) Shutdown() { var wait chan struct{} er.lock.Lock() if er.endingSentinel != nil { wait = er.endingSentinel.sinked } er.lock.Unlock() if wait != nil { er.log.Info("Waiting sentinel before shutdown") waitloop: for { select { case <-wait: break waitloop case <-time.After(shutdownTimeout): er.log.Warn("Timeout waiting for assured events processed") break waitloop } } } er.broadcaster.Shutdown() er.broadcaster = nil er.lock.Lock() er.endingSentinel = nil er.lock.Unlock() } func (er *eventRecorder) sentinelWatch(event *corev1api.Event) bool { er.lock.Lock() defer er.lock.Unlock() if er.endingSentinel == nil { return false } if er.endingSentinel.m == event.Message && er.endingSentinel.r == event.Reason && er.endingSentinel.t == event.Type { close(er.endingSentinel.sinked) return true } return false } func (es *eventSink) Create(event *corev1api.Event) (*corev1api.Event, error) { if es.recorder.sentinelWatch(event) { return event, nil } return es.sink.CreateWithEventNamespaceWithContext(context.Background(), event) } func (es *eventSink) Update(event *corev1api.Event) (*corev1api.Event, error) { return es.sink.UpdateWithEventNamespaceWithContext(context.Background(), event) } func (es *eventSink) Patch(event *corev1api.Event, data []byte) (*corev1api.Event, error) { return es.sink.PatchWithEventNamespaceWithContext(context.Background(), event, data) } ================================================ FILE: pkg/util/kube/event_handler.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package kube import ( "context" "k8s.io/client-go/util/workqueue" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) // EnqueueRequestsFromMapUpdateFunc has the same purpose with handler.EnqueueRequestsFromMapFunc. // It's simpler on Update event because mapAndEnqueue is called once with the new object. EnqueueRequestsFromMapFunc is called twice with the old and new object. func EnqueueRequestsFromMapUpdateFunc(fn handler.MapFunc) handler.EventHandler { return TypedEnqueueRequestsFromMapUpdateFunc(fn) } func TypedEnqueueRequestsFromMapUpdateFunc[object any, request comparable](fn handler.TypedMapFunc[object, request]) handler.TypedEventHandler[object, request] { return &enqueueRequestsFromMapFunc[object, request]{ toRequests: fn, } } var _ handler.EventHandler = &enqueueRequestsFromMapFunc[client.Object, reconcile.Request]{} type enqueueRequestsFromMapFunc[object any, request comparable] struct { toRequests handler.TypedMapFunc[object, request] } // Create implements EventHandler. func (e *enqueueRequestsFromMapFunc[object, request]) Create(ctx context.Context, evt event.TypedCreateEvent[object], q workqueue.TypedRateLimitingInterface[request]) { e.mapAndEnqueue(ctx, q, evt.Object) } // Update implements EventHandler. func (e *enqueueRequestsFromMapFunc[object, request]) Update(ctx context.Context, evt event.TypedUpdateEvent[object], q workqueue.TypedRateLimitingInterface[request]) { e.mapAndEnqueue(ctx, q, evt.ObjectNew) } // Delete implements EventHandler. func (e *enqueueRequestsFromMapFunc[object, request]) Delete(ctx context.Context, evt event.TypedDeleteEvent[object], q workqueue.TypedRateLimitingInterface[request]) { e.mapAndEnqueue(ctx, q, evt.Object) } // Generic implements EventHandler. func (e *enqueueRequestsFromMapFunc[object, request]) Generic(ctx context.Context, evt event.TypedGenericEvent[object], q workqueue.TypedRateLimitingInterface[request]) { e.mapAndEnqueue(ctx, q, evt.Object) } func (e *enqueueRequestsFromMapFunc[object, request]) mapAndEnqueue(ctx context.Context, q workqueue.TypedRateLimitingInterface[request], obj object) { reqs := map[request]struct{}{} for _, req := range e.toRequests(ctx, obj) { _, ok := reqs[req] if !ok { q.Add(req) reqs[req] = struct{}{} } } } ================================================ FILE: pkg/util/kube/event_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package kube import ( "fmt" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/fake" corev1api "k8s.io/api/core/v1" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func TestEvent(t *testing.T) { type testEvent struct { warning bool reason string message string ending bool } cases := []struct { name string events []testEvent generateDiff int generateSame int generateEnding bool expected int }{ { name: "update events, different message", events: []testEvent{ { warning: false, reason: "Progress", message: "progress-1", }, { warning: false, reason: "Progress", message: "progress-2", ending: true, }, }, expected: 1, }, { name: "create events, different reason", events: []testEvent{ { warning: false, reason: "action-1-1", message: "fake-message", }, { warning: false, reason: "action-1-2", message: "fake-message", ending: true, }, }, expected: 2, }, { name: "create events, different warning", events: []testEvent{ { warning: false, reason: "action-2-1", message: "fake-message", }, { warning: true, reason: "action-2-1", message: "fake-message", ending: true, }, }, expected: 2, }, { name: "endingEvent, double entrance", events: []testEvent{ { warning: false, reason: "action-2-1", message: "fake-message", ending: true, }, { warning: true, reason: "action-2-1", message: "fake-message", ending: true, }, }, expected: -1, }, { name: "auto generate 200", generateDiff: 200, generateEnding: true, expected: 201, }, { name: "auto generate 200, update", generateSame: 200, generateEnding: true, expected: 2, }, } shutdownTimeout = time.Second * 5 for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { client := fake.NewSimpleClientset() scheme := runtime.NewScheme() err := corev1api.AddToScheme(scheme) require.NoError(t, err) recorder := NewEventRecorder(client, scheme, "source-1", "fake-node", velerotest.NewLogger()) pod := &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-ns", Name: "fake-pod", UID: types.UID("fake-uid"), }, Spec: corev1api.PodSpec{ NodeName: "fake-node", }, } _, err = client.CoreV1().Pods("fake-ns").Create(t.Context(), pod, metav1.CreateOptions{}) require.NoError(t, err) for i := 0; i < tc.generateDiff; i++ { tc.events = append(tc.events, testEvent{ reason: fmt.Sprintf("fake-reason-%v", i), message: fmt.Sprintf("fake-message-%v", i), }) } for i := 0; i < tc.generateSame; i++ { tc.events = append(tc.events, testEvent{ reason: "fake-reason", message: fmt.Sprintf("fake-message-%v", i), }) } if tc.generateEnding { tc.events = append(tc.events, testEvent{ reason: "fake-ending-reason", message: "fake-ending-message", ending: true, }) } for _, e := range tc.events { if e.ending { recorder.EndingEvent(pod, e.warning, e.reason, e.message) } else { recorder.Event(pod, e.warning, e.reason, e.message) } } recorder.Shutdown() items, err := client.CoreV1().Events("fake-ns").List(t.Context(), metav1.ListOptions{}) require.NoError(t, err) if tc.expected != len(items.Items) { for _, i := range items.Items { fmt.Printf("event (%s, %s, %s)\n", i.Type, i.Message, i.Reason) } } if tc.expected >= 0 { assert.Len(t, items.Items, tc.expected) } }) } } ================================================ FILE: pkg/util/kube/list_watch.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package kube import ( "context" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" kbclient "sigs.k8s.io/controller-runtime/pkg/client" ) type InternalLW struct { Client kbclient.WithWatch Namespace string ObjectList kbclient.ObjectList } func (lw *InternalLW) Watch(options metav1.ListOptions) (watch.Interface, error) { return lw.Client.Watch(context.Background(), lw.ObjectList, &kbclient.ListOptions{Raw: &options, Namespace: lw.Namespace}) } func (lw *InternalLW) List(options metav1.ListOptions) (runtime.Object, error) { err := lw.Client.List(context.Background(), lw.ObjectList, &kbclient.ListOptions{Raw: &options, Namespace: lw.Namespace}) return lw.ObjectList, err } ================================================ FILE: pkg/util/kube/list_watch_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package kube import ( "testing" "time" "github.com/stretchr/testify/require" "k8s.io/client-go/tools/cache" kbclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func TestInternalLW(t *testing.T) { stop := make(chan struct{}) client := velerotest.NewFakeControllerRuntimeClient(t).(kbclient.WithWatch) lw := InternalLW{ Client: client, Namespace: "velero", ObjectList: new(velerov1api.BackupList), } restoreInformer := cache.NewSharedInformer(&lw, &velerov1api.BackupList{}, time.Second) go restoreInformer.Run(stop) time.Sleep(1 * time.Second) close(stop) backupList := new(velerov1api.BackupList) err := client.List(t.Context(), backupList) require.NoError(t, err) _, err = client.Watch(t.Context(), backupList) require.NoError(t, err) } ================================================ FILE: pkg/util/kube/mocks/Client.go ================================================ // Code generated by mockery v2.42.1. DO NOT EDIT. package mocks import ( context "context" client "sigs.k8s.io/controller-runtime/pkg/client" meta "k8s.io/apimachinery/pkg/api/meta" mock "github.com/stretchr/testify/mock" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" types "k8s.io/apimachinery/pkg/types" ) // Client is an autogenerated mock type for the Client type type Client struct { mock.Mock } // Create provides a mock function with given fields: ctx, obj, opts func (_m *Client) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { _va := make([]interface{}, len(opts)) for _i := range opts { _va[_i] = opts[_i] } var _ca []interface{} _ca = append(_ca, ctx, obj) _ca = append(_ca, _va...) ret := _m.Called(_ca...) if len(ret) == 0 { panic("no return value specified for Create") } var r0 error if rf, ok := ret.Get(0).(func(context.Context, client.Object, ...client.CreateOption) error); ok { r0 = rf(ctx, obj, opts...) } else { r0 = ret.Error(0) } return r0 } // Delete provides a mock function with given fields: ctx, obj, opts func (_m *Client) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error { _va := make([]interface{}, len(opts)) for _i := range opts { _va[_i] = opts[_i] } var _ca []interface{} _ca = append(_ca, ctx, obj) _ca = append(_ca, _va...) ret := _m.Called(_ca...) if len(ret) == 0 { panic("no return value specified for Delete") } var r0 error if rf, ok := ret.Get(0).(func(context.Context, client.Object, ...client.DeleteOption) error); ok { r0 = rf(ctx, obj, opts...) } else { r0 = ret.Error(0) } return r0 } // DeleteAllOf provides a mock function with given fields: ctx, obj, opts func (_m *Client) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error { _va := make([]interface{}, len(opts)) for _i := range opts { _va[_i] = opts[_i] } var _ca []interface{} _ca = append(_ca, ctx, obj) _ca = append(_ca, _va...) ret := _m.Called(_ca...) if len(ret) == 0 { panic("no return value specified for DeleteAllOf") } var r0 error if rf, ok := ret.Get(0).(func(context.Context, client.Object, ...client.DeleteAllOfOption) error); ok { r0 = rf(ctx, obj, opts...) } else { r0 = ret.Error(0) } return r0 } // Get provides a mock function with given fields: ctx, key, obj, opts func (_m *Client) Get(ctx context.Context, key types.NamespacedName, obj client.Object, opts ...client.GetOption) error { _va := make([]interface{}, len(opts)) for _i := range opts { _va[_i] = opts[_i] } var _ca []interface{} _ca = append(_ca, ctx, key, obj) _ca = append(_ca, _va...) ret := _m.Called(_ca...) if len(ret) == 0 { panic("no return value specified for Get") } var r0 error if rf, ok := ret.Get(0).(func(context.Context, types.NamespacedName, client.Object, ...client.GetOption) error); ok { r0 = rf(ctx, key, obj, opts...) } else { r0 = ret.Error(0) } return r0 } // GroupVersionKindFor provides a mock function with given fields: obj func (_m *Client) GroupVersionKindFor(obj runtime.Object) (schema.GroupVersionKind, error) { ret := _m.Called(obj) if len(ret) == 0 { panic("no return value specified for GroupVersionKindFor") } var r0 schema.GroupVersionKind var r1 error if rf, ok := ret.Get(0).(func(runtime.Object) (schema.GroupVersionKind, error)); ok { return rf(obj) } if rf, ok := ret.Get(0).(func(runtime.Object) schema.GroupVersionKind); ok { r0 = rf(obj) } else { r0 = ret.Get(0).(schema.GroupVersionKind) } if rf, ok := ret.Get(1).(func(runtime.Object) error); ok { r1 = rf(obj) } else { r1 = ret.Error(1) } return r0, r1 } // IsObjectNamespaced provides a mock function with given fields: obj func (_m *Client) IsObjectNamespaced(obj runtime.Object) (bool, error) { ret := _m.Called(obj) if len(ret) == 0 { panic("no return value specified for IsObjectNamespaced") } var r0 bool var r1 error if rf, ok := ret.Get(0).(func(runtime.Object) (bool, error)); ok { return rf(obj) } if rf, ok := ret.Get(0).(func(runtime.Object) bool); ok { r0 = rf(obj) } else { r0 = ret.Get(0).(bool) } if rf, ok := ret.Get(1).(func(runtime.Object) error); ok { r1 = rf(obj) } else { r1 = ret.Error(1) } return r0, r1 } // List provides a mock function with given fields: ctx, list, opts func (_m *Client) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { _va := make([]interface{}, len(opts)) for _i := range opts { _va[_i] = opts[_i] } var _ca []interface{} _ca = append(_ca, ctx, list) _ca = append(_ca, _va...) ret := _m.Called(_ca...) if len(ret) == 0 { panic("no return value specified for List") } var r0 error if rf, ok := ret.Get(0).(func(context.Context, client.ObjectList, ...client.ListOption) error); ok { r0 = rf(ctx, list, opts...) } else { r0 = ret.Error(0) } return r0 } // Patch provides a mock function with given fields: ctx, obj, patch, opts func (_m *Client) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { _va := make([]interface{}, len(opts)) for _i := range opts { _va[_i] = opts[_i] } var _ca []interface{} _ca = append(_ca, ctx, obj, patch) _ca = append(_ca, _va...) ret := _m.Called(_ca...) if len(ret) == 0 { panic("no return value specified for Patch") } var r0 error if rf, ok := ret.Get(0).(func(context.Context, client.Object, client.Patch, ...client.PatchOption) error); ok { r0 = rf(ctx, obj, patch, opts...) } else { r0 = ret.Error(0) } return r0 } // RESTMapper provides a mock function with given fields: func (_m *Client) RESTMapper() meta.RESTMapper { ret := _m.Called() if len(ret) == 0 { panic("no return value specified for RESTMapper") } var r0 meta.RESTMapper if rf, ok := ret.Get(0).(func() meta.RESTMapper); ok { r0 = rf() } else { if ret.Get(0) != nil { r0 = ret.Get(0).(meta.RESTMapper) } } return r0 } // Scheme provides a mock function with given fields: func (_m *Client) Scheme() *runtime.Scheme { ret := _m.Called() if len(ret) == 0 { panic("no return value specified for Scheme") } var r0 *runtime.Scheme if rf, ok := ret.Get(0).(func() *runtime.Scheme); ok { r0 = rf() } else { if ret.Get(0) != nil { r0 = ret.Get(0).(*runtime.Scheme) } } return r0 } // Status provides a mock function with given fields: func (_m *Client) Status() client.SubResourceWriter { ret := _m.Called() if len(ret) == 0 { panic("no return value specified for Status") } var r0 client.SubResourceWriter if rf, ok := ret.Get(0).(func() client.SubResourceWriter); ok { r0 = rf() } else { if ret.Get(0) != nil { r0 = ret.Get(0).(client.SubResourceWriter) } } return r0 } // SubResource provides a mock function with given fields: subResource func (_m *Client) SubResource(subResource string) client.SubResourceClient { ret := _m.Called(subResource) if len(ret) == 0 { panic("no return value specified for SubResource") } var r0 client.SubResourceClient if rf, ok := ret.Get(0).(func(string) client.SubResourceClient); ok { r0 = rf(subResource) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(client.SubResourceClient) } } return r0 } // Update provides a mock function with given fields: ctx, obj, opts func (_m *Client) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { _va := make([]interface{}, len(opts)) for _i := range opts { _va[_i] = opts[_i] } var _ca []interface{} _ca = append(_ca, ctx, obj) _ca = append(_ca, _va...) ret := _m.Called(_ca...) if len(ret) == 0 { panic("no return value specified for Update") } var r0 error if rf, ok := ret.Get(0).(func(context.Context, client.Object, ...client.UpdateOption) error); ok { r0 = rf(ctx, obj, opts...) } else { r0 = ret.Error(0) } return r0 } // NewClient creates a new instance of Client. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. func NewClient(t interface { mock.TestingT Cleanup(func()) }) *Client { mock := &Client{} mock.Mock.Test(t) t.Cleanup(func() { mock.AssertExpectations(t) }) return mock } ================================================ FILE: pkg/util/kube/mocks.go ================================================ package kube import ( "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "sigs.k8s.io/controller-runtime/pkg/client" ) // Client knows how to perform CRUD operations on Kubernetes objects. // //go:generate mockery --name=Client type Client interface { client.Reader client.Writer client.StatusClient client.SubResourceClientConstructor // Scheme returns the scheme this client is using. Scheme() *runtime.Scheme // RESTMapper returns the rest this client is using. RESTMapper() meta.RESTMapper // GroupVersionKindFor returns the GroupVersionKind for the given object. GroupVersionKindFor(obj runtime.Object) (schema.GroupVersionKind, error) // IsObjectNamespaced returns true if the GroupVersionKind of the object is namespaced. IsObjectNamespaced(obj runtime.Object) (bool, error) } ================================================ FILE: pkg/util/kube/node.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package kube import ( "context" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) const ( NodeOSLinux = "linux" NodeOSWindows = "windows" NodeOSLabel = "kubernetes.io/os" ) var realNodeOSMap = map[string]string{ "linux": NodeOSLinux, "windows": NodeOSWindows, } func IsLinuxNode(ctx context.Context, nodeName string, client client.Client) error { node := &corev1api.Node{} if err := client.Get(ctx, types.NamespacedName{Name: nodeName}, node); err != nil { return errors.Wrapf(err, "error getting node %s", nodeName) } os, found := node.Labels[NodeOSLabel] if !found { return errors.Errorf("no os type label for node %s", nodeName) } if getRealOS(os) != NodeOSLinux { return errors.Errorf("os type %s for node %s is not linux", os, nodeName) } return nil } func WithLinuxNode(ctx context.Context, client client.Client, log logrus.FieldLogger) bool { return withOSNode(ctx, client, NodeOSLinux, log) } func WithWindowsNode(ctx context.Context, client client.Client, log logrus.FieldLogger) bool { return withOSNode(ctx, client, NodeOSWindows, log) } func withOSNode(ctx context.Context, client client.Client, osType string, log logrus.FieldLogger) bool { nodeList := new(corev1api.NodeList) if err := client.List(ctx, nodeList); err != nil { log.Warnf("Failed to list nodes, cannot decide existence of nodes of OS %s", osType) return false } allNodeLabeled := true for _, node := range nodeList.Items { os, found := node.Labels[NodeOSLabel] if getRealOS(os) == osType { return true } if !found { allNodeLabeled = false } } if !allNodeLabeled { log.Warnf("Not all nodes have os type label, cannot decide existence of nodes of OS %s", osType) } return false } func GetNodeOS(ctx context.Context, nodeName string, nodeClient corev1client.CoreV1Interface) (string, error) { node, err := nodeClient.Nodes().Get(context.Background(), nodeName, metav1.GetOptions{}) if err != nil { return "", errors.Wrapf(err, "error getting node %s", nodeName) } if node.Labels == nil { return "", nil } return getRealOS(node.Labels[NodeOSLabel]), nil } func HasNodeWithOS(ctx context.Context, os string, nodeClient corev1client.CoreV1Interface) error { if os == "" { return errors.New("invalid node OS") } nodes, err := nodeClient.Nodes().List(ctx, metav1.ListOptions{}) if err != nil { return errors.Wrapf(err, "error listing nodes with OS %s", os) } for _, node := range nodes.Items { osLabel, found := node.Labels[NodeOSLabel] if !found { continue } if getRealOS(osLabel) == os { return nil } } return errors.Errorf("node with OS %s doesn't exist", os) } func getRealOS(osLabel string) string { if os, found := realNodeOSMap[osLabel]; !found { return NodeOSLinux } else { return os } } ================================================ FILE: pkg/util/kube/node_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package kube import ( "testing" "github.com/pkg/errors" "github.com/stretchr/testify/assert" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "github.com/vmware-tanzu/velero/pkg/builder" kubeClientFake "k8s.io/client-go/kubernetes/fake" clientTesting "k8s.io/client-go/testing" clientFake "sigs.k8s.io/controller-runtime/pkg/client/fake" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func TestIsLinuxNode(t *testing.T) { nodeNoOSLabel := builder.ForNode("fake-node").Result() nodeWindows := builder.ForNode("fake-node").Labels(map[string]string{"kubernetes.io/os": "windows"}).Result() nodeLinux := builder.ForNode("fake-node").Labels(map[string]string{"kubernetes.io/os": "linux"}).Result() scheme := runtime.NewScheme() corev1api.AddToScheme(scheme) tests := []struct { name string kubeClientObj []runtime.Object err string }{ { name: "error getting node", err: "error getting node fake-node: nodes \"fake-node\" not found", }, { name: "no os label", kubeClientObj: []runtime.Object{ nodeNoOSLabel, }, err: "no os type label for node fake-node", }, { name: "os label does not match", kubeClientObj: []runtime.Object{ nodeWindows, }, err: "os type windows for node fake-node is not linux", }, { name: "succeed", kubeClientObj: []runtime.Object{ nodeLinux, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeClientBuilder := clientFake.NewClientBuilder() fakeClientBuilder = fakeClientBuilder.WithScheme(scheme) fakeClient := fakeClientBuilder.WithRuntimeObjects(test.kubeClientObj...).Build() err := IsLinuxNode(t.Context(), "fake-node", fakeClient) if err != nil { assert.EqualError(t, err, test.err) } else { assert.NoError(t, err) } }) } } func TestWithLinuxNode(t *testing.T) { nodeWindows := builder.ForNode("fake-node-1").Labels(map[string]string{"kubernetes.io/os": "windows"}).Result() nodeLinux := builder.ForNode("fake-node-2").Labels(map[string]string{"kubernetes.io/os": "linux"}).Result() scheme := runtime.NewScheme() corev1api.AddToScheme(scheme) tests := []struct { name string kubeClientObj []runtime.Object result bool }{ { name: "error listing node", }, { name: "with node of other type", kubeClientObj: []runtime.Object{ nodeWindows, }, }, { name: "with node of the same type", kubeClientObj: []runtime.Object{ nodeWindows, nodeLinux, }, result: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeClientBuilder := clientFake.NewClientBuilder() fakeClientBuilder = fakeClientBuilder.WithScheme(scheme) fakeClient := fakeClientBuilder.WithRuntimeObjects(test.kubeClientObj...).Build() result := withOSNode(t.Context(), fakeClient, "linux", velerotest.NewLogger()) assert.Equal(t, test.result, result) }) } } func TestGetNodeOSType(t *testing.T) { nodeNoOSLabel := builder.ForNode("fake-node").Result() nodeWindows := builder.ForNode("fake-node").Labels(map[string]string{"kubernetes.io/os": "windows"}).Result() nodeLinux := builder.ForNode("fake-node").Labels(map[string]string{"kubernetes.io/os": "linux"}).Result() scheme := runtime.NewScheme() corev1api.AddToScheme(scheme) tests := []struct { name string kubeClientObj []runtime.Object err string expectedOSType string }{ { name: "error getting node", err: "error getting node fake-node: nodes \"fake-node\" not found", }, { name: "no os label", kubeClientObj: []runtime.Object{ nodeNoOSLabel, }, }, { name: "windows node", kubeClientObj: []runtime.Object{ nodeWindows, }, expectedOSType: "windows", }, { name: "linux node", kubeClientObj: []runtime.Object{ nodeLinux, }, expectedOSType: "linux", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeKubeClient := kubeClientFake.NewSimpleClientset(test.kubeClientObj...) osType, err := GetNodeOS(t.Context(), "fake-node", fakeKubeClient.CoreV1()) if err != nil { assert.EqualError(t, err, test.err) } else { assert.Equal(t, test.expectedOSType, osType) } }) } } func TestHasNodeWithOS(t *testing.T) { nodeNoOSLabel := builder.ForNode("fake-node-1").Result() nodeWindows := builder.ForNode("fake-node-2").Labels(map[string]string{"kubernetes.io/os": "windows"}).Result() nodeLinux := builder.ForNode("fake-node-3").Labels(map[string]string{"kubernetes.io/os": "linux"}).Result() scheme := runtime.NewScheme() corev1api.AddToScheme(scheme) tests := []struct { name string kubeClientObj []runtime.Object kubeReactors []reactor os string err string }{ { name: "os is empty", err: "invalid node OS", }, { name: "error to list node", kubeReactors: []reactor{ { verb: "list", resource: "nodes", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, errors.New("fake-list-error") }, }, }, os: "linux", err: "error listing nodes with OS linux: fake-list-error", }, { name: "no expected node - no node", os: "linux", err: "node with OS linux doesn't exist", }, { name: "no expected node - no node with label", kubeClientObj: []runtime.Object{ nodeNoOSLabel, nodeWindows, }, os: "linux", err: "node with OS linux doesn't exist", }, { name: "succeed", kubeClientObj: []runtime.Object{ nodeNoOSLabel, nodeWindows, nodeLinux, }, os: "windows", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeKubeClient := kubeClientFake.NewSimpleClientset(test.kubeClientObj...) for _, reactor := range test.kubeReactors { fakeKubeClient.Fake.PrependReactor(reactor.verb, reactor.resource, reactor.reactorFunc) } err := HasNodeWithOS(t.Context(), test.os, fakeKubeClient.CoreV1()) if test.err != "" { assert.EqualError(t, err, test.err) } else { assert.NoError(t, err) } }) } } ================================================ FILE: pkg/util/kube/periodical_enqueue_source.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package kube import ( "context" "fmt" "reflect" "time" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/util/workqueue" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) func NewPeriodicalEnqueueSource( logger logrus.FieldLogger, client client.Client, objList client.ObjectList, period time.Duration, option PeriodicalEnqueueSourceOption) *PeriodicalEnqueueSource { return &PeriodicalEnqueueSource{ logger: logger.WithField("resource", reflect.TypeOf(objList).String()), Client: client, objList: objList, period: period, option: option, } } // PeriodicalEnqueueSource is an implementation of interface sigs.k8s.io/controller-runtime/pkg/source/Source // It reads the specific resources from Kubernetes/cache and enqueues them into the queue to trigger // the reconcile logic periodically type PeriodicalEnqueueSource struct { client.Client logger logrus.FieldLogger objList client.ObjectList period time.Duration option PeriodicalEnqueueSourceOption } type PeriodicalEnqueueSourceOption struct { OrderFunc func(objList client.ObjectList) client.ObjectList Predicates []predicate.Predicate // the predicates only apply to the GenericEvent } // Start enqueue items periodically func (p *PeriodicalEnqueueSource) Start(ctx context.Context, q workqueue.TypedRateLimitingInterface[reconcile.Request]) error { go wait.Until(func() { p.logger.Debug("enqueueing resources ...") // empty the list otherwise the result of the new list call will be appended if err := meta.SetList(p.objList, nil); err != nil { p.logger.WithError(err).Error("error reset resource list") return } if err := p.List(ctx, p.objList); err != nil { p.logger.WithError(err).Error("error listing resources") return } if meta.LenList(p.objList) == 0 { p.logger.Debug("no resources, skip") return } if p.option.OrderFunc != nil { p.objList = p.option.OrderFunc(p.objList) } if err := meta.EachListItem(p.objList, func(object runtime.Object) error { obj, ok := object.(client.Object) if !ok { p.logger.Error("%s's type isn't metav1.Object", object.GetObjectKind().GroupVersionKind().String()) return nil } event := event.GenericEvent{Object: obj} for _, predicate := range p.option.Predicates { if !predicate.Generic(event) { p.logger.Debugf("skip enqueue object %s/%s due to the predicate.", obj.GetNamespace(), obj.GetName()) return nil } } q.Add(ctrl.Request{ NamespacedName: types.NamespacedName{ Namespace: obj.GetNamespace(), Name: obj.GetName(), }, }) p.logger.Debugf("resource %s/%s enqueued", obj.GetNamespace(), obj.GetName()) return nil }); err != nil { p.logger.WithError(err).Error("error enqueueing resources") return } }, p.period, ctx.Done()) return nil } func (p *PeriodicalEnqueueSource) String() string { if p.objList != nil { return fmt.Sprintf("periodical enqueue source: %T", p.objList) } return "periodical enqueue source: unknown type" } ================================================ FILE: pkg/util/kube/periodical_enqueue_source_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package kube import ( "testing" "time" "context" "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/util/workqueue" crclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" "github.com/vmware-tanzu/velero/internal/storage" velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) func TestStart(t *testing.T) { require.NoError(t, velerov1.AddToScheme(scheme.Scheme)) ctx, cancelFunc := context.WithCancel(t.Context()) client := (&fake.ClientBuilder{}).Build() queue := workqueue.NewTypedRateLimitingQueue(workqueue.DefaultTypedItemBasedRateLimiter[reconcile.Request]()) source := NewPeriodicalEnqueueSource(logrus.WithContext(ctx).WithField("controller", "PES_TEST"), client, &velerov1.ScheduleList{}, 1*time.Second, PeriodicalEnqueueSourceOption{}) require.NoError(t, source.Start(ctx, queue)) // no resources time.Sleep(1 * time.Second) require.Equal(t, 0, queue.Len()) // contain one resource require.NoError(t, client.Create(ctx, &velerov1.Schedule{ ObjectMeta: metav1.ObjectMeta{ Name: "schedule", }, })) time.Sleep(2 * time.Second) require.Equal(t, 1, queue.Len()) // context canceled, the enqueue source shouldn't run anymore item, _ := queue.Get() queue.Forget(item) require.Equal(t, 0, queue.Len()) cancelFunc() time.Sleep(2 * time.Second) require.Equal(t, 0, queue.Len()) } func TestPredicate(t *testing.T) { require.NoError(t, velerov1.AddToScheme(scheme.Scheme)) ctx, cancelFunc := context.WithCancel(t.Context()) client := (&fake.ClientBuilder{}).Build() queue := workqueue.NewTypedRateLimitingQueue(workqueue.DefaultTypedItemBasedRateLimiter[reconcile.Request]()) pred := NewGenericEventPredicate(func(object crclient.Object) bool { location := object.(*velerov1.BackupStorageLocation) return storage.IsReadyToValidate(location.Spec.ValidationFrequency, location.Status.LastValidationTime, 1*time.Minute, logrus.WithContext(ctx).WithField("BackupStorageLocation", location.Name)) }) source := NewPeriodicalEnqueueSource( logrus.WithContext(ctx).WithField("controller", "PES_TEST"), client, &velerov1.BackupStorageLocationList{}, 1*time.Second, PeriodicalEnqueueSourceOption{ Predicates: []predicate.Predicate{pred}, }, ) require.NoError(t, source.Start(ctx, queue)) // Should not patch a backup storage location object status phase // if the location's validation frequency is specifically set to zero require.NoError(t, client.Create(ctx, &velerov1.BackupStorageLocation{ ObjectMeta: metav1.ObjectMeta{ Name: "location1", Namespace: "default", }, Spec: velerov1.BackupStorageLocationSpec{ ValidationFrequency: &metav1.Duration{Duration: 0}, }, Status: velerov1.BackupStorageLocationStatus{ LastValidationTime: &metav1.Time{Time: time.Now()}, }, })) time.Sleep(2 * time.Second) require.Equal(t, 0, queue.Len()) cancelFunc() } func TestOrder(t *testing.T) { require.NoError(t, velerov1.AddToScheme(scheme.Scheme)) ctx, cancelFunc := context.WithCancel(t.Context()) client := (&fake.ClientBuilder{}).Build() queue := workqueue.NewTypedRateLimitingQueue(workqueue.DefaultTypedItemBasedRateLimiter[reconcile.Request]()) source := NewPeriodicalEnqueueSource( logrus.WithContext(ctx).WithField("controller", "PES_TEST"), client, &velerov1.BackupStorageLocationList{}, 1*time.Second, PeriodicalEnqueueSourceOption{ OrderFunc: func(objList crclient.ObjectList) crclient.ObjectList { locationList := &velerov1.BackupStorageLocationList{} objArray := make([]runtime.Object, 0) // Generate BSL array. locations, _ := meta.ExtractList(objList) // Move default BSL to tail of array. objArray = append(objArray, locations[1]) objArray = append(objArray, locations[0]) meta.SetList(locationList, objArray) return locationList }, }, ) require.NoError(t, source.Start(ctx, queue)) // Should not patch a backup storage location object status phase // if the location's validation frequency is specifically set to zero require.NoError(t, client.Create(ctx, &velerov1.BackupStorageLocation{ ObjectMeta: metav1.ObjectMeta{ Name: "location1", Namespace: "default", }, Spec: velerov1.BackupStorageLocationSpec{ ValidationFrequency: &metav1.Duration{Duration: 0}, }, Status: velerov1.BackupStorageLocationStatus{ LastValidationTime: &metav1.Time{Time: time.Now()}, }, })) require.NoError(t, client.Create(ctx, &velerov1.BackupStorageLocation{ ObjectMeta: metav1.ObjectMeta{ Name: "location2", Namespace: "default", }, Spec: velerov1.BackupStorageLocationSpec{ ValidationFrequency: &metav1.Duration{Duration: 0}, Default: true, }, Status: velerov1.BackupStorageLocationStatus{ LastValidationTime: &metav1.Time{Time: time.Now()}, }, })) time.Sleep(2 * time.Second) first, _ := queue.Get() bsl := &velerov1.BackupStorageLocation{} require.Equal(t, "location2", first.Name) require.NoError(t, client.Get(ctx, first.NamespacedName, bsl)) require.True(t, bsl.Spec.Default) cancelFunc() } ================================================ FILE: pkg/util/kube/pod.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package kube import ( "context" "fmt" "io" "os" "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" ) type LoadAffinity struct { // NodeSelector specifies the label selector to match nodes NodeSelector metav1.LabelSelector `json:"nodeSelector"` // StorageClass specifies the VGDPs the LoadAffinity applied to. If the StorageClass doesn't have value, it applies to all. If not, it applies to only the VGDPs that use this StorageClass. StorageClass string `json:"storageClass"` } type PodResources struct { CPURequest string `json:"cpuRequest,omitempty"` CPULimit string `json:"cpuLimit,omitempty"` MemoryRequest string `json:"memoryRequest,omitempty"` MemoryLimit string `json:"memoryLimit,omitempty"` EphemeralStorageRequest string `json:"ephemeralStorageRequest,omitempty"` EphemeralStorageLimit string `json:"ephemeralStorageLimit,omitempty"` } // IsPodRunning does a well-rounded check to make sure the specified pod is running stably. // If not, return the error found func IsPodRunning(pod *corev1api.Pod) error { return isPodScheduledInStatus(pod, func(pod *corev1api.Pod) error { if pod.Status.Phase != corev1api.PodRunning { return errors.New("pod is not running") } return nil }) } // IsPodScheduled does a well-rounded check to make sure the specified pod has been scheduled into a node and in a stable status. // If not, return the error found func IsPodScheduled(pod *corev1api.Pod) error { return isPodScheduledInStatus(pod, func(pod *corev1api.Pod) error { if pod.Status.Phase != corev1api.PodRunning && pod.Status.Phase != corev1api.PodPending { return errors.New("pod is not running or pending") } return nil }) } func isPodScheduledInStatus(pod *corev1api.Pod, statusCheckFunc func(*corev1api.Pod) error) error { if pod == nil { return errors.New("invalid input pod") } if pod.Spec.NodeName == "" { return errors.Errorf("pod is not scheduled, name=%s, namespace=%s, phase=%s", pod.Name, pod.Namespace, pod.Status.Phase) } if err := statusCheckFunc(pod); err != nil { return errors.Wrapf(err, "pod is not in the expected status, name=%s, namespace=%s, phase=%s", pod.Name, pod.Namespace, pod.Status.Phase) } if pod.DeletionTimestamp != nil { return errors.Errorf("pod is being terminated, name=%s, namespace=%s, phase=%s", pod.Name, pod.Namespace, pod.Status.Phase) } return nil } // DeletePodIfAny deletes a pod by name if it exists, and log an error when the deletion fails func DeletePodIfAny(ctx context.Context, podGetter corev1client.CoreV1Interface, podName string, podNamespace string, log logrus.FieldLogger) { err := podGetter.Pods(podNamespace).Delete(ctx, podName, metav1.DeleteOptions{}) if err != nil { if apierrors.IsNotFound(err) { log.WithError(err).Debugf("Abort deleting pod, it doesn't exist %s/%s", podNamespace, podName) } else { log.WithError(err).Errorf("Failed to delete pod %s/%s", podNamespace, podName) } } } // EnsureDeletePod asserts the existence of a pod by name, deletes it and waits for its disappearance and returns errors on any failure func EnsureDeletePod(ctx context.Context, podGetter corev1client.CoreV1Interface, pod string, namespace string, timeout time.Duration) error { err := podGetter.Pods(namespace).Delete(ctx, pod, metav1.DeleteOptions{}) if err != nil { return errors.Wrapf(err, "error to delete pod %s", pod) } var updated *corev1api.Pod err = wait.PollUntilContextTimeout(ctx, waitInternal, timeout, true, func(ctx context.Context) (bool, error) { po, err := podGetter.Pods(namespace).Get(ctx, pod, metav1.GetOptions{}) if err != nil { if apierrors.IsNotFound(err) { return true, nil } return false, errors.Wrapf(err, "error to get pod %s", pod) } updated = po return false, nil }) if err != nil { if errors.Is(err, context.DeadlineExceeded) { return errors.Errorf("timeout to assure pod %s is deleted, finalizers in pod %v", pod, updated.Finalizers) } else { return errors.Wrapf(err, "error to assure pod is deleted, %s", pod) } } return nil } // IsPodUnrecoverable checks if the pod is in an abnormal state and could not be recovered // It could not cover all the cases but we could add more cases in the future func IsPodUnrecoverable(pod *corev1api.Pod, log logrus.FieldLogger) (bool, string) { // Check the Phase field if pod.Status.Phase == corev1api.PodFailed || pod.Status.Phase == corev1api.PodUnknown { message := "" if pod.Status.Message != "" { message += pod.Status.Message + "/" } message += GetPodTerminateMessage(pod) log.Warnf("Pod is in abnormal state %s, message [%s]", pod.Status.Phase, message) return true, fmt.Sprintf("Pod is in abnormal state [%s], message [%s]", pod.Status.Phase, message) } // removed "Unschedulable" check since unschedulable condition isn't always permanent // Check the Status field for _, containerStatus := range pod.Status.ContainerStatuses { // If the container's image state is ImagePullBackOff, it indicates an image pull failure if containerStatus.State.Waiting != nil && (containerStatus.State.Waiting.Reason == "ImagePullBackOff" || containerStatus.State.Waiting.Reason == "ErrImageNeverPull") { log.Warnf("Container %s in Pod %s/%s is in pull image failed with reason %s", containerStatus.Name, pod.Namespace, pod.Name, containerStatus.State.Waiting.Reason) return true, fmt.Sprintf("Container %s in Pod %s/%s is in pull image failed with reason %s", containerStatus.Name, pod.Namespace, pod.Name, containerStatus.State.Waiting.Reason) } } return false, "" } // GetPodContainerTerminateMessage returns the terminate message for a specific container of a pod func GetPodContainerTerminateMessage(pod *corev1api.Pod, container string) string { message := "" for _, containerStatus := range pod.Status.ContainerStatuses { if containerStatus.Name == container { if containerStatus.State.Terminated != nil { message = containerStatus.State.Terminated.Message } break } } return message } // GetPodTerminateMessage returns the terminate message for all containers of a pod func GetPodTerminateMessage(pod *corev1api.Pod) string { message := "" for _, containerStatus := range pod.Status.ContainerStatuses { if containerStatus.State.Terminated != nil { if containerStatus.State.Terminated.Message != "" { message += containerStatus.State.Terminated.Message + "/" } } } return message } func getPodLogReader(ctx context.Context, podGetter corev1client.CoreV1Interface, pod string, namespace string, logOptions *corev1api.PodLogOptions) (io.ReadCloser, error) { request := podGetter.Pods(namespace).GetLogs(pod, logOptions) return request.Stream(ctx) } var podLogReaderGetter = getPodLogReader // CollectPodLogs collects logs of the specified container of a pod and write to the output func CollectPodLogs(ctx context.Context, podGetter corev1client.CoreV1Interface, pod string, namespace string, container string, output io.Writer) error { logIndicator := fmt.Sprintf("***************************begin pod logs[%s/%s]***************************\n", pod, container) if _, err := output.Write([]byte(logIndicator)); err != nil { return errors.Wrap(err, "error to write begin pod log indicator") } logOptions := &corev1api.PodLogOptions{ Container: container, } if input, err := podLogReaderGetter(ctx, podGetter, pod, namespace, logOptions); err != nil { logIndicator = fmt.Sprintf("No present log retrieved, err: %v\n", err) } else { if _, err := io.Copy(output, input); err != nil { return errors.Wrap(err, "error to copy input") } logIndicator = "" } logIndicator += fmt.Sprintf("***************************end pod logs[%s/%s]***************************\n", pod, container) if _, err := output.Write([]byte(logIndicator)); err != nil { return errors.Wrap(err, "error to write end pod log indicator") } return nil } func ToSystemAffinity(loadAffinity *LoadAffinity, volumeTopology *corev1api.NodeSelector) *corev1api.Affinity { requirements := []corev1api.NodeSelectorRequirement{} if loadAffinity != nil { for k, v := range loadAffinity.NodeSelector.MatchLabels { requirements = append(requirements, corev1api.NodeSelectorRequirement{ Key: k, Values: []string{v}, Operator: corev1api.NodeSelectorOpIn, }) } for _, exp := range loadAffinity.NodeSelector.MatchExpressions { requirements = append(requirements, corev1api.NodeSelectorRequirement{ Key: exp.Key, Values: exp.Values, Operator: corev1api.NodeSelectorOperator(exp.Operator), }) } } result := new(corev1api.Affinity) result.NodeAffinity = new(corev1api.NodeAffinity) result.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution = new(corev1api.NodeSelector) if volumeTopology != nil { result.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = append(result.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms, volumeTopology.NodeSelectorTerms...) } else if len(requirements) > 0 { result.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = make([]corev1api.NodeSelectorTerm, 1) } else { return nil } for i := range result.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms { result.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[i].MatchExpressions = append(result.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[i].MatchExpressions, requirements...) } return result } func DiagnosePod(pod *corev1api.Pod, events *corev1api.EventList) string { diag := fmt.Sprintf("Pod %s/%s, phase %s, node name %s, message %s\n", pod.Namespace, pod.Name, pod.Status.Phase, pod.Spec.NodeName, pod.Status.Message) for _, condition := range pod.Status.Conditions { diag += fmt.Sprintf("Pod condition %s, status %s, reason %s, message %s\n", condition.Type, condition.Status, condition.Reason, condition.Message) } if events != nil { for _, e := range events.Items { if e.InvolvedObject.UID == pod.UID && e.Type == corev1api.EventTypeWarning { diag += fmt.Sprintf("Pod event reason %s, message %s\n", e.Reason, e.Message) } } } return diag } var funcExit = os.Exit var funcCreateFile = os.Create func ExitPodWithMessage(logger logrus.FieldLogger, succeed bool, message string, a ...any) { exitCode := 0 if !succeed { exitCode = 1 } toWrite := fmt.Sprintf(message, a...) podFile, err := funcCreateFile("/dev/termination-log") if err != nil { logger.WithError(err).Error("Failed to create termination log file") exitCode = 1 } else { if _, err := podFile.WriteString(toWrite); err != nil { logger.WithError(err).Error("Failed to write error to termination log file") exitCode = 1 } podFile.Close() } funcExit(exitCode) } // GetLoadAffinityByStorageClass retrieves the LoadAffinity from the parameter affinityList. // The function first try to find by the scName. If there is no such LoadAffinity, // it will try to get the LoadAffinity whose StorageClass has no value. func GetLoadAffinityByStorageClass( affinityList []*LoadAffinity, scName string, logger logrus.FieldLogger, ) *LoadAffinity { var globalAffinity *LoadAffinity for _, affinity := range affinityList { if affinity.StorageClass == scName { logger.WithField("StorageClass", scName).Info("Found pod's affinity setting per StorageClass.") return affinity } if affinity.StorageClass == "" && globalAffinity == nil { globalAffinity = affinity } } if globalAffinity != nil { logger.Info("Use the Global affinity for pod.") } else { logger.Info("No Affinity is found for pod.") } return globalAffinity } ================================================ FILE: pkg/util/kube/pod_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package kube import ( "context" "fmt" "io" "os" "path/filepath" "reflect" "strings" "testing" "time" "github.com/google/uuid" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" clientTesting "k8s.io/client-go/testing" velerotest "github.com/vmware-tanzu/velero/pkg/test" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" ) func TestEnsureDeletePod(t *testing.T) { podObject := &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-ns", Name: "fake-pod", }, } podObjectWithFinalizer := &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-ns", Name: "fake-pod", Finalizers: []string{"fake-finalizer-1", "fake-finalizer-2"}, }, } tests := []struct { name string clientObj []runtime.Object podName string namespace string reactors []reactor err string }{ { name: "delete fail", podName: "fake-pod", namespace: "fake-ns", err: "error to delete pod fake-pod: pods \"fake-pod\" not found", }, { name: "wait timeout", podName: "fake-pod", namespace: "fake-ns", clientObj: []runtime.Object{podObjectWithFinalizer}, reactors: []reactor{ { verb: "delete", resource: "pods", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, nil }, }, }, err: "timeout to assure pod fake-pod is deleted, finalizers in pod [fake-finalizer-1 fake-finalizer-2]", }, { name: "wait timeout, no finalizer", podName: "fake-pod", namespace: "fake-ns", clientObj: []runtime.Object{podObject}, reactors: []reactor{ { verb: "delete", resource: "pods", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, nil }, }, }, err: "timeout to assure pod fake-pod is deleted, finalizers in pod []", }, { name: "wait fail", podName: "fake-pod", namespace: "fake-ns", clientObj: []runtime.Object{podObject}, reactors: []reactor{ { verb: "get", resource: "pods", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, errors.New("fake-get-error") }, }, }, err: "error to assure pod is deleted, fake-pod: error to get pod fake-pod: fake-get-error", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeKubeClient := fake.NewSimpleClientset(test.clientObj...) for _, reactor := range test.reactors { fakeKubeClient.Fake.PrependReactor(reactor.verb, reactor.resource, reactor.reactorFunc) } var kubeClient kubernetes.Interface = fakeKubeClient err := EnsureDeletePod(t.Context(), kubeClient.CoreV1(), test.podName, test.namespace, time.Millisecond) if err != nil { assert.EqualError(t, err, test.err) } else { assert.NoError(t, err) } }) } } func TestIsPodRunning(t *testing.T) { tests := []struct { name string pod *corev1api.Pod err string }{ { name: "pod is nil", err: "invalid input pod", }, { name: "pod is not scheduled", pod: &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-ns", Name: "fake-pod", }, Status: corev1api.PodStatus{ Phase: "fake-phase", }, }, err: "pod is not scheduled, name=fake-pod, namespace=fake-ns, phase=fake-phase", }, { name: "pod is not running", pod: &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-ns", Name: "fake-pod", }, Spec: corev1api.PodSpec{ NodeName: "fake-node", }, Status: corev1api.PodStatus{ Phase: "fake-phase", }, }, err: "pod is not in the expected status, name=fake-pod, namespace=fake-ns, phase=fake-phase: pod is not running", }, { name: "pod is being deleted", pod: &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-ns", Name: "fake-pod", DeletionTimestamp: &metav1.Time{Time: time.Now()}, }, Spec: corev1api.PodSpec{ NodeName: "fake-node", }, Status: corev1api.PodStatus{ Phase: corev1api.PodRunning, }, }, err: "pod is being terminated, name=fake-pod, namespace=fake-ns, phase=Running", }, { name: "success", pod: &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-ns", Name: "fake-pod", }, Spec: corev1api.PodSpec{ NodeName: "fake-node", }, Status: corev1api.PodStatus{ Phase: corev1api.PodRunning, }, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { err := IsPodRunning(test.pod) if err != nil { assert.EqualError(t, err, test.err) } else { assert.NoError(t, err) } }) } } func TestIsPodScheduled(t *testing.T) { tests := []struct { name string pod *corev1api.Pod err string }{ { name: "pod is nil", err: "invalid input pod", }, { name: "pod is not scheduled", pod: &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-ns", Name: "fake-pod", }, Status: corev1api.PodStatus{ Phase: "fake-phase", }, }, err: "pod is not scheduled, name=fake-pod, namespace=fake-ns, phase=fake-phase", }, { name: "pod is not running or pending", pod: &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-ns", Name: "fake-pod", }, Spec: corev1api.PodSpec{ NodeName: "fake-node", }, Status: corev1api.PodStatus{ Phase: "fake-phase", }, }, err: "pod is not in the expected status, name=fake-pod, namespace=fake-ns, phase=fake-phase: pod is not running or pending", }, { name: "pod is being deleted", pod: &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-ns", Name: "fake-pod", DeletionTimestamp: &metav1.Time{Time: time.Now()}, }, Spec: corev1api.PodSpec{ NodeName: "fake-node", }, Status: corev1api.PodStatus{ Phase: corev1api.PodRunning, }, }, err: "pod is being terminated, name=fake-pod, namespace=fake-ns, phase=Running", }, { name: "success on running", pod: &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-ns", Name: "fake-pod", }, Spec: corev1api.PodSpec{ NodeName: "fake-node", }, Status: corev1api.PodStatus{ Phase: corev1api.PodRunning, }, }, }, { name: "success on pending", pod: &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-ns", Name: "fake-pod", }, Spec: corev1api.PodSpec{ NodeName: "fake-node", }, Status: corev1api.PodStatus{ Phase: corev1api.PodPending, }, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { err := IsPodScheduled(test.pod) if err != nil { assert.EqualError(t, err, test.err) } else { assert.NoError(t, err) } }) } } func TestDeletePodIfAny(t *testing.T) { tests := []struct { name string podName string podNamespace string kubeClientObj []runtime.Object kubeReactors []reactor logMessage string logLevel string logError string }{ { name: "get fail", podName: "fake-pod", podNamespace: "fake-namespace", logMessage: "Abort deleting pod, it doesn't exist fake-namespace/fake-pod", logLevel: "level=debug", }, { name: "delete fail", podName: "fake-pod", podNamespace: "fake-namespace", kubeReactors: []reactor{ { verb: "delete", resource: "pods", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, errors.New("fake-delete-error") }, }, }, logMessage: "Failed to delete pod fake-namespace/fake-pod", logLevel: "level=error", logError: "error=fake-delete-error", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...) for _, reactor := range test.kubeReactors { fakeKubeClient.Fake.PrependReactor(reactor.verb, reactor.resource, reactor.reactorFunc) } var kubeClient kubernetes.Interface = fakeKubeClient logMessage := "" DeletePodIfAny(t.Context(), kubeClient.CoreV1(), test.podName, test.podNamespace, velerotest.NewSingleLogger(&logMessage)) if len(test.logMessage) > 0 { assert.Contains(t, logMessage, test.logMessage) } if len(test.logLevel) > 0 { assert.Contains(t, logMessage, test.logLevel) } if len(test.logError) > 0 { assert.Contains(t, logMessage, test.logError) } }) } } func TestIsPodUnrecoverable(t *testing.T) { tests := []struct { name string pod *corev1api.Pod want bool }{ { name: "pod is in failed state", pod: &corev1api.Pod{ Status: corev1api.PodStatus{ Phase: corev1api.PodFailed, }, }, want: true, }, { name: "pod is in unknown state", pod: &corev1api.Pod{ Status: corev1api.PodStatus{ Phase: corev1api.PodUnknown, }, }, want: true, }, { name: "container image pull failure", pod: &corev1api.Pod{ Status: corev1api.PodStatus{ ContainerStatuses: []corev1api.ContainerStatus{ {State: corev1api.ContainerState{Waiting: &corev1api.ContainerStateWaiting{Reason: "ImagePullBackOff"}}}, }, }, }, want: true, }, { name: "container image pull failure with different reason", pod: &corev1api.Pod{ Status: corev1api.PodStatus{ ContainerStatuses: []corev1api.ContainerStatus{ {State: corev1api.ContainerState{Waiting: &corev1api.ContainerStateWaiting{Reason: "ErrImageNeverPull"}}}, }, }, }, want: true, }, { name: "container image pull failure with different reason", pod: &corev1api.Pod{ Status: corev1api.PodStatus{ ContainerStatuses: []corev1api.ContainerStatus{ {State: corev1api.ContainerState{Waiting: &corev1api.ContainerStateWaiting{Reason: "OtherReason"}}}, }, }, }, want: false, }, { name: "pod is normal", pod: &corev1api.Pod{ Status: corev1api.PodStatus{ Phase: corev1api.PodRunning, ContainerStatuses: []corev1api.ContainerStatus{ {Ready: true, State: corev1api.ContainerState{Running: &corev1api.ContainerStateRunning{}}}, }, }, }, want: false, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { got, _ := IsPodUnrecoverable(test.pod, velerotest.NewLogger()) assert.Equal(t, test.want, got) }) } } func TestGetPodTerminateMessage(t *testing.T) { tests := []struct { name string pod *corev1api.Pod message string }{ { name: "empty message when no container status", pod: &corev1api.Pod{ Status: corev1api.PodStatus{ Phase: corev1api.PodFailed, }, }, }, { name: "empty message when no termination status", pod: &corev1api.Pod{ Status: corev1api.PodStatus{ ContainerStatuses: []corev1api.ContainerStatus{ {Name: "container-1", State: corev1api.ContainerState{Waiting: &corev1api.ContainerStateWaiting{Reason: "ImagePullBackOff"}}}, }, }, }, }, { name: "empty message when no termination message", pod: &corev1api.Pod{ Status: corev1api.PodStatus{ ContainerStatuses: []corev1api.ContainerStatus{ {Name: "container-1", State: corev1api.ContainerState{Terminated: &corev1api.ContainerStateTerminated{Reason: "fake-reason"}}}, }, }, }, }, { name: "with termination message", pod: &corev1api.Pod{ Status: corev1api.PodStatus{ ContainerStatuses: []corev1api.ContainerStatus{ {Name: "container-1", State: corev1api.ContainerState{Terminated: &corev1api.ContainerStateTerminated{Message: "message-1"}}}, {Name: "container-2", State: corev1api.ContainerState{Terminated: &corev1api.ContainerStateTerminated{Message: "message-2"}}}, {Name: "container-3", State: corev1api.ContainerState{Terminated: &corev1api.ContainerStateTerminated{Message: "message-3"}}}, }, }, }, message: "message-1/message-2/message-3/", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { message := GetPodTerminateMessage(test.pod) assert.Equal(t, test.message, message) }) } } func TestGetPodContainerTerminateMessage(t *testing.T) { tests := []struct { name string pod *corev1api.Pod container string message string }{ { name: "empty message when no container status", pod: &corev1api.Pod{ Status: corev1api.PodStatus{ Phase: corev1api.PodFailed, }, }, }, { name: "empty message when no termination status", pod: &corev1api.Pod{ Status: corev1api.PodStatus{ ContainerStatuses: []corev1api.ContainerStatus{ {Name: "container-1", State: corev1api.ContainerState{Waiting: &corev1api.ContainerStateWaiting{Reason: "ImagePullBackOff"}}}, }, }, }, container: "container-1", }, { name: "empty message when no termination message", pod: &corev1api.Pod{ Status: corev1api.PodStatus{ ContainerStatuses: []corev1api.ContainerStatus{ {Name: "container-1", State: corev1api.ContainerState{Terminated: &corev1api.ContainerStateTerminated{Reason: "fake-reason"}}}, }, }, }, container: "container-1", }, { name: "not matched container name", pod: &corev1api.Pod{ Status: corev1api.PodStatus{ ContainerStatuses: []corev1api.ContainerStatus{ {Name: "container-1", State: corev1api.ContainerState{Terminated: &corev1api.ContainerStateTerminated{Message: "message-1"}}}, {Name: "container-2", State: corev1api.ContainerState{Terminated: &corev1api.ContainerStateTerminated{Message: "message-2"}}}, {Name: "container-3", State: corev1api.ContainerState{Terminated: &corev1api.ContainerStateTerminated{Message: "message-3"}}}, }, }, }, container: "container-0", }, { name: "with termination message", pod: &corev1api.Pod{ Status: corev1api.PodStatus{ ContainerStatuses: []corev1api.ContainerStatus{ {Name: "container-1", State: corev1api.ContainerState{Terminated: &corev1api.ContainerStateTerminated{Message: "message-1"}}}, {Name: "container-2", State: corev1api.ContainerState{Terminated: &corev1api.ContainerStateTerminated{Message: "message-2"}}}, {Name: "container-3", State: corev1api.ContainerState{Terminated: &corev1api.ContainerStateTerminated{Message: "message-3"}}}, }, }, }, container: "container-2", message: "message-2", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { message := GetPodContainerTerminateMessage(test.pod, test.container) assert.Equal(t, test.message, message) }) } } type fakePodLog struct { getError error readError error beginWriteError error endWriteError error writeError error logMessage string outputMessage string readPos int } func (fp *fakePodLog) GetPodLogReader(ctx context.Context, podGetter corev1client.CoreV1Interface, pod string, namespace string, logOptions *corev1api.PodLogOptions) (io.ReadCloser, error) { if fp.getError != nil { return nil, fp.getError } return fp, nil } func (fp *fakePodLog) Read(p []byte) (n int, err error) { if fp.readError != nil { return -1, fp.readError } if fp.readPos == len(fp.logMessage) { return 0, io.EOF } copy(p, []byte(fp.logMessage)) fp.readPos += len(fp.logMessage) return len(fp.logMessage), nil } func (fp *fakePodLog) Close() error { return nil } func (fp *fakePodLog) Write(p []byte) (n int, err error) { message := string(p) if strings.Contains(message, "begin pod logs") { if fp.beginWriteError != nil { return -1, fp.beginWriteError } } else if strings.Contains(message, "end pod logs") { if fp.endWriteError != nil { return -1, fp.endWriteError } } else { if fp.writeError != nil { return -1, fp.writeError } } fp.outputMessage += message return len(message), nil } func TestCollectPodLogs(t *testing.T) { tests := []struct { name string pod string container string getError error readError error beginWriteError error endWriteError error writeError error readMessage string message string expectErr string }{ { name: "error to write begin indicator", beginWriteError: errors.New("fake-write-error-01"), expectErr: "error to write begin pod log indicator: fake-write-error-01", }, { name: "error to get log", pod: "fake-pod", container: "fake-container", getError: errors.New("fake-get-error"), message: "***************************begin pod logs[fake-pod/fake-container]***************************\nNo present log retrieved, err: fake-get-error\n***************************end pod logs[fake-pod/fake-container]***************************\n", }, { name: "error to read pod log", pod: "fake-pod", container: "fake-container", readError: errors.New("fake-read-error"), expectErr: "error to copy input: fake-read-error", }, { name: "error to write pod log", pod: "fake-pod", container: "fake-container", writeError: errors.New("fake-write-error-03"), readMessage: "fake pod message 01\n fake pod message 02\n fake pod message 03\n", expectErr: "error to copy input: fake-write-error-03", }, { name: "error to write end indicator", pod: "fake-pod", container: "fake-container", endWriteError: errors.New("fake-write-error-02"), readMessage: "fake pod message 01\n fake pod message 02\n fake pod message 03\n", expectErr: "error to write end pod log indicator: fake-write-error-02", }, { name: "succeed", pod: "fake-pod", container: "fake-container", readMessage: "fake pod message 01\n fake pod message 02\n fake pod message 03\n", message: "***************************begin pod logs[fake-pod/fake-container]***************************\nfake pod message 01\n fake pod message 02\n fake pod message 03\n***************************end pod logs[fake-pod/fake-container]***************************\n", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fp := &fakePodLog{ getError: test.getError, readError: test.readError, beginWriteError: test.beginWriteError, endWriteError: test.endWriteError, writeError: test.writeError, logMessage: test.readMessage, } podLogReaderGetter = fp.GetPodLogReader err := CollectPodLogs(t.Context(), nil, test.pod, "", test.container, fp) if test.expectErr != "" { assert.EqualError(t, err, test.expectErr) } else { require.NoError(t, err) assert.Equal(t, fp.outputMessage, test.message) } }) } } func TestToSystemAffinity(t *testing.T) { tests := []struct { name string loadAffinity *LoadAffinity volumeTopology *corev1api.NodeSelector expected *corev1api.Affinity }{ { name: "loadAffinity is nil", }, { name: "loadAffinity is empty", loadAffinity: &LoadAffinity{}, }, { name: "with match label", loadAffinity: &LoadAffinity{ NodeSelector: metav1.LabelSelector{ MatchLabels: map[string]string{ "key-1": "value-1", }, }, }, expected: &corev1api.Affinity{ NodeAffinity: &corev1api.NodeAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ NodeSelectorTerms: []corev1api.NodeSelectorTerm{ { MatchExpressions: []corev1api.NodeSelectorRequirement{ { Key: "key-1", Values: []string{"value-1"}, Operator: corev1api.NodeSelectorOpIn, }, }, }, }, }, }, }, }, { name: "with match expression", loadAffinity: &LoadAffinity{ NodeSelector: metav1.LabelSelector{ MatchLabels: map[string]string{ "key-2": "value-2", }, MatchExpressions: []metav1.LabelSelectorRequirement{ { Key: "key-3", Values: []string{"value-3-1", "value-3-2"}, Operator: metav1.LabelSelectorOpNotIn, }, { Key: "key-4", Values: []string{"value-4-1", "value-4-2", "value-4-3"}, Operator: metav1.LabelSelectorOpDoesNotExist, }, }, }, }, expected: &corev1api.Affinity{ NodeAffinity: &corev1api.NodeAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ NodeSelectorTerms: []corev1api.NodeSelectorTerm{ { MatchExpressions: []corev1api.NodeSelectorRequirement{ { Key: "key-2", Values: []string{"value-2"}, Operator: corev1api.NodeSelectorOpIn, }, { Key: "key-3", Values: []string{"value-3-1", "value-3-2"}, Operator: corev1api.NodeSelectorOpNotIn, }, { Key: "key-4", Values: []string{"value-4-1", "value-4-2", "value-4-3"}, Operator: corev1api.NodeSelectorOpDoesNotExist, }, }, }, }, }, }, }, }, { name: "with olume topology", volumeTopology: &corev1api.NodeSelector{ NodeSelectorTerms: []corev1api.NodeSelectorTerm{ { MatchExpressions: []corev1api.NodeSelectorRequirement{ { Key: "key-5", Values: []string{"value-5-1", "value-5-2", "value-5-3"}, Operator: corev1api.NodeSelectorOpGt, }, { Key: "key-6", Values: []string{"value-5-1", "value-5-2", "value-5-3"}, Operator: corev1api.NodeSelectorOpGt, }, }, }, { MatchExpressions: []corev1api.NodeSelectorRequirement{ { Key: "key-7", Values: []string{"value-7-1", "value-7-2", "value-7-3"}, Operator: corev1api.NodeSelectorOpGt, }, { Key: "key-8", Values: []string{"value-8-1", "value-8-2", "value-8-3"}, Operator: corev1api.NodeSelectorOpGt, }, }, }, { MatchFields: []corev1api.NodeSelectorRequirement{ { Key: "key-9", Values: []string{"value-9-1", "value-9-2", "value-9-3"}, Operator: corev1api.NodeSelectorOpGt, }, { Key: "key-a", Values: []string{"value-a-1", "value-a-2", "value-a-3"}, Operator: corev1api.NodeSelectorOpGt, }, }, }, }, }, expected: &corev1api.Affinity{ NodeAffinity: &corev1api.NodeAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ NodeSelectorTerms: []corev1api.NodeSelectorTerm{ { MatchExpressions: []corev1api.NodeSelectorRequirement{ { Key: "key-5", Values: []string{"value-5-1", "value-5-2", "value-5-3"}, Operator: corev1api.NodeSelectorOpGt, }, { Key: "key-6", Values: []string{"value-5-1", "value-5-2", "value-5-3"}, Operator: corev1api.NodeSelectorOpGt, }, }, }, { MatchExpressions: []corev1api.NodeSelectorRequirement{ { Key: "key-7", Values: []string{"value-7-1", "value-7-2", "value-7-3"}, Operator: corev1api.NodeSelectorOpGt, }, { Key: "key-8", Values: []string{"value-8-1", "value-8-2", "value-8-3"}, Operator: corev1api.NodeSelectorOpGt, }, }, }, { MatchFields: []corev1api.NodeSelectorRequirement{ { Key: "key-9", Values: []string{"value-9-1", "value-9-2", "value-9-3"}, Operator: corev1api.NodeSelectorOpGt, }, { Key: "key-a", Values: []string{"value-a-1", "value-a-2", "value-a-3"}, Operator: corev1api.NodeSelectorOpGt, }, }, }, }, }, }, }, }, { name: "with match expression and volume topology", loadAffinity: &LoadAffinity{ NodeSelector: metav1.LabelSelector{ MatchLabels: map[string]string{ "key-2": "value-2", }, MatchExpressions: []metav1.LabelSelectorRequirement{ { Key: "key-3", Values: []string{"value-3-1", "value-3-2"}, Operator: metav1.LabelSelectorOpNotIn, }, { Key: "key-4", Values: []string{"value-4-1", "value-4-2", "value-4-3"}, Operator: metav1.LabelSelectorOpDoesNotExist, }, }, }, }, volumeTopology: &corev1api.NodeSelector{ NodeSelectorTerms: []corev1api.NodeSelectorTerm{ { MatchExpressions: []corev1api.NodeSelectorRequirement{ { Key: "key-5", Values: []string{"value-5-1", "value-5-2", "value-5-3"}, Operator: corev1api.NodeSelectorOpGt, }, { Key: "key-6", Values: []string{"value-5-1", "value-5-2", "value-5-3"}, Operator: corev1api.NodeSelectorOpGt, }, }, }, { MatchExpressions: []corev1api.NodeSelectorRequirement{ { Key: "key-7", Values: []string{"value-7-1", "value-7-2", "value-7-3"}, Operator: corev1api.NodeSelectorOpGt, }, { Key: "key-8", Values: []string{"value-8-1", "value-8-2", "value-8-3"}, Operator: corev1api.NodeSelectorOpGt, }, }, }, { MatchFields: []corev1api.NodeSelectorRequirement{ { Key: "key-9", Values: []string{"value-9-1", "value-9-2", "value-9-3"}, Operator: corev1api.NodeSelectorOpGt, }, { Key: "key-a", Values: []string{"value-a-1", "value-a-2", "value-a-3"}, Operator: corev1api.NodeSelectorOpGt, }, }, }, }, }, expected: &corev1api.Affinity{ NodeAffinity: &corev1api.NodeAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ NodeSelectorTerms: []corev1api.NodeSelectorTerm{ { MatchExpressions: []corev1api.NodeSelectorRequirement{ { Key: "key-5", Values: []string{"value-5-1", "value-5-2", "value-5-3"}, Operator: corev1api.NodeSelectorOpGt, }, { Key: "key-6", Values: []string{"value-5-1", "value-5-2", "value-5-3"}, Operator: corev1api.NodeSelectorOpGt, }, { Key: "key-2", Values: []string{"value-2"}, Operator: corev1api.NodeSelectorOpIn, }, { Key: "key-3", Values: []string{"value-3-1", "value-3-2"}, Operator: corev1api.NodeSelectorOpNotIn, }, { Key: "key-4", Values: []string{"value-4-1", "value-4-2", "value-4-3"}, Operator: corev1api.NodeSelectorOpDoesNotExist, }, }, }, { MatchExpressions: []corev1api.NodeSelectorRequirement{ { Key: "key-7", Values: []string{"value-7-1", "value-7-2", "value-7-3"}, Operator: corev1api.NodeSelectorOpGt, }, { Key: "key-8", Values: []string{"value-8-1", "value-8-2", "value-8-3"}, Operator: corev1api.NodeSelectorOpGt, }, { Key: "key-2", Values: []string{"value-2"}, Operator: corev1api.NodeSelectorOpIn, }, { Key: "key-3", Values: []string{"value-3-1", "value-3-2"}, Operator: corev1api.NodeSelectorOpNotIn, }, { Key: "key-4", Values: []string{"value-4-1", "value-4-2", "value-4-3"}, Operator: corev1api.NodeSelectorOpDoesNotExist, }, }, }, { MatchExpressions: []corev1api.NodeSelectorRequirement{ { Key: "key-2", Values: []string{"value-2"}, Operator: corev1api.NodeSelectorOpIn, }, { Key: "key-3", Values: []string{"value-3-1", "value-3-2"}, Operator: corev1api.NodeSelectorOpNotIn, }, { Key: "key-4", Values: []string{"value-4-1", "value-4-2", "value-4-3"}, Operator: corev1api.NodeSelectorOpDoesNotExist, }, }, MatchFields: []corev1api.NodeSelectorRequirement{ { Key: "key-9", Values: []string{"value-9-1", "value-9-2", "value-9-3"}, Operator: corev1api.NodeSelectorOpGt, }, { Key: "key-a", Values: []string{"value-a-1", "value-a-2", "value-a-3"}, Operator: corev1api.NodeSelectorOpGt, }, }, }, }, }, }, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { affinity := ToSystemAffinity(test.loadAffinity, test.volumeTopology) assert.True(t, reflect.DeepEqual(affinity, test.expected)) }) } } func TestDiagnosePod(t *testing.T) { testCases := []struct { name string pod *corev1api.Pod events *corev1api.EventList expected string }{ { name: "pod with all info but event", pod: &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-pod", Namespace: "fake-ns", }, Spec: corev1api.PodSpec{ NodeName: "fake-node", }, Status: corev1api.PodStatus{ Phase: corev1api.PodPending, Conditions: []corev1api.PodCondition{ { Type: corev1api.PodInitialized, Status: corev1api.ConditionTrue, Reason: "fake-reason-1", Message: "fake-message-1", }, { Type: corev1api.PodScheduled, Status: corev1api.ConditionFalse, Reason: "fake-reason-2", Message: "fake-message-2", }, }, Message: "fake-message-3", }, }, expected: "Pod fake-ns/fake-pod, phase Pending, node name fake-node, message fake-message-3\nPod condition Initialized, status True, reason fake-reason-1, message fake-message-1\nPod condition PodScheduled, status False, reason fake-reason-2, message fake-message-2\n", }, { name: "pod with all info and empty event list", pod: &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-pod", Namespace: "fake-ns", }, Spec: corev1api.PodSpec{ NodeName: "fake-node", }, Status: corev1api.PodStatus{ Phase: corev1api.PodPending, Conditions: []corev1api.PodCondition{ { Type: corev1api.PodInitialized, Status: corev1api.ConditionTrue, Reason: "fake-reason-1", Message: "fake-message-1", }, { Type: corev1api.PodScheduled, Status: corev1api.ConditionFalse, Reason: "fake-reason-2", Message: "fake-message-2", }, }, Message: "fake-message-3", }, }, events: &corev1api.EventList{}, expected: "Pod fake-ns/fake-pod, phase Pending, node name fake-node, message fake-message-3\nPod condition Initialized, status True, reason fake-reason-1, message fake-message-1\nPod condition PodScheduled, status False, reason fake-reason-2, message fake-message-2\n", }, { name: "pod with all info and events", pod: &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-pod", Namespace: "fake-ns", UID: "fake-pod-uid", }, Spec: corev1api.PodSpec{ NodeName: "fake-node", }, Status: corev1api.PodStatus{ Phase: corev1api.PodPending, Conditions: []corev1api.PodCondition{ { Type: corev1api.PodInitialized, Status: corev1api.ConditionTrue, Reason: "fake-reason-1", Message: "fake-message-1", }, { Type: corev1api.PodScheduled, Status: corev1api.ConditionFalse, Reason: "fake-reason-2", Message: "fake-message-2", }, }, Message: "fake-message-3", }, }, events: &corev1api.EventList{Items: []corev1api.Event{ { InvolvedObject: corev1api.ObjectReference{UID: "fake-uid-1"}, Type: corev1api.EventTypeWarning, Reason: "reason-1", Message: "message-1", }, { InvolvedObject: corev1api.ObjectReference{UID: "fake-uid-2"}, Type: corev1api.EventTypeWarning, Reason: "reason-2", Message: "message-2", }, { InvolvedObject: corev1api.ObjectReference{UID: "fake-pod-uid"}, Type: corev1api.EventTypeWarning, Reason: "reason-3", Message: "message-3", }, { InvolvedObject: corev1api.ObjectReference{UID: "fake-pod-uid"}, Type: corev1api.EventTypeNormal, Reason: "reason-4", Message: "message-4", }, { InvolvedObject: corev1api.ObjectReference{UID: "fake-pod-uid"}, Type: corev1api.EventTypeNormal, Reason: "reason-5", Message: "message-5", }, { InvolvedObject: corev1api.ObjectReference{UID: "fake-pod-uid"}, Type: corev1api.EventTypeWarning, Reason: "reason-6", Message: "message-6", }, }}, expected: "Pod fake-ns/fake-pod, phase Pending, node name fake-node, message fake-message-3\nPod condition Initialized, status True, reason fake-reason-1, message fake-message-1\nPod condition PodScheduled, status False, reason fake-reason-2, message fake-message-2\nPod event reason reason-3, message message-3\nPod event reason reason-6, message message-6\n", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { diag := DiagnosePod(tc.pod, tc.events) assert.Equal(t, tc.expected, diag) }) } } type exitWithMessageMock struct { createErr error writeFail bool filePath string exitCode int } func (em *exitWithMessageMock) Exit(code int) { em.exitCode = code } func (em *exitWithMessageMock) CreateFile(name string) (*os.File, error) { if em.createErr != nil { return nil, em.createErr } if em.writeFail { return os.OpenFile(em.filePath, os.O_CREATE|os.O_RDONLY, 0500) } else { return os.Create(em.filePath) } } func TestExitPodWithMessage(t *testing.T) { tests := []struct { name string message string succeed bool args []any createErr error writeFail bool expectedExitCode int expectedMessage string }{ { name: "create pod file failed", createErr: errors.New("fake-create-file-error"), succeed: true, expectedExitCode: 1, }, { name: "write pod file failed", writeFail: true, succeed: true, expectedExitCode: 1, }, { name: "not succeed", message: "fake-message-1, arg-1 %s, arg-2 %v, arg-3 %v", args: []any{ "arg-1-1", 10, false, }, expectedExitCode: 1, expectedMessage: fmt.Sprintf("fake-message-1, arg-1 %s, arg-2 %v, arg-3 %v", "arg-1-1", 10, false), }, { name: "not succeed", message: "fake-message-2, arg-1 %s, arg-2 %v, arg-3 %v", args: []any{ "arg-1-2", 20, true, }, succeed: true, expectedMessage: fmt.Sprintf("fake-message-2, arg-1 %s, arg-2 %v, arg-3 %v", "arg-1-2", 20, true), }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { podFile := filepath.Join(os.TempDir(), uuid.NewString()) em := exitWithMessageMock{ createErr: test.createErr, writeFail: test.writeFail, filePath: podFile, } funcExit = em.Exit funcCreateFile = em.CreateFile ExitPodWithMessage(velerotest.NewLogger(), test.succeed, test.message, test.args...) assert.Equal(t, test.expectedExitCode, em.exitCode) if test.createErr == nil && !test.writeFail { reader, err := os.Open(podFile) require.NoError(t, err) message, err := io.ReadAll(reader) require.NoError(t, err) reader.Close() assert.Equal(t, test.expectedMessage, string(message)) } }) } } func TestGetLoadAffinityByStorageClass(t *testing.T) { tests := []struct { name string affinityList []*LoadAffinity scName string expectedAffinity *LoadAffinity }{ { name: "get global affinity", affinityList: []*LoadAffinity{ { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { Key: "kubernetes.io/arch", Operator: metav1.LabelSelectorOpIn, Values: []string{"amd64"}, }, }, }, StorageClass: "storage-class-01", }, { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { Key: "kubernetes.io/os", Operator: metav1.LabelSelectorOpIn, Values: []string{"Linux"}, }, }, }, }, }, scName: "", expectedAffinity: &LoadAffinity{ NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { Key: "kubernetes.io/os", Operator: metav1.LabelSelectorOpIn, Values: []string{"Linux"}, }, }, }, }, }, { name: "get affinity for StorageClass but only global affinity exists", affinityList: []*LoadAffinity{ { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { Key: "kubernetes.io/os", Operator: metav1.LabelSelectorOpIn, Values: []string{"Linux"}, }, }, }, }, { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { Key: "kubernetes.io/arch", Operator: metav1.LabelSelectorOpIn, Values: []string{"amd64"}, }, }, }, }, { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { Key: "kubernetes.io/os", Operator: metav1.LabelSelectorOpIn, Values: []string{"Windows"}, }, }, }, }, }, scName: "storage-class-01", expectedAffinity: &LoadAffinity{ NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { Key: "kubernetes.io/os", Operator: metav1.LabelSelectorOpIn, Values: []string{"Linux"}, }, }, }, }, }, { name: "get affinity for StorageClass", affinityList: []*LoadAffinity{ { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { Key: "kubernetes.io/control-plane=", Operator: metav1.LabelSelectorOpIn, Values: []string{""}, }, }, }, }, { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { Key: "kubernetes.io/os", Operator: metav1.LabelSelectorOpIn, Values: []string{"Linux"}, }, }, }, StorageClass: "storage-class-01", }, { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { Key: "kubernetes.io/arch", Operator: metav1.LabelSelectorOpIn, Values: []string{"amd64"}, }, }, }, StorageClass: "invalid-storage-class", }, }, scName: "storage-class-01", expectedAffinity: &LoadAffinity{ NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { Key: "kubernetes.io/os", Operator: metav1.LabelSelectorOpIn, Values: []string{"Linux"}, }, }, }, StorageClass: "storage-class-01", }, }, { name: "Cannot find a match Affinity", affinityList: []*LoadAffinity{ { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { Key: "kubernetes.io/arch", Operator: metav1.LabelSelectorOpIn, Values: []string{"amd64"}, }, }, }, StorageClass: "invalid-storage-class", }, }, scName: "storage-class-01", expectedAffinity: nil, }, { name: "affinityList is nil", affinityList: nil, scName: "storage-class-01", expectedAffinity: nil, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { result := GetLoadAffinityByStorageClass(test.affinityList, test.scName, velerotest.NewLogger()) assert.Equal(t, test.expectedAffinity, result) }) } } ================================================ FILE: pkg/util/kube/predicate.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package kube import ( "reflect" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/predicate" ) // SpecChangePredicate implements a default update predicate function on Spec change // As Velero doesn't enable subresource in CRDs, we cannot use the object's metadata.generation field to check the spec change // More details about the generation field refer to https://github.com/kubernetes-sigs/controller-runtime/blob/v0.12.2/pkg/predicate/predicate.go#L156 type SpecChangePredicate struct { predicate.Funcs } func (SpecChangePredicate) Update(e event.UpdateEvent) bool { if e.ObjectOld == nil { return false } if e.ObjectNew == nil { return false } oldSpec := reflect.ValueOf(e.ObjectOld).Elem().FieldByName("Spec") // contains no field named "Spec", return false directly if oldSpec.IsZero() { return false } newSpec := reflect.ValueOf(e.ObjectNew).Elem().FieldByName("Spec") return !reflect.DeepEqual(oldSpec.Interface(), newSpec.Interface()) } // NewAllEventPredicate creates a new Predicate that checks all the events with the provided func func NewAllEventPredicate(f func(object client.Object) bool) predicate.Predicate { return predicate.Funcs{ CreateFunc: func(event event.CreateEvent) bool { return f(event.Object) }, DeleteFunc: func(event event.DeleteEvent) bool { return f(event.Object) }, UpdateFunc: func(event event.UpdateEvent) bool { return f(event.ObjectNew) }, GenericFunc: func(event event.GenericEvent) bool { return f(event.Object) }, } } // FalsePredicate always returns false for all kinds of events type FalsePredicate struct{} // Create always returns false func (f FalsePredicate) Create(event.CreateEvent) bool { return false } // Delete always returns false func (f FalsePredicate) Delete(event.DeleteEvent) bool { return false } // Update always returns false func (f FalsePredicate) Update(event.UpdateEvent) bool { return false } // Generic always returns false func (f FalsePredicate) Generic(event.GenericEvent) bool { return false } // NewGenericEventPredicate creates a new Predicate that checks the Generic event with the provided func func NewGenericEventPredicate(f func(object client.Object) bool) predicate.Predicate { return predicate.Funcs{ GenericFunc: func(event event.GenericEvent) bool { return f(event.Object) }, } } // NewUpdateEventPredicate creates a new Predicate that checks the Update event with the provided func func NewUpdateEventPredicate( f func(objectOld client.Object, objectNew client.Object) bool, ) predicate.Predicate { return predicate.Funcs{ UpdateFunc: func(event event.UpdateEvent) bool { return f(event.ObjectOld, event.ObjectNew) }, } } func NewCreateEventPredicate( f func(object client.Object) bool, ) predicate.Predicate { return predicate.Funcs{ CreateFunc: func(event event.CreateEvent) bool { return f(event.Object) }, } } ================================================ FILE: pkg/util/kube/predicate_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package kube import ( "testing" "time" "github.com/stretchr/testify/assert" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) func TestSpecChangePredicate(t *testing.T) { cases := []struct { name string oldObj client.Object newObj client.Object changed bool }{ { name: "Contains no spec field", oldObj: &velerov1.BackupStorageLocation{ ObjectMeta: metav1.ObjectMeta{ Name: "bsl01", }, }, newObj: &velerov1.BackupStorageLocation{ ObjectMeta: metav1.ObjectMeta{ Name: "bsl01", }, }, changed: false, }, { name: "ObjectMetas are different, Specs are same", oldObj: &velerov1.BackupStorageLocation{ ObjectMeta: metav1.ObjectMeta{ Name: "bsl01", Annotations: map[string]string{"key1": "value1"}, }, Spec: velerov1.BackupStorageLocationSpec{ Provider: "azure", }, }, newObj: &velerov1.BackupStorageLocation{ ObjectMeta: metav1.ObjectMeta{ Name: "bsl01", Annotations: map[string]string{"key2": "value2"}, }, Spec: velerov1.BackupStorageLocationSpec{ Provider: "azure", }, }, changed: false, }, { name: "Statuses are different, Specs are same", oldObj: &velerov1.BackupStorageLocation{ Spec: velerov1.BackupStorageLocationSpec{ Provider: "azure", }, Status: velerov1.BackupStorageLocationStatus{ Phase: velerov1.BackupStorageLocationPhaseAvailable, }, }, newObj: &velerov1.BackupStorageLocation{ Spec: velerov1.BackupStorageLocationSpec{ Provider: "azure", }, Status: velerov1.BackupStorageLocationStatus{ Phase: velerov1.BackupStorageLocationPhaseUnavailable, }, }, changed: false, }, { name: "Specs are different", oldObj: &velerov1.BackupStorageLocation{ Spec: velerov1.BackupStorageLocationSpec{ Provider: "azure", }, }, newObj: &velerov1.BackupStorageLocation{ Spec: velerov1.BackupStorageLocationSpec{ Provider: "aws", }, }, changed: true, }, { name: "Specs are same", oldObj: &velerov1.BackupStorageLocation{ Spec: velerov1.BackupStorageLocationSpec{ Provider: "azure", Config: map[string]string{"key": "value"}, Credential: &corev1api.SecretKeySelector{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "secret", }, Key: "credential", }, StorageType: velerov1.StorageType{ ObjectStorage: &velerov1.ObjectStorageLocation{ Bucket: "bucket1", Prefix: "prefix", CACert: []byte{'a'}, }, }, Default: true, AccessMode: velerov1.BackupStorageLocationAccessModeReadWrite, BackupSyncPeriod: &metav1.Duration{ Duration: 1 * time.Minute, }, ValidationFrequency: &metav1.Duration{ Duration: 1 * time.Minute, }, }, }, newObj: &velerov1.BackupStorageLocation{ Spec: velerov1.BackupStorageLocationSpec{ Provider: "azure", Config: map[string]string{"key": "value"}, Credential: &corev1api.SecretKeySelector{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "secret", }, Key: "credential", }, StorageType: velerov1.StorageType{ ObjectStorage: &velerov1.ObjectStorageLocation{ Bucket: "bucket1", Prefix: "prefix", CACert: []byte{'a'}, }, }, Default: true, AccessMode: velerov1.BackupStorageLocationAccessModeReadWrite, BackupSyncPeriod: &metav1.Duration{ Duration: 1 * time.Minute, }, ValidationFrequency: &metav1.Duration{ Duration: 1 * time.Minute, }, }, }, changed: false, }, } predicate := SpecChangePredicate{} for _, c := range cases { t.Run(c.name, func(t *testing.T) { changed := predicate.Update(event.UpdateEvent{ ObjectOld: c.oldObj, ObjectNew: c.newObj, }) assert.Equal(t, c.changed, changed) }) } } func TestNewAllEventPredicate(t *testing.T) { predicate := NewAllEventPredicate(func(object client.Object) bool { return false }) assert.False(t, predicate.Create(event.CreateEvent{})) assert.False(t, predicate.Update(event.UpdateEvent{})) assert.False(t, predicate.Delete(event.DeleteEvent{})) assert.False(t, predicate.Generic(event.GenericEvent{})) } func TestNewGenericEventPredicate(t *testing.T) { predicate := NewGenericEventPredicate(func(object client.Object) bool { return false }) assert.False(t, predicate.Generic(event.GenericEvent{})) assert.True(t, predicate.Update(event.UpdateEvent{})) assert.True(t, predicate.Create(event.CreateEvent{})) assert.True(t, predicate.Delete(event.DeleteEvent{})) } func TestNewUpdateEventPredicate(t *testing.T) { predicate := NewUpdateEventPredicate( func(client.Object, client.Object) bool { return false }, ) assert.False(t, predicate.Update(event.UpdateEvent{})) assert.True(t, predicate.Create(event.CreateEvent{})) assert.True(t, predicate.Delete(event.DeleteEvent{})) assert.True(t, predicate.Generic(event.GenericEvent{})) } func TestNewCreateEventPredicate(t *testing.T) { predicate := NewCreateEventPredicate( func(client.Object) bool { return false }, ) assert.False(t, predicate.Create(event.CreateEvent{})) assert.True(t, predicate.Update(event.UpdateEvent{})) assert.True(t, predicate.Generic(event.GenericEvent{})) assert.True(t, predicate.Delete(event.DeleteEvent{})) } ================================================ FILE: pkg/util/kube/priority_class.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package kube import ( "context" "github.com/sirupsen/logrus" schedulingv1 "k8s.io/api/scheduling/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" "sigs.k8s.io/controller-runtime/pkg/client" ) // ValidatePriorityClass checks if the specified priority class exists in the cluster // Returns true if the priority class exists or if priorityClassName is empty // Returns false if the priority class doesn't exist or validation fails // Logs warnings when the priority class doesn't exist func ValidatePriorityClass(ctx context.Context, kubeClient kubernetes.Interface, priorityClassName string, logger logrus.FieldLogger) bool { if priorityClassName == "" { return true } _, err := kubeClient.SchedulingV1().PriorityClasses().Get(ctx, priorityClassName, metav1.GetOptions{}) if err != nil { if apierrors.IsNotFound(err) { logger.Warnf("Priority class %q not found in cluster. Pod creation may fail if the priority class doesn't exist when pods are scheduled.", priorityClassName) } else { logger.WithError(err).Warnf("Failed to validate priority class %q", priorityClassName) } return false } logger.Infof("Validated priority class %q exists in cluster", priorityClassName) return true } // ValidatePriorityClassWithClient checks if the specified priority class exists in the cluster using controller-runtime client // Returns nil if the priority class exists or if priorityClassName is empty // Returns error if the priority class doesn't exist or validation fails func ValidatePriorityClassWithClient(ctx context.Context, cli client.Client, priorityClassName string) error { if priorityClassName == "" { return nil } priorityClass := &schedulingv1.PriorityClass{} err := cli.Get(ctx, types.NamespacedName{Name: priorityClassName}, priorityClass) return err } ================================================ FILE: pkg/util/kube/priority_class_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package kube import ( "fmt" "testing" "github.com/stretchr/testify/assert" schedulingv1 "k8s.io/api/scheduling/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes/fake" k8stesting "k8s.io/client-go/testing" velerotesting "github.com/vmware-tanzu/velero/pkg/test" ) func TestValidatePriorityClass(t *testing.T) { tests := []struct { name string priorityClassName string existingPCs []runtime.Object clientReactors []k8stesting.ReactionFunc expectedLogs []string expectedLogLevel string expectedResult bool }{ { name: "empty priority class name should return without logging", priorityClassName: "", existingPCs: nil, expectedLogs: nil, expectedResult: true, }, { name: "existing priority class should log info message", priorityClassName: "high-priority", existingPCs: []runtime.Object{ &schedulingv1.PriorityClass{ ObjectMeta: metav1.ObjectMeta{ Name: "high-priority", }, Value: 100, }, }, expectedLogs: []string{"Validated priority class \\\"high-priority\\\" exists in cluster"}, expectedLogLevel: "info", expectedResult: true, }, { name: "non-existing priority class should log warning", priorityClassName: "does-not-exist", existingPCs: nil, expectedLogs: []string{"Priority class \\\"does-not-exist\\\" not found in cluster. Pod creation may fail if the priority class doesn't exist when pods are scheduled."}, expectedLogLevel: "warning", expectedResult: false, }, { name: "API error should log warning with error", priorityClassName: "test-priority", existingPCs: nil, clientReactors: []k8stesting.ReactionFunc{ func(action k8stesting.Action) (handled bool, ret runtime.Object, err error) { if action.GetVerb() == "get" && action.GetResource().Resource == "priorityclasses" { return true, nil, fmt.Errorf("API server error") } return false, nil, nil }, }, expectedLogs: []string{"Failed to validate priority class \\\"test-priority\\\""}, expectedLogLevel: "warning", expectedResult: false, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { // Create fake client with existing priority classes kubeClient := fake.NewSimpleClientset(test.existingPCs...) // Add any custom reactors for _, reactor := range test.clientReactors { kubeClient.PrependReactor("*", "*", reactor) } // Create test logger with buffer buffer := []string{} logger := velerotesting.NewMultipleLogger(&buffer) // Call the function result := ValidatePriorityClass(t.Context(), kubeClient, test.priorityClassName, logger) // Check result assert.Equal(t, test.expectedResult, result, "ValidatePriorityClass returned unexpected result") // Check logs if test.expectedLogs == nil { assert.Empty(t, buffer) } else { assert.Len(t, buffer, len(test.expectedLogs)) for i, expectedLog := range test.expectedLogs { assert.Contains(t, buffer[i], expectedLog) if test.expectedLogLevel == "info" { assert.Contains(t, buffer[i], "level=info") } else if test.expectedLogLevel == "warning" { assert.Contains(t, buffer[i], "level=warning") } } } }) } } ================================================ FILE: pkg/util/kube/pvc_pv.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package kube import ( "context" "encoding/json" "fmt" "strings" "time" jsonpatch "github.com/evanphx/json-patch/v5" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" crclient "sigs.k8s.io/controller-runtime/pkg/client" storagev1api "k8s.io/api/storage/v1" storagev1 "k8s.io/client-go/kubernetes/typed/storage/v1" ) const ( waitInternal = 2 * time.Second ) // DeletePVAndPVCIfAny deletes PVC and delete the bound PV if it exists and log an error when the deletion fails. // It first sets the reclaim policy of the PV to Delete, then PV will be deleted automatically when PVC is deleted. // If ensureTimeout is not 0, it waits until the PVC is deleted or timeout. func DeletePVAndPVCIfAny(ctx context.Context, client corev1client.CoreV1Interface, pvcName, pvcNamespace string, ensureTimeout time.Duration, log logrus.FieldLogger) { pvcObj, err := client.PersistentVolumeClaims(pvcNamespace).Get(ctx, pvcName, metav1.GetOptions{}) if err != nil { if apierrors.IsNotFound(err) { log.WithError(err).Debugf("Abort deleting PV and PVC, for related PVC doesn't exist, %s/%s", pvcNamespace, pvcName) return } else { log.Warnf("failed to get pvc %s/%s with err %v", pvcNamespace, pvcName, err) return } } if pvcObj.Spec.VolumeName == "" { log.Warnf("failed to delete PV, for related PVC %s/%s has no bind volume name", pvcNamespace, pvcName) } else { pvObj, err := client.PersistentVolumes().Get(ctx, pvcObj.Spec.VolumeName, metav1.GetOptions{}) if err != nil { log.Warnf("failed to delete PV %s with err %v", pvcObj.Spec.VolumeName, err) } else { _, err = SetPVReclaimPolicy(ctx, client, pvObj, corev1api.PersistentVolumeReclaimDelete) if err != nil { log.Warnf("failed to set reclaim policy of PV %s to delete with err %v", pvObj.Name, err) } } } if err := EnsureDeletePVC(ctx, client, pvcName, pvcNamespace, ensureTimeout); err != nil { log.Warnf("failed to delete pvc %s/%s with err %v", pvcNamespace, pvcName, err) } if err := EnsurePVDeleted(ctx, client, pvcObj.Spec.VolumeName, ensureTimeout); err != nil { log.Warnf("pv %s was not removed with err %v", pvcObj.Spec.VolumeName, err) } } // WaitPVCBound wait for binding of a PVC specified by name and returns the bound PV object func WaitPVCBound(ctx context.Context, pvcGetter corev1client.CoreV1Interface, pvGetter corev1client.CoreV1Interface, pvc string, namespace string, timeout time.Duration) (*corev1api.PersistentVolume, error) { var updated *corev1api.PersistentVolumeClaim err := wait.PollUntilContextTimeout(ctx, waitInternal, timeout, true, func(ctx context.Context) (bool, error) { tmpPVC, err := pvcGetter.PersistentVolumeClaims(namespace).Get(ctx, pvc, metav1.GetOptions{}) if err != nil { return false, errors.Wrapf(err, "error to get pvc %s/%s", namespace, pvc) } if tmpPVC.Spec.VolumeName == "" { return false, nil } updated = tmpPVC return true, nil }) if err != nil { return nil, errors.Wrap(err, "error to wait for rediness of PVC") } pv, err := pvGetter.PersistentVolumes().Get(ctx, updated.Spec.VolumeName, metav1.GetOptions{}) if err != nil { return nil, errors.Wrap(err, "error to get PV") } return pv, err } // DeletePVIfAny deletes a PV by name if it exists, and log an error when the deletion fails func DeletePVIfAny(ctx context.Context, pvGetter corev1client.CoreV1Interface, pvName string, log logrus.FieldLogger) { err := pvGetter.PersistentVolumes().Delete(ctx, pvName, metav1.DeleteOptions{}) if err != nil { if apierrors.IsNotFound(err) { log.WithError(err).Debugf("Abort deleting PV, it doesn't exist, %s", pvName) } else { log.WithError(err).Errorf("Failed to delete PV %s", pvName) } } } // EnsureDeletePVC asserts the existence of a PVC by name, deletes it and waits for its disappearance and returns errors on any failure // If timeout is 0, it doesn't wait and return nil func EnsureDeletePVC(ctx context.Context, pvcGetter corev1client.CoreV1Interface, pvcName string, namespace string, timeout time.Duration) error { err := pvcGetter.PersistentVolumeClaims(namespace).Delete(ctx, pvcName, metav1.DeleteOptions{}) if err != nil { return errors.Wrapf(err, "error to delete pvc %s", pvcName) } if timeout == 0 { return nil } var updated *corev1api.PersistentVolumeClaim err = wait.PollUntilContextTimeout(ctx, waitInternal, timeout, true, func(ctx context.Context) (bool, error) { pvc, err := pvcGetter.PersistentVolumeClaims(namespace).Get(ctx, pvcName, metav1.GetOptions{}) if err != nil { if apierrors.IsNotFound(err) { return true, nil } return false, errors.Wrapf(err, "error to get pvc %s", pvcName) } updated = pvc return false, nil }) if err != nil { if errors.Is(err, context.DeadlineExceeded) { return errors.Errorf("timeout to assure pvc %s is deleted, finalizers in pvc %v", pvcName, updated.Finalizers) } else { return errors.Wrapf(err, "error to ensure pvc deleted for %s", pvcName) } } return nil } // EnsurePVDeleted ensures a PV has been deleted. This function is supposed to be called after EnsureDeletePVC // If timeout is 0, it doesn't wait and return nil func EnsurePVDeleted(ctx context.Context, pvGetter corev1client.CoreV1Interface, pvName string, timeout time.Duration) error { if timeout == 0 { return nil } err := wait.PollUntilContextTimeout(ctx, waitInternal, timeout, true, func(ctx context.Context) (bool, error) { _, err := pvGetter.PersistentVolumes().Get(ctx, pvName, metav1.GetOptions{}) if err != nil { if apierrors.IsNotFound(err) { return true, nil } return false, errors.Wrapf(err, "error to get pv %s", pvName) } return false, nil }) if err != nil { if errors.Is(err, context.DeadlineExceeded) { return errors.Errorf("timeout to assure pv %s is deleted", pvName) } else { return errors.Wrapf(err, "error to ensure pv is deleted for %s", pvName) } } return nil } // RebindPVC rebinds a PVC by modifying its VolumeName to the specific PV func RebindPVC(ctx context.Context, pvcGetter corev1client.CoreV1Interface, pvc *corev1api.PersistentVolumeClaim, pv string) (*corev1api.PersistentVolumeClaim, error) { origBytes, err := json.Marshal(pvc) if err != nil { return nil, errors.Wrap(err, "error marshaling original PVC") } updated := pvc.DeepCopy() updated.Spec.VolumeName = pv delete(updated.Annotations, KubeAnnBindCompleted) delete(updated.Annotations, KubeAnnBoundByController) updatedBytes, err := json.Marshal(updated) if err != nil { return nil, errors.Wrap(err, "error marshaling updated PV") } patchBytes, err := jsonpatch.CreateMergePatch(origBytes, updatedBytes) if err != nil { return nil, errors.Wrap(err, "error creating json merge patch for PV") } updated, err = pvcGetter.PersistentVolumeClaims(pvc.Namespace).Patch(ctx, pvc.Name, types.MergePatchType, patchBytes, metav1.PatchOptions{}) if err != nil { return nil, errors.Wrap(err, "error patching PVC") } return updated, nil } // ResetPVBinding resets the binding info of a PV and adds the required labels so as to make it ready for binding func ResetPVBinding(ctx context.Context, pvGetter corev1client.CoreV1Interface, pv *corev1api.PersistentVolume, labels map[string]string, pvc *corev1api.PersistentVolumeClaim) (*corev1api.PersistentVolume, error) { origBytes, err := json.Marshal(pv) if err != nil { return nil, errors.Wrap(err, "error marshaling original PV") } updated := pv.DeepCopy() updated.Spec.ClaimRef = &corev1api.ObjectReference{ Kind: pvc.Kind, Namespace: pvc.Namespace, Name: pvc.Name, } delete(updated.Annotations, KubeAnnBoundByController) if labels != nil { if updated.Labels == nil { updated.Labels = make(map[string]string) } for k, v := range labels { if _, ok := updated.Labels[k]; !ok { updated.Labels[k] = v } } } updatedBytes, err := json.Marshal(updated) if err != nil { return nil, errors.Wrap(err, "error marshaling updated PV") } patchBytes, err := jsonpatch.CreateMergePatch(origBytes, updatedBytes) if err != nil { return nil, errors.Wrap(err, "error creating json merge patch for PV") } updated, err = pvGetter.PersistentVolumes().Patch(ctx, pv.Name, types.MergePatchType, patchBytes, metav1.PatchOptions{}) if err != nil { return nil, errors.Wrap(err, "error patching PV") } return updated, nil } // SetPVReclaimPolicy sets the specified reclaim policy to a PV func SetPVReclaimPolicy(ctx context.Context, pvGetter corev1client.CoreV1Interface, pv *corev1api.PersistentVolume, policy corev1api.PersistentVolumeReclaimPolicy) (*corev1api.PersistentVolume, error) { if pv.Spec.PersistentVolumeReclaimPolicy == policy { return nil, nil } origBytes, err := json.Marshal(pv) if err != nil { return nil, errors.Wrap(err, "error marshaling original PV") } updated := pv.DeepCopy() updated.Spec.PersistentVolumeReclaimPolicy = policy updatedBytes, err := json.Marshal(updated) if err != nil { return nil, errors.Wrap(err, "error marshaling updated PV") } patchBytes, err := jsonpatch.CreateMergePatch(origBytes, updatedBytes) if err != nil { return nil, errors.Wrap(err, "error creating json merge patch for PV") } updated, err = pvGetter.PersistentVolumes().Patch(ctx, pv.Name, types.MergePatchType, patchBytes, metav1.PatchOptions{}) if err != nil { return nil, errors.Wrap(err, "error patching PV") } return updated, nil } // WaitPVCConsumed waits for a PVC to be consumed by a pod so that the selected node is set by the pod scheduling; or does // nothing if the consuming doesn't affect the PV provision. // The latest PVC and the selected node will be returned. func WaitPVCConsumed( ctx context.Context, pvcGetter corev1client.CoreV1Interface, pvc string, namespace string, storageClient storagev1.StorageV1Interface, timeout time.Duration, ignoreConsume bool, ) (string, *corev1api.PersistentVolumeClaim, error) { selectedNode := "" var updated *corev1api.PersistentVolumeClaim var storageClass *storagev1api.StorageClass err := wait.PollUntilContextTimeout(ctx, waitInternal, timeout, true, func(ctx context.Context) (bool, error) { tmpPVC, err := pvcGetter.PersistentVolumeClaims(namespace).Get(ctx, pvc, metav1.GetOptions{}) if err != nil { return false, errors.Wrapf(err, "error to get pvc %s/%s", namespace, pvc) } if !ignoreConsume { if tmpPVC.Spec.StorageClassName != nil && storageClass == nil { storageClass, err = storageClient.StorageClasses().Get(ctx, *tmpPVC.Spec.StorageClassName, metav1.GetOptions{}) if err != nil { return false, errors.Wrapf(err, "error to get storage class %s", *tmpPVC.Spec.StorageClassName) } } if storageClass != nil { if storageClass.VolumeBindingMode != nil && *storageClass.VolumeBindingMode == storagev1api.VolumeBindingWaitForFirstConsumer { selectedNode = tmpPVC.Annotations[KubeAnnSelectedNode] if selectedNode == "" { return false, nil } } } } updated = tmpPVC return true, nil }) if err != nil { return "", nil, errors.Wrap(err, "error to wait for PVC") } return selectedNode, updated, err } // WaitPVBound wait for binding of a PV specified by name and returns the bound PV object func WaitPVBound(ctx context.Context, pvGetter corev1client.CoreV1Interface, pvName string, pvcName string, pvcNamespace string, timeout time.Duration) (*corev1api.PersistentVolume, error) { var updated *corev1api.PersistentVolume err := wait.PollUntilContextTimeout(ctx, waitInternal, timeout, true, func(ctx context.Context) (bool, error) { tmpPV, err := pvGetter.PersistentVolumes().Get(ctx, pvName, metav1.GetOptions{}) if err != nil { return false, errors.Wrapf(err, "failed to get pv %s", pvName) } if tmpPV.Spec.ClaimRef == nil { return false, nil } if tmpPV.Status.Phase != corev1api.VolumeBound { return false, nil } if tmpPV.Spec.ClaimRef.Name != pvcName { return false, errors.Errorf("pv has been bound by unexpected pvc %s/%s", tmpPV.Spec.ClaimRef.Namespace, tmpPV.Spec.ClaimRef.Name) } if tmpPV.Spec.ClaimRef.Namespace != pvcNamespace { return false, errors.Errorf("pv has been bound by unexpected pvc %s/%s", tmpPV.Spec.ClaimRef.Namespace, tmpPV.Spec.ClaimRef.Name) } updated = tmpPV return true, nil }) if err != nil { return nil, errors.Wrap(err, "error to wait for bound of PV") } else { return updated, nil } } // IsPVCBound returns true if the specified PVC has been bound func IsPVCBound(pvc *corev1api.PersistentVolumeClaim) bool { return pvc.Spec.VolumeName != "" } // MakePodPVCAttachment returns the volume mounts and devices for a pod needed to attach a PVC func MakePodPVCAttachment(volumeName string, volumeMode *corev1api.PersistentVolumeMode, readOnly bool) ([]corev1api.VolumeMount, []corev1api.VolumeDevice, string) { var volumeMounts []corev1api.VolumeMount var volumeDevices []corev1api.VolumeDevice volumePath := "/" + volumeName if volumeMode != nil && *volumeMode == corev1api.PersistentVolumeBlock { volumeDevices = []corev1api.VolumeDevice{{ Name: volumeName, DevicePath: volumePath, }} } else { volumeMounts = []corev1api.VolumeMount{{ Name: volumeName, MountPath: volumePath, ReadOnly: readOnly, }} } return volumeMounts, volumeDevices, volumePath } // GetPVForPVC returns the PersistentVolume backing a PVC // returns PV, error. // PV will be nil on error func GetPVForPVC( pvc *corev1api.PersistentVolumeClaim, crClient crclient.Client, ) (*corev1api.PersistentVolume, error) { if pvc.Spec.VolumeName == "" { return nil, errors.Errorf("PVC %s/%s has no volume backing this claim", pvc.Namespace, pvc.Name) } if pvc.Status.Phase != corev1api.ClaimBound { return nil, errors.Errorf("PVC %s/%s is in phase %v and is not bound to a volume", pvc.Namespace, pvc.Name, pvc.Status.Phase) } pv := &corev1api.PersistentVolume{} err := crClient.Get( context.TODO(), crclient.ObjectKey{Name: pvc.Spec.VolumeName}, pv, ) if err != nil { return nil, errors.Wrapf(err, "failed to get PV %s for PVC %s/%s", pvc.Spec.VolumeName, pvc.Namespace, pvc.Name) } return pv, nil } func GetPVCForPodVolume(vol *corev1api.Volume, pod *corev1api.Pod, crClient crclient.Client) (*corev1api.PersistentVolumeClaim, error) { if vol.PersistentVolumeClaim == nil { return nil, errors.Errorf("volume %s/%s has no PVC associated with it", pod.Namespace, vol.Name) } pvc := &corev1api.PersistentVolumeClaim{} err := crClient.Get( context.TODO(), crclient.ObjectKey{Name: vol.PersistentVolumeClaim.ClaimName, Namespace: pod.Namespace}, pvc, ) if err != nil { return nil, errors.Wrapf(err, "failed to get PVC %s for Volume %s/%s", vol.PersistentVolumeClaim.ClaimName, pod.Namespace, vol.Name) } return pvc, nil } func DiagnosePVC(pvc *corev1api.PersistentVolumeClaim, events *corev1api.EventList) string { diag := fmt.Sprintf("PVC %s/%s, phase %s, binding to %s\n", pvc.Namespace, pvc.Name, pvc.Status.Phase, pvc.Spec.VolumeName) if events != nil { for _, e := range events.Items { if e.InvolvedObject.UID == pvc.UID && e.Type == corev1api.EventTypeWarning { diag += fmt.Sprintf("PVC event reason %s, message %s\n", e.Reason, e.Message) } } } return diag } func DiagnosePV(pv *corev1api.PersistentVolume) string { diag := fmt.Sprintf("PV %s, phase %s, reason %s, message %s\n", pv.Name, pv.Status.Phase, pv.Status.Reason, pv.Status.Message) return diag } func GetPVCAttachingNodeOS(pvc *corev1api.PersistentVolumeClaim, nodeClient corev1client.CoreV1Interface, storageClient storagev1.StorageV1Interface, log logrus.FieldLogger) string { var nodeOS string var scFsType string if pvc.Spec.VolumeMode != nil && *pvc.Spec.VolumeMode == corev1api.PersistentVolumeBlock { log.Infof("Use linux node for block mode PVC %s/%s", pvc.Namespace, pvc.Name) return NodeOSLinux } if pvc.Spec.VolumeName == "" { log.Warnf("PVC %s/%s is not bound to a PV", pvc.Namespace, pvc.Name) } if pvc.Spec.StorageClassName == nil { log.Warnf("PVC %s/%s is not with storage class", pvc.Namespace, pvc.Name) } nodeName := "" if pvc.Spec.VolumeName != "" { if n, err := GetPVAttachedNode(context.Background(), pvc.Spec.VolumeName, storageClient); err != nil { log.WithError(err).Warnf("Failed to get attached node for PVC %s/%s", pvc.Namespace, pvc.Name) } else { nodeName = n } } if nodeName == "" { if value := pvc.Annotations[KubeAnnSelectedNode]; value != "" { nodeName = value } } if nodeName != "" { if os, err := GetNodeOS(context.Background(), nodeName, nodeClient); err != nil { log.WithError(err).Warnf("Failed to get os from node %s for PVC %s/%s", nodeName, pvc.Namespace, pvc.Name) } else { nodeOS = os } } if pvc.Spec.StorageClassName != nil { if sc, err := storageClient.StorageClasses().Get(context.Background(), *pvc.Spec.StorageClassName, metav1.GetOptions{}); err != nil { log.WithError(err).Warnf("Failed to get storage class %s for PVC %s/%s", *pvc.Spec.StorageClassName, pvc.Namespace, pvc.Name) } else if sc.Parameters != nil { scFsType = strings.ToLower(sc.Parameters["csi.storage.k8s.io/fstype"]) } } if nodeOS != "" { log.Infof("Deduced node os %s from selected/attached node for PVC %s/%s (fsType %s)", nodeOS, pvc.Namespace, pvc.Name, scFsType) return nodeOS } if scFsType == "ntfs" { log.Infof("Deduced Windows node os from fsType %s for PVC %s/%s", scFsType, pvc.Namespace, pvc.Name) return NodeOSWindows } if scFsType != "" { log.Infof("Deduced linux node os from fsType %s for PVC %s/%s", scFsType, pvc.Namespace, pvc.Name) return NodeOSLinux } log.Warnf("Cannot deduce node os for PVC %s/%s, default to linux", pvc.Namespace, pvc.Name) return NodeOSLinux } func GetPVAttachedNode(ctx context.Context, pv string, storageClient storagev1.StorageV1Interface) (string, error) { vaList, err := storageClient.VolumeAttachments().List(ctx, metav1.ListOptions{}) if err != nil { return "", errors.Wrapf(err, "error listing volumeattachment") } for _, va := range vaList.Items { if va.Spec.Source.PersistentVolumeName != nil && *va.Spec.Source.PersistentVolumeName == pv { return va.Spec.NodeName, nil } } return "", nil } func GetPVAttachedNodes(ctx context.Context, pv string, storageClient storagev1.StorageV1Interface) ([]string, error) { vaList, err := storageClient.VolumeAttachments().List(ctx, metav1.ListOptions{}) if err != nil { return nil, errors.Wrapf(err, "error listing volumeattachment") } nodes := []string{} for _, va := range vaList.Items { if va.Spec.Source.PersistentVolumeName != nil && *va.Spec.Source.PersistentVolumeName == pv { nodes = append(nodes, va.Spec.NodeName) } } return nodes, nil } func GetVolumeTopology(ctx context.Context, volumeClient corev1client.CoreV1Interface, storageClient storagev1.StorageV1Interface, pvName string, scName string) (*corev1api.NodeSelector, error) { if pvName == "" || scName == "" { return nil, errors.Errorf("invalid parameter, pv %s, sc %s", pvName, scName) } sc, err := storageClient.StorageClasses().Get(ctx, scName, metav1.GetOptions{}) if err != nil { return nil, errors.Wrapf(err, "error getting storage class %s", scName) } if sc.VolumeBindingMode == nil || *sc.VolumeBindingMode != storagev1api.VolumeBindingWaitForFirstConsumer { return nil, nil } pv, err := volumeClient.PersistentVolumes().Get(ctx, pvName, metav1.GetOptions{}) if err != nil { return nil, errors.Wrapf(err, "error getting PV %s", pvName) } if pv.Spec.NodeAffinity == nil { return nil, nil } return pv.Spec.NodeAffinity.Required, nil } ================================================ FILE: pkg/util/kube/pvc_pv_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package kube import ( "testing" "time" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" corev1api "k8s.io/api/core/v1" storagev1api "k8s.io/api/storage/v1" clientTesting "k8s.io/client-go/testing" "github.com/vmware-tanzu/velero/pkg/builder" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) type reactor struct { verb string resource string reactorFunc clientTesting.ReactionFunc } func TestWaitPVCBound(t *testing.T) { pvcObject := &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-namespace", Name: "fake-pvc", }, } pvcObjectBound := &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-namespace", Name: "fake-pvc", }, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "fake-pv", }, } pvObj := &corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-pv", }, } tests := []struct { name string pvcName string pvcNamespace string kubeClientObj []runtime.Object kubeReactors []reactor expected *corev1api.PersistentVolume err string }{ { name: "wait pvc error", pvcName: "fake-pvc", pvcNamespace: "fake-namespace", err: "error to wait for rediness of PVC: error to get pvc fake-namespace/fake-pvc: persistentvolumeclaims \"fake-pvc\" not found", }, { name: "wait pvc timeout", pvcName: "fake-pvc", pvcNamespace: "fake-namespace", kubeClientObj: []runtime.Object{ pvcObject, }, err: "error to wait for rediness of PVC: context deadline exceeded", }, { name: "get pv fail", pvcName: "fake-pvc", pvcNamespace: "fake-namespace", kubeClientObj: []runtime.Object{ pvcObjectBound, }, err: "error to get PV: persistentvolumes \"fake-pv\" not found", }, { name: "success", pvcName: "fake-pvc", pvcNamespace: "fake-namespace", kubeClientObj: []runtime.Object{ pvcObjectBound, pvObj, }, expected: pvObj, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...) for _, reactor := range test.kubeReactors { fakeKubeClient.Fake.PrependReactor(reactor.verb, reactor.resource, reactor.reactorFunc) } var kubeClient kubernetes.Interface = fakeKubeClient pv, err := WaitPVCBound(t.Context(), kubeClient.CoreV1(), kubeClient.CoreV1(), test.pvcName, test.pvcNamespace, time.Millisecond) if err != nil { require.EqualError(t, err, test.err) } else { require.NoError(t, err) } assert.Equal(t, test.expected, pv) }) } } func TestWaitPVCConsumed(t *testing.T) { storageClass := "fake-storage-class" bindModeImmediate := storagev1api.VolumeBindingImmediate bindModeWait := storagev1api.VolumeBindingWaitForFirstConsumer pvcObject := &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-namespace", Name: "fake-pvc-1", }, } pvcObjectWithSC := &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-namespace", Name: "fake-pvc-2", }, Spec: corev1api.PersistentVolumeClaimSpec{ StorageClassName: &storageClass, }, } scObjWithoutBindMode := &storagev1api.StorageClass{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-storage-class", }, } scObjWaitBind := &storagev1api.StorageClass{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-storage-class", }, VolumeBindingMode: &bindModeWait, } scObjWithImmidateBinding := &storagev1api.StorageClass{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-storage-class", }, VolumeBindingMode: &bindModeImmediate, } pvcObjectWithSCAndAnno := &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-namespace", Name: "fake-pvc-3", Annotations: map[string]string{"volume.kubernetes.io/selected-node": "fake-node-1"}, }, Spec: corev1api.PersistentVolumeClaimSpec{ StorageClassName: &storageClass, }, } tests := []struct { name string pvcName string pvcNamespace string kubeClientObj []runtime.Object kubeReactors []reactor expectedPVC *corev1api.PersistentVolumeClaim selectedNode string ignoreWaitForFirstConsumer bool err string }{ { name: "get pvc error", pvcName: "fake-pvc", pvcNamespace: "fake-namespace", err: "error to wait for PVC: error to get pvc fake-namespace/fake-pvc: persistentvolumeclaims \"fake-pvc\" not found", }, { name: "success when no sc", pvcName: "fake-pvc-1", pvcNamespace: "fake-namespace", kubeClientObj: []runtime.Object{ pvcObject, }, expectedPVC: pvcObject, }, { name: "success when ignore wait for first consumer", pvcName: "fake-pvc-2", pvcNamespace: "fake-namespace", ignoreWaitForFirstConsumer: true, kubeClientObj: []runtime.Object{ pvcObjectWithSC, }, expectedPVC: pvcObjectWithSC, }, { name: "get sc fail", pvcName: "fake-pvc-2", pvcNamespace: "fake-namespace", kubeClientObj: []runtime.Object{ pvcObjectWithSC, }, err: "error to wait for PVC: error to get storage class fake-storage-class: storageclasses.storage.k8s.io \"fake-storage-class\" not found", }, { name: "success on sc without binding mode", pvcName: "fake-pvc-2", pvcNamespace: "fake-namespace", kubeClientObj: []runtime.Object{ pvcObjectWithSC, scObjWithoutBindMode, }, expectedPVC: pvcObjectWithSC, }, { name: "success on sc without immediate binding mode", pvcName: "fake-pvc-2", pvcNamespace: "fake-namespace", kubeClientObj: []runtime.Object{ pvcObjectWithSC, scObjWithImmidateBinding, }, expectedPVC: pvcObjectWithSC, }, { name: "pvc annotation miss", pvcName: "fake-pvc-2", pvcNamespace: "fake-namespace", kubeClientObj: []runtime.Object{ pvcObjectWithSC, scObjWaitBind, }, err: "error to wait for PVC: context deadline exceeded", }, { name: "success on sc without wait binding mode", pvcName: "fake-pvc-3", pvcNamespace: "fake-namespace", kubeClientObj: []runtime.Object{ pvcObjectWithSCAndAnno, scObjWaitBind, }, expectedPVC: pvcObjectWithSCAndAnno, selectedNode: "fake-node-1", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...) for _, reactor := range test.kubeReactors { fakeKubeClient.Fake.PrependReactor(reactor.verb, reactor.resource, reactor.reactorFunc) } var kubeClient kubernetes.Interface = fakeKubeClient selectedNode, pvc, err := WaitPVCConsumed(t.Context(), kubeClient.CoreV1(), test.pvcName, test.pvcNamespace, kubeClient.StorageV1(), time.Millisecond, test.ignoreWaitForFirstConsumer) if err != nil { require.EqualError(t, err, test.err) } else { require.NoError(t, err) } assert.Equal(t, test.expectedPVC, pvc) assert.Equal(t, test.selectedNode, selectedNode) }) } } func TestDeletePVCIfAny(t *testing.T) { pvObject := &corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-pv", Annotations: map[string]string{ KubeAnnBoundByController: "true", }, }, Spec: corev1api.PersistentVolumeSpec{ ClaimRef: &corev1api.ObjectReference{ Kind: "fake-kind", Namespace: "fake-ns", Name: "fake-pvc", }, }, } pvcObject := &corev1api.PersistentVolumeClaim{ TypeMeta: metav1.TypeMeta{ Kind: "fake-kind-1", }, ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-namespace", Name: "fake-pvc", Annotations: map[string]string{ KubeAnnBindCompleted: "true", KubeAnnBoundByController: "true", }, }, } pvcWithVolume := pvcObject.DeepCopy() pvcWithVolume.Spec.VolumeName = "fake-pv" tests := []struct { name string pvcName string pvcNamespace string pvName string kubeClientObj []runtime.Object kubeReactors []reactor logMessage string logLevel string logError string ensureTimeout time.Duration }{ { name: "pvc not found", pvcName: "fake-pvc", pvcNamespace: "fake-namespace", logMessage: "Abort deleting PV and PVC, for related PVC doesn't exist, fake-namespace/fake-pvc", logLevel: "level=debug", }, { name: "failed to get pvc", pvcName: "fake-pvc", pvcNamespace: "fake-namespace", kubeReactors: []reactor{ { verb: "get", resource: "persistentvolumeclaims", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, errors.New("fake-get-error") }, }, }, logMessage: "failed to get pvc fake-namespace/fake-pvc with err fake-get-error", logLevel: "level=warning", }, { name: "pvc has no volume name", pvcName: "fake-pvc", pvcNamespace: "fake-namespace", pvName: "fake-pv", kubeClientObj: []runtime.Object{ pvcObject, pvObject, }, logMessage: "failed to delete PV, for related PVC fake-namespace/fake-pvc has no bind volume name", logLevel: "level=warning", }, { name: "failed to delete pvc", pvcName: "fake-pvc", pvcNamespace: "fake-namespace", pvName: "fake-pv", kubeReactors: []reactor{ { verb: "delete", resource: "persistentvolumeclaims", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, errors.New("fake-delete-error") }, }, }, kubeClientObj: []runtime.Object{ pvcWithVolume, pvObject, }, logMessage: "failed to delete pvc fake-namespace/fake-pvc with err error to delete pvc fake-pvc: fake-delete-error", logLevel: "level=warning", }, { name: "failed to get pv", pvcName: "fake-pvc", pvcNamespace: "fake-namespace", pvName: "fake-pv", kubeReactors: []reactor{ { verb: "get", resource: "persistentvolumes", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, errors.New("fake-get-error") }, }, }, kubeClientObj: []runtime.Object{ pvcWithVolume, pvObject, }, logMessage: "failed to delete PV fake-pv with err fake-get-error", logLevel: "level=warning", }, { name: "set reclaim policy fail", pvcName: "fake-pvc", pvcNamespace: "fake-namespace", pvName: "fake-pv", kubeClientObj: []runtime.Object{ pvcWithVolume, pvObject, }, kubeReactors: []reactor{ { verb: "patch", resource: "persistentvolumes", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, pvObject, errors.New("fake-patch-error") }, }, }, logMessage: "failed to set reclaim policy of PV fake-pv to delete with err error patching PV: fake-patch-error", logLevel: "level=warning", logError: "fake-patch-error", }, { name: "delete pv pvc success", pvcName: "fake-pvc", pvcNamespace: "fake-namespace", pvName: "fake-pv", kubeClientObj: []runtime.Object{ pvcWithVolume, pvObject, }, }, { name: "delete pv pvc success but wait fail", pvcName: "fake-pvc", pvcNamespace: "fake-namespace", pvName: "fake-pv", kubeClientObj: []runtime.Object{ pvcWithVolume, pvObject, }, kubeReactors: []reactor{ { verb: "delete", resource: "persistentvolumeclaims", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, pvcWithVolume, nil }, }, }, ensureTimeout: time.Second, logMessage: "failed to delete pvc fake-namespace/fake-pvc with err timeout to assure pvc fake-pvc is deleted, finalizers in pvc []", logLevel: "level=warning", }, { name: "delete pv pvc success, wait won't succeed but ensureTimeout is 0", pvcName: "fake-pvc", pvcNamespace: "fake-namespace", pvName: "fake-pv", kubeClientObj: []runtime.Object{ pvcWithVolume, pvObject, }, kubeReactors: []reactor{ { verb: "delete", resource: "persistentvolumeclaims", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, pvcWithVolume, nil }, }, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...) for _, reactor := range test.kubeReactors { fakeKubeClient.Fake.PrependReactor(reactor.verb, reactor.resource, reactor.reactorFunc) } var kubeClient kubernetes.Interface = fakeKubeClient logMessage := "" DeletePVAndPVCIfAny(t.Context(), kubeClient.CoreV1(), test.pvcName, test.pvcNamespace, test.ensureTimeout, velerotest.NewSingleLogger(&logMessage)) if len(test.logMessage) > 0 { assert.Contains(t, logMessage, test.logMessage) } if len(test.logLevel) > 0 { assert.Contains(t, logMessage, test.logLevel) } if len(test.logError) > 0 { assert.Contains(t, logMessage, test.logError) } }) } } func TestDeletePVIfAny(t *testing.T) { tests := []struct { name string pvName string kubeClientObj []runtime.Object kubeReactors []reactor logMessage string logLevel string logError string }{ { name: "get fail", pvName: "fake-pv", logMessage: "Abort deleting PV, it doesn't exist, fake-pv", logLevel: "level=debug", }, { name: "delete fail", pvName: "fake-pv", kubeReactors: []reactor{ { verb: "delete", resource: "persistentvolumes", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, errors.New("fake-delete-error") }, }, }, logMessage: "Failed to delete PV fake-pv", logLevel: "level=error", logError: "error=fake-delete-error", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...) for _, reactor := range test.kubeReactors { fakeKubeClient.Fake.PrependReactor(reactor.verb, reactor.resource, reactor.reactorFunc) } var kubeClient kubernetes.Interface = fakeKubeClient logMessage := "" DeletePVIfAny(t.Context(), kubeClient.CoreV1(), test.pvName, velerotest.NewSingleLogger(&logMessage)) if len(test.logMessage) > 0 { assert.Contains(t, logMessage, test.logMessage) } if len(test.logLevel) > 0 { assert.Contains(t, logMessage, test.logLevel) } if len(test.logError) > 0 { assert.Contains(t, logMessage, test.logError) } }) } } func TestEnsureDeletePVC(t *testing.T) { pvcObject := &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-ns", Name: "fake-pvc", }, } pvcObjectWithFinalizer := &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-ns", Name: "fake-pvc", Finalizers: []string{"fake-finalizer-1", "fake-finalizer-2"}, }, } tests := []struct { name string clientObj []runtime.Object pvcName string namespace string reactors []reactor timeout time.Duration err string }{ { name: "delete fail", pvcName: "fake-pvc", namespace: "fake-ns", err: "error to delete pvc fake-pvc: persistentvolumeclaims \"fake-pvc\" not found", }, { name: "0 timeout", pvcName: "fake-pvc", namespace: "fake-ns", clientObj: []runtime.Object{pvcObject}, reactors: []reactor{ { verb: "delete", resource: "persistentvolumeclaims", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, pvcObject, nil }, }, }, }, { name: "wait fail", pvcName: "fake-pvc", namespace: "fake-ns", clientObj: []runtime.Object{pvcObject}, timeout: time.Millisecond, reactors: []reactor{ { verb: "get", resource: "persistentvolumeclaims", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, errors.New("fake-get-error") }, }, }, err: "error to ensure pvc deleted for fake-pvc: error to get pvc fake-pvc: fake-get-error", }, { name: "wait timeout", pvcName: "fake-pvc", namespace: "fake-ns", clientObj: []runtime.Object{pvcObjectWithFinalizer}, timeout: time.Millisecond, reactors: []reactor{ { verb: "delete", resource: "persistentvolumeclaims", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, pvcObject, nil }, }, }, err: "timeout to assure pvc fake-pvc is deleted, finalizers in pvc [fake-finalizer-1 fake-finalizer-2]", }, { name: "wait timeout, no finalizer", pvcName: "fake-pvc", namespace: "fake-ns", clientObj: []runtime.Object{pvcObject}, timeout: time.Millisecond, reactors: []reactor{ { verb: "delete", resource: "persistentvolumeclaims", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, pvcObject, nil }, }, }, err: "timeout to assure pvc fake-pvc is deleted, finalizers in pvc []", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeKubeClient := fake.NewSimpleClientset(test.clientObj...) for _, reactor := range test.reactors { fakeKubeClient.Fake.PrependReactor(reactor.verb, reactor.resource, reactor.reactorFunc) } var kubeClient kubernetes.Interface = fakeKubeClient err := EnsureDeletePVC(t.Context(), kubeClient.CoreV1(), test.pvcName, test.namespace, test.timeout) if err != nil { assert.EqualError(t, err, test.err) } else { assert.NoError(t, err) } }) } } func TestEnsureDeletePV(t *testing.T) { pvObject := &corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-pv", }, } tests := []struct { name string clientObj []runtime.Object pvName string reactors []reactor timeout time.Duration err string }{ { name: "get fail", pvName: "fake-pv", err: "error to get pv fake-pv: persistentvolumes \"fake-pv\" not found", }, { name: "0 timeout", pvName: "fake-pv", clientObj: []runtime.Object{pvObject}, reactors: []reactor{ { verb: "get", resource: "persistentvolumes", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, pvObject, nil }, }, }, }, { name: "wait fail", pvName: "fake-pv", clientObj: []runtime.Object{pvObject}, timeout: time.Millisecond, reactors: []reactor{ { verb: "get", resource: "persistentvolumes", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, errors.New("fake-get-error") }, }, }, err: "error to ensure pv is deleted for fake-pv: error to get pv fake-pv: fake-get-error", }, { name: "wait timeout", pvName: "fake-pv", clientObj: []runtime.Object{pvObject}, timeout: time.Millisecond, reactors: []reactor{ { verb: "get", resource: "persistentvolumes", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, pvObject, nil }, }, }, err: "timeout to assure pv fake-pv is deleted", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeKubeClient := fake.NewSimpleClientset(test.clientObj...) for _, reactor := range test.reactors { fakeKubeClient.Fake.PrependReactor(reactor.verb, reactor.resource, reactor.reactorFunc) } var kubeClient kubernetes.Interface = fakeKubeClient err := EnsurePVDeleted(t.Context(), kubeClient.CoreV1(), test.pvName, test.timeout) if err != nil { assert.EqualError(t, err, test.err) } else { assert.NoError(t, err) } }) } } func TestRebindPVC(t *testing.T) { pvcObject := &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-ns", Name: "fake-pvc", Annotations: map[string]string{ KubeAnnBindCompleted: "true", KubeAnnBoundByController: "true", }, }, } tests := []struct { name string clientObj []runtime.Object pvc *corev1api.PersistentVolumeClaim pv string reactors []reactor result *corev1api.PersistentVolumeClaim err string }{ { name: "path fail", pvc: pvcObject, pv: "fake-pv", clientObj: []runtime.Object{pvcObject}, reactors: []reactor{ { verb: "patch", resource: "persistentvolumeclaims", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, errors.New("fake-patch-error") }, }, }, err: "error patching PVC: fake-patch-error", }, { name: "succeed", pvc: pvcObject, pv: "fake-pv", clientObj: []runtime.Object{pvcObject}, result: &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-ns", Name: "fake-pvc", }, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "fake-pv", }, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeKubeClient := fake.NewSimpleClientset(test.clientObj...) for _, reactor := range test.reactors { fakeKubeClient.Fake.PrependReactor(reactor.verb, reactor.resource, reactor.reactorFunc) } var kubeClient kubernetes.Interface = fakeKubeClient result, err := RebindPVC(t.Context(), kubeClient.CoreV1(), test.pvc, test.pv) if err != nil { require.EqualError(t, err, test.err) } else { require.NoError(t, err) } assert.Equal(t, test.result, result) }) } } func TestResetPVBinding(t *testing.T) { pvObject := &corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-pv", Annotations: map[string]string{ KubeAnnBoundByController: "true", }, }, Spec: corev1api.PersistentVolumeSpec{ ClaimRef: &corev1api.ObjectReference{ Kind: "fake-kind", Namespace: "fake-ns", Name: "fake-pvc", }, }, } pvcObject := &corev1api.PersistentVolumeClaim{ TypeMeta: metav1.TypeMeta{ Kind: "fake-kind-1", }, ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-ns-1", Name: "fake-pvc-1", Annotations: map[string]string{ KubeAnnBindCompleted: "true", KubeAnnBoundByController: "true", }, }, } tests := []struct { name string clientObj []runtime.Object pv *corev1api.PersistentVolume pvc *corev1api.PersistentVolumeClaim labels map[string]string reactors []reactor result *corev1api.PersistentVolume err string }{ { name: "path fail", pv: pvObject, pvc: pvcObject, clientObj: []runtime.Object{pvObject}, reactors: []reactor{ { verb: "patch", resource: "persistentvolumes", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, errors.New("fake-patch-error") }, }, }, err: "error patching PV: fake-patch-error", }, { name: "succeed", pv: pvObject, pvc: pvcObject, labels: map[string]string{ "fake-label-1": "fake-value-1", "fake-label-2": "fake-value-2", }, clientObj: []runtime.Object{pvObject}, result: &corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-pv", Labels: map[string]string{ "fake-label-1": "fake-value-1", "fake-label-2": "fake-value-2", }, }, Spec: corev1api.PersistentVolumeSpec{ ClaimRef: &corev1api.ObjectReference{ Kind: "fake-kind-1", Namespace: "fake-ns-1", Name: "fake-pvc-1", }, }, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeKubeClient := fake.NewSimpleClientset(test.clientObj...) for _, reactor := range test.reactors { fakeKubeClient.Fake.PrependReactor(reactor.verb, reactor.resource, reactor.reactorFunc) } var kubeClient kubernetes.Interface = fakeKubeClient result, err := ResetPVBinding(t.Context(), kubeClient.CoreV1(), test.pv, test.labels, test.pvc) if err != nil { require.EqualError(t, err, test.err) } else { require.NoError(t, err) } assert.Equal(t, test.result, result) }) } } func TestSetPVReclaimPolicy(t *testing.T) { pvObject := &corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-pv", }, Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeReclaimPolicy: corev1api.PersistentVolumeReclaimRetain, }, } tests := []struct { name string clientObj []runtime.Object pv *corev1api.PersistentVolume policy corev1api.PersistentVolumeReclaimPolicy reactors []reactor result *corev1api.PersistentVolume err string }{ { name: "policy not changed", pv: pvObject, policy: corev1api.PersistentVolumeReclaimRetain, }, { name: "path fail", pv: pvObject, policy: corev1api.PersistentVolumeReclaimDelete, clientObj: []runtime.Object{pvObject}, reactors: []reactor{ { verb: "patch", resource: "persistentvolumes", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, errors.New("fake-patch-error") }, }, }, err: "error patching PV: fake-patch-error", }, { name: "succeed", pv: pvObject, policy: corev1api.PersistentVolumeReclaimDelete, clientObj: []runtime.Object{pvObject}, result: &corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-pv", }, Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeReclaimPolicy: corev1api.PersistentVolumeReclaimDelete, }, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeKubeClient := fake.NewSimpleClientset(test.clientObj...) for _, reactor := range test.reactors { fakeKubeClient.Fake.PrependReactor(reactor.verb, reactor.resource, reactor.reactorFunc) } var kubeClient kubernetes.Interface = fakeKubeClient result, err := SetPVReclaimPolicy(t.Context(), kubeClient.CoreV1(), test.pv, test.policy) if err != nil { require.EqualError(t, err, test.err) } else { require.NoError(t, err) } assert.Equal(t, test.result, result) }) } } func TestWaitPVBound(t *testing.T) { tests := []struct { name string pvName string pvcName string pvcNamespace string kubeClientObj []runtime.Object kubeReactors []reactor expectedPV *corev1api.PersistentVolume err string }{ { name: "get pv error", pvName: "fake-pv", err: "error to wait for bound of PV: failed to get pv fake-pv: persistentvolumes \"fake-pv\" not found", }, { name: "pvc claimRef miss", pvName: "fake-pv", kubeClientObj: []runtime.Object{ &corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-pv", }, }, }, err: "error to wait for bound of PV: context deadline exceeded", }, { name: "pvc status not bound", pvName: "fake-pv", kubeClientObj: []runtime.Object{ &corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-pv", }, }, }, err: "error to wait for bound of PV: context deadline exceeded", }, { name: "pvc claimRef pvc name mismatch", pvName: "fake-pv", pvcName: "fake-pvc", pvcNamespace: "fake-ns", kubeClientObj: []runtime.Object{ &corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-pv", }, Spec: corev1api.PersistentVolumeSpec{ ClaimRef: &corev1api.ObjectReference{ Kind: "fake-kind", Namespace: "fake-ns", Name: "fake-pvc-1", }, }, Status: corev1api.PersistentVolumeStatus{ Phase: "Bound", }, }, }, err: "error to wait for bound of PV: pv has been bound by unexpected pvc fake-ns/fake-pvc-1", }, { name: "pvc claimRef pvc namespace mismatch", pvName: "fake-pv", pvcName: "fake-pvc", pvcNamespace: "fake-ns", kubeClientObj: []runtime.Object{ &corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-pv", }, Spec: corev1api.PersistentVolumeSpec{ ClaimRef: &corev1api.ObjectReference{ Kind: "fake-kind", Namespace: "fake-ns-1", Name: "fake-pvc", }, }, Status: corev1api.PersistentVolumeStatus{ Phase: "Bound", }, }, }, err: "error to wait for bound of PV: pv has been bound by unexpected pvc fake-ns-1/fake-pvc", }, { name: "success", pvName: "fake-pv", pvcName: "fake-pvc", pvcNamespace: "fake-ns", kubeClientObj: []runtime.Object{ &corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-pv", }, Spec: corev1api.PersistentVolumeSpec{ ClaimRef: &corev1api.ObjectReference{ Kind: "fake-kind", Name: "fake-pvc", Namespace: "fake-ns", }, }, Status: corev1api.PersistentVolumeStatus{ Phase: "Bound", }, }, }, expectedPV: &corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-pv", }, Spec: corev1api.PersistentVolumeSpec{ ClaimRef: &corev1api.ObjectReference{ Kind: "fake-kind", Name: "fake-pvc", Namespace: "fake-ns", }, }, Status: corev1api.PersistentVolumeStatus{ Phase: "Bound", }, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...) for _, reactor := range test.kubeReactors { fakeKubeClient.Fake.PrependReactor(reactor.verb, reactor.resource, reactor.reactorFunc) } var kubeClient kubernetes.Interface = fakeKubeClient pv, err := WaitPVBound(t.Context(), kubeClient.CoreV1(), test.pvName, test.pvcName, test.pvcNamespace, time.Millisecond) if err != nil { require.EqualError(t, err, test.err) } else { require.NoError(t, err) } assert.Equal(t, test.expectedPV, pv) }) } } func TestIsPVCBound(t *testing.T) { tests := []struct { name string pvc *corev1api.PersistentVolumeClaim expect bool }{ { name: "expect bound", pvc: &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-ns", Name: "fake-pvc", }, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "fake-volume", }, }, expect: true, }, { name: "expect not bound", pvc: &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-ns", Name: "fake-pvc", }, }, expect: false, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { result := IsPVCBound(test.pvc) assert.Equal(t, test.expect, result) }) } } var ( csiStorageClass = "csi-hostpath-sc" ) func TestGetPVForPVC(t *testing.T) { boundPVC := &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: "test-csi-pvc", Namespace: "default", }, Spec: corev1api.PersistentVolumeClaimSpec{ AccessModes: []corev1api.PersistentVolumeAccessMode{corev1api.ReadWriteOnce}, Resources: corev1api.VolumeResourceRequirements{ Requests: corev1api.ResourceList{}, }, StorageClassName: &csiStorageClass, VolumeName: "test-csi-7d28e566-ade7-4ed6-9e15-2e44d2fbcc08", }, Status: corev1api.PersistentVolumeClaimStatus{ Phase: corev1api.ClaimBound, AccessModes: []corev1api.PersistentVolumeAccessMode{corev1api.ReadWriteOnce}, Capacity: corev1api.ResourceList{}, }, } matchingPV := &corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "test-csi-7d28e566-ade7-4ed6-9e15-2e44d2fbcc08", }, Spec: corev1api.PersistentVolumeSpec{ AccessModes: []corev1api.PersistentVolumeAccessMode{corev1api.ReadWriteOnce}, Capacity: corev1api.ResourceList{}, ClaimRef: &corev1api.ObjectReference{ Kind: "PersistentVolumeClaim", Name: "test-csi-pvc", Namespace: "default", ResourceVersion: "1027", UID: "7d28e566-ade7-4ed6-9e15-2e44d2fbcc08", }, PersistentVolumeSource: corev1api.PersistentVolumeSource{ CSI: &corev1api.CSIPersistentVolumeSource{ Driver: "hostpath.csi.k8s.io", FSType: "ext4", VolumeAttributes: map[string]string{ "storage.kubernetes.io/csiProvisionerIdentity": "1582049697841-8081-hostpath.csi.k8s.io", }, VolumeHandle: "e61f2b48-527a-11ea-b54f-cab6317018f1", }, }, PersistentVolumeReclaimPolicy: corev1api.PersistentVolumeReclaimDelete, StorageClassName: csiStorageClass, }, Status: corev1api.PersistentVolumeStatus{ Phase: corev1api.VolumeBound, }, } pvcWithNoVolumeName := &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: "no-vol-pvc", Namespace: "default", }, Spec: corev1api.PersistentVolumeClaimSpec{ AccessModes: []corev1api.PersistentVolumeAccessMode{corev1api.ReadWriteOnce}, Resources: corev1api.VolumeResourceRequirements{ Requests: corev1api.ResourceList{}, }, StorageClassName: &csiStorageClass, }, Status: corev1api.PersistentVolumeClaimStatus{}, } unboundPVC := &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: "unbound-pvc", Namespace: "default", }, Spec: corev1api.PersistentVolumeClaimSpec{ AccessModes: []corev1api.PersistentVolumeAccessMode{corev1api.ReadWriteOnce}, Resources: corev1api.VolumeResourceRequirements{ Requests: corev1api.ResourceList{}, }, StorageClassName: &csiStorageClass, VolumeName: "test-csi-7d28e566-ade7-4ed6-9e15-2e44d2fbcc08", }, Status: corev1api.PersistentVolumeClaimStatus{ Phase: corev1api.ClaimPending, AccessModes: []corev1api.PersistentVolumeAccessMode{corev1api.ReadWriteOnce}, Capacity: corev1api.ResourceList{}, }, } testCases := []struct { name string inPVC *corev1api.PersistentVolumeClaim expectError bool expectedPV *corev1api.PersistentVolume }{ { name: "should find PV matching the PVC", inPVC: boundPVC, expectError: false, expectedPV: matchingPV, }, { name: "should fail to find PV for PVC with no volumeName", inPVC: pvcWithNoVolumeName, expectError: true, expectedPV: nil, }, { name: "should fail to find PV for PVC not in bound phase", inPVC: unboundPVC, expectError: true, expectedPV: nil, }, } objs := []runtime.Object{boundPVC, matchingPV, pvcWithNoVolumeName, unboundPVC} fakeClient := velerotest.NewFakeControllerRuntimeClient(t, objs...) for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { actualPV, actualError := GetPVForPVC(tc.inPVC, fakeClient) if tc.expectError { require.Error(t, actualError, "Want error; Got nil error") assert.Nilf(t, actualPV, "Want PV: nil; Got PV: %q", actualPV) return } require.NoErrorf(t, actualError, "Want: nil error; Got: %v", actualError) assert.Equalf(t, actualPV.Name, tc.expectedPV.Name, "Want PV with name %q; Got PV with name %q", tc.expectedPV.Name, actualPV.Name) }) } } func TestGetPVCForPodVolume(t *testing.T) { sampleVol := &corev1api.Volume{ Name: "sample-volume", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "sample-pvc", }, }, } sampleVol2 := &corev1api.Volume{ Name: "sample-volume", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "sample-pvc-1", }, }, } sampleVol3 := &corev1api.Volume{ Name: "sample-volume", VolumeSource: corev1api.VolumeSource{}, } samplePod := &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "sample-pod", Namespace: "sample-ns", }, Spec: corev1api.PodSpec{ Containers: []corev1api.Container{ { Name: "sample-container", Image: "sample-image", VolumeMounts: []corev1api.VolumeMount{ { Name: "sample-vm", MountPath: "/etc/pod-info", }, }, }, }, Volumes: []corev1api.Volume{ { Name: "sample-volume", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "sample-pvc", }, }, }, }, }, } matchingPVC := &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: "sample-pvc", Namespace: "sample-ns", }, Spec: corev1api.PersistentVolumeClaimSpec{ AccessModes: []corev1api.PersistentVolumeAccessMode{corev1api.ReadWriteOnce}, Resources: corev1api.VolumeResourceRequirements{ Requests: corev1api.ResourceList{}, }, StorageClassName: &csiStorageClass, VolumeName: "test-csi-7d28e566-ade7-4ed6-9e15-2e44d2fbcc08", }, Status: corev1api.PersistentVolumeClaimStatus{ Phase: corev1api.ClaimBound, AccessModes: []corev1api.PersistentVolumeAccessMode{corev1api.ReadWriteOnce}, Capacity: corev1api.ResourceList{}, }, } testCases := []struct { name string vol *corev1api.Volume pod *corev1api.Pod expectedPVC *corev1api.PersistentVolumeClaim expectedError bool }{ { name: "should find PVC for volume", vol: sampleVol, pod: samplePod, expectedPVC: matchingPVC, expectedError: false, }, { name: "should not find PVC for volume not found error case", vol: sampleVol2, pod: samplePod, expectedPVC: nil, expectedError: true, }, { name: "should not find PVC vol has no PVC, error case", vol: sampleVol3, pod: samplePod, expectedPVC: nil, expectedError: true, }, } objs := []runtime.Object{matchingPVC} fakeClient := velerotest.NewFakeControllerRuntimeClient(t, objs...) for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { actualPVC, actualError := GetPVCForPodVolume(tc.vol, samplePod, fakeClient) if tc.expectedError { require.Error(t, actualError, "Want error; Got nil error") assert.Nilf(t, actualPVC, "Want PV: nil; Got PV: %q", actualPVC) return } require.NoErrorf(t, actualError, "Want: nil error; Got: %v", actualError) assert.Equalf(t, actualPVC.Name, tc.expectedPVC.Name, "Want PVC with name %q; Got PVC with name %q", tc.expectedPVC.Name, actualPVC) }) } } func TestMakePodPVCAttachment(t *testing.T) { testCases := []struct { name string volumeName string volumeMode corev1api.PersistentVolumeMode readOnly bool expectedVolumeMount []corev1api.VolumeMount expectedVolumeDevice []corev1api.VolumeDevice expectedVolumePath string }{ { name: "no volume mode specified", volumeName: "volume-1", readOnly: true, expectedVolumeMount: []corev1api.VolumeMount{ { Name: "volume-1", MountPath: "/volume-1", ReadOnly: true, }, }, expectedVolumePath: "/volume-1", }, { name: "fs mode specified", volumeName: "volume-2", volumeMode: corev1api.PersistentVolumeFilesystem, readOnly: true, expectedVolumeMount: []corev1api.VolumeMount{ { Name: "volume-2", MountPath: "/volume-2", ReadOnly: true, }, }, expectedVolumePath: "/volume-2", }, { name: "block volume mode specified", volumeName: "volume-3", volumeMode: corev1api.PersistentVolumeBlock, expectedVolumeDevice: []corev1api.VolumeDevice{ { Name: "volume-3", DevicePath: "/volume-3", }, }, expectedVolumePath: "/volume-3", }, { name: "fs mode specified with readOnly as false", volumeName: "volume-4", readOnly: false, volumeMode: corev1api.PersistentVolumeFilesystem, expectedVolumeMount: []corev1api.VolumeMount{ { Name: "volume-4", MountPath: "/volume-4", ReadOnly: false, }, }, expectedVolumePath: "/volume-4", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { var volMode *corev1api.PersistentVolumeMode if tc.volumeMode != "" { volMode = &tc.volumeMode } mount, device, path := MakePodPVCAttachment(tc.volumeName, volMode, tc.readOnly) assert.Equal(t, tc.expectedVolumeMount, mount) assert.Equal(t, tc.expectedVolumeDevice, device) assert.Equal(t, tc.expectedVolumePath, path) if tc.expectedVolumeMount != nil { assert.Equal(t, tc.expectedVolumeMount[0].ReadOnly, tc.readOnly) } }) } } func TestDiagnosePVC(t *testing.T) { testCases := []struct { name string pvc *corev1api.PersistentVolumeClaim events *corev1api.EventList expected string }{ { name: "pvc with all info but events", pvc: &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-pvc", Namespace: "fake-ns", }, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "fake-pv", }, Status: corev1api.PersistentVolumeClaimStatus{ Phase: corev1api.ClaimPending, }, }, expected: "PVC fake-ns/fake-pvc, phase Pending, binding to fake-pv\n", }, { name: "pvc with all info and empty events", pvc: &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-pvc", Namespace: "fake-ns", }, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "fake-pv", }, Status: corev1api.PersistentVolumeClaimStatus{ Phase: corev1api.ClaimPending, }, }, events: &corev1api.EventList{}, expected: "PVC fake-ns/fake-pvc, phase Pending, binding to fake-pv\n", }, { name: "pvc with all info and events", pvc: &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-pvc", Namespace: "fake-ns", UID: "fake-pvc-uid", }, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "fake-pv", }, Status: corev1api.PersistentVolumeClaimStatus{ Phase: corev1api.ClaimPending, }, }, events: &corev1api.EventList{Items: []corev1api.Event{ { InvolvedObject: corev1api.ObjectReference{UID: "fake-uid-1"}, Type: corev1api.EventTypeWarning, Reason: "reason-1", Message: "message-1", }, { InvolvedObject: corev1api.ObjectReference{UID: "fake-uid-2"}, Type: corev1api.EventTypeWarning, Reason: "reason-2", Message: "message-2", }, { InvolvedObject: corev1api.ObjectReference{UID: "fake-pvc-uid"}, Type: corev1api.EventTypeWarning, Reason: "reason-3", Message: "message-3", }, { InvolvedObject: corev1api.ObjectReference{UID: "fake-pvc-uid"}, Type: corev1api.EventTypeNormal, Reason: "reason-4", Message: "message-4", }, { InvolvedObject: corev1api.ObjectReference{UID: "fake-pvc-uid"}, Type: corev1api.EventTypeNormal, Reason: "reason-5", Message: "message-5", }, { InvolvedObject: corev1api.ObjectReference{UID: "fake-pvc-uid"}, Type: corev1api.EventTypeWarning, Reason: "reason-6", Message: "message-6", }, }}, expected: "PVC fake-ns/fake-pvc, phase Pending, binding to fake-pv\nPVC event reason reason-3, message message-3\nPVC event reason reason-6, message message-6\n", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { diag := DiagnosePVC(tc.pvc, tc.events) assert.Equal(t, tc.expected, diag) }) } } func TestDiagnosePV(t *testing.T) { testCases := []struct { name string pv *corev1api.PersistentVolume expected string }{ { name: "pv with all info", pv: &corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-pv", }, Status: corev1api.PersistentVolumeStatus{ Phase: corev1api.VolumePending, Message: "fake-message", Reason: "fake-reason", }, }, expected: "PV fake-pv, phase Pending, reason fake-reason, message fake-message\n", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { diag := DiagnosePV(tc.pv) assert.Equal(t, tc.expected, diag) }) } } func TestGetPVCAttachingNodeOS(t *testing.T) { storageClass := "fake-storage-class" nodeNoOSLabel := builder.ForNode("fake-node").Result() nodeWindows := builder.ForNode("fake-node").Labels(map[string]string{"kubernetes.io/os": "windows"}).Result() pvcObj := &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-namespace", Name: "fake-pvc", }, } blockMode := corev1api.PersistentVolumeBlock pvcObjBlockMode := &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-namespace", Name: "fake-pvc", }, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeMode: &blockMode, }, } pvName := "fake-volume-name" pvcObjWithAll := &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-namespace", Name: "fake-pvc", Annotations: map[string]string{KubeAnnSelectedNode: "fake-node"}, }, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: pvName, StorageClassName: &storageClass, }, } scObjWithoutFSType := &storagev1api.StorageClass{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-storage-class", }, } scObjWithFSType := &storagev1api.StorageClass{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-storage-class", }, Parameters: map[string]string{"csi.storage.k8s.io/fstype": "ntfs"}, } scObjWithFSTypeExt := &storagev1api.StorageClass{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-storage-class", }, Parameters: map[string]string{"csi.storage.k8s.io/fstype": "ext4"}, } volAttachEmpty := &storagev1api.VolumeAttachment{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-volume-attach-1", }, } volAttachWithVolume := &storagev1api.VolumeAttachment{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-volume-attach-2", }, Spec: storagev1api.VolumeAttachmentSpec{ Source: storagev1api.VolumeAttachmentSource{ PersistentVolumeName: &pvName, }, NodeName: "fake-node", }, } otherPVName := "other-volume-name" volAttachWithOtherVolume := &storagev1api.VolumeAttachment{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-volume-attach-3", }, Spec: storagev1api.VolumeAttachmentSpec{ Source: storagev1api.VolumeAttachmentSource{ PersistentVolumeName: &otherPVName, }, }, } tests := []struct { name string pvc *corev1api.PersistentVolumeClaim kubeClientObj []runtime.Object expectedNodeOS string }{ { name: "no selected node, volume name and storage class", pvc: pvcObj, expectedNodeOS: NodeOSLinux, }, { name: "fallback", pvc: pvcObjWithAll, expectedNodeOS: NodeOSLinux, }, { name: "with selected node, but node without label", pvc: pvcObjWithAll, kubeClientObj: []runtime.Object{ nodeNoOSLabel, }, expectedNodeOS: NodeOSLinux, }, { name: "volume attachment exist, but get node os fails", pvc: pvcObjWithAll, kubeClientObj: []runtime.Object{ scObjWithFSType, volAttachWithVolume, }, expectedNodeOS: NodeOSWindows, }, { name: "volume attachment exist, node without label", pvc: pvcObjWithAll, kubeClientObj: []runtime.Object{ nodeNoOSLabel, scObjWithFSType, volAttachWithVolume, }, expectedNodeOS: NodeOSWindows, }, { name: "sc without fsType", pvc: pvcObjWithAll, kubeClientObj: []runtime.Object{ scObjWithoutFSType, }, expectedNodeOS: NodeOSLinux, }, { name: "deduce from node os by selected node", pvc: pvcObjWithAll, kubeClientObj: []runtime.Object{ nodeWindows, scObjWithFSTypeExt, }, expectedNodeOS: NodeOSWindows, }, { name: "deduce from sc", pvc: pvcObjWithAll, kubeClientObj: []runtime.Object{ nodeNoOSLabel, scObjWithFSType, }, expectedNodeOS: NodeOSWindows, }, { name: "deduce from attached node os", pvc: pvcObjWithAll, kubeClientObj: []runtime.Object{ nodeWindows, scObjWithFSTypeExt, volAttachEmpty, volAttachWithVolume, volAttachWithOtherVolume, }, expectedNodeOS: NodeOSWindows, }, { name: "block access", pvc: pvcObjBlockMode, expectedNodeOS: NodeOSLinux, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...) var kubeClient kubernetes.Interface = fakeKubeClient nodeOS := GetPVCAttachingNodeOS(test.pvc, kubeClient.CoreV1(), kubeClient.StorageV1(), velerotest.NewLogger()) assert.Equal(t, test.expectedNodeOS, nodeOS) }) } } func TestGetVolumeTopology(t *testing.T) { pvWithoutNodeAffinity := &corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-pv", }, } pvWithNodeAffinity := &corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-pv", }, Spec: corev1api.PersistentVolumeSpec{ NodeAffinity: &corev1api.VolumeNodeAffinity{ Required: &corev1api.NodeSelector{ NodeSelectorTerms: []corev1api.NodeSelectorTerm{ { MatchExpressions: []corev1api.NodeSelectorRequirement{ { Key: "fake-key", }, }, }, }, }, }, }, } scObjWithoutVolumeBind := &storagev1api.StorageClass{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-storage-class", }, } volumeBindImmediate := storagev1api.VolumeBindingImmediate scObjWithImeediateBind := &storagev1api.StorageClass{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-storage-class", }, VolumeBindingMode: &volumeBindImmediate, } volumeBindWffc := storagev1api.VolumeBindingWaitForFirstConsumer scObjWithWffcBind := &storagev1api.StorageClass{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-storage-class", }, VolumeBindingMode: &volumeBindWffc, } tests := []struct { name string pvName string scName string kubeClientObj []runtime.Object expectedErr string expected *corev1api.NodeSelector }{ { name: "invalid pvName", scName: "fake-storage-class", expectedErr: "invalid parameter, pv , sc fake-storage-class", }, { name: "invalid scName", pvName: "fake-pv", expectedErr: "invalid parameter, pv fake-pv, sc ", }, { name: "no sc", pvName: "fake-pv", scName: "fake-storage-class", expectedErr: "error getting storage class fake-storage-class: storageclasses.storage.k8s.io \"fake-storage-class\" not found", }, { name: "sc without binding mode", pvName: "fake-pv", scName: "fake-storage-class", kubeClientObj: []runtime.Object{scObjWithoutVolumeBind}, }, { name: "sc without immediate binding mode", pvName: "fake-pv", scName: "fake-storage-class", kubeClientObj: []runtime.Object{scObjWithImeediateBind}, }, { name: "get pv fail", pvName: "fake-pv", scName: "fake-storage-class", kubeClientObj: []runtime.Object{scObjWithWffcBind}, expectedErr: "error getting PV fake-pv: persistentvolumes \"fake-pv\" not found", }, { name: "pv with no affinity", pvName: "fake-pv", scName: "fake-storage-class", kubeClientObj: []runtime.Object{ scObjWithWffcBind, pvWithoutNodeAffinity, }, }, { name: "pv with affinity", pvName: "fake-pv", scName: "fake-storage-class", kubeClientObj: []runtime.Object{ scObjWithWffcBind, pvWithNodeAffinity, }, expected: &corev1api.NodeSelector{ NodeSelectorTerms: []corev1api.NodeSelectorTerm{ { MatchExpressions: []corev1api.NodeSelectorRequirement{ { Key: "fake-key", }, }, }, }, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...) var kubeClient kubernetes.Interface = fakeKubeClient affinity, err := GetVolumeTopology(t.Context(), kubeClient.CoreV1(), kubeClient.StorageV1(), test.pvName, test.scName) if test.expectedErr != "" { assert.EqualError(t, err, test.expectedErr) } else { assert.Equal(t, test.expected, affinity) } }) } } ================================================ FILE: pkg/util/kube/resource_deletionstatus_tracker.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package kube import ( "fmt" "sync" "k8s.io/apimachinery/pkg/util/sets" ) // resourceDeletionStatusTracker keeps track of items pending deletion. type ResourceDeletionStatusTracker interface { // Add informs the tracker that a polling is in progress to check namespace deletion status. Add(kind, ns, name string) // Delete informs the tracker that a namespace deletion is completed. Delete(kind, ns, name string) // Contains returns true if the tracker is tracking the namespace deletion progress. Contains(kind, ns, name string) bool } type resourceDeletionStatusTracker struct { lock sync.RWMutex isNameSpacePresentInPollingSet sets.Set[string] } // NewResourceDeletionStatusTracker returns a new ResourceDeletionStatusTracker. func NewResourceDeletionStatusTracker() ResourceDeletionStatusTracker { return &resourceDeletionStatusTracker{ isNameSpacePresentInPollingSet: sets.New[string](), } } func (bt *resourceDeletionStatusTracker) Add(kind, ns, name string) { bt.lock.Lock() defer bt.lock.Unlock() bt.isNameSpacePresentInPollingSet.Insert(resourceDeletionStatusTrackerKey(kind, ns, name)) } func (bt *resourceDeletionStatusTracker) Delete(kind, ns, name string) { bt.lock.Lock() defer bt.lock.Unlock() bt.isNameSpacePresentInPollingSet.Delete(resourceDeletionStatusTrackerKey(kind, ns, name)) } func (bt *resourceDeletionStatusTracker) Contains(kind, ns, name string) bool { bt.lock.RLock() defer bt.lock.RUnlock() return bt.isNameSpacePresentInPollingSet.Has(resourceDeletionStatusTrackerKey(kind, ns, name)) } func resourceDeletionStatusTrackerKey(kind, ns, name string) string { return fmt.Sprintf("%s/%s/%s", kind, ns, name) } ================================================ FILE: pkg/util/kube/resource_requirements.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package kube import ( "github.com/pkg/errors" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" "github.com/vmware-tanzu/velero/pkg/constant" ) // ParseCPUAndMemoryResources is a helper function that parses CPU and memory requests and limits, // using default values for ephemeral storage. func ParseCPUAndMemoryResources(cpuRequest, memRequest, cpuLimit, memLimit string) (corev1api.ResourceRequirements, error) { return ParseResourceRequirements( cpuRequest, memRequest, constant.DefaultEphemeralStorageRequest, cpuLimit, memLimit, constant.DefaultEphemeralStorageLimit, ) } // ParseResourceRequirements takes a set of CPU, memory, ephemeral storage requests and limit string // values and returns a ResourceRequirements struct to be used in a Container. // An error is returned if we cannot parse the request/limit. func ParseResourceRequirements( cpuRequest, memRequest, ephemeralStorageRequest, cpuLimit, memLimit, ephemeralStorageLimit string, ) (corev1api.ResourceRequirements, error) { resources := corev1api.ResourceRequirements{ Requests: corev1api.ResourceList{}, Limits: corev1api.ResourceList{}, } parsedCPURequest, err := resource.ParseQuantity(cpuRequest) if err != nil { return resources, errors.Wrapf(err, `couldn't parse CPU request "%s"`, cpuRequest) } parsedMemRequest, err := resource.ParseQuantity(memRequest) if err != nil { return resources, errors.Wrapf(err, `couldn't parse memory request "%s"`, memRequest) } parsedEphemeralStorageRequest, err := resource.ParseQuantity(ephemeralStorageRequest) if err != nil { return resources, errors.Wrapf(err, `couldn't parse ephemeral storage request "%s"`, ephemeralStorageRequest) } parsedCPULimit, err := resource.ParseQuantity(cpuLimit) if err != nil { return resources, errors.Wrapf(err, `couldn't parse CPU limit "%s"`, cpuLimit) } parsedMemLimit, err := resource.ParseQuantity(memLimit) if err != nil { return resources, errors.Wrapf(err, `couldn't parse memory limit "%s"`, memLimit) } parsedEphemeralStorageLimit, err := resource.ParseQuantity(ephemeralStorageLimit) if err != nil { return resources, errors.Wrapf(err, `couldn't parse ephemeral storage limit "%s"`, ephemeralStorageLimit) } // A quantity of 0 is treated as unbounded unbounded := resource.MustParse("0") if parsedCPULimit != unbounded && parsedCPURequest.Cmp(parsedCPULimit) > 0 { return resources, errors.WithStack(errors.Errorf(`CPU request "%s" must be less than or equal to CPU limit "%s"`, cpuRequest, cpuLimit)) } if parsedMemLimit != unbounded && parsedMemRequest.Cmp(parsedMemLimit) > 0 { return resources, errors.WithStack(errors.Errorf(`Memory request "%s" must be less than or equal to Memory limit "%s"`, memRequest, memLimit)) } if parsedEphemeralStorageLimit != unbounded && parsedEphemeralStorageRequest.Cmp(parsedEphemeralStorageLimit) > 0 { return resources, errors.WithStack(errors.Errorf(`Ephemeral storage request "%s" must be less than or equal to Ephemeral storage limit "%s"`, ephemeralStorageRequest, ephemeralStorageLimit)) } // Only set resources if they are not unbounded if parsedCPURequest != unbounded { resources.Requests[corev1api.ResourceCPU] = parsedCPURequest } if parsedMemRequest != unbounded { resources.Requests[corev1api.ResourceMemory] = parsedMemRequest } if parsedEphemeralStorageRequest != unbounded { resources.Requests[corev1api.ResourceEphemeralStorage] = parsedEphemeralStorageRequest } if parsedCPULimit != unbounded { resources.Limits[corev1api.ResourceCPU] = parsedCPULimit } if parsedMemLimit != unbounded { resources.Limits[corev1api.ResourceMemory] = parsedMemLimit } if parsedEphemeralStorageLimit != unbounded { resources.Limits[corev1api.ResourceEphemeralStorage] = parsedEphemeralStorageLimit } return resources, nil } ================================================ FILE: pkg/util/kube/resource_requirements_test.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package kube import ( "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" ) func TestParseResourceRequirements(t *testing.T) { type args struct { cpuRequest string memRequest string ephemeralStorageRequest string cpuLimit string memLimit string ephemeralStorageLimit string } tests := []struct { name string args args wantErr bool expected *corev1api.ResourceRequirements }{ {"unbounded quantities", args{"0", "0", "0", "0", "0", "0"}, false, &corev1api.ResourceRequirements{ Requests: corev1api.ResourceList{}, Limits: corev1api.ResourceList{}, }}, {"valid quantities", args{"100m", "128Mi", "5Gi", "200m", "256Mi", "10Gi"}, false, nil}, {"CPU request with unbounded limit", args{"100m", "128Mi", "5Gi", "0", "256Mi", "10Gi"}, false, &corev1api.ResourceRequirements{ Requests: corev1api.ResourceList{ corev1api.ResourceCPU: resource.MustParse("100m"), corev1api.ResourceMemory: resource.MustParse("128Mi"), corev1api.ResourceEphemeralStorage: resource.MustParse("5Gi"), }, Limits: corev1api.ResourceList{ corev1api.ResourceMemory: resource.MustParse("256Mi"), corev1api.ResourceEphemeralStorage: resource.MustParse("10Gi"), }, }}, {"Mem request with unbounded limit", args{"100m", "128Mi", "5Gi", "200m", "0", "10Gi"}, false, &corev1api.ResourceRequirements{ Requests: corev1api.ResourceList{ corev1api.ResourceCPU: resource.MustParse("100m"), corev1api.ResourceMemory: resource.MustParse("128Mi"), corev1api.ResourceEphemeralStorage: resource.MustParse("5Gi"), }, Limits: corev1api.ResourceList{ corev1api.ResourceCPU: resource.MustParse("200m"), corev1api.ResourceEphemeralStorage: resource.MustParse("10Gi"), }, }}, {"Ephemeral storage request with unbounded limit", args{"100m", "128Mi", "5Gi", "200m", "256Mi", "0"}, false, &corev1api.ResourceRequirements{ Requests: corev1api.ResourceList{ corev1api.ResourceCPU: resource.MustParse("100m"), corev1api.ResourceMemory: resource.MustParse("128Mi"), corev1api.ResourceEphemeralStorage: resource.MustParse("5Gi"), }, Limits: corev1api.ResourceList{ corev1api.ResourceCPU: resource.MustParse("200m"), corev1api.ResourceMemory: resource.MustParse("256Mi"), }, }}, {"CPU/Mem/EphemeralStorage requests with unbounded limits", args{"100m", "128Mi", "5Gi", "0", "0", "0"}, false, &corev1api.ResourceRequirements{ Requests: corev1api.ResourceList{ corev1api.ResourceCPU: resource.MustParse("100m"), corev1api.ResourceMemory: resource.MustParse("128Mi"), corev1api.ResourceEphemeralStorage: resource.MustParse("5Gi"), }, Limits: corev1api.ResourceList{}, }}, {"invalid quantity", args{"100m", "invalid", "1Gi", "200m", "256Mi", "valid"}, true, nil}, {"CPU request greater than limit", args{"300m", "128Mi", "5Gi", "200m", "256Mi", "10Gi"}, true, nil}, {"memory request greater than limit", args{"100m", "512Mi", "5Gi", "200m", "256Mi", "10Gi"}, true, nil}, {"ephemeral storage request greater than limit", args{"100m", "128Mi", "10Gi", "200m", "256Mi", "5Gi"}, true, nil}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got, err := ParseResourceRequirements(tt.args.cpuRequest, tt.args.memRequest, tt.args.ephemeralStorageRequest, tt.args.cpuLimit, tt.args.memLimit, tt.args.ephemeralStorageLimit) if tt.wantErr { assert.Error(t, err) return } require.NoError(t, err) var expected corev1api.ResourceRequirements if tt.expected == nil { expected = corev1api.ResourceRequirements{ Requests: corev1api.ResourceList{ corev1api.ResourceCPU: resource.MustParse(tt.args.cpuRequest), corev1api.ResourceMemory: resource.MustParse(tt.args.memRequest), corev1api.ResourceEphemeralStorage: resource.MustParse(tt.args.ephemeralStorageRequest), }, Limits: corev1api.ResourceList{ corev1api.ResourceCPU: resource.MustParse(tt.args.cpuLimit), corev1api.ResourceMemory: resource.MustParse(tt.args.memLimit), corev1api.ResourceEphemeralStorage: resource.MustParse(tt.args.ephemeralStorageLimit), }, } } else { expected = *tt.expected } assert.Equal(t, expected, got) }) } } ================================================ FILE: pkg/util/kube/secrets.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package kube import ( "context" "github.com/pkg/errors" corev1api "k8s.io/api/core/v1" kbclient "sigs.k8s.io/controller-runtime/pkg/client" ) func GetSecret(client kbclient.Client, namespace, name string) (*corev1api.Secret, error) { secret := &corev1api.Secret{} if err := client.Get(context.TODO(), kbclient.ObjectKey{ Namespace: namespace, Name: name, }, secret); err != nil { return nil, err } return secret, nil } func GetSecretKey(client kbclient.Client, namespace string, selector *corev1api.SecretKeySelector) ([]byte, error) { secret, err := GetSecret(client, namespace, selector.Name) if err != nil { return nil, err } key, found := secret.Data[selector.Key] if !found { return nil, errors.Errorf("%q secret is missing data for key %q", selector.Name, selector.Key) } return key, nil } ================================================ FILE: pkg/util/kube/secrets_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package kube import ( "testing" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" "github.com/vmware-tanzu/velero/pkg/builder" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func TestGetSecretKey(t *testing.T) { testCases := []struct { name string secrets []*corev1api.Secret namespace string selector *corev1api.SecretKeySelector expectedData string expectedErr string }{ { name: "error is returned when secret doesn't exist", secrets: []*corev1api.Secret{ builder.ForSecret("ns-1", "secret1").Data(map[string][]byte{ "key1": []byte("ns1-secretdata1"), }).Result(), }, namespace: "ns-1", selector: builder.ForSecretKeySelector("non-existent-secret", "key2").Result(), expectedErr: "secrets \"non-existent-secret\" not found", }, { name: "error is returned when key is missing from secret", secrets: []*corev1api.Secret{ builder.ForSecret("ns-1", "secret1").Data(map[string][]byte{ "key1": []byte("ns1-secretdata1"), }).Result(), }, namespace: "ns-1", selector: builder.ForSecretKeySelector("secret1", "non-existent-key").Result(), expectedErr: "\"secret1\" secret is missing data for key \"non-existent-key\"", }, { name: "key specified in selector is returned", secrets: []*corev1api.Secret{ builder.ForSecret("ns-1", "secret1").Data(map[string][]byte{ "key1": []byte("ns1-secretdata1"), "key2": []byte("ns1-secretdata2"), }).Result(), builder.ForSecret("ns-2", "secret1").Data(map[string][]byte{ "key1": []byte("ns2-secretdata1"), "key2": []byte("ns2-secretdata2"), }).Result(), }, namespace: "ns-2", selector: builder.ForSecretKeySelector("secret1", "key2").Result(), expectedData: "ns2-secretdata2", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { fakeClient := velerotest.NewFakeControllerRuntimeClient(t) for _, secret := range tc.secrets { require.NoError(t, fakeClient.Create(t.Context(), secret)) } data, err := GetSecretKey(fakeClient, tc.namespace, tc.selector) if tc.expectedErr != "" { require.Nil(t, data) require.EqualError(t, err, tc.expectedErr) } else { require.NoError(t, err) require.Equal(t, tc.expectedData, string(data)) } }) } } ================================================ FILE: pkg/util/kube/security_context.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package kube import ( "strconv" "github.com/pkg/errors" corev1api "k8s.io/api/core/v1" "sigs.k8s.io/yaml" ) func ParseSecurityContext(runAsUser string, runAsGroup string, allowPrivilegeEscalation string, secCtx string) (corev1api.SecurityContext, error) { securityContext := corev1api.SecurityContext{} if runAsUser != "" { parsedRunAsUser, err := strconv.ParseInt(runAsUser, 10, 64) if err != nil { return securityContext, errors.WithStack(errors.Errorf(`Security context runAsUser "%s" is not a number`, runAsUser)) } securityContext.RunAsUser = &parsedRunAsUser } if runAsGroup != "" { parsedRunAsGroup, err := strconv.ParseInt(runAsGroup, 10, 64) if err != nil { return securityContext, errors.WithStack(errors.Errorf(`Security context runAsGroup "%s" is not a number`, runAsGroup)) } securityContext.RunAsGroup = &parsedRunAsGroup } if allowPrivilegeEscalation != "" { parsedAllowPrivilegeEscalation, err := strconv.ParseBool(allowPrivilegeEscalation) if err != nil { return securityContext, errors.WithStack(errors.Errorf(`Security context allowPrivilegeEscalation "%s" is not a boolean`, allowPrivilegeEscalation)) } securityContext.AllowPrivilegeEscalation = &parsedAllowPrivilegeEscalation } if secCtx != "" { err := yaml.UnmarshalStrict([]byte(secCtx), &securityContext) if err != nil { return securityContext, errors.WithStack(errors.Errorf(`Security context secCtx error: "%s"`, err)) } } return securityContext, nil } ================================================ FILE: pkg/util/kube/security_context_test.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package kube import ( "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" "github.com/vmware-tanzu/velero/pkg/util/boolptr" ) func TestParseSecurityContext(t *testing.T) { type args struct { runAsUser string runAsGroup string allowPrivilegeEscalation string secCtx string } tests := []struct { name string args args wantErr bool expected *corev1api.SecurityContext }{ { "valid security context", args{"1001", "999", "true", ``}, false, &corev1api.SecurityContext{ RunAsUser: pointInt64(1001), RunAsGroup: pointInt64(999), AllowPrivilegeEscalation: boolptr.True(), }, }, { "valid security context with override runAsUser", args{"1001", "999", "true", `runAsUser: 2000`}, false, &corev1api.SecurityContext{ RunAsUser: pointInt64(2000), RunAsGroup: pointInt64(999), AllowPrivilegeEscalation: boolptr.True(), }, }, { "valid securityContext with comments only secCtx key", args{"", "", "", ` capabilities: drop: - ALL add: - cap1 - cap2 seLinuxOptions: user: userLabel role: roleLabel type: typeLabel level: levelLabel # user www-data runAsUser: 3333 # group www-data runAsGroup: 3333 runAsNonRoot: true readOnlyRootFilesystem: true allowPrivilegeEscalation: false`}, false, &corev1api.SecurityContext{ RunAsUser: pointInt64(3333), RunAsGroup: pointInt64(3333), Capabilities: &corev1api.Capabilities{ Drop: []corev1api.Capability{"ALL"}, Add: []corev1api.Capability{"cap1", "cap2"}, }, SELinuxOptions: &corev1api.SELinuxOptions{ User: "userLabel", Role: "roleLabel", Type: "typeLabel", Level: "levelLabel", }, RunAsNonRoot: boolptr.True(), ReadOnlyRootFilesystem: boolptr.True(), AllowPrivilegeEscalation: boolptr.False(), }, }, { "valid securityContext with comments only secCtx key check seLinuxOptions is correctly parsed", args{"", "", "", ` capabilities: drop: - ALL add: - cap1 - cap2 seLinuxOptions: user: userLabelFail role: roleLabel type: typeLabel level: levelLabel # user www-data runAsUser: 3333 # group www-data runAsGroup: 3333 runAsNonRoot: true readOnlyRootFilesystem: true allowPrivilegeEscalation: false`}, true, &corev1api.SecurityContext{ RunAsUser: pointInt64(3333), RunAsGroup: pointInt64(3333), Capabilities: &corev1api.Capabilities{ Drop: []corev1api.Capability{"ALL"}, Add: []corev1api.Capability{"cap1", "cap2"}, }, SELinuxOptions: &corev1api.SELinuxOptions{ User: "userLabel", Role: "roleLabel", Type: "typeLabel", Level: "levelLabel", }, RunAsNonRoot: boolptr.True(), ReadOnlyRootFilesystem: boolptr.True(), AllowPrivilegeEscalation: boolptr.False(), }, }, { "valid securityContext with secCtx key override runAsUser runAsGroup and allowPrivilegeEscalation", args{"1001", "999", "true", ` capabilities: drop: - ALL add: - cap1 - cap2 seLinuxOptions: user: userLabel role: roleLabel type: typeLabel level: levelLabel # user www-data runAsUser: 3333 # group www-data runAsGroup: 3333 runAsNonRoot: true readOnlyRootFilesystem: true allowPrivilegeEscalation: false`}, false, &corev1api.SecurityContext{ RunAsUser: pointInt64(3333), RunAsGroup: pointInt64(3333), Capabilities: &corev1api.Capabilities{ Drop: []corev1api.Capability{"ALL"}, Add: []corev1api.Capability{"cap1", "cap2"}, }, SELinuxOptions: &corev1api.SELinuxOptions{ User: "userLabel", Role: "roleLabel", Type: "typeLabel", Level: "levelLabel", }, RunAsNonRoot: boolptr.True(), ReadOnlyRootFilesystem: boolptr.True(), AllowPrivilegeEscalation: boolptr.False(), }, }, { "another valid security context", args{"1001", "999", "false", ""}, false, &corev1api.SecurityContext{ RunAsUser: pointInt64(1001), RunAsGroup: pointInt64(999), AllowPrivilegeEscalation: boolptr.False(), }, }, {"security context without runAsGroup", args{"1001", "", "", ""}, false, &corev1api.SecurityContext{ RunAsUser: pointInt64(1001), }}, {"security context without runAsUser", args{"", "999", "", ""}, false, &corev1api.SecurityContext{ RunAsGroup: pointInt64(999), }}, {"empty context without runAsUser", args{"", "", "", ""}, false, &corev1api.SecurityContext{}}, { "invalid securityContext secCtx unknown key", args{"", "", "", ` capabilitiesUnknownkey: drop: - ALL add: - cap1 - cap2 # user www-data runAsUser: 3333 # group www-data runAsGroup: 3333 runAsNonRoot: true readOnlyRootFilesystem: true allowPrivilegeEscalation: false`}, true, nil, }, { "invalid securityContext secCtx wrong value type string instead of bool", args{"", "", "", ` capabilitiesUnknownkey: drop: - ALL add: - cap1 - cap2 # user www-data runAsUser: 3333 # group www-data runAsGroup: 3333 runAsNonRoot: plop readOnlyRootFilesystem: true allowPrivilegeEscalation: false`}, true, nil, }, { "invalid securityContext secCtx wrong value type string instead of int", args{"", "", "", ` capabilitiesUnknownkey: drop: - ALL add: - cap1 - cap2 # user www-data runAsUser: plop # group www-data runAsGroup: 3333 runAsNonRoot: true readOnlyRootFilesystem: true allowPrivilegeEscalation: false`}, true, nil, }, {"invalid security context runAsUser", args{"not a number", "", "", ""}, true, nil}, {"invalid security context runAsGroup", args{"", "not a number", "", ""}, true, nil}, {"invalid security context allowPrivilegeEscalation", args{"", "", "not a bool", ""}, true, nil}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got, err := ParseSecurityContext(tt.args.runAsUser, tt.args.runAsGroup, tt.args.allowPrivilegeEscalation, tt.args.secCtx) if err != nil && tt.wantErr { assert.Error(t, err) return } require.NoError(t, err) if tt.expected == nil { tt.expected = &corev1api.SecurityContext{} } if tt.wantErr { assert.NotEqual(t, *tt.expected, got) return } assert.Equal(t, *tt.expected, got) }) } } func pointInt64(i int64) *int64 { return &i } ================================================ FILE: pkg/util/kube/utils.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package kube import ( "context" "encoding/json" "fmt" "strings" "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/label" "github.com/vmware-tanzu/velero/pkg/uploader" "github.com/vmware-tanzu/velero/pkg/util/filesystem" ) // These annotations are taken from the Kubernetes persistent volume/persistent volume claim controller. // They cannot be directly importing because they are part of the kubernetes/kubernetes package, and importing that package is unsupported. // Their values are well-known and slow changing. They're duplicated here as constants to provide compile-time checking. // Originals can be found in kubernetes/kubernetes/pkg/controller/volume/persistentvolume/util/util.go. const ( KubeAnnBindCompleted = "pv.kubernetes.io/bind-completed" KubeAnnBoundByController = "pv.kubernetes.io/bound-by-controller" KubeAnnDynamicallyProvisioned = "pv.kubernetes.io/provisioned-by" KubeAnnMigratedTo = "pv.kubernetes.io/migrated-to" KubeAnnSelectedNode = "volume.kubernetes.io/selected-node" ) // VolumeSnapshotContentManagedByLabel is applied by the snapshot controller // to the VolumeSnapshotContent object in case distributed snapshotting is enabled. // The value contains the name of the node that handles the snapshot for the volume local to that node. const VolumeSnapshotContentManagedByLabel = "snapshot.storage.kubernetes.io/managed-by" var ErrorPodVolumeIsNotPVC = errors.New("pod volume is not a PVC") // NamespaceAndName returns a string in the format / func NamespaceAndName(objMeta metav1.Object) string { if objMeta.GetNamespace() == "" { return objMeta.GetName() } return fmt.Sprintf("%s/%s", objMeta.GetNamespace(), objMeta.GetName()) } // EnsureNamespaceExistsAndIsReady attempts to create the provided Kubernetes namespace. // It returns three values: // - a bool indicating whether or not the namespace is ready, // - a bool indicating whether or not the namespace was created // - an error if one occurred. // // examples: // // namespace already exists and is not ready, this function will return (false, false, nil). // If the namespace exists and is marked for deletion, this function will wait up to the timeout for it to fully delete. func EnsureNamespaceExistsAndIsReady(namespace *corev1api.Namespace, client corev1client.NamespaceInterface, timeout time.Duration, resourceDeletionStatusTracker ResourceDeletionStatusTracker) (ready bool, nsCreated bool, err error) { // nsCreated tells whether the namespace was created by this method // required for keeping track of number of restored items // if namespace is marked for deletion, and we timed out, report an error var terminatingNamespace bool var namespaceAlreadyInDeletionTracker bool err = wait.PollUntilContextTimeout(context.Background(), time.Second, timeout, true, func(ctx context.Context) (bool, error) { clusterNS, err := client.Get(ctx, namespace.Name, metav1.GetOptions{}) // if namespace is marked for deletion, and we timed out, report an error if apierrors.IsNotFound(err) { // Namespace isn't in cluster, we're good to create. return true, nil } if err != nil { // Return the err and exit the loop. return true, err } if clusterNS != nil && (clusterNS.GetDeletionTimestamp() != nil || clusterNS.Status.Phase == corev1api.NamespaceTerminating) { if resourceDeletionStatusTracker.Contains(clusterNS.Kind, clusterNS.Name, clusterNS.Name) { namespaceAlreadyInDeletionTracker = true return true, errors.Errorf("namespace %s is already present in the polling set, skipping execution", namespace.Name) } // Marked for deletion, keep waiting terminatingNamespace = true return false, nil } // clusterNS found, is not nil, and not marked for deletion, therefore we shouldn't create it. ready = true return true, nil }) // err will be set if we timed out or encountered issues retrieving the namespace, if err != nil { if terminatingNamespace { // If the namespace is marked for deletion, and we timed out, adding it in tracker resourceDeletionStatusTracker.Add(namespace.Kind, namespace.Name, namespace.Name) return false, nsCreated, errors.Wrapf(err, "timed out waiting for terminating namespace %s to disappear before restoring", namespace.Name) } else if namespaceAlreadyInDeletionTracker { // If the namespace is already in the tracker, return an error. return false, nsCreated, errors.Wrapf(err, "skipping polling for terminating namespace %s", namespace.Name) } return false, nsCreated, errors.Wrapf(err, "error getting namespace %s", namespace.Name) } // In the case the namespace already exists and isn't marked for deletion, assume it's ready for use. if ready { return true, nsCreated, nil } clusterNS, err := client.Create(context.TODO(), namespace, metav1.CreateOptions{}) if apierrors.IsAlreadyExists(err) { if clusterNS != nil && (clusterNS.GetDeletionTimestamp() != nil || clusterNS.Status.Phase == corev1api.NamespaceTerminating) { // Somehow created after all our polling and marked for deletion, return an error return false, nsCreated, errors.Errorf("namespace %s created and marked for termination after timeout", namespace.Name) } } else if err != nil { return false, nsCreated, errors.Wrapf(err, "error creating namespace %s", namespace.Name) } else { nsCreated = true } // The namespace created successfully return true, nsCreated, nil } // GetVolumeDirectory gets the name of the directory on the host, under /var/lib/kubelet/pods//volumes/, // where the specified volume lives. // For volumes with a CSIVolumeSource, append "/mount" to the directory name. func GetVolumeDirectory(ctx context.Context, log logrus.FieldLogger, pod *corev1api.Pod, volumeName string, kubeClient kubernetes.Interface) (string, error) { pvc, pv, volume, err := GetPodPVCVolume(ctx, log, pod, volumeName, kubeClient) if err != nil { // This case implies the administrator created the PV and attached it directly, without PVC. // Note that only one VolumeSource can be populated per Volume on a pod if err == ErrorPodVolumeIsNotPVC { if volume.VolumeSource.CSI != nil { return volume.Name + "/mount", nil } return volume.Name, nil } return "", errors.WithStack(err) } // Most common case is that we have a PVC VolumeSource, and we need to check the PV it points to for a CSI source. // PV's been created with a CSI source. isProvisionedByCSI, err := isProvisionedByCSI(log, pv, kubeClient) if err != nil { return "", errors.WithStack(err) } if isProvisionedByCSI { if pv.Spec.VolumeMode != nil && *pv.Spec.VolumeMode == corev1api.PersistentVolumeBlock { return pvc.Spec.VolumeName, nil } return pvc.Spec.VolumeName + "/mount", nil } return pvc.Spec.VolumeName, nil } // GetVolumeMode gets the uploader.PersistentVolumeMode of the volume. func GetVolumeMode(ctx context.Context, log logrus.FieldLogger, pod *corev1api.Pod, volumeName string, kubeClient kubernetes.Interface) ( uploader.PersistentVolumeMode, error) { _, pv, _, err := GetPodPVCVolume(ctx, log, pod, volumeName, kubeClient) if err != nil { if err == ErrorPodVolumeIsNotPVC { return uploader.PersistentVolumeFilesystem, nil } return "", errors.WithStack(err) } if pv.Spec.VolumeMode != nil && *pv.Spec.VolumeMode == corev1api.PersistentVolumeBlock { return uploader.PersistentVolumeBlock, nil } return uploader.PersistentVolumeFilesystem, nil } // GetPodPVCVolume gets the PVC, PV and volume for a pod volume name. // Returns pod volume in case of ErrorPodVolumeIsNotPVC error func GetPodPVCVolume(ctx context.Context, log logrus.FieldLogger, pod *corev1api.Pod, volumeName string, kubeClient kubernetes.Interface) ( *corev1api.PersistentVolumeClaim, *corev1api.PersistentVolume, *corev1api.Volume, error) { var volume *corev1api.Volume for i := range pod.Spec.Volumes { if pod.Spec.Volumes[i].Name == volumeName { volume = &pod.Spec.Volumes[i] break } } if volume == nil { return nil, nil, nil, errors.New("volume not found in pod") } if volume.VolumeSource.PersistentVolumeClaim == nil { return nil, nil, volume, ErrorPodVolumeIsNotPVC // There is a pod volume but it is not a PVC } pvc, err := kubeClient.CoreV1().PersistentVolumeClaims(pod.Namespace).Get(ctx, volume.VolumeSource.PersistentVolumeClaim.ClaimName, metav1.GetOptions{}) if err != nil { return nil, nil, nil, errors.WithStack(err) } pv, err := kubeClient.CoreV1().PersistentVolumes().Get(ctx, pvc.Spec.VolumeName, metav1.GetOptions{}) if err != nil { return nil, nil, nil, errors.WithStack(err) } return pvc, pv, volume, nil } // isProvisionedByCSI function checks whether this is a CSI PV by annotation. // Either "pv.kubernetes.io/provisioned-by" or "pv.kubernetes.io/migrated-to" indicates // PV is provisioned by CSI. func isProvisionedByCSI(log logrus.FieldLogger, pv *corev1api.PersistentVolume, kubeClient kubernetes.Interface) (bool, error) { if pv.Spec.CSI != nil { return true, nil } // Although the pv.Spec.CSI is nil, the volume could be provisioned by a CSI driver when enabling the CSI migration // Refer to https://github.com/vmware-tanzu/velero/issues/4496 for more details if pv.Annotations != nil { driverName := pv.Annotations[KubeAnnDynamicallyProvisioned] migratedDriver := pv.Annotations[KubeAnnMigratedTo] if len(driverName) > 0 || len(migratedDriver) > 0 { list, err := kubeClient.StorageV1().CSIDrivers().List(context.TODO(), metav1.ListOptions{}) if err != nil { return false, err } for _, driver := range list.Items { if driverName == driver.Name || migratedDriver == driver.Name { log.Debugf("the annotation %s or %s equals to %s indicates the volume is provisioned by a CSI driver", KubeAnnDynamicallyProvisioned, KubeAnnMigratedTo, driver.Name) return true, nil } } } } return false, nil } // SinglePathMatch checks whether pass-in volume path is valid. // Check whether there is only one match by the path's pattern. func SinglePathMatch(path string, fs filesystem.Interface, log logrus.FieldLogger) (string, error) { matches, err := fs.Glob(path) if err != nil { return "", errors.WithStack(err) } if len(matches) != 1 { return "", errors.Errorf("expected one matching path: %s, got %d", path, len(matches)) } log.Debugf("This is a valid volume path: %s.", matches[0]) return matches[0], nil } // IsV1CRDReady checks a v1 CRD to see if it's ready, with both the Established and NamesAccepted conditions. func IsV1CRDReady(crd *apiextv1.CustomResourceDefinition) bool { var isEstablished, namesAccepted bool for _, cond := range crd.Status.Conditions { if cond.Type == apiextv1.Established && cond.Status == apiextv1.ConditionTrue { isEstablished = true } if cond.Type == apiextv1.NamesAccepted && cond.Status == apiextv1.ConditionTrue { namesAccepted = true } } return (isEstablished && namesAccepted) } // IsV1Beta1CRDReady checks a v1beta1 CRD to see if it's ready, with both the Established and NamesAccepted conditions. func IsV1Beta1CRDReady(crd *apiextv1beta1.CustomResourceDefinition) bool { var isEstablished, namesAccepted bool for _, cond := range crd.Status.Conditions { if cond.Type == apiextv1beta1.Established && cond.Status == apiextv1beta1.ConditionTrue { isEstablished = true } if cond.Type == apiextv1beta1.NamesAccepted && cond.Status == apiextv1beta1.ConditionTrue { namesAccepted = true } } return (isEstablished && namesAccepted) } // IsCRDReady triggers IsV1Beta1CRDReady/IsV1CRDReady according to the version of the input param func IsCRDReady(crd *unstructured.Unstructured) (bool, error) { ver := crd.GroupVersionKind().Version switch ver { case "v1beta1": v1beta1crd := &apiextv1beta1.CustomResourceDefinition{} err := runtime.DefaultUnstructuredConverter.FromUnstructured(crd.Object, v1beta1crd) if err != nil { return false, err } return IsV1Beta1CRDReady(v1beta1crd), nil case "v1": v1crd := &apiextv1.CustomResourceDefinition{} err := runtime.DefaultUnstructuredConverter.FromUnstructured(crd.Object, v1crd) if err != nil { return false, err } return IsV1CRDReady(v1crd), nil default: return false, fmt.Errorf("unable to handle CRD with version %s", ver) } } // AddAnnotations adds the supplied key-values to the annotations on the object func AddAnnotations(o *metav1.ObjectMeta, vals map[string]string) { if o.Annotations == nil { o.Annotations = make(map[string]string) } for k, v := range vals { o.Annotations[k] = v } } // AddLabels adds the supplied key-values to the labels on the object func AddLabels(o *metav1.ObjectMeta, vals map[string]string) { if o.Labels == nil { o.Labels = make(map[string]string) } for k, v := range vals { o.Labels[k] = label.GetValidName(v) } } func HasBackupLabel(o *metav1.ObjectMeta, backupName string) bool { if o.Labels == nil || len(strings.TrimSpace(backupName)) == 0 { return false } return o.Labels[velerov1api.BackupNameLabel] == label.GetValidName(backupName) } func VerifyJSONConfigs(ctx context.Context, namespace string, crClient client.Client, configName string, configType any) error { cm := new(corev1api.ConfigMap) err := crClient.Get(ctx, client.ObjectKey{Namespace: namespace, Name: configName}, cm) if err != nil { return errors.Wrapf(err, "fail to find ConfigMap %s", configName) } if cm.Data == nil { return errors.Errorf("data is not available in ConfigMap %s", configName) } // Verify all the keys in ConfigMap's data. jsonString := "" for _, v := range cm.Data { jsonString = v configs := configType err = json.Unmarshal([]byte(jsonString), configs) if err != nil { return errors.Wrapf(err, "error to unmarshall data from ConfigMap %s", configName) } } return nil } ================================================ FILE: pkg/util/kube/utils_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package kube import ( "encoding/json" "testing" "time" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" storagev1api "k8s.io/api/storage/v1" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/kubernetes/fake" "github.com/vmware-tanzu/velero/pkg/builder" velerotest "github.com/vmware-tanzu/velero/pkg/test" "github.com/vmware-tanzu/velero/pkg/uploader" ) func TestNamespaceAndName(t *testing.T) { //TODO } func TestEnsureNamespaceExistsAndIsReady(t *testing.T) { tests := []struct { name string expectNSFound bool nsPhase corev1api.NamespacePhase nsDeleting bool expectCreate bool alreadyExists bool expectedResult bool expectedCreatedResult bool nsAlreadyInTerminationTracker bool ResourceDeletionStatusTracker ResourceDeletionStatusTracker }{ { name: "namespace found, not deleting", expectNSFound: true, expectedResult: true, expectedCreatedResult: false, }, { name: "namespace found, terminating phase", expectNSFound: true, nsPhase: corev1api.NamespaceTerminating, expectedResult: false, expectedCreatedResult: false, }, { name: "namespace found, deletiontimestamp set", expectNSFound: true, nsDeleting: true, expectedResult: false, expectedCreatedResult: false, }, { name: "namespace not found, successfully created", expectCreate: true, expectedResult: true, expectedCreatedResult: true, }, { name: "namespace not found initially, create returns already exists error, returned namespace is ready", alreadyExists: true, expectedResult: true, expectedCreatedResult: false, }, { name: "namespace not found initially, create returns already exists error, returned namespace is terminating", alreadyExists: true, nsPhase: corev1api.NamespaceTerminating, expectedResult: false, expectedCreatedResult: false, }, { name: "same namespace found earlier, terminating phase already tracked", expectNSFound: true, nsPhase: corev1api.NamespaceTerminating, expectedResult: false, expectedCreatedResult: false, nsAlreadyInTerminationTracker: true, }, } resourceDeletionStatusTracker := NewResourceDeletionStatusTracker() for _, test := range tests { t.Run(test.name, func(t *testing.T) { namespace := &corev1api.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: "test", }, } if test.nsPhase != "" { namespace.Status.Phase = test.nsPhase } if test.nsDeleting { namespace.SetDeletionTimestamp(&metav1.Time{Time: time.Now()}) } timeout := time.Millisecond nsClient := &velerotest.FakeNamespaceClient{} defer nsClient.AssertExpectations(t) if test.expectNSFound { nsClient.On("Get", "test", metav1.GetOptions{}).Return(namespace, nil) } else { nsClient.On("Get", "test", metav1.GetOptions{}).Return(&corev1api.Namespace{}, apierrors.NewNotFound(schema.GroupResource{Resource: "namespaces"}, "test")) } if test.alreadyExists { nsClient.On("Create", namespace).Return(namespace, apierrors.NewAlreadyExists(schema.GroupResource{Resource: "namespaces"}, "test")) } if test.expectCreate { nsClient.On("Create", namespace).Return(namespace, nil) } if test.nsAlreadyInTerminationTracker { resourceDeletionStatusTracker.Add(namespace.Kind, "test", "test") } result, nsCreated, _ := EnsureNamespaceExistsAndIsReady(namespace, nsClient, timeout, resourceDeletionStatusTracker) assert.Equal(t, test.expectedResult, result) assert.Equal(t, test.expectedCreatedResult, nsCreated) }) } } // TestGetVolumeDirectorySuccess tests that the GetVolumeDirectory function // returns a volume's name or a volume's name plus '/mount' when a PVC is present. func TestGetVolumeDirectorySuccess(t *testing.T) { tests := []struct { name string pod *corev1api.Pod pvc *corev1api.PersistentVolumeClaim pv *corev1api.PersistentVolume want string }{ { name: "Non-CSI volume with a PVC/PV returns the volume's name", pod: builder.ForPod("ns-1", "my-pod").Volumes(builder.ForVolume("my-vol").PersistentVolumeClaimSource("my-pvc").Result()).Result(), pvc: builder.ForPersistentVolumeClaim("ns-1", "my-pvc").VolumeName("a-pv").Result(), pv: builder.ForPersistentVolume("a-pv").Result(), want: "a-pv", }, { name: "CSI volume with a PVC/PV appends '/mount' to the volume name", pod: builder.ForPod("ns-1", "my-pod").Volumes(builder.ForVolume("my-vol").PersistentVolumeClaimSource("my-pvc").Result()).Result(), pvc: builder.ForPersistentVolumeClaim("ns-1", "my-pvc").VolumeName("a-pv").Result(), pv: builder.ForPersistentVolume("a-pv").CSI("csi.test.com", "provider-volume-id").Result(), want: "a-pv/mount", }, { name: "Block CSI volume with a PVC/PV does not append '/mount' to the volume name", pod: builder.ForPod("ns-1", "my-pod").Volumes(builder.ForVolume("my-vol").PersistentVolumeClaimSource("my-pvc").Result()).Result(), pvc: builder.ForPersistentVolumeClaim("ns-1", "my-pvc").VolumeName("a-pv").Result(), pv: builder.ForPersistentVolume("a-pv").CSI("csi.test.com", "provider-volume-id").VolumeMode(corev1api.PersistentVolumeBlock).Result(), want: "a-pv", }, { name: "CSI volume mounted without a PVC appends '/mount' to the volume name", pod: builder.ForPod("ns-1", "my-pod").Volumes(builder.ForVolume("my-vol").CSISource("csi.test.com").Result()).Result(), want: "my-vol/mount", }, { name: "Non-CSI volume without a PVC returns the volume name", pod: builder.ForPod("ns-1", "my-pod").Volumes(builder.ForVolume("my-vol").Result()).Result(), want: "my-vol", }, { name: "Volume with CSI annotation appends '/mount' to the volume name", pod: builder.ForPod("ns-1", "my-pod").Volumes(builder.ForVolume("my-vol").PersistentVolumeClaimSource("my-pvc").Result()).Result(), pvc: builder.ForPersistentVolumeClaim("ns-1", "my-pvc").VolumeName("a-pv").Result(), pv: builder.ForPersistentVolume("a-pv").ObjectMeta(builder.WithAnnotations(KubeAnnDynamicallyProvisioned, "csi.test.com")).Result(), want: "a-pv/mount", }, { name: "Volume with CSI annotation 'pv.kubernetes.io/migrated-to' appends '/mount' to the volume name", pod: builder.ForPod("ns-1", "my-pod").Volumes(builder.ForVolume("my-vol").PersistentVolumeClaimSource("my-pvc").Result()).Result(), pvc: builder.ForPersistentVolumeClaim("ns-1", "my-pvc").VolumeName("a-pv").Result(), pv: builder.ForPersistentVolume("a-pv").ObjectMeta(builder.WithAnnotations(KubeAnnMigratedTo, "csi.test.com")).Result(), want: "a-pv/mount", }, } csiDriver := storagev1api.CSIDriver{ ObjectMeta: metav1.ObjectMeta{Name: "csi.test.com"}, } for _, tc := range tests { objs := []runtime.Object{&csiDriver} if tc.pvc != nil { objs = append(objs, tc.pvc) } if tc.pv != nil { objs = append(objs, tc.pv) } fakeKubeClient := fake.NewSimpleClientset(objs...) // Function under test dir, err := GetVolumeDirectory(t.Context(), logrus.StandardLogger(), tc.pod, tc.pod.Spec.Volumes[0].Name, fakeKubeClient) require.NoError(t, err) assert.Equal(t, tc.want, dir) } } // TestGetVolumeModeSuccess tests the GetVolumeMode function func TestGetVolumeModeSuccess(t *testing.T) { tests := []struct { name string pod *corev1api.Pod pvc *corev1api.PersistentVolumeClaim pv *corev1api.PersistentVolume want uploader.PersistentVolumeMode }{ { name: "Filesystem PVC volume", pod: builder.ForPod("ns-1", "my-pod").Volumes(builder.ForVolume("my-vol").PersistentVolumeClaimSource("my-pvc").Result()).Result(), pvc: builder.ForPersistentVolumeClaim("ns-1", "my-pvc").VolumeName("a-pv").Result(), pv: builder.ForPersistentVolume("a-pv").VolumeMode(corev1api.PersistentVolumeFilesystem).Result(), want: uploader.PersistentVolumeFilesystem, }, { name: "Block PVC volume", pod: builder.ForPod("ns-1", "my-pod").Volumes(builder.ForVolume("my-vol").PersistentVolumeClaimSource("my-pvc").Result()).Result(), pvc: builder.ForPersistentVolumeClaim("ns-1", "my-pvc").VolumeName("a-pv").Result(), pv: builder.ForPersistentVolume("a-pv").VolumeMode(corev1api.PersistentVolumeBlock).Result(), want: uploader.PersistentVolumeBlock, }, { name: "Pod volume without a PVC", pod: builder.ForPod("ns-1", "my-pod").Volumes(builder.ForVolume("my-vol").Result()).Result(), want: uploader.PersistentVolumeFilesystem, }, } for _, tc := range tests { objs := []runtime.Object{} if tc.pvc != nil { objs = append(objs, tc.pvc) } if tc.pv != nil { objs = append(objs, tc.pv) } fakeKubeClient := fake.NewSimpleClientset(objs...) // Function under test mode, err := GetVolumeMode(t.Context(), logrus.StandardLogger(), tc.pod, tc.pod.Spec.Volumes[0].Name, fakeKubeClient) require.NoError(t, err) assert.Equal(t, tc.want, mode) } } func TestIsV1Beta1CRDReady(t *testing.T) { tests := []struct { name string crd *apiextv1beta1.CustomResourceDefinition want bool }{ { name: "CRD is not established & not accepting names - not ready", crd: builder.ForCustomResourceDefinitionV1Beta1("MyCRD").Result(), want: false, }, { name: "CRD is established & not accepting names - not ready", crd: builder.ForCustomResourceDefinitionV1Beta1("MyCRD"). Condition(builder.ForCustomResourceDefinitionV1Beta1Condition().Type(apiextv1beta1.Established).Status(apiextv1beta1.ConditionTrue).Result()).Result(), want: false, }, { name: "CRD is not established & accepting names - not ready", crd: builder.ForCustomResourceDefinitionV1Beta1("MyCRD"). Condition(builder.ForCustomResourceDefinitionV1Beta1Condition().Type(apiextv1beta1.NamesAccepted).Status(apiextv1beta1.ConditionTrue).Result()).Result(), want: false, }, { name: "CRD is established & accepting names - ready", crd: builder.ForCustomResourceDefinitionV1Beta1("MyCRD"). Condition(builder.ForCustomResourceDefinitionV1Beta1Condition().Type(apiextv1beta1.Established).Status(apiextv1beta1.ConditionTrue).Result()). Condition(builder.ForCustomResourceDefinitionV1Beta1Condition().Type(apiextv1beta1.NamesAccepted).Status(apiextv1beta1.ConditionTrue).Result()). Result(), want: true, }, } for _, tc := range tests { result := IsV1Beta1CRDReady(tc.crd) assert.Equal(t, tc.want, result) } } func TestIsV1CRDReady(t *testing.T) { tests := []struct { name string crd *apiextv1.CustomResourceDefinition want bool }{ { name: "CRD is not established & not accepting names - not ready", crd: builder.ForV1CustomResourceDefinition("MyCRD").Result(), want: false, }, { name: "CRD is established & not accepting names - not ready", crd: builder.ForV1CustomResourceDefinition("MyCRD"). Condition(builder.ForV1CustomResourceDefinitionCondition().Type(apiextv1.Established).Status(apiextv1.ConditionTrue).Result()).Result(), want: false, }, { name: "CRD is not established & accepting names - not ready", crd: builder.ForV1CustomResourceDefinition("MyCRD"). Condition(builder.ForV1CustomResourceDefinitionCondition().Type(apiextv1.NamesAccepted).Status(apiextv1.ConditionTrue).Result()).Result(), want: false, }, { name: "CRD is established & accepting names - ready", crd: builder.ForV1CustomResourceDefinition("MyCRD"). Condition(builder.ForV1CustomResourceDefinitionCondition().Type(apiextv1.Established).Status(apiextv1.ConditionTrue).Result()). Condition(builder.ForV1CustomResourceDefinitionCondition().Type(apiextv1.NamesAccepted).Status(apiextv1.ConditionTrue).Result()). Result(), want: true, }, } for _, tc := range tests { result := IsV1CRDReady(tc.crd) assert.Equal(t, tc.want, result) } } func TestIsCRDReady(t *testing.T) { v1beta1tests := []struct { name string crd *apiextv1beta1.CustomResourceDefinition want bool }{ { name: "v1beta1CRD is not established & not accepting names - not ready", crd: builder.ForCustomResourceDefinitionV1Beta1("MyCRD").Result(), want: false, }, { name: "v1beta1CRD is established & not accepting names - not ready", crd: builder.ForCustomResourceDefinitionV1Beta1("MyCRD"). Condition(builder.ForCustomResourceDefinitionV1Beta1Condition().Type(apiextv1beta1.Established).Status(apiextv1beta1.ConditionTrue).Result()).Result(), want: false, }, { name: "v1beta1CRD is not established & accepting names - not ready", crd: builder.ForCustomResourceDefinitionV1Beta1("MyCRD"). Condition(builder.ForCustomResourceDefinitionV1Beta1Condition().Type(apiextv1beta1.NamesAccepted).Status(apiextv1beta1.ConditionTrue).Result()).Result(), want: false, }, { name: "v1beta1CRD is established & accepting names - ready", crd: builder.ForCustomResourceDefinitionV1Beta1("MyCRD"). Condition(builder.ForCustomResourceDefinitionV1Beta1Condition().Type(apiextv1beta1.Established).Status(apiextv1beta1.ConditionTrue).Result()). Condition(builder.ForCustomResourceDefinitionV1Beta1Condition().Type(apiextv1beta1.NamesAccepted).Status(apiextv1beta1.ConditionTrue).Result()). Result(), want: true, }, } for _, tc := range v1beta1tests { m, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.crd) require.NoError(t, err) result, err := IsCRDReady(&unstructured.Unstructured{Object: m}) require.NoError(t, err) assert.Equal(t, tc.want, result) } v1tests := []struct { name string crd *apiextv1.CustomResourceDefinition want bool }{ { name: "v1CRD is not established & not accepting names - not ready", crd: builder.ForV1CustomResourceDefinition("MyCRD").Result(), want: false, }, { name: "v1CRD is established & not accepting names - not ready", crd: builder.ForV1CustomResourceDefinition("MyCRD"). Condition(builder.ForV1CustomResourceDefinitionCondition().Type(apiextv1.Established).Status(apiextv1.ConditionTrue).Result()).Result(), want: false, }, { name: "v1CRD is not established & accepting names - not ready", crd: builder.ForV1CustomResourceDefinition("MyCRD"). Condition(builder.ForV1CustomResourceDefinitionCondition().Type(apiextv1.NamesAccepted).Status(apiextv1.ConditionTrue).Result()).Result(), want: false, }, { name: "v1CRD is established & accepting names - ready", crd: builder.ForV1CustomResourceDefinition("MyCRD"). Condition(builder.ForV1CustomResourceDefinitionCondition().Type(apiextv1.Established).Status(apiextv1.ConditionTrue).Result()). Condition(builder.ForV1CustomResourceDefinitionCondition().Type(apiextv1.NamesAccepted).Status(apiextv1.ConditionTrue).Result()). Result(), want: true, }, } for _, tc := range v1tests { m, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.crd) require.NoError(t, err) result, err := IsCRDReady(&unstructured.Unstructured{Object: m}) require.NoError(t, err) assert.Equal(t, tc.want, result) } // input param is unrecognized resBytes := []byte(` { "apiVersion": "apiextensions.k8s.io/v9", "kind": "CustomResourceDefinition", "metadata": { "name": "foos.example.foo.com" }, "spec": { "group": "example.foo.com", "version": "v1alpha1", "scope": "Namespaced", "names": { "plural": "foos", "singular": "foo", "kind": "Foo" }, "validation": { "openAPIV3Schema": { "required": [ "spec" ], "properties": { "spec": { "required": [ "bar" ], "properties": { "bar": { "type": "integer", "minimum": 1 } } } } } } } } `) obj := &unstructured.Unstructured{} err := json.Unmarshal(resBytes, obj) require.NoError(t, err) _, err = IsCRDReady(obj) assert.Error(t, err) } func TestSinglePathMatch(t *testing.T) { fakeFS := velerotest.NewFakeFileSystem() fakeFS.MkdirAll("testDir1/subpath", 0755) fakeFS.MkdirAll("testDir2/subpath", 0755) _, err := SinglePathMatch("./*/subpath", fakeFS, logrus.StandardLogger()) require.ErrorContains(t, err, "expected one matching path") } func TestAddAnnotations(t *testing.T) { annotationValues := map[string]string{ "k1": "v1", "k2": "v2", "k3": "v3", "k4": "v4", "k5": "v5", } testCases := []struct { name string o metav1.ObjectMeta toAdd map[string]string }{ { name: "should create a new annotation map when annotation is nil", o: metav1.ObjectMeta{ Annotations: nil, }, toAdd: annotationValues, }, { name: "should add all supplied annotations into empty annotation", o: metav1.ObjectMeta{ Annotations: map[string]string{}, }, toAdd: annotationValues, }, { name: "should add all supplied annotations to existing annotation", o: metav1.ObjectMeta{ Annotations: map[string]string{ "k100": "v100", "k200": "v200", "k300": "v300", }, }, toAdd: annotationValues, }, { name: "should overwrite some existing annotations", o: metav1.ObjectMeta{ Annotations: map[string]string{ "k100": "v100", "k2": "v200", "k300": "v300", }, }, toAdd: annotationValues, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { AddAnnotations(&tc.o, tc.toAdd) for k, v := range tc.toAdd { actual, exists := tc.o.Annotations[k] assert.True(t, exists) assert.Equal(t, v, actual) } }) } } func TestAddLabels(t *testing.T) { labelValues := map[string]string{ "l1": "v1", "l2": "v2", "l3": "v3", "l4": "v4", "l5": "v5", } testCases := []struct { name string o metav1.ObjectMeta toAdd map[string]string }{ { name: "should create a new labels map when labels is nil", o: metav1.ObjectMeta{ Labels: nil, }, toAdd: labelValues, }, { name: "should add all supplied labels into empty labels", o: metav1.ObjectMeta{ Labels: map[string]string{}, }, toAdd: labelValues, }, { name: "should add all supplied labels to existing labels", o: metav1.ObjectMeta{ Labels: map[string]string{ "l100": "v100", "l200": "v200", "l300": "v300", }, }, toAdd: labelValues, }, { name: "should overwrite some existing labels", o: metav1.ObjectMeta{ Labels: map[string]string{ "l100": "v100", "l2": "v200", "l300": "v300", }, }, toAdd: labelValues, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { AddLabels(&tc.o, tc.toAdd) for k, v := range tc.toAdd { actual, exists := tc.o.Labels[k] assert.True(t, exists) assert.Equal(t, v, actual) } }) } } func TestHasBackupLabel(t *testing.T) { testCases := []struct { name string o metav1.ObjectMeta backupName string expected bool }{ { name: "object has no labels", o: metav1.ObjectMeta{}, expected: false, }, { name: "object has no velero backup label", backupName: "csi-b1", o: metav1.ObjectMeta{ Labels: map[string]string{ "l100": "v100", "l2": "v200", "l300": "v300", }, }, expected: false, }, { name: "object has velero backup label but value not equal to backup name", backupName: "csi-b1", o: metav1.ObjectMeta{ Labels: map[string]string{ "velero.io/backup-name": "does-not-match", "l100": "v100", "l2": "v200", "l300": "v300", }, }, expected: false, }, { name: "object has backup label with matching backup name value", backupName: "does-match", o: metav1.ObjectMeta{ Labels: map[string]string{ "velero.io/backup-name": "does-match", "l100": "v100", "l2": "v200", "l300": "v300", }, }, expected: true, }, } for _, tc := range testCases { actual := HasBackupLabel(&tc.o, tc.backupName) assert.Equal(t, tc.expected, actual) } } func TestVerifyJsonConfigs(t *testing.T) { testCases := []struct { name string configMapName string configMap *corev1api.ConfigMap configType any expectedErr string }{ { name: "ConfigMap not exist", configMapName: "non-exist", expectedErr: "fail to find ConfigMap non-exist: configmaps \"non-exist\" not found", }, { name: "ConfigMap doesn't have data", configMapName: "no-data", expectedErr: "data is not available in ConfigMap no-data", configMap: builder.ForConfigMap("velero", "no-data").Result(), }, { name: "ConfigMap data is invalid", configMapName: "invalid", expectedErr: "error to unmarshall data from ConfigMap invalid: unexpected end of JSON input", configMap: builder.ForConfigMap("velero", "invalid").Data("global", "{\"podResources\": {\"cpuRequest\": \"100m\", \"cpuLimit\": \"200m\", \"memoryRequest\": \"100Mi\", \"memoryLimit\": \"200Mi\"}, \"keepLatestMaintenanceJobs\": 1}", "other", "{\"podResources\": {\"cpuRequest\": \"100m\", \"cpuLimit\": \"200m\", \"memoryRequest\": \"100Mi\", \"memoryLimit\": \"200Mi\"}, \"keepLatestMaintenanceJobs: 1}").Result(), }, { name: "Normal case", configMapName: "normal", configMap: builder.ForConfigMap("velero", "normal").Data("global", "{\"podResources\": {\"cpuRequest\": \"100m\", \"cpuLimit\": \"200m\", \"memoryRequest\": \"100Mi\", \"memoryLimit\": \"200Mi\"}, \"keepLatestMaintenanceJobs\": 1}", "other", "{\"podResources\": {\"cpuRequest\": \"100m\", \"cpuLimit\": \"200m\", \"memoryRequest\": \"100Mi\", \"memoryLimit\": \"200Mi\"}, \"keepLatestMaintenanceJobs\": 1}").Result(), configType: make(map[string]any), }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { objects := make([]runtime.Object, 0) if tc.configMap != nil { objects = append(objects, tc.configMap) } fakeClient := velerotest.NewFakeControllerRuntimeClient(t, objects...) err := VerifyJSONConfigs(t.Context(), "velero", fakeClient, tc.configMapName, tc.configMap) if len(tc.expectedErr) > 0 { require.EqualError(t, err, tc.expectedErr) } else { require.NoError(t, err) } }) } } ================================================ FILE: pkg/util/logging/default_logger.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package logging import ( "os" "github.com/sirupsen/logrus" ) // DefaultHooks returns a slice of the default // logrus hooks to be used by a logger. func DefaultHooks(merge bool) []logrus.Hook { hooks := []logrus.Hook{ &LogLocationHook{}, &ErrorLocationHook{}, } if merge { hooks = append(hooks, &MergeHook{}) } return hooks } // DefaultLogger returns a Logger with the default properties // and hooks. The desired output format is passed as a LogFormat Enum. func DefaultLogger(level logrus.Level, format Format) *logrus.Logger { return createLogger(level, format, false) } // DefaultLogger returns a Logger with the default properties // and hooks, and also a hook to support log merge. // The desired output format is passed as a LogFormat Enum. func DefaultMergeLogger(level logrus.Level, format Format) *logrus.Logger { return createLogger(level, format, true) } func createLogger(level logrus.Level, format Format, merge bool) *logrus.Logger { logger := logrus.New() if format == FormatJSON { logger.Formatter = new(logrus.JSONFormatter) // Error hooks inject nested fields under "error.*" with the error // string message at "error". // // This structure is incompatible with recent Elasticsearch versions // where dots in field names are automatically expanded to objects; // field "error" cannot be both a string and an object at the same // time. // // ELK being a popular choice for log ingestion and a common reason // for enabling JSON logging in the first place, we avoid this mapping // problem by nesting the error's message at "error.message". // // This also follows the Elastic Common Schema (ECS) recommendation. // https://www.elastic.co/guide/en/ecs/current/ecs-error.html logrus.ErrorKey = "error.message" } else { logrus.ErrorKey = "error" } // Make sure the output is set to stdout so log messages don't show up as errors in cloud log dashboards. logger.Out = os.Stdout logger.Level = level for _, hook := range DefaultHooks(merge) { logger.Hooks.Add(hook) } return logger } ================================================ FILE: pkg/util/logging/default_logger_test.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package logging import ( "os" "testing" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" ) func TestDefaultLogger(t *testing.T) { formatFlag := NewFormatFlag() for _, testFormat := range formatFlag.AllowedValues() { formatFlag.Set(testFormat) logger := DefaultLogger(logrus.InfoLevel, formatFlag.Parse()) assert.Equal(t, logrus.InfoLevel, logger.Level) assert.Equal(t, os.Stdout, logger.Out) for _, level := range logrus.AllLevels { assert.Equal(t, DefaultHooks(false), logger.Hooks[level]) } } } func TestDefaultMergeLogger(t *testing.T) { formatFlag := NewFormatFlag() for _, testFormat := range formatFlag.AllowedValues() { formatFlag.Set(testFormat) logger := DefaultMergeLogger(logrus.InfoLevel, formatFlag.Parse()) assert.Equal(t, logrus.InfoLevel, logger.Level) assert.Equal(t, os.Stdout, logger.Out) assert.Equal(t, DefaultHooks(true), logger.Hooks[ListeningLevel]) } } ================================================ FILE: pkg/util/logging/dual_mode_logger.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package logging import ( "compress/gzip" "io" "os" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) // DualModeLogger is a thread safe logger interface to write logs to dual targets, one of which // is a persist file, so that the log could be further transferred. type DualModeLogger interface { logrus.FieldLogger // DoneForPersist stops outputting logs to the persist file DoneForPersist(log logrus.FieldLogger) // GetPersistFile moves the persist file pointer to beginning and returns it GetPersistFile() (*os.File, error) // Dispose closes the temp file pointer and removes the file Dispose(log logrus.FieldLogger) } type tempFileLogger struct { logrus.FieldLogger logger *logrus.Logger file *os.File w *gzip.Writer } // NewTempFileLogger creates a DualModeLogger instance that writes logs to both Stdout and a file in the temp folder. func NewTempFileLogger(logLevel logrus.Level, logFormat Format, hook *LogHook, fields logrus.Fields) (DualModeLogger, error) { file, err := os.CreateTemp("", "") if err != nil { return nil, errors.Wrap(err, "error creating temp file") } w := gzip.NewWriter(file) logger := DefaultLogger(logLevel, logFormat) logger.Out = io.MultiWriter(os.Stdout, w) if hook != nil { logger.Hooks.Add(hook) } return &tempFileLogger{ FieldLogger: logger.WithFields(fields), logger: logger, file: file, w: w, }, nil } func (p *tempFileLogger) DoneForPersist(log logrus.FieldLogger) { p.logger.SetOutput(os.Stdout) if err := p.w.Close(); err != nil { log.WithError(err).Warn("error closing gzip writer") } } func (p *tempFileLogger) GetPersistFile() (*os.File, error) { if _, err := p.file.Seek(0, 0); err != nil { return nil, errors.Wrap(err, "error resetting log file offset to 0") } return p.file, nil } func (p *tempFileLogger) Dispose(log logrus.FieldLogger) { p.w.Close() closeAndRemoveFile(p.file, log) } func closeAndRemoveFile(file *os.File, log logrus.FieldLogger) { if file == nil { log.Debug("Skipping removal of temp log file due to nil file pointer") return } if err := file.Close(); err != nil { log.WithError(err).WithField("file", file.Name()).Warn("error closing temp log file") } if err := os.Remove(file.Name()); err != nil { log.WithError(err).WithField("file", file.Name()).Warn("error removing temp log file") } } ================================================ FILE: pkg/util/logging/dual_mode_logger_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package logging import ( "compress/gzip" "io" "os" "testing" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func TestDualModeLogger(t *testing.T) { logMsgExpect := "Expected message in log" logMsgUnexpect := "Unexpected message in log" logger, err := NewTempFileLogger(logrus.DebugLevel, FormatText, nil, logrus.Fields{}) require.NoError(t, err) logger.Info(logMsgExpect) logger.DoneForPersist(velerotest.NewLogger()) logger.Info(logMsgUnexpect) logFile, err := logger.GetPersistFile() require.NoError(t, err) logStr, err := readLogString(logFile) require.NoError(t, err) assert.Contains(t, logStr, logMsgExpect) assert.NotContains(t, logStr, logMsgUnexpect) logger.Dispose(velerotest.NewLogger()) _, err = os.Stat(logFile.Name()) assert.True(t, os.IsNotExist(err)) } func readLogString(file *os.File) (string, error) { gzr, err := gzip.NewReader(file) if err != nil { return "", err } buffer := make([]byte, 1024) _, err = gzr.Read(buffer) if err != io.EOF { return "", err } return string(buffer[:]), nil } ================================================ FILE: pkg/util/logging/error_location_hook.go ================================================ /* Copyright 2017, 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package logging import ( "fmt" "strconv" "strings" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) const ( errorFileField = "error.file" errorFunctionField = "error.function" ) // ErrorLocationHook is a logrus hook that attaches error location information // to log entries if an error is being logged and it has stack-trace information // (i.e. if it originates from or is wrapped by github.com/pkg/errors, or if it // implements the errorLocationer interface, like errors returned from plugins // typically do). type ErrorLocationHook struct{} func (h *ErrorLocationHook) Levels() []logrus.Level { return logrus.AllLevels } func (h *ErrorLocationHook) Fire(entry *logrus.Entry) error { errObj, ok := entry.Data[logrus.ErrorKey] if !ok { return nil } if _, ok := entry.Data[errorFileField]; ok { // If there is already an error file field, preserve it instead of overwriting it. This field will already exist if // the log message occurred in the server half of a plugin. return nil } if _, ok := entry.Data[errorFunctionField]; ok { // If there is already an error function field, preserve it instead of overwriting it. This field will already exist if // the log message occurred in the server half of a plugin. return nil } err, ok := errObj.(error) if !ok { // if the value isn't an error type, skip trying to get location info, // and just let it be logged as whatever it was return nil } if errorLocationer, ok := err.(errorLocationer); ok { entry.Data[errorFileField] = fmt.Sprintf("%s:%d", errorLocationer.File(), errorLocationer.Line()) entry.Data[errorFunctionField] = errorLocationer.Function() return nil } if stackErr := getInnermostTrace(err); stackErr != nil { location := GetFrameLocationInfo(stackErr.StackTrace()[0]) entry.Data[errorFileField] = fmt.Sprintf("%s:%d", location.File, location.Line) entry.Data[errorFunctionField] = location.Function } return nil } // LocationInfo specifies the location of a line // of code. type LocationInfo struct { File string Function string Line int } // GetFrameLocationInfo returns the location of a frame. func GetFrameLocationInfo(frame errors.Frame) LocationInfo { // see https://godoc.org/github.com/pkg/errors#Frame.Format for // details on formatting verbs functionNameAndFileAndLine := fmt.Sprintf("%+v", frame) newLineIndex := strings.Index(functionNameAndFileAndLine, "\n") functionName := functionNameAndFileAndLine[0:newLineIndex] tabIndex := strings.LastIndex(functionNameAndFileAndLine, "\t") fileAndLine := strings.Split(functionNameAndFileAndLine[tabIndex+1:], ":") line, err := strconv.Atoi(fileAndLine[1]) if err != nil { line = -1 } return LocationInfo{ File: fileAndLine[0], Function: functionName, Line: line, } } type errorLocationer interface { File() string Line() int32 Function() string } type stackTracer interface { error StackTrace() errors.StackTrace } type causer interface { Cause() error } // getInnermostTrace returns the innermost error that // has a stack trace attached func getInnermostTrace(err error) stackTracer { var tracer stackTracer for { t, isTracer := err.(stackTracer) if isTracer { tracer = t } c, isCauser := err.(causer) if isCauser { err = c.Cause() } else { return tracer } } } ================================================ FILE: pkg/util/logging/error_location_hook_test.go ================================================ /* Copyright 2017, 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package logging import ( "errors" "testing" pkgerrs "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestFire(t *testing.T) { tests := []struct { name string preEntryFields map[string]any expectedEntryFields map[string]any expectedErr bool }{ { name: "no error", preEntryFields: map[string]any{"foo": "bar"}, expectedEntryFields: map[string]any{"foo": "bar"}, }, { name: "basic (non-pkg/errors) error", preEntryFields: map[string]any{logrus.ErrorKey: errors.New("a normal error")}, expectedEntryFields: map[string]any{logrus.ErrorKey: errors.New("a normal error")}, }, { name: "non-error logged in error field", preEntryFields: map[string]any{logrus.ErrorKey: "not an error"}, expectedEntryFields: map[string]any{logrus.ErrorKey: "not an error"}, expectedErr: false, }, { name: "pkg/errors error", preEntryFields: map[string]any{logrus.ErrorKey: pkgerrs.New("a pkg/errors error")}, expectedEntryFields: map[string]any{ logrus.ErrorKey: pkgerrs.New("a pkg/errors error"), errorFileField: "", errorFunctionField: "github.com/vmware-tanzu/velero/pkg/util/logging.TestFire", }, }, { name: "already have error file and function fields", preEntryFields: map[string]any{ logrus.ErrorKey: pkgerrs.New("a pkg/errors error"), errorFileField: "some_file.go:123", errorFunctionField: "SomeFunction", }, expectedEntryFields: map[string]any{ logrus.ErrorKey: pkgerrs.New("a pkg/errors error"), errorFileField: "some_file.go:123", errorFunctionField: "SomeFunction", }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { hook := &ErrorLocationHook{} entry := &logrus.Entry{ Data: logrus.Fields(test.preEntryFields), } // method under test err := hook.Fire(entry) require.Equal(t, test.expectedErr, err != nil) require.Len(t, entry.Data, len(test.expectedEntryFields)) for key, expectedValue := range test.expectedEntryFields { actualValue, found := entry.Data[key] assert.True(t, found, "expected key not found: %s", key) switch key { // test existence of this field only since testing the value // is fragile case errorFileField: case logrus.ErrorKey: if err, ok := expectedValue.(error); ok { assert.Equal(t, err.Error(), actualValue.(error).Error()) } else { assert.Equal(t, expectedValue, actualValue) } default: assert.Equal(t, expectedValue, actualValue) } } }) } } func TestGetInnermostTrace(t *testing.T) { newError := func() error { return errors.New("a normal error") } tests := []struct { name string err error expectedRes error }{ { name: "normal error", err: newError(), expectedRes: nil, }, { name: "pkg/errs error", err: pkgerrs.New("a pkg/errs error"), expectedRes: pkgerrs.New("a pkg/errs error"), }, { name: "one level of stack-ing a normal error", err: pkgerrs.WithStack(newError()), expectedRes: pkgerrs.WithStack(newError()), }, { name: "two levels of stack-ing a normal error", err: pkgerrs.WithStack(pkgerrs.WithStack(newError())), expectedRes: pkgerrs.WithStack(newError()), }, { name: "one level of stack-ing a pkg/errors error", err: pkgerrs.WithStack(pkgerrs.New("a pkg/errs error")), expectedRes: pkgerrs.New("a pkg/errs error"), }, { name: "two levels of stack-ing a pkg/errors error", err: pkgerrs.WithStack(pkgerrs.WithStack(pkgerrs.New("a pkg/errs error"))), expectedRes: pkgerrs.New("a pkg/errs error"), }, { name: "two levels of wrapping a normal error", err: pkgerrs.Wrap(pkgerrs.Wrap(newError(), "wrap 1"), "wrap 2"), expectedRes: pkgerrs.Wrap(newError(), "wrap 1"), }, { name: "two levels of wrapping a pkg/errors error", err: pkgerrs.Wrap(pkgerrs.Wrap(pkgerrs.New("a pkg/errs error"), "wrap 1"), "wrap 2"), expectedRes: pkgerrs.New("a pkg/errs error"), }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { res := getInnermostTrace(test.err) if test.expectedRes == nil { require.NoError(t, res) return } assert.Equal(t, test.expectedRes.Error(), res.Error()) }) } } ================================================ FILE: pkg/util/logging/format_flag.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package logging import "github.com/vmware-tanzu/velero/pkg/cmd/util/flag" // Format is a string representation of the desired output format for logs type Format string const ( FormatText Format = "text" FormatJSON Format = "json" defaultValue Format = FormatText ) // FormatFlag is a command-line flag for setting the logrus // log format. type FormatFlag struct { *flag.Enum defaultValue Format } // NewFormatFlag constructs a new log level flag. func NewFormatFlag() *FormatFlag { return &FormatFlag{ Enum: flag.NewEnum( string(defaultValue), string(FormatText), string(FormatJSON), ), defaultValue: defaultValue, } } // Parse returns the flag's value as a Format. func (f *FormatFlag) Parse() Format { return Format(f.String()) } ================================================ FILE: pkg/util/logging/hclog_level_hook.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package logging import ( "github.com/sirupsen/logrus" ) // HcLogLevelHook adds an hclog-compatible field ("@level") containing // the log level. Note that if you use this, you SHOULD NOT use // logrus.JSONFormatter's FieldMap to set the level key to "@level" because // that will result in the hclog-compatible info written here being // overwritten. type HcLogLevelHook struct{} func (h *HcLogLevelHook) Levels() []logrus.Level { return logrus.AllLevels } func (h *HcLogLevelHook) Fire(entry *logrus.Entry) error { switch entry.Level { // logrus uses "warning" to represent WarnLevel, // which is not compatible with hclog's "warn". case logrus.WarnLevel: entry.Data["@level"] = "warn" default: entry.Data["@level"] = entry.Level.String() } return nil } ================================================ FILE: pkg/util/logging/log_counter_hook.go ================================================ /* Copyright 2019 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package logging import ( "errors" "fmt" "sync" "github.com/sirupsen/logrus" "github.com/vmware-tanzu/velero/pkg/util/results" ) // LogHook is a logrus hook that counts the number of log // statements that have been written at each logrus level. It also // maintains log entries at each logrus level in result structure. type LogHook struct { mu sync.RWMutex counts map[logrus.Level]int entries map[logrus.Level]*results.Result } // NewLogHook returns a pointer to an initialized LogHook. func NewLogHook() *LogHook { return &LogHook{ counts: make(map[logrus.Level]int), entries: make(map[logrus.Level]*results.Result), } } // Levels returns the logrus levels that the hook should be fired for. func (h *LogHook) Levels() []logrus.Level { return logrus.AllLevels } // Fire executes the hook's logic. func (h *LogHook) Fire(entry *logrus.Entry) error { h.mu.Lock() defer h.mu.Unlock() h.counts[entry.Level]++ if h.entries[entry.Level] == nil { h.entries[entry.Level] = &results.Result{} } namespace, isNamespacePresent := entry.Data["namespace"] errorField, isErrorFieldPresent := entry.Data["error"] // When JSON logging format is enabled, error message is placed at "error.message" instead of "error" errorMsgField, isErrorMsgFieldPresent := entry.Data["error.message"] resourceField, isResourceFieldPresent := entry.Data["resource"] nameField, isNameFieldPresent := entry.Data["name"] msgField, isMsgFieldPresent := entry.Message, true entryMessage := "" if isResourceFieldPresent { entryMessage = fmt.Sprintf("%s resource: /%s", entryMessage, resourceField.(string)) } if isNameFieldPresent { entryMessage = fmt.Sprintf("%s name: /%s", entryMessage, nameField.(string)) } if isMsgFieldPresent { entryMessage = fmt.Sprintf("%s message: /%v", entryMessage, msgField) } if isErrorFieldPresent { entryMessage = fmt.Sprintf("%s error: /%v", entryMessage, errorField) } if isErrorMsgFieldPresent { entryMessage = fmt.Sprintf("%s error: /%v", entryMessage, errorMsgField) } if isNamespacePresent { h.entries[entry.Level].Add(namespace.(string), errors.New(entryMessage)) } else { h.entries[entry.Level].AddVeleroError(errors.New(entryMessage)) } return nil } // GetCount returns the number of log statements that have been // written at the specific level provided. func (h *LogHook) GetCount(level logrus.Level) int { h.mu.RLock() defer h.mu.RUnlock() return h.counts[level] } // GetEntries returns the log statements that have been // written at the specific level provided. func (h *LogHook) GetEntries(level logrus.Level) results.Result { h.mu.RLock() defer h.mu.RUnlock() response, isPresent := h.entries[level] if isPresent { return *response } return results.Result{} } ================================================ FILE: pkg/util/logging/log_counter_hook_test.go ================================================ package logging import ( "errors" "fmt" "testing" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/vmware-tanzu/velero/pkg/util/results" ) func TestLogHook_Fire(t *testing.T) { hook := NewLogHook() entry := &logrus.Entry{ Level: logrus.ErrorLevel, Data: logrus.Fields{ "namespace": "test-namespace", "error": errors.New("test-error"), "resource": "test-resource", "name": "test-name", }, Message: "test-message", } err := hook.Fire(entry) require.NoError(t, err) // Verify the counts assert.Equal(t, 1, hook.counts[logrus.ErrorLevel]) // Verify the entries expectedResult := &results.Result{} expectedResult.Add("test-namespace", fmt.Errorf(" resource: /test-resource name: /test-name message: /test-message error: /%v", entry.Data["error"])) assert.Equal(t, expectedResult, hook.entries[logrus.ErrorLevel]) entry1 := &logrus.Entry{ Level: logrus.ErrorLevel, Data: logrus.Fields{ "error.message": errors.New("test-error"), "resource": "test-resource", "name": "test-name", }, Message: "test-message", } err = hook.Fire(entry1) require.NoError(t, err) // Verify the counts assert.Equal(t, 2, hook.counts[logrus.ErrorLevel]) // Verify the entries expectedResult = &results.Result{} expectedResult.Add("test-namespace", fmt.Errorf(" resource: /test-resource name: /test-name message: /test-message error: /%v", entry.Data["error"])) expectedResult.AddVeleroError(fmt.Errorf(" resource: /test-resource name: /test-name message: /test-message error: /%v", entry1.Data["error.message"])) assert.Equal(t, expectedResult, hook.entries[logrus.ErrorLevel]) } func TestLogHook_Levels(t *testing.T) { hook := NewLogHook() levels := hook.Levels() expectedLevels := []logrus.Level{ logrus.PanicLevel, logrus.FatalLevel, logrus.ErrorLevel, logrus.WarnLevel, logrus.InfoLevel, logrus.DebugLevel, logrus.TraceLevel, } assert.Equal(t, expectedLevels, levels) } func TestLogHook_GetCount(t *testing.T) { hook := NewLogHook() // Set up test data hook.counts[logrus.ErrorLevel] = 5 hook.counts[logrus.WarnLevel] = 10 // Test GetCount for ErrorLevel count := hook.GetCount(logrus.ErrorLevel) assert.Equal(t, 5, count) // Test GetCount for WarnLevel count = hook.GetCount(logrus.WarnLevel) assert.Equal(t, 10, count) // Test GetCount for other levels count = hook.GetCount(logrus.InfoLevel) assert.Equal(t, 0, count) count = hook.GetCount(logrus.DebugLevel) assert.Equal(t, 0, count) } func TestLogHook_GetEntries(t *testing.T) { hook := NewLogHook() // Set up test data entry := &logrus.Entry{ Level: logrus.ErrorLevel, Data: logrus.Fields{ "namespace": "test-namespace", "error": errors.New("test-error"), "resource": "test-resource", "name": "test-name", }, Message: "test-message", } expectedResult := &results.Result{} expectedResult.Add("test-namespace", fmt.Errorf(" resource: /test-resource name: /test-name message: /test-message error: /%v", entry.Data["error"])) hook.entries[logrus.ErrorLevel] = expectedResult // Test GetEntries for ErrorLevel result := hook.GetEntries(logrus.ErrorLevel) assert.Equal(t, *expectedResult, result) // Test GetEntries for WarnLevel result = hook.GetEntries(logrus.WarnLevel) assert.Equal(t, results.Result{}, result) // Test GetEntries for other levels result = hook.GetEntries(logrus.InfoLevel) assert.Equal(t, results.Result{}, result) result = hook.GetEntries(logrus.DebugLevel) assert.Equal(t, results.Result{}, result) } ================================================ FILE: pkg/util/logging/log_level_flag.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package logging import ( "sort" "strings" "github.com/sirupsen/logrus" "github.com/vmware-tanzu/velero/pkg/cmd/util/flag" ) var sortedLogLevels = sortLogLevels() // LevelFlag is a command-line flag for setting the logrus // log level. type LevelFlag struct { *flag.Enum defaultValue logrus.Level } // LogLevelFlag constructs a new log level flag. func LogLevelFlag(defaultValue logrus.Level) *LevelFlag { return &LevelFlag{ Enum: flag.NewEnum(defaultValue.String(), sortedLogLevels...), defaultValue: defaultValue, } } // Parse returns the flag's value as a logrus.Level. func (f *LevelFlag) Parse() logrus.Level { if parsed, err := logrus.ParseLevel(f.String()); err == nil { return parsed } // This should theoretically never happen assuming the enum flag // is constructed correctly because the enum flag will not allow // an invalid value to be set. logrus.Errorf("log-level flag has invalid value %s", strings.ToUpper(f.String())) return f.defaultValue } // sortLogLevels returns a string slice containing all of the valid logrus // log levels (based on logrus.AllLevels), sorted in ascending order of severity. func sortLogLevels() []string { var ( sortedLogLevels = make([]logrus.Level, len(logrus.AllLevels)) logLevelsStrings []string ) copy(sortedLogLevels, logrus.AllLevels) // logrus.Panic has the lowest value, so the compare function uses ">" sort.Slice(sortedLogLevels, func(i, j int) bool { return sortedLogLevels[i] > sortedLogLevels[j] }) for _, level := range sortedLogLevels { logLevelsStrings = append(logLevelsStrings, level.String()) } return logLevelsStrings } ================================================ FILE: pkg/util/logging/log_location_hook.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package logging import ( "fmt" "runtime" "strings" "github.com/sirupsen/logrus" ) const ( logSourceField = "logSource" logSourceSetMarkerField = "@logSourceSetBy" logrusPackage = "github.com/sirupsen/logrus" veleroPackage = "github.com/vmware-tanzu/velero/" veleroPackageLen = len(veleroPackage) ) // LogLocationHook is a logrus hook that attaches location information // to log entries, i.e. the file and line number of the logrus log call. // This hook is designed for use in both the Velero server and Velero plugin // implementations. When triggered within a plugin, a marker field will // be set on the log entry indicating that the location came from a plugin. // The Velero server instance will not overwrite location information if // it sees this marker. type LogLocationHook struct { loggerName string } // WithLoggerName gives the hook a name to use when setting the marker field // on a log entry indicating the location has been recorded by a plugin. This // should only be used when setting up a hook for a logger used in a plugin. func (h *LogLocationHook) WithLoggerName(name string) *LogLocationHook { h.loggerName = name return h } func (h *LogLocationHook) Levels() []logrus.Level { return logrus.AllLevels } func (h *LogLocationHook) Fire(entry *logrus.Entry) error { pcs := make([]uintptr, 64) // skip 2 frames: // runtime.Callers // github.com/vmware-tanzu/velero/pkg/util/logging/(*LogLocationHook).Fire n := runtime.Callers(2, pcs) // re-slice pcs based on the number of entries written frames := runtime.CallersFrames(pcs[:n]) // now traverse up the call stack looking for the first non-logrus // func, which will be the logrus invoker var ( frame runtime.Frame more = true ) for more { frame, more = frames.Next() if strings.Contains(frame.File, logrusPackage) { continue } // set the marker field if we're within a plugin indicating that // the location comes from the plugin. if h.loggerName != "" { entry.Data[logSourceSetMarkerField] = h.loggerName } // record the log statement location if we're within a plugin OR if // we're in Velero server and not logging something that has the marker // set (which would indicate the log statement is coming from a plugin). if h.loggerName != "" || getLogSourceSetMarker(entry) == "" { file := removeVeleroPackagePrefix(frame.File) entry.Data[logSourceField] = fmt.Sprintf("%s:%d", file, frame.Line) } // if we're in the Velero server, remove the marker field since we don't // want to record it in the actual log. if h.loggerName == "" { delete(entry.Data, logSourceSetMarkerField) } break } return nil } func getLogSourceSetMarker(entry *logrus.Entry) string { nameVal, found := entry.Data[logSourceSetMarkerField] if !found { return "" } if name, ok := nameVal.(string); ok { return name } return fmt.Sprintf("%s", nameVal) } func removeVeleroPackagePrefix(file string) string { if index := strings.Index(file, veleroPackage); index != -1 { // strip off .../github.com/vmware-tanzu/velero/ so we just have pkg/... return file[index+veleroPackageLen:] } return file } ================================================ FILE: pkg/util/logging/log_location_hook_test.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package logging import ( "testing" "github.com/stretchr/testify/assert" ) func TestRemoveVeleroPackagePrefix(t *testing.T) { assert.Equal(t, "pkg/foo.go", removeVeleroPackagePrefix("github.com/vmware-tanzu/velero/pkg/foo.go")) assert.Equal(t, "github.com/vmware-tanzu/velero-plugin-example/foo.go", removeVeleroPackagePrefix("github.com/vmware-tanzu/velero-plugin-example/foo.go")) } ================================================ FILE: pkg/util/logging/log_merge_hook.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package logging import ( "bytes" "io" "os" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) const ( ListeningLevel = logrus.ErrorLevel ListeningMessage = "merge-log-57847fd0-0c7c-48e3-b5f7-984b293d8376" LogSourceKey = "log-source" ) // MergeHook is used to redirect a batch of logs to another logger atomically. // It hooks a log with ListeningMessage message, once the message is hit it replaces // the logger's output to HookWriter so that HookWriter retrieves the logs from a file indicated // by LogSourceKey field. type MergeHook struct { } type hookWriter struct { orgWriter io.Writer source string logger *logrus.Logger } func newHookWriter(orgWriter io.Writer, source string, logger *logrus.Logger) io.Writer { return &hookWriter{ orgWriter: orgWriter, source: source, logger: logger, } } func (h *MergeHook) Levels() []logrus.Level { return []logrus.Level{ListeningLevel} } func (h *MergeHook) Fire(entry *logrus.Entry) error { if entry.Message != ListeningMessage { return nil } source, exist := entry.Data[LogSourceKey] if !exist { return nil } entry.Logger.SetOutput(newHookWriter(entry.Logger.Out, source.(string), entry.Logger)) return nil } func (w *hookWriter) Write(p []byte) (n int, err error) { if !bytes.Contains(p, []byte(ListeningMessage)) { return w.orgWriter.Write(p) } defer func() { w.logger.Out = w.orgWriter }() sourceFile, err := os.OpenFile(w.source, os.O_RDONLY, 0400) if err != nil { return 0, err } defer sourceFile.Close() total := 0 buffer := make([]byte, 2048) for { read, err := sourceFile.Read(buffer) if err == io.EOF { return total, nil } if err != nil { return total, errors.Wrapf(err, "error to read source file %s at pos %v", w.source, total) } written, err := w.orgWriter.Write(buffer[0:read]) if err != nil { return total, errors.Wrapf(err, "error to write log at pos %v", total) } if written != read { return total, errors.Errorf("error to write log at pos %v, read %v but written %v", total, read, written) } total += read } } ================================================ FILE: pkg/util/logging/log_merge_hook_test.go ================================================ /* Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package logging import ( "fmt" "os" "testing" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestMergeHook_Fire(t *testing.T) { tests := []struct { name string entry logrus.Entry expectHook bool }{ { name: "normal message", entry: logrus.Entry{ Level: logrus.ErrorLevel, Message: "fake-message", }, expectHook: false, }, { name: "normal source", entry: logrus.Entry{ Level: logrus.ErrorLevel, Message: ListeningMessage, Data: logrus.Fields{"fake-key": "fake-value"}, }, expectHook: false, }, { name: "hook hit", entry: logrus.Entry{ Level: logrus.ErrorLevel, Message: ListeningMessage, Data: logrus.Fields{LogSourceKey: "any-value"}, Logger: &logrus.Logger{}, }, expectHook: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { hook := &MergeHook{} // method under test err := hook.Fire(&test.entry) require.NoError(t, err) if test.expectHook { assert.NotNil(t, test.entry.Logger.Out.(*hookWriter)) } }) } } type fakeWriter struct { p []byte writeError error writtenLen int } func (fw *fakeWriter) Write(p []byte) (n int, err error) { if fw.writeError != nil || fw.writtenLen != -1 { return fw.writtenLen, fw.writeError } fw.p = append(fw.p, p...) return len(p), nil } func TestMergeHook_Write(t *testing.T) { sourceFile, err := os.CreateTemp(t.TempDir(), "") require.NoError(t, err) logMessage := "fake-message-1\nfake-message-2" _, err = sourceFile.WriteString(logMessage) require.NoError(t, err) tests := []struct { name string content []byte source string writeErr error writtenLen int expectError string needRollBackHook bool }{ { name: "normal message", content: []byte("fake-message"), writtenLen: -1, }, { name: "failed to open source file", content: []byte(ListeningMessage), source: "non-exist", needRollBackHook: true, expectError: "open non-exist: no such file or directory", }, { name: "write error", content: []byte(ListeningMessage), source: sourceFile.Name(), writeErr: errors.New("fake-error"), expectError: "error to write log at pos 0: fake-error", needRollBackHook: true, }, { name: "write len mismatch", content: []byte(ListeningMessage), source: sourceFile.Name(), writtenLen: 100, expectError: fmt.Sprintf("error to write log at pos 0, read %v but written 100", len(logMessage)), needRollBackHook: true, }, { name: "success", content: []byte(ListeningMessage), source: sourceFile.Name(), writtenLen: -1, needRollBackHook: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { writer := hookWriter{ orgWriter: &fakeWriter{ writeError: test.writeErr, writtenLen: test.writtenLen, }, source: test.source, logger: &logrus.Logger{}, } n, err := writer.Write(test.content) if test.expectError == "" { require.NoError(t, err) expectStr := string(test.content) if expectStr == ListeningMessage { expectStr = logMessage } assert.Len(t, expectStr, n) fakeWriter := writer.orgWriter.(*fakeWriter) writtenStr := string(fakeWriter.p) assert.Equal(t, writtenStr, expectStr) } else { require.EqualError(t, err, test.expectError) } if test.needRollBackHook { assert.Equal(t, writer.logger.Out, writer.orgWriter) } }) } } ================================================ FILE: pkg/util/podvolume/pod_volume.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package podvolume import ( "context" "strings" "sync" "github.com/pkg/errors" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" crclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/util" ) // PVCPodCache provides a cached mapping from PVC to the pods that use it. // This cache is built once per backup to avoid repeated pod listings which // cause O(N*M) performance issues when there are many PVCs and pods. type PVCPodCache struct { mu sync.RWMutex // cache maps namespace -> pvcName -> []Pod cache map[string]map[string][]corev1api.Pod // built indicates whether the cache has been populated built bool } // NewPVCPodCache creates a new empty PVC to Pod cache. func NewPVCPodCache() *PVCPodCache { return &PVCPodCache{ cache: make(map[string]map[string][]corev1api.Pod), built: false, } } // BuildCacheForNamespaces builds the cache by listing pods once per namespace. // This is much more efficient than listing pods for each PVC lookup. func (c *PVCPodCache) BuildCacheForNamespaces( ctx context.Context, namespaces []string, crClient crclient.Client, ) error { c.mu.Lock() defer c.mu.Unlock() for _, ns := range namespaces { podList := new(corev1api.PodList) if err := crClient.List( ctx, podList, &crclient.ListOptions{Namespace: ns}, ); err != nil { return errors.Wrapf(err, "failed to list pods in namespace %s", ns) } if c.cache[ns] == nil { c.cache[ns] = make(map[string][]corev1api.Pod) } // Build mapping from PVC name to pods for i := range podList.Items { pod := podList.Items[i] for _, v := range pod.Spec.Volumes { if v.PersistentVolumeClaim != nil { pvcName := v.PersistentVolumeClaim.ClaimName c.cache[ns][pvcName] = append(c.cache[ns][pvcName], pod) } } } } c.built = true return nil } // GetPodsUsingPVC retrieves pods using a specific PVC from the cache. // Returns nil slice if the PVC is not found in the cache. func (c *PVCPodCache) GetPodsUsingPVC(namespace, pvcName string) []corev1api.Pod { c.mu.RLock() defer c.mu.RUnlock() if nsPods, ok := c.cache[namespace]; ok { if pods, ok := nsPods[pvcName]; ok { // Return a copy to avoid race conditions result := make([]corev1api.Pod, len(pods)) copy(result, pods) return result } } return nil } // IsBuilt returns true if the cache has been built. func (c *PVCPodCache) IsBuilt() bool { c.mu.RLock() defer c.mu.RUnlock() return c.built } // IsNamespaceBuilt returns true if the cache has been built for the given namespace. func (c *PVCPodCache) IsNamespaceBuilt(namespace string) bool { c.mu.RLock() defer c.mu.RUnlock() _, ok := c.cache[namespace] return ok } // BuildCacheForNamespace builds the cache for a single namespace lazily. // This is used by plugins where namespaces are encountered one at a time. // If the namespace is already cached, this is a no-op. func (c *PVCPodCache) BuildCacheForNamespace( ctx context.Context, namespace string, crClient crclient.Client, ) error { // Check if already built (read lock first for performance) c.mu.RLock() if _, ok := c.cache[namespace]; ok { c.mu.RUnlock() return nil } c.mu.RUnlock() // Need to build - acquire write lock c.mu.Lock() defer c.mu.Unlock() // Double-check after acquiring write lock if _, ok := c.cache[namespace]; ok { return nil } podList := new(corev1api.PodList) if err := crClient.List( ctx, podList, &crclient.ListOptions{Namespace: namespace}, ); err != nil { return errors.Wrapf(err, "failed to list pods in namespace %s", namespace) } c.cache[namespace] = make(map[string][]corev1api.Pod) // Build mapping from PVC name to pods for i := range podList.Items { pod := podList.Items[i] for _, v := range pod.Spec.Volumes { if v.PersistentVolumeClaim != nil { pvcName := v.PersistentVolumeClaim.ClaimName c.cache[namespace][pvcName] = append(c.cache[namespace][pvcName], pod) } } } // Mark as built for GetPodsUsingPVCWithCache fallback logic c.built = true return nil } // GetVolumesByPod returns a list of volume names to backup for the provided pod. func GetVolumesByPod(pod *corev1api.Pod, defaultVolumesToFsBackup, backupExcludePVC bool, volsToProcessByLegacyApproach []string) ([]string, []string) { // tracks the volumes that have been explicitly opted out of backup via the annotation in the pod optedOutVolumes := make([]string, 0) if !defaultVolumesToFsBackup { return GetVolumesToBackup(pod), optedOutVolumes } volsToExclude := GetVolumesToExclude(pod) podVolumes := []string{} // Identify volume to process // For normal case all the pod volume will be processed // For case when volsToProcessByLegacyApproach is non-empty then only those volume will be processed volsToProcess := GetVolumesToProcess(pod.Spec.Volumes, volsToProcessByLegacyApproach) for _, pv := range volsToProcess { // cannot backup hostpath volumes as they are not mounted into /var/lib/kubelet/pods // and therefore not accessible to the node agent daemon set. if pv.HostPath != nil { continue } // don't backup volumes mounting secrets. Secrets will be backed up separately. if pv.Secret != nil { continue } // don't backup volumes mounting ConfigMaps. ConfigMaps will be backed up separately. if pv.ConfigMap != nil { continue } // don't backup volumes mounted as projected volumes, all data in those come from kube state. if pv.Projected != nil { continue } // don't backup DownwardAPI volumes, all data in those come from kube state. if pv.DownwardAPI != nil { continue } if pv.PersistentVolumeClaim != nil && backupExcludePVC { continue } // don't backup volumes that are included in the exclude list. if util.Contains(volsToExclude, pv.Name) { optedOutVolumes = append(optedOutVolumes, pv.Name) continue } // don't include volumes that mount the default service account token. if strings.HasPrefix(pv.Name, "default-token") { continue } podVolumes = append(podVolumes, pv.Name) } return podVolumes, optedOutVolumes } // GetVolumesToBackup returns a list of volume names to backup for // the provided pod. // Deprecated: Use GetVolumesByPod instead. func GetVolumesToBackup(obj metav1.Object) []string { annotations := obj.GetAnnotations() if annotations == nil { return nil } backupsValue := annotations[velerov1api.VolumesToBackupAnnotation] if backupsValue == "" { return nil } return strings.Split(backupsValue, ",") } func GetVolumesToExclude(obj metav1.Object) []string { annotations := obj.GetAnnotations() if annotations == nil { return nil } return strings.Split(annotations[velerov1api.VolumesToExcludeAnnotation], ",") } // IsPVCDefaultToFSBackupWithCache checks if a PVC should default to fs-backup based on pod annotations. // If cache is nil or not built, it falls back to listing pods directly. // Note: In the main backup path, the cache is always built (via NewVolumeHelperImplWithNamespaces), // so the fallback is only used by plugins that don't need cache optimization. func IsPVCDefaultToFSBackupWithCache( pvcNamespace, pvcName string, crClient crclient.Client, defaultVolumesToFsBackup bool, cache *PVCPodCache, ) (bool, error) { var pods []corev1api.Pod var err error // Use cache if available, otherwise fall back to direct lookup if cache != nil && cache.IsBuilt() { pods = cache.GetPodsUsingPVC(pvcNamespace, pvcName) } else { pods, err = getPodsUsingPVCDirect(pvcNamespace, pvcName, crClient) if err != nil { return false, errors.WithStack(err) } } return checkPodsForFSBackup(pods, pvcName, defaultVolumesToFsBackup) } // checkPodsForFSBackup is a helper function that checks if any pod using the PVC // has the volume selected for fs-backup. func checkPodsForFSBackup(pods []corev1api.Pod, pvcName string, defaultVolumesToFsBackup bool) (bool, error) { for index := range pods { vols, _ := GetVolumesByPod(&pods[index], defaultVolumesToFsBackup, false, []string{}) if len(vols) > 0 { volName, err := getPodVolumeNameForPVC(pods[index], pvcName) if err != nil { return false, err } if util.Contains(vols, volName) { return true, nil } } } return false, nil } func getPodVolumeNameForPVC(pod corev1api.Pod, pvcName string) (string, error) { for _, v := range pod.Spec.Volumes { if v.PersistentVolumeClaim != nil && v.PersistentVolumeClaim.ClaimName == pvcName { return v.Name, nil } } return "", errors.Errorf("Pod %s/%s does not use PVC %s/%s", pod.Namespace, pod.Name, pod.Namespace, pvcName) } // GetPodsUsingPVCWithCache returns all pods that use the specified PVC. // If cache is available and built, it uses the cache for O(1) lookup. // Otherwise, it falls back to listing pods directly. // Note: In the main backup path, the cache is always built (via NewVolumeHelperImplWithNamespaces), // so the fallback is only used by plugins that don't need cache optimization. func GetPodsUsingPVCWithCache( pvcNamespace, pvcName string, crClient crclient.Client, cache *PVCPodCache, ) ([]corev1api.Pod, error) { // Use cache if available if cache != nil && cache.IsBuilt() { pods := cache.GetPodsUsingPVC(pvcNamespace, pvcName) if pods == nil { return []corev1api.Pod{}, nil } return pods, nil } // Fall back to direct lookup (for plugins without cache) return getPodsUsingPVCDirect(pvcNamespace, pvcName, crClient) } // getPodsUsingPVCDirect returns all pods in the given namespace that use the specified PVC. // This is an internal function that lists all pods in the namespace and filters them. func getPodsUsingPVCDirect( pvcNamespace, pvcName string, crClient crclient.Client, ) ([]corev1api.Pod, error) { podsUsingPVC := []corev1api.Pod{} podList := new(corev1api.PodList) if err := crClient.List( context.TODO(), podList, &crclient.ListOptions{Namespace: pvcNamespace}, ); err != nil { return nil, err } for _, p := range podList.Items { for _, v := range p.Spec.Volumes { if v.PersistentVolumeClaim != nil && v.PersistentVolumeClaim.ClaimName == pvcName { podsUsingPVC = append(podsUsingPVC, p) } } } return podsUsingPVC, nil } func GetVolumesToProcess(volumes []corev1api.Volume, volsToProcessByLegacyApproach []string) []corev1api.Volume { volsToProcess := make([]corev1api.Volume, 0) // return empty list when no volumes associated with pod if len(volumes) == 0 { return volsToProcess } // legacy approach as a fallback option case if len(volsToProcessByLegacyApproach) > 0 { for _, vol := range volumes { // don't process volumes that are already matched for supported action in volume policy approach if !util.Contains(volsToProcessByLegacyApproach, vol.Name) { continue } // add volume that is not processed in volume policy approach volsToProcess = append(volsToProcess, vol) } return volsToProcess } // normal case return the list as in return volumes } ================================================ FILE: pkg/util/podvolume/pod_volume_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package podvolume import ( "sort" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func TestGetVolumesToBackup(t *testing.T) { tests := []struct { name string annotations map[string]string expected []string }{ { name: "nil annotations", annotations: nil, expected: nil, }, { name: "no volumes to backup", annotations: map[string]string{"foo": "bar"}, expected: nil, }, { name: "one volume to backup", annotations: map[string]string{"foo": "bar", velerov1api.VolumesToBackupAnnotation: "volume-1"}, expected: []string{"volume-1"}, }, { name: "multiple volumes to backup", annotations: map[string]string{"foo": "bar", velerov1api.VolumesToBackupAnnotation: "volume-1,volume-2,volume-3"}, expected: []string{"volume-1", "volume-2", "volume-3"}, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { pod := &corev1api.Pod{} pod.Annotations = test.annotations res := GetVolumesToBackup(pod) // sort to ensure good compare of slices sort.Strings(test.expected) sort.Strings(res) assert.Equal(t, test.expected, res) }) } } func TestGetVolumesByPod(t *testing.T) { testCases := []struct { name string pod *corev1api.Pod expected struct { included []string optedOut []string } defaultVolumesToFsBackup bool backupExcludePVC bool }{ { name: "should get PVs from VolumesToBackupAnnotation when defaultVolumesToFsBackup is false", defaultVolumesToFsBackup: false, pod: &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ velerov1api.VolumesToBackupAnnotation: "pvbPV1,pvbPV2,pvbPV3", }, }, }, expected: struct { included []string optedOut []string }{ included: []string{"pvbPV1", "pvbPV2", "pvbPV3"}, optedOut: []string{}, }, }, { name: "should get all pod volumes when defaultVolumesToFsBackup is true and no PVs are excluded", defaultVolumesToFsBackup: true, pod: &corev1api.Pod{ Spec: corev1api.PodSpec{ Volumes: []corev1api.Volume{ // PVB Volumes {Name: "pvbPV1"}, {Name: "pvbPV2"}, {Name: "pvbPV3"}, }, }, }, expected: struct { included []string optedOut []string }{ included: []string{"pvbPV1", "pvbPV2", "pvbPV3"}, optedOut: []string{}, }, }, { name: "should get all pod volumes except ones excluded when defaultVolumesToFsBackup is true", defaultVolumesToFsBackup: true, pod: &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ velerov1api.VolumesToExcludeAnnotation: "nonPvbPV1,nonPvbPV2,nonPvbPV3", }, }, Spec: corev1api.PodSpec{ Volumes: []corev1api.Volume{ // PVB Volumes {Name: "pvbPV1"}, {Name: "pvbPV2"}, {Name: "pvbPV3"}, /// Excluded from PVB through annotation {Name: "nonPvbPV1"}, {Name: "nonPvbPV2"}, {Name: "nonPvbPV3"}, }, }, }, expected: struct { included []string optedOut []string }{ included: []string{"pvbPV1", "pvbPV2", "pvbPV3"}, optedOut: []string{"nonPvbPV1", "nonPvbPV2", "nonPvbPV3"}, }, }, { name: "should exclude default service account token from pod volume backup", defaultVolumesToFsBackup: true, pod: &corev1api.Pod{ Spec: corev1api.PodSpec{ Volumes: []corev1api.Volume{ // PVB Volumes {Name: "pvbPV1"}, {Name: "pvbPV2"}, {Name: "pvbPV3"}, /// Excluded from PVB because volume mounting default service account token {Name: "default-token-5xq45"}, }, }, }, expected: struct { included []string optedOut []string }{ included: []string{"pvbPV1", "pvbPV2", "pvbPV3"}, optedOut: []string{}, }, }, { name: "should exclude host path volumes from pod volume backups", defaultVolumesToFsBackup: true, pod: &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ velerov1api.VolumesToExcludeAnnotation: "nonPvbPV1,nonPvbPV2,nonPvbPV3", }, }, Spec: corev1api.PodSpec{ Volumes: []corev1api.Volume{ // PVB Volumes {Name: "pvbPV1"}, {Name: "pvbPV2"}, {Name: "pvbPV3"}, /// Excluded from pod volume backup through annotation {Name: "nonPvbPV1"}, {Name: "nonPvbPV2"}, {Name: "nonPvbPV3"}, // Excluded from pod volume backup because hostpath {Name: "hostPath1", VolumeSource: corev1api.VolumeSource{HostPath: &corev1api.HostPathVolumeSource{Path: "/hostpathVol"}}}, }, }, }, expected: struct { included []string optedOut []string }{ included: []string{"pvbPV1", "pvbPV2", "pvbPV3"}, optedOut: []string{"nonPvbPV1", "nonPvbPV2", "nonPvbPV3"}, }, }, { name: "should exclude volumes mounting secrets", defaultVolumesToFsBackup: true, pod: &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ velerov1api.VolumesToExcludeAnnotation: "nonPvbPV1,nonPvbPV2,nonPvbPV3", }, }, Spec: corev1api.PodSpec{ Volumes: []corev1api.Volume{ // PVB Volumes {Name: "pvbPV1"}, {Name: "pvbPV2"}, {Name: "pvbPV3"}, /// Excluded from pod volume backup through annotation {Name: "nonPvbPV1"}, {Name: "nonPvbPV2"}, {Name: "nonPvbPV3"}, // Excluded from pod volume backup because hostpath {Name: "superSecret", VolumeSource: corev1api.VolumeSource{Secret: &corev1api.SecretVolumeSource{SecretName: "super-secret"}}}, }, }, }, expected: struct { included []string optedOut []string }{ included: []string{"pvbPV1", "pvbPV2", "pvbPV3"}, optedOut: []string{"nonPvbPV1", "nonPvbPV2", "nonPvbPV3"}, }, }, { name: "should exclude volumes mounting ConfigMaps", defaultVolumesToFsBackup: true, pod: &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ velerov1api.VolumesToExcludeAnnotation: "nonPvbPV1,nonPvbPV2,nonPvbPV3", }, }, Spec: corev1api.PodSpec{ Volumes: []corev1api.Volume{ // PVB Volumes {Name: "pvbPV1"}, {Name: "pvbPV2"}, {Name: "pvbPV3"}, /// Excluded from pod volume backup through annotation {Name: "nonPvbPV1"}, {Name: "nonPvbPV2"}, {Name: "nonPvbPV3"}, // Excluded from pod volume backup because hostpath {Name: "appCOnfig", VolumeSource: corev1api.VolumeSource{ConfigMap: &corev1api.ConfigMapVolumeSource{LocalObjectReference: corev1api.LocalObjectReference{Name: "app-config"}}}}, }, }, }, expected: struct { included []string optedOut []string }{ included: []string{"pvbPV1", "pvbPV2", "pvbPV3"}, optedOut: []string{"nonPvbPV1", "nonPvbPV2", "nonPvbPV3"}, }, }, { name: "should exclude projected volumes", defaultVolumesToFsBackup: true, pod: &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ velerov1api.VolumesToExcludeAnnotation: "nonPvbPV1,nonPvbPV2,nonPvbPV3", }, }, Spec: corev1api.PodSpec{ Volumes: []corev1api.Volume{ {Name: "pvbPV1"}, {Name: "pvbPV2"}, {Name: "pvbPV3"}, { Name: "projected", VolumeSource: corev1api.VolumeSource{ Projected: &corev1api.ProjectedVolumeSource{ Sources: []corev1api.VolumeProjection{{ Secret: &corev1api.SecretProjection{ LocalObjectReference: corev1api.LocalObjectReference{}, Items: nil, Optional: nil, }, DownwardAPI: nil, ConfigMap: nil, ServiceAccountToken: nil, }}, DefaultMode: nil, }, }, }, }, }, }, expected: struct { included []string optedOut []string }{ included: []string{"pvbPV1", "pvbPV2", "pvbPV3"}, optedOut: []string{}, }, }, { name: "should exclude DownwardAPI volumes", defaultVolumesToFsBackup: true, pod: &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ velerov1api.VolumesToExcludeAnnotation: "nonPvbPV1,nonPvbPV2,nonPvbPV3", }, }, Spec: corev1api.PodSpec{ Volumes: []corev1api.Volume{ {Name: "pvbPV1"}, {Name: "pvbPV2"}, {Name: "pvbPV3"}, { Name: "downwardAPI", VolumeSource: corev1api.VolumeSource{ DownwardAPI: &corev1api.DownwardAPIVolumeSource{ Items: []corev1api.DownwardAPIVolumeFile{ { Path: "labels", FieldRef: &corev1api.ObjectFieldSelector{ APIVersion: "v1", FieldPath: "metadata.labels", }, }, }, }, }, }, }, }, }, expected: struct { included []string optedOut []string }{ included: []string{"pvbPV1", "pvbPV2", "pvbPV3"}, optedOut: []string{}, }, }, { name: "should exclude PVC volume when backup excludes PVC resource", defaultVolumesToFsBackup: true, backupExcludePVC: true, pod: &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ velerov1api.VolumesToExcludeAnnotation: "nonPvbPV1,nonPvbPV2,nonPvbPV3", }, }, Spec: corev1api.PodSpec{ Volumes: []corev1api.Volume{ {Name: "pvbPV1"}, {Name: "pvbPV2"}, {Name: "pvbPV3"}, { Name: "downwardAPI", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "testPVC", }, }, }, }, }, }, expected: struct { included []string optedOut []string }{ included: []string{"pvbPV1", "pvbPV2", "pvbPV3"}, optedOut: []string{}, }, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { actualIncluded, actualOptedOut := GetVolumesByPod(tc.pod, tc.defaultVolumesToFsBackup, tc.backupExcludePVC, []string{}) sort.Strings(tc.expected.included) sort.Strings(actualIncluded) assert.Equal(t, tc.expected.included, actualIncluded) sort.Strings(tc.expected.optedOut) sort.Strings(actualOptedOut) assert.Equal(t, tc.expected.optedOut, actualOptedOut) }) } } func TestGetPodVolumeNameForPVC(t *testing.T) { testCases := []struct { name string pod corev1api.Pod pvcName string expectError bool expectedVolumeName string }{ { name: "should get volume name for pod with multiple PVCs", pod: corev1api.Pod{ Spec: corev1api.PodSpec{ Volumes: []corev1api.Volume{ { Name: "csi-vol1", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "csi-pvc1", }, }, }, { Name: "csi-vol2", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "csi-pvc2", }, }, }, { Name: "csi-vol3", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "csi-pvc3", }, }, }, }, }, }, pvcName: "csi-pvc2", expectedVolumeName: "csi-vol2", expectError: false, }, { name: "should get volume name from pod using exactly one PVC", pod: corev1api.Pod{ Spec: corev1api.PodSpec{ Volumes: []corev1api.Volume{ { Name: "csi-vol1", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "csi-pvc1", }, }, }, }, }, }, pvcName: "csi-pvc1", expectedVolumeName: "csi-vol1", expectError: false, }, { name: "should return error for pod with no PVCs", pod: corev1api.Pod{ Spec: corev1api.PodSpec{}, }, pvcName: "csi-pvc2", expectError: true, }, { name: "should return error for pod with no matching PVC", pod: corev1api.Pod{ Spec: corev1api.PodSpec{ Volumes: []corev1api.Volume{ { Name: "csi-vol1", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "csi-pvc1", }, }, }, }, }, }, pvcName: "mismatch-pvc", expectError: true, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { actualVolumeName, err := getPodVolumeNameForPVC(tc.pod, tc.pvcName) if tc.expectError && err == nil { assert.Error(t, err, "Want error; Got nil error") return } assert.Equalf(t, tc.expectedVolumeName, actualVolumeName, "unexpected podVolumename returned. Want %s; Got %s", tc.expectedVolumeName, actualVolumeName) }) } } func TestGetVolumesToProcess(t *testing.T) { testCases := []struct { name string volumes []corev1api.Volume volsToProcessByLegacyApproach []string expectedVolumes []corev1api.Volume }{ { name: "pod has 2 volumes empty volsToProcessByLegacyApproach list return 2 volumes", volumes: []corev1api.Volume{ { Name: "sample-volume-1", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "sample-pvc-1", }, }, }, { Name: "sample-volume-2", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "sample-pvc-2", }, }, }, }, volsToProcessByLegacyApproach: []string{}, expectedVolumes: []corev1api.Volume{ { Name: "sample-volume-1", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "sample-pvc-1", }, }, }, { Name: "sample-volume-2", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "sample-pvc-2", }, }, }, }, }, { name: "pod has 2 volumes non-empty volsToProcessByLegacyApproach list returns 1 volumes", volumes: []corev1api.Volume{ { Name: "sample-volume-1", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "sample-pvc-1", }, }, }, { Name: "sample-volume-2", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "sample-pvc-2", }, }, }, }, volsToProcessByLegacyApproach: []string{"sample-volume-2"}, expectedVolumes: []corev1api.Volume{ { Name: "sample-volume-2", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "sample-pvc-2", }, }, }, }, }, { name: "empty case, return empty list", volumes: []corev1api.Volume{}, volsToProcessByLegacyApproach: []string{}, expectedVolumes: []corev1api.Volume{}, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { actualVolumes := GetVolumesToProcess(tc.volumes, tc.volsToProcessByLegacyApproach) assert.Equal(t, tc.expectedVolumes, actualVolumes, "Want Volumes List %v; Got Volumes List %v", tc.expectedVolumes, actualVolumes) }) } } func TestPVCPodCache_BuildAndGet(t *testing.T) { objs := []runtime.Object{ &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "pod1", Namespace: "default", }, Spec: corev1api.PodSpec{ Volumes: []corev1api.Volume{ { Name: "vol1", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "pvc1", }, }, }, }, }, }, &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "pod2", Namespace: "default", }, Spec: corev1api.PodSpec{ Volumes: []corev1api.Volume{ { Name: "vol1", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "pvc1", }, }, }, { Name: "vol2", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "pvc2", }, }, }, }, }, }, &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "pod3", Namespace: "default", }, Spec: corev1api.PodSpec{ Volumes: []corev1api.Volume{ { Name: "vol1", VolumeSource: corev1api.VolumeSource{ EmptyDir: &corev1api.EmptyDirVolumeSource{}, }, }, }, }, }, &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "pod4", Namespace: "other-ns", }, Spec: corev1api.PodSpec{ Volumes: []corev1api.Volume{ { Name: "vol1", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "pvc1", }, }, }, }, }, }, } fakeClient := velerotest.NewFakeControllerRuntimeClient(t, objs...) testCases := []struct { name string namespaces []string pvcNamespace string pvcName string expectedPodCount int }{ { name: "should find 2 pods using pvc1 in default namespace", namespaces: []string{"default", "other-ns"}, pvcNamespace: "default", pvcName: "pvc1", expectedPodCount: 2, }, { name: "should find 1 pod using pvc2 in default namespace", namespaces: []string{"default", "other-ns"}, pvcNamespace: "default", pvcName: "pvc2", expectedPodCount: 1, }, { name: "should find 1 pod using pvc1 in other-ns", namespaces: []string{"default", "other-ns"}, pvcNamespace: "other-ns", pvcName: "pvc1", expectedPodCount: 1, }, { name: "should find 0 pods for non-existent PVC", namespaces: []string{"default", "other-ns"}, pvcNamespace: "default", pvcName: "non-existent", expectedPodCount: 0, }, { name: "should find 0 pods for non-existent namespace", namespaces: []string{"default", "other-ns"}, pvcNamespace: "non-existent-ns", pvcName: "pvc1", expectedPodCount: 0, }, { name: "should find 0 pods when namespace not in cache", namespaces: []string{"default"}, pvcNamespace: "other-ns", pvcName: "pvc1", expectedPodCount: 0, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { cache := NewPVCPodCache() err := cache.BuildCacheForNamespaces(t.Context(), tc.namespaces, fakeClient) require.NoError(t, err) assert.True(t, cache.IsBuilt()) pods := cache.GetPodsUsingPVC(tc.pvcNamespace, tc.pvcName) assert.Len(t, pods, tc.expectedPodCount, "unexpected number of pods") }) } } func TestGetPodsUsingPVCWithCache(t *testing.T) { objs := []runtime.Object{ &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "pod1", Namespace: "default", }, Spec: corev1api.PodSpec{ Volumes: []corev1api.Volume{ { Name: "vol1", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "pvc1", }, }, }, }, }, }, &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "pod2", Namespace: "default", }, Spec: corev1api.PodSpec{ Volumes: []corev1api.Volume{ { Name: "vol1", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "pvc1", }, }, }, }, }, }, } fakeClient := velerotest.NewFakeControllerRuntimeClient(t, objs...) testCases := []struct { name string pvcNamespace string pvcName string buildCache bool useNilCache bool expectedPodCount int }{ { name: "returns cached results when cache is available", pvcNamespace: "default", pvcName: "pvc1", buildCache: true, useNilCache: false, expectedPodCount: 2, }, { name: "falls back to direct lookup when cache is nil", pvcNamespace: "default", pvcName: "pvc1", buildCache: false, useNilCache: true, expectedPodCount: 2, }, { name: "falls back to direct lookup when cache is not built", pvcNamespace: "default", pvcName: "pvc1", buildCache: false, useNilCache: false, expectedPodCount: 2, }, { name: "returns empty slice for non-existent PVC with cache", pvcNamespace: "default", pvcName: "non-existent", buildCache: true, useNilCache: false, expectedPodCount: 0, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { var cache *PVCPodCache if !tc.useNilCache { cache = NewPVCPodCache() if tc.buildCache { err := cache.BuildCacheForNamespaces(t.Context(), []string{"default"}, fakeClient) require.NoError(t, err) } } pods, err := GetPodsUsingPVCWithCache(tc.pvcNamespace, tc.pvcName, fakeClient, cache) require.NoError(t, err) assert.Len(t, pods, tc.expectedPodCount, "unexpected number of pods") }) } } func TestIsPVCDefaultToFSBackupWithCache(t *testing.T) { objs := []runtime.Object{ &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "pod1", Namespace: "default", Annotations: map[string]string{ "backup.velero.io/backup-volumes": "vol1", }, }, Spec: corev1api.PodSpec{ Volumes: []corev1api.Volume{ { Name: "vol1", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "pvc1", }, }, }, }, }, }, &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "pod2", Namespace: "default", }, Spec: corev1api.PodSpec{ Volumes: []corev1api.Volume{ { Name: "vol1", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "pvc2", }, }, }, }, }, }, } fakeClient := velerotest.NewFakeControllerRuntimeClient(t, objs...) testCases := []struct { name string pvcNamespace string pvcName string defaultVolumesToFsBackup bool buildCache bool useNilCache bool expectedResult bool }{ { name: "returns true for PVC with opt-in annotation using cache", pvcNamespace: "default", pvcName: "pvc1", defaultVolumesToFsBackup: false, buildCache: true, useNilCache: false, expectedResult: true, }, { name: "returns false for PVC without annotation using cache", pvcNamespace: "default", pvcName: "pvc2", defaultVolumesToFsBackup: false, buildCache: true, useNilCache: false, expectedResult: false, }, { name: "returns true for any PVC with defaultVolumesToFsBackup using cache", pvcNamespace: "default", pvcName: "pvc2", defaultVolumesToFsBackup: true, buildCache: true, useNilCache: false, expectedResult: true, }, { name: "falls back to direct lookup when cache is nil", pvcNamespace: "default", pvcName: "pvc1", defaultVolumesToFsBackup: false, buildCache: false, useNilCache: true, expectedResult: true, }, { name: "returns false for non-existent PVC", pvcNamespace: "default", pvcName: "non-existent", defaultVolumesToFsBackup: false, buildCache: true, useNilCache: false, expectedResult: false, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { var cache *PVCPodCache if !tc.useNilCache { cache = NewPVCPodCache() if tc.buildCache { err := cache.BuildCacheForNamespaces(t.Context(), []string{"default"}, fakeClient) require.NoError(t, err) } } result, err := IsPVCDefaultToFSBackupWithCache(tc.pvcNamespace, tc.pvcName, fakeClient, tc.defaultVolumesToFsBackup, cache) require.NoError(t, err) assert.Equal(t, tc.expectedResult, result) }) } } // TestIsNamespaceBuilt tests the IsNamespaceBuilt method for lazy per-namespace caching. func TestIsNamespaceBuilt(t *testing.T) { cache := NewPVCPodCache() // Initially no namespace should be built assert.False(t, cache.IsNamespaceBuilt("ns1"), "namespace should not be built initially") assert.False(t, cache.IsNamespaceBuilt("ns2"), "namespace should not be built initially") // Create a fake client with a pod in ns1 pod := &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "test-pod", Namespace: "ns1", }, Spec: corev1api.PodSpec{ Volumes: []corev1api.Volume{ { Name: "vol1", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "pvc1", }, }, }, }, }, } fakeClient := velerotest.NewFakeControllerRuntimeClient(t, pod) // Build cache for ns1 err := cache.BuildCacheForNamespace(t.Context(), "ns1", fakeClient) require.NoError(t, err) // ns1 should be built, ns2 should not assert.True(t, cache.IsNamespaceBuilt("ns1"), "namespace ns1 should be built") assert.False(t, cache.IsNamespaceBuilt("ns2"), "namespace ns2 should not be built") // Build cache for ns2 (empty namespace) err = cache.BuildCacheForNamespace(t.Context(), "ns2", fakeClient) require.NoError(t, err) // Both should now be built assert.True(t, cache.IsNamespaceBuilt("ns1"), "namespace ns1 should still be built") assert.True(t, cache.IsNamespaceBuilt("ns2"), "namespace ns2 should now be built") } // TestBuildCacheForNamespace tests the lazy per-namespace cache building. func TestBuildCacheForNamespace(t *testing.T) { tests := []struct { name string pods []runtime.Object namespace string expectedPVCs map[string]int // pvcName -> expected pod count expectError bool }{ { name: "build cache for namespace with pods using PVCs", namespace: "ns1", pods: []runtime.Object{ &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "pod1", Namespace: "ns1"}, Spec: corev1api.PodSpec{ Volumes: []corev1api.Volume{ { Name: "vol1", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "pvc1", }, }, }, }, }, }, &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "pod2", Namespace: "ns1"}, Spec: corev1api.PodSpec{ Volumes: []corev1api.Volume{ { Name: "vol1", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "pvc1", }, }, }, }, }, }, }, expectedPVCs: map[string]int{"pvc1": 2}, }, { name: "build cache for empty namespace", namespace: "empty-ns", pods: []runtime.Object{}, expectedPVCs: map[string]int{}, }, { name: "build cache ignores pods without PVCs", namespace: "ns1", pods: []runtime.Object{ &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "pod1", Namespace: "ns1"}, Spec: corev1api.PodSpec{ Volumes: []corev1api.Volume{ { Name: "config-vol", VolumeSource: corev1api.VolumeSource{ ConfigMap: &corev1api.ConfigMapVolumeSource{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "my-config", }, }, }, }, }, }, }, }, expectedPVCs: map[string]int{}, }, { name: "build cache only for specified namespace", namespace: "ns1", pods: []runtime.Object{ &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "pod1", Namespace: "ns1"}, Spec: corev1api.PodSpec{ Volumes: []corev1api.Volume{ { Name: "vol1", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "pvc1", }, }, }, }, }, }, &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "pod2", Namespace: "ns2"}, Spec: corev1api.PodSpec{ Volumes: []corev1api.Volume{ { Name: "vol1", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "pvc2", }, }, }, }, }, }, }, expectedPVCs: map[string]int{"pvc1": 1}, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { fakeClient := velerotest.NewFakeControllerRuntimeClient(t, tc.pods...) cache := NewPVCPodCache() // Build cache for the namespace err := cache.BuildCacheForNamespace(t.Context(), tc.namespace, fakeClient) if tc.expectError { require.Error(t, err) return } require.NoError(t, err) // Verify namespace is marked as built assert.True(t, cache.IsNamespaceBuilt(tc.namespace)) // Verify PVC to pod mappings for pvcName, expectedCount := range tc.expectedPVCs { pods := cache.GetPodsUsingPVC(tc.namespace, pvcName) assert.Len(t, pods, expectedCount, "unexpected pod count for PVC %s", pvcName) } // Calling BuildCacheForNamespace again should be a no-op err = cache.BuildCacheForNamespace(t.Context(), tc.namespace, fakeClient) require.NoError(t, err) }) } } // TestBuildCacheForNamespaceIdempotent verifies that building cache multiple times is safe. func TestBuildCacheForNamespaceIdempotent(t *testing.T) { pod := &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "pod1", Namespace: "ns1"}, Spec: corev1api.PodSpec{ Volumes: []corev1api.Volume{ { Name: "vol1", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ ClaimName: "pvc1", }, }, }, }, }, } fakeClient := velerotest.NewFakeControllerRuntimeClient(t, pod) cache := NewPVCPodCache() // Build cache multiple times - should be idempotent for i := 0; i < 3; i++ { err := cache.BuildCacheForNamespace(t.Context(), "ns1", fakeClient) require.NoError(t, err) assert.True(t, cache.IsNamespaceBuilt("ns1")) pods := cache.GetPodsUsingPVC("ns1", "pvc1") assert.Len(t, pods, 1, "should have exactly 1 pod using pvc1") } } ================================================ FILE: pkg/util/results/result.go ================================================ /* Copyright 2019, 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package results // Result is a collection of messages that were generated during // execution of a backup or restore. This will typically store either // warning or error messages. type Result struct { // Velero is a slice of messages related to the operation of Velero // itself (for example, messages related to connecting to the // cloud, reading a backup file, etc.) Velero []string `json:"velero,omitempty"` // Cluster is a slice of messages related to backup or restore of // cluster-scoped resources. Cluster []string `json:"cluster,omitempty"` // Namespaces is a map of namespace name to slice of messages // related to backup or restore namespace-scoped resources. Namespaces map[string][]string `json:"namespaces,omitempty"` } // Merge combines two Result objects into one // by appending the corresponding lists to one another. func (r *Result) Merge(other *Result) { r.Cluster = append(r.Cluster, other.Cluster...) r.Velero = append(r.Velero, other.Velero...) for k, v := range other.Namespaces { if r.Namespaces == nil { r.Namespaces = make(map[string][]string) } r.Namespaces[k] = append(r.Namespaces[k], v...) } } // AddVeleroError appends an error to the provided Result's Velero list. func (r *Result) AddVeleroError(err error) { r.Velero = append(r.Velero, err.Error()) } // Add appends an error to the provided Result, either within // the cluster-scoped list (if ns == "") or within the provided namespace's // entry. func (r *Result) Add(ns string, e error) { if ns == "" { r.Cluster = append(r.Cluster, e.Error()) } else { if r.Namespaces == nil { r.Namespaces = make(map[string][]string) } r.Namespaces[ns] = append(r.Namespaces[ns], e.Error()) } } // IsEmpty returns true if all collections of messages are empty func (r *Result) IsEmpty() bool { return len(r.Velero) == 0 && len(r.Cluster) == 0 && len(r.Namespaces) == 0 } ================================================ FILE: pkg/util/results/result_test.go ================================================ /* Copyright 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package results import ( "testing" "github.com/pkg/errors" "github.com/stretchr/testify/assert" ) func TestMerge(t *testing.T) { tests := []struct { name string result *Result other *Result want *Result }{ { name: "when an empty result is merged into a non-empty result, the result does not change", result: &Result{ Cluster: []string{"foo"}, Namespaces: map[string][]string{ "ns-1": {"bar"}, "ns-2": {"baz"}, }, }, other: &Result{}, want: &Result{ Cluster: []string{"foo"}, Namespaces: map[string][]string{ "ns-1": {"bar"}, "ns-2": {"baz"}, }, }, }, { name: "when a non-empty result is merged into an result, the result looks like the non-empty result", result: &Result{}, other: &Result{ Cluster: []string{"foo"}, Namespaces: map[string][]string{ "ns-1": {"bar"}, "ns-2": {"baz"}, }, }, want: &Result{ Cluster: []string{"foo"}, Namespaces: map[string][]string{ "ns-1": {"bar"}, "ns-2": {"baz"}, }, }, }, { name: "when two non-empty results are merged, the result is the union of the two", result: &Result{ Cluster: []string{"cluster-err-1"}, Namespaces: map[string][]string{ "ns-1": {"ns-1-err-1"}, "ns-2": {"ns-2-err-1"}, "ns-3": {"ns-3-err-1"}, }, }, other: &Result{ Cluster: []string{"cluster-err-2"}, Namespaces: map[string][]string{ "ns-1": {"ns-1-err-2"}, "ns-2": {"ns-2-err-2"}, "ns-4": {"ns-4-err-1"}, }, }, want: &Result{ Cluster: []string{"cluster-err-1", "cluster-err-2"}, Namespaces: map[string][]string{ "ns-1": {"ns-1-err-1", "ns-1-err-2"}, "ns-2": {"ns-2-err-1", "ns-2-err-2"}, "ns-3": {"ns-3-err-1"}, "ns-4": {"ns-4-err-1"}, }, }, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { tc.result.Merge(tc.other) assert.Equal(t, tc.want, tc.result) }) } } func TestAddVeleroError(t *testing.T) { tests := []struct { name string result *Result err error want *Result }{ { name: "when AddVeleroError is called for a result with no velero errors, the result has the new error added properly", result: &Result{}, err: errors.New("foo"), want: &Result{Velero: []string{"foo"}}, }, { name: "when AddVeleroError is called for a result with existing velero errors, the result has the new error appended properly", result: &Result{Velero: []string{"bar"}}, err: errors.New("foo"), want: &Result{Velero: []string{"bar", "foo"}}, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { tc.result.AddVeleroError(tc.err) assert.Equal(t, tc.want, tc.result) }) } } func TestAdd(t *testing.T) { tests := []struct { name string result *Result ns string err error want *Result }{ { name: "when Add is called for a result with no existing errors and an empty namespace, the error is added to the cluster-scoped list", result: &Result{}, ns: "", err: errors.New("foo"), want: &Result{Cluster: []string{"foo"}}, }, { name: "when Add is called for a result with some existing errors and an empty namespace, the error is added to the cluster-scoped list", result: &Result{Cluster: []string{"bar"}}, ns: "", err: errors.New("foo"), want: &Result{Cluster: []string{"bar", "foo"}}, }, { name: "when Add is called for a result with no existing errors and a non-empty namespace, the error is added to the namespace list", result: &Result{}, ns: "ns-1", err: errors.New("foo"), want: &Result{ Namespaces: map[string][]string{ "ns-1": {"foo"}, }, }, }, { name: "when Add is called for a result with some existing errors and a non-empty namespace, the error is added to the namespace list", result: &Result{ Namespaces: map[string][]string{ "ns-1": {"bar"}, "ns-2": {"baz"}, }, }, ns: "ns-1", err: errors.New("foo"), want: &Result{ Namespaces: map[string][]string{ "ns-1": {"bar", "foo"}, "ns-2": {"baz"}, }, }, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { tc.result.Add(tc.ns, tc.err) assert.Equal(t, tc.want, tc.result) }) } } func TestIsEmpty(t *testing.T) { result := &Result{ Velero: nil, Cluster: nil, Namespaces: nil, } assert.True(t, result.IsEmpty()) result = &Result{ Velero: []string{"error"}, Cluster: nil, Namespaces: nil, } assert.False(t, result.IsEmpty()) } ================================================ FILE: pkg/util/scheme.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package util import ( "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" ) var VeleroScheme = runtime.NewScheme() func init() { localSchemeBuilder := runtime.SchemeBuilder{ v1.AddToScheme, v2alpha1.AddToScheme, } utilruntime.Must(localSchemeBuilder.AddToScheme(VeleroScheme)) } ================================================ FILE: pkg/util/stringptr/stringptr.go ================================================ /* Copyright 2017 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package stringptr const NilString = "" func GetString(str *string) string { if str == nil { return NilString } else { return *str } } ================================================ FILE: pkg/util/stringslice/stringslice.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package stringslice // Has returns true if the `items` slice contains the // value `val`, or false otherwise. func Has(items []string, val string) bool { for _, itm := range items { if itm == val { return true } } return false } // Except returns a new string slice that contains all of the entries // from `items` except `val`. func Except(items []string, val string) []string { // Default the capacity the len(items) instead of len(items)-1 in case items does not contain val. newItems := make([]string, 0, len(items)) for _, itm := range items { if itm != val { newItems = append(newItems, itm) } } return newItems } ================================================ FILE: pkg/util/stringslice/stringslice_test.go ================================================ /* Copyright 2018 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package stringslice import ( "testing" "github.com/stretchr/testify/assert" ) func TestHas(t *testing.T) { items := []string{} assert.False(t, Has(items, "a")) items = []string{"a", "b", "c"} assert.True(t, Has(items, "a")) assert.True(t, Has(items, "b")) assert.True(t, Has(items, "c")) assert.False(t, Has(items, "d")) } func TestExcept(t *testing.T) { items := []string{} except := Except(items, "asdf") assert.Empty(t, except) items = []string{"a", "b", "c"} assert.Equal(t, []string{"b", "c"}, Except(items, "a")) assert.Equal(t, []string{"a", "c"}, Except(items, "b")) assert.Equal(t, []string{"a", "b"}, Except(items, "c")) assert.Equal(t, []string{"a", "b", "c"}, Except(items, "d")) } ================================================ FILE: pkg/util/third_party.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package util var ThirdPartyLabels = []string{ "azure.workload.identity/use", } var ThirdPartyAnnotations = []string{ "iam.amazonaws.com/role", } var ThirdPartyTolerations = []string{ "kubernetes.azure.com/scalesetpriority", "CriticalAddonsOnly", } const ( VSphereCNSFastCloneAnno = "csi.vsphere.volume/fast-provisioning" ) ================================================ FILE: pkg/util/util.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package util import ( "crypto/sha256" "encoding/hex" ) func Contains(slice []string, key string) bool { for _, i := range slice { if i == key { return true } } return false } // GenerateSha256FromRestoreUIDAndVsName Use the restore UID and the VS Name to generate // the new VSC and VS name. By this way, VS and VSC RIA action can get the same VSC name. func GenerateSha256FromRestoreUIDAndVsName(restoreUID string, vsName string) string { sha256Bytes := sha256.Sum256([]byte(restoreUID + "/" + vsName)) return hex.EncodeToString(sha256Bytes[:]) } ================================================ FILE: pkg/util/util_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package util import ( "testing" "github.com/stretchr/testify/assert" ) func TestContains(t *testing.T) { testCases := []struct { name string inSlice []string inKey string expectedResult bool }{ { name: "should find the key", inSlice: []string{"key1", "key2", "key3", "key4", "key5"}, inKey: "key3", expectedResult: true, }, { name: "should not find the key in non-empty slice", inSlice: []string{"key1", "key2", "key3", "key4", "key5"}, inKey: "key300", expectedResult: false, }, { name: "should not find key in empty slice", inSlice: []string{}, inKey: "key300", expectedResult: false, }, { name: "should not find key in nil slice", inSlice: nil, inKey: "key300", expectedResult: false, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { actualResult := Contains(tc.inSlice, tc.inKey) assert.Equal(t, tc.expectedResult, actualResult) }) } } ================================================ FILE: pkg/util/velero/restore/util.go ================================================ package restore import ( api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) func IsResourcePolicyValid(resourcePolicy string) bool { if resourcePolicy == string(api.PolicyTypeNone) || resourcePolicy == string(api.PolicyTypeUpdate) { return true } return false } ================================================ FILE: pkg/util/velero/restore/util_test.go ================================================ package restore import ( "testing" "github.com/stretchr/testify/require" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) func TestIsResourcePolicyValid(t *testing.T) { require.True(t, IsResourcePolicyValid(string(velerov1api.PolicyTypeNone))) require.True(t, IsResourcePolicyValid(string(velerov1api.PolicyTypeUpdate))) require.False(t, IsResourcePolicyValid("")) } ================================================ FILE: pkg/util/velero/velero.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package velero import ( appsv1api "k8s.io/api/apps/v1" corev1api "k8s.io/api/core/v1" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) // GetNodeSelectorFromVeleroServer get the node selector from the Velero server deployment func GetNodeSelectorFromVeleroServer(deployment *appsv1api.Deployment) map[string]string { return deployment.Spec.Template.Spec.NodeSelector } // GetTolerationsFromVeleroServer get the tolerations from the Velero server deployment func GetTolerationsFromVeleroServer(deployment *appsv1api.Deployment) []corev1api.Toleration { return deployment.Spec.Template.Spec.Tolerations } // GetAffinityFromVeleroServer get the affinity from the Velero server deployment func GetAffinityFromVeleroServer(deployment *appsv1api.Deployment) *corev1api.Affinity { return deployment.Spec.Template.Spec.Affinity } // GetEnvVarsFromVeleroServer get the environment variables from the Velero server deployment func GetEnvVarsFromVeleroServer(deployment *appsv1api.Deployment) []corev1api.EnvVar { for _, container := range deployment.Spec.Template.Spec.Containers { // We only have one container in the Velero server deployment return container.Env } return nil } // GetEnvFromSourcesFromVeleroServer get the environment sources from the Velero server deployment func GetEnvFromSourcesFromVeleroServer(deployment *appsv1api.Deployment) []corev1api.EnvFromSource { for _, container := range deployment.Spec.Template.Spec.Containers { // We only have one container in the Velero server deployment return container.EnvFrom } return nil } // GetVolumeMountsFromVeleroServer get the volume mounts from the Velero server deployment func GetVolumeMountsFromVeleroServer(deployment *appsv1api.Deployment) []corev1api.VolumeMount { for _, container := range deployment.Spec.Template.Spec.Containers { // We only have one container in the Velero server deployment return container.VolumeMounts } return nil } // GetPodSecurityContextsFromVeleroServer get the pod security context from the Velero server deployment func GetPodSecurityContextsFromVeleroServer(deployment *appsv1api.Deployment) *corev1api.PodSecurityContext { return deployment.Spec.Template.Spec.SecurityContext } // GetContainerSecurityContextsFromVeleroServer get the security context from the Velero server deployment func GetContainerSecurityContextsFromVeleroServer(deployment *appsv1api.Deployment) *corev1api.SecurityContext { for _, container := range deployment.Spec.Template.Spec.Containers { // We only have one container in the Velero server deployment return container.SecurityContext } return nil } // GetVolumesFromVeleroServer get the volumes from the Velero server deployment func GetVolumesFromVeleroServer(deployment *appsv1api.Deployment) []corev1api.Volume { return deployment.Spec.Template.Spec.Volumes } // GetServiceAccountFromVeleroServer get the service account from the Velero server deployment func GetServiceAccountFromVeleroServer(deployment *appsv1api.Deployment) string { return deployment.Spec.Template.Spec.ServiceAccountName } // GetImagePullSecretsFromVeleroServer get the image pull secrets from the Velero server deployment func GetImagePullSecretsFromVeleroServer(deployment *appsv1api.Deployment) []corev1api.LocalObjectReference { return deployment.Spec.Template.Spec.ImagePullSecrets } // getVeleroServerImage get the image of the Velero server deployment func GetVeleroServerImage(deployment *appsv1api.Deployment) string { return deployment.Spec.Template.Spec.Containers[0].Image } // GetVeleroServerLables get the labels of the Velero server deployment func GetVeleroServerLables(deployment *appsv1api.Deployment) map[string]string { return deployment.Spec.Template.Labels } // GetVeleroServerAnnotations get the annotations of the Velero server deployment func GetVeleroServerAnnotations(deployment *appsv1api.Deployment) map[string]string { return deployment.Spec.Template.Annotations } // GetVeleroServerLabelValue returns the value of specified label of Velero server deployment func GetVeleroServerLabelValue(deployment *appsv1api.Deployment, key string) string { if deployment.Spec.Template.Labels == nil { return "" } return deployment.Spec.Template.Labels[key] } // GetVeleroServerAnnotationValue returns the value of specified annotation of Velero server deployment func GetVeleroServerAnnotationValue(deployment *appsv1api.Deployment, key string) string { if deployment.Spec.Template.Annotations == nil { return "" } return deployment.Spec.Template.Annotations[key] } func BSLIsAvailable(bsl velerov1api.BackupStorageLocation) bool { return bsl.Status.Phase == velerov1api.BackupStorageLocationPhaseAvailable } ================================================ FILE: pkg/util/velero/velero_test.go ================================================ /* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package velero import ( "reflect" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" appsv1api "k8s.io/api/apps/v1" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/util/boolptr" ) func TestGetNodeSelectorFromVeleroServer(t *testing.T) { tests := []struct { name string deploy *appsv1api.Deployment want map[string]string }{ { name: "no node selector", deploy: &appsv1api.Deployment{ Spec: appsv1api.DeploymentSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ NodeSelector: map[string]string{}, }, }, }, }, want: map[string]string{}, }, { name: "node selector", deploy: &appsv1api.Deployment{ Spec: appsv1api.DeploymentSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ NodeSelector: map[string]string{ "foo": "bar", }, }, }, }, }, want: map[string]string{ "foo": "bar", }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { got := GetNodeSelectorFromVeleroServer(test.deploy) if len(got) != len(test.want) { t.Errorf("expected node selector to have %d elements, got %d", len(test.want), len(got)) } for k, v := range test.want { if got[k] != v { t.Errorf("expected node selector to have key %s with value %s, got %s", k, v, got[k]) } } }) } } func TestGetTolerationsFromVeleroServer(t *testing.T) { tests := []struct { name string deploy *appsv1api.Deployment want []corev1api.Toleration }{ { name: "no tolerations", deploy: &appsv1api.Deployment{ Spec: appsv1api.DeploymentSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ Tolerations: []corev1api.Toleration{}, }, }, }, }, want: []corev1api.Toleration{}, }, { name: "tolerations", deploy: &appsv1api.Deployment{ Spec: appsv1api.DeploymentSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ Tolerations: []corev1api.Toleration{ { Key: "foo", Operator: "Exists", }, }, }, }, }, }, want: []corev1api.Toleration{ { Key: "foo", Operator: "Exists", }, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { got := GetTolerationsFromVeleroServer(test.deploy) if len(got) != len(test.want) { t.Errorf("expected tolerations to have %d elements, got %d", len(test.want), len(got)) } for i, want := range test.want { if got[i] != want { t.Errorf("expected toleration at index %d to be %v, got %v", i, want, got[i]) } } }) } } func TestGetAffinityFromVeleroServer(t *testing.T) { tests := []struct { name string deploy *appsv1api.Deployment want *corev1api.Affinity }{ { name: "no affinity", deploy: &appsv1api.Deployment{ Spec: appsv1api.DeploymentSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ Affinity: nil, }, }, }, }, want: nil, }, { name: "affinity", deploy: &appsv1api.Deployment{ Spec: appsv1api.DeploymentSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ Affinity: &corev1api.Affinity{ NodeAffinity: &corev1api.NodeAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ NodeSelectorTerms: []corev1api.NodeSelectorTerm{ { MatchExpressions: []corev1api.NodeSelectorRequirement{ { Key: "foo", Operator: "In", Values: []string{"bar"}, }, }, }, }, }, }, }, }, }, }, }, want: &corev1api.Affinity{ NodeAffinity: &corev1api.NodeAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ NodeSelectorTerms: []corev1api.NodeSelectorTerm{ { MatchExpressions: []corev1api.NodeSelectorRequirement{ { Key: "foo", Operator: "In", Values: []string{"bar"}, }, }, }, }, }, }, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { got := GetAffinityFromVeleroServer(test.deploy) if test.want != nil { require.NotNilf(t, got, "expected affinity to be %v, got nil", test.want) if test.want.NodeAffinity != nil { require.NotNilf(t, got.NodeAffinity, "expected node affinity to be %v, got nil", test.want.NodeAffinity) if test.want.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution != nil { require.NotNilf(t, got.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution, "expected required during scheduling ignored during execution to be %v, got nil", test.want.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution) assert.Truef(t, reflect.DeepEqual(got.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution, test.want.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution), "expected required during scheduling ignored during execution to be %v, got %v", test.want.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution, got.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution) } else { assert.Nilf(t, got.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution, "expected required during scheduling ignored during execution to be nil, got %v", got.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution) } } else { assert.Nilf(t, got.NodeAffinity, "expected node affinity to be nil, got %v", got.NodeAffinity) } } else { assert.Nilf(t, got, "expected affinity to be nil, got %v", got) } }) } } func TestGetEnvVarsFromVeleroServer(t *testing.T) { tests := []struct { name string deploy *appsv1api.Deployment want []corev1api.EnvVar }{ { name: "no env vars", deploy: &appsv1api.Deployment{ Spec: appsv1api.DeploymentSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ Containers: []corev1api.Container{ { Env: []corev1api.EnvVar{}, }, }, }, }, }, }, want: []corev1api.EnvVar{}, }, { name: "env vars", deploy: &appsv1api.Deployment{ Spec: appsv1api.DeploymentSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ Containers: []corev1api.Container{ { Env: []corev1api.EnvVar{ { Name: "foo", Value: "bar", }, }, }, }, }, }, }, }, want: []corev1api.EnvVar{ { Name: "foo", Value: "bar", }, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { got := GetEnvVarsFromVeleroServer(test.deploy) if len(got) != len(test.want) { t.Errorf("expected env vars to have %d elements, got %d", len(test.want), len(got)) } for i, want := range test.want { if got[i] != want { t.Errorf("expected env var at index %d to be %v, got %v", i, want, got[i]) } } }) } } func TestGetEnvFromSourcesFromVeleroServer(t *testing.T) { tests := []struct { name string deploy *appsv1api.Deployment expected []corev1api.EnvFromSource }{ { name: "no env vars", deploy: &appsv1api.Deployment{ Spec: appsv1api.DeploymentSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ Containers: []corev1api.Container{ { EnvFrom: []corev1api.EnvFromSource{}, }, }, }, }, }, }, expected: []corev1api.EnvFromSource{}, }, { name: "configmap", deploy: &appsv1api.Deployment{ Spec: appsv1api.DeploymentSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ Containers: []corev1api.Container{ { EnvFrom: []corev1api.EnvFromSource{ { ConfigMapRef: &corev1api.ConfigMapEnvSource{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "foo", }, }, }, }, }, }, }, }, }, }, expected: []corev1api.EnvFromSource{ { ConfigMapRef: &corev1api.ConfigMapEnvSource{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "foo", }, }, }, }, }, { name: "secret", deploy: &appsv1api.Deployment{ Spec: appsv1api.DeploymentSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ Containers: []corev1api.Container{ { EnvFrom: []corev1api.EnvFromSource{ { SecretRef: &corev1api.SecretEnvSource{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "foo", }, }, }, }, }, }, }, }, }, }, expected: []corev1api.EnvFromSource{ { SecretRef: &corev1api.SecretEnvSource{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "foo", }, }, }, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { result := GetEnvFromSourcesFromVeleroServer(test.deploy) assert.Equal(t, test.expected, result) }) } } func TestGetVolumeMountsFromVeleroServer(t *testing.T) { tests := []struct { name string deploy *appsv1api.Deployment want []corev1api.VolumeMount }{ { name: "no volume mounts", deploy: &appsv1api.Deployment{ Spec: appsv1api.DeploymentSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ Containers: []corev1api.Container{ { VolumeMounts: []corev1api.VolumeMount{}, }, }, }, }, }, }, want: []corev1api.VolumeMount{}, }, { name: "volume mounts", deploy: &appsv1api.Deployment{ Spec: appsv1api.DeploymentSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ Containers: []corev1api.Container{ { VolumeMounts: []corev1api.VolumeMount{ { Name: "foo", MountPath: "/bar", }, }, }, }, }, }, }, }, want: []corev1api.VolumeMount{ { Name: "foo", MountPath: "/bar", }, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { got := GetVolumeMountsFromVeleroServer(test.deploy) if len(got) != len(test.want) { t.Errorf("expected volume mounts to have %d elements, got %d", len(test.want), len(got)) } for i, want := range test.want { if got[i] != want { t.Errorf("expected volume mount at index %d to be %v, got %v", i, want, got[i]) } } }) } } func TestGetVolumesFromVeleroServer(t *testing.T) { tests := []struct { name string deploy *appsv1api.Deployment want []corev1api.Volume }{ { name: "no volumes", deploy: &appsv1api.Deployment{ Spec: appsv1api.DeploymentSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ Volumes: []corev1api.Volume{}, }, }, }, }, want: []corev1api.Volume{}, }, { name: "volumes", deploy: &appsv1api.Deployment{ Spec: appsv1api.DeploymentSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ Volumes: []corev1api.Volume{ { Name: "foo", }, }, }, }, }, }, want: []corev1api.Volume{ { Name: "foo", }, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { got := GetVolumesFromVeleroServer(test.deploy) if len(got) != len(test.want) { t.Errorf("expected volumes to have %d elements, got %d", len(test.want), len(got)) } for i, want := range test.want { if got[i] != want { t.Errorf("expected volume at index %d to be %v, got %v", i, want, got[i]) } } }) } } func TestGetPodSecurityContextsFromVeleroServer(t *testing.T) { tests := []struct { name string deploy *appsv1api.Deployment want *corev1api.PodSecurityContext }{ { name: "no security context", deploy: &appsv1api.Deployment{ Spec: appsv1api.DeploymentSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ SecurityContext: nil, }, }, }, }, want: nil, }, { name: "security context", deploy: &appsv1api.Deployment{ Spec: appsv1api.DeploymentSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ SecurityContext: &corev1api.PodSecurityContext{ RunAsNonRoot: boolptr.True(), }, }, }, }, }, want: &corev1api.PodSecurityContext{ RunAsNonRoot: boolptr.True(), }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { got := GetPodSecurityContextsFromVeleroServer(test.deploy) assert.Equal(t, test.want, got) }) } } func TestGetContainerSecurityContextsFromVeleroServer(t *testing.T) { tests := []struct { name string deploy *appsv1api.Deployment want *corev1api.SecurityContext }{ { name: "no container", deploy: &appsv1api.Deployment{ Spec: appsv1api.DeploymentSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ Containers: []corev1api.Container{}, }, }, }, }, want: nil, }, { name: "no security context", deploy: &appsv1api.Deployment{ Spec: appsv1api.DeploymentSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ Containers: []corev1api.Container{ { SecurityContext: nil, }, }, }, }, }, }, want: nil, }, { name: "security context", deploy: &appsv1api.Deployment{ Spec: appsv1api.DeploymentSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ Containers: []corev1api.Container{ { SecurityContext: &corev1api.SecurityContext{ RunAsNonRoot: boolptr.True(), }, }, }, }, }, }, }, want: &corev1api.SecurityContext{ RunAsNonRoot: boolptr.True(), }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { got := GetContainerSecurityContextsFromVeleroServer(test.deploy) assert.Equal(t, test.want, got) }) } } func TestGetServiceAccountFromVeleroServer(t *testing.T) { tests := []struct { name string deploy *appsv1api.Deployment want string }{ { name: "no service account", deploy: &appsv1api.Deployment{ Spec: appsv1api.DeploymentSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ ServiceAccountName: "", }, }, }, }, want: "", }, { name: "service account", deploy: &appsv1api.Deployment{ Spec: appsv1api.DeploymentSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ ServiceAccountName: "foo", }, }, }, }, want: "foo", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { got := GetServiceAccountFromVeleroServer(test.deploy) if got != test.want { t.Errorf("expected service account to be %s, got %s", test.want, got) } }) } } func TestGetImagePullSecretsFromVeleroServer(t *testing.T) { tests := []struct { name string deploy *appsv1api.Deployment want []corev1api.LocalObjectReference }{ { name: "no image pull secrets", deploy: &appsv1api.Deployment{ Spec: appsv1api.DeploymentSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ ServiceAccountName: "", }, }, }, }, want: nil, }, { name: "image pull secrets", deploy: &appsv1api.Deployment{ Spec: appsv1api.DeploymentSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ ImagePullSecrets: []corev1api.LocalObjectReference{ { Name: "imagePullSecret1", }, { Name: "imagePullSecret2", }, }, }, }, }, }, want: []corev1api.LocalObjectReference{ { Name: "imagePullSecret1", }, { Name: "imagePullSecret2", }, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { got := GetImagePullSecretsFromVeleroServer(test.deploy) require.Equal(t, test.want, got) }) } } func TestGetVeleroServerImage(t *testing.T) { tests := []struct { name string deploy *appsv1api.Deployment want string }{ { name: "velero server image", deploy: &appsv1api.Deployment{ Spec: appsv1api.DeploymentSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ Containers: []corev1api.Container{ { Image: "velero/velero:latest", }, }, }, }, }, }, want: "velero/velero:latest", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { got := GetVeleroServerImage(test.deploy) if got != test.want { t.Errorf("expected velero server image to be %s, got %s", test.want, got) } }) } } func TestGetVeleroServerLables(t *testing.T) { tests := []struct { name string deployment *appsv1api.Deployment expected map[string]string }{ { name: "Empty Labels", deployment: &appsv1api.Deployment{ Spec: appsv1api.DeploymentSpec{ Template: corev1api.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{}, }, }, }, }, expected: map[string]string{}, }, { name: "Non-empty Labels", deployment: &appsv1api.Deployment{ Spec: appsv1api.DeploymentSpec{ Template: corev1api.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ "app": "velero", "component": "server", }, }, }, }, }, expected: map[string]string{ "app": "velero", "component": "server", }, }, } // Run tests for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { result := GetVeleroServerLables(tt.deployment) assert.Equal(t, tt.expected, result) }) } } func TestGetVeleroServerAnnotations(t *testing.T) { tests := []struct { name string deployment *appsv1api.Deployment expected map[string]string }{ { name: "Empty Labels", deployment: &appsv1api.Deployment{ Spec: appsv1api.DeploymentSpec{ Template: corev1api.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{}, }, }, }, }, expected: map[string]string{}, }, { name: "Non-empty Labels", deployment: &appsv1api.Deployment{ Spec: appsv1api.DeploymentSpec{ Template: corev1api.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ "app": "velero", "component": "server", }, }, }, }, }, expected: map[string]string{ "app": "velero", "component": "server", }, }, } // Run tests for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { result := GetVeleroServerAnnotations(tt.deployment) assert.Equal(t, tt.expected, result) }) } } func TestGetVeleroServerLabelValue(t *testing.T) { tests := []struct { name string deployment *appsv1api.Deployment expected string }{ { name: "nil Labels", deployment: &appsv1api.Deployment{}, expected: "", }, { name: "no label key", deployment: &appsv1api.Deployment{ Spec: appsv1api.DeploymentSpec{ Template: corev1api.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{}, }, }, }, }, expected: "", }, { name: "with label key", deployment: &appsv1api.Deployment{ Spec: appsv1api.DeploymentSpec{ Template: corev1api.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{"fake-key": "fake-value"}, }, }, }, }, expected: "fake-value", }, } // Run tests for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { result := GetVeleroServerLabelValue(tt.deployment, "fake-key") assert.Equal(t, tt.expected, result) }) } } func TestBSLIsAvailable(t *testing.T) { availableBSL := builder.ForBackupStorageLocation("velero", "available").Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result() unavailableBSL := builder.ForBackupStorageLocation("velero", "unavailable").Phase(velerov1api.BackupStorageLocationPhaseUnavailable).Result() assert.True(t, BSLIsAvailable(*availableBSL)) assert.False(t, BSLIsAvailable(*unavailableBSL)) } ================================================ FILE: pkg/util/wildcard/expand.go ================================================ package wildcard import ( "errors" "strings" "github.com/gobwas/glob" "k8s.io/apimachinery/pkg/util/sets" ) func ShouldExpandWildcards(includes []string, excludes []string) bool { wildcardFound := false for _, include := range includes { // Special case: "*" alone means "match all" - don't expand if include == "*" { return false } if containsWildcardPattern(include) { wildcardFound = true } } for _, exclude := range excludes { if containsWildcardPattern(exclude) { wildcardFound = true } } return wildcardFound } // containsWildcardPattern checks if a pattern contains any wildcard symbols // Supported patterns: *, ?, [abc] // Note: . and + are treated as literal characters (not wildcards) // Note: ** and consecutive asterisks are NOT supported (will cause validation error) func containsWildcardPattern(pattern string) bool { return strings.ContainsAny(pattern, "*?[") } func validateWildcardPatterns(patterns []string) error { for _, pattern := range patterns { if err := ValidateNamespaceName(pattern); err != nil { return err } } return nil } func ValidateNamespaceName(pattern string) error { // Check for invalid characters that are not supported in glob patterns if strings.ContainsAny(pattern, "|()!{},") { return errors.New("wildcard pattern contains unsupported characters: |, (, ), !, {, }, ,") } // Check for consecutive asterisks (2 or more) if strings.Contains(pattern, "**") { return errors.New("wildcard pattern contains consecutive asterisks (only single * allowed)") } // Check for malformed brace patterns if err := validateBracePatterns(pattern); err != nil { return err } return nil } // validateBracePatterns checks for malformed brace patterns like unclosed braces or empty braces // Also validates bracket patterns [] for character classes func validateBracePatterns(pattern string) error { bracketDepth := 0 for i := 0; i < len(pattern); i++ { if pattern[i] == '[' { bracketStart := i bracketDepth++ // Scan ahead to find the matching closing bracket and validate content for j := i + 1; j < len(pattern) && bracketDepth > 0; j++ { if pattern[j] == ']' { bracketDepth-- if bracketDepth == 0 { // Found matching closing bracket - validate content content := pattern[bracketStart+1 : j] if content == "" { return errors.New("wildcard pattern contains empty bracket pattern '[]'") } // Skip to the closing bracket i = j break } } } // If we exited the loop without finding a match (bracketDepth > 0), bracket is unclosed if bracketDepth > 0 { return errors.New("wildcard pattern contains unclosed bracket '['") } // i is now positioned at the closing bracket; the outer loop will increment it } else if pattern[i] == ']' { // Found a closing bracket without a matching opening bracket return errors.New("wildcard pattern contains unmatched closing bracket ']'") } } return nil } func ExpandWildcards(activeNamespaces []string, includes []string, excludes []string) ([]string, []string, error) { expandedIncludes, err := expandWildcards(includes, activeNamespaces) if err != nil { return nil, nil, err } expandedExcludes, err := expandWildcards(excludes, activeNamespaces) if err != nil { return nil, nil, err } return expandedIncludes, expandedExcludes, nil } // expands wildcard patterns into a list of namespaces, while normally passing non-wildcard patterns func expandWildcards(patterns []string, activeNamespaces []string) ([]string, error) { if len(patterns) == 0 { return nil, nil } // Validate patterns before processing if err := validateWildcardPatterns(patterns); err != nil { return nil, err } matchedSet := make(map[string]struct{}) for _, pattern := range patterns { // If the pattern is a non-wildcard pattern, we can just add it to the result if !containsWildcardPattern(pattern) { matchedSet[pattern] = struct{}{} continue } // Compile glob pattern g, err := glob.Compile(pattern) if err != nil { return nil, err } // Match against all namespaces for _, ns := range activeNamespaces { if g.Match(ns) { matchedSet[ns] = struct{}{} } } } // Convert set to slice result := make([]string, 0, len(matchedSet)) for ns := range matchedSet { result = append(result, ns) } return result, nil } // GetWildcardResult returns the final list of namespaces after applying wildcard include/exclude logic func GetWildcardResult(expandedIncludes []string, expandedExcludes []string) []string { // Set check: set of expandedIncludes - set of expandedExcludes expandedIncludesSet := sets.New(expandedIncludes...) expandedExcludesSet := sets.New(expandedExcludes...) selectedNamespacesSet := expandedIncludesSet.Difference(expandedExcludesSet) // Convert the set to a slice selectedNamespaces := make([]string, 0, selectedNamespacesSet.Len()) for ns := range selectedNamespacesSet { selectedNamespaces = append(selectedNamespaces, ns) } return selectedNamespaces } ================================================ FILE: pkg/util/wildcard/expand_test.go ================================================ package wildcard import ( "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestShouldExpandWildcards(t *testing.T) { tests := []struct { name string includes []string excludes []string expected bool }{ { name: "no wildcards", includes: []string{"ns1", "ns2"}, excludes: []string{"ns3", "ns4"}, expected: false, }, { name: "includes has star - should not expand", includes: []string{"*"}, excludes: []string{"ns1"}, expected: false, }, { name: "includes has star after a wildcard pattern - should not expand", includes: []string{"ns*", "*"}, excludes: []string{"ns1"}, expected: false, }, { name: "includes has wildcard pattern", includes: []string{"ns*"}, excludes: []string{"ns1"}, expected: true, }, { name: "excludes has wildcard pattern", includes: []string{"ns1"}, excludes: []string{"ns*"}, expected: true, }, { name: "both have wildcard patterns", includes: []string{"app-*"}, excludes: []string{"test-*"}, expected: true, }, { name: "includes has star and wildcard - star takes precedence", includes: []string{"*", "ns*"}, excludes: []string{}, expected: false, }, { name: "double asterisk should be detected as wildcard", includes: []string{"**"}, excludes: []string{}, expected: true, // ** is a wildcard pattern (but will error during validation) }, { name: "empty slices", includes: []string{}, excludes: []string{}, expected: false, }, { name: "complex wildcard patterns", includes: []string{"*-prod"}, excludes: []string{"test-*-staging"}, expected: true, }, { name: "question mark wildcard", includes: []string{"ns?"}, excludes: []string{}, expected: true, // question mark is now considered a wildcard }, { name: "character class wildcard", includes: []string{"ns[abc]"}, excludes: []string{}, expected: true, // character class is considered wildcard }, { name: "brace alternatives wildcard", includes: []string{"ns{prod,staging}"}, excludes: []string{}, expected: false, // brace alternatives are not supported }, { name: "dot is literal - not wildcard", includes: []string{"app.prod"}, excludes: []string{}, expected: false, // dot is literal, not wildcard }, { name: "plus is literal - not wildcard", includes: []string{"app+"}, excludes: []string{}, expected: false, // plus is literal, not wildcard }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { result := ShouldExpandWildcards(tt.includes, tt.excludes) assert.Equal(t, tt.expected, result) }) } } func TestExpandWildcards(t *testing.T) { tests := []struct { name string activeNamespaces []string includes []string excludes []string expectedIncludes []string expectedExcludes []string expectError bool }{ { name: "no wildcards", activeNamespaces: []string{"ns1", "ns2", "ns3"}, includes: []string{"ns1", "ns4"}, excludes: []string{"ns2"}, expectedIncludes: []string{"ns1", "ns4"}, expectedExcludes: []string{"ns2"}, expectError: false, }, { name: "wildcard in includes", activeNamespaces: []string{"app-prod", "app-staging", "db-prod", "test-ns"}, includes: []string{"app-*"}, excludes: []string{"test-ns"}, expectedIncludes: []string{"app-prod", "app-staging"}, expectedExcludes: []string{"test-ns"}, expectError: false, }, { name: "wildcard in excludes", activeNamespaces: []string{"app-prod", "app-staging", "db-prod", "test-ns"}, includes: []string{"app-prod"}, excludes: []string{"*-staging"}, expectedIncludes: []string{"app-prod"}, expectedExcludes: []string{"app-staging"}, expectError: false, }, { name: "wildcards in both", activeNamespaces: []string{"app-prod", "app-staging", "db-prod", "db-staging", "test-ns"}, includes: []string{"*-prod"}, excludes: []string{"*-staging"}, expectedIncludes: []string{"app-prod", "db-prod"}, expectedExcludes: []string{"app-staging", "db-staging"}, expectError: false, }, { name: "star pattern in includes", activeNamespaces: []string{"ns1", "ns2", "ns3"}, includes: []string{"*"}, excludes: []string{}, expectedIncludes: []string{"ns1", "ns2", "ns3"}, expectedExcludes: nil, expectError: false, }, { name: "empty active namespaces", activeNamespaces: []string{}, includes: []string{"app-*"}, excludes: []string{"test-*"}, expectedIncludes: nil, expectedExcludes: nil, expectError: false, }, { name: "empty includes and excludes", activeNamespaces: []string{"ns1", "ns2"}, includes: []string{}, excludes: []string{}, expectedIncludes: nil, expectedExcludes: nil, expectError: false, }, { name: "complex patterns", activeNamespaces: []string{"my-app-prod", "my-app-staging", "your-app-prod", "system-ns"}, includes: []string{"*-app-*"}, excludes: []string{"*-staging"}, expectedIncludes: []string{"my-app-prod", "my-app-staging", "your-app-prod"}, expectedExcludes: []string{"my-app-staging"}, expectError: false, }, { name: "double asterisk should error", activeNamespaces: []string{"ns1", "ns2", "ns3"}, includes: []string{"**"}, excludes: []string{}, expectedIncludes: nil, expectedExcludes: nil, expectError: true, // ** is invalid }, { name: "double asterisk in pattern should error", activeNamespaces: []string{"ns1", "ns2", "ns3"}, includes: []string{"app-**"}, excludes: []string{}, expectedIncludes: nil, expectedExcludes: nil, expectError: true, // app-** contains ** which is invalid }, { name: "question mark patterns", activeNamespaces: []string{"ns1", "ns2", "ns10", "test"}, includes: []string{"ns?"}, excludes: []string{}, expectedIncludes: []string{"ns1", "ns2"}, // ? matches single character expectedExcludes: nil, expectError: false, }, { name: "character class patterns", activeNamespaces: []string{"nsa", "nsb", "nsc", "nsx", "ns1"}, includes: []string{"ns[abc]"}, excludes: []string{}, expectedIncludes: []string{"nsa", "nsb", "nsc"}, // [abc] matches a, b, or c expectedExcludes: nil, expectError: false, }, { name: "brace alternative patterns", activeNamespaces: []string{"app-prod", "app-staging", "app-dev", "db-prod"}, includes: []string{"app-{prod,staging}"}, excludes: []string{}, expectedIncludes: nil, expectedExcludes: nil, expectError: true, }, { name: "literal dot and plus patterns", activeNamespaces: []string{"app.prod", "app-prod", "app+", "app"}, includes: []string{"app.prod", "app+"}, excludes: []string{}, expectedIncludes: []string{"app.prod", "app+"}, // . and + are literal expectedExcludes: nil, expectError: false, }, { name: "unsupported regex patterns should error", activeNamespaces: []string{"ns1", "ns2"}, includes: []string{"ns(1|2)"}, excludes: []string{}, expectedIncludes: nil, expectedExcludes: nil, expectError: true, // |, (, ) are not supported }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { includes, excludes, err := ExpandWildcards(tt.activeNamespaces, tt.includes, tt.excludes) if tt.expectError { assert.Error(t, err) return } require.NoError(t, err) assert.ElementsMatch(t, tt.expectedIncludes, includes) assert.ElementsMatch(t, tt.expectedExcludes, excludes) }) } } func TestExpandWildcardsPrivate(t *testing.T) { tests := []struct { name string patterns []string activeNamespaces []string expected []string expectError bool }{ { name: "empty patterns", patterns: []string{}, activeNamespaces: []string{"ns1", "ns2"}, expected: nil, expectError: false, }, { name: "non-wildcard patterns", patterns: []string{"ns1", "ns3"}, activeNamespaces: []string{"ns1", "ns2"}, expected: []string{"ns1", "ns3"}, // includes ns3 even if not in active expectError: false, }, { name: "star pattern", patterns: []string{"*"}, activeNamespaces: []string{"ns1", "ns2", "ns3"}, expected: []string{"ns1", "ns2", "ns3"}, expectError: false, }, { name: "simple wildcard", patterns: []string{"app-*"}, activeNamespaces: []string{"app-prod", "app-staging", "db-prod"}, expected: []string{"app-prod", "app-staging"}, expectError: false, }, { name: "multiple patterns", patterns: []string{"app-*", "db-prod", "*-test"}, activeNamespaces: []string{"app-prod", "app-staging", "db-prod", "service-test", "other"}, expected: []string{"app-prod", "app-staging", "db-prod", "service-test"}, expectError: false, }, { name: "wildcard with no matches", patterns: []string{"missing-*"}, activeNamespaces: []string{"app-prod", "db-staging"}, expected: []string{}, // returns empty slice, not nil expectError: false, }, { name: "duplicate matches from multiple patterns", patterns: []string{"app-*", "*-prod"}, activeNamespaces: []string{"app-prod", "app-staging", "db-prod"}, expected: []string{"app-prod", "app-staging", "db-prod"}, // no duplicates expectError: false, }, { name: "question mark pattern - glob wildcard", patterns: []string{"ns?"}, activeNamespaces: []string{"ns1", "ns2", "ns10"}, expected: []string{"ns1", "ns2"}, // ? is a glob pattern for single character expectError: false, }, { name: "character class patterns", patterns: []string{"ns[12]"}, activeNamespaces: []string{"ns1", "ns2", "ns3", "nsa"}, expected: []string{"ns1", "ns2"}, // [12] matches 1 or 2 expectError: false, }, { name: "character range patterns", patterns: []string{"ns[a-c]"}, activeNamespaces: []string{"nsa", "nsb", "nsc", "nsd", "ns1"}, expected: []string{"nsa", "nsb", "nsc"}, // [a-c] matches a to c expectError: false, }, { name: "double asterisk should error", patterns: []string{"**"}, activeNamespaces: []string{"app-prod", "app.staging", "db/prod"}, expected: nil, expectError: true, // ** is not allowed }, { name: "unsupported regex symbols should error", patterns: []string{"ns(1|2)"}, activeNamespaces: []string{"ns1", "ns2"}, expected: nil, expectError: true, // |, (, ) not supported }, { name: "double asterisk should error", patterns: []string{"**"}, activeNamespaces: []string{"ns1", "ns2"}, expected: nil, expectError: true, // ** not allowed }, { name: "double asterisk in pattern should error", patterns: []string{"app-**-prod"}, activeNamespaces: []string{"app-prod"}, expected: nil, expectError: true, // ** not allowed anywhere }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { result, err := expandWildcards(tt.patterns, tt.activeNamespaces) if tt.expectError { assert.Error(t, err) return } require.NoError(t, err) if tt.expected == nil { assert.Nil(t, result) } else if len(tt.expected) == 0 { assert.Empty(t, result) } else { assert.ElementsMatch(t, tt.expected, result) } }) } } func TestValidateBracePatterns(t *testing.T) { tests := []struct { name string pattern string expectError bool errorMsg string }{ // Valid square bracket patterns { name: "valid square bracket pattern", pattern: "ns[abc]", expectError: false, }, { name: "valid square bracket pattern with range", pattern: "ns[a-z]", expectError: false, }, { name: "valid square bracket pattern with numbers", pattern: "ns[0-9]", expectError: false, }, { name: "valid square bracket pattern with mixed", pattern: "ns[a-z0-9]", expectError: false, }, { name: "valid square bracket pattern with single character", pattern: "ns[a]", expectError: false, }, { name: "valid square bracket pattern with text before and after", pattern: "prefix-[abc]-suffix", expectError: false, }, // Unclosed opening brackets { name: "unclosed opening bracket at end", pattern: "ns[abc", expectError: true, errorMsg: "unclosed bracket", }, { name: "unclosed opening bracket at start", pattern: "[abc", expectError: true, errorMsg: "unclosed bracket", }, { name: "unclosed opening bracket in middle", pattern: "ns[abc-test", expectError: true, errorMsg: "unclosed bracket", }, // Unmatched closing brackets { name: "unmatched closing bracket at end", pattern: "ns-abc]", expectError: true, errorMsg: "unmatched closing bracket", }, { name: "unmatched closing bracket at start", pattern: "]ns-abc", expectError: true, errorMsg: "unmatched closing bracket", }, { name: "unmatched closing bracket in middle", pattern: "ns-]abc", expectError: true, errorMsg: "unmatched closing bracket", }, { name: "extra closing bracket after valid pair", pattern: "ns[abc]]", expectError: true, errorMsg: "unmatched closing bracket", }, // Empty bracket patterns { name: "completely empty brackets", pattern: "ns[]", expectError: true, errorMsg: "empty bracket pattern", }, { name: "empty brackets at start", pattern: "[]ns", expectError: true, errorMsg: "empty bracket pattern", }, { name: "empty brackets standalone", pattern: "[]", expectError: true, errorMsg: "empty bracket pattern", }, // Edge cases { name: "empty pattern", pattern: "", expectError: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { err := validateBracePatterns(tt.pattern) if tt.expectError { require.Error(t, err, "Expected error for pattern: %s", tt.pattern) if tt.errorMsg != "" { assert.Contains(t, err.Error(), tt.errorMsg, "Error message should contain: %s", tt.errorMsg) } } else { assert.NoError(t, err, "Expected no error for pattern: %s", tt.pattern) } }) } } // Edge case tests func TestExpandWildcardsEdgeCases(t *testing.T) { t.Run("nil inputs", func(t *testing.T) { includes, excludes, err := ExpandWildcards(nil, nil, nil) require.NoError(t, err) assert.Nil(t, includes) assert.Nil(t, excludes) }) t.Run("empty string patterns", func(t *testing.T) { activeNamespaces := []string{"ns1", "ns2"} result, err := expandWildcards([]string{""}, activeNamespaces) require.NoError(t, err) assert.ElementsMatch(t, []string{""}, result) // empty string is treated as literal }) t.Run("whitespace patterns", func(t *testing.T) { activeNamespaces := []string{"ns1", " ", "ns2"} result, err := expandWildcards([]string{" "}, activeNamespaces) require.NoError(t, err) assert.ElementsMatch(t, []string{" "}, result) }) t.Run("special characters in namespace names", func(t *testing.T) { activeNamespaces := []string{"ns-1", "ns_2", "ns.3", "ns@4"} result, err := expandWildcards([]string{"ns*"}, activeNamespaces) require.NoError(t, err) assert.ElementsMatch(t, []string{"ns-1", "ns_2", "ns.3", "ns@4"}, result) }) t.Run("mixed literal and wildcard patterns", func(t *testing.T) { activeNamespaces := []string{"app.prod", "app-prod", "app_prod", "test.ns"} result, err := expandWildcards([]string{"app.prod", "app?prod"}, activeNamespaces) require.NoError(t, err) assert.ElementsMatch(t, []string{"app.prod", "app-prod", "app_prod"}, result) }) t.Run("conservative asterisk validation", func(t *testing.T) { tests := []struct { name string pattern string shouldError bool }{ {"single asterisk", "*", false}, {"double asterisk", "**", true}, {"triple asterisk", "***", true}, {"quadruple asterisk", "****", true}, {"mixed with double", "app-**", true}, {"double in middle", "app-**-prod", true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { _, err := expandWildcards([]string{tt.pattern}, []string{"test"}) if tt.shouldError { assert.Error(t, err) } else { assert.NoError(t, err) } }) } }) t.Run("malformed pattern validation", func(t *testing.T) { tests := []struct { name string pattern string shouldError bool }{ {"unclosed bracket", "ns[abc", true}, {"valid bracket", "ns[abc]", false}, {"empty bracket", "ns[]", true}, // empty brackets are invalid } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { _, err := expandWildcards([]string{tt.pattern}, []string{"test"}) if tt.shouldError { assert.Error(t, err, "Expected error for pattern: %s", tt.pattern) } else { assert.NoError(t, err, "Expected no error for pattern: %s", tt.pattern) } }) } }) } ================================================ FILE: restore-hooks_product-requirements.md ================================================ Velero Restore Hooks - PRD (Product Requirements Document) MVP Feature Set Relates to: [https://github.com/vmware-tanzu/velero/issues/2116](https://github.com/vmware-tanzu/velero/issues/2116) # **Change tracking** _This is a live document, you can reach me on the following channels for more information or for any questions:_ * _Email: [bstephanie@vmware.com](mailto:bstephanie@vmware.com)_ * _Kubernetes slack: _ * _Channel: #velero_ * _@Stephanie Bauman_ * _VMware internal slack:_ * _@bstephanie_ # **Relates to Git Issues:** * [https://app.zenhub.com/workspaces/velero-5c59c15e39d47b774b5864e3/issues/vmware-tanzu/velero/2465](https://app.zenhub.com/workspaces/velero-5c59c15e39d47b774b5864e3/issues/vmware-tanzu/velero/2465) * [https://github.com/vmware-tanzu/velero/pull/2465/files?short_path=140e0c6#diff-140e0c6b370f250ee97f6ecafc8dbb7a](https://github.com/vmware-tanzu/velero/pull/2465/files?short_path=140e0c6#diff-140e0c6b370f250ee97f6ecafc8dbb7a) * [https://github.com/vmware-tanzu/velero/issues/1150](https://github.com/vmware-tanzu/velero/issues/1150) Background Velero supports restore operations but there are gaps in the process. Gaps in the restore process require users to manually carry out steps to start, clean up, and end the restore process. Other gaps in the restore process can cause issues with application performance for applications running in a pod when a restore operation is carried out. On a restore, Velero currently does not include hooks to execute a pre- or -post restore script. As a result, users are required to perform additional actions following a velero restore operation. Some gaps that currently exist in the Velero restore process are: * Users can create a restore operation but has no option to customize or automate commands during the start of the restore operation * Users can perform post restore operations but have no option to customize or automate commands during the end of the restore operation Strategic Fit Adding a restore hook action today would allow Velero to unpack the data that was backed up in an automated way by enabling Velero to execute commands in containers during a restore. This will also improve the restore operations on a container and mitigate against any negative performance impacts of apps running in the container during restore. Purpose / Goal The purpose of this feature is to improve the extensibility and user experience of pre and post restore operations for Velero users. Goals for this feature include: * Enhance application performance during and following restore events * Provide pre-restore hooks for customizing start restore operations * Provide actions for things like retry actions, event logging, etc... during restore operations * Provide observability/status of restore commands run in restored pods * Extend restore logs to include status, error codes and necessary metadata for restore commands run during restore operations for enhanced troubleshooting capabilities * Provide post-restore hooks for customizing end of restore operations * Include pre-populated data that has been serialized by a Velero backup hook into a container or external service prior to allowing a restore to be marked as completed. Non-goals Feature Description This feature will automate the restore operations/processes in Velero, and will provide restore hook actions in Velero that allows users to execute restore commands on a container. Restore hooks include pre-hook actions and post-hook actions. This feature will mirror the actions of a Velero backup by allowing Velero to check for any restore hooks specified for a pod. Assumptions * Restore operations will be run at the pod level instead of at the volume level. Some databases require the pod to be running and in some cases a user cannot manipulate a volume without the pod running. We need to support more than one type of database for this feature and so we need to ensure that this works broadly as opposed to providing support only for specific dbs. * Velero will be responsible for invoking the restore hook. MVP Use Cases The following use cases must be included as part of the Velero restore hooks MVP (minimum viable product). **Note: **Processing of concurrent vs sequential workloads is slated later in the Velero roadmap (see [https://github.com/vmware-tanzu/velero/pull/2548/files](https://github.com/vmware-tanzu/velero/pull/2548/files)). The MVP for this feature set will align with restore of single workloads vs concurrent workload restores. A second epic will be created to address the concurrent restore operations and will be added to the backlog for priority visibility. **Note: **Please refer to the Requirements section of this document for more details on what items are P0 (must have in the MVP), P1 (should not ship without for the MVP), P2 (nice to haves). **USE CASE 1** **Title:** Run restore hook before pod restart. **Description: **As a user, I would like to run a restore hook before the applications in my pod are restarted. **______________________________________________________________** **USE CASE 2** **Title: **Allow restore hook to run on non-kubernetes databases **Description: **As a user, I would like to run restore hook operations even on databases that are external to Kubernetes (such as postgres, elastic, etc…). **______________________________________________________________** **USE CASE 3** **Title: **Run restore at pod level. **Description: **As a user, I would like to make sure that I can run the restore hook at the pod level. And, I would like the option to run this with an annotation flag in line or using an init container.** **The restore pre-hook should allow the user to run the command on the container where the pre-hook should be executed. Similar to the backup hooks, this hook should run to default to run on the first container in the pod. **______________________________________________________________** **USE CASE 4** **Title:** Specify container in which to run pre-hook **Description: **As a user, if I do not want to run the pre-hook command on the first container in the pod (default container), I would like the option to annotate the specific container that i would like the hook to run in. **______________________________________________________________** **USE CASE 5** **Title: **Check for latest snapshot **Description: **As a user, I would like Velero to run a check for the latest snapshot in object storage prior to starting restore operations on a pod. **______________________________________________________________** **USE CASE 6** **Title: **Display/surface output from restore hooks/restore status **Description:** As a user, I would like to see the output of the restore hooks/status of my restore surfaced from the pod volume restore status. Including statuses: Pending, Running/In Progress, Succeeded, Failed, Unknown. **______________________________________________________________** **USE CASE 7** **Title: **Restore metadata **Description: **As a user, I would like to have the metadata of the contents of what was restored using Velero. _Note: Kubernetes patterns may result in some snapshot metadata being overwritten during restore operations._ **______________________________________________________________** **USE CASE 8* **Title: **Increase default restore and retry limits. **Description: **As a user, I would like to increase the default restore retry and timeout limits from the default values to some other value I would like to specify. _Note: See use case 11 for the default value specifications. _ **______________________________________________________________** User Experience The following is representative of what the user experience could look like for Velero restore pre-hooks and post-hooks. **_Note: These examples are representative and are not to be considered for use in pre- and post- restore hook operations until the technical design is complete._** **Restore Pre-Hooks** Container Command ``` pre.hook.restore.velero.io/container kubectl patch backupstoragelocation \ --namespace velero \ --type merge \ --patch '{"spec":{"accessMode":"ReadOnly"}}' ``` Command Execute ``` pre.hook.restore.velero.io/command ``` Includes commands for: * Create * Create from most recent backup * Create from specific backup - allow user to list backups * Set backup storage location to read only ``` kubectl patch backupstoragelocation \ --namespace velero \ --type merge \ --patch '{"spec":{"accessMode":"ReadOnly"}} ``` * Set backup storage location to read-write ``` kubectl patch backupstoragelocation \ --namespace velero \ --type merge \ --patch '{"spec":{"accessMode":"ReadWrite"}}' ``` Error handling ``` pre.hook.restore.velero.io/on-error ``` Timeout ``` pre.hook.restore.velero.io/retry ``` Requirements **_P0_** = must not ship without (absolute requirement for MVP, engineering requirements for long term viability usually fall in here for ex., and incompletion nearing or by deadline means delaying code freeze and GA date) **_P1_** = should not ship without (required for feature to achieve general adoption, should be scoped into feature but can be pushed back to later iterations if needed) **_P2_** = nice to have (opportunistic, should not be scoped into the overall feature if it has dependencies and could cause delay) **P0 Requirements** P0. Use Case 1 - Run restore hook before pod restart. P0. Use Case 3- Run restore at pod level. P0. Use Case 5 - Check for latest snapshot P0. Use Case 9 - ** **Display/surface restore status P0. Use Case 10 - Restore metadata P0. Use Case 11 - Retry restore upon restore failure/error/timeout P0. Use Case 12 - Increase default restore and retry limits. **P1 Requirements** P1. Use Case 2 - Allow restore hook to run on non-kubernetes databases P1. Use Case 4 - Specify container in which to run pre-hook P1. Use Case 6 - Specify backup snapshot to use for restore P1. Use Case 7 - ** **Include or exclude namespaces from restore **P2 Requirements** P2. **Out of scope** The following requirements are out of scope for the Velero Restore Hooks MVP: 1. Verifying the integrity of a backup, resource, or other artifact will not be included in the scope of this effort. 2. Verifying the integrity of a snapshot using Kubernetes hash checks. 3. Running concurrent restore operations (for the MVP) a secondary epic will be opened to align better with the concurrent workload operations currently set on the Velero roadmap for Q4 timeframe. **Questions** 1. For USE CASE 1: Init vs app containers - if multiple containers are specified for a pod kubelet will run each init container sequentially - does this have an impact on things like concurrent workload processing? 2. Can velero allow a user to specify a specific backup if the most recent backup is not desired in a restore? 3. If a backup specified for a restore operation fails, can velero retry and pick up the next most recent backup in the restore? 4. Can velero provide a delta between the two backups if a different backup needs to be picked up (other than the most recent because the most recent backup cannot be accessed?) 5. What types of errors can velero surface about backups, namespaces, pods, resources, if a backup has an issue with it preventing a restore from being done? For questions, please contact michaelmi@vmware.com, [bstephanie@vmware.com](mailto:bstephanie@vmware.com) ================================================ FILE: site/Dockerfile ================================================ FROM klakegg/hugo:0.73.0-ext-ubuntu WORKDIR /srv/hugo EXPOSE 1313 ================================================ FILE: site/README-HUGO.md ================================================ # Running in Docker To run this site in a Docker container, you can use `make serve-docs` from the root directory. # Dependencies for MacOS Install the following for an easy to use dev environment: * `brew install hugo` # Dependencies for Linux If you are running a build on Ubuntu you will need the following packages: * hugo # Local Development 1. Clone down your own fork, or clone the main repo `git clone https://github.com/vmware-tanzu/velero` and add your own remote. 1. `cd velero/site` 1. Serve the site and watch for markup/sass changes `hugo serve`. 1. View your website at http://127.0.0.1:1313/ 1. Commit any changes and push everything to your fork. 1. Once you're ready, submit a PR of your changes. Netlify will automatically generate a preview of your changes. # Jetbrains IDE setup (IntelliJ, Goland, etc) 1. Install the `Hugo Integration` plugin: https://plugins.jetbrains.com/plugin/13215-hugo-integration - Under `Preferences...` -> `Plugins` 1. Create a new configuration: - Click `Edit Configurations...` - Click the `+` button to create a new configuration and select `Hugo` - Select `hugo serve` and make sure it is running under the `site` directory - Save and run the new Configuration - View your website at http://127.0.0.1:1313/ - Any changes in `site` will reload the website automatically # Adding a New Docs Version To add a new set of versioned docs to go with a new Velero release: 1. In the root of the repository, run: ```bash # set to the appropriate version numbers NEW_DOCS_VERSION=vX.Y VELERO_VERSION=vX.Y.Z make gen-docs ``` 1. [Pre-release only] In `site/config.yaml`, revert the change to the `latest` field, so the pre-release docs do not become the default. ================================================ FILE: site/assets/_scss/_styles.scss ================================================ @import "./site/settings/variables"; // 3rd party @import "./bootstrap-4.1.3/bootstrap.scss"; // Common @import "./site/common/fonts"; @import "./site/common/core"; @import "./site/common/type"; // Layouts @import "./site/layouts/container"; @import "./site/layouts/documentation"; @import "./site/layouts/docsearch"; // Objects @import "./site/objects/header"; @import "./site/objects/footer"; @import "./site/objects/section"; @import "./site/objects/home-hero"; @import "./site/objects/card"; @import "./site/objects/alternating-cards"; @import "./site/objects/button"; @import "./site/objects/thumbnail-grid"; @import "./site/objects/post"; // Utilities @import "./site/utilities/image"; @import "./site/utilities/type"; ================================================ FILE: site/assets/_scss/bootstrap-4.1.3/_alert.scss ================================================ // // Base styles // .alert { position: relative; padding: $alert-padding-y $alert-padding-x; margin-bottom: $alert-margin-bottom; border: $alert-border-width solid transparent; @include border-radius($alert-border-radius); } // Headings for larger alerts .alert-heading { // Specified to prevent conflicts of changing $headings-color color: inherit; } // Provide class for links that match alerts .alert-link { font-weight: $alert-link-font-weight; } // Dismissible alerts // // Expand the right padding and account for the close button's positioning. .alert-dismissible { padding-right: ($close-font-size + $alert-padding-x * 2); // Adjust close link position .close { position: absolute; top: 0; right: 0; padding: $alert-padding-y $alert-padding-x; color: inherit; } } // Alternate styles // // Generate contextual modifier classes for colorizing the alert. @each $color, $value in $theme-colors { .alert-#{$color} { @include alert-variant(theme-color-level($color, $alert-bg-level), theme-color-level($color, $alert-border-level), theme-color-level($color, $alert-color-level)); } } ================================================ FILE: site/assets/_scss/bootstrap-4.1.3/_badge.scss ================================================ // Base class // // Requires one of the contextual, color modifier classes for `color` and // `background-color`. .badge { display: inline-block; padding: $badge-padding-y $badge-padding-x; font-size: $badge-font-size; font-weight: $badge-font-weight; line-height: 1; text-align: center; white-space: nowrap; vertical-align: baseline; @include border-radius($badge-border-radius); // Empty badges collapse automatically &:empty { display: none; } } // Quick fix for badges in buttons .btn .badge { position: relative; top: -1px; } // Pill badges // // Make them extra rounded with a modifier to replace v3's badges. .badge-pill { padding-right: $badge-pill-padding-x; padding-left: $badge-pill-padding-x; @include border-radius($badge-pill-border-radius); } // Colors // // Contextual variations (linked badges get darker on :hover). @each $color, $value in $theme-colors { .badge-#{$color} { @include badge-variant($value); } } ================================================ FILE: site/assets/_scss/bootstrap-4.1.3/_breadcrumb.scss ================================================ .breadcrumb { display: flex; flex-wrap: wrap; padding: $breadcrumb-padding-y $breadcrumb-padding-x; margin-bottom: $breadcrumb-margin-bottom; list-style: none; background-color: $breadcrumb-bg; @include border-radius($breadcrumb-border-radius); } .breadcrumb-item { // The separator between breadcrumbs (by default, a forward-slash: "/") + .breadcrumb-item { padding-left: $breadcrumb-item-padding; &::before { display: inline-block; // Suppress underlining of the separator in modern browsers padding-right: $breadcrumb-item-padding; color: $breadcrumb-divider-color; content: $breadcrumb-divider; } } // IE9-11 hack to properly handle hyperlink underlines for breadcrumbs built // without `